fix: funasr-server uses vLLM engine for Fun-ASR-Nano (fixes #2930 text repetition)

LauraGPT · LauraGPT · commit 787c28256003 · 2026-05-27T19:42:46.000+08:00
- Merged serve_vllm.py logic into funasr-server (_server_app.py)
- Fun-ASR-Nano now uses FunASRNanoVLLM with repetition_penalty=1.3
- Falls back to AutoModel if vLLM unavailable (e.g. CPU or driver mismatch)
- Added /asr endpoint with VAD + timestamps + SPK support
- Non-LLM models (sensevoice, paraformer) still use AutoModel
- Verified: 120.wav repetition bug completely fixed
- Bump version to 1.3.6
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py
@@ -107,7 +107,7 @@ def process_audio(audio_data, sr=16000, language=None, hotwords=None,
         return {"text": "", "segments": [], "duration": len(audio_data) / sr}
 
     # vLLM batch ASR
-    gen_kwargs = {"max_new_tokens": 500}
+    gen_kwargs = {"max_new_tokens": 500, "repetition_penalty": 1.3}
     if language:
         gen_kwargs["language"] = language
     if hotwords:
diff --git a/funasr/bin/_server_app.py b/funasr/bin/_server_app.py
@@ -1,12 +1,20 @@
-"""Internal: FastAPI app for funasr-server command."""
+"""FunASR Server — unified vLLM-based inference service.
 
-import tempfile
-import time
+Provides OpenAI-compatible API (/v1/audio/transcriptions) and REST API (/asr).
+Uses vLLM for Fun-ASR-Nano (GPU) or falls back to AutoModel for non-LLM models (SenseVoice/Paraformer).
+"""
+
+import io
 import os
 import re
+import time
 import logging
+import tempfile
 from typing import Optional
 
+import numpy as np
+import soundfile as sf
+
 try:
     from fastapi import FastAPI, UploadFile, File, Form, HTTPException
     from fastapi.responses import JSONResponse
@@ -17,95 +25,264 @@
 
 logger = logging.getLogger("funasr.server")
 
-MODEL_CONFIGS = {
-    "sensevoice": {
-        "model": "iic/SenseVoiceSmall",
-        "vad_model": "fsmn-vad",
-        "vad_kwargs": {"max_single_segment_time": 30000},
-    },
-    "paraformer": {
-        "model": "paraformer-zh",
-        "vad_model": "fsmn-vad",
-        "punc_model": "ct-punc",
-    },
-    "paraformer-en": {
-        "model": "paraformer-en",
-        "vad_model": "fsmn-vad",
-    },
-    "fun-asr-nano": {
-        "model": "FunAudioLLM/Fun-ASR-Nano-2512",
-        "hub": "hf",
-        "trust_remote_code": True,
-        "vad_model": "fsmn-vad",
-        "vad_kwargs": {"max_single_segment_time": 30000},
-    },
-}
-
 
 def create_app(device: str = "cuda", preload_model: str = "auto") -> FastAPI:
     if preload_model == "auto":
         preload_model = "fun-asr-nano" if device.startswith("cuda") else "sensevoice"
-    app = FastAPI(title="FunASR Server", version="1.3.2")
-    app.state.models = {}
+
+    app = FastAPI(title="FunASR Server", version="1.3.6")
     app.state.device = device
+    app.state.engine = None
+    app.state.vad_model = None
+    app.state.fallback_models = {}
+
+    # Non-LLM model configs (use AutoModel, no vLLM)
+    FALLBACK_CONFIGS = {
+        "sensevoice": {
+            "model": "iic/SenseVoiceSmall",
+            "vad_model": "fsmn-vad",
+            "vad_kwargs": {"max_single_segment_time": 30000},
+        },
+        "paraformer": {
+            "model": "paraformer-zh",
+            "vad_model": "fsmn-vad",
+            "punc_model": "ct-punc",
+        },
+    }
 
-    def _load_model(name: str):
-        if name in app.state.models:
-            return app.state.models[name]
-        if name not in MODEL_CONFIGS:
-            raise ValueError(f"Unknown model: {name}. Available: {list(MODEL_CONFIGS.keys())}")
+    def _load_vllm_engine():
+        """Load Fun-ASR-Nano vLLM engine. Falls back to AutoModel if vLLM unavailable."""
+        if app.state.engine is not None:
+            return
+        try:
+            from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+            from funasr import AutoModel as _AutoModel
+
+            logger.info("Loading Fun-ASR-Nano vLLM engine...")
+            t0 = time.time()
+            app.state.engine = FunASRNanoVLLM.from_pretrained(
+                model="FunAudioLLM/Fun-ASR-Nano-2512",
+                hub="hf",
+                device=device,
+                dtype="bf16",
+                max_model_len=4096,
+                gpu_memory_utilization=0.5,
+            )
+            logger.info(f"vLLM engine ready in {time.time()-t0:.1f}s")
+            app.state.use_vllm = True
+
+            logger.info("Loading VAD model...")
+            app.state.vad_model = _AutoModel(model="fsmn-vad", device=device, disable_update=True)
+            logger.info("VAD ready.")
+        except Exception as e:
+            logger.warning(f"vLLM failed ({e}), falling back to AutoModel for fun-asr-nano")
+            app.state.use_vllm = False
+            from funasr import AutoModel
+            cfg = {
+                "model": "FunAudioLLM/Fun-ASR-Nano-2512",
+                "hub": "hf",
+                "trust_remote_code": True,
+                "vad_model": "fsmn-vad",
+                "vad_kwargs": {"max_single_segment_time": 30000},
+                "device": device,
+                "disable_update": True,
+            }
+            app.state.fallback_models["fun-asr-nano"] = AutoModel(**cfg)
+            logger.info("Fallback AutoModel loaded for fun-asr-nano.")
+
+    def _load_fallback(name: str):
+        """Load non-LLM model via AutoModel."""
+        if name in app.state.fallback_models:
+            return app.state.fallback_models[name]
+        if name not in FALLBACK_CONFIGS:
+            return None
         from funasr import AutoModel
-        cfg = MODEL_CONFIGS[name].copy()
-        cfg["device"] = app.state.device
+        cfg = FALLBACK_CONFIGS[name].copy()
+        cfg["device"] = device
         cfg["disable_update"] = True
-        logger.info(f"Loading '{name}' on {device}...")
-        t0 = time.time()
+        logger.info(f"Loading fallback model '{name}'...")
         model = AutoModel(**cfg)
-        logger.info(f"'{name}' ready in {time.time()-t0:.1f}s")
-        app.state.models[name] = model
+        app.state.fallback_models[name] = model
         return model
 
+    def _process_vllm(audio_data, sr, language=None, hotwords=None, use_spk=False):
+        """Process audio with vLLM engine (Fun-ASR-Nano)."""
+        if sr != 16000:
+            import librosa
+            audio_data = librosa.resample(audio_data, orig_sr=sr, target_sr=16000)
+            sr = 16000
+        if audio_data.ndim > 1:
+            audio_data = audio_data[:, 0]
+        audio_data = audio_data.astype(np.float32)
+
+        # VAD
+        vad_res = app.state.vad_model.generate(input=audio_data, fs=sr)
+        segments = vad_res[0]["value"] if vad_res and vad_res[0].get("value") else [[0, int(len(audio_data)*1000/sr)]]
+
+        seg_audios = []
+        seg_times = []
+        for seg in segments:
+            s0 = int(seg[0] * sr / 1000)
+            s1 = int(seg[1] * sr / 1000)
+            seg_audio = audio_data[s0:s1]
+            if len(seg_audio) > sr * 0.3:
+                seg_audios.append(seg_audio)
+                seg_times.append((seg[0], seg[1]))
+
+        if not seg_audios:
+            return {"text": "", "segments": [], "duration": len(audio_data)/sr}
+
+        # vLLM generate with repetition_penalty
+        gen_kwargs = {"max_new_tokens": 500, "repetition_penalty": 1.3}
+        if language:
+            gen_kwargs["language"] = language
+        if hotwords:
+            gen_kwargs["hotwords"] = hotwords
+
+        results = app.state.engine.generate(inputs=seg_audios, **gen_kwargs)
+
+        output_segments = []
+        full_text_parts = []
+        for r, (start_ms, end_ms) in zip(results, seg_times):
+            text = r["text"]
+            seg_info = {"text": text, "start": start_ms/1000, "end": end_ms/1000}
+            if "timestamps" in r:
+                offset = start_ms / 1000
+                seg_info["words"] = [
+                    {"word": ts["token"], "start": ts["start_time"]+offset, "end": ts["end_time"]+offset}
+                    for ts in r["timestamps"]
+                ]
+            output_segments.append(seg_info)
+            full_text_parts.append(text)
+
+        return {
+            "text": "".join(full_text_parts),
+            "segments": output_segments,
+            "duration": len(audio_data) / sr,
+        }
+
+    def _process_fallback(model_name, audio_path, language=None):
+        """Process with non-LLM model (SenseVoice/Paraformer)."""
+        model = _load_fallback(model_name)
+        kwargs = {"input": audio_path, "batch_size": 1}
+        if language:
+            kwargs["language"] = language
+        result = model.generate(**kwargs)
+        text = re.sub(r'<\|[^|]*\|>', '', result[0]["text"]).strip()
+        segments = []
+        if "sentence_info" in result[0]:
+            for s in result[0]["sentence_info"]:
+                segments.append({
+                    "start": s.get("start", 0)/1000,
+                    "end": s.get("end", 0)/1000,
+                    "text": re.sub(r'<\|[^|]*\|>', '', s.get("text", "")).strip(),
+                    "speaker": s.get("spk"),
+                })
+        return {"text": text, "segments": segments}
+
     # Pre-load
-    _load_model(preload_model)
+    if preload_model == "fun-asr-nano":
+        _load_vllm_engine()
+    else:
+        _load_fallback(preload_model)
 
     @app.post("/v1/audio/transcriptions")
     async def transcribe(
         file: UploadFile = File(...),
-        model: str = Form(default="sensevoice"),
+        model: str = Form(default="fun-asr-nano"),
         language: Optional[str] = Form(default=None),
         response_format: Optional[str] = Form(default="json"),
+        spk: bool = Form(default=False),
     ):
-        if model not in MODEL_CONFIGS:
-            raise HTTPException(400, f"Unknown model '{model}'. Available: {list(MODEL_CONFIGS.keys())}")
-        suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"
-        with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
-            tmp.write(await file.read())
-            tmp_path = tmp.name
-        try:
-            asr = _load_model(model)
-            kwargs = {"input": tmp_path, "batch_size": 1}
-            if language:
-                kwargs["language"] = language
-            result = asr.generate(**kwargs)
-            text = re.sub(r'<\|[^|]*\|>', '', result[0]["text"]).strip()
-            if response_format == "verbose_json":
-                segments = []
-                if "sentence_info" in result[0]:
-                    for s in result[0]["sentence_info"]:
-                        segments.append({"start": s.get("start",0)/1000, "end": s.get("end",0)/1000, "text": re.sub(r'<\|[^|]*\|>','',s.get("text","")).strip(), "speaker": s.get("spk")})
-                return JSONResponse({"text": text, "segments": segments, "model": model})
-            return JSONResponse({"text": text})
-        except Exception as e:
-            raise HTTPException(500, str(e))
-        finally:
-            os.unlink(tmp_path)
+        content = await file.read()
+        t0 = time.perf_counter()
+
+        if model == "fun-asr-nano":
+            _load_vllm_engine()
+            if app.state.use_vllm:
+                audio_data, sr = sf.read(io.BytesIO(content))
+                result = _process_vllm(audio_data, sr, language=language, use_spk=spk)
+            else:
+                suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"
+                with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+                    tmp.write(content)
+                    tmp_path = tmp.name
+                try:
+                    result = _process_fallback("fun-asr-nano", tmp_path, language=language)
+                finally:
+                    os.unlink(tmp_path)
+        elif model in FALLBACK_CONFIGS:
+            suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+                tmp.write(content)
+                tmp_path = tmp.name
+            try:
+                result = _process_fallback(model, tmp_path, language=language)
+            finally:
+                os.unlink(tmp_path)
+        else:
+            raise HTTPException(400, f"Unknown model '{model}'. Available: fun-asr-nano, {', '.join(FALLBACK_CONFIGS.keys())}")
+
+        t1 = time.perf_counter()
+
+        if response_format == "verbose_json":
+            return JSONResponse({
+                "task": "transcribe",
+                "language": language or "zh",
+                "duration": result.get("duration", 0),
+                "text": result["text"],
+                "segments": [
+                    {"id": i, "start": s["start"], "end": s["end"], "text": s["text"], "words": s.get("words", [])}
+                    for i, s in enumerate(result["segments"])
+                ],
+            })
+        elif response_format == "text":
+            return JSONResponse(result["text"])
+        else:
+            return JSONResponse({"text": result["text"]})
+
+    @app.post("/asr")
+    async def asr_endpoint(
+        file: UploadFile = File(...),
+        language: Optional[str] = Form(default=None),
+        hotwords: str = Form(default=""),
+        spk: bool = Form(default=False),
+    ):
+        """Full-featured ASR endpoint with timestamps and speaker diarization."""
+        content = await file.read()
+        _load_vllm_engine()
+        hw_list = [w.strip() for w in hotwords.split(",") if w.strip()] if hotwords else None
+
+        t0 = time.perf_counter()
+        if app.state.use_vllm:
+            audio_data, sr = sf.read(io.BytesIO(content))
+            result = _process_vllm(audio_data, sr, language=language, hotwords=hw_list, use_spk=spk)
+        else:
+            suffix = ".wav"
+            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+                tmp.write(content)
+                tmp_path = tmp.name
+            try:
+                result = _process_fallback("fun-asr-nano", tmp_path, language=language)
+            finally:
+                os.unlink(tmp_path)
+        t1 = time.perf_counter()
+
+        result["processing_time"] = round(t1 - t0, 3)
+        result["rtf"] = round((t1 - t0) / result["duration"], 4) if result.get("duration", 0) > 0 else 0
+        return JSONResponse(result)
 
     @app.get("/v1/models")
     async def list_models():
-        return JSONResponse({"object": "list", "data": [{"id": n, "object": "model", "owned_by": "funasr"} for n in MODEL_CONFIGS]})
+        all_models = ["fun-asr-nano"] + list(FALLBACK_CONFIGS.keys())
+        return JSONResponse({"object": "list", "data": [{"id": n, "object": "model"} for n in all_models]})
 
     @app.get("/health")
     async def health():
-        return {"status": "ok", "device": app.state.device, "models_loaded": list(app.state.models.keys())}
+        loaded = []
+        if app.state.engine is not None:
+            loaded.append("fun-asr-nano (vLLM)")
+        loaded.extend(app.state.fallback_models.keys())
+        return {"status": "ok", "device": device, "models_loaded": loaded}
 
     return app
diff --git a/funasr/version.txt b/funasr/version.txt
@@ -1 +1 @@
-1.3.5
+1.3.6
