[voice] feat: add managed voice cloning profiles (#10799)

* feat(ui): add voice library workflow

Give administrators a production-ready flow to record or upload consented reference audio, manage reusable profiles, inspect API usage, discover compatible models, and hand a saved voice directly to text-to-speech.

Assisted-by: Codex:gpt-5

* feat(voice): add managed voice cloning profiles

Make reusable reference voices manageable through the admin API instead of requiring model-directory and YAML edits. Discover compatible installed and gallery models from server-side backend capabilities, retain explicit model configuration controls, and stage saved references for supported backends.

Expose profile management through REST and MCP, document backend-specific behavior, and cover the workflow from profile creation through real Qwen3-TTS synthesis. Harden the agent-job HTTP test against completion racing cancellation.

Assisted-by: Codex:gpt-5

---------

Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

Author: localai-bot

backend/go/crispasr/gocrispasr.go

@@ -530,11 +530,51 @@ func setVoice(voice string) {
 	}
 }
 
+// applyRequestVoice distinguishes named speakers from per-request reference
+// WAVs. The latter are used by F5-TTS and require the exact transcript under
+// the cross-backend params.ref_text contract.
+func applyRequestVoice(req *pb.TTSRequest) error {
+	voice := strings.TrimSpace(req.Voice)
+	if voice == "" {
+		return nil
+	}
+
+	info, statErr := os.Stat(voice)
+	looksLikeFile := filepath.IsAbs(voice) || strings.EqualFold(filepath.Ext(voice), ".wav")
+	if statErr == nil && info.Mode().IsRegular() {
+		refText := ""
+		if req.Params != nil {
+			refText = strings.TrimSpace(req.Params["ref_text"])
+			if refText == "" {
+				refText = strings.TrimSpace(req.Params["voice_text"])
+			}
+		}
+		if refText == "" {
+			return fmt.Errorf("crispasr: params.ref_text is required with a reference voice WAV")
+		}
+		if rc := CppTTSSetVoiceFile(voice, refText); rc < 0 {
+			return fmt.Errorf("crispasr: failed to apply reference voice %q (rc=%d)", voice, rc)
+		}
+		return nil
+	}
+	if looksLikeFile {
+		if statErr != nil {
+			return fmt.Errorf("crispasr: reference voice %q: %w", voice, statErr)
+		}
+		return fmt.Errorf("crispasr: reference voice %q is not a regular file", voice)
+	}
+
+	setVoice(voice)
+	return nil
+}
+
 func (w *CrispASR) TTS(req *pb.TTSRequest) error {
 	if req.Dst == "" {
 		return fmt.Errorf("crispasr: TTS requires a destination path")
 	}
-	setVoice(req.Voice)
+	if err := applyRequestVoice(req); err != nil {
+		return err
+	}
 	pcm, err := w.synthesize(req.Text)
 	if err != nil {
 		return err
@@ -553,7 +593,9 @@ func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error {
 	if req.Text == "" {
 		return fmt.Errorf("crispasr: TTSStream requires text")
 	}
-	setVoice(req.Voice)
+	if err := applyRequestVoice(req); err != nil {
+		return err
+	}
 	pcm, err := w.synthesize(req.Text)
 	if err != nil {
 		return err

backend/go/crispasr/gocrispasr_test.go

@@ -172,6 +172,32 @@ var _ = Describe("CrispASR", func() {
 	})
 
 	Context("TTS", func() {
+		It("applies a per-request reference WAV and transcript", func() {
+			refWAV := filepath.Join(GinkgoT().TempDir(), "reference.wav")
+			Expect(os.WriteFile(refWAV, []byte("fixture"), 0o600)).To(Succeed())
+
+			original := CppTTSSetVoiceFile
+			DeferCleanup(func() { CppTTSSetVoiceFile = original })
+			var gotPath, gotText string
+			CppTTSSetVoiceFile = func(path, refText string) int {
+				gotPath, gotText = path, refText
+				return 0
+			}
+
+			Expect(applyRequestVoice(&pb.TTSRequest{
+				Voice:  refWAV,
+				Params: map[string]string{"ref_text": "The exact words in the clip."},
+			})).To(Succeed())
+			Expect(gotPath).To(Equal(refWAV))
+			Expect(gotText).To(Equal("The exact words in the clip."))
+		})
+
+		It("rejects a reference WAV without a transcript", func() {
+			refWAV := filepath.Join(GinkgoT().TempDir(), "reference.wav")
+			Expect(os.WriteFile(refWAV, []byte("fixture"), 0o600)).To(Succeed())
+			Expect(applyRequestVoice(&pb.TTSRequest{Voice: refWAV})).To(MatchError(ContainSubstring("params.ref_text")))
+		})
+
 		It("synthesizes a non-empty WAV", func() {
 			ttsModel := ttsModelOrSkip()
 			ensureLibLoaded()

backend/go/qwen3-tts-cpp/README.md

@@ -0,0 +1,85 @@
+# Qwen3-TTS C++ backend
+
+This backend runs Qwen3-TTS GGUF models through
+[qwentts.cpp](https://github.com/ServeurpersoCom/qwentts.cpp). It supports
+24 kHz speech generation, streaming, named speakers, voice design, and
+reference-audio cloning depending on the model variant.
+
+## Gallery models
+
+The following Base models accept LocalAI Voice Library profiles:
+
+- `qwen3-tts-cpp`
+- `qwen3-tts-cpp-0.6b-base-q4`
+- `qwen3-tts-cpp-1.7b-base`
+- `qwen3-tts-cpp-1.7b-base-q4`
+
+Gallery models containing `customvoice` or `voicedesign` implement those Qwen
+modes instead and are not advertised as raw reference-audio models.
+
+Install a Base model with:
+
+```bash
+local-ai models install qwen3-tts-cpp
+```
+
+## Model configuration
+
+Base filenames are detected automatically. Set `tts.voice_cloning` only when a
+verified private conversion has a name that does not identify it as a Base or
+VoiceClone model:
+
+```yaml
+name: private-qwen-voice
+backend: qwen3-tts-cpp
+parameters:
+  model: qwen-private/talker.gguf
+known_usecases:
+  - tts
+tts:
+  voice_cloning: true
+  audio_path: voices/default-reference.wav  # optional model-wide fallback
+```
+
+The tokenizer GGUF is auto-discovered when its filename contains `tokenizer`
+and it is stored beside the talker. Otherwise set
+`options: ["tokenizer:qwen-private/tokenizer.gguf"]`.
+
+`tts.voice_cloning: false` removes a model from Voice Library compatibility
+results and rejects saved `localai://voice-profiles/...` references. It does
+not disable Qwen's named-speaker or VoiceDesign modes. Setting it to `true`
+cannot add cloning to a backend that lacks LocalAI's reference-audio contract.
+
+Request precedence is: a request `voice`, then `tts.voice`, then
+`tts.audio_path`. A saved profile supplies its private WAV and exact transcript
+for that request without changing the model YAML.
+
+## API example
+
+Create or select a profile in **Operate → Voice Library**, then pass its stable
+URI to either speech endpoint:
+
+```bash
+curl http://localhost:8080/v1/audio/speech \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "qwen3-tts-cpp",
+    "input": "This request uses a saved reference voice.",
+    "voice": "localai://voice-profiles/PROFILE_ID"
+  }' \
+  --output speech.wav
+```
+
+## Native end-to-end test
+
+The labeled test loads real GGUFs, synthesizes speech, streams audio, and
+exercises cloning with a generated 24 kHz reference WAV:
+
+```bash
+make -C backend/go/qwen3-tts-cpp qwen3-tts-cpp
+
+QWEN3TTS_MODEL=/path/to/qwen-talker-0.6b-base-Q8_0.gguf \
+QWEN3TTS_CODEC=/path/to/qwen-tokenizer-12hz-Q8_0.gguf \
+QWEN3TTS_LIBRARY=backend/go/qwen3-tts-cpp/libgoqwen3ttscpp-fallback.so \
+  go test ./backend/go/qwen3-tts-cpp -ginkgo.label-filter=e2e
+```

backend/python/chatterbox/backend.py

@@ -199,7 +199,9 @@ def TTS(self, request, context):
 
             if "language" in self.options:
                 kwargs["language_id"] = self.options["language"]
-            if self.AudioPath is not None:
+            if request.voice and os.path.isfile(request.voice):
+                kwargs["audio_prompt_path"] = request.voice
+            elif self.AudioPath is not None:
                 kwargs["audio_prompt_path"] = self.AudioPath
 
             # add options to kwargs
@@ -211,6 +213,8 @@ def TTS(self, request, context):
             # strings on the wire and are coerced to float/int/bool.
             if hasattr(request, "params") and request.params:
                 for key, value in request.params.items():
+                    if key == "ref_text":
+                        continue
                     kwargs[key] = coerce_param_value(value)
 
             # Check if text exceeds 250 characters

backend/python/coqui/backend.py

@@ -79,14 +79,19 @@ def TTS(self, request, context):
             if self.tts.is_multi_lingual and lang is None:
                return backend_pb2.Result(success=False, message=f"Model is multi-lingual, but no language was provided")
 
-            # if model is multi-speaker, use speaker_wav or the speaker_id from request.voice
-            if self.tts.is_multi_speaker and self.AudioPath is None and request.voice is None:
+            # A path-shaped per-request voice is a cloning reference; otherwise
+            # preserve Coqui's named-speaker behavior.
+            request_voice = request.voice if request.voice else ""
+            speaker_wav = request_voice if os.path.isfile(request_voice) else self.AudioPath
+            if self.tts.is_multi_speaker and speaker_wav is None and not request_voice:
                 return backend_pb2.Result(success=False, message=f"Model is multi-speaker, but no speaker was provided")
 
-            if self.tts.is_multi_speaker and request.voice is not None:
-               self.tts.tts_to_file(text=request.text, speaker=request.voice, language=lang, file_path=request.dst)
+            if speaker_wav is not None:
+                self.tts.tts_to_file(text=request.text, speaker_wav=speaker_wav, language=lang, file_path=request.dst)
+            elif self.tts.is_multi_speaker and request_voice:
+               self.tts.tts_to_file(text=request.text, speaker=request_voice, language=lang, file_path=request.dst)
             else:
-                self.tts.tts_to_file(text=request.text, speaker_wav=self.AudioPath, language=lang, file_path=request.dst)
+                self.tts.tts_to_file(text=request.text, language=lang, file_path=request.dst)
         except Exception as err:
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
         return backend_pb2.Result(success=True)

backend/python/faster-qwen3-tts/backend.py

@@ -83,20 +83,23 @@ def LoadModel(self, request, context):
         return backend_pb2.Result(message="Model loaded successfully", success=True)
 
     def _get_ref_audio_path(self, request):
-        if not self.audio_path:
+        # A per-request voice path is the canonical LocalAI voice-profile
+        # contract. Keep AudioPath as the backwards-compatible YAML fallback.
+        audio_path = request.voice if hasattr(request, "voice") and request.voice else self.audio_path
+        if not audio_path:
             return None
-        if os.path.isabs(self.audio_path):
-            return self.audio_path
+        if os.path.isabs(audio_path):
+            return audio_path
         if self.model_file:
             model_file_base = os.path.dirname(self.model_file)
-            ref_path = os.path.join(model_file_base, self.audio_path)
+            ref_path = os.path.join(model_file_base, audio_path)
             if os.path.exists(ref_path):
                 return ref_path
         if self.model_path:
-            ref_path = os.path.join(self.model_path, self.audio_path)
+            ref_path = os.path.join(self.model_path, audio_path)
             if os.path.exists(ref_path):
                 return ref_path
-        return self.audio_path
+        return audio_path
 
     def TTS(self, request, context):
         try:
@@ -122,13 +125,13 @@ def TTS(self, request, context):
                     success=False,
                     message="AudioPath is required for voice clone (set in LoadModel)"
                 )
-            ref_text = self.options.get("ref_text")
-            if not ref_text and hasattr(request, 'ref_text') and request.ref_text:
-                ref_text = request.ref_text
+            ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
+            if not ref_text:
+                ref_text = self.options.get("ref_text")
             if not ref_text:
                 return backend_pb2.Result(
                     success=False,
-                    message="ref_text is required for voice clone (set via LoadModel Options, e.g. ref_text:Your reference transcript)"
+                    message="ref_text is required for voice clone (set in request.params or LoadModel options)"
                 )
 
             chunk_size = self.options.get("chunk_size")

backend/python/fish-speech/backend.py

@@ -267,6 +267,8 @@ def LoadModel(self, request, context):
 
     def _get_ref_audio_path(self, voice_name=None):
         """Get reference audio path from voices dict or stored AudioPath."""
+        if voice_name and os.path.isfile(voice_name):
+            return voice_name
         if voice_name and voice_name in self.voices:
             audio_path = self.voices[voice_name]["audio"]
 
@@ -332,7 +334,19 @@ def TTS(self, request, context):
             references = []
             voice_name = request.voice if request.voice else None
 
-            if voice_name and voice_name in self.voices:
+            if voice_name and os.path.isfile(voice_name):
+                ref_audio_path = self._get_ref_audio_path(voice_name)
+                with open(ref_audio_path, "rb") as f:
+                    audio_bytes = f.read()
+                ref_text = request.params.get("ref_text", "") if hasattr(request, "params") else ""
+                references.append(
+                    ServeReferenceAudio(audio=audio_bytes, text=ref_text)
+                )
+                print(
+                    f"[INFO] Using per-request reference audio: {ref_audio_path}",
+                    file=sys.stderr,
+                )
+            elif voice_name and voice_name in self.voices:
                 ref_audio_path = self._get_ref_audio_path(voice_name)
                 if ref_audio_path and os.path.exists(ref_audio_path):
                     with open(ref_audio_path, "rb") as f:
@@ -350,7 +364,9 @@ def TTS(self, request, context):
                 if ref_audio_path and os.path.exists(ref_audio_path):
                     with open(ref_audio_path, "rb") as f:
                         audio_bytes = f.read()
-                    ref_text = self.options.get("ref_text", "")
+                    ref_text = request.params.get("ref_text", "") if hasattr(request, "params") else ""
+                    if not ref_text:
+                        ref_text = self.options.get("ref_text", "")
                     references.append(
                         ServeReferenceAudio(audio=audio_bytes, text=ref_text)
                     )

backend/python/neutts/backend.py

@@ -119,9 +119,18 @@ def TTS(self, request, context):
             # add options to kwargs
             kwargs.update(self.options)
 
-            ref_codes = self.model.encode_reference(self.AudioPath)
-
-            wav = self.model.infer(request.text, ref_codes, self.ref_text)
+            ref_audio = request.voice if request.voice else self.AudioPath
+            if not ref_audio:
+                return backend_pb2.Result(success=False, message="reference audio is required")
+            ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
+            if not ref_text:
+                ref_text = self.ref_text
+            if not ref_text:
+                return backend_pb2.Result(success=False, message="ref_text is required")
+
+            ref_codes = self.model.encode_reference(ref_audio)
+
+            wav = self.model.infer(request.text, ref_codes, ref_text)
 
             sf.write(request.dst, wav, 24000)            
         except Exception as err:

backend/python/qwen-tts/backend.py

@@ -362,7 +362,7 @@ def _detect_mode(self, request):
         # model_type explicitly set
         if self.model_type == "CustomVoice":
             return "CustomVoice"
-        if self.model_type == "VoiceClone":
+        if self.model_type in ("VoiceClone", "Base"):
             return "VoiceClone"
         if self.model_type == "VoiceDesign":
             return "VoiceDesign"
@@ -380,6 +380,8 @@ def _detect_mode(self, request):
 
     def _get_ref_audio_path(self, request, voice_name=None):
         """Get reference audio path from stored AudioPath or from voices dict."""
+        if hasattr(request, "voice") and request.voice and os.path.isfile(request.voice):
+            return request.voice
         # If voice_name is provided and exists in voices dict, use that
         if voice_name and voice_name in self.voices:
             audio_path = self.voices[voice_name]["audio"]
@@ -735,6 +737,8 @@ def TTS(self, request, context):
             # model receives the types it expects. These override YAML-derived kwargs.
             if hasattr(request, "params") and request.params:
                 for key, value in request.params.items():
+                    if key == "ref_text":
+                        continue
                     generation_kwargs[key] = coerce_param_value(value)
 
             # Generate audio based on mode
@@ -743,7 +747,8 @@ def TTS(self, request, context):
 
                 # Check if multi-voice mode is active (voices dict is populated)
                 voice_name = None
-                if self.voices:
+                request_voice_path = request.voice if request.voice and os.path.isfile(request.voice) else None
+                if self.voices and request_voice_path is None:
                     # Get voice from request (priority) or options
                     voice_name = request.voice if request.voice else None
                     if not voice_name:
@@ -775,11 +780,9 @@ def TTS(self, request, context):
                 if voice_name and voice_name in self.voices:
                     ref_text_source = self.voices[voice_name]["ref_text"]
                 else:
-                    ref_text_source = self.options.get("ref_text", None)
+                    ref_text_source = request.params.get("ref_text") if hasattr(request, "params") else None
                     if not ref_text_source:
-                        # Try to get from request if available
-                        if hasattr(request, "ref_text") and request.ref_text:
-                            ref_text_source = request.ref_text
+                        ref_text_source = self.options.get("ref_text", None)
 
                 if not ref_text_source:
                     # x_vector_only_mode doesn't require ref_text