feat(proxy): adapt VoxCPM clone through TTS API

skyguan92 · skyguan92 · commit 81da6095d599 · 2026-05-14T18:47:40.000+08:00
diff --git a/internal/inferencehttp/routes.go b/internal/inferencehttp/routes.go
@@ -8,6 +8,7 @@ import (
 	"io"
 	"log/slog"
 	"mime"
+	"mime/multipart"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
@@ -19,6 +20,7 @@ const maxTTSRequestBody = 16 << 20
 const (
 	adapterLiteTTSHTTP = "litetts_http"
 	adapterMooERGRPC   = "mooer_grpc"
+	adapterVoxCPMClone = "voxcpm_clone"
 )
 
 // RegisterRoutes returns a function that registers AIMA inference proxy routes.
@@ -74,10 +76,15 @@ func (d *Deps) handleTTS(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if d.adapterFor(model, r.URL.Path) == adapterLiteTTSHTTP {
+	adapter := d.adapterFor(model, r.URL.Path)
+	if adapter == adapterLiteTTSHTTP {
 		d.handleLiteTTS(w, r, backend, raw)
 		return
 	}
+	if adapter == adapterVoxCPMClone && hasTTSReferenceAudio(raw) {
+		d.handleVoxCPMClone(w, r, backend, raw, body)
+		return
+	}
 
 	switch r.URL.Path {
 	case "/v1/tts":
@@ -504,6 +511,122 @@ func (d *Deps) forwardTTSJSON(w http.ResponseWriter, r *http.Request, backend *B
 	writeBackendResponse(w, resp, respBody)
 }
 
+func (d *Deps) handleVoxCPMClone(w http.ResponseWriter, r *http.Request, backend *Backend, raw map[string]any, requestBody []byte) {
+	body, contentType, err := buildVoxCPMCloneRequest(raw)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	resp, respBody, err := d.callBackend(r, backend.Address, "/v1/clone", contentType, body)
+	if err != nil {
+		slog.Warn("aima proxy: VoxCPM clone backend request failed", "backend", backend.Address, "err", err)
+		http.Error(w, "backend unreachable", http.StatusBadGateway)
+		return
+	}
+
+	if r.URL.Path == "/v1/tts" && resp.StatusCode >= 200 && resp.StatusCode < 300 && isAudioContent(resp.Header.Get("Content-Type")) {
+		writeAudioJSON(w, respBody, requestBody, resp.Header.Get("Content-Type"), resp.StatusCode)
+		return
+	}
+	if r.URL.Path == "/v1/audio/speech" && resp.StatusCode >= 200 && resp.StatusCode < 300 && writeAudioFromJSON(w, respBody, requestBody, resp.StatusCode) {
+		return
+	}
+	writeBackendResponse(w, resp, respBody)
+}
+
+func buildVoxCPMCloneRequest(raw map[string]any) ([]byte, string, error) {
+	text := extractTTSText(raw)
+	if text == "" {
+		return nil, "", fmt.Errorf(`{"error":"missing or invalid input field"}`)
+	}
+	refAudio := firstTTSString(raw, "reference_audio", "ref_audio")
+	if refAudio == "" {
+		return nil, "", fmt.Errorf(`{"error":"missing or invalid reference_audio field"}`)
+	}
+	audio, filename, err := decodeReferenceAudio(refAudio)
+	if err != nil {
+		return nil, "", err
+	}
+
+	var body bytes.Buffer
+	writer := multipart.NewWriter(&body)
+	if err := writer.WriteField("text", text); err != nil {
+		return nil, "", err
+	}
+	if refText := firstTTSString(raw, "reference_text", "ref_text"); refText != "" {
+		if err := writer.WriteField("ref_text", refText); err != nil {
+			return nil, "", err
+		}
+	}
+	for _, key := range []string{"response_format", "temperature", "cfg", "max_length"} {
+		if value, ok := raw[key]; ok {
+			if err := writer.WriteField(key, fmt.Sprint(value)); err != nil {
+				return nil, "", err
+			}
+		}
+	}
+	part, err := writer.CreateFormFile("ref_audio", filename)
+	if err != nil {
+		return nil, "", err
+	}
+	if _, err := part.Write(audio); err != nil {
+		return nil, "", err
+	}
+	if err := writer.Close(); err != nil {
+		return nil, "", err
+	}
+	return body.Bytes(), writer.FormDataContentType(), nil
+}
+
+func hasTTSReferenceAudio(raw map[string]any) bool {
+	return firstTTSString(raw, "reference_audio", "ref_audio") != ""
+}
+
+func firstTTSString(raw map[string]any, keys ...string) string {
+	for _, key := range keys {
+		if value, _ := raw[key].(string); strings.TrimSpace(value) != "" {
+			return strings.TrimSpace(value)
+		}
+	}
+	return ""
+}
+
+func decodeReferenceAudio(value string) ([]byte, string, error) {
+	value = strings.TrimSpace(value)
+	if strings.HasPrefix(strings.ToLower(value), "data:") {
+		return decodeReferenceAudioDataURL(value)
+	}
+	audio, err := base64.StdEncoding.DecodeString(value)
+	if err != nil {
+		return nil, "", fmt.Errorf(`{"error":"reference_audio must be a data URL or base64 audio"}`)
+	}
+	return audio, "reference.wav", nil
+}
+
+func decodeReferenceAudioDataURL(value string) ([]byte, string, error) {
+	comma := strings.IndexByte(value, ',')
+	if comma < 0 {
+		return nil, "", fmt.Errorf(`{"error":"invalid reference_audio data URL"}`)
+	}
+	meta := value[len("data:"):comma]
+	payload := value[comma+1:]
+	if !strings.Contains(strings.ToLower(meta), ";base64") {
+		return nil, "", fmt.Errorf(`{"error":"reference_audio data URL must be base64 encoded"}`)
+	}
+	audio, err := base64.StdEncoding.DecodeString(payload)
+	if err != nil {
+		return nil, "", fmt.Errorf(`{"error":"invalid reference_audio base64 data"}`)
+	}
+
+	contentType := strings.TrimSpace(strings.Split(meta, ";")[0])
+	format := audioFormatFromContentType(contentType)
+	if format == "" {
+		format = "wav"
+	}
+	return audio, "reference." + format, nil
+}
+
 func (d *Deps) callBackend(r *http.Request, targetAddr, targetPath, contentType string, body []byte) (*http.Response, []byte, error) {
 	if !strings.HasPrefix(targetAddr, "http://") && !strings.HasPrefix(targetAddr, "https://") {
 		targetAddr = "http://" + targetAddr
diff --git a/internal/inferencehttp/routes_test.go b/internal/inferencehttp/routes_test.go
@@ -3,6 +3,7 @@ package inferencehttp
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"io"
 	"mime/multipart"
@@ -386,6 +387,145 @@ func TestHandleTTSJSONFallsBackToSpeechAndWrapsAudio(t *testing.T) {
 	}
 }
 
+func TestHandleTTSVoxCPMCloneJSONRoutesToCloneAndWrapsAudio(t *testing.T) {
+	var (
+		gotPath        string
+		gotContentType string
+		gotFields      map[string][]string
+		gotRefAudio    []byte
+		gotRefName     string
+	)
+	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		gotContentType = r.Header.Get("Content-Type")
+		if err := r.ParseMultipartForm(1 << 20); err != nil {
+			t.Fatalf("ParseMultipartForm: %v", err)
+		}
+		gotFields = r.MultipartForm.Value
+		file, header, err := r.FormFile("ref_audio")
+		if err != nil {
+			t.Fatalf("FormFile ref_audio: %v", err)
+		}
+		defer file.Close()
+		gotRefName = header.Filename
+		gotRefAudio, err = io.ReadAll(file)
+		if err != nil {
+			t.Fatalf("ReadAll ref_audio: %v", err)
+		}
+		w.Header().Set("Content-Type", "audio/wav")
+		_, _ = w.Write([]byte("RIFFclone"))
+	}))
+	defer backend.Close()
+
+	deps := &Deps{
+		Backends: staticBackendLister{backends: map[string]*Backend{
+			"voxcpm2": {
+				ModelName: "voxcpm2",
+				Address:   strings.TrimPrefix(backend.URL, "http://"),
+				Ready:     true,
+			},
+		}},
+		Catalog: staticCatalog{adapters: map[string][]Adapter{
+			"voxcpm2": {{Path: "/v1/tts", Kind: adapterVoxCPMClone}},
+		}},
+	}
+
+	mux := http.NewServeMux()
+	RegisterRoutes(deps)(mux)
+
+	reqBody := `{"model":"voxcpm2","text":"hello","response_format":"wav","reference_audio":"data:audio/wav;base64,UklGRg==","reference_text":"sample voice"}`
+	req := httptest.NewRequest(http.MethodPost, "/v1/tts", strings.NewReader(reqBody))
+	req.Header.Set("Content-Type", "application/json")
+	w := httptest.NewRecorder()
+	mux.ServeHTTP(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("status = %d, want %d; body=%s", w.Code, http.StatusOK, w.Body.String())
+	}
+	if gotPath != "/v1/clone" {
+		t.Fatalf("backend path = %q, want /v1/clone", gotPath)
+	}
+	if !strings.HasPrefix(gotContentType, "multipart/form-data; boundary=") {
+		t.Fatalf("content-type = %q, want multipart boundary", gotContentType)
+	}
+	field := func(name string) string {
+		if values := gotFields[name]; len(values) > 0 {
+			return values[0]
+		}
+		return ""
+	}
+	if field("text") != "hello" {
+		t.Fatalf("text field = %q, want hello", field("text"))
+	}
+	if field("ref_text") != "sample voice" {
+		t.Fatalf("ref_text field = %q, want sample voice", field("ref_text"))
+	}
+	if field("response_format") != "wav" {
+		t.Fatalf("response_format field = %q, want wav", field("response_format"))
+	}
+	if gotRefName != "reference.wav" {
+		t.Fatalf("ref_audio filename = %q, want reference.wav", gotRefName)
+	}
+	if string(gotRefAudio) != "RIFF" {
+		t.Fatalf("ref_audio = %q, want RIFF", string(gotRefAudio))
+	}
+	var resp map[string]any
+	if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+		t.Fatalf("Unmarshal response: %v", err)
+	}
+	if resp["audio_base64"] != base64.StdEncoding.EncodeToString([]byte("RIFFclone")) {
+		t.Fatalf("audio_base64 = %#v, want encoded RIFFclone", resp["audio_base64"])
+	}
+	if resp["format"] != "wav" {
+		t.Fatalf("format = %#v, want wav", resp["format"])
+	}
+}
+
+func TestHandleTTSSpeechVoxCPMCloneReturnsAudio(t *testing.T) {
+	var gotPath string
+	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		w.Header().Set("Content-Type", "audio/wav")
+		_, _ = w.Write([]byte("RIFFclone"))
+	}))
+	defer backend.Close()
+
+	deps := &Deps{
+		Backends: staticBackendLister{backends: map[string]*Backend{
+			"voxcpm2": {
+				ModelName: "voxcpm2",
+				Address:   strings.TrimPrefix(backend.URL, "http://"),
+				Ready:     true,
+			},
+		}},
+		Catalog: staticCatalog{adapters: map[string][]Adapter{
+			"voxcpm2": {{Path: "/v1/audio/speech", Kind: adapterVoxCPMClone}},
+		}},
+	}
+
+	mux := http.NewServeMux()
+	RegisterRoutes(deps)(mux)
+
+	reqBody := `{"model":"voxcpm2","input":"hello","reference_audio":"UklGRg=="}`
+	req := httptest.NewRequest(http.MethodPost, "/v1/audio/speech", strings.NewReader(reqBody))
+	req.Header.Set("Content-Type", "application/json")
+	w := httptest.NewRecorder()
+	mux.ServeHTTP(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("status = %d, want %d; body=%s", w.Code, http.StatusOK, w.Body.String())
+	}
+	if gotPath != "/v1/clone" {
+		t.Fatalf("backend path = %q, want /v1/clone", gotPath)
+	}
+	if got := w.Body.String(); got != "RIFFclone" {
+		t.Fatalf("body = %q, want RIFFclone", got)
+	}
+	if ct := w.Header().Get("Content-Type"); ct != "audio/wav" {
+		t.Fatalf("content-type = %q, want audio/wav", ct)
+	}
+}
+
 func TestHandleTTSSpeechFallsBackToJSONAndDecodesAudio(t *testing.T) {
 	var paths []string
 	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
