Skip to content

Commit 4cee67e

Browse files
tjbcktjbck
committed
feat: mistral tts
1 parent 730e52a commit 4cee67e

4 files changed

Lines changed: 184 additions & 0 deletions

File tree

‎backend/open_webui/config.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3940,6 +3940,18 @@ class BannerModel(BaseModel):
39403940
os.getenv('AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT', 'audio-24khz-160kbitrate-mono-mp3'),
39413941
)
39423942

3943+
AUDIO_TTS_MISTRAL_API_KEY = PersistentConfig(
3944+
'AUDIO_TTS_MISTRAL_API_KEY',
3945+
'audio.tts.mistral.api_key',
3946+
os.getenv('AUDIO_TTS_MISTRAL_API_KEY', ''),
3947+
)
3948+
3949+
AUDIO_TTS_MISTRAL_API_BASE_URL = PersistentConfig(
3950+
'AUDIO_TTS_MISTRAL_API_BASE_URL',
3951+
'audio.tts.mistral.api_base_url',
3952+
os.getenv('AUDIO_TTS_MISTRAL_API_BASE_URL', 'https://api.mistral.ai/v1'),
3953+
)
3954+
39433955

39443956
####################################
39453957
# LDAP

‎backend/open_webui/main.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,8 @@
212212
AUDIO_TTS_AZURE_SPEECH_REGION,
213213
AUDIO_TTS_AZURE_SPEECH_BASE_URL,
214214
AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT,
215+
AUDIO_TTS_MISTRAL_API_KEY,
216+
AUDIO_TTS_MISTRAL_API_BASE_URL,
215217
PLAYWRIGHT_WS_URL,
216218
PLAYWRIGHT_TIMEOUT,
217219
FIRECRAWL_API_BASE_URL,
@@ -1284,6 +1286,9 @@ async def lifespan(app: FastAPI):
12841286
app.state.config.TTS_AZURE_SPEECH_BASE_URL = AUDIO_TTS_AZURE_SPEECH_BASE_URL
12851287
app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT = AUDIO_TTS_AZURE_SPEECH_OUTPUT_FORMAT
12861288

1289+
app.state.config.TTS_MISTRAL_API_KEY = AUDIO_TTS_MISTRAL_API_KEY
1290+
app.state.config.TTS_MISTRAL_API_BASE_URL = AUDIO_TTS_MISTRAL_API_BASE_URL
1291+
12871292

12881293
app.state.faster_whisper_model = None
12891294
app.state.speech_synthesiser = None

‎backend/open_webui/routers/audio.py‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,8 @@ class TTSConfigForm(BaseModel):
168168
AZURE_SPEECH_REGION: str
169169
AZURE_SPEECH_BASE_URL: str
170170
AZURE_SPEECH_OUTPUT_FORMAT: str
171+
MISTRAL_API_KEY: str
172+
MISTRAL_API_BASE_URL: str
171173

172174

173175
class STTConfigForm(BaseModel):
@@ -208,6 +210,8 @@ async def get_audio_config(request: Request, user=Depends(get_admin_user)):
208210
'AZURE_SPEECH_REGION': request.app.state.config.TTS_AZURE_SPEECH_REGION,
209211
'AZURE_SPEECH_BASE_URL': request.app.state.config.TTS_AZURE_SPEECH_BASE_URL,
210212
'AZURE_SPEECH_OUTPUT_FORMAT': request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
213+
'MISTRAL_API_KEY': request.app.state.config.TTS_MISTRAL_API_KEY,
214+
'MISTRAL_API_BASE_URL': request.app.state.config.TTS_MISTRAL_API_BASE_URL,
211215
},
212216
'stt': {
213217
'OPENAI_API_BASE_URL': request.app.state.config.STT_OPENAI_API_BASE_URL,
@@ -242,6 +246,8 @@ async def update_audio_config(request: Request, form_data: AudioConfigUpdateForm
242246
request.app.state.config.TTS_AZURE_SPEECH_REGION = form_data.tts.AZURE_SPEECH_REGION
243247
request.app.state.config.TTS_AZURE_SPEECH_BASE_URL = form_data.tts.AZURE_SPEECH_BASE_URL
244248
request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT = form_data.tts.AZURE_SPEECH_OUTPUT_FORMAT
249+
request.app.state.config.TTS_MISTRAL_API_KEY = form_data.tts.MISTRAL_API_KEY
250+
request.app.state.config.TTS_MISTRAL_API_BASE_URL = form_data.tts.MISTRAL_API_BASE_URL
245251

246252
request.app.state.config.STT_OPENAI_API_BASE_URL = form_data.stt.OPENAI_API_BASE_URL
247253
request.app.state.config.STT_OPENAI_API_KEY = form_data.stt.OPENAI_API_KEY
@@ -280,6 +286,8 @@ async def update_audio_config(request: Request, form_data: AudioConfigUpdateForm
280286
'AZURE_SPEECH_REGION': request.app.state.config.TTS_AZURE_SPEECH_REGION,
281287
'AZURE_SPEECH_BASE_URL': request.app.state.config.TTS_AZURE_SPEECH_BASE_URL,
282288
'AZURE_SPEECH_OUTPUT_FORMAT': request.app.state.config.TTS_AZURE_SPEECH_OUTPUT_FORMAT,
289+
'MISTRAL_API_KEY': request.app.state.config.TTS_MISTRAL_API_KEY,
290+
'MISTRAL_API_BASE_URL': request.app.state.config.TTS_MISTRAL_API_BASE_URL,
283291
},
284292
'stt': {
285293
'OPENAI_API_BASE_URL': request.app.state.config.STT_OPENAI_API_BASE_URL,
@@ -551,6 +559,76 @@ async def speech(request: Request, user=Depends(get_verified_user)):
551559

552560
return FileResponse(file_path)
553561

562+
elif request.app.state.config.TTS_ENGINE == 'mistral':
563+
api_key = request.app.state.config.TTS_MISTRAL_API_KEY
564+
api_base_url = request.app.state.config.TTS_MISTRAL_API_BASE_URL or 'https://api.mistral.ai/v1'
565+
566+
if not api_key:
567+
raise HTTPException(
568+
status_code=400,
569+
detail='Mistral API key is required for Mistral TTS',
570+
)
571+
572+
try:
573+
timeout = aiohttp.ClientTimeout(total=AIOHTTP_CLIENT_TIMEOUT)
574+
async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session:
575+
mistral_payload = {
576+
'input': payload.get('input', ''),
577+
'model': request.app.state.config.TTS_MODEL or 'mistral-tts-latest',
578+
'voice_id': payload.get('voice', ''),
579+
'response_format': 'mp3',
580+
}
581+
582+
r = await session.post(
583+
url=f'{api_base_url}/audio/speech',
584+
json=mistral_payload,
585+
headers={
586+
'Content-Type': 'application/json',
587+
'Authorization': f'Bearer {api_key}',
588+
},
589+
ssl=AIOHTTP_CLIENT_SESSION_SSL,
590+
)
591+
592+
r.raise_for_status()
593+
594+
res = await r.json()
595+
audio_data = res.get('audio_data', '')
596+
if not audio_data:
597+
raise ValueError('No audio_data in Mistral TTS response')
598+
599+
audio_bytes = base64.b64decode(audio_data)
600+
601+
async with aiofiles.open(file_path, 'wb') as f:
602+
await f.write(audio_bytes)
603+
604+
async with aiofiles.open(file_body_path, 'w') as f:
605+
await f.write(json.dumps(payload))
606+
607+
return FileResponse(file_path)
608+
609+
except Exception as e:
610+
log.exception(e)
611+
detail = None
612+
613+
status_code = 500
614+
detail = 'Open WebUI: Server Connection Error'
615+
616+
if r is not None:
617+
status_code = r.status
618+
619+
try:
620+
res = await r.json()
621+
if 'error' in res:
622+
detail = f'External: {res["error"]}'
623+
elif 'message' in res:
624+
detail = f'External: {res["message"]}'
625+
except Exception:
626+
detail = f'External: {e}'
627+
628+
raise HTTPException(
629+
status_code=status_code,
630+
detail=detail,
631+
)
554632

555633
def transcription_handler(request, file_path, metadata, user=None):
556634
filename = os.path.basename(file_path)
@@ -1238,6 +1316,8 @@ def get_available_models(request: Request) -> list[dict]:
12381316
available_models = [{'name': model['name'], 'id': model['model_id']} for model in models]
12391317
except requests.RequestException as e:
12401318
log.error(f'Error fetching voices: {str(e)}')
1319+
elif request.app.state.config.TTS_ENGINE == 'mistral':
1320+
available_models = [{'id': 'mistral-tts-latest'}]
12411321
return available_models
12421322

12431323

@@ -1301,6 +1381,29 @@ def get_available_voices(request) -> dict:
13011381
available_voices[voice['ShortName']] = f'{voice["DisplayName"]} ({voice["ShortName"]})'
13021382
except requests.RequestException as e:
13031383
log.error(f'Error fetching voices: {str(e)}')
1384+
elif request.app.state.config.TTS_ENGINE == 'mistral':
1385+
api_key = request.app.state.config.TTS_MISTRAL_API_KEY
1386+
api_base_url = request.app.state.config.TTS_MISTRAL_API_BASE_URL or 'https://api.mistral.ai/v1'
1387+
1388+
if api_key:
1389+
try:
1390+
response = requests.get(
1391+
f'{api_base_url}/audio/voices',
1392+
headers={
1393+
'Authorization': f'Bearer {api_key}',
1394+
},
1395+
timeout=AIOHTTP_CLIENT_TIMEOUT_MODEL_LIST,
1396+
)
1397+
response.raise_for_status()
1398+
voices_data = response.json()
1399+
1400+
for voice in voices_data:
1401+
voice_id = voice.get('voice_id', voice.get('id', ''))
1402+
voice_name = voice.get('name', voice_id)
1403+
if voice_id:
1404+
available_voices[voice_id] = voice_name
1405+
except requests.RequestException as e:
1406+
log.error(f'Error fetching Mistral voices: {str(e)}')
13041407

13051408
return available_voices
13061409

‎src/lib/components/admin/Settings/Audio.svelte‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
let TTS_AZURE_SPEECH_REGION = '';
3838
let TTS_AZURE_SPEECH_BASE_URL = '';
3939
let TTS_AZURE_SPEECH_OUTPUT_FORMAT = '';
40+
let TTS_MISTRAL_API_KEY = '';
41+
let TTS_MISTRAL_API_BASE_URL = '';
4042
4143
let STT_OPENAI_API_BASE_URL = '';
4244
let STT_OPENAI_API_KEY = '';
@@ -124,6 +126,8 @@
124126
AZURE_SPEECH_REGION: TTS_AZURE_SPEECH_REGION,
125127
AZURE_SPEECH_BASE_URL: TTS_AZURE_SPEECH_BASE_URL,
126128
AZURE_SPEECH_OUTPUT_FORMAT: TTS_AZURE_SPEECH_OUTPUT_FORMAT,
129+
MISTRAL_API_KEY: TTS_MISTRAL_API_KEY,
130+
MISTRAL_API_BASE_URL: TTS_MISTRAL_API_BASE_URL,
127131
SPLIT_ON: TTS_SPLIT_ON
128132
},
129133
stt: {
@@ -176,6 +180,8 @@
176180
TTS_AZURE_SPEECH_REGION = res.tts.AZURE_SPEECH_REGION;
177181
TTS_AZURE_SPEECH_BASE_URL = res.tts.AZURE_SPEECH_BASE_URL;
178182
TTS_AZURE_SPEECH_OUTPUT_FORMAT = res.tts.AZURE_SPEECH_OUTPUT_FORMAT;
183+
TTS_MISTRAL_API_KEY = res.tts.MISTRAL_API_KEY;
184+
TTS_MISTRAL_API_BASE_URL = res.tts.MISTRAL_API_BASE_URL;
179185
180186
STT_OPENAI_API_BASE_URL = res.stt.OPENAI_API_BASE_URL;
181187
STT_OPENAI_API_KEY = res.stt.OPENAI_API_KEY;
@@ -517,6 +523,9 @@
517523
if (e.target?.value === 'openai') {
518524
TTS_VOICE = 'alloy';
519525
TTS_MODEL = 'tts-1';
526+
} else if (e.target?.value === 'mistral') {
527+
TTS_VOICE = '';
528+
TTS_MODEL = 'mistral-tts-latest';
520529
} else {
521530
TTS_VOICE = '';
522531
TTS_MODEL = '';
@@ -528,6 +537,7 @@
528537
<option value="openai">{$i18n.t('OpenAI')}</option>
529538
<option value="elevenlabs">{$i18n.t('ElevenLabs')}</option>
530539
<option value="azure">{$i18n.t('Azure AI Speech')}</option>
540+
<option value="mistral">{$i18n.t('MistralAI')}</option>
531541
</select>
532542
</div>
533543
</div>
@@ -585,6 +595,19 @@
585595
</div>
586596
</div>
587597
</div>
598+
{:else if TTS_ENGINE === 'mistral'}
599+
<div>
600+
<div class="mt-1 flex gap-2 mb-1">
601+
<input
602+
class="flex-1 w-full bg-transparent outline-hidden"
603+
placeholder={$i18n.t('API Base URL')}
604+
bind:value={TTS_MISTRAL_API_BASE_URL}
605+
required
606+
/>
607+
608+
<SensitiveInput placeholder={$i18n.t('API Key')} bind:value={TTS_MISTRAL_API_KEY} />
609+
</div>
610+
</div>
588611
{/if}
589612

590613
<div class="mb-2">
@@ -791,6 +814,47 @@
791814
</div>
792815
</div>
793816
</div>
817+
{:else if TTS_ENGINE === 'mistral'}
818+
<div class=" flex gap-2">
819+
<div class="w-full">
820+
<div class=" mb-1.5 text-xs font-medium">{$i18n.t('TTS Voice')}</div>
821+
<div class="flex w-full">
822+
<div class="flex-1">
823+
<input
824+
list="voice-list"
825+
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
826+
bind:value={TTS_VOICE}
827+
placeholder={$i18n.t('Select a voice')}
828+
/>
829+
830+
<datalist id="voice-list">
831+
{#each voices as voice}
832+
<option value={voice.id}>{voice.name}</option>
833+
{/each}
834+
</datalist>
835+
</div>
836+
</div>
837+
</div>
838+
<div class="w-full">
839+
<div class=" mb-1.5 text-xs font-medium">{$i18n.t('TTS Model')}</div>
840+
<div class="flex w-full">
841+
<div class="flex-1">
842+
<input
843+
list="tts-model-list"
844+
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
845+
bind:value={TTS_MODEL}
846+
placeholder={$i18n.t('Select a model')}
847+
/>
848+
849+
<datalist id="tts-model-list">
850+
{#each models as model}
851+
<option value={model.id} class="bg-gray-50 dark:bg-gray-700" />
852+
{/each}
853+
</datalist>
854+
</div>
855+
</div>
856+
</div>
857+
</div>
794858
{/if}
795859
</div>
796860

0 commit comments

Comments
 (0)