From 2fade5cddc25a82aa07395b533f1b0f3adeeaf48 Mon Sep 17 00:00:00 2001 From: tangtaizhong666 Date: Fri, 3 Jul 2026 07:30:48 +0800 Subject: [PATCH 1/2] fix: adapt MiMo STT to V2.5 models and reject non-WAV audio The MiMo-V2 series went offline on 2026-06-30, so the default STT model mimo-v2-omni fails for every default configuration. Switch the default to mimo-v2.5-asr, the dedicated speech recognition model whose official docs use exactly the bare input_audio payload this provider sends. For non-ASR multimodal models such as mimo-v2.5, the audio understanding docs require a text instruction alongside the audio, so restore the system/user transcription prompts for that model family only. Also validate that the resolved audio payload really is RIFF/WAVE before calling the API: when a platform voice file (e.g. Tencent SILK from QQ) slips through the WAV conversion chain unchanged, fail locally with an actionable error instead of the opaque HTTP 400 from the API. Fixes #9113 --- astrbot/core/config/default.py | 2 +- .../core/provider/sources/mimo_api_common.py | 45 ++++++- .../provider/sources/mimo_stt_api_source.py | 54 ++++++-- tests/test_mimo_api_sources.py | 122 +++++++++++++++++- 4 files changed, 204 insertions(+), 19 deletions(-) diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index 7fb847dccd..6156601248 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -1593,7 +1593,7 @@ "enable": False, "api_key": "", "api_base": "https://api.xiaomimimo.com/v1", - "model": "mimo-v2-omni", + "model": "mimo-v2.5-asr", "timeout": "20", "proxy": "", }, diff --git a/astrbot/core/provider/sources/mimo_api_common.py b/astrbot/core/provider/sources/mimo_api_common.py index e2dc6f92bd..36901a5a9c 100644 --- a/astrbot/core/provider/sources/mimo_api_common.py +++ b/astrbot/core/provider/sources/mimo_api_common.py @@ -1,3 +1,4 @@ +import base64 from pathlib import Path import httpx @@ -10,7 +11,16 @@ DEFAULT_MIMO_TTS_MODEL = "mimo-v2-tts" DEFAULT_MIMO_TTS_VOICE = "mimo_default" DEFAULT_MIMO_TTS_SEED_TEXT = "Hello, MiMo, have you had lunch?" -DEFAULT_MIMO_STT_MODEL = "mimo-v2-omni" +# The MiMo-V2 series went offline on 2026-06-30; mimo-v2.5-asr is the +# dedicated speech recognition model per the official model lineup. +DEFAULT_MIMO_STT_MODEL = "mimo-v2.5-asr" +DEFAULT_MIMO_STT_SYSTEM_PROMPT = ( + "You are a speech transcription assistant. " + "Transcribe the spoken content from the audio exactly and return only the transcription text." +) +DEFAULT_MIMO_STT_USER_PROMPT = ( + "Please transcribe the content of the audio and return only the transcription text." +) class MiMoAPIError(Exception): @@ -67,9 +77,42 @@ async def prepare_audio_input(audio_source: str) -> tuple[str, list[Path]]: ) if audio_data is None: raise ValueError(f"Invalid audio data: {describe_media_ref(audio_source)}") + _validate_wav_payload(audio_data.base64_data, audio_source) return audio_data.to_data_url(), [] +def _validate_wav_payload(base64_data: str, audio_source: str) -> None: + """Reject audio payloads whose bytes are not RIFF/WAVE. + + MiMo only accepts wav/mp3 audio. When a platform voice file (e.g. Tencent + SILK from QQ) slips through the WAV conversion chain unchanged, the API + replies with an opaque HTTP 400, so fail locally with the real reason. + + Args: + base64_data: Base64-encoded audio payload about to be sent. + audio_source: Original media reference, used in error messages. + + Raises: + MiMoAPIError: Raised when the payload is not valid WAV data. + """ + try: + header = base64.b64decode(base64_data[:64]) + except Exception: + header = b"" + if len(header) >= 12 and header[:4] == b"RIFF" and header[8:12] == b"WAVE": + return + if header.startswith((b"#!SILK_V3", b"\x02#!SILK_V3")): + raise MiMoAPIError( + "Audio for MiMo STT is still Tencent SILK data after WAV conversion; " + "check that the silk-python package is installed and working: " + f"{describe_media_ref(audio_source)}" + ) + raise MiMoAPIError( + "Audio for MiMo STT could not be converted to WAV " + f"(unrecognized audio bytes): {describe_media_ref(audio_source)}" + ) + + def cleanup_files(paths: list[Path]) -> None: for path in paths: try: diff --git a/astrbot/core/provider/sources/mimo_stt_api_source.py b/astrbot/core/provider/sources/mimo_stt_api_source.py index e267cf1e68..8417e7a988 100644 --- a/astrbot/core/provider/sources/mimo_stt_api_source.py +++ b/astrbot/core/provider/sources/mimo_stt_api_source.py @@ -4,6 +4,8 @@ from .mimo_api_common import ( DEFAULT_MIMO_API_BASE, DEFAULT_MIMO_STT_MODEL, + DEFAULT_MIMO_STT_SYSTEM_PROMPT, + DEFAULT_MIMO_STT_USER_PROMPT, MiMoAPIError, build_api_url, build_headers, @@ -33,23 +35,49 @@ def __init__( self.set_model(provider_config.get("model", DEFAULT_MIMO_STT_MODEL)) self.client = create_http_client(self.timeout, self.proxy) + def _is_asr_model(self) -> bool: + return "asr" in (self.model_name or "").lower() + + def _build_messages(self, audio_data_url: str) -> list[dict]: + audio_content = { + "type": "input_audio", + "input_audio": { + "data": audio_data_url, + }, + } + if self._is_asr_model(): + # Dedicated ASR models (speech-recognition docs) take bare audio. + return [ + { + "role": "user", + "content": [audio_content], + }, + ] + # Multimodal models such as mimo-v2.5 (audio-understanding docs) + # require a text instruction alongside the audio, otherwise the API + # rejects the request. + return [ + { + "role": "system", + "content": DEFAULT_MIMO_STT_SYSTEM_PROMPT, + }, + { + "role": "user", + "content": [ + audio_content, + { + "type": "text", + "text": DEFAULT_MIMO_STT_USER_PROMPT, + }, + ], + }, + ] + async def get_text(self, audio_url: str) -> str: audio_data_url, cleanup_paths = await prepare_audio_input(audio_url) payload = { "model": self.model_name, - "messages": [ - { - "role": "user", - "content": [ - { - "type": "input_audio", - "input_audio": { - "data": audio_data_url, - }, - }, - ], - }, - ], + "messages": self._build_messages(audio_data_url), "max_completion_tokens": 1024, } diff --git a/tests/test_mimo_api_sources.py b/tests/test_mimo_api_sources.py index c0b7abc7c2..da1a947e1e 100644 --- a/tests/test_mimo_api_sources.py +++ b/tests/test_mimo_api_sources.py @@ -1,4 +1,5 @@ import asyncio +import base64 from types import SimpleNamespace import pytest @@ -11,7 +12,9 @@ from astrbot.core.provider.sources.mimo_stt_api_source import ProviderMiMoSTTAPI from astrbot.core.provider.sources.mimo_tts_api_source import ProviderMiMoTTSAPI -MIMO_STT_TEST_AUDIO_DATA_URL = "data:audio/wav;base64,ZmFrZQ==" +MIMO_STT_TEST_WAV_HEADER = b"RIFF\x24\x08\x00\x00WAVEfmt " +MIMO_STT_TEST_AUDIO_BASE64 = base64.b64encode(MIMO_STT_TEST_WAV_HEADER).decode() +MIMO_STT_TEST_AUDIO_DATA_URL = f"data:audio/wav;base64,{MIMO_STT_TEST_AUDIO_BASE64}" def _make_tts_provider(overrides: dict | None = None) -> ProviderMiMoTTSAPI: @@ -33,7 +36,7 @@ def _make_stt_provider(overrides: dict | None = None) -> ProviderMiMoSTTAPI: provider_config = { "id": "test-mimo-stt", "type": "mimo_stt_api", - "model": "mimo-v2-omni", + "model": "mimo-v2.5-asr", "api_key": "test-key", } if overrides: @@ -196,7 +199,8 @@ def json(self): @pytest.mark.asyncio -async def test_mimo_stt_payload_includes_audio_only(monkeypatch): +async def test_mimo_stt_asr_model_payload_includes_audio_only(monkeypatch): + """专用 ASR 模型按官方语音识别文档只传 input_audio，不带任何提示词。""" provider = _make_stt_provider( { "mimo-stt-system-prompt": "system prompt", @@ -248,10 +252,91 @@ async def fake_post(_url, headers=None, json=None): ] +def test_mimo_stt_default_model_is_v25_asr(): + """mimo-v2-omni 已于 2026-06-30 下线，默认模型应为 mimo-v2.5-asr。""" + provider = ProviderMiMoSTTAPI( + provider_config={ + "id": "test-mimo-stt", + "type": "mimo_stt_api", + "api_key": "test-key", + }, + provider_settings={}, + ) + try: + assert provider.model_name == "mimo-v2.5-asr" + finally: + asyncio.run(provider.terminate()) + + +@pytest.mark.asyncio +async def test_mimo_stt_multimodal_model_payload_includes_transcription_prompts( + monkeypatch, +): + """非 ASR 模型（如 mimo-v2.5）按官方音频理解文档要求携带 system 与 text 指令。""" + provider = _make_stt_provider({"model": "mimo-v2.5"}) + + captured: dict = {} + + async def fake_prepare_audio_input(_audio_source: str): + return MIMO_STT_TEST_AUDIO_DATA_URL, [] + + class _Response: + status_code = 200 + text = '{"choices":[{"message":{"content":"transcribed text"}}]}' + + def raise_for_status(self): + return None + + def json(self): + return {"choices": [{"message": {"content": "transcribed text"}}]} + + async def fake_post(_url, headers=None, json=None): + captured["json"] = json + return _Response() + + monkeypatch.setattr( + "astrbot.core.provider.sources.mimo_stt_api_source.prepare_audio_input", + fake_prepare_audio_input, + ) + provider.client = SimpleNamespace(post=fake_post) + + result = await provider.get_text("/tmp/test.wav") + + assert result == "transcribed text" + assert captured["json"]["messages"] == [ + { + "role": "system", + "content": ( + "You are a speech transcription assistant. " + "Transcribe the spoken content from the audio exactly " + "and return only the transcription text." + ), + }, + { + "role": "user", + "content": [ + { + "type": "input_audio", + "input_audio": { + "data": MIMO_STT_TEST_AUDIO_DATA_URL, + }, + }, + { + "type": "text", + "text": ( + "Please transcribe the content of the audio " + "and return only the transcription text." + ), + }, + ], + }, + ] + + @pytest.mark.asyncio async def test_mimo_stt_prepare_audio_input_returns_data_url(monkeypatch): class _ResolvedAudio: - base64_data = "ZmFrZQ==" + base64_data = MIMO_STT_TEST_AUDIO_BASE64 mime_type = "audio/wav" format = "wav" @@ -284,6 +369,35 @@ async def to_base64_data(self, **kwargs): assert cleanup_paths == [] +@pytest.mark.asyncio +async def test_mimo_stt_prepare_audio_input_rejects_non_wav_payload(monkeypatch): + """上游 SILK→WAV 转换静默失败时应本地报错，而不是把坏字节发给 API（#9113）。""" + silk_base64 = base64.b64encode(b"\x02#!SILK_V3" + b"\x00" * 16).decode() + + class _ResolvedAudio: + base64_data = silk_base64 + mime_type = "audio/wav" + format = "wav" + + def to_data_url(self): + return f"data:audio/wav;base64,{silk_base64}" + + class _Resolver: + def __init__(self, *_args, **_kwargs): + pass + + async def to_base64_data(self, **_kwargs): + return _ResolvedAudio() + + monkeypatch.setattr( + "astrbot.core.provider.sources.mimo_api_common.MediaResolver", + _Resolver, + ) + + with pytest.raises(MiMoAPIError, match="SILK"): + await prepare_audio_input("/tmp/test.wav") + + @pytest.mark.asyncio async def test_mimo_stt_get_text_uses_reasoning_content(monkeypatch): provider = _make_stt_provider() From df2e6709028d31f1b9897d5aab3cacb614adffa6 Mon Sep 17 00:00:00 2001 From: tangtaizong666 <212687958+tangtaizong666@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:56:36 +0800 Subject: [PATCH 2/2] fix: accept unpadded MiMo wav headers --- astrbot/core/provider/sources/mimo_api_common.py | 10 +++++++++- tests/test_mimo_api_sources.py | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/astrbot/core/provider/sources/mimo_api_common.py b/astrbot/core/provider/sources/mimo_api_common.py index 36901a5a9c..9ae260ccee 100644 --- a/astrbot/core/provider/sources/mimo_api_common.py +++ b/astrbot/core/provider/sources/mimo_api_common.py @@ -81,6 +81,14 @@ async def prepare_audio_input(audio_source: str) -> tuple[str, list[Path]]: return audio_data.to_data_url(), [] +def _decode_base64_header(base64_data: str) -> bytes: + chunk = "".join(base64_data[:64].split()) + padding = len(chunk) % 4 + if padding: + chunk += "=" * (4 - padding) + return base64.b64decode(chunk) + + def _validate_wav_payload(base64_data: str, audio_source: str) -> None: """Reject audio payloads whose bytes are not RIFF/WAVE. @@ -96,7 +104,7 @@ def _validate_wav_payload(base64_data: str, audio_source: str) -> None: MiMoAPIError: Raised when the payload is not valid WAV data. """ try: - header = base64.b64decode(base64_data[:64]) + header = _decode_base64_header(base64_data) except Exception: header = b"" if len(header) >= 12 and header[:4] == b"RIFF" and header[8:12] == b"WAVE": diff --git a/tests/test_mimo_api_sources.py b/tests/test_mimo_api_sources.py index da1a947e1e..9c6f7c1a8b 100644 --- a/tests/test_mimo_api_sources.py +++ b/tests/test_mimo_api_sources.py @@ -6,6 +6,7 @@ from astrbot.core.provider.sources.mimo_api_common import ( MiMoAPIError, + _validate_wav_payload, build_headers, prepare_audio_input, ) @@ -398,6 +399,12 @@ async def to_base64_data(self, **_kwargs): await prepare_audio_input("/tmp/test.wav") +def test_mimo_stt_wav_validation_accepts_unpadded_base64_header(): + wav_base64 = base64.b64encode(MIMO_STT_TEST_WAV_HEADER).decode().rstrip("=") + + _validate_wav_payload(wav_base64, "/tmp/test.wav") + + @pytest.mark.asyncio async def test_mimo_stt_get_text_uses_reasoning_content(monkeypatch): provider = _make_stt_provider()