Skip to content

Commit d3182ce

Browse files
IgorSwatMateusz Słuszniak
andauthored
feat!: major performance & accuracy improvements in speech-to-text module (#1132)
## Description This PR introduces several changes to the speech-to-text module based on Whisper models: - CoreML integration - models re-exported to CoreML backend, bringing significant performance upgrade for iOS devices. - New streaming algorithm - eliminates duplicates in streaming output, resulting in a major quality improvement of the live streaming mode. - Changes in demo apps: removed faulty 'voice mode' screen in LLM demo app, refactored speech to text screen in 'speech' app by adding new CoreML models to selection bar and changing the default model for iOS devices. - Minor code improvements in speech-to-text module ### Introduces a breaking change? - [x] Yes - [ ] No **Change**: removes predefined constants for quantized models. **Justification**: the quantized models differ very slightly from the original ones, introducing unnecessary complexity in this case. ### Type of change - [x] Bug fix (change which fixes an issue) - [x] New feature (change which adds functionality) - [x] Documentation update (improves or adds clarity to existing documentation) - [x] Other (chores, tests, code style improvements etc.) ### Tested on - [x] iOS - [x] Android ### Testing instructions Run demo app to test the live streaming mode. ### Screenshots <!-- Add screenshots here, if applicable --> ### Related issues #1124 ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have updated the documentation accordingly - [x] My changes generate no new warnings ### Additional notes --------- Co-authored-by: Mateusz Słuszniak <msluszniak1@gmail.com>
1 parent d04a36e commit d3182ce

25 files changed

Lines changed: 670 additions & 1358 deletions

File tree

apps/llm/app/index.tsx

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,6 @@ export default function Home() {
2929
>
3030
<Text style={styles.buttonText}>LLM Structured Output</Text>
3131
</TouchableOpacity>
32-
<TouchableOpacity
33-
style={styles.button}
34-
onPress={() => router.navigate('voice_chat/')}
35-
>
36-
<Text style={styles.buttonText}>Voice Chat</Text>
37-
</TouchableOpacity>
3832
<TouchableOpacity
3933
style={styles.button}
4034
onPress={() => router.navigate('multimodal_llm/')}

apps/llm/app/voice_chat/index.tsx

Lines changed: 0 additions & 311 deletions
This file was deleted.

apps/speech/screens/SpeechToTextScreen.tsx

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@ import { SafeAreaProvider, SafeAreaView } from 'react-native-safe-area-context';
1414
import {
1515
useSpeechToText,
1616
WHISPER_TINY_EN,
17-
WHISPER_TINY_EN_QUANTIZED,
17+
WHISPER_TINY_EN_COREML,
1818
WHISPER_BASE_EN,
19+
WHISPER_BASE_EN_COREML,
1920
WHISPER_SMALL_EN,
21+
WHISPER_SMALL_EN_COREML,
2022
TranscriptionResult,
2123
SpeechToTextProps,
2224
} from 'react-native-executorch';
@@ -25,10 +27,12 @@ import { ModelPicker, ModelOption } from '../components/ModelPicker';
2527
type STTModelSources = SpeechToTextProps['model'];
2628

2729
const MODELS: ModelOption<STTModelSources>[] = [
28-
{ label: 'Whisper Tiny', value: WHISPER_TINY_EN },
29-
{ label: 'Whisper Tiny Q', value: WHISPER_TINY_EN_QUANTIZED },
30-
{ label: 'Whisper Base', value: WHISPER_BASE_EN },
31-
{ label: 'Whisper Small', value: WHISPER_SMALL_EN },
30+
{ label: 'Whisper Tiny EN (XNNPACK)', value: WHISPER_TINY_EN },
31+
{ label: 'Whisper Tiny EN (CoreML)', value: WHISPER_TINY_EN_COREML },
32+
{ label: 'Whisper Base EN (XNNPACK)', value: WHISPER_BASE_EN },
33+
{ label: 'Whisper Base EN (CoreML)', value: WHISPER_BASE_EN_COREML },
34+
{ label: 'Whisper Small EN (XNNPACK)', value: WHISPER_SMALL_EN },
35+
{ label: 'Whisper Small EN (CoreML)', value: WHISPER_SMALL_EN_COREML },
3236
];
3337
import FontAwesome from '@expo/vector-icons/FontAwesome';
3438
import {
@@ -45,9 +49,12 @@ import ErrorBanner from '../components/ErrorBanner';
4549

4650
const isSimulator = DeviceInfo.isEmulatorSync();
4751

52+
const DEFAULT_MODEL =
53+
Platform.OS === 'ios' ? WHISPER_BASE_EN_COREML : WHISPER_TINY_EN;
54+
4855
export const SpeechToTextScreen = ({ onBack }: { onBack: () => void }) => {
4956
const [selectedModel, setSelectedModel] =
50-
useState<STTModelSources>(WHISPER_TINY_EN);
57+
useState<STTModelSources>(DEFAULT_MODEL);
5158

5259
const model = useSpeechToText({
5360
model: selectedModel,
@@ -148,7 +155,7 @@ export const SpeechToTextScreen = ({ onBack }: { onBack: () => void }) => {
148155
recorder.current.onAudioReady(
149156
{
150157
sampleRate,
151-
bufferLength: 0.1 * sampleRate,
158+
bufferLength: 0.1 * sampleRate, // 100 ms
152159
channelCount: 1,
153160
},
154161
({ buffer }) => {
@@ -178,6 +185,7 @@ export const SpeechToTextScreen = ({ onBack }: { onBack: () => void }) => {
178185
try {
179186
const streamIter = model.stream({
180187
verbose: enableTimestamps,
188+
timeout: 100,
181189
});
182190

183191
for await (const { committed, nonCommitted } of streamIter) {

0 commit comments

Comments
 (0)