Merge branch 'v0.4.0-rc1' into @md/s2t_streaming

Author: mdydek

src/constants/modelUrls.ts

@@ -59,9 +59,14 @@ export const WHISPER_TINY_DECODER =
   'https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.3.0/xnnpack/whisper_tiny_en_xnnpack_decoder.pte';
 export const WHISPER_TINY_ENCODER =
   'https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny.en/resolve/v0.3.0/xnnpack/whisper_tiny_en_xnnpack_encoder.pte';
+export const WHISPER_TINY_MULTILINGUAL_ENCODER =
+  'https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.4.0/xnnpack/xnnpack_whisper_encoder.pte';
+export const WHISPER_TINY_MULTILINGUAL_DECODER =
+  'https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.4.0/xnnpack/xnnpack_whisper_decoder.pte';
+export const WHISPER_TINY_MULTILINGUAL_TOKENIZER =
+  'https://huggingface.co/software-mansion/react-native-executorch-whisper-tiny/resolve/v0.4.0/tokenizer.json';
 
 // OCR
-
 export const DETECTOR_CRAFT_1280 =
   'https://huggingface.co/software-mansion/react-native-executorch-detector-craft/resolve/v0.3.0/xnnpack/xnnpack_craft_1280.pte';
 export const DETECTOR_CRAFT_800 =

src/constants/sttDefaults.ts

@@ -5,52 +5,62 @@ import {
   WHISPER_TINY_ENCODER,
   WHISPER_TINY_DECODER,
   WHISPER_TOKENIZER,
+  WHISPER_TINY_MULTILINGUAL_ENCODER,
+  WHISPER_TINY_MULTILINGUAL_DECODER,
+  WHISPER_TINY_MULTILINGUAL_TOKENIZER,
 } from './modelUrls';
+import { AvailableModels, ModelConfig } from '../types/stt';
 
 export const SAMPLE_RATE = 16_000;
 export const SECOND = SAMPLE_RATE;
 export const HAMMING_DIST_THRESHOLD = 1;
 
-export interface ModelConfig {
+const whisperTinyModelConfig = {
   sources: {
-    encoder: string;
-    decoder: string;
-  };
+    encoder: WHISPER_TINY_ENCODER,
+    decoder: WHISPER_TINY_DECODER,
+  },
   tokenizer: {
-    source: string;
-    sos: number;
-    eos: number;
-    specialChar: string;
-  };
-}
+    source: WHISPER_TOKENIZER,
+    bos: 50257, // FIXME: this is a placeholder and needs to be changed
+    eos: 50256, // FIXME: this is a placeholder and needs to be changed
+  },
+  isMultilingual: false,
+};
+
+const moonshineTinyModelConfig = {
+  sources: {
+    encoder: MOONSHINE_TINY_ENCODER,
+    decoder: MOONSHINE_TINY_DECODER,
+  },
+  tokenizer: {
+    source: MOONSHINE_TOKENIZER,
+    bos: 1, // FIXME: this is a placeholder and needs to be changed
+    eos: 2, // FIXME: this is a placeholder and needs to be changed
+  },
+  isMultilingual: false,
+};
 
-export const MODEL_CONFIGS: { [key in 'moonshine' | 'whisper']: ModelConfig } =
-  {
-    moonshine: {
-      sources: {
-        encoder: MOONSHINE_TINY_ENCODER,
-        decoder: MOONSHINE_TINY_DECODER,
-      },
-      tokenizer: {
-        source: MOONSHINE_TOKENIZER,
-        sos: 1,
-        eos: 2,
-        specialChar: '\u2581',
-      },
-    },
-    whisper: {
-      sources: {
-        encoder: WHISPER_TINY_ENCODER,
-        decoder: WHISPER_TINY_DECODER,
-      },
-      tokenizer: {
-        source: WHISPER_TOKENIZER,
-        sos: 50257,
-        eos: 50256,
-        specialChar: 'Ġ',
-      },
-    },
-  };
+const whisperTinyMultilingualModelConfig = {
+  sources: {
+    encoder: WHISPER_TINY_MULTILINGUAL_ENCODER,
+    decoder: WHISPER_TINY_MULTILINGUAL_DECODER,
+  },
+  tokenizer: {
+    source: WHISPER_TINY_MULTILINGUAL_TOKENIZER,
+    bos: 50258, // FIXME: this is a placeholder and needs to be changed
+    eos: 50257, // FIXME: this is a placeholder and needs to be changed
+  },
+  isMultilingual: true,
+};
+
+export const MODEL_CONFIGS: {
+  [key in AvailableModels]: ModelConfig;
+} = {
+  moonshine: moonshineTinyModelConfig,
+  whisper: whisperTinyModelConfig,
+  whisperMultilingual: whisperTinyMultilingualModelConfig,
+};
 
 export const MODES = {
   fast: {
@@ -67,6 +77,8 @@ export const MODES = {
   },
 };
 
+export const NUM_TOKENS_TO_SLICE = 3;
+
 export enum STREAMING_ACTION {
   START,
   DATA,