Big refactor of TTS native code

Author: IgorSwat

apps/speech/screens/TextToSpeechScreen.tsx

@@ -60,8 +60,48 @@ const STEPS: ModelOption<number>[] = [
   { label: '16 (best quality)', value: 16 },
 ];
 
-// Supertonic renders audio at 44.1 kHz.
-const SAMPLE_RATE = 44100;
+// Kokoro: language-specific voices. Each voice bundles its own phonemizer
+// config, so only the voice picker is shown (no language/steps controls).
+const KOKORO_LANG_LABELS: Record<string, string> = {
+  en_us: '🇺🇸 US English',
+  en_gb: '🇬🇧 UK English',
+  fr: '🇫🇷 French',
+  es: '🇪🇸 Spanish',
+  it: '🇮🇹 Italian',
+  pt: '🇵🇹 Portuguese',
+  hi: '🇮🇳 Hindi',
+  pl: '🇵🇱 Polish',
+  de: '🇩🇪 German',
+};
+
+const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1);
+
+// `models.text_to_speech.kokoro.<lang>.<voice>` are factory functions
+// (`() => TextToSpeechModelConfig`); call them to get the actual configs.
+const kokoroVoicesByLang = models.text_to_speech.kokoro as unknown as Record<
+  string,
+  Record<string, () => TextToSpeechModelConfig>
+>;
+
+const KOKORO_VOICES: ModelOption<TextToSpeechModelConfig>[] = Object.entries(
+  kokoroVoicesByLang
+).flatMap(([lang, voices]) =>
+  Object.entries(voices).map(([name, factory]) => ({
+    label: `${KOKORO_LANG_LABELS[lang] ?? lang} · ${capitalize(name)}`,
+    value: factory(),
+  }))
+);
+
+type TtsModelType = 'supertonic' | 'kokoro';
+
+const TTS_MODEL_OPTIONS: ModelOption<TtsModelType>[] = [
+  { label: 'Supertonic', value: 'supertonic' },
+  { label: 'Kokoro', value: 'kokoro' },
+];
+
+// Supertonic renders at 44.1 kHz, Kokoro at 24 kHz.
+const SUPERTONIC_SAMPLE_RATE = 44100;
+const KOKORO_SAMPLE_RATE = 24000;
 
 import FontAwesome from '@expo/vector-icons/FontAwesome';
 import {
@@ -83,7 +123,7 @@ import ErrorBanner from '../components/ErrorBanner';
 const createAudioBufferFromVector = (
   audioVector: Float32Array,
   audioContext: AudioContext | null = null,
-  sampleRate: number = SAMPLE_RATE
+  sampleRate: number = SUPERTONIC_SAMPLE_RATE
 ): AudioBuffer => {
   if (audioContext == null) audioContext = new AudioContext({ sampleRate });
 
@@ -99,6 +139,8 @@ const createAudioBufferFromVector = (
 };
 
 export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => {
+  const [selectedTtsModel, setSelectedTtsModel] =
+    useState<TtsModelType>('supertonic');
   const [selectedSpeaker, setSelectedSpeaker] =
     useState<TextToSpeechModelConfig>(tts.m1());
   const [selectedLang, setSelectedLang] =
@@ -107,6 +149,19 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => {
 
   const model = useTextToSpeech(selectedSpeaker);
 
+  const sampleRate =
+    selectedTtsModel === 'supertonic'
+      ? SUPERTONIC_SAMPLE_RATE
+      : KOKORO_SAMPLE_RATE;
+
+  const handleSelectTtsModel = (type: TtsModelType) => {
+    if (type === selectedTtsModel) return;
+    setSelectedTtsModel(type);
+    setSelectedSpeaker(
+      type === 'supertonic' ? VOICES[0].value : KOKORO_VOICES[0].value
+    );
+  };
+
   const [inputText, setInputText] = useState('');
   const [isPlaying, setIsPlaying] = useState(false);
   const [readyToGenerate, setReadyToGenerate] = useState(false);
@@ -123,7 +178,7 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => {
       iosOptions: ['defaultToSpeaker'],
     });
 
-    const context = new AudioContext({ sampleRate: SAMPLE_RATE });
+    const context = new AudioContext({ sampleRate });
     audioContextRef.current = context;
     context.suspend();
 
@@ -138,7 +193,7 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => {
       audioContextRef.current = null;
       gainNodeRef.current = null;
     };
-  }, []);
+  }, [sampleRate]);
 
   useEffect(() => {
     setReadyToGenerate(!model.isGenerating && model.isReady && !isPlaying);
@@ -165,7 +220,7 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => {
           const audioBuffer = createAudioBufferFromVector(
             audioVec,
             audioContext,
-            SAMPLE_RATE
+            sampleRate
           );
 
           const source = (sourceRef.current =
@@ -192,8 +247,9 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => {
       await model.stream({
         text: inputText,
         speed: 1.0,
-        totalSteps,
-        lang: selectedLang,
+        ...(selectedTtsModel === 'supertonic'
+          ? { totalSteps, lang: selectedLang }
+          : {}),
         onNext,
         onEnd,
       });
@@ -234,29 +290,41 @@ export const TextToSpeechScreen = ({ onBack }: { onBack: () => void }) => {
           </View>
           <ErrorBanner message={error} onDismiss={() => setError(null)} />
 
+          <ModelPicker
+            label="Model"
+            models={TTS_MODEL_OPTIONS}
+            selectedModel={selectedTtsModel}
+            disabled={model.isGenerating || isPlaying}
+            onSelect={handleSelectTtsModel}
+          />
+
           <ModelPicker
             label="Voice"
-            models={VOICES}
+            models={selectedTtsModel === 'supertonic' ? VOICES : KOKORO_VOICES}
             selectedModel={selectedSpeaker}
             disabled={model.isGenerating}
             onSelect={(m) => setSelectedSpeaker(m)}
           />
 
-          <ModelPicker
-            label="Language"
-            models={LANGUAGES}
-            selectedModel={selectedLang}
-            disabled={isPlaying}
-            onSelect={(l) => setSelectedLang(l)}
-          />
+          {selectedTtsModel === 'supertonic' && (
+            <>
+              <ModelPicker
+                label="Language"
+                models={LANGUAGES}
+                selectedModel={selectedLang}
+                disabled={isPlaying}
+                onSelect={(l) => setSelectedLang(l)}
+              />
 
-          <ModelPicker
-            label="Steps"
-            models={STEPS}
-            selectedModel={totalSteps}
-            disabled={isPlaying}
-            onSelect={(s) => setTotalSteps(s)}
-          />
+              <ModelPicker
+                label="Steps"
+                models={STEPS}
+                selectedModel={totalSteps}
+                disabled={isPlaying}
+                onSelect={(s) => setTotalSteps(s)}
+              />
+            </>
+          )}
 
           <View style={styles.inputContainer}>
             <Text style={styles.inputLabel}>Enter text to synthesize</Text>

apps/speech/screens/VoiceActivityDetectionScreen.tsx

@@ -8,10 +8,7 @@ import {
   Platform,
 } from 'react-native';
 import { SafeAreaProvider, SafeAreaView } from 'react-native-safe-area-context';
-import {
-  models,
-  useVAD 
-} from 'react-native-executorch';
+import { models, useVAD } from 'react-native-executorch';
 import FontAwesome from '@expo/vector-icons/FontAwesome';
 import { AudioManager, AudioRecorder } from 'react-native-audio-api';
 import SWMIcon from '../assets/swm_icon.svg';

packages/react-native-executorch/common/rnexecutorch/models/text_to_speech/common/Constants.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <cstdint>
+#include <unordered_set>
+
+namespace rnexecutorch::models::text_to_speech::constants {
+
+// Special text characters - end of sentence markers
+inline const std::unordered_set<char32_t> kEndOfSentenceCharacters = {
+    U'.', U'?', U'!', U';',
+    U'…', // Ellipsis
+    U'|', // ASCII Pipe (often used as Hindi Purna Viram)
+    U'।', // Hindi Purna Viram (U+0964)
+    U'॥', // Hindi Deergh Viram (U+0965)
+    U'¿', // Spanish Inverted Question Mark (U+00BF)
+    U'¡', // Spanish Inverted Exclamation Mark (U+00A1)
+};
+
+// Special text characters - mid-sentence pause markers
+inline const std::unordered_set<char32_t> kPauseCharacters = {
+    U',', U':', U'-',
+    U'—', // Em Dash (U+2014)
+    U'«', // Left Guillemet (U+00AB)
+    U'»', // Right Guillemet (U+00BB)
+};
+
+} // namespace rnexecutorch::models::text_to_speech::constants

packages/react-native-executorch/common/rnexecutorch/models/text_to_speech/common/TextPartitioner.cpp

@@ -0,0 +1,105 @@
+#include "TextPartitioner.h"
+#include "Constants.h"
+
+#include <algorithm>
+#include <deque>
+#include <limits>
+#include <ranges>
+
+namespace rnexecutorch::models::text_to_speech {
+
+constexpr TextPartitioner::Cost INF = 1e7;
+
+TextPartitioner::TextPartitioner(const TextPartitionerConfig &cfg)
+    : config_(cfg) {}
+
+TextPartitioner::Partition TextPartitioner::partition(std::u32string_view input,
+                                                      size_t limit) const {
+  if (input.empty()) {
+    return {input, {}};
+  }
+
+  size_t n = input.size();
+  std::vector<std::pair<Cost, int64_t>> dp(n, {INF, -1});
+
+  std::deque<size_t> eosPoints, pausePoints, whitePoints;
+
+  // Helper function to estimate the cost of given partitioning.
+  auto costFn = [this, limit](Cost acc, size_t beg, int64_t prevBp, int64_t bp,
+                              size_t end, Separator sep) -> Cost {
+    if (end - bp > limit) {
+      return INF;
+    }
+
+    Cost sepPenalty = sep == Separator::EOS     ? config_.eosCost
+                      : sep == Separator::PAUSE ? config_.pauseCost
+                      : sep == Separator::WHITE ? config_.whiteCost
+                                                : 0;
+
+    int64_t rightmostRangeLength = end - bp;
+    int64_t prevRangeLength = bp - prevBp;
+
+    int64_t latency = std::max(static_cast<int64_t>(0),
+                               rightmostRangeLength - prevRangeLength);
+    int64_t discount =
+        config_.tokenDiscountFactor *
+        std::max(static_cast<int64_t>(0), config_.tokenDiscountRange - bp - 1);
+
+    return acc +
+           static_cast<Cost>(latency * discount / config_.tokenDiscountRange) +
+           sepPenalty;
+  };
+
+  for (size_t i = 0; i < n; ++i) {
+    auto &[bestCost, prevBpIdx] = dp[i];
+
+    bestCost = costFn(0, 0, -1, -1, i + 1, Separator::NO_SEP);
+
+    for (auto *q : {&eosPoints, &pausePoints, &whitePoints}) {
+      while (!q->empty() && q->front() + limit < i) {
+        q->pop_front();
+      }
+
+      Separator sep = q == &eosPoints     ? Separator::EOS
+                      : q == &pausePoints ? Separator::PAUSE
+                                          : Separator::WHITE;
+      for (size_t breakIdx : (*q)) {
+        auto cost = costFn(dp[breakIdx].first, 0, dp[breakIdx].second, breakIdx,
+                           i, sep);
+        if (cost < bestCost && breakIdx > 0) {
+          bestCost = cost;
+          prevBpIdx = breakIdx;
+        }
+      }
+    }
+
+    char32_t c = input[i];
+    if (constants::kEndOfSentenceCharacters.contains(c)) {
+      eosPoints.push_back(i);
+    } else if (constants::kPauseCharacters.contains(c)) {
+      pausePoints.push_back(i);
+    } else if (c < 256 && std::isspace(static_cast<char>(c))) {
+      whitePoints.push_back(i);
+    }
+  }
+
+  std::vector<std::pair<size_t, size_t>> segments;
+  int64_t currBp = dp.back().second;
+  size_t lastIdx = n;
+
+  // Backtracking
+  while (currBp != -1) {
+    size_t start = static_cast<size_t>(currBp + 1);
+    segments.emplace_back(start, lastIdx - start);
+    lastIdx = currBp + 1;
+    currBp = dp[currBp].second;
+  }
+  segments.emplace_back(0, lastIdx);
+
+  // Because of backtracking, the segments are placed in reversed order.
+  std::ranges::reverse(segments);
+
+  return {input, std::move(segments)};
+}
+
+} // namespace rnexecutorch::models::text_to_speech