modified llm example

Author: Mateusz Kopciński

examples/llm/App.tsx

@@ -14,9 +14,11 @@ import {
 } from 'react-native';
 import LLMScreen from './screens/LLMScreen';
 import LLMToolCallingScreen from './screens/LLMToolCallingScreen';
+import VoiceChatScreen from './screens/VocieChatScreen';
 
 enum Mode {
   LLM,
+  LLM_VOICE_CHAT,
   LLM_TOOL_CALLING,
 }
 
@@ -39,6 +41,9 @@ export default function App() {
       case Mode.LLM:
         return <LLMScreen setIsGenerating={setIsGenerating} />;
 
+      case Mode.LLM_VOICE_CHAT:
+        return <VoiceChatScreen setIsGenerating={setIsGenerating} />;
+
       case Mode.LLM_TOOL_CALLING:
         return <LLMToolCallingScreen setIsGenerating={setIsGenerating} />;
 
@@ -61,7 +66,7 @@ export default function App() {
             {!isGenerating ? (
               <View style={styles.wheelPickerContainer}>
                 <ScrollPicker
-                  dataSource={['Chat with LLM', 'Tool calling']}
+                  dataSource={['Chat with LLM', 'Talk to LLM', 'Tool calling']}
                   onValueChange={(_, selectedIndex) => {
                     handleModeChange(selectedIndex);
                   }}

examples/llm/ios/Podfile.lock

@@ -1401,7 +1401,7 @@ PODS:
     - React-jsiexecutor
     - React-RCTFBReactNativeSpec
     - ReactCommon/turbomodule/core
-  - react-native-executorch (0.3.3):
+  - react-native-executorch (0.3.1-stt-12):
     - DoubleConversion
     - glog
     - hermes-engine
@@ -2395,7 +2395,7 @@ SPEC CHECKSUMS:
   React-logger: 8edfcedc100544791cd82692ca5a574240a16219
   React-Mapbuffer: c3f4b608e4a59dd2f6a416ef4d47a14400194468
   React-microtasksnativemodule: 054f34e9b82f02bd40f09cebd4083828b5b2beb6
-  react-native-executorch: d0c3dffa0a4a4111ea9c7b97f3fbf088a48d3b2a
+  react-native-executorch: 8835fcfdfc71b1d42d30525ee047b2811c359cb8
   react-native-safe-area-context: 562163222d999b79a51577eda2ea8ad2c32b4d06
   React-NativeModulesApple: 2c4377e139522c3d73f5df582e4f051a838ff25e
   React-oscompat: ef5df1c734f19b8003e149317d041b8ce1f7d29c

examples/llm/package.json

@@ -20,7 +20,7 @@
     "react": "19.0.0",
     "react-native": "^0.79.2",
     "react-native-audio-api": "0.5.7",
-    "react-native-executorch": "^0.3.1",
+    "react-native-executorch": "^0.3.2",
     "react-native-live-audio-stream": "^1.1.1",
     "react-native-loading-spinner-overlay": "^3.0.1",
     "react-native-markdown-display": "^7.0.2",

examples/llm/screens/LLMScreen.tsx

@@ -13,98 +13,31 @@ import {
 import SWMIcon from '../assets/icons/swm_icon.svg';
 import Spinner from 'react-native-loading-spinner-overlay';
 import {
-  STREAMING_ACTION,
-  useSpeechToText,
   useLLM,
   QWEN3_0_6B_QUANTIZED,
   QWEN3_TOKENIZER,
   QWEN3_TOKENIZER_CONFIG,
 } from 'react-native-executorch';
 import PauseIcon from '../assets/icons/pause_icon.svg';
-import MicIcon from '../assets/icons/mic_icon.svg';
 import SendIcon from '../assets/icons/send_icon.svg';
-import StopIcon from '../assets/icons/stop_icon.svg';
 import ColorPalette from '../colors';
 import Messages from '../components/Messages';
-import LiveAudioStream from 'react-native-live-audio-stream';
-import { Buffer } from 'buffer';
-
-const audioStreamOptions = {
-  sampleRate: 16000,
-  channels: 1,
-  bitsPerSample: 16,
-  audioSource: 1,
-  bufferSize: 16000,
-};
-
-const startStreamingAudio = (options: any, onChunk: (data: string) => void) => {
-  LiveAudioStream.init(options);
-  LiveAudioStream.on('data', onChunk);
-  LiveAudioStream.start();
-};
-
-const float32ArrayFromPCMBinaryBuffer = (b64EncodedBuffer: string) => {
-  const b64DecodedChunk = Buffer.from(b64EncodedBuffer, 'base64');
-  const int16Array = new Int16Array(b64DecodedChunk.buffer);
-
-  const float32Array = new Float32Array(int16Array.length);
-  for (let i = 0; i < int16Array.length; i++) {
-    float32Array[i] = Math.max(
-      -1,
-      Math.min(1, (int16Array[i] / audioStreamOptions.bufferSize) * 8)
-    );
-  }
-  return float32Array;
-};
 
 export default function LLMScreen({
   setIsGenerating,
 }: {
   setIsGenerating: React.Dispatch<React.SetStateAction<boolean>>;
 }) {
-  const [isRecording, setIsRecording] = useState(false);
   const [isTextInputFocused, setIsTextInputFocused] = useState(false);
   const [userInput, setUserInput] = useState('');
   const textInputRef = useRef<TextInput>(null);
-  const messageRecorded = useRef<boolean>(false);
 
   const llm = useLLM({
     modelSource: QWEN3_0_6B_QUANTIZED,
     tokenizerSource: QWEN3_TOKENIZER,
     tokenizerConfigSource: QWEN3_TOKENIZER_CONFIG,
-    chatConfig: {
-      contextWindowLength: 6,
-    },
-  });
-  const speechToText = useSpeechToText({
-    modelName: 'moonshine',
-    windowSize: 3,
-    overlapSeconds: 1.2,
   });
 
-  const onChunk = (data: string) => {
-    const float32Chunk = float32ArrayFromPCMBinaryBuffer(data);
-    speechToText.streamingTranscribe(
-      STREAMING_ACTION.DATA,
-      Array.from(float32Chunk)
-    );
-  };
-
-  const handleRecordPress = async () => {
-    if (isRecording) {
-      setIsRecording(false);
-      LiveAudioStream.stop();
-      messageRecorded.current = true;
-      await llm.sendMessage(
-        await speechToText.streamingTranscribe(STREAMING_ACTION.STOP)
-      );
-    } else {
-      setIsRecording(true);
-      startStreamingAudio(audioStreamOptions, onChunk);
-      await speechToText.streamingTranscribe(STREAMING_ACTION.START);
-    }
-  };
-
   useEffect(() => {
     setIsGenerating(llm.isGenerating);
   }, [llm.isGenerating, setIsGenerating]);
@@ -118,10 +51,10 @@ export default function LLMScreen({
     }
   };
 
-  return !llm.isReady || !speechToText.isReady ? (
+  return !llm.isReady ? (
     <Spinner
-      visible={!llm.isReady || !speechToText.isReady}
-      textContent={`Loading the model ${(llm.downloadProgress * 100).toFixed(0)} %\nLoading the speech model ${(speechToText.downloadProgress * 100).toFixed(0)} %`}
+      visible={!llm.isReady}
+      textContent={`Loading the model ${(llm.downloadProgress * 100).toFixed(0)} %\n`}
     />
   ) : (
     <TouchableWithoutFeedback onPress={Keyboard.dismiss}>
@@ -134,17 +67,10 @@ export default function LLMScreen({
           <SWMIcon width={45} height={45} />
           <Text style={styles.textModelName}>Qwen 3 x Whisper</Text>
         </View>
-        {llm.messageHistory.length || speechToText.sequence ? (
+        {llm.messageHistory.length ? (
           <View style={styles.chatContainer}>
             <Messages
-              chatHistory={
-                speechToText.isGenerating
-                  ? [
-                      ...llm.messageHistory,
-                      { role: 'user', content: speechToText.sequence },
-                    ]
-                  : llm.messageHistory
-              }
+              chatHistory={llm.messageHistory}
               llmResponse={llm.response}
               isGenerating={llm.isGenerating}
               deleteMessage={llm.deleteMessage}
@@ -162,13 +88,13 @@ export default function LLMScreen({
           <TextInput
             onFocus={() => setIsTextInputFocused(true)}
             onBlur={() => setIsTextInputFocused(false)}
-            editable={!isRecording && !llm.isGenerating}
+            editable={!!llm.isGenerating}
             style={{
               ...styles.textInput,
               borderColor: isTextInputFocused
                 ? ColorPalette.blueDark
                 : ColorPalette.blueLight,
-              display: isRecording ? 'none' : 'flex',
+              display: 'flex',
             }}
             placeholder="Your message"
             placeholderTextColor={'#C1C6E5'}
@@ -180,19 +106,6 @@ export default function LLMScreen({
             <TouchableOpacity onPress={llm.interrupt}>
               <PauseIcon height={40} width={40} padding={4} margin={8} />
             </TouchableOpacity>
-          ) : !userInput ? (
-            <TouchableOpacity
-              style={
-                !isRecording ? styles.recordTouchable : styles.recordingInfo
-              }
-              onPress={handleRecordPress}
-            >
-              {isRecording ? (
-                <StopIcon height={40} width={40} padding={4} margin={8} />
-              ) : (
-                <MicIcon height={40} width={40} padding={4} margin={8} />
-              )}
-            </TouchableOpacity>
           ) : (
             <TouchableOpacity
               style={styles.recordTouchable}
@@ -208,9 +121,6 @@ export default function LLMScreen({
 }
 
 const styles = StyleSheet.create({
-  container: {
-    flex: 1,
-  },
   keyboardAvoidingView: {
     flex: 1,
   },
@@ -262,20 +172,9 @@ const styles = StyleSheet.create({
     color: ColorPalette.primary,
     padding: 16,
   },
-  fromUrlTouchable: {
-    height: '100%',
-    justifyContent: 'center',
-    alignItems: 'flex-start',
-  },
   recordTouchable: {
     height: '100%',
     justifyContent: 'center',
     alignItems: 'center',
   },
-  recordingInfo: {
-    width: '100%',
-    display: 'flex',
-    justifyContent: 'center',
-    alignItems: 'center',
-  },
 });