fixed S2T streaming

improved error handling, minor refactoring

Author: Mateusz Kopciński

examples/llm/ios/Podfile.lock

@@ -1401,7 +1401,7 @@ PODS:
     - React-jsiexecutor
     - React-RCTFBReactNativeSpec
     - ReactCommon/turbomodule/core
-  - react-native-executorch (0.3.1-stt-7):
+  - react-native-executorch (0.3.3):
     - DoubleConversion
     - glog
     - hermes-engine
@@ -1823,6 +1823,82 @@ PODS:
     - React-logger (= 0.79.2)
     - React-perflogger (= 0.79.2)
     - React-utils (= 0.79.2)
+  - RNAudioAPI (0.5.7):
+    - DoubleConversion
+    - glog
+    - hermes-engine
+    - RCT-Folly (= 2024.11.18.00)
+    - RCTRequired
+    - RCTTypeSafety
+    - React-Core
+    - React-debug
+    - React-Fabric
+    - React-featureflags
+    - React-graphics
+    - React-hermes
+    - React-ImageManager
+    - React-jsi
+    - React-NativeModulesApple
+    - React-RCTFabric
+    - React-renderercss
+    - React-rendererdebug
+    - React-utils
+    - ReactCodegen
+    - ReactCommon/turbomodule/bridging
+    - ReactCommon/turbomodule/core
+    - RNAudioAPI/audioapi (= 0.5.7)
+    - Yoga
+  - RNAudioAPI/audioapi (0.5.7):
+    - DoubleConversion
+    - glog
+    - hermes-engine
+    - RCT-Folly (= 2024.11.18.00)
+    - RCTRequired
+    - RCTTypeSafety
+    - React-Core
+    - React-debug
+    - React-Fabric
+    - React-featureflags
+    - React-graphics
+    - React-hermes
+    - React-ImageManager
+    - React-jsi
+    - React-NativeModulesApple
+    - React-RCTFabric
+    - React-renderercss
+    - React-rendererdebug
+    - React-utils
+    - ReactCodegen
+    - ReactCommon/turbomodule/bridging
+    - ReactCommon/turbomodule/core
+    - RNAudioAPI/audioapi/ios (= 0.5.7)
+    - Yoga
+  - RNAudioAPI/audioapi/ios (0.5.7):
+    - DoubleConversion
+    - glog
+    - hermes-engine
+    - RCT-Folly (= 2024.11.18.00)
+    - RCTRequired
+    - RCTTypeSafety
+    - React-Core
+    - React-debug
+    - React-Fabric
+    - React-featureflags
+    - React-graphics
+    - React-hermes
+    - React-ImageManager
+    - React-jsi
+    - React-NativeModulesApple
+    - React-RCTFabric
+    - React-renderercss
+    - React-rendererdebug
+    - React-utils
+    - ReactCodegen
+    - ReactCommon/turbomodule/bridging
+    - ReactCommon/turbomodule/core
+    - Yoga
+  - RNLiveAudioStream (1.1.1):
+    - React
   - RNReanimated (3.17.5):
     - DoubleConversion
     - glog
@@ -2083,6 +2159,8 @@ DEPENDENCIES:
   - ReactAppDependencyProvider (from `build/generated/ios`)
   - ReactCodegen (from `build/generated/ios`)
   - ReactCommon/turbomodule/core (from `../node_modules/react-native/ReactCommon`)
+  - RNAudioAPI (from `../node_modules/react-native-audio-api`)
+  - RNLiveAudioStream (from `../node_modules/react-native-live-audio-stream`)
   - RNReanimated (from `../node_modules/react-native-reanimated`)
   - RNSVG (from `../node_modules/react-native-svg`)
   - Yoga (from `../node_modules/react-native/ReactCommon/yoga`)
@@ -2256,6 +2334,10 @@ EXTERNAL SOURCES:
     :path: build/generated/ios
   ReactCommon:
     :path: "../node_modules/react-native/ReactCommon"
+  RNAudioAPI:
+    :path: "../node_modules/react-native-audio-api"
+  RNLiveAudioStream:
+    :path: "../node_modules/react-native-live-audio-stream"
   RNReanimated:
     :path: "../node_modules/react-native-reanimated"
   RNSVG:
@@ -2313,7 +2395,7 @@ SPEC CHECKSUMS:
   React-logger: 8edfcedc100544791cd82692ca5a574240a16219
   React-Mapbuffer: c3f4b608e4a59dd2f6a416ef4d47a14400194468
   React-microtasksnativemodule: 054f34e9b82f02bd40f09cebd4083828b5b2beb6
-  react-native-executorch: 8bca350ccbb66246dd7366fb6514fed57083930e
+  react-native-executorch: d0c3dffa0a4a4111ea9c7b97f3fbf088a48d3b2a
   react-native-safe-area-context: 562163222d999b79a51577eda2ea8ad2c32b4d06
   React-NativeModulesApple: 2c4377e139522c3d73f5df582e4f051a838ff25e
   React-oscompat: ef5df1c734f19b8003e149317d041b8ce1f7d29c
@@ -2346,6 +2428,8 @@ SPEC CHECKSUMS:
   ReactAppDependencyProvider: 04d5eb15eb46be6720e17a4a7fa92940a776e584
   ReactCodegen: c63eda03ba1d94353fb97b031fc84f75a0d125ba
   ReactCommon: 76d2dc87136d0a667678668b86f0fca0c16fdeb0
+  RNAudioAPI: 2e3fd4bf75aa5717791babb30126707504996f09
+  RNLiveAudioStream: 93ac2bb6065be9018d0b00157b220f11cebc1513
   RNReanimated: 2313402fe27fecb7237619e9c6fcee3177f08a65
   RNSVG: 794f269526df9ddc1f79b3d1a202b619df0368e3
   SocketRocket: d4aabe649be1e368d1318fdf28a022d714d65748

examples/llm/ios/llm.xcodeproj/project.pbxproj

@@ -23,8 +23,8 @@
 		13B07F961A680F5B00A75B9A /* llm.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = llm.app; sourceTree = BUILT_PRODUCTS_DIR; };
 		13B07FB51A68108700A75B9A /* Images.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Images.xcassets; path = llm/Images.xcassets; sourceTree = "<group>"; };
 		13B07FB61A68108700A75B9A /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = Info.plist; path = llm/Info.plist; sourceTree = "<group>"; };
-		52F93D50E948F855530BE970 /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; includeInIndex = 1; name = PrivacyInfo.xcprivacy; path = llm/PrivacyInfo.xcprivacy; sourceTree = "<group>"; };
-		66FD0DA88DE041D4B2294CB4 /* Aeonik-Medium.otf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = undefined; includeInIndex = 0; lastKnownFileType = unknown; name = "Aeonik-Medium.otf"; path = "../assets/fonts/Aeonik-Medium.otf"; sourceTree = "<group>"; };
+		52F93D50E948F855530BE970 /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xml; name = PrivacyInfo.xcprivacy; path = llm/PrivacyInfo.xcprivacy; sourceTree = "<group>"; };
+		66FD0DA88DE041D4B2294CB4 /* Aeonik-Medium.otf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = "Aeonik-Medium.otf"; path = "../assets/fonts/Aeonik-Medium.otf"; sourceTree = "<group>"; };
 		99F00DD47E42AF45816C58B8 /* Pods-llm.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-llm.release.xcconfig"; path = "Target Support Files/Pods-llm/Pods-llm.release.xcconfig"; sourceTree = "<group>"; };
 		A8F7447548358E44DADA5029 /* Pods-llm.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-llm.debug.xcconfig"; path = "Target Support Files/Pods-llm/Pods-llm.debug.xcconfig"; sourceTree = "<group>"; };
 		AA286B85B6C04FC6940260E9 /* SplashScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = SplashScreen.storyboard; path = llm/SplashScreen.storyboard; sourceTree = "<group>"; };
@@ -33,7 +33,7 @@
 		ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; };
 		F11748412D0307B40044C1D9 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = AppDelegate.swift; path = llm/AppDelegate.swift; sourceTree = "<group>"; };
 		F11748442D0722820044C1D9 /* llm-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "llm-Bridging-Header.h"; path = "llm/llm-Bridging-Header.h"; sourceTree = "<group>"; };
-		F73A1A19BDF34985A5AAA585 /* Aeonik-Regular.otf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = undefined; includeInIndex = 0; lastKnownFileType = unknown; name = "Aeonik-Regular.otf"; path = "../assets/fonts/Aeonik-Regular.otf"; sourceTree = "<group>"; };
+		F73A1A19BDF34985A5AAA585 /* Aeonik-Regular.otf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = 9; includeInIndex = 0; lastKnownFileType = unknown; name = "Aeonik-Regular.otf"; path = "../assets/fonts/Aeonik-Regular.otf"; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -117,7 +117,6 @@
 				66FD0DA88DE041D4B2294CB4 /* Aeonik-Medium.otf */,
 			);
 			name = Resources;
-			path = "";
 			sourceTree = "<group>";
 		};
 		B83BB53798F9A57DA3D5DBEC /* Pods */ = {
@@ -126,7 +125,6 @@
 				A8F7447548358E44DADA5029 /* Pods-llm.debug.xcconfig */,
 				99F00DD47E42AF45816C58B8 /* Pods-llm.release.xcconfig */,
 			);
-			name = Pods;
 			path = Pods;
 			sourceTree = "<group>";
 		};
@@ -353,6 +351,7 @@
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_ENTITLEMENTS = llm/llm.entitlements;
 				CURRENT_PROJECT_VERSION = 1;
+				DEVELOPMENT_TEAM = J5FM626PE2;
 				ENABLE_BITCODE = NO;
 				GCC_PREPROCESSOR_DEFINITIONS = (
 					"$(inherited)",
@@ -389,6 +388,7 @@
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_ENTITLEMENTS = llm/llm.entitlements;
 				CURRENT_PROJECT_VERSION = 1;
+				DEVELOPMENT_TEAM = J5FM626PE2;
 				INFOPLIST_FILE = llm/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 15.1;
 				LD_RUNPATH_SEARCH_PATHS = (

examples/llm/ios/llm.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEWorkspaceSharedSettings_AutocreateContextsIfNeeded</key>
+	<false/>
+</dict>
+</plist>

examples/llm/ios/llm/Info.plist

@@ -52,6 +52,8 @@
     <string>Allow $(PRODUCT_NAME) to access your reminders</string>
     <key>NSRemindersUsageDescription</key>
     <string>Allow $(PRODUCT_NAME) to access your reminders</string>
+    <key>NSMicrophoneUsageDescription</key>
+    <string>We need your permission to use the microphone.</string>
     <key>UIAppFonts</key>
     <array>
       <string>Aeonik-Regular.otf</string>

examples/llm/package.json

@@ -19,7 +19,9 @@
     "metro-config": "^0.81.0",
     "react": "19.0.0",
     "react-native": "^0.79.2",
-    "react-native-executorch": "/Users/kopcion/swm-ai/react-native-executorch/react-native-executorch-0.3.1-stt-7.tgz",
+    "react-native-audio-api": "0.5.7",
+    "react-native-executorch": "^0.3.1",
+    "react-native-live-audio-stream": "^1.1.1",
     "react-native-loading-spinner-overlay": "^3.0.1",
     "react-native-markdown-display": "^7.0.2",
     "react-native-reanimated": "^3.17.5",

examples/llm/screens/LLMScreen.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState } from 'react';
+import { useRef, useState } from 'react';
 import {
   Keyboard,
   KeyboardAvoidingView,
@@ -14,12 +14,12 @@ import { SafeAreaView } from 'react-native-safe-area-context';
 import SWMIcon from '../assets/icons/swm_icon.svg';
 import Spinner from 'react-native-loading-spinner-overlay';
 import {
-  // STREAMING_ACTION,
+  STREAMING_ACTION,
   useSpeechToText,
-  LLAMA3_2_1B_QLORA,
-  LLAMA3_2_1B_TOKENIZER,
   useLLM,
-  LLAMA3_2_TOKENIZER_CONFIG,
+  QWEN3_0_6B_QUANTIZED,
+  QWEN3_TOKENIZER,
+  QWEN3_TOKENIZER_CONFIG,
 } from 'react-native-executorch';
 import PauseIcon from '../assets/icons/pause_icon.svg';
 import MicIcon from '../assets/icons/mic_icon.svg';
@@ -29,7 +29,6 @@ import ColorPalette from '../colors';
 import Messages from '../components/Messages';
 import LiveAudioStream from 'react-native-live-audio-stream';
 import { Buffer } from 'buffer';
-// import * as Speech from 'expo-speech';
 
 const audioStreamOptions = {
   sampleRate: 16000,
@@ -60,59 +59,48 @@ const float32ArrayFromPCMBinaryBuffer = (b64EncodedBuffer: string) => {
 };
 
 export default function ChatScreen() {
-  // const [currentMessage, setCurrentMessage] = useState('');
   const [isRecording, setIsRecording] = useState(false);
   const [isTextInputFocused, setIsTextInputFocused] = useState(false);
   const [userInput, setUserInput] = useState('');
   const textInputRef = useRef<TextInput>(null);
   const messageRecorded = useRef<boolean>(false);
   const llm = useLLM({
-    modelSource: LLAMA3_2_1B_QLORA,
-    tokenizerSource: LLAMA3_2_1B_TOKENIZER,
-    tokenizerConfigSource: LLAMA3_2_TOKENIZER_CONFIG,
+    modelSource: QWEN3_0_6B_QUANTIZED,
+    tokenizerSource: QWEN3_TOKENIZER,
+    tokenizerConfigSource: QWEN3_TOKENIZER_CONFIG,
     chatConfig: {
       contextWindowLength: 6,
     },
   });
   const speechToText = useSpeechToText({
-    modelName: 'whisper',
-    windowSize: 5,
+    modelName: 'moonshine',
+    windowSize: 3,
     overlapSeconds: 1.2,
   });
 
   const onChunk = (data: string) => {
     const float32Chunk = float32ArrayFromPCMBinaryBuffer(data);
-    speechToText.transcribe(Array.from(float32Chunk));
-    // speechToText.streamingTranscribe(
-    //   STREAMING_ACTION.DATA,
-    //   Array.from(float32Chunk)
-    // );
+    speechToText.streamingTranscribe(
+      STREAMING_ACTION.DATA,
+      Array.from(float32Chunk)
+    );
   };
 
   const handleRecordPress = async () => {
     if (isRecording) {
       setIsRecording(false);
       LiveAudioStream.stop();
       messageRecorded.current = true;
-      // await llm.generate(
-      //   await speechToText.streamingTranscribe(STREAMING_ACTION.STOP)
-      // );
+      await llm.sendMessage(
+        await speechToText.streamingTranscribe(STREAMING_ACTION.STOP)
+      );
     } else {
       setIsRecording(true);
       startStreamingAudio(audioStreamOptions, onChunk);
-      // await speechToText.streamingTranscribe(STREAMING_ACTION.START);
+      await speechToText.streamingTranscribe(STREAMING_ACTION.START);
     }
   };
 
-  // const sendMessage = async () => {
-  //   setUserInput('');
-  //   textInputRef.current?.clear();
-  //   try {
-  //     await llm.sendMessage(userInput);
-  //   } catch (e) {
-  //     console.error(e);
-  //   }
-  // };
   const sendMessage = async () => {
     if (userInput) {
       llm.sendMessage(userInput);
@@ -122,28 +110,6 @@ export default function ChatScreen() {
     }
   };
 
-  // useEffect(() => {
-  //   if (llm.response && !llm.isGenerating) {
-  //     appendToMessageHistory(llm.response, 'assistant');
-  //   }
-  // }, [llm.response, llm.isGenerating]);
-
-  // const modifyLastMessage = (content: string) => {
-  //   setCurrentMessage((prevMessage) => prevMessage + content);
-  // };
-
-  useEffect(() => {
-    if (speechToText.sequence.length && !speechToText.isGenerating) {
-      llm.sendMessage(speechToText.sequence);
-    }
-  }, [speechToText.sequence, speechToText.isGenerating, llm.sendMessage]); //eslint-disable-line react-hooks/exhaustive-deps
-
-  // const appendToMessageHistory = (content: string, role: SenderType) => {
-  //   setChatHistory((prevHistory) => [...prevHistory, { role, content }]);
-  //   if (role == 'assistant' && messageRecorded.current)
-  //     Speech.speak(content, { language: 'en-US' });
-  // };
-
   return !llm.isReady || !speechToText.isReady ? (
     <Spinner
       visible={!llm.isReady || !speechToText.isReady}
@@ -161,13 +127,17 @@ export default function ChatScreen() {
             <SWMIcon width={45} height={45} />
             <Text style={styles.textModelName}>llm 3.2 1B QLoRA x Whisper</Text>
           </View>
-          {llm.messageHistory.length ? (
+          {llm.messageHistory.length || speechToText.sequence ? (
             <View style={styles.chatContainer}>
               <Messages
-                chatHistory={[
-                  ...llm.messageHistory,
-                  { role: 'user', content: speechToText.sequence },
-                ]}
+                chatHistory={
+                  speechToText.isGenerating
+                    ? [
+                        ...llm.messageHistory,
+                        { role: 'user', content: speechToText.sequence },
+                      ]
+                    : llm.messageHistory
+                }
                 llmResponse={llm.response}
                 isGenerating={llm.isGenerating}
                 deleteMessage={llm.deleteMessage}