diff --git a/examples/android/RunAnywhereAI/app/build.gradle.kts b/examples/android/RunAnywhereAI/app/build.gradle.kts index 335c2aee44..ab9503a53d 100644 --- a/examples/android/RunAnywhereAI/app/build.gradle.kts +++ b/examples/android/RunAnywhereAI/app/build.gradle.kts @@ -97,6 +97,9 @@ dependencies { implementation(libs.androidx.lifecycle.viewmodel.compose) implementation(libs.androidx.navigation.compose) implementation(libs.kotlinx.serialization.json) + // files(...) AARs carry no POM; declare coroutines 1.11.0 directly so it outranks + // the older transitive core from androidx (SDK is compiled against 1.11.0). + implementation(libs.kotlinx.coroutines.android) implementation(libs.proto.wire.runtime) testImplementation(libs.junit) diff --git a/examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/RunAnywhereApplication.kt b/examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/RunAnywhereApplication.kt index 15a8c34109..b7bbfbe08a 100644 --- a/examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/RunAnywhereApplication.kt +++ b/examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/RunAnywhereApplication.kt @@ -74,7 +74,11 @@ class RunAnywhereApplication : Application() { RunAnywhere.initialize( apiKey = BuildConfig.RUNANYWHERE_API_KEY, baseURL = BuildConfig.RUNANYWHERE_BASE_URL, - environment = SDKEnvironment.SDK_ENVIRONMENT_PRODUCTION, + // STAGING (not PRODUCTION) so the railway *development* backend in + // local.properties is honored and telemetry lands — matches the + // Flutter example. PRODUCTION expects a production backend/auth and + // also disables console logging. + environment = SDKEnvironment.SDK_ENVIRONMENT_STAGING, ) // Production env disables SDK console logging entirely; without this // debug builds emit zero SDK logs to logcat, which makes on-device diff --git a/examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/ui/screens/voice/VoiceScreen.kt b/examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/ui/screens/voice/VoiceScreen.kt index 576cfe3e63..04856ba6d8 100644 --- a/examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/ui/screens/voice/VoiceScreen.kt +++ b/examples/android/RunAnywhereAI/app/src/main/java/com/runanywhere/runanywhereai/ui/screens/voice/VoiceScreen.kt @@ -62,13 +62,16 @@ fun VoiceScreen() { viewModel(key = "voice-stt", factory = ModelSelectionViewModel.Factory(ModelSelectionContext.STT)) val ttsVm: ModelSelectionViewModel = viewModel(key = "voice-tts", factory = ModelSelectionViewModel.Factory(ModelSelectionContext.TTS)) + val vadVm: ModelSelectionViewModel = + viewModel(key = "voice-vad", factory = ModelSelectionViewModel.Factory(ModelSelectionContext.VAD)) var sheet by remember { mutableStateOf(null) } val listState = rememberLazyListState() val llmName = GlobalState.model.loaded?.name val sttName = sttVm.state.models.firstOrNull { it.id == sttVm.state.currentModelId }?.name val ttsVoice = ttsVm.state.models.firstOrNull { it.id == ttsVm.state.currentModelId } - val ready = llmName != null && sttName != null && ttsVoice != null + val vadName = vadVm.state.models.firstOrNull { it.id == vadVm.state.currentModelId }?.name + val ready = llmName != null && sttName != null && ttsVoice != null && vadName != null val permissionLauncher = rememberLauncherForActivityResult( ActivityResultContracts.RequestPermission(), @@ -110,7 +113,9 @@ fun VoiceScreen() { SetupRow(RACIcons.Outline.Brain, "Speech to text", sttName, onClick = { sheet = sttVm }) Divider() SetupRow(RACIcons.Outline.Robot, "Voice", ttsVoice?.name, onClick = { sheet = ttsVm }) - } + Divider() + SetupRow(RACIcons.Outline.Activity, "Voice activity (VAD)", vadName, onClick = { sheet = vadVm }) +} } Box(modifier = Modifier.weight(1f), contentAlignment = Alignment.Center) { diff --git a/examples/android/RunAnywhereAI/gradle/libs.versions.toml b/examples/android/RunAnywhereAI/gradle/libs.versions.toml index 73186dbd23..3418532023 100644 --- a/examples/android/RunAnywhereAI/gradle/libs.versions.toml +++ b/examples/android/RunAnywhereAI/gradle/libs.versions.toml @@ -11,13 +11,13 @@ kotlin = "2.4.0" # AndroidX core coreKtx = "1.19.0" activityCompose = "1.13.0" -lifecycleRuntimeKtx = "2.10.0" +lifecycleRuntimeKtx = "2.11.0" # Security securityCrypto = "1.1.0" # Compose -composeBom = "2026.05.01" +composeBom = "2026.06.00" # Navigation & serialization navigationCompose = "2.9.8" @@ -37,7 +37,7 @@ espressoCore = "3.7.0" wire = "6.4.0" # Networking -okhttp = "5.3.2" +okhttp = "5.4.0" # Document parsing (RAG ingestion) pdfbox = "2.0.27.0" diff --git a/examples/android/RunAnywhereAI/gradle/wrapper/gradle-wrapper.properties b/examples/android/RunAnywhereAI/gradle/wrapper/gradle-wrapper.properties index b52fb7e713..eb84db68da 100644 --- a/examples/android/RunAnywhereAI/gradle/wrapper/gradle-wrapper.properties +++ b/examples/android/RunAnywhereAI/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.0-bin.zip networkTimeout=10000 retries=0 retryBackOffMs=500 diff --git a/examples/flutter/RunAnywhereAI/.gitignore b/examples/flutter/RunAnywhereAI/.gitignore index 5d05bc30a9..b4542badb3 100644 --- a/examples/flutter/RunAnywhereAI/.gitignore +++ b/examples/flutter/RunAnywhereAI/.gitignore @@ -68,3 +68,6 @@ android/settings.gradle.kts # FVM Version Cache .fvm/ + +# Local --dart-define secrets (API key / base URL) +dart_defines.json diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/AndroidManifest.xml b/examples/flutter/RunAnywhereAI/android/app/src/main/AndroidManifest.xml index 40a1640f32..e1793a9606 100644 --- a/examples/flutter/RunAnywhereAI/android/app/src/main/AndroidManifest.xml +++ b/examples/flutter/RunAnywhereAI/android/app/src/main/AndroidManifest.xml @@ -8,6 +8,7 @@ android:label="RunAnywhere AI" android:name="${applicationName}" android:icon="@mipmap/ic_launcher" + android:roundIcon="@mipmap/ic_launcher_round" android:extractNativeLibs="false" android:enableOnBackInvokedCallback="true"> diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/java/io/flutter/plugins/GeneratedPluginRegistrant.java b/examples/flutter/RunAnywhereAI/android/app/src/main/java/io/flutter/plugins/GeneratedPluginRegistrant.java index 8f2b535466..bbd1a2d934 100644 --- a/examples/flutter/RunAnywhereAI/android/app/src/main/java/io/flutter/plugins/GeneratedPluginRegistrant.java +++ b/examples/flutter/RunAnywhereAI/android/app/src/main/java/io/flutter/plugins/GeneratedPluginRegistrant.java @@ -51,19 +51,14 @@ public static void registerWith(@NonNull FlutterEngine flutterEngine) { Log.e(TAG, "Error registering plugin image_picker_android, io.flutter.plugins.imagepicker.ImagePickerPlugin", e); } try { - flutterEngine.getPlugins().add(new com.github.dart_lang.jni.JniPlugin()); - } catch (Exception e) { - Log.e(TAG, "Error registering plugin jni, com.github.dart_lang.jni.JniPlugin", e); - } - try { - flutterEngine.getPlugins().add(new com.github.dart_lang.jni_flutter.JniFlutterPlugin()); + flutterEngine.getPlugins().add(new dev.fluttercommunity.plus.packageinfo.PackageInfoPlugin()); } catch (Exception e) { - Log.e(TAG, "Error registering plugin jni_flutter, com.github.dart_lang.jni_flutter.JniFlutterPlugin", e); + Log.e(TAG, "Error registering plugin package_info_plus, dev.fluttercommunity.plus.packageinfo.PackageInfoPlugin", e); } try { - flutterEngine.getPlugins().add(new dev.fluttercommunity.plus.packageinfo.PackageInfoPlugin()); + flutterEngine.getPlugins().add(new io.flutter.plugins.pathprovider.PathProviderPlugin()); } catch (Exception e) { - Log.e(TAG, "Error registering plugin package_info_plus, dev.fluttercommunity.plus.packageinfo.PackageInfoPlugin", e); + Log.e(TAG, "Error registering plugin path_provider_android, io.flutter.plugins.pathprovider.PathProviderPlugin", e); } try { flutterEngine.getPlugins().add(new com.baseflow.permissionhandler.PermissionHandlerPlugin()); diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/drawable/ic_launcher_foreground.xml b/examples/flutter/RunAnywhereAI/android/app/src/main/res/drawable/ic_launcher_foreground.xml deleted file mode 100644 index 1fa802f9fd..0000000000 --- a/examples/flutter/RunAnywhereAI/android/app/src/main/res/drawable/ic_launcher_foreground.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml deleted file mode 100644 index 5ed0a2df70..0000000000 --- a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml deleted file mode 100644 index 5ed0a2df70..0000000000 --- a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher.png index 08cd6f2bfd..cb0cbf6d59 100644 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher.png and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png deleted file mode 100644 index 08cd6f2bfd..0000000000 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png new file mode 100644 index 0000000000..67c4925ac2 Binary files /dev/null and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-hdpi/ic_launcher_round.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher.png index 08cd6f2bfd..c6f59d4841 100644 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher.png and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png deleted file mode 100644 index 08cd6f2bfd..0000000000 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png new file mode 100644 index 0000000000..54c9d85d5b Binary files /dev/null and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-mdpi/ic_launcher_round.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png index 08cd6f2bfd..33df0290c1 100644 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png deleted file mode 100644 index 08cd6f2bfd..0000000000 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png new file mode 100644 index 0000000000..faa0371669 Binary files /dev/null and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xhdpi/ic_launcher_round.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png index 08cd6f2bfd..172a969f0c 100644 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png deleted file mode 100644 index 08cd6f2bfd..0000000000 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png new file mode 100644 index 0000000000..f605f26aa6 Binary files /dev/null and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png index 08cd6f2bfd..21617fdbf2 100644 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png deleted file mode 100644 index 08cd6f2bfd..0000000000 Binary files a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png and /dev/null differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png new file mode 100644 index 0000000000..2501997c06 Binary files /dev/null and b/examples/flutter/RunAnywhereAI/android/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.png differ diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/values-night/styles.xml b/examples/flutter/RunAnywhereAI/android/app/src/main/res/values-night/styles.xml index 06952be745..62b9a93400 100644 --- a/examples/flutter/RunAnywhereAI/android/app/src/main/res/values-night/styles.xml +++ b/examples/flutter/RunAnywhereAI/android/app/src/main/res/values-night/styles.xml @@ -5,6 +5,8 @@ @drawable/launch_background + @android:color/black + false diff --git a/examples/flutter/RunAnywhereAI/android/app/src/main/res/values/styles.xml b/examples/flutter/RunAnywhereAI/android/app/src/main/res/values/styles.xml index 7819bd331c..d2c816f88c 100644 --- a/examples/flutter/RunAnywhereAI/android/app/src/main/res/values/styles.xml +++ b/examples/flutter/RunAnywhereAI/android/app/src/main/res/values/styles.xml @@ -5,9 +5,13 @@ @drawable/launch_background + @android:color/white + true diff --git a/examples/flutter/RunAnywhereAI/android/build.gradle b/examples/flutter/RunAnywhereAI/android/build.gradle index f6ae56bdec..6b4c558052 100644 --- a/examples/flutter/RunAnywhereAI/android/build.gradle +++ b/examples/flutter/RunAnywhereAI/android/build.gradle @@ -1,12 +1,12 @@ buildscript { - ext.kotlin_version = '2.1.21' + ext.kotlin_version = '2.2.20' repositories { google() mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:8.9.1' + classpath 'com.android.tools.build:gradle:8.11.1' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" } } diff --git a/examples/flutter/RunAnywhereAI/android/gradle/wrapper/gradle-wrapper.properties b/examples/flutter/RunAnywhereAI/android/gradle/wrapper/gradle-wrapper.properties index efdcc4ace9..74b269f35e 100644 --- a/examples/flutter/RunAnywhereAI/android/gradle/wrapper/gradle-wrapper.properties +++ b/examples/flutter/RunAnywhereAI/android/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-all.zip diff --git a/examples/flutter/RunAnywhereAI/android/settings.gradle b/examples/flutter/RunAnywhereAI/android/settings.gradle index b64bd30938..ff03f22be3 100644 --- a/examples/flutter/RunAnywhereAI/android/settings.gradle +++ b/examples/flutter/RunAnywhereAI/android/settings.gradle @@ -18,8 +18,8 @@ pluginManagement { plugins { id "dev.flutter.flutter-plugin-loader" version "1.0.0" - id "com.android.application" version "8.9.1" apply false - id "org.jetbrains.kotlin.android" version "2.1.21" apply false + id "com.android.application" version "8.11.1" apply false + id "org.jetbrains.kotlin.android" version "2.2.20" apply false } include ":app" diff --git a/examples/flutter/RunAnywhereAI/assets/fonts/figtree.ttf b/examples/flutter/RunAnywhereAI/assets/fonts/figtree.ttf new file mode 100644 index 0000000000..f93a4b6cd8 Binary files /dev/null and b/examples/flutter/RunAnywhereAI/assets/fonts/figtree.ttf differ diff --git a/examples/flutter/RunAnywhereAI/assets/fonts/figtree_italic.ttf b/examples/flutter/RunAnywhereAI/assets/fonts/figtree_italic.ttf new file mode 100644 index 0000000000..06d4ee3a1c Binary files /dev/null and b/examples/flutter/RunAnywhereAI/assets/fonts/figtree_italic.ttf differ diff --git a/examples/flutter/RunAnywhereAI/assets/fonts/maple_mono.ttf b/examples/flutter/RunAnywhereAI/assets/fonts/maple_mono.ttf new file mode 100644 index 0000000000..8373098c0b Binary files /dev/null and b/examples/flutter/RunAnywhereAI/assets/fonts/maple_mono.ttf differ diff --git a/examples/flutter/RunAnywhereAI/ios/Flutter/AppFrameworkInfo.plist b/examples/flutter/RunAnywhereAI/ios/Flutter/AppFrameworkInfo.plist index 1dc6cf7652..391a902b2b 100644 --- a/examples/flutter/RunAnywhereAI/ios/Flutter/AppFrameworkInfo.plist +++ b/examples/flutter/RunAnywhereAI/ios/Flutter/AppFrameworkInfo.plist @@ -20,7 +20,5 @@ ???? CFBundleVersion 1.0 - MinimumOSVersion - 13.0 diff --git a/examples/flutter/RunAnywhereAI/ios/Runner/AppDelegate.swift b/examples/flutter/RunAnywhereAI/ios/Runner/AppDelegate.swift index b636303481..c30b367ec0 100644 --- a/examples/flutter/RunAnywhereAI/ios/Runner/AppDelegate.swift +++ b/examples/flutter/RunAnywhereAI/ios/Runner/AppDelegate.swift @@ -1,13 +1,16 @@ -import UIKit import Flutter +import UIKit @main -@objc class AppDelegate: FlutterAppDelegate { +@objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate { override func application( _ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? ) -> Bool { - GeneratedPluginRegistrant.register(with: self) return super.application(application, didFinishLaunchingWithOptions: launchOptions) } + + func didInitializeImplicitFlutterEngine(_ engineBridge: FlutterImplicitEngineBridge) { + GeneratedPluginRegistrant.register(with: engineBridge.pluginRegistry) + } } diff --git a/examples/flutter/RunAnywhereAI/ios/Runner/GeneratedPluginRegistrant.m b/examples/flutter/RunAnywhereAI/ios/Runner/GeneratedPluginRegistrant.m index 7f41c9b8b7..5ec7b1d98b 100644 --- a/examples/flutter/RunAnywhereAI/ios/Runner/GeneratedPluginRegistrant.m +++ b/examples/flutter/RunAnywhereAI/ios/Runner/GeneratedPluginRegistrant.m @@ -48,6 +48,12 @@ @import package_info_plus; #endif +#if __has_include() +#import +#else +@import path_provider_foundation; +#endif + #if __has_include() #import #else @@ -106,6 +112,7 @@ + (void)registerWithRegistry:(NSObject*)registry { [FlutterSecureStorageDarwinPlugin registerWithRegistrar:[registry registrarForPlugin:@"FlutterSecureStorageDarwinPlugin"]]; [FLTImagePickerPlugin registerWithRegistrar:[registry registrarForPlugin:@"FLTImagePickerPlugin"]]; [FPPPackageInfoPlusPlugin registerWithRegistrar:[registry registrarForPlugin:@"FPPPackageInfoPlusPlugin"]]; + [PathProviderPlugin registerWithRegistrar:[registry registrarForPlugin:@"PathProviderPlugin"]]; [PermissionHandlerPlugin registerWithRegistrar:[registry registrarForPlugin:@"PermissionHandlerPlugin"]]; [RecordIosPlugin registerWithRegistrar:[registry registrarForPlugin:@"RecordIosPlugin"]]; [RunAnywherePlugin registerWithRegistrar:[registry registrarForPlugin:@"RunAnywherePlugin"]]; diff --git a/examples/flutter/RunAnywhereAI/ios/Runner/Info.plist b/examples/flutter/RunAnywhereAI/ios/Runner/Info.plist index ee26af2840..29f58e3a49 100644 --- a/examples/flutter/RunAnywhereAI/ios/Runner/Info.plist +++ b/examples/flutter/RunAnywhereAI/ios/Runner/Info.plist @@ -32,6 +32,27 @@ This app needs microphone access for voice assistant features. NSSpeechRecognitionUsageDescription This app needs speech recognition for voice assistant features. + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + UIWindowSceneSessionRoleApplication + + + UISceneClassName + UIWindowScene + UISceneConfigurationName + flutter + UISceneDelegateClassName + FlutterSceneDelegate + UISceneStoryboardFile + Main + + + + UIApplicationSupportsIndirectInputEvents UILaunchStoryboardName diff --git a/examples/flutter/RunAnywhereAI/lib/app/content_view.dart b/examples/flutter/RunAnywhereAI/lib/app/content_view.dart index a120bf5b16..7f3a4edd88 100644 --- a/examples/flutter/RunAnywhereAI/lib/app/content_view.dart +++ b/examples/flutter/RunAnywhereAI/lib/app/content_view.dart @@ -20,19 +20,41 @@ class ContentView extends StatefulWidget { class _ContentViewState extends State { int _selectedTab = 0; - // Tab pages matching iOS structure. - final List _pages = const [ - ChatInterfaceView(), // Tab 0: Chat (LLM) - VisionHubView(), // Tab 1: Vision (VLM) - VoiceAssistantView(), // Tab 2: Voice Assistant (STT + LLM + TTS) - MoreView(), // Tab 3: More hub - CombinedSettingsView(), // Tab 4: Settings - ]; + // Tabs are built lazily on first visit and kept alive afterwards. A hidden + // tab's initState / view-model work (animations, event-bus subscriptions, + // periodic refreshes) must not run until the user selects that tab. + // IndexedStack still preserves each tab's state once it has been built. + final Set _visitedTabs = {0}; + + Widget _buildTab(int index) { + switch (index) { + case 0: + return const ChatInterfaceView(); // Chat (LLM) + case 1: + return const VisionHubView(); // Vision (VLM) + case 2: + return const VoiceAssistantView(); // Voice Assistant (STT + LLM + TTS) + case 3: + return const MoreView(); // More hub + case 4: + return const CombinedSettingsView(); // Settings + default: + return const SizedBox.shrink(); + } + } @override Widget build(BuildContext context) { return Scaffold( - body: IndexedStack(index: _selectedTab, children: _pages), + body: IndexedStack( + index: _selectedTab, + children: List.generate( + 5, + (index) => _visitedTabs.contains(index) + ? _buildTab(index) + : const SizedBox.shrink(), + ), + ), bottomNavigationBar: NavigationBar( selectedIndex: _selectedTab, // B-FL-14-002: explicit height + vertical padding so the @@ -43,6 +65,7 @@ class _ContentViewState extends State { onDestinationSelected: (index) { setState(() { _selectedTab = index; + _visitedTabs.add(index); }); }, destinations: const [ diff --git a/examples/flutter/RunAnywhereAI/lib/app/runanywhere_ai_app.dart b/examples/flutter/RunAnywhereAI/lib/app/runanywhere_ai_app.dart index 05e3c76236..e49b7f1add 100644 --- a/examples/flutter/RunAnywhereAI/lib/app/runanywhere_ai_app.dart +++ b/examples/flutter/RunAnywhereAI/lib/app/runanywhere_ai_app.dart @@ -53,14 +53,14 @@ class _RunAnywhereAIAppState extends State { await _registerBackends(); - final customApiKey = await KeychainHelper.loadString(KeychainKeys.apiKey); - final customBaseURL = await KeychainHelper.loadString( - KeychainKeys.baseURL, - ); + final customApiKey = + await KeychainHelper.loadString(KeychainKeys.apiKey) ?? + DefaultConfig.runanywhereApiKey; + final customBaseURL = + await KeychainHelper.loadString(KeychainKeys.baseURL) ?? + DefaultConfig.runanywhereBaseUrl; final hasCustomConfig = - customApiKey != null && customApiKey.isNotEmpty && - customBaseURL != null && customBaseURL.isNotEmpty && !_looksLikePlaceholder(customApiKey) && !_looksLikePlaceholder(customBaseURL); @@ -73,9 +73,12 @@ class _RunAnywhereAIAppState extends State { await RunAnywhere.initialize( apiKey: customApiKey, baseURL: normalizedURL, - environment: SDKEnvironment.SDK_ENVIRONMENT_PRODUCTION, + // Staging (not Production) so the custom base URL is honored AND + // local logging stays on — Production sets enableLocalLogging:false, + // hiding all SDK/telemetry logs. Development would ignore baseURL. + environment: SDKEnvironment.SDK_ENVIRONMENT_STAGING, ); - debugPrint('✅ SDK initialized with CUSTOM configuration (production)'); + debugPrint('✅ SDK initialized with CUSTOM configuration (staging)'); } else { await RunAnywhere.initialize(); debugPrint('✅ SDK initialized in DEVELOPMENT mode'); diff --git a/examples/flutter/RunAnywhereAI/lib/core/services/model_catalog_bootstrap.dart b/examples/flutter/RunAnywhereAI/lib/core/services/model_catalog_bootstrap.dart index 33209d8a53..3718e9cbb7 100644 --- a/examples/flutter/RunAnywhereAI/lib/core/services/model_catalog_bootstrap.dart +++ b/examples/flutter/RunAnywhereAI/lib/core/services/model_catalog_bootstrap.dart @@ -17,10 +17,10 @@ abstract final class ModelCatalogBootstrap { static Future registerAll() async { if (_modulesRegistered) { - debugPrint('📦 Catalog already registered — skipping'); + debugPrint('Catalog already registered — skipping'); return; } - debugPrint('📦 Registering modules with their models...'); + debugPrint('Registering modules with their models...'); // --- LLM models (LlamaCpp backend) ------------------------------------ await _registerLLM( @@ -141,7 +141,7 @@ abstract final class ModelCatalogBootstrap { framework: InferenceFramework.INFERENCE_FRAMEWORK_LLAMA_CPP, memoryRequirement: 2000000000, ); - debugPrint('✅ LLM models registered'); + debugPrint('LLM models registered'); // --- VLM models (multi-modal, multi-file) ----------------------------- await _registerArchive( @@ -193,7 +193,7 @@ abstract final class ModelCatalogBootstrap { modality: ModelCategory.MODEL_CATEGORY_MULTIMODAL, memoryRequirement: 600000000, ); - debugPrint('✅ VLM models registered'); + debugPrint('VLM models registered'); // --- STT models (Sherpa-ONNX) ----------------------------------------- await _registerArchive( @@ -246,7 +246,7 @@ abstract final class ModelCatalogBootstrap { // guard on a valid ~2.3 MB download. memoryRequirement: 2327524, ); - debugPrint('✅ Sherpa STT/TTS + Silero VAD models registered'); + debugPrint('Sherpa STT/TTS + Silero VAD models registered'); // --- ONNX Embedding (RAG) --------------------------------------------- // MiniLM needs model.onnx + vocab.txt in the same folder for the C++ @@ -270,14 +270,14 @@ abstract final class ModelCatalogBootstrap { modality: ModelCategory.MODEL_CATEGORY_EMBEDDING, memoryRequirement: 25500000, ); - debugPrint('✅ ONNX Embedding models registered'); + debugPrint('ONNX Embedding models registered'); // --- LoRA adapters ------------------------------------------------------ // Mirrors iOS `registerLoraAdapters` / Android `ModelBootstrap.seedLora`. await _registerLoraAdapters(); - debugPrint('✅ LoRA adapters registered'); + debugPrint('LoRA adapters registered'); - debugPrint('🎉 All modules and models registered'); + debugPrint('All modules and models registered'); _modulesRegistered = true; } @@ -300,7 +300,7 @@ abstract final class ModelCatalogBootstrap { try { await RunAnywhere.lora.registerArtifact(adapter); } catch (e) { - debugPrint('⚠️ Failed to register LoRA adapter: $e'); + debugPrint('Failed to register LoRA adapter: $e'); } } @@ -329,7 +329,7 @@ abstract final class ModelCatalogBootstrap { supportsLora: supportsLora, ); } catch (e) { - debugPrint('⚠️ Failed to register model $id: $e'); + debugPrint('Failed to register model $id: $e'); } } @@ -355,7 +355,7 @@ abstract final class ModelCatalogBootstrap { memoryRequirement: memoryRequirement, ); } catch (e) { - debugPrint('⚠️ Failed to register archive model $id: $e'); + debugPrint('Failed to register archive model $id: $e'); } } @@ -392,7 +392,7 @@ abstract final class ModelCatalogBootstrap { memoryRequirement: memoryRequirement, ); } catch (e) { - debugPrint('⚠️ Failed to register multi-file model $id: $e'); + debugPrint('Failed to register multi-file model $id: $e'); } } } diff --git a/examples/flutter/RunAnywhereAI/lib/core/utilities/constants.dart b/examples/flutter/RunAnywhereAI/lib/core/utilities/constants.dart index e64688da29..5203c4bf39 100644 --- a/examples/flutter/RunAnywhereAI/lib/core/utilities/constants.dart +++ b/examples/flutter/RunAnywhereAI/lib/core/utilities/constants.dart @@ -2,6 +2,23 @@ // // Application-wide constant values. +/// Default backend config, supplied at build time via --dart-define so no +/// secret is committed (mirrors the Android example's gitignored +/// local.properties → BuildConfig.RUNANYWHERE_API_KEY / _BASE_URL). Empty when +/// not provided, in which case the app falls back to the SDK's dev defaults. +/// +/// flutter run \ +/// --dart-define=RUNANYWHERE_API_KEY=runa_prod_... \ +/// --dart-define=RUNANYWHERE_BASE_URL=https://...up.railway.app +class DefaultConfig { + DefaultConfig._(); + + static const String runanywhereApiKey = + String.fromEnvironment('RUNANYWHERE_API_KEY'); + static const String runanywhereBaseUrl = + String.fromEnvironment('RUNANYWHERE_BASE_URL'); +} + /// Keychain keys for secure storage class KeychainKeys { KeychainKeys._(); diff --git a/examples/flutter/RunAnywhereAI/lib/features/benchmarks/llm_benchmark_provider.dart b/examples/flutter/RunAnywhereAI/lib/features/benchmarks/llm_benchmark_provider.dart index b77d706046..c036877f1a 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/benchmarks/llm_benchmark_provider.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/benchmarks/llm_benchmark_provider.dart @@ -4,9 +4,9 @@ import 'package:runanywhere_ai/features/benchmarks/benchmark_types.dart'; /// Benchmarks LLM generation with short/medium/long token counts. /// -/// Mirrors iOS `LLMBenchmarkProvider.swift`: load → warmup → streamed +/// Mirrors iOS `LLMBenchmarkProvider.swift`: load warmup streamed /// generation aggregated into TTFT / tokens-per-second / prefill / decode -/// throughput → unload. +/// throughput unload. class LLMBenchmarkProvider implements BenchmarkScenarioProvider { @override BenchmarkCategory get category => BenchmarkCategory.llm; diff --git a/examples/flutter/RunAnywhereAI/lib/features/benchmarks/tts_benchmark_provider.dart b/examples/flutter/RunAnywhereAI/lib/features/benchmarks/tts_benchmark_provider.dart index b46f13a365..ef27ba850c 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/benchmarks/tts_benchmark_provider.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/benchmarks/tts_benchmark_provider.dart @@ -4,9 +4,9 @@ import 'package:runanywhere_ai/features/benchmarks/benchmark_types.dart'; /// Benchmarks TTS synthesis with short and medium text inputs. /// -/// Mirrors iOS `TTSBenchmarkProvider.swift`: load voice → synthesize (not -/// speak — no playback) → report latency, produced audio duration, and -/// character throughput → unload. +/// Mirrors iOS `TTSBenchmarkProvider.swift`: load voice synthesize (not +/// speak — no playback) report latency, produced audio duration, and +/// character throughput unload. class TTSBenchmarkProvider implements BenchmarkScenarioProvider { @override BenchmarkCategory get category => BenchmarkCategory.tts; diff --git a/examples/flutter/RunAnywhereAI/lib/features/chat/chat_view_model.dart b/examples/flutter/RunAnywhereAI/lib/features/chat/chat_view_model.dart index 213e3a3a6e..2ece2f485f 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/chat/chat_view_model.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/chat/chat_view_model.dart @@ -147,6 +147,20 @@ class ChatViewModel extends ChangeNotifier { // LLMViewModel.subscribeToModelLifecycle). _lifecycleSubscription = sdk.RunAnywhere.events.modelLifecycle.listen((change) { + // iOS parity (LLMViewModel.handleModelLifecycle): only react to LLM + // component changes. The lifecycle echo published by currentModel() + // when nothing is loaded carries an unspecified component, so this + // also drops it. + if (change.component != sdk.SDKComponent.SDK_COMPONENT_LLM && + change.event.category != sdk.EventCategory.EVENT_CATEGORY_LLM) { + return; + } + // syncModelState() calls currentModel(), which itself re-publishes a + // lifecycle event. Re-sync only on an actual transition so that echo + // cannot feed back into an infinite query -> publish loop. + final loaded = change.kind == sdk.ModelLifecycleChangeKind.loaded; + if (loaded && change.modelId == _loadedModelId) return; + if (!loaded && _loadedModelId == null) return; unawaited(syncModelState()); }); diff --git a/examples/flutter/RunAnywhereAI/lib/features/models/model_list_view_model.dart b/examples/flutter/RunAnywhereAI/lib/features/models/model_list_view_model.dart index 8ac8f365ee..ec7c59a82c 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/models/model_list_view_model.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/models/model_list_view_model.dart @@ -54,13 +54,13 @@ class ModelListViewModel extends ChangeNotifier { _availableModels = sdkModels; debugPrint( - '✅ Loaded ${_availableModels.length} models from SDK registry'); + 'Loaded ${_availableModels.length} models from SDK registry'); for (final model in _availableModels) { debugPrint( ' - ${model.name} (${model.category.displayName}) [${model.preferredFramework.displayName}] ready: ${model.isReadyOnDevice}'); } } catch (e) { - debugPrint('❌ Failed to load models from SDK: $e'); + debugPrint('Failed to load models from SDK: $e'); _errorMessage = 'Failed to load models: $e'; _availableModels = []; } @@ -81,10 +81,10 @@ class ModelListViewModel extends ChangeNotifier { } _availableFrameworks = frameworks.toList(); debugPrint( - '✅ Available frameworks: ${_availableFrameworks.map((f) => f.displayName).join(", ")}'); + 'Available frameworks: ${_availableFrameworks.map((f) => f.displayName).join(", ")}'); notifyListeners(); } catch (e) { - debugPrint('❌ Failed to load frameworks: $e'); + debugPrint('Failed to load frameworks: $e'); _availableFrameworks = []; notifyListeners(); } @@ -107,7 +107,7 @@ class ModelListViewModel extends ChangeNotifier { try { await loadModel(model); setCurrentModel(model); - debugPrint('✅ Model ${model.name} selected and loaded'); + debugPrint('Model ${model.name} selected and loaded'); } catch (e) { _errorMessage = 'Failed to load model: $e'; notifyListeners(); @@ -121,7 +121,7 @@ class ModelListViewModel extends ChangeNotifier { void Function(double) progressHandler, ) async { if (_downloadingModels.contains(model.id)) { - debugPrint('⚠️ Model ${model.id} is already downloading'); + debugPrint('Model ${model.id} is already downloading'); return; } @@ -130,7 +130,7 @@ class ModelListViewModel extends ChangeNotifier { notifyListeners(); try { - debugPrint('📥 Starting download for model: ${model.name}'); + debugPrint('Starting download for model: ${model.name}'); await for (final progress in sdk.RunAnywhere.downloads.start(model.id)) { final totalBytes = progress.totalBytes.toInt(); @@ -144,7 +144,7 @@ class ModelListViewModel extends ChangeNotifier { // Check if completed or failed if (progress.stage == sdk.DownloadStage.DOWNLOAD_STAGE_COMPLETED) { - debugPrint('✅ Download completed for model: ${model.name}'); + debugPrint('Download completed for model: ${model.name}'); break; } else if (progress.stage == sdk.DownloadStage.DOWNLOAD_STAGE_UNSPECIFIED && @@ -156,9 +156,9 @@ class ModelListViewModel extends ChangeNotifier { // Update model with local path after download await loadModelsFromRegistry(); - debugPrint('✅ Model ${model.name} download complete'); + debugPrint('Model ${model.name} download complete'); } catch (e) { - debugPrint('❌ Failed to download model ${model.id}: $e'); + debugPrint('Failed to download model ${model.id}: $e'); _errorMessage = 'Download failed: $e'; } finally { _downloadingModels.remove(model.id); @@ -170,16 +170,16 @@ class ModelListViewModel extends ChangeNotifier { /// Delete a downloaded model using SDK Future deleteModel(ModelInfo model) async { try { - debugPrint('🗑️ Deleting model: ${model.name}'); + debugPrint('Deleting model: ${model.name}'); await sdk.RunAnywhere.deleteModel(model.id); // Refresh models from registry await loadModelsFromRegistry(); - debugPrint('✅ Model ${model.name} deleted successfully'); + debugPrint('Model ${model.name} deleted successfully'); } catch (e) { - debugPrint('❌ Failed to delete model: $e'); + debugPrint('Failed to delete model: $e'); _errorMessage = 'Failed to delete model: $e'; notifyListeners(); } @@ -208,12 +208,12 @@ class ModelListViewModel extends ChangeNotifier { }; if (alreadyLoadedId == model.id) { - debugPrint('♻️ Model ${model.name} already loaded — skipping reload'); + debugPrint('Model ${model.name} already loaded — skipping reload'); _currentModel = model; return; } - debugPrint('⏳ Loading model: ${model.name}'); + debugPrint('Loading model: ${model.name}'); switch (model.category) { case ModelCategory.MODEL_CATEGORY_LANGUAGE: @@ -239,9 +239,9 @@ class ModelListViewModel extends ChangeNotifier { } _currentModel = model; - debugPrint('✅ Model ${model.name} loaded successfully'); + debugPrint('Model ${model.name} loaded successfully'); } catch (e) { - debugPrint('❌ Failed to load model ${model.id}: $e'); + debugPrint('Failed to load model ${model.id}: $e'); _errorMessage = 'Failed to load model: $e'; rethrow; } finally { diff --git a/examples/flutter/RunAnywhereAI/lib/features/models/model_selection_sheet.dart b/examples/flutter/RunAnywhereAI/lib/features/models/model_selection_sheet.dart index 19477c464e..5cd7a73e91 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/models/model_selection_sheet.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/models/model_selection_sheet.dart @@ -421,7 +421,7 @@ class _ModelSelectionSheetState extends State { // Call the callback - this is where the actual model loading happens // The callback knows the correct context and how to load the model - debugPrint('🎯 Model selected: ${model.id}, calling callback to load'); + debugPrint('Model selected: ${model.id}, calling callback to load'); await widget.onModelSelected(model); if (mounted) { @@ -435,7 +435,7 @@ class _ModelSelectionSheetState extends State { }); } } catch (e) { - debugPrint('❌ Failed to load model: $e'); + debugPrint('Failed to load model: $e'); setState(() { _isLoadingModel = false; _loadingProgress = ''; diff --git a/examples/flutter/RunAnywhereAI/lib/features/rag/rag_view_model.dart b/examples/flutter/RunAnywhereAI/lib/features/rag/rag_view_model.dart index 841ebdf39a..581c679f4a 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/rag/rag_view_model.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/rag/rag_view_model.dart @@ -2,6 +2,7 @@ // // Coordinates document extraction, SDK pipeline lifecycle, and query state. +import 'dart:async'; import 'dart:io'; import 'package:flutter/foundation.dart'; @@ -96,6 +97,9 @@ class RAGViewModel extends ChangeNotifier { _llmSupportsThinking = llmModel.supportsThinking; } catch (e) { _error = e.toString(); + // Tear down any partially-created pipeline so a failed ingest doesn't + // leave an orphaned C++ pipeline session behind. + await RunAnywhere.rag.destroyPipeline(); } finally { _isLoadingDocument = false; notifyListeners(); @@ -170,4 +174,12 @@ class RAGViewModel extends ChangeNotifier { _llmSupportsThinking = false; notifyListeners(); } + + @override + void dispose() { + // Destroy the C++ RAG pipeline so it isn't leaked when the screen is popped + // without an explicit clearDocument(). dispose() is sync, so fire-and-forget. + unawaited(RunAnywhere.rag.destroyPipeline()); + super.dispose(); + } } diff --git a/examples/flutter/RunAnywhereAI/lib/features/solutions/solutions_view.dart b/examples/flutter/RunAnywhereAI/lib/features/solutions/solutions_view.dart index 06453518ed..62d51417f3 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/solutions/solutions_view.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/solutions/solutions_view.dart @@ -30,16 +30,16 @@ class _SolutionsViewState extends State { Future _runSolution(String name, String yaml) async { if (_isRunning) return; setState(() => _isRunning = true); - _append('→ $name: creating solution from YAML…'); + _append('$name: creating solution from YAML…'); try { final handle = await RunAnywhere.solutions.run(yaml: yaml); - _append('✓ $name: handle created. Calling start()…'); + _append('$name: handle created. Calling start()…'); handle.start(); - _append('✓ $name: started. Tearing down (demo).'); + _append('$name: started. Tearing down (demo).'); handle.destroy(); - _append('✓ $name: destroyed.'); + _append('$name: destroyed.'); } catch (e) { - _append('✗ $name: $e'); + _append('$name: $e'); } finally { if (mounted) { setState(() => _isRunning = false); diff --git a/examples/flutter/RunAnywhereAI/lib/features/vision/vlm_camera_view.dart b/examples/flutter/RunAnywhereAI/lib/features/vision/vlm_camera_view.dart index 899347c7f6..365b24d41d 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/vision/vlm_camera_view.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/vision/vlm_camera_view.dart @@ -1,6 +1,6 @@ import 'dart:async'; +import 'dart:io'; -import 'package:camera/camera.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:image_picker/image_picker.dart'; @@ -11,15 +11,13 @@ import 'package:runanywhere_ai/features/models/model_selection_sheet.dart'; import 'package:runanywhere_ai/features/models/model_types.dart'; import 'package:runanywhere_ai/features/vision/vlm_view_model.dart'; -/// VLMCameraView - Camera view for Vision Language Model +/// VLMCameraView - Vision Language Model screen. /// -/// Mirrors iOS VLMCameraView.swift exactly: -/// - Camera preview (45% screen height) -/// - Description panel with streaming text -/// - Control bar with 4 buttons (Photos, Main action, Live, Model) -/// - Model-required screen when no model loaded -/// - Auto-streaming with 2.5s interval -/// - Single capture and gallery modes +/// Mirrors the Android Kotlin VisionScreen: the user supplies an image from the +/// device gallery or the device camera app (via the OS picker — no in-app +/// camera preview), enters a prompt, and runs a streamed description over the +/// loaded VLM model. Styled with the app design system so it stays consistent +/// with the chat screen (theme-driven surfaces, no hardcoded black/white). class VLMCameraView extends StatefulWidget { const VLMCameraView({super.key}); @@ -28,32 +26,17 @@ class VLMCameraView extends StatefulWidget { } class _VLMCameraViewState extends State { - late VLMViewModel _viewModel; + late final VLMViewModel _viewModel; + final ImagePicker _picker = ImagePicker(); + late final TextEditingController _promptController; @override void initState() { super.initState(); _viewModel = VLMViewModel(); - - // Listen to view model changes + _promptController = TextEditingController(text: _viewModel.prompt); _viewModel.addListener(_onViewModelChanged); - - // Initialize - debugPrint('VLM streaming completed'); - _initializeAsync(); - } - - void _initializeAsync() { - unawaited( - _viewModel.checkModelStatus().then((_) async { - if (mounted) { - await _viewModel.checkCameraAuthorization(context); - if (_viewModel.isCameraAuthorized) { - unawaited(_viewModel.initializeCamera()); - } - } - }), - ); + unawaited(_viewModel.checkModelStatus()); } void _onViewModelChanged() { @@ -64,26 +47,19 @@ class _VLMCameraViewState extends State { @override void dispose() { - _viewModel.stopAutoStreaming(); - _viewModel.disposeCamera(); _viewModel.removeListener(_onViewModelChanged); _viewModel.dispose(); + _promptController.dispose(); super.dispose(); } @override Widget build(BuildContext context) { return Scaffold( - backgroundColor: Colors.black, appBar: _buildAppBar(), - body: Stack( - children: [ - if (_viewModel.isModelLoaded) - _buildMainContent() - else - _buildModelRequiredContent(), - ], - ), + body: _viewModel.isModelLoaded + ? _buildMainContent() + : _buildModelRequiredContent(), ); } @@ -92,21 +68,23 @@ class _VLMCameraViewState extends State { PreferredSizeWidget _buildAppBar() { return AppBar( title: const Text('Vision AI'), - backgroundColor: Colors.black, - foregroundColor: Colors.white, actions: [ if (_viewModel.loadedModelName != null) - Padding( - padding: const EdgeInsets.only(right: AppSpacing.large), - child: Center( + Center( + child: Padding( + padding: const EdgeInsets.symmetric(horizontal: AppSpacing.small), child: Text( _viewModel.loadedModelName!, - style: AppTypography.caption(context).copyWith( - color: Colors.grey, - ), + style: AppTypography.caption(context) + .copyWith(color: AppColors.textSecondary(context)), ), ), ), + IconButton( + icon: const Icon(Icons.swap_horiz), + tooltip: 'Change model', + onPressed: _onSelectModel, + ), ], ); } @@ -114,375 +92,250 @@ class _VLMCameraViewState extends State { // MARK: - Main Content Widget _buildMainContent() { - return Column( - children: [ - // Camera preview (45% screen height) - _buildCameraPreview(), - - // Description panel (flexible) - Expanded(child: _buildDescriptionPanel()), - - // Control bar (fixed at bottom) - _buildControlBar(), - ], - ); - } - - // MARK: - Camera Preview - - Widget _buildCameraPreview() { - final screenHeight = MediaQuery.of(context).size.height; - final cameraHeight = screenHeight * 0.45; - - return SizedBox( - height: cameraHeight, - child: Stack( - children: [ - // Camera preview or permission view - if (_viewModel.isCameraAuthorized) - _buildCameraPreviewContent() - else - _buildCameraPermissionView(), - - // Processing overlay - if (_viewModel.isProcessing) _buildProcessingOverlay(), - ], - ), - ); - } - - Widget _buildCameraPreviewContent() { - if (!_viewModel.isCameraInitialized || - _viewModel.cameraController == null) { - return const Center( - child: CircularProgressIndicator(color: Colors.white), - ); - } - - return Container( - color: Colors.black, - child: CameraPreview(_viewModel.cameraController!), - ); - } - - Widget _buildCameraPermissionView() { - return Center( + return SingleChildScrollView( + padding: const EdgeInsets.all(AppSpacing.large), child: Column( - mainAxisAlignment: MainAxisAlignment.center, + crossAxisAlignment: CrossAxisAlignment.stretch, children: [ - const Icon(Icons.camera_alt, size: 60, color: Colors.grey), - const SizedBox(height: AppSpacing.mediumLarge), - Text( - 'Camera Access Required', - style: - AppTypography.headline(context).copyWith(color: Colors.white), - ), + _buildImagePreview(), + const SizedBox(height: AppSpacing.large), + _buildSourceButtons(), + const SizedBox(height: AppSpacing.large), + _buildPromptField(), const SizedBox(height: AppSpacing.mediumLarge), - ElevatedButton( - onPressed: () { - unawaited( - _viewModel.checkCameraAuthorization(context).then((_) { - if (_viewModel.isCameraAuthorized) { - unawaited(_viewModel.initializeCamera()); - } - }), - ); - }, - child: const Text('Open Settings'), - ), + _buildDescribeButton(), + const SizedBox(height: AppSpacing.large), + _buildResult(), ], ), ); } - Widget _buildProcessingOverlay() { - return Positioned( - bottom: AppSpacing.large, - left: 0, - right: 0, - child: Center( - child: Container( - padding: const EdgeInsets.symmetric( - horizontal: AppSpacing.large, - vertical: AppSpacing.mediumLarge, - ), - decoration: BoxDecoration( - color: Colors.black.withValues(alpha: 0.7), - borderRadius: BorderRadius.circular(AppSpacing.cornerRadiusModal), - ), - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - const SizedBox( - width: 16, - height: 16, - child: CircularProgressIndicator( - strokeWidth: 2, - color: Colors.white, - ), - ), - const SizedBox(width: AppSpacing.smallMedium), - Text( - 'Analyzing...', - style: AppTypography.subheadline(context).copyWith( - color: Colors.white, - ), - ), - ], - ), - ), - ), - ); - } - - // MARK: - Description Panel - - Widget _buildDescriptionPanel() { - return Container( - color: AppColors.backgroundPrimary(context), - padding: const EdgeInsets.symmetric( - horizontal: AppSpacing.large, - vertical: AppSpacing.regular, - ), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - // Header row: "Description" + LIVE indicator + Copy button - Row( - children: [ - Text( - 'Description', - style: AppTypography.headline(context).copyWith( - fontWeight: FontWeight.w600, - ), - ), - const SizedBox(width: AppSpacing.small), - if (_viewModel.isAutoStreamingEnabled) _buildLiveIndicator(), - const Spacer(), - if (_viewModel.currentDescription.isNotEmpty) _buildCopyButton(), - ], - ), - const SizedBox(height: AppSpacing.mediumLarge), + // MARK: - Image Preview - // Scrollable description text - Expanded( - child: SingleChildScrollView( - child: Text( - _viewModel.currentDescription.isEmpty - ? 'Tap the button to describe what your camera sees' - : _viewModel.currentDescription, - style: AppTypography.body(context).copyWith( - color: _viewModel.currentDescription.isEmpty - ? AppColors.textSecondary(context) - : AppColors.textPrimary(context), + Widget _buildImagePreview() { + final path = _viewModel.selectedImagePath; + return AspectRatio( + aspectRatio: 4 / 3, + child: Container( + decoration: BoxDecoration( + color: AppColors.backgroundGray6(context), + borderRadius: BorderRadius.circular(AppSpacing.cornerRadiusModal), + border: Border.all(color: AppColors.separator(context)), + ), + clipBehavior: Clip.antiAlias, + child: path != null + ? Image.file(File(path), fit: BoxFit.cover, width: double.infinity) + : Center( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + Icons.image_outlined, + size: 56, + color: AppColors.textSecondary(context), + ), + const SizedBox(height: AppSpacing.smallMedium), + Text( + 'Pick or capture an image', + style: AppTypography.subheadline(context) + .copyWith(color: AppColors.textSecondary(context)), + ), + ], ), ), - ), - ), - - // Error text - if (_viewModel.error != null) ...[ - const SizedBox(height: AppSpacing.mediumLarge), - Text( - _viewModel.error!, - style: AppTypography.caption(context).copyWith( - color: Colors.red, - ), - ), - ], - ], ), ); } - Widget _buildLiveIndicator() { + // MARK: - Source Buttons (gallery + device camera) + + Widget _buildSourceButtons() { + final disabled = _viewModel.isProcessing; return Row( - mainAxisSize: MainAxisSize.min, children: [ - Container( - width: 8, - height: 8, - decoration: const BoxDecoration( - color: Colors.green, - shape: BoxShape.circle, + Expanded( + child: OutlinedButton.icon( + onPressed: disabled ? null : () => _pickImage(ImageSource.gallery), + icon: const Icon(Icons.photo_library_outlined), + label: const Text('Gallery'), + style: OutlinedButton.styleFrom( + padding: + const EdgeInsets.symmetric(vertical: AppSpacing.mediumLarge), + ), ), ), - const SizedBox(width: 4), - Text( - 'LIVE', - style: AppTypography.caption2Bold(context).copyWith( - color: Colors.green, + const SizedBox(width: AppSpacing.mediumLarge), + Expanded( + child: OutlinedButton.icon( + onPressed: disabled ? null : () => _pickImage(ImageSource.camera), + icon: const Icon(Icons.photo_camera_outlined), + label: const Text('Camera'), + style: OutlinedButton.styleFrom( + padding: + const EdgeInsets.symmetric(vertical: AppSpacing.mediumLarge), + ), ), ), ], ); } - Widget _buildCopyButton() { - return IconButton( - icon: const Icon(Icons.copy, size: 18), - color: AppColors.textSecondary(context), - onPressed: () { - unawaited(Clipboard.setData( - ClipboardData(text: _viewModel.currentDescription))); + Future _pickImage(ImageSource source) async { + try { + final xFile = await _picker.pickImage(source: source); + if (xFile != null) { + _viewModel.setSelectedImage(xFile.path); + } + } catch (e) { + if (mounted) { unawaited( ScaffoldMessenger.of(context) - .showSnackBar( - const SnackBar( - content: Text('Description copied to clipboard'), - duration: Duration(seconds: 2), - ), - ) + .showSnackBar(SnackBar(content: Text('Failed to pick image: $e'))) .closed .then((_) => null), ); - }, - ); + } + } } - // MARK: - Control Bar - - Widget _buildControlBar() { - return Container( - color: Colors.black, - padding: const EdgeInsets.symmetric(vertical: AppSpacing.large), - child: Row( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - _buildPhotosButton(), - const SizedBox(width: 32), - _buildMainActionButton(), - const SizedBox(width: 32), - _buildLiveToggleButton(), - const SizedBox(width: 32), - _buildModelButton(), - ], + // MARK: - Prompt + + Widget _buildPromptField() { + return TextField( + controller: _promptController, + onChanged: (value) => _viewModel.prompt = value, + minLines: 1, + maxLines: 3, + decoration: InputDecoration( + hintText: 'Describe this image…', + border: OutlineInputBorder( + borderRadius: BorderRadius.circular(AppSpacing.cornerRadiusBubble), + ), + contentPadding: const EdgeInsets.symmetric( + horizontal: AppSpacing.large, + vertical: AppSpacing.mediumLarge, + ), ), ); } - Widget _buildPhotosButton() { - return GestureDetector( - onTap: _onPhotosButtonTap, - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - const Icon(Icons.photo, size: 28, color: Colors.white), - const SizedBox(height: 4), - Text( - 'Photos', - style: AppTypography.caption2(context).copyWith( - color: Colors.white, - ), - ), - ], + // MARK: - Describe / Stop + + Widget _buildDescribeButton() { + final isProcessing = _viewModel.isProcessing; + final canDescribe = _viewModel.selectedImagePath != null && !isProcessing; + return FilledButton.icon( + onPressed: isProcessing + ? () => unawaited(_viewModel.cancelGeneration()) + : (canDescribe + ? () => unawaited(_viewModel.describeSelectedImage()) + : null), + icon: Icon(isProcessing ? Icons.stop : Icons.auto_awesome), + label: Text(isProcessing ? 'Stop' : 'Describe'), + style: FilledButton.styleFrom( + padding: const EdgeInsets.symmetric(vertical: AppSpacing.mediumLarge), ), ); } - Future _onPhotosButtonTap() async { - final picker = ImagePicker(); - final xFile = await picker.pickImage(source: ImageSource.gallery); - if (xFile != null) { - await _viewModel.describePickedImage(xFile.path); - } - } - - Widget _buildMainActionButton() { - final isProcessing = _viewModel.isProcessing; - final isAutoStreaming = _viewModel.isAutoStreamingEnabled; - final isDisabled = isProcessing && !isAutoStreaming; - - Color buttonColor; - if (isAutoStreaming) { - buttonColor = Colors.red; - } else if (isProcessing) { - buttonColor = Colors.grey; - } else { - buttonColor = Colors.orange; - } + // MARK: - Result - return GestureDetector( - onTap: isDisabled ? null : _onMainActionButtonTap, - child: Container( - width: 64, - height: 64, + Widget _buildResult() { + if (_viewModel.error != null) { + return Container( + width: double.infinity, + padding: const EdgeInsets.all(AppSpacing.large), decoration: BoxDecoration( - color: buttonColor, - shape: BoxShape.circle, + color: AppColors.primaryRed.withValues(alpha: 0.1), + borderRadius: BorderRadius.circular(AppSpacing.cornerRadiusModal), + border: Border.all(color: AppColors.primaryRed.withValues(alpha: 0.3)), ), - child: Center( - child: isProcessing - ? const SizedBox( - width: 24, - height: 24, - child: CircularProgressIndicator( - strokeWidth: 2, - color: Colors.white, - ), - ) - : Icon( - isAutoStreaming ? Icons.stop : Icons.auto_awesome, - color: Colors.white, - size: isAutoStreaming ? 28 : 32, - ), + child: Text( + _viewModel.error!, + style: AppTypography.caption(context) + .copyWith(color: AppColors.primaryRed), ), - ), - ); - } + ); + } - Future _onMainActionButtonTap() async { - if (_viewModel.isAutoStreamingEnabled) { - _viewModel.stopAutoStreaming(); - } else { - await _viewModel.describeCurrentFrame(); + if (_viewModel.isProcessing && _viewModel.currentDescription.isEmpty) { + return Row( + children: [ + const SizedBox( + width: 16, + height: 16, + child: CircularProgressIndicator(strokeWidth: 2), + ), + const SizedBox(width: AppSpacing.smallMedium), + Text( + 'Analyzing image…', + style: AppTypography.subheadline(context), + ), + ], + ); } - } - Widget _buildLiveToggleButton() { - final isActive = _viewModel.isAutoStreamingEnabled; - final color = isActive ? Colors.green : Colors.white; + if (_viewModel.currentDescription.isEmpty) { + return const SizedBox.shrink(); + } - return GestureDetector( - onTap: () { - _viewModel.toggleAutoStreaming(); - }, + return Container( + width: double.infinity, + padding: const EdgeInsets.all(AppSpacing.large), + decoration: BoxDecoration( + color: AppColors.backgroundGray6(context), + borderRadius: BorderRadius.circular(AppSpacing.cornerRadiusModal), + border: Border.all(color: AppColors.separator(context)), + ), child: Column( - mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.start, children: [ - Icon(Icons.circle, size: 28, color: color), - const SizedBox(height: 4), + Row( + children: [ + Text( + 'Description', + style: AppTypography.headline(context) + .copyWith(fontWeight: FontWeight.w600), + ), + const Spacer(), + IconButton( + icon: const Icon(Icons.copy, size: 18), + color: AppColors.textSecondary(context), + visualDensity: VisualDensity.compact, + onPressed: _copyDescription, + ), + ], + ), + const SizedBox(height: AppSpacing.smallMedium), Text( - 'Live', - style: AppTypography.caption2(context).copyWith( - color: color, - ), + _viewModel.currentDescription, + style: AppTypography.body(context), ), ], ), ); } - Widget _buildModelButton() { - return GestureDetector( - onTap: _onModelButtonTap, - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - const Icon(Icons.view_in_ar, size: 28, color: Colors.white), - const SizedBox(height: 4), - Text( - 'Model', - style: AppTypography.caption2(context).copyWith( - color: Colors.white, + void _copyDescription() { + unawaited( + Clipboard.setData(ClipboardData(text: _viewModel.currentDescription))); + unawaited( + ScaffoldMessenger.of(context) + .showSnackBar( + const SnackBar( + content: Text('Description copied to clipboard'), + duration: Duration(seconds: 2), ), - ), - ], - ), + ) + .closed + .then((_) => null), ); } - Future _onModelButtonTap() async { + // MARK: - Model Selection + + Future _onSelectModel() async { await showModalBottomSheet( context: context, isScrollControlled: true, @@ -491,11 +344,6 @@ class _VLMCameraViewState extends State { context: ModelSelectionContext.vlm, onModelSelected: (model) async { await _viewModel.onModelSelected(model.id, model.name, this.context); - // Initialize camera if authorized after model is loaded - if (_viewModel.isCameraAuthorized && - !_viewModel.isCameraInitialized) { - unawaited(_viewModel.initializeCamera()); - } }, ), ); @@ -505,43 +353,42 @@ class _VLMCameraViewState extends State { Widget _buildModelRequiredContent() { return Center( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - const Icon( - Icons.center_focus_strong, - size: 60, - color: Colors.orange, - ), - const SizedBox(height: AppSpacing.xLarge), - Text( - 'Vision AI', - style: AppTypography.titleBold(context).copyWith( - color: Colors.white, + child: Padding( + padding: const EdgeInsets.all(AppSpacing.xxLarge), + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + Icon( + Icons.center_focus_strong, + size: 60, + color: AppColors.textSecondary(context), ), - ), - const SizedBox(height: AppSpacing.mediumLarge), - Text( - 'Select a vision model to describe images', - style: AppTypography.subheadline(context).copyWith( - color: Colors.grey, + const SizedBox(height: AppSpacing.xLarge), + Text( + 'Vision AI', + style: AppTypography.titleBold(context), ), - ), - const SizedBox(height: AppSpacing.xxLarge), - ElevatedButton.icon( - onPressed: _onModelButtonTap, - icon: const Icon(Icons.auto_awesome), - label: const Text('Select Model'), - style: ElevatedButton.styleFrom( - backgroundColor: Colors.orange, - foregroundColor: Colors.white, - padding: const EdgeInsets.symmetric( - horizontal: AppSpacing.xxLarge, - vertical: AppSpacing.mediumLarge, + const SizedBox(height: AppSpacing.smallMedium), + Text( + 'Select a vision model to describe images', + style: AppTypography.subheadline(context) + .copyWith(color: AppColors.textSecondary(context)), + textAlign: TextAlign.center, + ), + const SizedBox(height: AppSpacing.xxLarge), + FilledButton.icon( + onPressed: _onSelectModel, + icon: const Icon(Icons.auto_awesome), + label: const Text('Select Model'), + style: FilledButton.styleFrom( + padding: const EdgeInsets.symmetric( + horizontal: AppSpacing.xxLarge, + vertical: AppSpacing.mediumLarge, + ), ), ), - ), - ], + ], + ), ), ); } diff --git a/examples/flutter/RunAnywhereAI/lib/features/vision/vlm_view_model.dart b/examples/flutter/RunAnywhereAI/lib/features/vision/vlm_view_model.dart index 49fcd9113f..9fc5609767 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/vision/vlm_view_model.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/vision/vlm_view_model.dart @@ -1,20 +1,15 @@ import 'dart:async'; -import 'package:camera/camera.dart'; import 'package:flutter/material.dart'; import 'package:runanywhere/runanywhere.dart' as sdk; -import 'package:runanywhere_ai/core/services/permission_service.dart'; -/// VLMViewModel - State management for VLM camera view +/// VLMViewModel - State management for the VLM screen. /// -/// Mirrors iOS VLMViewModel.swift exactly: -/// - Camera management (authorization, initialization, disposal) -/// - Model status tracking (loaded state, model name) -/// - Single capture mode (camera frame → description) -/// - Gallery photo mode (picked image → detailed description) -/// - Auto-streaming mode (live 2.5s interval captures) -/// - Token-by-token streaming display -/// - Error handling and cancellation +/// Mirrors the Android Kotlin VisionViewModel: the user supplies an image from +/// the device gallery or the device camera app (no in-app camera preview), +/// then runs a streamed description over the loaded VLM model. Image picking +/// happens in the view via `image_picker`; this view model owns the selected +/// image path, the prompt, and the streamed inference state. class VLMViewModel extends ChangeNotifier { // MARK: - State Properties @@ -23,9 +18,8 @@ class VLMViewModel extends ChangeNotifier { bool _isProcessing = false; String _currentDescription = ''; String? _error; - bool _isCameraAuthorized = false; - bool _isCameraInitialized = false; - bool _isAutoStreamingEnabled = false; + String? _selectedImagePath; + String _prompt = 'Describe this image in detail.'; // Getters bool get isModelLoaded => _isModelLoaded; @@ -33,97 +27,34 @@ class VLMViewModel extends ChangeNotifier { bool get isProcessing => _isProcessing; String get currentDescription => _currentDescription; String? get error => _error; - bool get isCameraAuthorized => _isCameraAuthorized; - bool get isCameraInitialized => _isCameraInitialized; - bool get isAutoStreamingEnabled => _isAutoStreamingEnabled; + String? get selectedImagePath => _selectedImagePath; + String get prompt => _prompt; - // MARK: - Camera Management - - CameraController? _cameraController; - CameraController? get cameraController => _cameraController; - - Timer? _autoStreamTimer; - static const autoStreamInterval = Duration(seconds: 2, milliseconds: 500); - - // MARK: - Camera Initialization - - /// Initialize camera with back camera (or first available) - /// Request BGRA format (preferred for iOS, Android may fallback to YUV) - Future initializeCamera() async { - try { - final cameras = await availableCameras(); - if (cameras.isEmpty) { - debugPrint('❌ No cameras available'); - return; - } - - // Select back camera (or first available) - final camera = cameras.firstWhere( - (c) => c.lensDirection == CameraLensDirection.back, - orElse: () => cameras.first, - ); - - // Create controller with BGRA format request (iOS preferred, Android fallback to YUV) - _cameraController = CameraController( - camera, - ResolutionPreset.medium, - imageFormatGroup: ImageFormatGroup.bgra8888, - ); - - await _cameraController!.initialize(); - - _isCameraInitialized = true; - notifyListeners(); - - debugPrint('✅ Camera initialized: ${camera.lensDirection}'); - } catch (e) { - debugPrint('❌ Camera initialization failed: $e'); - _error = 'Failed to initialize camera: $e'; - notifyListeners(); - } - } - - /// Dispose camera controller - void disposeCamera() { - unawaited(_cameraController?.dispose()); - _cameraController = null; - _isCameraInitialized = false; - notifyListeners(); - } - - /// Check and request camera permission - Future checkCameraAuthorization(BuildContext context) async { - _isCameraAuthorized = - await PermissionService.shared.requestCameraPermission(context); - notifyListeners(); + set prompt(String value) { + _prompt = value; } // MARK: - Model Management - /// Check if VLM model is loaded + /// Check if a VLM model is loaded. Future checkModelStatus() async { _isModelLoaded = sdk.RunAnywhere.vlm.isLoaded; - if (_isModelLoaded) { - _loadedModelName = sdk.RunAnywhere.vlm.currentModelId; - } else { - _loadedModelName = null; - } + _loadedModelName = _isModelLoaded ? sdk.RunAnywhere.vlm.currentModelId : null; notifyListeners(); } - /// Handle model selection from sheet - /// Takes the app's ModelInfo and loads the SDK model by ID + /// Handle model selection from the model sheet — loads the SDK model by id. Future onModelSelected( String modelId, String modelName, BuildContext context) async { try { - debugPrint('🎯 Loading VLM model: $modelId'); + debugPrint('Loading VLM model: $modelId'); await sdk.RunAnywhere.vlm.load(modelId); _isModelLoaded = true; _loadedModelName = modelName; notifyListeners(); - debugPrint('✅ VLM model loaded: $modelName'); + debugPrint('VLM model loaded: $modelName'); } catch (e) { - debugPrint('❌ Failed to load VLM model: $e'); + debugPrint('Failed to load VLM model: $e'); _error = 'Failed to load model: $e'; notifyListeners(); if (context.mounted) { @@ -139,77 +70,44 @@ class VLMViewModel extends ChangeNotifier { } } - // MARK: - Image Processing - Single Capture + // MARK: - Image Selection - /// Describe the current camera frame (single capture mode) - /// Matches iOS describeCurrentFrame() - Future describeCurrentFrame() async { - if (_isProcessing || !_isCameraInitialized || _cameraController == null) { - return; - } - - _isProcessing = true; - _error = null; + /// Set the image (gallery pick or camera capture) to describe. Clears any + /// previous description so the preview and result stay in sync. + void setSelectedImage(String path) { + _selectedImagePath = path; _currentDescription = ''; + _error = null; notifyListeners(); + } - try { - // Capture image from camera - final xFile = await _cameraController!.takePicture(); - - // Create VLMImage from file path - final image = sdk.VLMImage(filePath: xFile.path); - - final events = sdk.RunAnywhere.vlm.processImageStream( - image, - prompt: 'Describe what you see briefly.', - options: sdk.VLMGenerationOptions(maxTokens: 200), - ); - - // Listen to stream events and append token payloads. - final buffer = StringBuffer(_currentDescription); - await for (final event in events) { - if (event.token.isEmpty) continue; - buffer.write(event.token); - _currentDescription = buffer.toString(); - notifyListeners(); - } + // MARK: - Image Processing - debugPrint( - '✅ Single capture complete: ${_currentDescription.length} chars'); - debugPrint('VLM streaming completed'); - } catch (e) { - debugPrint('❌ Single capture error: $e'); - _error = e.toString(); - notifyListeners(); - } finally { - _isProcessing = false; - notifyListeners(); + /// Describe the currently selected image with the current prompt, streaming + /// tokens into [currentDescription] as they arrive. + Future describeSelectedImage() async { + final path = _selectedImagePath; + if (path == null || _isProcessing) { + return; } - } - - // MARK: - Image Processing - Gallery Photo + final prompt = + _prompt.trim().isEmpty ? 'Describe this image in detail.' : _prompt.trim(); - /// Describe a picked image from gallery - /// Matches iOS describeImage(_:) - Future describePickedImage(String imagePath) async { _isProcessing = true; _error = null; _currentDescription = ''; notifyListeners(); try { - // Create VLMImage from file path - final image = sdk.VLMImage(filePath: imagePath); + final image = sdk.VLMImage(filePath: path); final events = sdk.RunAnywhere.vlm.processImageStream( image, - prompt: 'Describe this image in detail.', + prompt: prompt, options: sdk.VLMGenerationOptions(maxTokens: 300), ); - // Listen to stream events and append token payloads. - final buffer = StringBuffer(_currentDescription); + final buffer = StringBuffer(); await for (final event in events) { if (event.token.isEmpty) continue; buffer.write(event.token); @@ -217,11 +115,9 @@ class VLMViewModel extends ChangeNotifier { notifyListeners(); } - debugPrint( - '✅ Gallery photo described: ${_currentDescription.length} chars'); - debugPrint('VLM streaming completed'); + debugPrint('VLM describe complete: ${_currentDescription.length} chars'); } catch (e) { - debugPrint('❌ Gallery photo error: $e'); + debugPrint('VLM describe error: $e'); _error = e.toString(); notifyListeners(); } finally { @@ -230,108 +126,11 @@ class VLMViewModel extends ChangeNotifier { } } - // MARK: - Auto-Streaming (Live Mode) - - /// Toggle auto-streaming mode - /// Matches iOS toggleAutoStreaming() - void toggleAutoStreaming() { - _isAutoStreamingEnabled = !_isAutoStreamingEnabled; - notifyListeners(); - - if (_isAutoStreamingEnabled) { - _startAutoStreaming(); - } else { - stopAutoStreaming(); - } - } - - /// Start auto-streaming with periodic timer - void _startAutoStreaming() { - _autoStreamTimer?.cancel(); - _autoStreamTimer = Timer.periodic(autoStreamInterval, (timer) { - if (!_isProcessing) { - unawaited(_describeCurrentFrameForAutoStream()); - } - }); - debugPrint( - '🔴 Auto-streaming started (${autoStreamInterval.inMilliseconds}ms interval)'); - } - - /// Stop auto-streaming - void stopAutoStreaming() { - _autoStreamTimer?.cancel(); - _autoStreamTimer = null; - _isAutoStreamingEnabled = false; - notifyListeners(); - debugPrint('⏹️ Auto-streaming stopped'); - } - - /// Describe current frame for auto-stream (live mode) - /// Matches iOS describeCurrentFrameForAutoStream() - /// - Shorter prompt for quick responses - /// - Don't clear description (smooth transition) - /// - Errors only logged, not shown to user - Future _describeCurrentFrameForAutoStream() async { - if (_isProcessing || !_isCameraInitialized || _cameraController == null) { - return; - } - - _isProcessing = true; - notifyListeners(); - - // Build new description in local var (per iOS pattern) - String newDescription = ''; - - try { - // Capture image from camera - final xFile = await _cameraController!.takePicture(); - - // Create VLMImage from file path - final image = sdk.VLMImage(filePath: xFile.path); - - final events = sdk.RunAnywhere.vlm.processImageStream( - image, - prompt: 'Describe what you see in one sentence.', - options: sdk.VLMGenerationOptions(maxTokens: 100), - ); - - // Listen to stream events and build description from token payloads. - final buffer = StringBuffer(newDescription); - await for (final event in events) { - if (event.token.isEmpty) continue; - buffer.write(event.token); - newDescription = buffer.toString(); - _currentDescription = newDescription; - notifyListeners(); - } - - debugPrint( - '🔴 Auto-stream capture complete: ${newDescription.length} chars'); - debugPrint('VLM streaming completed'); - } catch (e) { - // Only log errors in auto-stream mode (per iOS pattern) - debugPrint('⚠️ Auto-stream error (non-critical): $e'); - // Don't set _error in auto-stream mode - } finally { - _isProcessing = false; - notifyListeners(); - } - } - // MARK: - Cancellation - /// Cancel ongoing VLM generation + /// Cancel an ongoing VLM generation. Future cancelGeneration() async { unawaited(sdk.RunAnywhere.vlm.cancelVLMGeneration()); - debugPrint('🛑 VLM generation cancelled'); - } - - // MARK: - Cleanup - - @override - void dispose() { - _autoStreamTimer?.cancel(); - unawaited(_cameraController?.dispose()); - super.dispose(); + debugPrint('VLM generation cancelled'); } } diff --git a/examples/flutter/RunAnywhereAI/lib/features/voice/voice_agent_view_model.dart b/examples/flutter/RunAnywhereAI/lib/features/voice/voice_agent_view_model.dart index d853219aed..7f6b2d0751 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/voice/voice_agent_view_model.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/voice/voice_agent_view_model.dart @@ -62,17 +62,20 @@ class VoiceAgentViewModel extends ChangeNotifier { UiModelLoadState sttModelState = UiModelLoadState.notLoaded; UiModelLoadState llmModelState = UiModelLoadState.notLoaded; UiModelLoadState ttsModelState = UiModelLoadState.notLoaded; + UiModelLoadState vadModelState = UiModelLoadState.notLoaded; String currentSTTModel = 'Not loaded'; String currentLLMModel = 'Not loaded'; String currentTTSModel = 'Not loaded'; + String currentVADModel = 'Not loaded'; // --- Computed properties (for the view) ------------------------------------------- bool get allModelsLoaded => sttModelState == UiModelLoadState.loaded && llmModelState == UiModelLoadState.loaded && - ttsModelState == UiModelLoadState.loaded; + ttsModelState == UiModelLoadState.loaded && + vadModelState == UiModelLoadState.loaded; bool get isActive => sessionState != UiVoiceSessionState.disconnected && @@ -102,6 +105,7 @@ class VoiceAgentViewModel extends ChangeNotifier { final llmModelId = sdk.RunAnywhere.llm.currentModelId; final sttModelId = sdk.RunAnywhere.stt.currentModelId; final ttsVoiceId = sdk.RunAnywhere.tts.currentVoiceId; + final vadModelId = sdk.RunAnywhere.vad.currentModelId; sttModelState = sttModelId != null ? UiModelLoadState.loaded @@ -112,10 +116,14 @@ class VoiceAgentViewModel extends ChangeNotifier { ttsModelState = ttsVoiceId != null ? UiModelLoadState.loaded : UiModelLoadState.notLoaded; + vadModelState = vadModelId != null + ? UiModelLoadState.loaded + : UiModelLoadState.notLoaded; currentSTTModel = sttModelId ?? 'Not loaded'; currentLLMModel = llmModelId ?? 'Not loaded'; currentTTSModel = ttsVoiceId ?? 'Not loaded'; + currentVADModel = vadModelId ?? 'Not loaded'; _notify(); } catch (e) { debugPrint('Failed to get component states: $e'); @@ -146,7 +154,7 @@ class VoiceAgentViewModel extends ChangeNotifier { try { if (!sdk.RunAnywhere.voice.isReady) { sessionState = UiVoiceSessionState.error; - errorMessage = 'Please load STT, LLM, and TTS models first'; + errorMessage = 'Please load STT, LLM, TTS, and VAD models first'; _notify(); return; } @@ -187,6 +195,13 @@ class VoiceAgentViewModel extends ChangeNotifier { debugPrint('Voice cleanup: $e'); } + // Commit any assistant response that finished generating but hadn't yet + // reached PIPELINE_STATE_SPEAKING (where it is normally committed), so a + // stop in that window doesn't silently drop the turn. Safe from double-add: + // the subscription is already cancelled, and a committed turn leaves + // assistantResponse empty. + _commitAssistantResponse(); + sessionState = UiVoiceSessionState.disconnected; currentTranscript = ''; assistantResponse = ''; @@ -238,6 +253,13 @@ class VoiceAgentViewModel extends ChangeNotifier { case sdk.VoiceEvent_Payload.userSaid: final text = event.userSaid.text; if (text.isNotEmpty) { + // A new user utterance means the previous assistant reply is done. + // Commit it as its own bubble BEFORE appending this user turn — the + // mic-driver / process-turn path doesn't reliably emit + // agent-response/SPEAKING signals, so without this the next reply's + // tokens accumulate into the previous reply's buffer and the bubbles + // merge. Idempotent: a no-op if already committed elsewhere. + _commitAssistantResponse(); conversation.add( ConversationTurn( role: proto.MessageRole.MESSAGE_ROLE_USER, @@ -273,14 +295,29 @@ class VoiceAgentViewModel extends ChangeNotifier { _notify(); break; + case sdk.VoiceEvent_Payload.agentResponseStarted: + // A new assistant reply is starting — flush any prior uncommitted + // response into its own turn first, so consecutive replies never merge + // into one bubble (the next assistant tokens then accumulate fresh). + _commitAssistantResponse(); + sessionState = UiVoiceSessionState.processing; + _notify(); + break; + + case sdk.VoiceEvent_Payload.agentResponseCompleted: + // Canonical end-of-reply: commit the accumulated tokens as their own + // assistant turn. Idempotent with the SPEAKING-state commit below — the + // first to fire commits and clears; the other sees an empty buffer. + _commitAssistantResponse(); + _notify(); + break; + case sdk.VoiceEvent_Payload.interrupted: case sdk.VoiceEvent_Payload.metrics: case sdk.VoiceEvent_Payload.componentStateChanged: case sdk.VoiceEvent_Payload.sessionError: case sdk.VoiceEvent_Payload.sessionStarted: case sdk.VoiceEvent_Payload.sessionStopped: - case sdk.VoiceEvent_Payload.agentResponseStarted: - case sdk.VoiceEvent_Payload.agentResponseCompleted: case sdk.VoiceEvent_Payload.turnLifecycle: case sdk.VoiceEvent_Payload.componentProgress: case sdk.VoiceEvent_Payload.notSet: @@ -288,6 +325,21 @@ class VoiceAgentViewModel extends ChangeNotifier { } } + /// Commit the accumulated assistant tokens as a distinct ASSISTANT turn, then + /// clear the buffer. No-op when empty, so it is safe to call from every + /// turn-boundary signal (response-started/completed, SPEAKING, stop) without + /// producing duplicate or merged bubbles. + void _commitAssistantResponse() { + if (assistantResponse.isEmpty) return; + conversation.add( + ConversationTurn( + role: proto.MessageRole.MESSAGE_ROLE_ASSISTANT, + text: assistantResponse, + ), + ); + assistantResponse = ''; + } + void _handlePipelineState(sdk.PipelineState state) { switch (state) { case sdk.PipelineState.PIPELINE_STATE_IDLE: @@ -302,15 +354,7 @@ class VoiceAgentViewModel extends ChangeNotifier { case sdk.PipelineState.PIPELINE_STATE_SPEAKING: sessionState = UiVoiceSessionState.speaking; // Flush accumulated assistant tokens as a completed turn. - if (assistantResponse.isNotEmpty) { - conversation.add( - ConversationTurn( - role: proto.MessageRole.MESSAGE_ROLE_ASSISTANT, - text: assistantResponse, - ), - ); - assistantResponse = ''; - } + _commitAssistantResponse(); _notify(); break; case sdk.PipelineState.PIPELINE_STATE_STOPPED: diff --git a/examples/flutter/RunAnywhereAI/lib/features/voice/voice_assistant_view.dart b/examples/flutter/RunAnywhereAI/lib/features/voice/voice_assistant_view.dart index 5b4f7970cc..27194cf47a 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/voice/voice_assistant_view.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/voice/voice_assistant_view.dart @@ -34,6 +34,9 @@ class _VoiceAssistantViewState extends State late AnimationController _pulseController; late Animation _pulseAnimation; + // Keeps the conversation pinned to the newest message. + final ScrollController _scrollController = ScrollController(); + @override void initState() { super.initState(); @@ -54,9 +57,23 @@ class _VoiceAssistantViewState extends State _viewModel.removeListener(_syncPulseAnimation); _viewModel.dispose(); _pulseController.dispose(); + _scrollController.dispose(); super.dispose(); } + /// Pin the conversation list to the newest message after the frame paints. + void _scrollToBottom() { + WidgetsBinding.instance.addPostFrameCallback((_) { + if (_scrollController.hasClients) { + _scrollController.animateTo( + _scrollController.position.maxScrollExtent, + duration: const Duration(milliseconds: 200), + curve: Curves.easeOut, + ); + } + }); + } + void _syncPulseAnimation() { if (_viewModel.isActive) { if (!_pulseController.isAnimating) { @@ -136,6 +153,22 @@ class _VoiceAssistantViewState extends State ); } + void _showVADModelSelection() { + unawaited( + showModalBottomSheet( + context: context, + isScrollControlled: true, + backgroundColor: Colors.transparent, + builder: (context) => ModelSelectionSheet( + context: ModelSelectionContext.vad, + onModelSelected: (model) async { + await _viewModel.refreshComponentStates(); + }, + ), + ), + ); + } + @override Widget build(BuildContext context) { return ListenableBuilder( @@ -231,6 +264,17 @@ class _VoiceAssistantViewState extends State color: AppColors.primaryPurple, onTap: _showTTSModelSelection, ), + const SizedBox(height: AppSpacing.large), + + // VAD Model + _buildModelConfigRow( + icon: Icons.hearing, + label: 'Voice Activity (VAD)', + modelName: _viewModel.currentVADModel, + state: _viewModel.vadModelState, + color: AppColors.statusOrange, + onTap: _showVADModelSelection, + ), const SizedBox(height: AppSpacing.smallMedium), const Spacer(), @@ -505,7 +549,10 @@ class _VoiceAssistantViewState extends State ); } + _scrollToBottom(); + return ListView( + controller: _scrollController, padding: const EdgeInsets.all(AppSpacing.large), children: [ // Past conversation turns @@ -534,34 +581,51 @@ class _VoiceAssistantViewState extends State Widget _buildConversationBubble(ConversationTurn turn) { final isUser = turn.role == proto.MessageRole.MESSAGE_ROLE_USER; - final speaker = isUser ? 'You' : 'Assistant'; + const radius = AppSpacing.cornerRadiusBubble; - return Padding( - padding: const EdgeInsets.only(bottom: AppSpacing.large), - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - Text( - speaker, - style: AppTypography.caption(context).copyWith( - color: AppColors.textSecondary(context), - fontWeight: FontWeight.w500, - ), + final bubble = ConstrainedBox( + constraints: BoxConstraints( + maxWidth: MediaQuery.of(context).size.width * 0.76, + ), + child: Container( + padding: const EdgeInsets.symmetric( + horizontal: AppSpacing.large, + vertical: AppSpacing.mediumLarge, + ), + decoration: BoxDecoration( + gradient: isUser + ? LinearGradient( + colors: [ + AppColors.userBubbleGradientStart, + AppColors.userBubbleGradientEnd, + ], + begin: Alignment.topLeft, + end: Alignment.bottomRight, + ) + : null, + color: isUser ? null : AppColors.assistantBubbleBg(context), + borderRadius: BorderRadius.only( + topLeft: const Radius.circular(radius), + topRight: const Radius.circular(radius), + bottomLeft: Radius.circular(isUser ? radius : 4), + bottomRight: Radius.circular(isUser ? 4 : radius), ), - const SizedBox(height: 6), - Container( - padding: const EdgeInsets.all(AppSpacing.mediumLarge), - decoration: BoxDecoration( - color: isUser - ? AppColors.backgroundGray5(context) - : AppColors.primaryBlue.withValues(alpha: 0.08), - borderRadius: BorderRadius.circular( - AppSpacing.cornerRadiusBubble, - ), - ), - child: Text(turn.text, style: AppTypography.body(context)), + ), + child: Text( + turn.text, + style: AppTypography.body(context).copyWith( + color: isUser ? Colors.white : AppColors.textPrimary(context), ), - ], + ), + ), + ); + + return Padding( + padding: const EdgeInsets.only(bottom: AppSpacing.mediumLarge), + child: Row( + mainAxisAlignment: + isUser ? MainAxisAlignment.end : MainAxisAlignment.start, + children: [bubble], ), ); } diff --git a/examples/flutter/RunAnywhereAI/lib/features/voice/voice_component_view_model_base.dart b/examples/flutter/RunAnywhereAI/lib/features/voice/voice_component_view_model_base.dart index 6522d322d0..5457bbddc1 100644 --- a/examples/flutter/RunAnywhereAI/lib/features/voice/voice_component_view_model_base.dart +++ b/examples/flutter/RunAnywhereAI/lib/features/voice/voice_component_view_model_base.dart @@ -124,7 +124,15 @@ abstract class VoiceComponentViewModelBase extends ChangeNotifier { } switch (change.kind) { case sdk.ModelLifecycleChangeKind.loaded: - unawaited(applyCurrentModelSnapshot('loaded')); + // Resolve from the event payload, never re-query the SDK here: + // `modelLifecycle.current()` itself publishes a loaded lifecycle + // event, so calling it from inside this handler feeds back into an + // infinite loaded -> current() -> loaded loop that ANRs the app. + // The change carries the model id; resolve the rest from the + // catalog and apply through `applyLoadedModel` so the subclass + // overrides (STT live-mode, TTS system-voice) still run. + applyLoadedModel(resolveLoadedModel(change.modelId)); + debugPrint('Voice component model loaded: ${change.modelId}'); case sdk.ModelLifecycleChangeKind.unloaded: clearLoadedModel(); debugPrint('Voice component model unloaded'); @@ -141,10 +149,11 @@ abstract class VoiceComponentViewModelBase extends ChangeNotifier { applyCurrentModelSnapshot('already loaded'); /// Resolve the current model for this modality via the SDK snapshot and - /// apply it to published state. Shared by [checkInitialModelState] (cold - /// start) and the lifecycle-loaded listener — the lifecycle change only - /// carries modelId/kind, so the full `ModelInfo` is re-resolved through the - /// SDK rather than passed from the event. + /// apply it to published state. Used ONLY at cold start + /// ([checkInitialModelState]); a one-shot `current()` outside an event + /// handler is safe. The lifecycle-loaded listener must NOT call this: + /// `modelLifecycle.current()` re-publishes a loaded event, which from inside + /// the handler loops forever — it resolves from the event payload instead. @protected Future applyCurrentModelSnapshot(String reason) async { final result = await sdk.RunAnywhere.modelLifecycle.current( @@ -194,6 +203,19 @@ abstract class VoiceComponentViewModelBase extends ChangeNotifier { notify(); } + /// Resolve the full [ModelInfo] for a just-loaded [modelId] from the shared + /// catalog, falling back to an id-only model when the catalog has no entry. + /// Used by the lifecycle-loaded listener, which only carries the id — this + /// avoids re-querying the SDK (which would loop) while still giving the + /// subclass `applyLoadedModel` overrides a populated name/framework. + @protected + ModelInfo resolveLoadedModel(String modelId) { + final matches = ModelListViewModel.shared.availableModels + .where((m) => m.id == modelId); + if (matches.isNotEmpty) return matches.first; + return ModelInfo()..id = modelId; + } + /// Clear published state for an unloaded model. @protected void clearLoadedModel() { diff --git a/examples/flutter/RunAnywhereAI/lib/generated/solutions_yaml.dart b/examples/flutter/RunAnywhereAI/lib/generated/solutions_yaml.dart index e4fdbeca42..f291f46453 100644 --- a/examples/flutter/RunAnywhereAI/lib/generated/solutions_yaml.dart +++ b/examples/flutter/RunAnywhereAI/lib/generated/solutions_yaml.dart @@ -7,7 +7,7 @@ abstract final class SolutionsYaml { static const String voiceAgent = r''' # RunAnywhere Solution — canonical voice agent. # -# Matches the VAD → STT → LLM → TTS DAG built by the hand-rolled +# Matches the VAD STT LLM TTS DAG built by the hand-rolled # voice_agent_pipeline.cpp; SolutionRunner compiles this YAML into an # equivalent GraphScheduler graph via PipelineExecutor. # @@ -45,7 +45,7 @@ voice_agent: static const String rag = r''' # RunAnywhere Solution — retrieval-augmented generation. # -# Mirrors the Query → Embed → Retrieve → ContextBuild → LLM DAG T4.6 +# Mirrors the Query Embed Retrieve ContextBuild LLM DAG T4.6 # landed. SolutionRunner expands this into a PipelineSpec (see # solution_converter.cpp::expand_rag) and compiles it into a # GraphScheduler graph. diff --git a/examples/react-native/RunAnywhereAI/App.tsx b/examples/react-native/RunAnywhereAI/App.tsx index acd92f07f5..4472a9fa5c 100644 --- a/examples/react-native/RunAnywhereAI/App.tsx +++ b/examples/react-native/RunAnywhereAI/App.tsx @@ -18,14 +18,15 @@ import { View, Text, StyleSheet, - ActivityIndicator, TouchableOpacity, - Platform, + StatusBar, } from 'react-native'; -import { NavigationContainer } from '@react-navigation/native'; import Icon from 'react-native-vector-icons/Ionicons'; import { SafeAreaProvider } from 'react-native-safe-area-context'; -import TabNavigator from './src/navigation/TabNavigator'; +import { GestureHandlerRootView } from 'react-native-gesture-handler'; +import RootNavigator from './src/navigation/RootNavigator'; +import IntroScreen from './src/features/intro/IntroScreen'; +import { ThemeProvider, useTheme } from './src/theme/system'; import { Colors } from './src/theme/colors'; import { Typography } from './src/theme/typography'; import { @@ -72,14 +73,6 @@ try { ); } -// Make Genie optional (Android/Snapdragon only) -let Genie: OptionalBackend | null = null; -try { - Genie = require('@runanywhere/genie').Genie as OptionalBackend; -} catch { - logDiagnostic('[App] Genie NPU backend not available'); -} - let ONNX: OptionalBackend | null = null; try { ONNX = require('@runanywhere/onnx').ONNX as OptionalBackend; @@ -93,28 +86,13 @@ import { } from './src/screens/SettingsScreen'; import { logDiagnostic } from './src/utils/diagnostics'; -type InitState = 'loading' | 'ready' | 'error'; +// Default backend config for the development Railway backend (mirrors the +// Android example's gitignored local.properties → BuildConfig.RUNANYWHERE_*). +// Used when no custom configuration has been saved in Settings. +const DEFAULT_BASE_URL = 'YOUR_PRODUCTION_BASE_URL'; +const DEFAULT_API_KEY = 'YOUR_PRODUCTION_API_KEY'; -const InitializationLoadingView: React.FC = () => ( - - - - - - RunAnywhere AI - Initializing SDK... - - - -); +type InitState = 'loading' | 'ready' | 'error'; const InitializationErrorView: React.FC<{ error: string; @@ -149,15 +127,6 @@ async function registerBackends(): Promise { ); } - let genieRegistered = false; - if (Platform.OS === 'android' && Genie && Genie.isAvailable) { - const result = await Genie.register(); - genieRegistered = result !== false; - logDiagnostic( - '[App] Genie backend registered; NPU model catalog is pending generated registry/catalog support' - ); - } - const onnxResult = ONNX ? await ONNX.register() : false; const onnxRegistered = onnxResult !== false; if (!ONNX) { @@ -168,7 +137,7 @@ async function registerBackends(): Promise { ); } - return { llamaRegistered, onnxRegistered, genieRegistered }; + return { llamaRegistered, onnxRegistered }; } const App: React.FC = () => { @@ -192,21 +161,27 @@ const App: React.FC = () => { const customBaseURL = await getStoredBaseURL(); const hasCustomConfig = await hasCustomConfiguration(); + const effectiveApiKey = + hasCustomConfig && customApiKey ? customApiKey : DEFAULT_API_KEY; + const effectiveBaseURL = + hasCustomConfig && customBaseURL ? customBaseURL : DEFAULT_BASE_URL; + if ( - hasCustomConfig && - customApiKey && - customBaseURL && - hasUsableBackendConfig({ apiKey: customApiKey, baseURL: customBaseURL }) + hasUsableBackendConfig({ + apiKey: effectiveApiKey, + baseURL: effectiveBaseURL, + }) ) { - console.log('[App] Found custom API configuration'); + console.log('[App] Found backend configuration'); + // Staging (not Production) so the custom base URL is honored AND + // local logging stays on — Production sets enableLocalLogging:false, + // hiding all SDK/telemetry logs. Development would ignore baseURL. await RunAnywhere.initialize({ - apiKey: customApiKey, - baseURL: customBaseURL, - environment: SDKEnvironment.SDK_ENVIRONMENT_PRODUCTION, + apiKey: effectiveApiKey, + baseURL: effectiveBaseURL, + environment: SDKEnvironment.SDK_ENVIRONMENT_STAGING, }); - console.log( - '[App] SDK initialized with custom configuration (production)' - ); + console.log('[App] SDK initialized with backend configuration (staging)'); } else { await RunAnywhere.initialize({ apiKey: '', @@ -249,65 +224,51 @@ const App: React.FC = () => { return () => clearTimeout(timeoutId); }, [initializeSDK]); + let content: React.ReactNode; if (initState === 'loading') { - return ( - - - + content = ; + } else if (initState === 'error') { + content = ( + ); + } else { + content = ; } - if (initState === 'error') { - return ( + return ( + - + + + {content} + - ); - } + + ); +}; +/** + * Edge-to-edge status bar driven by the theme. System bars are transparent and + * content draws behind them; only barStyle (icon color) is honored on Android + * 15+, so it follows light/dark to keep the clock/battery/nav glyphs visible. + */ +const ThemedStatusBar: React.FC = () => { + const { dark } = useTheme(); return ( - - - - - + ); }; const styles = StyleSheet.create({ - loadingContainer: { + root: { flex: 1, - backgroundColor: Colors.backgroundPrimary, - justifyContent: 'center', - alignItems: 'center', - }, - loadingContent: { - alignItems: 'center', - }, - iconContainer: { - width: IconSize.huge, - height: IconSize.huge, - borderRadius: IconSize.huge / 2, - backgroundColor: Colors.badgeBlue, - justifyContent: 'center', - alignItems: 'center', - marginBottom: Spacing.xLarge, - }, - loadingTitle: { - ...Typography.title, - color: Colors.textPrimary, - marginBottom: Spacing.small, - }, - loadingSubtitle: { - ...Typography.body, - color: Colors.textSecondary, - marginBottom: Spacing.xLarge, - }, - spinner: { - marginTop: Spacing.large, }, errorContainer: { flex: 1, diff --git a/examples/react-native/RunAnywhereAI/android/app/build.gradle b/examples/react-native/RunAnywhereAI/android/app/build.gradle index 078d464e32..17e91030db 100644 --- a/examples/react-native/RunAnywhereAI/android/app/build.gradle +++ b/examples/react-native/RunAnywhereAI/android/app/build.gradle @@ -28,7 +28,6 @@ def packageNameMap = [ "core" : "core", "llamacpp": "llamacpp", "onnx" : "onnx", - "genie" : "genie", ] task syncSdkNativeLibs { @@ -103,7 +102,6 @@ def enableProguardInReleaseBuilds = false * The preferred build flavor of JavaScriptCore (JSC) */ def jscFlavor = 'io.github.react-native-community:jsc-android:2026004.+' -def enableRunAnywhereGenie = (project.findProperty("runanywhere.enableGenie") ?: "false").toString().toBoolean() android { ndkVersion rootProject.ext.ndkVersion @@ -166,15 +164,11 @@ android { "**/librac_backend_onnx.so", // ONNX backend shared by multiple modules "**/librac_commons.so", "**/libomp.so", - "**/libcdsprpc.so" + "**/libcdsprpc.so", + "**/libGenie.so", + "**/libQnn*.so", + "**/librac_backend_genie*.so" ] - if (!enableRunAnywhereGenie) { - excludes += [ - "**/libGenie.so", - "**/libQnn*.so", - "**/librac_backend_genie*.so" - ] - } } // Use AGP's modern native packaging. The SDK Android modules also replace @@ -200,9 +194,6 @@ dependencies { implementation project(':runanywhere_core') implementation project(':runanywhere_llamacpp') implementation project(':runanywhere_onnx') - if (enableRunAnywhereGenie) { - implementation project(':runanywhere_genie') - } def isHermesEnabled = project.hasProperty("hermesEnabled") ? project.hermesEnabled.toBoolean() : true // Expose hermesEnabled for other modules (like react-native-worklets) diff --git a/examples/react-native/RunAnywhereAI/android/app/src/main/assets/fonts/Figtree-Italic.ttf b/examples/react-native/RunAnywhereAI/android/app/src/main/assets/fonts/Figtree-Italic.ttf new file mode 100644 index 0000000000..06d4ee3a1c Binary files /dev/null and b/examples/react-native/RunAnywhereAI/android/app/src/main/assets/fonts/Figtree-Italic.ttf differ diff --git a/examples/react-native/RunAnywhereAI/android/app/src/main/assets/fonts/Figtree.ttf b/examples/react-native/RunAnywhereAI/android/app/src/main/assets/fonts/Figtree.ttf new file mode 100644 index 0000000000..f93a4b6cd8 Binary files /dev/null and b/examples/react-native/RunAnywhereAI/android/app/src/main/assets/fonts/Figtree.ttf differ diff --git a/examples/react-native/RunAnywhereAI/android/app/src/main/assets/fonts/MapleMono.ttf b/examples/react-native/RunAnywhereAI/android/app/src/main/assets/fonts/MapleMono.ttf new file mode 100644 index 0000000000..8373098c0b Binary files /dev/null and b/examples/react-native/RunAnywhereAI/android/app/src/main/assets/fonts/MapleMono.ttf differ diff --git a/examples/react-native/RunAnywhereAI/android/gradle.properties.example b/examples/react-native/RunAnywhereAI/android/gradle.properties.example index 4727b9c102..e73668f5f4 100644 --- a/examples/react-native/RunAnywhereAI/android/gradle.properties.example +++ b/examples/react-native/RunAnywhereAI/android/gradle.properties.example @@ -11,10 +11,6 @@ android.enableJetifier=false # Set to true to use locally built native libraries # Set to false to download from GitHub releases runanywhere.useLocalNatives=true -# Closed-source Genie/QNN Android prebuilts are excluded by default until a -# 16 KB ELF-aligned package is available. Re-enable with this property plus -# RUNANYWHERE_ENABLE_GENIE=1 for React Native autolinking. -runanywhere.enableGenie=false # Gradle JVM Settings # Increase memory for React Native builds diff --git a/examples/react-native/RunAnywhereAI/android/link-assets-manifest.json b/examples/react-native/RunAnywhereAI/android/link-assets-manifest.json new file mode 100644 index 0000000000..5912cba444 --- /dev/null +++ b/examples/react-native/RunAnywhereAI/android/link-assets-manifest.json @@ -0,0 +1,17 @@ +{ + "migIndex": 1, + "data": [ + { + "path": "assets/fonts/Figtree-Italic.ttf", + "sha1": "478151ddbbdac0bcfe6d33c6e5ebd179feee721c" + }, + { + "path": "assets/fonts/Figtree.ttf", + "sha1": "371ddf33582284211bcdd664470591441720988d" + }, + { + "path": "assets/fonts/MapleMono.ttf", + "sha1": "ab9daff90e67a9e7c49b66a67557121f4144fdb8" + } + ] +} \ No newline at end of file diff --git a/examples/react-native/RunAnywhereAI/android/settings.gradle b/examples/react-native/RunAnywhereAI/android/settings.gradle index f83bcfd787..ad05e3ce01 100644 --- a/examples/react-native/RunAnywhereAI/android/settings.gradle +++ b/examples/react-native/RunAnywhereAI/android/settings.gradle @@ -9,15 +9,6 @@ rootProject.name = 'RunAnywhereAI' include ':app' includeBuild('../node_modules/@react-native/gradle-plugin') -def localGradleProperties = new Properties() -def localGradlePropertiesFile = new File(settingsDir, "gradle.properties") -if (localGradlePropertiesFile.exists()) { - localGradlePropertiesFile.withInputStream { localGradleProperties.load(it) } -} -def enableRunAnywhereGenie = (gradle.startParameter.projectProperties["runanywhere.enableGenie"] - ?: localGradleProperties.getProperty("runanywhere.enableGenie") - ?: "false").toBoolean() - // Manually include nitro-modules (excluded from autolinking to avoid Turbo codegen) include ':react-native-nitro-modules' project(':react-native-nitro-modules').projectDir = new File(rootProject.projectDir, '../node_modules/react-native-nitro-modules/android') @@ -31,8 +22,3 @@ project(':runanywhere_llamacpp').projectDir = new File(rootProject.projectDir, ' include ':runanywhere_onnx' project(':runanywhere_onnx').projectDir = new File(rootProject.projectDir, '../node_modules/@runanywhere/onnx/android') - -if (enableRunAnywhereGenie) { - include ':runanywhere_genie' - project(':runanywhere_genie').projectDir = new File(rootProject.projectDir, '../node_modules/@runanywhere/genie/android') -} diff --git a/examples/react-native/RunAnywhereAI/assets/fonts/Figtree-Italic.ttf b/examples/react-native/RunAnywhereAI/assets/fonts/Figtree-Italic.ttf new file mode 100644 index 0000000000..06d4ee3a1c Binary files /dev/null and b/examples/react-native/RunAnywhereAI/assets/fonts/Figtree-Italic.ttf differ diff --git a/examples/react-native/RunAnywhereAI/assets/fonts/Figtree.ttf b/examples/react-native/RunAnywhereAI/assets/fonts/Figtree.ttf new file mode 100644 index 0000000000..f93a4b6cd8 Binary files /dev/null and b/examples/react-native/RunAnywhereAI/assets/fonts/Figtree.ttf differ diff --git a/examples/react-native/RunAnywhereAI/assets/fonts/MapleMono.ttf b/examples/react-native/RunAnywhereAI/assets/fonts/MapleMono.ttf new file mode 100644 index 0000000000..8373098c0b Binary files /dev/null and b/examples/react-native/RunAnywhereAI/assets/fonts/MapleMono.ttf differ diff --git a/examples/react-native/RunAnywhereAI/babel.config.js b/examples/react-native/RunAnywhereAI/babel.config.js index f7b3da3b33..9bea612293 100644 --- a/examples/react-native/RunAnywhereAI/babel.config.js +++ b/examples/react-native/RunAnywhereAI/babel.config.js @@ -1,3 +1,5 @@ module.exports = { presets: ['module:@react-native/babel-preset'], + // Required by react-native-reanimated v4 (worklets). Must be the last plugin. + plugins: ['react-native-worklets/plugin'], }; diff --git a/examples/react-native/RunAnywhereAI/index.js b/examples/react-native/RunAnywhereAI/index.js index 9b73932914..01d39071ef 100644 --- a/examples/react-native/RunAnywhereAI/index.js +++ b/examples/react-native/RunAnywhereAI/index.js @@ -2,6 +2,8 @@ * @format */ +// Must be the first import (react-native-gesture-handler requirement). +import 'react-native-gesture-handler'; import { AppRegistry } from 'react-native'; import App from './App'; import { name as appName } from './app.json'; diff --git a/examples/react-native/RunAnywhereAI/ios/RunAnywhereAI.xcodeproj/project.pbxproj b/examples/react-native/RunAnywhereAI/ios/RunAnywhereAI.xcodeproj/project.pbxproj index 6a9a65b82b..cc2f6bbe72 100644 --- a/examples/react-native/RunAnywhereAI/ios/RunAnywhereAI.xcodeproj/project.pbxproj +++ b/examples/react-native/RunAnywhereAI/ios/RunAnywhereAI.xcodeproj/project.pbxproj @@ -12,8 +12,14 @@ 131BD3E69C56FB3D2407D933 /* libPods-RunAnywhereAI-RunAnywhereAITests.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 22B57BB26B1AA1AAAD263C4E /* libPods-RunAnywhereAI-RunAnywhereAITests.a */; }; 13B07FBC1A68108700A75B9A /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB01A68108700A75B9A /* AppDelegate.swift */; }; 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB51A68108700A75B9A /* Images.xcassets */; }; + 2E85DD22A18E406887A7C196 /* Figtree.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 8BF95992387943D3A4344FFE /* Figtree.ttf */; }; + 31FE98DC2C9C407FB812B3C9 /* Figtree-Italic.ttf in Resources */ = {isa = PBXBuildFile; fileRef = C93356376CC9457C9AC44F86 /* Figtree-Italic.ttf */; }; + 36293524965A46A4B5A96AA5 /* Figtree.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 8BF95992387943D3A4344FFE /* Figtree.ttf */; }; + 5387F1DFAAC3441BA3D3240C /* Figtree-Italic.ttf in Resources */ = {isa = PBXBuildFile; fileRef = C93356376CC9457C9AC44F86 /* Figtree-Italic.ttf */; }; 5B7EAD8CABC3ABC09C3EBCB0 /* libPods-RunAnywhereAI.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 9E75FBBD9CE4CFEFC37E3C16 /* libPods-RunAnywhereAI.a */; }; 81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */; }; + A081B5C032C04D09B5E70F0F /* MapleMono.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 2F2E57DBB1744F5AA3380581 /* MapleMono.ttf */; }; + B0629D4718BB4063BAA49E2D /* MapleMono.ttf in Resources */ = {isa = PBXBuildFile; fileRef = 2F2E57DBB1744F5AA3380581 /* MapleMono.ttf */; }; CCDD001122334455 /* DocumentService.swift in Sources */ = {isa = PBXBuildFile; fileRef = CCDD001122334456 /* DocumentService.swift */; }; CCDD001122334457 /* DocumentService.m in Sources */ = {isa = PBXBuildFile; fileRef = CCDD001122334458 /* DocumentService.m */; }; /* End PBXBuildFile section */ @@ -40,11 +46,14 @@ 13B07FB81A68108700A75B9A /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = PrivacyInfo.xcprivacy; path = RunAnywhereAI/PrivacyInfo.xcprivacy; sourceTree = ""; }; 22B57BB26B1AA1AAAD263C4E /* libPods-RunAnywhereAI-RunAnywhereAITests.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-RunAnywhereAI-RunAnywhereAITests.a"; sourceTree = BUILT_PRODUCTS_DIR; }; 2B933AC90856DEE721474248 /* Pods-RunAnywhereAI-RunAnywhereAITests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunAnywhereAI-RunAnywhereAITests.release.xcconfig"; path = "Target Support Files/Pods-RunAnywhereAI-RunAnywhereAITests/Pods-RunAnywhereAI-RunAnywhereAITests.release.xcconfig"; sourceTree = ""; }; + 2F2E57DBB1744F5AA3380581 /* MapleMono.ttf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = undefined; includeInIndex = 0; lastKnownFileType = unknown; name = MapleMono.ttf; path = ../assets/fonts/MapleMono.ttf; sourceTree = ""; }; 5A502ADF42182DE7D087E596 /* Pods-RunAnywhereAI.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunAnywhereAI.debug.xcconfig"; path = "Target Support Files/Pods-RunAnywhereAI/Pods-RunAnywhereAI.debug.xcconfig"; sourceTree = ""; }; 7B92E6D53088A71A1B54CCAB /* Pods-RunAnywhereAI-RunAnywhereAITests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunAnywhereAI-RunAnywhereAITests.debug.xcconfig"; path = "Target Support Files/Pods-RunAnywhereAI-RunAnywhereAITests/Pods-RunAnywhereAI-RunAnywhereAITests.debug.xcconfig"; sourceTree = ""; }; 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = LaunchScreen.storyboard; path = RunAnywhereAI/LaunchScreen.storyboard; sourceTree = ""; }; + 8BF95992387943D3A4344FFE /* Figtree.ttf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = undefined; includeInIndex = 0; lastKnownFileType = unknown; name = Figtree.ttf; path = ../assets/fonts/Figtree.ttf; sourceTree = ""; }; 9E75FBBD9CE4CFEFC37E3C16 /* libPods-RunAnywhereAI.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-RunAnywhereAI.a"; sourceTree = BUILT_PRODUCTS_DIR; }; AABB001122334459 /* RunAnywhereAI-Bridging-Header.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = "RunAnywhereAI-Bridging-Header.h"; path = "RunAnywhereAI/RunAnywhereAI-Bridging-Header.h"; sourceTree = ""; }; + C93356376CC9457C9AC44F86 /* Figtree-Italic.ttf */ = {isa = PBXFileReference; explicitFileType = undefined; fileEncoding = undefined; includeInIndex = 0; lastKnownFileType = unknown; name = "Figtree-Italic.ttf"; path = "../assets/fonts/Figtree-Italic.ttf"; sourceTree = ""; }; CCDD001122334456 /* DocumentService.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = DocumentService.swift; path = RunAnywhereAI/DocumentService.swift; sourceTree = ""; }; CCDD001122334458 /* DocumentService.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = DocumentService.m; path = RunAnywhereAI/DocumentService.m; sourceTree = ""; }; ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; }; @@ -114,6 +123,17 @@ name = Frameworks; sourceTree = ""; }; + 70589C1467784F73A9BA4885 /* Resources */ = { + isa = PBXGroup; + children = ( + 8BF95992387943D3A4344FFE /* Figtree.ttf */, + C93356376CC9457C9AC44F86 /* Figtree-Italic.ttf */, + 2F2E57DBB1744F5AA3380581 /* MapleMono.ttf */, + ); + name = Resources; + path = ""; + sourceTree = ""; + }; 832341AE1AAA6A7D00B99B32 /* Libraries */ = { isa = PBXGroup; children = ( @@ -130,6 +150,7 @@ 83CBBA001A601CBA00E9B192 /* Products */, 2D16E6871FA4F8E400B85C8A /* Frameworks */, BBD78D7AC51CEA395F1C20DB /* Pods */, + 70589C1467784F73A9BA4885 /* Resources */, ); indentWidth = 2; sourceTree = ""; @@ -242,6 +263,9 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( + 2E85DD22A18E406887A7C196 /* Figtree.ttf in Resources */, + 31FE98DC2C9C407FB812B3C9 /* Figtree-Italic.ttf in Resources */, + A081B5C032C04D09B5E70F0F /* MapleMono.ttf in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -252,6 +276,9 @@ 81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */, 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */, 0006D45B6AD5438EA3B175BC /* PrivacyInfo.xcprivacy in Resources */, + 36293524965A46A4B5A96AA5 /* Figtree.ttf in Resources */, + 5387F1DFAAC3441BA3D3240C /* Figtree-Italic.ttf in Resources */, + B0629D4718BB4063BAA49E2D /* MapleMono.ttf in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/examples/react-native/RunAnywhereAI/ios/RunAnywhereAI/Info.plist b/examples/react-native/RunAnywhereAI/ios/RunAnywhereAI/Info.plist index 337d99ac53..3ccb472eda 100644 --- a/examples/react-native/RunAnywhereAI/ios/RunAnywhereAI/Info.plist +++ b/examples/react-native/RunAnywhereAI/ios/RunAnywhereAI/Info.plist @@ -32,7 +32,7 @@ NSLocationWhenInUseUsageDescription - + NSCameraUsageDescription Vision AI needs camera access to describe what you see NSMicrophoneUsageDescription @@ -46,6 +46,9 @@ UIAppFonts Ionicons.ttf + Figtree.ttf + Figtree-Italic.ttf + MapleMono.ttf UILaunchStoryboardName LaunchScreen diff --git a/examples/react-native/RunAnywhereAI/ios/RunAnywhereAITests/Info.plist b/examples/react-native/RunAnywhereAI/ios/RunAnywhereAITests/Info.plist index ba72822e87..47c3855952 100644 --- a/examples/react-native/RunAnywhereAI/ios/RunAnywhereAITests/Info.plist +++ b/examples/react-native/RunAnywhereAI/ios/RunAnywhereAITests/Info.plist @@ -20,5 +20,11 @@ ???? CFBundleVersion 1 + UIAppFonts + + Figtree.ttf + Figtree-Italic.ttf + MapleMono.ttf + diff --git a/examples/react-native/RunAnywhereAI/ios/link-assets-manifest.json b/examples/react-native/RunAnywhereAI/ios/link-assets-manifest.json new file mode 100644 index 0000000000..5912cba444 --- /dev/null +++ b/examples/react-native/RunAnywhereAI/ios/link-assets-manifest.json @@ -0,0 +1,17 @@ +{ + "migIndex": 1, + "data": [ + { + "path": "assets/fonts/Figtree-Italic.ttf", + "sha1": "478151ddbbdac0bcfe6d33c6e5ebd179feee721c" + }, + { + "path": "assets/fonts/Figtree.ttf", + "sha1": "371ddf33582284211bcdd664470591441720988d" + }, + { + "path": "assets/fonts/MapleMono.ttf", + "sha1": "ab9daff90e67a9e7c49b66a67557121f4144fdb8" + } + ] +} \ No newline at end of file diff --git a/examples/react-native/RunAnywhereAI/metro.config.js b/examples/react-native/RunAnywhereAI/metro.config.js index 2ef10bcf83..63daae90a0 100644 --- a/examples/react-native/RunAnywhereAI/metro.config.js +++ b/examples/react-native/RunAnywhereAI/metro.config.js @@ -14,6 +14,20 @@ const bufbuildWireCjs = path.join( 'dist/cjs/wire/binary-encoding.js' ); +const defaultConfig = getDefaultConfig(__dirname); +// Allow Metro to resolve .mjs/.cjs entry points (default sourceExts omit them). +defaultConfig.resolver.sourceExts.push('mjs', 'cjs'); + +// Don't crawl/watch native build output. The Android `.cxx`/`build` dirs churn +// during gradle builds (CMake TryCompile temp dirs created+deleted), which makes +// Metro's fallback file watcher crash with ENOENT. Excluding them keeps Metro +// stable whether or not watchman is installed. +defaultConfig.resolver.blockList = [ + /.*\/android\/\.cxx\/.*/, + /.*\/android\/build\/.*/, + /.*\/ios\/build\/.*/, +]; + /** * Metro configuration * https://reactnative.dev/docs/metro @@ -67,4 +81,4 @@ const config = { }, }; -module.exports = mergeConfig(getDefaultConfig(__dirname), config); +module.exports = mergeConfig(defaultConfig, config); diff --git a/examples/react-native/RunAnywhereAI/package.json b/examples/react-native/RunAnywhereAI/package.json index 66214d5b6d..dfb1e183cb 100644 --- a/examples/react-native/RunAnywhereAI/package.json +++ b/examples/react-native/RunAnywhereAI/package.json @@ -22,6 +22,7 @@ }, "dependencies": { "@bufbuild/protobuf": "^2.12.0", + "@gorhom/bottom-sheet": "^5.2.14", "@react-native-async-storage/async-storage": "^2.2.0", "@react-native-clipboard/clipboard": "^1.16.3", "@react-native-community/slider": "^5.2.0", @@ -30,20 +31,25 @@ "@react-navigation/native": "^7.2.4", "@react-navigation/native-stack": "^7.12.0", "@runanywhere/core": "workspace:*", - "@runanywhere/genie": "^0.1.1", "@runanywhere/llamacpp": "workspace:*", "@runanywhere/onnx": "workspace:*", "@runanywhere/proto-ts": "workspace:*", - "react": "19.2.3", + "lucide-react-native": "^1.21.0", + "react": "19.2.7", "react-native": "0.85.3", + "react-native-actions-sheet": "^10.1.2", "react-native-fs": "^2.20.0", + "react-native-gesture-handler": "^3.0.2", "react-native-image-picker": "^8.2.1", "react-native-nitro-modules": "^0.33.9", "react-native-permissions": "^5.4.4", + "react-native-reanimated": "^4.4.1", "react-native-safe-area-context": "^5.6.2", "react-native-screens": "~4.18.0", + "react-native-svg": "^15.15.5", "react-native-vector-icons": "^10.3.0", "react-native-vision-camera": "^4.7.3", + "react-native-worklets": "^0.9.2", "zustand": "^5.0.13" }, "devDependencies": { diff --git a/examples/react-native/RunAnywhereAI/react-native.config.js b/examples/react-native/RunAnywhereAI/react-native.config.js index 5e1e89cfe7..ba1eb973ac 100644 --- a/examples/react-native/RunAnywhereAI/react-native.config.js +++ b/examples/react-native/RunAnywhereAI/react-native.config.js @@ -1,16 +1,13 @@ /** * React Native configuration for RunAnywhere */ -const enableRunAnywhereGenie = - process.env.RUNANYWHERE_ENABLE_GENIE === '1' || - process.env.RUNANYWHERE_ENABLE_GENIE === 'true'; - module.exports = { project: { ios: { automaticPodsInstallation: true, }, }, + assets: ['./assets/fonts'], dependencies: { // Nitro modules requires Turbo codegen for iOS (NitroModulesSpec.h) 'react-native-nitro-modules': { @@ -19,13 +16,5 @@ module.exports = { ios: {}, }, }, - // Closed-source Genie/QNN Android prebuilts are not 16 KB ELF-aligned yet. - // Opt in only after compatible prebuilts are available by setting both - // RUNANYWHERE_ENABLE_GENIE=1 and -Prunanywhere.enableGenie=true. - '@runanywhere/genie': { - platforms: { - android: enableRunAnywhereGenie ? {} : null, - }, - }, }, }; diff --git a/examples/react-native/RunAnywhereAI/src/components/chat/ChatInput.tsx b/examples/react-native/RunAnywhereAI/src/components/chat/ChatInput.tsx index 8b99dea214..e10240a62c 100644 --- a/examples/react-native/RunAnywhereAI/src/components/chat/ChatInput.tsx +++ b/examples/react-native/RunAnywhereAI/src/components/chat/ChatInput.tsx @@ -1,166 +1,203 @@ /** - * ChatInput Component + * ChatInput — the chat composer: an auto-growing pill text field with a send / + * stop action button, themed on the design system. * - * Text input with send button for chat messages. - * - * Reference: iOS ChatInterfaceView input area + * Keyboard handling: tracks the IME height per-pixel via reanimated's + * useAnimatedKeyboard() so the bar rides up smoothly with the keyboard, and + * falls back to the bottom safe-area inset (gesture nav bar) when it's closed. + * Send turns into a stop control (white rounded square) while generating. */ - import React, { useState } from 'react'; -import { - View, - TextInput, - TouchableOpacity, - StyleSheet, - KeyboardAvoidingView, - Platform, -} from 'react-native'; -import Icon from 'react-native-vector-icons/Ionicons'; -import { Colors } from '../../theme/colors'; -import { Typography } from '../../theme/typography'; -import { - Spacing, - BorderRadius, - Padding, - ButtonHeight, - Layout, -} from '../../theme/spacing'; +import { View, TextInput, TouchableOpacity, StyleSheet } from 'react-native'; +import Animated, { + useAnimatedKeyboard, + useAnimatedStyle, +} from 'react-native-reanimated'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { Icon, useTheme } from '../../theme/system'; interface ChatInputProps { - /** Current input value */ value: string; - /** Callback when value changes */ onChangeText: (text: string) => void; - /** Callback when send button pressed */ onSend: () => void; - /** Callback when stop button pressed during generation */ onStop?: () => void; - /** Whether input is disabled */ disabled?: boolean; - /** Placeholder text */ placeholder?: string; - /** Whether currently sending/generating */ isLoading?: boolean; + /** When provided, renders a tool-calling toggle button at the start of the bar. */ + toolsEnabled?: boolean; + onToggleTools?: () => void; } +const MIN_HEIGHT = 24; +const MAX_HEIGHT = 120; + export const ChatInput: React.FC = ({ value, onChangeText, onSend, onStop, disabled = false, - placeholder = 'Type a message...', + placeholder = 'Type a message…', isLoading = false, + toolsEnabled = false, + onToggleTools, }) => { - const [inputHeight, setInputHeight] = useState(Layout.inputMinHeight); + const { colors, typography } = useTheme(); + const insets = useSafeAreaInsets(); + const keyboard = useAnimatedKeyboard(); + const [inputHeight, setInputHeight] = useState(MIN_HEIGHT); + const canSend = value.trim().length > 0 && !disabled && !isLoading; const canStop = isLoading && !!onStop; const canPressAction = canSend || canStop; + // Under edge-to-edge the window isn't resized for us, so lift the bar by the + // keyboard height. reanimated reports that height from the very bottom of the + // screen (including the gesture nav region), so subtract the bottom inset to + // sit snug on the keyboard. When closed, rest on the bottom safe-area inset. + const animatedStyle = useAnimatedStyle(() => ({ + paddingBottom: + keyboard.height.value > 0 + ? Math.max(keyboard.height.value - insets.bottom, 8) + : Math.max(insets.bottom, 8), + })); + const handleContentSizeChange = (event: { nativeEvent: { contentSize: { height: number } }; }) => { - const height = event.nativeEvent.contentSize.height; - // Clamp between min and max (4 lines max) - const clampedHeight = Math.min( - Math.max(height, Layout.inputMinHeight), - 120 - ); - setInputHeight(clampedHeight); + const h = event.nativeEvent.contentSize.height; + setInputHeight(Math.min(Math.max(h, MIN_HEIGHT), MAX_HEIGHT)); }; - const handleSend = () => { - if (canStop) { - onStop?.(); - } else if (canSend) { - onSend(); - } + const handleAction = () => { + if (canStop) onStop?.(); + else if (canSend) onSend(); }; return ( - - - + + {onToggleTools && ( + + + + )} + + - + + {canStop ? ( + + ) : ( - - + )} + - + ); }; const styles = StyleSheet.create({ container: { - backgroundColor: Colors.backgroundPrimary, - borderTopWidth: 1, - borderTopColor: Colors.borderLight, - paddingHorizontal: Padding.padding16, - paddingVertical: Padding.padding10, - paddingBottom: - Platform.OS === 'ios' ? Padding.padding20 : Padding.padding10, + borderTopWidth: StyleSheet.hairlineWidth, + paddingHorizontal: 12, + paddingTop: 10, }, - inputContainer: { + row: { flexDirection: 'row', alignItems: 'flex-end', - gap: Spacing.smallMedium, + gap: 8, }, - input: { + tool: { + width: 44, + height: 44, + borderRadius: 22, + justifyContent: 'center', + alignItems: 'center', + }, + field: { flex: 1, - backgroundColor: Colors.backgroundSecondary, - borderRadius: BorderRadius.large, - paddingHorizontal: Padding.padding16, - paddingTop: Padding.padding12, - paddingBottom: Padding.padding12, - ...Typography.body, - color: Colors.textPrimary, - maxHeight: 120, + minHeight: 44, + justifyContent: 'center', + borderRadius: 22, + paddingHorizontal: 16, }, - sendButton: { - width: ButtonHeight.regular, - height: ButtonHeight.regular, - borderRadius: ButtonHeight.regular / 2, + input: { + paddingTop: 8, + paddingBottom: 8, + maxHeight: MAX_HEIGHT, + }, + action: { + width: 44, + height: 44, + borderRadius: 22, justifyContent: 'center', alignItems: 'center', }, - sendButtonActive: { - backgroundColor: Colors.primaryBlue, - }, - sendButtonInactive: { - backgroundColor: Colors.backgroundGray5, + stop: { + width: 13, + height: 13, + borderRadius: 3, }, }); diff --git a/examples/react-native/RunAnywhereAI/src/components/chat/LoRASheet.tsx b/examples/react-native/RunAnywhereAI/src/components/chat/LoRASheet.tsx index e2ff103a1b..d462105e4e 100644 --- a/examples/react-native/RunAnywhereAI/src/components/chat/LoRASheet.tsx +++ b/examples/react-native/RunAnywhereAI/src/components/chat/LoRASheet.tsx @@ -15,7 +15,6 @@ import React, { useCallback, useEffect, useState } from 'react'; import { ActivityIndicator, - ScrollView, StyleSheet, Text, TouchableOpacity, @@ -23,7 +22,7 @@ import { } from 'react-native'; import Slider from '@react-native-community/slider'; import Icon from 'react-native-vector-icons/Ionicons'; -import { SafeAreaView } from 'react-native-safe-area-context'; +import { BottomSheet, BottomSheetScrollView } from '../ui/BottomSheet'; import { RunAnywhere } from '@runanywhere/core'; import { LoRAAdapterConfig, @@ -39,12 +38,15 @@ import { Typography } from '../../theme/typography'; import { Spacing, Padding, BorderRadius } from '../../theme/spacing'; interface LoRASheetProps { + visible: boolean; modelId: string | null; onClose: () => void; /** Lets the parent (ChatScreen badge) track the loaded-adapter count. */ onAdaptersChanged?: (adapters: LoRAAdapterInfo[]) => void; } +const LORA_SNAP_POINTS = ['75%']; + function formatBytes(bytes: number): string { if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)} MB`; if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)} KB`; @@ -57,6 +59,7 @@ function lastPathComponent(path: string): string { } export const LoRASheet: React.FC = ({ + visible, modelId, onClose, onAdaptersChanged, @@ -92,27 +95,35 @@ export const LoRASheet: React.FC = ({ /** Mirrors iOS refreshAvailableAdapters + refreshLoraAdapters. */ const refresh = useCallback(async () => { setError(null); + // LoRA `list()` is a loaded-service operation: it requires an LLM model to + // be loaded. Calling it with no model fails with "LoRA service is not + // loaded" and emits a spurious lora.failed/-230 telemetry event. Guard both + // catalog + list on modelId so we never touch the service before a load. + if (!modelId) { + setAvailableAdapters([]); + updateLoaded([]); + return; + } try { - if (modelId) { - const result = await RunAnywhere.lora.queryCatalog( - LoraAdapterCatalogQuery.fromPartial({ modelId }) - ); - if (!result.success) { - throw new Error(result.errorMessage || 'LoRA catalog query failed'); - } - setAvailableAdapters(result.entries); - } else { - setAvailableAdapters([]); + const result = await RunAnywhere.lora.queryCatalog( + LoraAdapterCatalogQuery.fromPartial({ modelId }) + ); + if (!result.success) { + throw new Error(result.errorMessage || 'LoRA catalog query failed'); } + setAvailableAdapters(result.entries); handleLoraState(await RunAnywhere.lora.list()); } catch (err) { setError(err instanceof Error ? err.message : String(err)); } - }, [modelId, handleLoraState]); + }, [modelId, handleLoraState, updateLoaded]); + // Only refresh once the user actually opens the sheet — the sheet is mounted + // (hidden) on the chat screen at startup, so an unconditional mount-refresh + // would hit the LoRA service before any model is loaded. useEffect(() => { - refresh(); - }, [refresh]); + if (visible) refresh(); + }, [visible, refresh]); /** Mirrors iOS downloadAndLoadAdapter -> loadLoraAdapter. */ const handleDownloadAndApply = useCallback( @@ -190,15 +201,16 @@ export const LoRASheet: React.FC = ({ ); return ( - - + + LoRA Adapters - - Done - - + {error && ( @@ -347,12 +359,18 @@ export const LoRASheet: React.FC = ({ )} - - + + ); }; const styles = StyleSheet.create({ + sheetHeader: { + paddingHorizontal: Padding.padding16, + paddingTop: Spacing.small, + paddingBottom: Spacing.medium, + alignItems: 'center', + }, container: { flex: 1, backgroundColor: Colors.backgroundPrimary, diff --git a/examples/react-native/RunAnywhereAI/src/components/chat/MessageBubble.tsx b/examples/react-native/RunAnywhereAI/src/components/chat/MessageBubble.tsx index f8b1174c1b..7e35d15611 100644 --- a/examples/react-native/RunAnywhereAI/src/components/chat/MessageBubble.tsx +++ b/examples/react-native/RunAnywhereAI/src/components/chat/MessageBubble.tsx @@ -1,282 +1,263 @@ /** - * MessageBubble Component + * MessageBubble — a single chat message, Claude/ChatGPT-style and themed for + * light/dark. * - * Displays a single chat message with role-specific styling. - * - * Reference: iOS MessageBubbleView.swift + * Assistant turns render flat and full-width (no bubble): a subtle framework + * label, the response text on the page background, an expandable thinking + * section, tool-call indicator, a thin streaming cursor and a muted tok/s meta. + * User turns render as a compact right-aligned bubble. Empty content (a turn + * that only ran tools, or one mid-stream) renders no text line. */ - -import React, { useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { - View, - Text, + Animated, + LayoutAnimation, StyleSheet, + Text, TouchableOpacity, - LayoutAnimation, + View, } from 'react-native'; -import Icon from 'react-native-vector-icons/Ionicons'; -import { Colors } from '../../theme/colors'; -import { Typography } from '../../theme/typography'; -import { Spacing, BorderRadius, Padding, Layout } from '../../theme/spacing'; +import { Icon, useTheme } from '../../theme/system'; import type { Message } from '../../types/chat'; import { MessageRole } from '../../types/chat'; import { ToolCallIndicator } from './ToolCallIndicator'; interface MessageBubbleProps { message: Message; - /** Maximum width as fraction of screen */ maxWidthFraction?: number; } -/** - * Format timestamp to relative or time string - */ -const formatTimestamp = (date: Date): string => { - const now = new Date(); - const diff = now.getTime() - date.getTime(); - const minutes = Math.floor(diff / 60000); +const formatTPS = (tps: number): string => + tps >= 100 ? `${Math.round(tps)} tok/s` : `${tps.toFixed(1)} tok/s`; - if (minutes < 1) return 'Just now'; - if (minutes < 60) return `${minutes}m ago`; +/** Three softly-pulsing dots shown while an assistant turn streams its first token. */ +const TypingDots: React.FC<{ color: string }> = ({ color }) => { + const progress = useRef(new Animated.Value(0)).current; - return date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }); -}; + useEffect(() => { + const loop = Animated.loop( + Animated.timing(progress, { + toValue: 3, + duration: 1050, + useNativeDriver: true, + }) + ); + loop.start(); + return () => loop.stop(); + }, [progress]); -/** - * Format tokens per second - */ -const formatTPS = (tps: number): string => { - if (tps >= 100) return `${Math.round(tps)} tok/s`; - return `${tps.toFixed(1)} tok/s`; + return ( + + {[0, 1, 2].map((i) => ( + + ))} + + ); }; export const MessageBubble: React.FC = ({ message, - maxWidthFraction = Layout.messageBubbleMaxWidth, + maxWidthFraction = 0.84, }) => { + const { colors, typography } = useTheme(); const [showThinking, setShowThinking] = useState(false); const isUser = message.role === MessageRole.User; const isAssistant = message.role === MessageRole.Assistant; const hasThinking = !!message.thinkingContent; + const hasContent = !!message.content?.trim(); + const tps = message.analytics?.performance.throughputTokensPerSec ?? 0; const toggleThinking = () => { LayoutAnimation.configureNext(LayoutAnimation.Presets.easeInEaseOut); setShowThinking(!showThinking); }; - return ( - - {/* Message Bubble */} - - {/* Model Info Badge (for assistant messages) */} - {isAssistant && - message.modelInfo && - message.modelInfo.frameworkDisplayName && ( - - - - {message.modelInfo.frameworkDisplayName} - - + // User: compact right-aligned bubble. + if (isUser) { + return ( + + + {hasContent && ( + + {message.content} + )} + + + ); + } - {/* Tool Call Indicator (for messages that used tools) */} - {isAssistant && message.toolCallInfo && ( - - )} - - {/* Thinking Section (expandable) */} - {hasThinking && ( - + {message.modelInfo?.frameworkDisplayName && ( + + + - - Thinking - {message.analytics?.thinkingTime && ( - - {(message.analytics.thinkingTime / 1000).toFixed(1)}s - - )} - - )} + {message.modelInfo.frameworkDisplayName} + + + )} - {showThinking && message.thinkingContent && ( - - {message.thinkingContent} - - )} + {message.toolCallInfo && ( + + )} - {/* Message Content */} - + + + Thinking + + {!!message.analytics?.thinkingTime && ( + + {(message.analytics.thinkingTime / 1000).toFixed(1)}s + + )} + + )} + + {showThinking && !!message.thinkingContent && ( + - {message.content} - - - {/* Streaming Indicator */} - {message.isStreaming && ( - - - - )} - - {/* Footer: Timestamp & Analytics */} - - {formatTimestamp(message.timestamp)} + {message.thinkingContent} - - {/* Analytics (for assistant messages) */} - {isAssistant && - message.analytics && - message.analytics.performance.throughputTokensPerSec > 0 && ( - - - {formatTPS( - message.analytics.performance.throughputTokensPerSec - )} - - - )} - + )} + + {hasContent ? ( + + {message.content} + + ) : ( + message.isStreaming && + )} + + {hasContent && message.isStreaming && ( + + )} + + {tps > 0 && ( + + {formatTPS(tps)} + + )} ); }; const styles = StyleSheet.create({ - container: { - marginVertical: Spacing.xSmall, - paddingHorizontal: Padding.padding16, + row: { + paddingHorizontal: 16, + marginVertical: 4, }, - userContainer: { + alignEnd: { alignItems: 'flex-end', }, - assistantContainer: { - alignItems: 'flex-start', - }, - bubble: { - borderRadius: BorderRadius.xLarge, - paddingHorizontal: Padding.padding14, - paddingVertical: Padding.padding10, - }, userBubble: { - backgroundColor: Colors.primaryBlue, - borderBottomRightRadius: BorderRadius.small, + borderRadius: 18, + borderBottomRightRadius: 6, + paddingHorizontal: 14, + paddingVertical: 9, }, - assistantBubble: { - backgroundColor: Colors.assistantBubbleBg, - borderBottomLeftRadius: BorderRadius.small, + assistant: { + paddingHorizontal: 16, + marginTop: 6, + marginBottom: 10, + gap: 6, }, - modelBadge: { + label: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.xSmall, - marginBottom: Spacing.small, - backgroundColor: Colors.badgeBlue, - alignSelf: 'flex-start', - paddingHorizontal: Spacing.small, - paddingVertical: Spacing.xxSmall, - borderRadius: BorderRadius.small, + gap: 5, }, - modelBadgeText: { - ...Typography.caption2, - color: Colors.primaryBlue, - fontWeight: '600', + body: { + lineHeight: 24, }, thinkingHeader: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.xSmall, - marginBottom: Spacing.small, - paddingVertical: Spacing.xSmall, - }, - thinkingLabel: { - ...Typography.caption, - color: Colors.textSecondary, - fontWeight: '600', - }, - thinkingTime: { - ...Typography.caption, - color: Colors.textTertiary, + gap: 6, }, thinkingContent: { - backgroundColor: Colors.backgroundSecondary, - borderRadius: BorderRadius.regular, - padding: Padding.padding10, - marginBottom: Spacing.smallMedium, - }, - thinkingText: { - ...Typography.footnote, - color: Colors.textSecondary, - fontStyle: 'italic', - }, - messageText: { - ...Typography.body, - }, - userText: { - color: Colors.textWhite, - }, - assistantText: { - color: Colors.textPrimary, - }, - streamingIndicator: { - marginTop: Spacing.xSmall, + borderRadius: 10, + padding: 10, }, cursor: { - width: 8, - height: 16, - backgroundColor: Colors.textSecondary, - opacity: 0.5, + width: 7, + height: 15, + borderRadius: 2, + opacity: 0.7, }, - footer: { + dots: { flexDirection: 'row', alignItems: 'center', - justifyContent: 'space-between', - marginTop: Spacing.small, + gap: 5, + paddingVertical: 4, }, - timestamp: { - ...Typography.caption2, + dot: { + width: 7, + height: 7, + borderRadius: 999, }, - userTimestamp: { - color: 'rgba(255, 255, 255, 0.7)', + meta: { + opacity: 0.7, + marginTop: 2, }, - assistantTimestamp: { - color: Colors.textTertiary, + bold: { + fontWeight: '700', }, - analytics: { - flexDirection: 'row', - alignItems: 'center', - gap: Spacing.small, - }, - analyticsText: { - ...Typography.caption2, - color: Colors.textTertiary, + italic: { + fontStyle: 'italic', }, }); diff --git a/examples/react-native/RunAnywhereAI/src/components/common/ModelRequiredOverlay.tsx b/examples/react-native/RunAnywhereAI/src/components/common/ModelRequiredOverlay.tsx index 4607258b23..d81b801ee2 100644 --- a/examples/react-native/RunAnywhereAI/src/components/common/ModelRequiredOverlay.tsx +++ b/examples/react-native/RunAnywhereAI/src/components/common/ModelRequiredOverlay.tsx @@ -1,95 +1,45 @@ /** - * ModelRequiredOverlay Component + * ModelRequiredOverlay — empty state shown when a modality has no model loaded. * - * Full-screen overlay shown when a model is required but not selected. - * - * Reference: iOS ModelRequiredOverlay + * Themed on the design system: a soft icon medallion, title + description, and a + * filled primary "Select a model" button that opens the model picker. Touches in + * empty regions pass through (box-none) so the chat header beneath stays tappable. */ - import React from 'react'; import { View, Text, TouchableOpacity, StyleSheet } from 'react-native'; -import Icon from 'react-native-vector-icons/Ionicons'; -import { Colors } from '../../theme/colors'; -import { Typography } from '../../theme/typography'; -import { - Spacing, - BorderRadius, - Padding, - IconSize, - ButtonHeight, -} from '../../theme/spacing'; +import { Icon, useTheme, type IconName } from '../../theme/system'; export type RequiredModelKind = 'llm' | 'stt' | 'tts' | 'vad' | 'vlm'; interface ModelRequiredOverlayProps { - /** Modality context for icon and text */ modality: RequiredModelKind; - /** Title text */ title?: string; - /** Description text */ description?: string; - /** Callback when select model button pressed */ onSelectModel: () => void; } -/** - * Get icon name based on modality - */ -const getModalityIcon = (modality: RequiredModelKind): string => { - switch (modality) { - case 'llm': - return 'chatbubble-ellipses-outline'; - case 'stt': - return 'mic-outline'; - case 'tts': - return 'volume-high-outline'; - case 'vad': - return 'pulse-outline'; - case 'vlm': - return 'eye-outline'; - default: - return 'cube-outline'; - } +const MODALITY_ICON: Record = { + llm: 'chat', + stt: 'transcribe', + tts: 'speak', + vad: 'vad', + vlm: 'vision', }; -/** - * Get default title based on modality - */ -const getDefaultTitle = (modality: RequiredModelKind): string => { - switch (modality) { - case 'llm': - return 'No Language Model Selected'; - case 'stt': - return 'No Speech Model Selected'; - case 'tts': - return 'No Voice Model Selected'; - case 'vad': - return 'No VAD Model Selected'; - case 'vlm': - return 'No Vision Model Selected'; - default: - return 'No Model Selected'; - } +const DEFAULT_TITLE: Record = { + llm: 'No language model selected', + stt: 'No speech model selected', + tts: 'No voice model selected', + vad: 'No VAD model selected', + vlm: 'No vision model selected', }; -/** - * Get default description based on modality - */ -const getDefaultDescription = (modality: RequiredModelKind): string => { - switch (modality) { - case 'llm': - return 'Select a language model to start chatting with AI on your device.'; - case 'stt': - return 'Select a speech recognition model to transcribe audio.'; - case 'tts': - return 'Select a text-to-speech model to generate audio.'; - case 'vad': - return 'Select a voice activity model to detect speech in microphone audio.'; - case 'vlm': - return 'Select a vision model to analyze images.'; - default: - return 'Select a model to get started.'; - } +const DEFAULT_DESCRIPTION: Record = { + llm: 'Select a language model to start chatting with AI on your device.', + stt: 'Select a speech recognition model to transcribe audio.', + tts: 'Select a text-to-speech model to generate audio.', + vad: 'Select a voice activity model to detect speech in microphone audio.', + vlm: 'Select a vision model to analyze images.', }; export const ModelRequiredOverlay: React.FC = ({ @@ -98,44 +48,55 @@ export const ModelRequiredOverlay: React.FC = ({ description, onSelectModel, }) => { - const iconName = getModalityIcon(modality); - const displayTitle = title || getDefaultTitle(modality); - const displayDescription = description || getDefaultDescription(modality); + const { colors, typography } = useTheme(); + const displayTitle = title ?? DEFAULT_TITLE[modality]; + const displayDescription = description ?? DEFAULT_DESCRIPTION[modality]; return ( - // B-RN-8-001 fix: pointerEvents="box-none" lets the absolutely-positioned - // overlay container pass touches through empty regions, so the parent - // ChatScreen header (rendered as a sibling beneath the overlay) still - // receives the History / New / Info icon taps. The inner button captures - // its own touch via pointerEvents="auto". - + - {/* Icon */} - + - {/* Title */} - {displayTitle} + + {displayTitle} + - {/* Description */} - {displayDescription} + + {displayDescription} + - {/* Select Model Button */} - - Select a Model + + + Select a model + @@ -144,52 +105,45 @@ export const ModelRequiredOverlay: React.FC = ({ const styles = StyleSheet.create({ container: { - ...StyleSheet.absoluteFill, - backgroundColor: Colors.backgroundPrimary, + flex: 1, justifyContent: 'center', alignItems: 'center', - padding: Padding.padding40, + padding: 40, }, content: { alignItems: 'center', - maxWidth: 300, + maxWidth: 320, }, - iconContainer: { - width: IconSize.huge, - height: IconSize.huge, - borderRadius: IconSize.huge / 2, - backgroundColor: Colors.backgroundSecondary, + medallion: { + width: 88, + height: 88, + borderRadius: 44, justifyContent: 'center', alignItems: 'center', - marginBottom: Spacing.xLarge, + marginBottom: 24, }, title: { - ...Typography.title3, - color: Colors.textPrimary, textAlign: 'center', - marginBottom: Spacing.medium, + fontWeight: '700', + marginBottom: 10, }, description: { - ...Typography.body, - color: Colors.textSecondary, textAlign: 'center', - marginBottom: Spacing.xxLarge, - lineHeight: 24, + lineHeight: 22, + marginBottom: 28, }, button: { flexDirection: 'row', alignItems: 'center', justifyContent: 'center', - gap: Spacing.smallMedium, - backgroundColor: Colors.primaryBlue, - paddingHorizontal: Padding.padding24, - height: ButtonHeight.regular, - borderRadius: BorderRadius.large, + gap: 8, + paddingHorizontal: 24, + height: 52, + borderRadius: 16, minWidth: 200, }, buttonText: { - ...Typography.headline, - color: Colors.textWhite, + fontWeight: '700', }, }); diff --git a/examples/react-native/RunAnywhereAI/src/components/model/ModelSelectionSheet.tsx b/examples/react-native/RunAnywhereAI/src/components/model/ModelSelectionSheet.tsx index b972b2f234..45ac543c3a 100644 --- a/examples/react-native/RunAnywhereAI/src/components/model/ModelSelectionSheet.tsx +++ b/examples/react-native/RunAnywhereAI/src/components/model/ModelSelectionSheet.tsx @@ -1,45 +1,29 @@ /** - * ModelSelectionSheet - Reusable model selection component + * ModelSelectionSheet — model picker presented in the app bottom sheet. * - * Reference: iOS Features/Models/ModelSelectionSheet.swift - * - * Features: - * - Device status section - * - Framework list with expansion - * - Model list with download/select actions - * - Loading overlay for model loading - * - Context-based filtering (LLM, STT, TTS, Voice, VLM, RAG Embedding, RAG LLM) + * Design: a device-storage summary card, then models split into "On device" + * (ready to use) and "Available" (downloadable) sections of rich rows — no + * framework dropdown/accordion. Context-based filtering (LLM, STT, TTS, Voice, + * VLM, RAG Embedding, RAG LLM) selects which models are shown. */ - -import React, { useState, useEffect, useCallback } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { - View, - Text, - Modal, + ActivityIndicator, StyleSheet, + Text, TouchableOpacity, - ScrollView, - ActivityIndicator, + View, } from 'react-native'; -import { SafeAreaView } from 'react-native-safe-area-context'; -import Icon from 'react-native-vector-icons/Ionicons'; -import { Colors } from '../../theme/colors'; -import { Typography, FontWeight } from '../../theme/typography'; -import { Spacing, Padding, BorderRadius } from '../../theme/spacing'; +import { BottomSheet, BottomSheetScrollView } from '../ui/BottomSheet'; +import { Icon, useTheme } from '../../theme/system'; import { DEFAULT_INFERENCE_FRAMEWORK, - getFrameworkColor, getFrameworkDisplayName, - getFrameworkIcon, getModelDownloadSizeBytes, getModelFormatLabel, getModelFrameworks, getPrimaryFramework, - isModelCompatibleWithFramework, } from '../../utils/modelDisplay'; - -// Import SDK types and values -// Import RunAnywhere SDK (Multi-Package Architecture) import { RunAnywhere } from '@runanywhere/core'; import { InferenceFramework, @@ -47,54 +31,51 @@ import { type ModelInfo as SDKModelInfo, } from '@runanywhere/proto-ts/model_types'; -// Canonical SDK methods (Swift parity). const downloadModelStreamHelper = RunAnywhere.downloadModelStream; const listModels = async (): Promise => (await RunAnywhere.listModels()).models?.models ?? []; +type StorageSnapshot = Awaited>; + +// Opens tall (long model lists); drag up to near-full. +const SNAP_POINTS = ['60%', '92%']; + /** - * Context for filtering frameworks and models based on the current experience/modality + * Context for filtering models based on the current experience/modality. */ export enum ModelSelectionContext { - LLM = 'llm', // Chat experience - show LLM frameworks - STT = 'stt', // Speech-to-Text - show STT frameworks - TTS = 'tts', // Text-to-Speech - show TTS frameworks - Voice = 'voice', // Voice Assistant - show all voice-related - VAD = 'vad', // Voice Activity Detection - VLM = 'vlm', // Vision - show VLM frameworks - RagEmbedding = 'ragEmbedding', // RAG embedding - ONNX embedding models only - RagLLM = 'ragLLM', // RAG generation - LlamaCpp language models only + LLM = 'llm', + STT = 'stt', + TTS = 'tts', + Voice = 'voice', + VAD = 'vad', + VLM = 'vlm', + RagEmbedding = 'ragEmbedding', + RagLLM = 'ragLLM', } -/** - * Get title for context - */ const getContextTitle = (context: ModelSelectionContext): string => { switch (context) { case ModelSelectionContext.LLM: - return 'Select LLM Model'; + return 'Select a model'; case ModelSelectionContext.STT: - return 'Select STT Model'; + return 'Select a speech model'; case ModelSelectionContext.TTS: - return 'Select TTS Model'; + return 'Select a voice'; case ModelSelectionContext.Voice: - return 'Select Model'; + return 'Select a model'; case ModelSelectionContext.VAD: - return 'Select VAD Model'; + return 'Select a VAD model'; case ModelSelectionContext.VLM: - return 'Select Vision Model'; + return 'Select a vision model'; case ModelSelectionContext.RagEmbedding: - return 'Select Embedding Model'; + return 'Select an embedding model'; case ModelSelectionContext.RagLLM: - return 'Select LLM Model'; + return 'Select a model'; } }; -/** - * Get category for SDK filtering (uses SDK's `ModelCategory` proto enum). - * Returns the proto-canonical numeric `ModelCategory` value or `null` to mean - * "show all". - */ +/** SDK category filter (proto `ModelCategory`); null = show all. */ const getCategoryForContext = ( context: ModelSelectionContext ): ModelCategory | null => { @@ -106,7 +87,7 @@ const getCategoryForContext = ( case ModelSelectionContext.TTS: return ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS; case ModelSelectionContext.Voice: - return null; // Show all + return null; case ModelSelectionContext.VAD: return ModelCategory.MODEL_CATEGORY_VOICE_ACTIVITY_DETECTION; case ModelSelectionContext.VLM: @@ -118,10 +99,7 @@ const getCategoryForContext = ( } }; -/** - * Get allowed frameworks for context. - * Returns null if all frameworks are acceptable. - */ +/** Framework restriction for a context; null = any framework. */ const getAllowedFrameworksForContext = ( context: ModelSelectionContext ): Set | null => { @@ -135,54 +113,40 @@ const getAllowedFrameworksForContext = ( } }; -/** - * Whether this context is a RAG context (no model pre-loading needed) - */ const isRAGContext = (context: ModelSelectionContext): boolean => context === ModelSelectionContext.RagEmbedding || context === ModelSelectionContext.RagLLM; -/** - * Framework info for display - */ -interface FrameworkDisplayInfo { - framework: InferenceFramework; - displayName: string; - iconName: string; - color: string; - modelCount: number; -} +const isOnDevice = (model: SDKModelInfo): boolean => + Boolean(model.isDownloaded || model.localPath); -/** - * Get framework display info - */ -const getFrameworkInfo = ( - framework: InferenceFramework, - modelCount: number -): FrameworkDisplayInfo => { - return { - framework, - displayName: getFrameworkDisplayName(framework), - iconName: getFrameworkIcon(framework), - color: getFrameworkColor(framework), - modelCount, - }; -}; - -/** - * Format bytes to human-readable string - */ const formatBytes = (bytes: number): string => { - if (bytes === 0) return '0 B'; + if (!bytes || bytes <= 0) return '0 B'; const k = 1024; const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; const i = Math.floor(Math.log(bytes) / Math.log(k)); return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`; }; +const modelSubtitle = (model: SDKModelInfo): string => { + const size = getModelDownloadSizeBytes(model); + const framework = getFrameworkDisplayName( + getPrimaryFramework(model, DEFAULT_INFERENCE_FRAMEWORK) + ); + return [ + size > 0 ? formatBytes(size) : null, + getModelFormatLabel(model.format), + framework, + ] + .filter(Boolean) + .join(' · '); +}; + interface ModelSelectionSheetProps { visible: boolean; context: ModelSelectionContext; + /** Id of the model currently loaded/in use, so its row shows "In use". */ + activeModelId?: string | null; onClose: () => void; onModelSelected: (model: SDKModelInfo) => Promise; } @@ -190,745 +154,529 @@ interface ModelSelectionSheetProps { export const ModelSelectionSheet: React.FC = ({ visible, context, + activeModelId, onClose, onModelSelected, }) => { - // State - const [availableModels, setAvailableModels] = useState([]); - const [expandedFramework, setExpandedFramework] = - useState(null); - const [isLoadingModel, setIsLoadingModel] = useState(false); - const [loadingProgress, _setLoadingProgress] = useState(''); - const [selectedModelId, setSelectedModelId] = useState(null); - // Track multiple downloads: modelId -> progress (0-1) - const [downloadingModels, setDownloadingModels] = useState< - Record - >({}); + const { colors, typography } = useTheme(); + const [models, setModels] = useState([]); + const [storage, setStorage] = useState(null); + const [downloading, setDownloading] = useState>({}); const [isLoading, setIsLoading] = useState(true); + const [loadingId, setLoadingId] = useState(null); + const [activeId, setActiveId] = useState(null); + // Model ids that have at least one compatible LoRA adapter in the catalog, + // so their rows can show a "LoRA" tag. listCatalog is a catalog op (no loaded + // model required), so it's safe to call here. + const [loraModelIds, setLoraModelIds] = useState>(new Set()); - /** - * Load available models - */ const loadData = useCallback(async () => { setIsLoading(true); try { - // Single canonical SDK query — category + (optional) framework - // filter applied registry-side. The previous triple-fallback - // (category → framework → all-models) papered over the registry - // catalog when chosen IDs didn't match the strict combination; - // empty results are now surfaced via the empty-state UI so the - // user sees that no model is registered for the selected mode. - const allModels = await listModels(); + const [allModels, storageInfo, loraCatalog] = await Promise.all([ + listModels(), + RunAnywhere.getStorageInfo().catch(() => null), + RunAnywhere.lora.listCatalog().catch(() => null), + ]); + setStorage(storageInfo); + + if (loraCatalog?.success) { + const ids = new Set(); + for (const entry of loraCatalog.entries) { + entry.compatibleModels.forEach((id) => ids.add(id)); + } + setLoraModelIds(ids); + } + const categoryFilter = getCategoryForContext(context); const allowedFrameworks = getAllowedFrameworksForContext(context); - - const filteredModels = categoryFilter - ? allModels.filter((m: SDKModelInfo) => { - const modelCategory = m.category; + const filtered = categoryFilter + ? allModels.filter((m) => { const categoryMatch = - modelCategory === categoryFilter || - String(modelCategory).toLowerCase() === + m.category === categoryFilter || + String(m.category).toLowerCase() === String(categoryFilter).toLowerCase(); if (!categoryMatch) return false; - - // Framework restriction (e.g., ONNX-only for embedding, - // LlamaCpp-only for RAG LLM). if (allowedFrameworks) { - const frameworks = getModelFrameworks(m); - if (!frameworks.some((fw) => allowedFrameworks.has(fw))) { - return false; - } + return getModelFrameworks(m).some((fw) => + allowedFrameworks.has(fw) + ); } return true; }) : allModels; - - setAvailableModels(filteredModels); + setModels(filtered); } catch (error) { - console.error('[ModelSelectionSheet] Error loading data:', error); + console.error('[ModelSelectionSheet] load failed:', error); } finally { setIsLoading(false); } }, [context]); - // Load data when visible or on mount - // This ensures models are loaded even if the sheet renders before becoming visible useEffect(() => { loadData(); }, [loadData]); - // Reload data when visibility changes to ensure fresh data useEffect(() => { - if (visible) { - loadData(); - } + if (visible) loadData(); }, [visible, loadData]); - // B-RN-Sheet-Routing: clear stale lists when context changes so we don't - // briefly render the previous tab's models while loadData is in flight. useEffect(() => { - setAvailableModels([]); - setExpandedFramework(null); - setSelectedModelId(null); + setModels([]); }, [context]); - /** - * Get frameworks with their model counts - */ - const getFrameworks = useCallback((): FrameworkDisplayInfo[] => { - const frameworkCounts = new Map(); - - availableModels.forEach((model: SDKModelInfo, _index: number) => { - const framework = getPrimaryFramework(model, DEFAULT_INFERENCE_FRAMEWORK); - - const count = frameworkCounts.get(framework) || 0; - frameworkCounts.set(framework, count + 1); - }); - - return Array.from(frameworkCounts.entries()) - .map(([framework, count]) => getFrameworkInfo(framework, count)) - .sort((a, b) => b.modelCount - a.modelCount); - }, [availableModels]); - - /** - * Get models for a specific framework - */ - const getModelsForFramework = useCallback( - (framework: InferenceFramework): SDKModelInfo[] => { - return availableModels.filter((model: SDKModelInfo) => { - return isModelCompatibleWithFramework(model, framework); - }); - }, - [availableModels] - ); - - /** - * Toggle framework expansion - */ - const toggleFramework = (framework: InferenceFramework) => { - setExpandedFramework(expandedFramework === framework ? null : framework); - }; - - /** - * Handle model selection - */ - const handleSelectModel = async (model: SDKModelInfo) => { - console.warn('[ModelSelectionSheet] Select tapped:', model.id); - if (!model.isDownloaded && !model.localPath) { - return; - } - - try { - if (isRAGContext(context)) { - // RAG models are referenced by file path at pipeline creation time, - // not pre-loaded into memory. Just pass the selection back and close. - await onModelSelected(model); - onClose(); - } else { - await onModelSelected(model); - } - } catch (error) { - console.error('[ModelSelectionSheet] Error selecting model:', error); - } - }; - + // E2E hooks. useEffect(() => { - const testGlobal = globalThis as typeof globalThis & { + const g = globalThis as typeof globalThis & { __testModels?: SDKModelInfo[]; __testOnModelSelected?: (model: SDKModelInfo) => Promise; }; - testGlobal.__testModels = availableModels; - testGlobal.__testOnModelSelected = onModelSelected; - }, [availableModels, onModelSelected]); - - /** - * Handle model download with real-time progress - * Supports multiple concurrent downloads - */ - const handleDownloadModel = async (model: SDKModelInfo) => { - // B-RN-3-002: log entry so a missing call is visible in metro/logcat - console.warn('[ModelSelectionSheet] Download tapped:', model.id); - // Add this model to downloading set - setDownloadingModels((prev) => ({ ...prev, [model.id]: 0 })); - - try { - // Manual async iteration — Hermes doesn't recognise NitroModules async iterables with for-await - const dlIter = downloadModelStreamHelper(model)[Symbol.asyncIterator](); - let dlResult = await dlIter.next(); - while (!dlResult.done) { - const progress = dlResult.value; - setDownloadingModels((prev) => ({ - ...prev, - [model.id]: progress.stageProgress ?? 0, - })); - console.warn( - `[Download] ${model.id}: ${Math.round((progress.stageProgress ?? 0) * 100)}% (${formatBytes(progress.bytesDownloaded)} / ${formatBytes(progress.totalBytes)})` - ); - dlResult = await dlIter.next(); + g.__testModels = models; + g.__testOnModelSelected = onModelSelected; + }, [models, onModelSelected]); + + const handleSheetClose = useCallback(() => { + onClose(); + }, [onClose]); + + const handleSelect = useCallback( + async (model: SDKModelInfo) => { + if (!isOnDevice(model)) return; + setLoadingId(model.id); + try { + await onModelSelected(model); + setActiveId(model.id); + if (isRAGContext(context)) onClose(); + } catch (error) { + console.error('[ModelSelectionSheet] select failed:', error); + } finally { + setLoadingId(null); } + }, + [context, onClose, onModelSelected] + ); - // Refresh models after download - await loadData(); - } catch (error) { - console.error('[ModelSelectionSheet] Error downloading model:', error); - } finally { - // Remove this model from downloading set - setDownloadingModels((prev) => { - const updated = { ...prev }; - delete updated[model.id]; - return updated; - }); - } - }; + const handleDownload = useCallback( + async (model: SDKModelInfo) => { + setDownloading((prev) => ({ ...prev, [model.id]: 0 })); + try { + const iter = downloadModelStreamHelper(model)[Symbol.asyncIterator](); + let step = await iter.next(); + while (!step.done) { + const p = step.value.stageProgress ?? 0; + setDownloading((prev) => ({ ...prev, [model.id]: p })); + step = await iter.next(); + } + await loadData(); + } catch (error) { + console.error('[ModelSelectionSheet] download failed:', error); + } finally { + setDownloading((prev) => { + const next = { ...prev }; + delete next[model.id]; + return next; + }); + } + }, + [loadData] + ); - /** - * Render framework row - */ - const renderFrameworkRow = (info: FrameworkDisplayInfo) => { - const isExpanded = expandedFramework === info.framework; + const onDeviceModels = useMemo(() => models.filter(isOnDevice), [models]); + const availableModels = useMemo( + () => models.filter((m) => !isOnDevice(m)), + [models] + ); + const renderRow = (model: SDKModelInfo, ready: boolean) => { + const progress = downloading[model.id]; + const isDownloading = progress !== undefined; return ( - - toggleFramework(info.framework)} - > - - - - - - {info.displayName} - - {info.modelCount} {info.modelCount === 1 ? 'model' : 'models'} + + ready ? handleSelect(model) : !isDownloading && handleDownload(model) + } + > + + + + + + + {model.name} + {loraModelIds.has(model.id) && ( + + + + LoRA + + + )} - - - - - {isExpanded && renderExpandedModels(info.framework)} - - ); - }; - - /** - * Render expanded models for a framework - */ - const renderExpandedModels = (framework: InferenceFramework) => { - const models = getModelsForFramework(framework); - - if (models.length === 0) { - return ( - - - No models available for this framework + + {isDownloading + ? `Downloading… ${Math.round((progress ?? 0) * 100)}%` + : modelSubtitle(model)} - - ); - } - - return ( - - {models.map((model) => renderModelRow(model))} - - ); - }; - - /** - * Render model row - */ - const renderModelRow = (model: SDKModelInfo) => { - const isDownloading = model.id in downloadingModels; - const downloadProgress = downloadingModels[model.id] ?? 0; - const isSelected = selectedModelId === model.id; - const canSelect = model.isDownloaded || model.localPath; - const modelInfoContent = ( - <> - - {model.name} - - - - {getModelDownloadSizeBytes(model) > 0 && ( - - + - - {formatBytes(getModelDownloadSizeBytes(model))} - )} - - - - {getModelFormatLabel(model.format)} - - - - {/* Download/Status indicator */} - - {isDownloading ? ( - - - - - Downloading... {Math.round(downloadProgress * 100)}% - - - {/* Progress bar */} - - - - - ) : canSelect ? ( - <> - - - Downloaded + {ready ? ( + loadingId === model.id ? ( + + ) : activeId === model.id || activeModelId === model.id ? ( + + + + In use - + ) : ( - <> - - - Available for download + + + Use - - )} - - - ); - - return ( - - {canSelect ? ( - handleSelectModel(model)} - disabled={isLoadingModel || isSelected} - accessible={false} - > - {modelInfoContent} - + + ) ) : ( - {modelInfoContent} + !isDownloading && ( + + ) )} - - {/* Action button */} - - {isDownloading ? ( - - - - ) : canSelect ? ( - handleSelectModel(model)} - disabled={isLoadingModel || isSelected} - accessible={true} - accessibilityLabel={`Select ${model.name}`} - accessibilityRole="button" - > - Select - - ) : ( - // B-RN-3-002: Removed `disabled={isLoadingModel}` — concurrent - // downloads are supported (each model has its own progress entry - // in `downloadingModels`), so a load-in-progress on a different - // model must not block downloads. Per-model `isDownloading` is - // managed via `downloadingModels[model.id]`. - handleDownloadModel(model)} - disabled={downloadingModels[model.id] !== undefined} - accessible={true} - accessibilityLabel={`Download ${model.name}`} - accessibilityRole="button" - > - Download - - )} - - + ); }; - /** - * Render loading overlay - */ - const renderLoadingOverlay = () => { - if (!isLoadingModel) return null; - - return ( - - - - Loading Model - {loadingProgress} + const renderSection = (title: string, list: SDKModelInfo[], ready: boolean) => + list.length > 0 && ( + + + {title} + + + {list.map((m) => renderRow(m, ready))} ); - }; - - const frameworks = getFrameworks(); return ( - { - // Ensure state is cleaned up when modal is dismissed - setIsLoadingModel(false); - setSelectedModelId(null); - // Don't clear downloads - they continue in background - }} + onClose={handleSheetClose} + snapPoints={SNAP_POINTS} > - - {/* Header */} - - + + + {getContextTitle(context)} + + + + + {isLoading ? ( + + - Cancel + Loading models… - + + ) : ( + <> + + + + + + + Device storage + + - {getContextTitle(context)} + + + {formatBytes(storage?.device?.freeBytes ?? 0)} + + + free + + + + of {formatBytes(storage?.device?.totalBytes ?? 0)} + + - - + + + - {/* Content */} - - {isLoading ? ( - - - Loading models... + + {formatBytes(storage?.totalModelsBytes ?? 0)} in models · {onDeviceModels.length} on device + - ) : ( - <> - {/* Frameworks Section */} - - Available Frameworks - - {frameworks.length > 0 ? ( - frameworks.map(renderFrameworkRow) - ) : ( - - - No frameworks available - - - )} - - - - )} - - {/* Loading Overlay */} - {renderLoadingOverlay()} - - + {renderSection('On device', onDeviceModels, true)} + {renderSection('Available', availableModels, false)} + + {models.length === 0 && ( + + No models available for this mode. + + )} + + )} + + ); }; const styles = StyleSheet.create({ - container: { - flex: 1, - backgroundColor: Colors.backgroundSecondary, - }, header: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - paddingHorizontal: Padding.padding16, - paddingVertical: Padding.padding12, - backgroundColor: Colors.backgroundPrimary, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - cancelButton: { - minWidth: 60, - }, - cancelText: { - ...Typography.body, - color: Colors.primaryBlue, - }, - textDisabled: { - opacity: 0.5, - }, - title: { - ...Typography.headline, - color: Colors.textPrimary, - }, - headerSpacer: { - minWidth: 60, + paddingHorizontal: 20, + paddingTop: 4, + paddingBottom: 12, }, content: { - flex: 1, + paddingHorizontal: 16, + paddingBottom: 24, + gap: 16, }, - // B-RN-3-004: ensure the last model row's CTA button clears the - // bottom tab bar so users on shorter devices don't have to overscroll - // to reveal it. - contentContainer: { - paddingBottom: 80, - }, - loadingContainer: { - flex: 1, + loading: { + paddingVertical: 48, alignItems: 'center', - justifyContent: 'center', - paddingVertical: Padding.padding60, - }, - loadingText: { - ...Typography.subheadline, - color: Colors.textSecondary, - marginTop: Spacing.medium, - }, - section: { - marginBottom: Spacing.large, + gap: 12, }, - sectionTitle: { - ...Typography.footnote, - color: Colors.textSecondary, - textTransform: 'uppercase', - marginHorizontal: Padding.padding16, - marginBottom: Spacing.small, - marginTop: Spacing.large, - }, - card: { - backgroundColor: Colors.backgroundPrimary, - marginHorizontal: Padding.padding16, - borderRadius: BorderRadius.medium, - overflow: 'hidden', + storageCard: { + borderRadius: 22, + padding: 18, + gap: 12, }, - frameworkRow: { + storageTop: { flexDirection: 'row', alignItems: 'center', - paddingVertical: Padding.padding12, - paddingHorizontal: Padding.padding16, - borderBottomWidth: StyleSheet.hairlineWidth, - borderBottomColor: Colors.borderLight, + gap: 10, }, - frameworkIcon: { - width: 36, - height: 36, - borderRadius: BorderRadius.regular, + storageIcon: { + width: 32, + height: 32, + borderRadius: 10, alignItems: 'center', justifyContent: 'center', }, - frameworkInfo: { - flex: 1, - marginLeft: Spacing.mediumLarge, - }, - frameworkName: { - ...Typography.body, - color: Colors.textPrimary, - }, - frameworkCount: { - ...Typography.caption, - color: Colors.textSecondary, - }, - modelsList: { - backgroundColor: Colors.backgroundSecondary, - paddingHorizontal: Padding.padding16, + storageLabel: { + textTransform: 'uppercase', + letterSpacing: 1, }, - modelRow: { + storageStatRow: { flexDirection: 'row', - alignItems: 'center', - backgroundColor: Colors.backgroundPrimary, - marginVertical: Spacing.xSmall, - paddingVertical: Padding.padding12, - paddingHorizontal: Padding.padding12, - borderRadius: BorderRadius.regular, - }, - dimmed: { - opacity: 0.6, + alignItems: 'baseline', + gap: 6, }, - modelInfo: { + flexSpacer: { flex: 1, }, - modelName: { - ...Typography.subheadline, - color: Colors.textPrimary, + usageTrack: { + height: 8, + borderRadius: 999, + overflow: 'hidden', }, - modelNameSelected: { - fontWeight: FontWeight.semibold, + usageFill: { + height: 8, + borderRadius: 999, }, - modelMeta: { - flexDirection: 'row', - alignItems: 'center', - gap: Spacing.small, - marginTop: Spacing.xSmall, + usePill: { + paddingHorizontal: 16, + paddingVertical: 7, + borderRadius: 999, }, - sizeTag: { + inUse: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.xxSmall, + gap: 4, + }, + bold: { + fontWeight: '700', }, - sizeText: { - ...Typography.caption2, - color: Colors.textSecondary, + italic: { + fontStyle: 'italic', }, - badge: { - backgroundColor: Colors.badgeGray, - paddingHorizontal: Spacing.small, - paddingVertical: Spacing.xxSmall, - borderRadius: BorderRadius.small, + section: { + gap: 8, }, - badgeText: { - ...Typography.caption2, - color: Colors.textSecondary, + sectionLabel: { + textTransform: 'uppercase', + paddingHorizontal: 4, }, - modelMetaText: { - ...Typography.caption2, - color: Colors.textSecondary, + card: { + borderRadius: 16, + overflow: 'hidden', }, - statusRow: { + row: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.xSmall, - marginTop: Spacing.xSmall, - }, - statusText: { - ...Typography.caption2, - color: Colors.textSecondary, + gap: 12, + paddingHorizontal: 12, + paddingVertical: 12, + }, + tile: { + width: 40, + height: 40, + borderRadius: 12, + alignItems: 'center', + justifyContent: 'center', }, - downloadProgressContainer: { + rowText: { flex: 1, + gap: 2, }, - downloadProgressRow: { + nameRow: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.xSmall, - }, - progressBarBackground: { - height: 4, - backgroundColor: Colors.backgroundGray5, - borderRadius: 2, - marginTop: 6, - overflow: 'hidden', - }, - progressBarFill: { - height: '100%', - backgroundColor: Colors.primaryBlue, - borderRadius: 2, - }, - actionButtons: { - marginLeft: Spacing.medium, - }, - selectButton: { - backgroundColor: Colors.primaryBlue, - paddingHorizontal: Padding.padding12, - paddingVertical: Padding.padding6, - borderRadius: BorderRadius.regular, - }, - selectButtonText: { - ...Typography.caption, - color: Colors.textWhite, - fontWeight: FontWeight.semibold, - }, - downloadButton: { - backgroundColor: Colors.primaryBlue, - paddingHorizontal: Padding.padding12, - paddingVertical: Padding.padding6, - borderRadius: BorderRadius.regular, - }, - downloadButtonText: { - ...Typography.caption, - color: Colors.textWhite, - fontWeight: FontWeight.semibold, - }, - buttonDisabled: { - opacity: 0.5, - }, - downloadingIndicator: { - padding: Padding.padding8, - }, - emptyModels: { - padding: Padding.padding16, - alignItems: 'center', - }, - emptyText: { - ...Typography.subheadline, - color: Colors.textSecondary, + gap: 6, }, - loadingOverlay: { - ...StyleSheet.absoluteFill, - backgroundColor: Colors.overlayMedium, - alignItems: 'center', - justifyContent: 'center', + nameText: { + flexShrink: 1, }, - loadingCard: { - backgroundColor: Colors.backgroundPrimary, - paddingHorizontal: Padding.padding40, - paddingVertical: Padding.padding30, - borderRadius: BorderRadius.large, + loraTag: { + flexDirection: 'row', alignItems: 'center', - shadowColor: '#000', - shadowOffset: { width: 0, height: 4 }, - shadowOpacity: 0.15, - shadowRadius: 12, - elevation: 8, + gap: 3, + paddingHorizontal: 7, + paddingVertical: 2, + borderRadius: 999, + }, + track: { + height: 3, + borderRadius: 999, + marginTop: 6, + overflow: 'hidden', }, - loadingTitle: { - ...Typography.headline, - color: Colors.textPrimary, - marginTop: Spacing.large, + trackFill: { + height: 3, + borderRadius: 999, }, - loadingMessage: { - ...Typography.subheadline, - color: Colors.textSecondary, - marginTop: Spacing.small, + empty: { textAlign: 'center', + paddingVertical: 32, }, }); diff --git a/examples/react-native/RunAnywhereAI/src/components/ui/BottomSheet.tsx b/examples/react-native/RunAnywhereAI/src/components/ui/BottomSheet.tsx new file mode 100644 index 0000000000..60e352e0f7 --- /dev/null +++ b/examples/react-native/RunAnywhereAI/src/components/ui/BottomSheet.tsx @@ -0,0 +1,84 @@ +/** + * BottomSheet — the app's reusable bottom sheet, built on react-native-actions-sheet + * (real snap points, drag, fling/momentum; works with reanimated 4, unlike + * @gorhom/bottom-sheet). Styled from our design tokens: surface panel with rounded + * top, themed grabber + scrim, snap points from the motion-consistent theme. + * + * Driven by a `visible`/`onClose` prop pair so it drops into the existing screen + * state pattern. Scrollable content should use the re-exported BottomSheetScrollView + * / BottomSheetFlatList (gesture-aware) so inner scrolling coordinates with the sheet. + */ +import React, { useEffect, useMemo, useRef } from 'react'; +import { View } from 'react-native'; +import ActionSheet, { type ActionSheetRef } from 'react-native-actions-sheet'; +import { useTheme } from '../../theme/system'; + +/** Our snap points are strings like '60%' / numbers; the lib wants number[] (percent). */ +function toSnapNumbers( + snapPoints?: Array +): number[] | undefined { + if (!snapPoints || snapPoints.length === 0) return undefined; + return snapPoints.map((p) => (typeof p === 'number' ? p : parseFloat(p))); +} + +export interface BottomSheetProps { + visible: boolean; + onClose: () => void; + /** Snap points (e.g. ['60%', '92%']); opens at the first. */ + snapPoints?: Array; + children: React.ReactNode; +} + +export const BottomSheet: React.FC = ({ + visible, + onClose, + snapPoints, + children, +}) => { + const { colors, dimens } = useTheme(); + const ref = useRef(null); + const points = useMemo(() => toSnapNumbers(snapPoints), [snapPoints]); + + useEffect(() => { + if (visible) { + ref.current?.show(); + } else { + ref.current?.hide(); + } + }, [visible]); + + return ( + + {children} + + ); +}; + +/** Content helpers — gesture-aware, drop-in for the migrated screens. */ +export const BottomSheetView = View; +export { + ScrollView as BottomSheetScrollView, + FlatList as BottomSheetFlatList, +} from 'react-native-actions-sheet'; + +export default BottomSheet; diff --git a/examples/react-native/RunAnywhereAI/src/features/chat/components/ChatHeader.tsx b/examples/react-native/RunAnywhereAI/src/features/chat/components/ChatHeader.tsx new file mode 100644 index 0000000000..d8061fa90e --- /dev/null +++ b/examples/react-native/RunAnywhereAI/src/features/chat/components/ChatHeader.tsx @@ -0,0 +1,178 @@ +/** + * ChatHeader — top bar for the chat screen. The title is a tappable model card + * (icon + name + a status dot: Ready / Generating… / Tap to choose) that opens + * the model picker; right-side actions are LoRA, analytics, history and new chat. + * Inspired by the Android app's ChatTopBar, themed for light/dark and safe-area + * aware so it sits correctly under the iOS notch / Android status bar. + */ +import React from 'react'; +import { StyleSheet, Text, TouchableOpacity, View } from 'react-native'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { Icon, useTheme } from '../../../theme/system'; + +interface ChatHeaderProps { + modelName?: string | null; + ready: boolean; + generating: boolean; + hasMessages: boolean; + onModelPress: () => void; + onAnalytics: () => void; + onHistory: () => void; + onNewChat: () => void; +} + +export const ChatHeader: React.FC = ({ + modelName, + ready, + generating, + hasMessages, + onModelPress, + onAnalytics, + onHistory, + onNewChat, +}) => { + const { colors, typography } = useTheme(); + const insets = useSafeAreaInsets(); + + const status = generating + ? 'Generating…' + : ready + ? 'Ready' + : modelName + ? 'Not loaded' + : 'Tap to choose'; + const dotColor = generating + ? colors.primary + : ready + ? colors.success + : colors.onSurfaceVariant; + + return ( + + + + + + + + + {modelName ?? 'Select model'} + + + + + {status} + + + + + + + + + + + + + + + + + + + ); +}; + +const styles = StyleSheet.create({ + bar: { + borderBottomWidth: StyleSheet.hairlineWidth, + }, + row: { + flexDirection: 'row', + alignItems: 'center', + justifyContent: 'space-between', + gap: 8, + paddingHorizontal: 12, + paddingVertical: 8, + }, + modelCard: { + flexShrink: 1, + maxWidth: 220, + flexDirection: 'row', + alignItems: 'center', + gap: 8, + paddingVertical: 5, + paddingLeft: 5, + paddingRight: 12, + borderRadius: 12, + }, + modelIcon: { + width: 32, + height: 32, + borderRadius: 9, + alignItems: 'center', + justifyContent: 'center', + }, + modelText: { + flex: 1, + }, + statusRow: { + flexDirection: 'row', + alignItems: 'center', + gap: 6, + marginTop: 1, + }, + dot: { + width: 7, + height: 7, + borderRadius: 999, + }, + actions: { + flexDirection: 'row', + alignItems: 'center', + }, + iconBtn: { + padding: 7, + }, + bold: { + fontWeight: '700', + }, +}); + +export default ChatHeader; diff --git a/examples/react-native/RunAnywhereAI/src/features/chat/components/PromptSuggestions.tsx b/examples/react-native/RunAnywhereAI/src/features/chat/components/PromptSuggestions.tsx new file mode 100644 index 0000000000..e0df687ffc --- /dev/null +++ b/examples/react-native/RunAnywhereAI/src/features/chat/components/PromptSuggestions.tsx @@ -0,0 +1,107 @@ +/** + * PromptSuggestions — a horizontal row of example-prompt pills shown above the + * chat composer while the conversation is empty. The set shown depends on mode, + * mirroring the Android app: LoRA active → uncensored prompts, tools enabled → + * tool-calling prompts, otherwise casual prompts. Tapping a pill sends it. + */ +import React from 'react'; +import { ScrollView, StyleSheet, Text, TouchableOpacity, View } from 'react-native'; +import { Icon, useTheme, type IconName } from '../../../theme/system'; + +interface Suggestion { + label: string; + prompt: string; + icon?: IconName; +} + +const CASUAL: Suggestion[] = [ + { label: 'Explain LLMs', prompt: 'Explain how large language models work, in simple terms.' }, + { label: 'Write a poem', prompt: 'Write a short poem about the ocean at night.' }, + { label: 'Summarize a story', prompt: 'Summarize Romeo and Juliet in three sentences.' }, + { label: 'Name ideas', prompt: 'Give me five creative names for a coffee shop.' }, +]; + +const TOOL: Suggestion[] = [ + { label: 'Weather in Tokyo', prompt: "What's the weather in Tokyo right now?", icon: 'cloud' }, + { label: 'Current time', prompt: 'What time is it right now?', icon: 'clock' }, + { label: 'Battery level', prompt: "What's my battery level?", icon: 'battery' }, + { label: 'Quick math', prompt: 'What is 15% of 240?', icon: 'calculator' }, +]; + +const UNCENSORED: Suggestion[] = [ + { label: 'Brutally honest', prompt: 'Give me brutally honest feedback on a weak startup idea.', icon: 'bolt' }, + { label: 'Dark joke', prompt: 'Tell me a dark joke.', icon: 'bolt' }, + { label: 'Hot take', prompt: 'Give me a controversial tech opinion and defend it hard.', icon: 'bolt' }, + { label: 'Roast me', prompt: 'Roast my code in one savage paragraph, no holding back.', icon: 'bolt' }, +]; + +interface PromptSuggestionsProps { + toolsEnabled: boolean; + loraActive: boolean; + onSelect: (prompt: string) => void; +} + +export const PromptSuggestions: React.FC = ({ + toolsEnabled, + loraActive, + onSelect, +}) => { + const { colors, typography } = useTheme(); + const items = loraActive ? UNCENSORED : toolsEnabled ? TOOL : CASUAL; + + return ( + + {items.map((s) => ( + onSelect(s.prompt)} + style={[styles.pill, { backgroundColor: colors.surfaceContainerHigh }]} + > + {s.icon && } + + {s.label} + + + ))} + + ); +}; + +const styles = StyleSheet.create({ + // A horizontal ScrollView stretches to fill a flex column unless capped; + // flexGrow:0 keeps it at its content height so it sits snug above the input. + scroll: { + flexGrow: 0, + }, + row: { + flexDirection: 'row', + alignItems: 'center', + gap: 8, + paddingHorizontal: 16, + paddingTop: 2, + paddingBottom: 6, + }, + pill: { + flexDirection: 'row', + alignItems: 'center', + gap: 6, + paddingHorizontal: 14, + paddingVertical: 8, + borderRadius: 999, + }, + label: { + fontWeight: '600', + }, +}); + +export default PromptSuggestions; diff --git a/examples/react-native/RunAnywhereAI/src/features/intro/IntroScreen.tsx b/examples/react-native/RunAnywhereAI/src/features/intro/IntroScreen.tsx new file mode 100644 index 0000000000..e599c2f0e1 --- /dev/null +++ b/examples/react-native/RunAnywhereAI/src/features/intro/IntroScreen.tsx @@ -0,0 +1,94 @@ +/** + * IntroScreen — shown while the SDK initializes. Minimal by design: the app name + * and an indeterminate horizontal progress bar. Fully themed (light/dark) and + * animated via the motion system. + */ +import React, { useEffect, useRef, useState } from 'react'; +import { Animated, StyleSheet, Text, View } from 'react-native'; +import { useTheme } from '../../theme/system'; + +const TRACK_HEIGHT = 4; +const BAR_FRACTION = 0.4; + +export const IntroScreen: React.FC = () => { + const { colors, typography, dimens, motion } = useTheme(); + const [trackWidth, setTrackWidth] = useState(0); + const progress = useRef(new Animated.Value(0)).current; + + useEffect(() => { + const loop = Animated.loop( + Animated.timing(progress, { + toValue: 1, + duration: motion.duration.extraLong, + easing: motion.easing.easeInOut, + useNativeDriver: true, + }) + ); + loop.start(); + return () => loop.stop(); + }, [progress, motion.duration.extraLong, motion.easing.easeInOut]); + + const barWidth = trackWidth * BAR_FRACTION; + const translateX = progress.interpolate({ + inputRange: [0, 1], + outputRange: [-barWidth, trackWidth], + }); + + return ( + + + RunAnywhere AI + + + setTrackWidth(e.nativeEvent.layout.width)} + style={[ + styles.track, + { + backgroundColor: colors.surfaceVariant, + borderRadius: dimens.radius.full, + }, + ]} + > + {trackWidth > 0 && ( + + )} + + + ); +}; + +const styles = StyleSheet.create({ + container: { + flex: 1, + alignItems: 'center', + justifyContent: 'center', + paddingHorizontal: 32, + }, + track: { + width: '60%', + maxWidth: 280, + height: TRACK_HEIGHT, + overflow: 'hidden', + }, + bar: { + height: TRACK_HEIGHT, + }, +}); + +export default IntroScreen; diff --git a/examples/react-native/RunAnywhereAI/src/navigation/BottomTabs.tsx b/examples/react-native/RunAnywhereAI/src/navigation/BottomTabs.tsx new file mode 100644 index 0000000000..33da341726 --- /dev/null +++ b/examples/react-native/RunAnywhereAI/src/navigation/BottomTabs.tsx @@ -0,0 +1,61 @@ +/** + * Bottom tab bar — the only navigator container in the graph. Three sections: + * Chat, Voice, More (More is the hub to every other screen). Tab changes fade. + */ +import React from 'react'; +import { createBottomTabNavigator } from '@react-navigation/bottom-tabs'; +import { Typography } from '../theme/typography'; +import { Icon, type IconName, useTheme } from '../theme/system'; +import type { TabParamList } from './navigation.types'; +import { ROUTES } from './routes'; +import ChatScreen from '../screens/ChatScreen'; +import VoiceAssistantScreen from '../screens/VoiceAssistantScreen'; +import MoreScreen from '../screens/MoreScreen'; + +const Tab = createBottomTabNavigator(); + +const TAB_ICONS: Record = { + Chat: 'chat', + Voice: 'voice', + More: 'more', +}; + +export const BottomTabs: React.FC = () => { + const { colors } = useTheme(); + return ( + ({ + headerShown: false, + animation: 'fade', + tabBarActiveTintColor: colors.primary, + tabBarInactiveTintColor: colors.onSurfaceVariant, + tabBarStyle: { + backgroundColor: colors.surface, + borderTopColor: colors.outlineVariant, + }, + tabBarLabelStyle: { ...Typography.caption2 }, + tabBarIcon: ({ color, size }) => ( + + ), + })} + > + + + + + ); +}; + +export default BottomTabs; diff --git a/examples/react-native/RunAnywhereAI/src/navigation/RootNavigator.tsx b/examples/react-native/RunAnywhereAI/src/navigation/RootNavigator.tsx new file mode 100644 index 0000000000..04f7e5b30e --- /dev/null +++ b/examples/react-native/RunAnywhereAI/src/navigation/RootNavigator.tsx @@ -0,0 +1,71 @@ +/** + * RootNavigator — the single source of truth for the whole navigation graph. + * One flat native-stack: the tab bar plus every feature screen as a direct + * sibling route (reachable in one navigate() call). Global fade transition. + * + * Header handling is transitional: screens that already render their own header + * (Transcribe/Speak/RAG/VAD/Storage/Vision/Settings) keep the stack header hidden + * — single clean header, and system back (Android button / iOS swipe) pops the + * stack. The three screens with no own header (VLM/Solutions/Benchmarks) use the + * stack header for title + back. A unified header lands in each screen's rebuild. + * Modal routes (ModelSelection/Lora/ConversationList/ChatAnalytics) are declared + * in the param list and get registered when ChatScreen is rebuilt. + */ +import React from 'react'; +import { NavigationContainer } from '@react-navigation/native'; +import { createNativeStackNavigator } from '@react-navigation/native-stack'; +import type { RootStackParamList } from './navigation.types'; +import { ROUTES } from './routes'; +import { screenFade } from './transitions'; +import { BottomTabs } from './BottomTabs'; +import VisionHubScreen from '../screens/VisionHubScreen'; +import VLMScreen from '../screens/VLMScreen'; +import STTScreen from '../screens/STTScreen'; +import TTSScreen from '../screens/TTSScreen'; +import VADScreen from '../screens/VADScreen'; +import RAGScreen from '../screens/RAGScreen'; +import StorageScreen from '../screens/StorageScreen'; +import SolutionsScreen from '../screens/SolutionsScreen'; +import BenchmarkScreen from '../screens/BenchmarkScreen'; +import SettingsScreen from '../screens/SettingsScreen'; + +const Stack = createNativeStackNavigator(); + +export const RootNavigator: React.FC = () => ( + + + + + {/* Own-header screens: stack header hidden, system back pops the stack. */} + + + + + + + + + {/* No own header: stack header provides title + back. */} + + + + + +); + +export default RootNavigator; diff --git a/examples/react-native/RunAnywhereAI/src/navigation/TabNavigator.tsx b/examples/react-native/RunAnywhereAI/src/navigation/TabNavigator.tsx deleted file mode 100644 index c8d062295f..0000000000 --- a/examples/react-native/RunAnywhereAI/src/navigation/TabNavigator.tsx +++ /dev/null @@ -1,210 +0,0 @@ -/** - * TabNavigator - Bottom Tab Navigation - * - * Reference: iOS ContentView.swift with 5 tabs: - * - Chat - * - Vision - * - Voice - * - More - * - Settings - */ - -import React from 'react'; -import { createBottomTabNavigator } from '@react-navigation/bottom-tabs'; -import { createNativeStackNavigator } from '@react-navigation/native-stack'; -import Icon from 'react-native-vector-icons/Ionicons'; -import { Colors } from '../theme/colors'; -import { Typography } from '../theme/typography'; -import type { - MoreStackParamList, - RootTabParamList, - SettingsStackParamList, - VisionStackParamList, -} from '../types'; - -import ChatScreen from '../screens/ChatScreen'; -import STTScreen from '../screens/STTScreen'; -import TTSScreen from '../screens/TTSScreen'; -import VoiceAssistantScreen from '../screens/VoiceAssistantScreen'; -import RAGScreen from '../screens/RAGScreen'; -import SolutionsScreen from '../screens/SolutionsScreen'; -import MoreScreen from '../screens/MoreScreen'; -import StorageScreen from '../screens/StorageScreen'; -import VADScreen from '../screens/VADScreen'; -import VisionHubScreen from '../screens/VisionHubScreen'; -import VLMScreen from '../screens/VLMScreen'; -import SettingsScreen from '../screens/SettingsScreen'; -import BenchmarkScreen from '../screens/BenchmarkScreen'; - -const Tab = createBottomTabNavigator(); -const VisionStack = createNativeStackNavigator(); -const MoreStack = createNativeStackNavigator(); -const SettingsStack = createNativeStackNavigator(); - -const tabIcons: Record< - keyof RootTabParamList, - { focused: string; unfocused: string } -> = { - Chat: { focused: 'chatbubble', unfocused: 'chatbubble-outline' }, - Vision: { focused: 'eye', unfocused: 'eye-outline' }, - Voice: { focused: 'mic', unfocused: 'mic-outline' }, - More: { - focused: 'ellipsis-horizontal', - unfocused: 'ellipsis-horizontal-outline', - }, - Settings: { focused: 'settings', unfocused: 'settings-outline' }, -}; - -const tabLabels: Record = { - Chat: 'Chat', - Vision: 'Vision', - Voice: 'Voice', - More: 'More', - Settings: 'Settings', -}; - -const renderTabBarIcon = ( - routeName: keyof RootTabParamList, - focused: boolean, - color: string, - size: number -) => { - const iconName = focused - ? tabIcons[routeName].focused - : tabIcons[routeName].unfocused; - return ; -}; - -export const TabNavigator: React.FC = () => { - return ( - ({ - tabBarIcon: ({ focused, color, size }) => - renderTabBarIcon(route.name, focused, color, size), - tabBarActiveTintColor: Colors.primaryBlue, - tabBarInactiveTintColor: Colors.textSecondary, - tabBarStyle: { - backgroundColor: Colors.backgroundPrimary, - borderTopColor: Colors.borderLight, - }, - tabBarLabelStyle: { - ...Typography.caption2, - }, - headerShown: false, - })} - > - - - - - - - ); -}; - -const SettingsStackScreen: React.FC = () => { - return ( - - - - - ); -}; - -const VisionStackScreen: React.FC = () => { - return ( - - - - - ); -}; - -const MoreStackScreen: React.FC = () => { - return ( - - - - - - - - - - ); -}; - -export default TabNavigator; diff --git a/examples/react-native/RunAnywhereAI/src/navigation/navigation.types.ts b/examples/react-native/RunAnywhereAI/src/navigation/navigation.types.ts new file mode 100644 index 0000000000..3a2cb32753 --- /dev/null +++ b/examples/react-native/RunAnywhereAI/src/navigation/navigation.types.ts @@ -0,0 +1,58 @@ +/** + * Typed param lists for the mono navigation graph. `RootStackParamList` is the + * flat root stack (tabs container + every feature screen + modals); `TabParamList` + * is the 3-tab bar. Params are `undefined` until a screen's rebuild defines its + * real contract. The global augmentation makes `useNavigation()` typed app-wide + * with no per-call generics. + */ +import type { + NavigatorScreenParams, + CompositeScreenProps, +} from '@react-navigation/native'; +import type { NativeStackScreenProps } from '@react-navigation/native-stack'; +import type { BottomTabScreenProps } from '@react-navigation/bottom-tabs'; + +export type TabParamList = { + Chat: undefined; + Voice: undefined; + More: undefined; +}; + +export type RootStackParamList = { + Tabs: NavigatorScreenParams | undefined; + + // Feature screens (flat, directly navigable). + Vision: undefined; + Vlm: undefined; + Transcribe: undefined; + Speak: undefined; + Vad: undefined; + Rag: undefined; + Storage: undefined; + Solutions: undefined; + Benchmarks: undefined; + Settings: undefined; + + // Modals. + ModelSelection: undefined; + Lora: undefined; + ConversationList: undefined; + ChatAnalytics: undefined; +}; + +/** Props for a screen on the root stack. */ +export type RootStackScreenProps = + NativeStackScreenProps; + +/** Props for a tab screen — composed so tab screens can also reach root routes. */ +export type TabScreenProps = CompositeScreenProps< + BottomTabScreenProps, + NativeStackScreenProps +>; + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace ReactNavigation { + interface RootParamList extends RootStackParamList {} + } +} diff --git a/examples/react-native/RunAnywhereAI/src/navigation/routes.ts b/examples/react-native/RunAnywhereAI/src/navigation/routes.ts new file mode 100644 index 0000000000..6dfae636a0 --- /dev/null +++ b/examples/react-native/RunAnywhereAI/src/navigation/routes.ts @@ -0,0 +1,35 @@ +/** + * Single registry of every navigable route. One flat graph: the 3-tab bar is the + * only container; every other screen is a direct sibling route on the root stack, + * reachable in one navigate() call. Modals share the same stack via a presentation + * group. Route names are referenced through ROUTES — never as string literals. + */ +export const ROUTES = { + // Root container for the bottom tab bar. + Tabs: 'Tabs', + + // Bottom-tab routes. + Chat: 'Chat', + Voice: 'Voice', + More: 'More', + + // Feature screens — flat siblings on the root stack (index 1, directly navigable). + Vision: 'Vision', + Vlm: 'Vlm', + Transcribe: 'Transcribe', + Speak: 'Speak', + Vad: 'Vad', + Rag: 'Rag', + Storage: 'Storage', + Solutions: 'Solutions', + Benchmarks: 'Benchmarks', + Settings: 'Settings', + + // Modal routes (presentation group, same root stack). + ModelSelection: 'ModelSelection', + Lora: 'Lora', + ConversationList: 'ConversationList', + ChatAnalytics: 'ChatAnalytics', +} as const; + +export type RouteName = (typeof ROUTES)[keyof typeof ROUTES]; diff --git a/examples/react-native/RunAnywhereAI/src/navigation/transitions.ts b/examples/react-native/RunAnywhereAI/src/navigation/transitions.ts new file mode 100644 index 0000000000..080fdf9a0f --- /dev/null +++ b/examples/react-native/RunAnywhereAI/src/navigation/transitions.ts @@ -0,0 +1,20 @@ +/** + * Shared navigation transitions. Every screen change fades (duration from the + * motion system) so the whole app has one consistent transition. Modals fade in + * as a modal presentation. Built on `motion` so timings match component animations. + */ +import type { NativeStackNavigationOptions } from '@react-navigation/native-stack'; +import { motion } from '../theme/system/motion'; + +/** Root-stack default: fade push/pop. */ +export const screenFade: NativeStackNavigationOptions = { + animation: 'fade', + animationDuration: motion.duration.medium, +}; + +/** Modal presentation with the same fade. */ +export const modalFade: NativeStackNavigationOptions = { + presentation: 'modal', + animation: 'fade', + animationDuration: motion.duration.medium, +}; diff --git a/examples/react-native/RunAnywhereAI/src/screens/BenchmarkScreen.tsx b/examples/react-native/RunAnywhereAI/src/screens/BenchmarkScreen.tsx index c821ac9fff..ba8fda0c9b 100644 --- a/examples/react-native/RunAnywhereAI/src/screens/BenchmarkScreen.tsx +++ b/examples/react-native/RunAnywhereAI/src/screens/BenchmarkScreen.tsx @@ -1,7 +1,7 @@ /** * BenchmarkScreen - on-device performance benchmarks. * - * RN MVP mirror of iOS `BenchmarkDashboardView` + `BenchmarkRunner`: + * RN mirror of Android BenchmarkDashboardScreen + BenchmarkDetailScreen: * runs (category x model x scenario) work items sequentially over downloaded * models and reports load time plus per-category key metrics. Scenario specs * mirror the iOS providers (LLMBenchmarkProvider / STTBenchmarkProvider / @@ -20,8 +20,8 @@ import { TouchableOpacity, View, } from 'react-native'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; import AsyncStorage from '@react-native-async-storage/async-storage'; -import Icon from 'react-native-vector-icons/Ionicons'; import { RunAnywhere } from '@runanywhere/core'; import { LLMGenerationOptions } from '@runanywhere/proto-ts/llm_options'; import { @@ -32,9 +32,7 @@ import { type ModelInfo as SDKModelInfo, } from '@runanywhere/proto-ts/model_types'; import { STTLanguage } from '@runanywhere/proto-ts/stt_options'; -import { Colors } from '../theme/colors'; -import { Typography } from '../theme/typography'; -import { Spacing, Padding, BorderRadius } from '../theme/spacing'; +import { Icon, useTheme } from '../theme/system'; import { silentAudioWav, sineWaveAudioWav, @@ -79,7 +77,6 @@ interface WorkItem { // Scenario execution — mirrors the iOS benchmark providers. // --------------------------------------------------------------------------- -// Prompts copied verbatim from iOS LLMBenchmarkProvider.swift. const LLM_SYSTEM_PROMPT = 'You are a helpful assistant. Always give extremely detailed, ' + 'thorough responses. Never stop early. Use the full response length available ' + @@ -108,7 +105,6 @@ async function loadBenchmarkModel( return Date.now() - loadStart; } -/** Mirrors iOS LLMBenchmarkProvider.execute. */ async function runLLMScenario( model: SDKModelInfo, maxTokens: number @@ -119,7 +115,6 @@ async function runLLMScenario( ModelCategory.MODEL_CATEGORY_LANGUAGE ); try { - // Warmup const warmupEvents = RunAnywhere.generateStream( 'Hello', LLMGenerationOptions.fromPartial({ maxTokens: 5, temperature: 0 }) @@ -128,7 +123,6 @@ async function runLLMScenario( if (event.isFinal) break; } - // Benchmark const benchStart = Date.now(); const events = RunAnywhere.generateStream( LLM_PROMPT, @@ -160,7 +154,6 @@ async function runLLMScenario( } } -/** Mirrors iOS STTBenchmarkProvider.execute. */ async function runSTTScenario( model: SDKModelInfo, type: 'silent' | 'sine' @@ -195,7 +188,6 @@ async function runSTTScenario( } } -// Texts copied verbatim from iOS TTSBenchmarkProvider.swift. const TTS_TEXTS: Record<'short' | 'medium', string> = { short: 'Hello, this is a test.', medium: @@ -203,7 +195,6 @@ const TTS_TEXTS: Record<'short' | 'medium', string> = { 'generate speech from text with remarkable quality and natural intonation.', }; -/** Mirrors iOS TTSBenchmarkProvider.execute. */ async function runTTSScenario( model: SDKModelInfo, length: 'short' | 'medium' @@ -230,7 +221,6 @@ async function runTTSScenario( } } -/** Scenario lists mirror the iOS providers. */ function scenariosForCategory(category: BenchmarkCategory): Array<{ name: string; run: (model: SDKModelInfo) => Promise<{ @@ -258,41 +248,43 @@ function scenariosForCategory(category: BenchmarkCategory): Array<{ } } +function formatTimestamp(ms: number): string { + const d = new Date(ms); + return d.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }) + + ', ' + + d.toLocaleTimeString('en-US', { hour: '2-digit', minute: '2-digit', hour12: false }); +} + // --------------------------------------------------------------------------- // Screen // --------------------------------------------------------------------------- export const BenchmarkScreen: React.FC = () => { - const [selectedCategories, setSelectedCategories] = useState< - Set - >(new Set(ALL_CATEGORIES)); - const [modelsByCategory, setModelsByCategory] = useState< - Record - >({ LLM: [], STT: [], TTS: [] }); - const [selectedModelIds, setSelectedModelIds] = useState>( - new Set() + const { colors, typography, dimens } = useTheme(); + const insets = useSafeAreaInsets(); + + const [selectedCategories, setSelectedCategories] = useState>( + new Set(ALL_CATEGORIES) ); + const [modelsByCategory, setModelsByCategory] = useState>( + { LLM: [], STT: [], TTS: [] } + ); + const [selectedModelIds, setSelectedModelIds] = useState>(new Set()); const [results, setResults] = useState([]); const [isRunning, setIsRunning] = useState(false); - const [progressText, setProgressText] = useState(null); + const [progressCurrent, setProgressCurrent] = useState(0); + const [progressTotal, setProgressTotal] = useState(0); + const [progressLabel, setProgressLabel] = useState(''); const [error, setError] = useState(null); const cancelRef = useRef(false); - /** - * Preflight: downloaded (on-disk) models grouped by category — mirrors - * iOS BenchmarkRunner.preflight / downloadedModels(for:in:). - */ const refreshModels = useCallback(async () => { setError(null); try { await RunAnywhere.refreshModelRegistry(); const listResult = await RunAnywhere.listModels(); const allModels = listResult.models?.models ?? []; - const grouped: Record = { - LLM: [], - STT: [], - TTS: [], - }; + const grouped: Record = { LLM: [], STT: [], TTS: [] }; for (const category of ALL_CATEGORIES) { grouped[category] = allModels.filter( (model) => @@ -301,12 +293,9 @@ export const BenchmarkScreen: React.FC = () => { ); } setModelsByCategory(grouped); - // Default-select every downloaded model. setSelectedModelIds( new Set( - ALL_CATEGORIES.flatMap((category) => - grouped[category].map((model) => model.id) - ) + ALL_CATEGORIES.flatMap((category) => grouped[category].map((model) => model.id)) ) ); } catch (err) { @@ -321,7 +310,7 @@ export const BenchmarkScreen: React.FC = () => { setResults(JSON.parse(raw) as BenchmarkResultEntry[]); } } catch { - // History is best-effort; a corrupt entry should not break the screen. + // History is best-effort. } }, []); @@ -363,7 +352,6 @@ export const BenchmarkScreen: React.FC = () => { }); }, []); - /** Expand (category x model x scenario) — mirrors iOS buildWorkItems. */ const buildWorkItems = useCallback((): WorkItem[] => { const items: WorkItem[] = []; for (const category of ALL_CATEGORIES) { @@ -373,12 +361,7 @@ export const BenchmarkScreen: React.FC = () => { ); for (const model of models) { for (const scenario of scenariosForCategory(category)) { - items.push({ - category, - model, - scenario: scenario.name, - run: scenario.run, - }); + items.push({ category, model, scenario: scenario.name, run: scenario.run }); } } } @@ -388,22 +371,20 @@ export const BenchmarkScreen: React.FC = () => { const handleRun = useCallback(async () => { const workItems = buildWorkItems(); if (workItems.length === 0) { - setError( - 'No benchmarks to run. Download models first, then select at least one.' - ); + setError('No benchmarks to run. Download models first, then select at least one.'); return; } setIsRunning(true); setError(null); cancelRef.current = false; + setProgressTotal(workItems.length); const newResults: BenchmarkResultEntry[] = []; for (let i = 0; i < workItems.length; i++) { if (cancelRef.current) break; const item = workItems[i]; - setProgressText( - `${i + 1}/${workItems.length} — ${item.scenario} · ${item.model.name}` - ); + setProgressCurrent(i + 1); + setProgressLabel(`${item.category} · ${item.scenario} · ${item.model.name}`); const base = { id: `${Date.now()}-${i}`, category: item.category, @@ -424,43 +405,98 @@ export const BenchmarkScreen: React.FC = () => { } } - setProgressText(null); setIsRunning(false); + setProgressCurrent(0); + setProgressTotal(0); + setProgressLabel(''); await persistResults([...newResults, ...results]); }, [buildWorkItems, persistResults, results]); const handleCancel = useCallback(() => { cancelRef.current = true; - setProgressText('Cancelling after current scenario…'); + setProgressLabel('Cancelling after current scenario…'); }, []); const handleClearResults = useCallback(async () => { await persistResults([]); }, [persistResults]); + const selectedCount = selectedCategories.size; + const progressFraction = progressTotal > 0 ? progressCurrent / progressTotal : 0; + + // Group results by category for display (mirrors KT detail screen). + const resultsByCategory = ALL_CATEGORIES.reduce>( + (acc, cat) => { + acc[cat] = results.filter((r) => r.category === cat); + return acc; + }, + { LLM: [], STT: [], TTS: [] } + ); + return ( - + + {/* Error banner */} {error && ( - - - {error} + + + + {error} + )} - {/* Category chips */} - CATEGORIES + {/* Device card */} + + + + + + + On-Device Benchmark + + + {ALL_CATEGORIES.flatMap((c) => modelsByCategory[c]).length} model{ALL_CATEGORIES.flatMap((c) => modelsByCategory[c]).length !== 1 ? 's' : ''} available + + + + + {/* Categories section */} + + Categories + {ALL_CATEGORIES.map((category) => { const selected = selectedCategories.has(category); return ( toggleCategory(category)} disabled={isRunning} + activeOpacity={0.7} > + {selected && ( + + )} {category} @@ -469,87 +505,215 @@ export const BenchmarkScreen: React.FC = () => { })} - {/* Downloaded models per selected category */} - {ALL_CATEGORIES.filter((category) => - selectedCategories.has(category) - ).map((category) => ( - - {category} MODELS + {/* Models per selected category */} + {ALL_CATEGORIES.filter((cat) => selectedCategories.has(cat)).map((category) => ( + + + {category} Models + {modelsByCategory[category].length === 0 ? ( - - No downloaded {category} models. Download from Settings → Model - Catalog first. + + No downloaded {category} models. Download from Settings → Model Catalog first. ) : ( - modelsByCategory[category].map((model) => { - const selected = selectedModelIds.has(model.id); - return ( - toggleModel(model.id)} - disabled={isRunning} - > - - {model.name} - - ); - }) + + {modelsByCategory[category].map((model, i) => { + const selected = selectedModelIds.has(model.id); + return ( + + {i > 0 && ( + + )} + toggleModel(model.id)} + disabled={isRunning} + activeOpacity={0.7} + > + + {selected && ( + + )} + + + {model.name} + + + + ); + })} + )} ))} - {/* Run / cancel */} + {/* Run / Running card */} {isRunning ? ( - - - {progressText && ( - {progressText} + + + + + {progressTotal > 0 ? `Running ${progressCurrent} / ${progressTotal}` : 'Running…'} + + + + + Cancel + + + + {progressTotal > 0 && ( + + + + )} + {progressLabel.length > 0 && ( + + {progressLabel} + )} - - Cancel - ) : ( - - - Run Benchmarks + 0 ? colors.primary : colors.surfaceContainerHigh }, + ]} + onPress={handleRun} + activeOpacity={0.8} + disabled={selectedCount === 0} + > + 0 ? colors.onPrimary : colors.onSurfaceVariant} + /> + 0 ? colors.onPrimary : colors.onSurfaceVariant }, + ]} + > + Run selected ({selectedCount}) + )} - {/* Results */} - {results.length > 0 && ( - - - RESULTS - - Clear All + {/* History */} + {results.length > 0 ? ( + + + + History + + + + Clear all + - {results.map((entry) => ( - - - - {entry.category} · {entry.scenario} - - - {entry.modelName} + + {ALL_CATEGORIES.map((cat) => { + const catResults = resultsByCategory[cat]; + if (catResults.length === 0) return null; + return ( + + + {cat} + {catResults.map((entry) => ( + + {/* Header row: success/fail icon + scenario + model */} + + + + + {entry.scenario} + + + {entry.modelName} · {formatTimestamp(entry.timestampMs)} + + + + + {/* Metric rows */} + {entry.errorMessage ? ( + + {entry.errorMessage} + + ) : ( + + + + Load time + + + {entry.loadTimeMs.toFixed(0)} ms + + + + + + Result + + + {entry.metricSummary} + + + + )} + + ))} - {entry.errorMessage ? ( - {entry.errorMessage} - ) : ( - - Load {entry.loadTimeMs.toFixed(0)} ms · {entry.metricSummary} - - )} - - ))} + ); + })} + ) : ( + !isRunning && ( + + No runs yet. Pick categories and run a benchmark across your downloaded models. + + ) )} ); @@ -558,141 +722,154 @@ export const BenchmarkScreen: React.FC = () => { const styles = StyleSheet.create({ container: { flex: 1, - backgroundColor: Colors.backgroundPrimary, }, content: { - padding: Padding.padding16, + paddingHorizontal: 16, + gap: 12, + }, + bold: { + fontWeight: '700', }, - errorBox: { + errorBanner: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.small, - backgroundColor: Colors.badgeRed, - borderRadius: BorderRadius.medium, - padding: Padding.padding12, - marginBottom: Spacing.medium, + gap: 8, + borderRadius: 12, + padding: 12, }, - errorText: { - ...Typography.caption, - color: Colors.primaryRed, + card: { + flexDirection: 'row', + alignItems: 'center', + gap: 12, + borderRadius: 20, + padding: 16, + }, + iconTile: { + width: 44, + height: 44, + borderRadius: 14, + alignItems: 'center', + justifyContent: 'center', + }, + cardText: { flex: 1, + gap: 2, }, - sectionHeader: { - ...Typography.caption, - color: Colors.textSecondary, - marginBottom: Spacing.small, + sectionLabel: { + marginBottom: 4, }, - chipRow: { + sectionHeaderRow: { flexDirection: 'row', - gap: Spacing.small, - marginBottom: Spacing.large, + alignItems: 'center', + justifyContent: 'space-between', + marginBottom: 4, }, - chip: { - borderWidth: 1, - borderColor: Colors.primaryBlue, - borderRadius: 16, - paddingHorizontal: Padding.padding16, - paddingVertical: 6, + section: { + gap: 8, }, - chipSelected: { - backgroundColor: Colors.primaryBlue, + chipRow: { + flexDirection: 'row', + gap: 8, + flexWrap: 'wrap', }, - chipText: { - ...Typography.caption, - color: Colors.primaryBlue, + filterChip: { + flexDirection: 'row', + alignItems: 'center', + gap: 6, + borderWidth: 1, + borderRadius: 999, + paddingHorizontal: 14, + paddingVertical: 7, }, - chipTextSelected: { - color: Colors.textWhite, + listCard: { + borderRadius: 16, + overflow: 'hidden', }, - modelSection: { - marginBottom: Spacing.large, + divider: { + height: StyleSheet.hairlineWidth, + marginLeft: 48, }, modelRow: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.small, - paddingVertical: Padding.padding8, + gap: 12, + paddingHorizontal: 14, + paddingVertical: 13, }, - modelName: { - ...Typography.subheadline, - color: Colors.textPrimary, - flex: 1, + checkbox: { + width: 22, + height: 22, + borderRadius: 6, + borderWidth: 2, + alignItems: 'center', + justifyContent: 'center', }, - emptyText: { - ...Typography.caption, - color: Colors.textTertiary, + runningCard: { + borderRadius: 20, + padding: 16, + gap: 10, }, - runArea: { + runningHeader: { + flexDirection: 'row', alignItems: 'center', - gap: Spacing.small, - marginVertical: Spacing.medium, + gap: 10, }, - progressText: { - ...Typography.caption, - color: Colors.textSecondary, - textAlign: 'center', + cancelPill: { + flexDirection: 'row', + alignItems: 'center', + gap: 5, + borderWidth: 1, + borderRadius: 999, + paddingHorizontal: 12, + paddingVertical: 6, }, - cancelButton: { - paddingHorizontal: Padding.padding16, - paddingVertical: Padding.padding8, + progressTrack: { + height: 4, + borderRadius: 999, + overflow: 'hidden', }, - cancelButtonText: { - ...Typography.subheadline, - color: Colors.primaryRed, + progressFill: { + height: 4, + borderRadius: 999, }, runButton: { flexDirection: 'row', alignItems: 'center', justifyContent: 'center', - gap: Spacing.small, - backgroundColor: Colors.primaryBlue, - borderRadius: BorderRadius.medium, - paddingVertical: Padding.padding12, - marginVertical: Spacing.medium, + gap: 8, + borderRadius: 999, + paddingVertical: 14, }, - runButtonText: { - ...Typography.headline, - color: Colors.textWhite, + resultCard: { + borderRadius: 16, + padding: 14, + gap: 10, }, - resultsHeaderRow: { + resultHeaderRow: { flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - }, - clearText: { - ...Typography.caption, - color: Colors.primaryRed, + alignItems: 'flex-start', + gap: 10, }, - resultCard: { - backgroundColor: Colors.backgroundSecondary, - borderRadius: BorderRadius.medium, - padding: Padding.padding12, - marginBottom: Spacing.small, + resultTitleBlock: { + flex: 1, + gap: 2, }, - resultTitleRow: { + metricGrid: { flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - gap: Spacing.small, - }, - resultScenario: { - ...Typography.subheadline, - color: Colors.textPrimary, + gap: 12, + alignItems: 'stretch', }, - resultModel: { - ...Typography.caption, - color: Colors.textSecondary, - flexShrink: 1, + metricCell: { + flex: 1, + gap: 2, }, - resultMetrics: { - ...Typography.caption, - color: Colors.textSecondary, - marginTop: 4, + metricCellDivider: { + width: StyleSheet.hairlineWidth, }, - resultError: { - ...Typography.caption, - color: Colors.primaryRed, - marginTop: 4, + emptyHistory: { + textAlign: 'center', + paddingVertical: 32, + paddingHorizontal: 24, }, }); diff --git a/examples/react-native/RunAnywhereAI/src/screens/ChatScreen.tsx b/examples/react-native/RunAnywhereAI/src/screens/ChatScreen.tsx index fe446e2558..9e2bc42141 100644 --- a/examples/react-native/RunAnywhereAI/src/screens/ChatScreen.tsx +++ b/examples/react-native/RunAnywhereAI/src/screens/ChatScreen.tsx @@ -40,10 +40,11 @@ import { import { Colors } from '../theme/colors'; import { Typography } from '../theme/typography'; import { Spacing, Padding, IconSize } from '../theme/spacing'; -import { ModelStatusBanner, ModelRequiredOverlay } from '../components/common'; +import { ModelRequiredOverlay } from '../components/common'; +import { ChatHeader } from '../features/chat/components/ChatHeader'; +import { PromptSuggestions } from '../features/chat/components/PromptSuggestions'; import { MessageBubble, - TypingIndicator, ChatInput, ToolCallingBadge, LoRASheet, @@ -136,6 +137,7 @@ export const ChatScreen: React.FC = () => { const [showModelSelection, setShowModelSelection] = useState(false); const [isInitialized, setIsInitialized] = useState(false); const [registeredToolCount, setRegisteredToolCount] = useState(0); + const [toolsEnabled, setToolsEnabled] = useState(false); // LoRA adapter management (mirrors iOS LLMViewModel.loraAdapters). const [showLoRASheet, setShowLoRASheet] = useState(false); const [loraAdapterCount, setLoraAdapterCount] = useState(0); @@ -276,10 +278,10 @@ export const ChatScreen: React.FC = () => { * Handle model selected from the sheet */ const handleModelSelected = useCallback(async (model: SDKModelInfo) => { - // Close the modal first to prevent UI issues - setShowModelSelection(false); - // Then load the model + // The sheet shows its own Loading spinner during the await; once the model + // is loaded we close it so the user lands directly in the chat. await loadModel(model); + setShowModelSelection(false); }, []); /** @@ -346,26 +348,49 @@ export const ChatScreen: React.FC = () => { } }; + // Tool-calling toggle (mirrors Android viewModel.toolsEnabled). Persisted so + // it survives navigation; the input bar's tool button flips it. + useEffect(() => { + AsyncStorage.getItem(APP_STORAGE_KEYS.TOOL_CALLING_ENABLED).then((v) => + setToolsEnabled(v === 'true') + ); + }, []); + + const handleToggleTools = useCallback(() => { + setToolsEnabled((prev) => { + const next = !prev; + void AsyncStorage.setItem( + APP_STORAGE_KEYS.TOOL_CALLING_ENABLED, + next ? 'true' : 'false' + ); + return next; + }); + }, []); + /** * Send a message and stream the response token-by-token. * Uses RunAnywhere.generateStream() for real-time streaming UI. * * generateWithTools() is still available on RunAnywhere for callers - * that genuinely need the batch tool-calling form. + * that genuinely need the batch tool-calling form. An optional prompt + * override lets prompt-suggestion pills send their text directly. */ - const handleSend = useCallback(async () => { - if (isLoading || !inputText.trim() || !currentConversation) return; + const handleSend = useCallback(async (promptOverride?: string) => { + const text = ( + typeof promptOverride === 'string' ? promptOverride : inputText + ).trim(); + if (isLoading || !text || !currentConversation) return; const userMessage: Message = { id: generateId(), role: MessageRole.User, - content: inputText.trim(), + content: text, timestamp: new Date(), }; // Add user message to conversation await addMessage(userMessage, currentConversation.id); - const prompt = inputText.trim(); + const prompt = text; setInputText(''); setIsLoading(true); @@ -389,10 +414,7 @@ export const ChatScreen: React.FC = () => { ); const registeredTools = await RunAnywhere.getRegisteredTools(); - const toolCallingEnabled = - (await AsyncStorage.getItem(APP_STORAGE_KEYS.TOOL_CALLING_ENABLED)) === - 'true'; - const shouldUseTools = toolCallingEnabled && registeredTools.length > 0; + const shouldUseTools = toolsEnabled && registeredTools.length > 0; const supportsThinking = currentModel?.supportsThinking ?? false; const wasThinkingMode = supportsThinking && options.thinkingModeEnabled; const disableThinking = supportsThinking && !options.thinkingModeEnabled; @@ -430,6 +452,7 @@ export const ChatScreen: React.FC = () => { role: MessageRole.Assistant, content: '', timestamp: new Date(), + isStreaming: true, modelInfo: { modelId: currentModel?.id || 'unknown', modelName: currentModel?.name || 'Unknown Model', @@ -520,6 +543,7 @@ export const ChatScreen: React.FC = () => { role: MessageRole.Assistant, content: accumulatedText, timestamp: new Date(), + isStreaming: true, modelInfo: { modelId: currentModel?.id || 'unknown', modelName: currentModel?.name || 'Unknown Model', @@ -623,6 +647,7 @@ export const ChatScreen: React.FC = () => { inputText, currentConversation, currentModel, + toolsEnabled, addMessage, updateMessage, updateConversation, @@ -698,166 +723,105 @@ export const ChatScreen: React.FC = () => { * Render header with actions */ const renderHeader = () => ( - - {/* Conversations list button */} - setShowConversationList(true)} - > - - - - {/* Title with conversation count */} - - Chat - {conversations.length > 1 && ( - - {conversations.length} conversations - - )} - - - - {/* New chat button */} - - - - - {/* Info button for chat analytics */} - - 0 ? Colors.primaryBlue : Colors.textTertiary - } - /> - - - + 0} + onModelPress={handleSelectModel} + onAnalytics={handleShowAnalytics} + onHistory={() => setShowConversationList(true)} + onNewChat={handleNewChat} + /> ); - // Show model required overlay if no model - if (!currentModel && !isModelLoading) { - return ( - - {renderHeader()} - - - {/* Conversation List Modal */} - setShowConversationList(false)} - > - setShowConversationList(false)} - onSelectConversation={handleSelectConversation} - /> - - - {/* Model Selection Sheet */} - setShowModelSelection(false)} - onModelSelected={handleModelSelected} - /> - - ); - } + const showOverlay = !currentModel && !isModelLoading; return ( - + {renderHeader()} - {/* Model Status Banner */} - - - {/* Messages List */} - item.id} - contentContainerStyle={[ - styles.messagesList, - messages.length === 0 && styles.emptyList, - ]} - ListEmptyComponent={renderEmptyState} - showsVerticalScrollIndicator={false} - /> - - {/* Typing Indicator */} - {isLoading && } - - {/* Tool Calling Badge (shows when tools are enabled) */} - {currentModel && registeredToolCount > 0 && ( - - )} - - {/* LoRA pill (mirrors iOS ChatMessageListView's LoRA row above input) */} - {currentModel && ( - - 0 && styles.loraPillActive, + {showOverlay ? ( + + ) : ( + <> + {/* Messages List */} + item.id} + contentContainerStyle={[ + styles.messagesList, + messages.length === 0 && styles.emptyList, ]} - onPress={() => setShowLoRASheet(true)} - > - 0 ? Colors.textWhite : Colors.primaryPurple - } + ListEmptyComponent={renderEmptyState} + showsVerticalScrollIndicator={false} + /> + + {/* Tool Calling Badge (shows when tools are enabled) */} + {currentModel && registeredToolCount > 0 && ( + + )} + + {/* LoRA pill (mirrors iOS ChatMessageListView's LoRA row above input) */} + {currentModel && ( + + 0 && styles.loraPillActive, + ]} + onPress={() => setShowLoRASheet(true)} + > + 0 ? Colors.textWhite : Colors.primaryPurple + } + /> + 0 && styles.loraPillTextActive, + ]} + > + {loraAdapterCount > 0 ? `LoRA x${loraAdapterCount}` : '+ LoRA'} + + + + )} + + {/* Example prompts (mode follows tool/LoRA state), shown on an empty chat */} + {currentModel && messages.length === 0 && ( + 0} + onSelect={(p) => void handleSend(p)} /> - 0 && styles.loraPillTextActive, - ]} - > - {loraAdapterCount > 0 ? `LoRA x${loraAdapterCount}` : '+ LoRA'} - - - + )} + + {/* Input Area */} + + )} - {/* Input Area */} - - {/* Analytics Modal */} { /> - {/* LoRA Adapter Management Modal */} - setShowLoRASheet(false)} - > - setShowLoRASheet(false)} - onAdaptersChanged={(adapters) => setLoraAdapterCount(adapters.length)} - /> - + modelId={currentModel?.id ?? null} + onClose={() => setShowLoRASheet(false)} + onAdaptersChanged={(adapters) => setLoraAdapterCount(adapters.length)} + /> - {/* Conversation List Modal */} - setShowConversationList(false)} - > - setShowConversationList(false)} - onSelectConversation={handleSelectConversation} - /> - + onClose={() => setShowConversationList(false)} + onSelectConversation={handleSelectConversation} + /> {/* Model Selection Sheet */} setShowModelSelection(false)} onModelSelected={handleModelSelected} /> @@ -947,11 +900,14 @@ const styles = StyleSheet.create({ headerButtonDisabled: { opacity: 0.5, }, + list: { + flex: 1, + }, messagesList: { paddingVertical: Spacing.medium, }, emptyList: { - flex: 1, + flexGrow: 1, justifyContent: 'center', }, emptyState: { @@ -981,7 +937,8 @@ const styles = StyleSheet.create({ loraRow: { flexDirection: 'row', paddingHorizontal: Padding.padding16, - paddingBottom: Spacing.small, + paddingTop: 2, + paddingBottom: 6, }, loraPill: { flexDirection: 'row', diff --git a/examples/react-native/RunAnywhereAI/src/screens/ConversationListScreen.tsx b/examples/react-native/RunAnywhereAI/src/screens/ConversationListScreen.tsx index 6bee72a66f..aa499a63f2 100644 --- a/examples/react-native/RunAnywhereAI/src/screens/ConversationListScreen.tsx +++ b/examples/react-native/RunAnywhereAI/src/screens/ConversationListScreen.tsx @@ -1,158 +1,59 @@ /** - * ConversationListScreen - Modal for managing conversations + * ConversationListScreen — conversation history in the app bottom sheet. * - * Reference: iOS Core/Services/ConversationStore.swift (ConversationListView) - * - * Features: - * - List all conversations with search - * - Create new conversation - * - Delete conversation with confirmation - * - Switch between conversations + * Search, start a new chat, switch conversations, and delete with confirmation. + * Rows show title, last-message preview, a relative date and the framework chip. */ - -import React, { useState, useCallback, useMemo } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import { - View, - Text, - FlatList, + Alert, StyleSheet, - TouchableOpacity, + Text, TextInput, - Alert, + TouchableOpacity, + View, } from 'react-native'; -import { SafeAreaView } from 'react-native-safe-area-context'; -import Icon from 'react-native-vector-icons/Ionicons'; -import { Colors } from '../theme/colors'; -import { Typography } from '../theme/typography'; -import { Spacing, Padding, BorderRadius, IconSize } from '../theme/spacing'; +import { BottomSheet, BottomSheetScrollView } from '../components/ui/BottomSheet'; +import { Icon, useTheme } from '../theme/system'; import type { Conversation } from '../types/chat'; import { useConversationStore, - getConversationSummary, getLastMessagePreview, formatRelativeDate, } from '../stores/conversationStore'; interface ConversationListScreenProps { + visible: boolean; onClose: () => void; onSelectConversation: (conversation: Conversation) => void; } -/** - * ConversationRow - Individual conversation item in list - * Matches iOS ConversationRow struct - */ -interface ConversationRowProps { - conversation: Conversation; - onPress: () => void; - onDelete: () => void; -} - -/** - * Stable separator component to avoid react/no-unstable-nested-components - */ -const ItemSeparator: React.FC = () => ; - -const ConversationRow: React.FC = ({ - conversation, - onPress, - onDelete, -}) => { - const handleDelete = useCallback(() => { - Alert.alert( - 'Delete Conversation', - `Are you sure you want to delete "${conversation.title}"? This cannot be undone.`, - [ - { text: 'Cancel', style: 'cancel' }, - { - text: 'Delete', - style: 'destructive', - onPress: onDelete, - }, - ] - ); - }, [conversation.title, onDelete]); - - return ( - - - {/* Title */} - - {conversation.title} - - - {/* Last message preview */} - - {getLastMessagePreview(conversation)} - - - {/* Summary line */} - - {getConversationSummary(conversation)} - - - {/* Bottom row: date and framework badge */} - - - {formatRelativeDate(conversation.updatedAt)} - - - {conversation.frameworkName && ( - - - {conversation.frameworkName} - - - )} - - - - {/* Delete button */} - - - - - ); -}; +const SNAP_POINTS = ['85%']; export const ConversationListScreen: React.FC = ({ + visible, onClose, onSelectConversation, }) => { + const { colors, typography } = useTheme(); const [searchQuery, setSearchQuery] = useState(''); - const { - conversations, - createConversation, - deleteConversation, - searchConversations, - } = useConversationStore(); + const { conversations, createConversation, deleteConversation, searchConversations } = + useConversationStore(); - // Filter conversations based on search - const filteredConversations = useMemo(() => { - if (!searchQuery.trim()) { - return conversations; - } - return searchConversations(searchQuery); - }, [conversations, searchQuery, searchConversations]); + const filtered = useMemo( + () => + searchQuery.trim() ? searchConversations(searchQuery) : conversations, + [conversations, searchQuery, searchConversations] + ); - /** - * Handle creating a new conversation - */ - const handleCreateConversation = useCallback(async () => { - const newConversation = await createConversation(); - onSelectConversation(newConversation); + const handleCreate = useCallback(async () => { + const created = await createConversation(); + onSelectConversation(created); onClose(); }, [createConversation, onSelectConversation, onClose]); - /** - * Handle selecting a conversation - */ - const handleSelectConversation = useCallback( + const handleSelect = useCallback( (conversation: Conversation) => { onSelectConversation(conversation); onClose(); @@ -160,244 +61,213 @@ export const ConversationListScreen: React.FC = ({ [onSelectConversation, onClose] ); - /** - * Handle deleting a conversation - */ - const handleDeleteConversation = useCallback( - async (conversationId: string) => { - await deleteConversation(conversationId); + const handleDelete = useCallback( + (conversation: Conversation) => { + Alert.alert( + 'Delete conversation', + `Delete "${conversation.title}"? This can't be undone.`, + [ + { text: 'Cancel', style: 'cancel' }, + { + text: 'Delete', + style: 'destructive', + onPress: () => void deleteConversation(conversation.id), + }, + ] + ); }, [deleteConversation] ); - /** - * Render a conversation item - */ - const renderConversation = ({ item }: { item: Conversation }) => ( - handleSelectConversation(item)} - onDelete={() => handleDeleteConversation(item.id)} - /> - ); - - /** - * Render empty state - */ - const renderEmptyState = () => ( - - - - - - {searchQuery ? 'No conversations found' : 'No conversations yet'} - - - {searchQuery - ? 'Try a different search term' - : 'Tap the + button to start a new conversation'} - - - ); - return ( - - {/* Header */} + - - Done - - - Conversations - - - + + Conversations + + + - {/* Search Bar */} - - + + {searchQuery.length > 0 && ( - setSearchQuery('')} - style={styles.clearButton} - > - + setSearchQuery('')} hitSlop={8}> + )} - {/* Conversations List */} - item.id} - contentContainerStyle={[ - styles.listContent, - filteredConversations.length === 0 && styles.emptyListContent, - ]} - ListEmptyComponent={renderEmptyState} - showsVerticalScrollIndicator={false} - ItemSeparatorComponent={ItemSeparator} - /> - + + {filtered.length === 0 ? ( + + + + {searchQuery ? 'No matches' : 'No conversations yet'} + + + {searchQuery + ? 'Try a different search term' + : 'Start a new chat with the + button'} + + + ) : ( + + {filtered.map((c, i) => ( + + {i > 0 && ( + + )} + handleSelect(c)} + > + + + {c.title} + + + {getLastMessagePreview(c)} + + + + {formatRelativeDate(c.updatedAt)} + + {!!c.frameworkName && ( + + + {c.frameworkName} + + + )} + + + handleDelete(c)} + hitSlop={8} + > + + + + + ))} + + )} + + ); }; const styles = StyleSheet.create({ - container: { - flex: 1, - backgroundColor: Colors.backgroundPrimary, - }, header: { flexDirection: 'row', alignItems: 'center', justifyContent: 'space-between', - paddingHorizontal: Padding.padding16, - paddingVertical: Padding.padding12, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, + paddingHorizontal: 20, + paddingTop: 4, + paddingBottom: 12, }, - headerButton: { - minWidth: 60, - }, - doneText: { - ...Typography.body, - color: Colors.primaryBlue, - fontWeight: '600', - }, - title: { - ...Typography.title2, - color: Colors.textPrimary, - textAlign: 'center', - }, - searchContainer: { + search: { flexDirection: 'row', alignItems: 'center', - backgroundColor: Colors.backgroundSecondary, - borderRadius: BorderRadius.medium, - marginHorizontal: Padding.padding16, - marginVertical: Padding.padding12, - paddingHorizontal: Padding.padding12, - }, - searchIcon: { - marginRight: Spacing.small, + gap: 8, + marginHorizontal: 16, + paddingHorizontal: 12, + borderRadius: 14, + marginBottom: 12, }, searchInput: { flex: 1, - ...Typography.body, - color: Colors.textPrimary, - paddingVertical: Padding.padding12, - }, - clearButton: { - padding: Spacing.small, + paddingVertical: 11, }, - listContent: { - paddingBottom: Spacing.large, + list: { + paddingHorizontal: 16, + paddingBottom: 24, }, - emptyListContent: { - flex: 1, - justifyContent: 'center', + card: { + borderRadius: 16, + overflow: 'hidden', }, - separator: { - height: 1, - backgroundColor: Colors.borderLight, - marginHorizontal: Padding.padding16, + divider: { + height: StyleSheet.hairlineWidth, + marginLeft: 14, }, - conversationRow: { + row: { flexDirection: 'row', alignItems: 'center', - paddingHorizontal: Padding.padding16, - paddingVertical: Padding.padding12, + gap: 8, + paddingHorizontal: 14, + paddingVertical: 12, }, - conversationContent: { + rowText: { flex: 1, - marginRight: Spacing.medium, - }, - conversationTitle: { - ...Typography.headline, - color: Colors.textPrimary, - marginBottom: Spacing.xSmall, + gap: 3, }, - conversationPreview: { - ...Typography.subheadline, - color: Colors.textSecondary, - marginBottom: Spacing.xSmall, - lineHeight: 20, - }, - conversationSummary: { - ...Typography.caption, - color: Colors.textTertiary, - marginBottom: Spacing.xSmall, - }, - conversationBottom: { + metaRow: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.small, + gap: 8, + marginTop: 2, }, - conversationDate: { - ...Typography.caption2, - color: Colors.textTertiary, - }, - frameworkBadge: { - backgroundColor: Colors.primaryBlue + '20', - paddingHorizontal: Spacing.small, + chip: { + paddingHorizontal: 8, paddingVertical: 2, - borderRadius: BorderRadius.small, - }, - frameworkBadgeText: { - ...Typography.caption2, - color: Colors.primaryBlue, - fontWeight: '600', + borderRadius: 999, }, - deleteButton: { - padding: Spacing.small, + deleteBtn: { + padding: 6, }, - emptyState: { + empty: { alignItems: 'center', - padding: Padding.padding40, + gap: 8, + paddingVertical: 48, }, - emptyIconContainer: { - width: IconSize.huge, - height: IconSize.huge, - borderRadius: IconSize.huge / 2, - backgroundColor: Colors.backgroundSecondary, - justifyContent: 'center', - alignItems: 'center', - marginBottom: Spacing.large, - }, - emptyTitle: { - ...Typography.title3, - color: Colors.textPrimary, - marginBottom: Spacing.small, - }, - emptySubtitle: { - ...Typography.body, - color: Colors.textSecondary, + emptyText: { textAlign: 'center', - maxWidth: 280, + maxWidth: 260, + }, + bold: { + fontWeight: '700', }, }); diff --git a/examples/react-native/RunAnywhereAI/src/screens/MoreScreen.tsx b/examples/react-native/RunAnywhereAI/src/screens/MoreScreen.tsx index d3fed3a72b..4ef17f52b0 100644 --- a/examples/react-native/RunAnywhereAI/src/screens/MoreScreen.tsx +++ b/examples/react-native/RunAnywhereAI/src/screens/MoreScreen.tsx @@ -1,134 +1,169 @@ import React from 'react'; -import { StyleSheet, Text, TouchableOpacity, View } from 'react-native'; -import type { NativeStackScreenProps } from '@react-navigation/native-stack'; -import Icon from 'react-native-vector-icons/Ionicons'; -import { SafeAreaView } from 'react-native-safe-area-context'; -import { Colors } from '../theme/colors'; -import { Typography } from '../theme/typography'; -import { Spacing, Padding, BorderRadius } from '../theme/spacing'; -import type { MoreStackParamList } from '../types'; +import { ScrollView, StyleSheet, Text, TouchableOpacity, View } from 'react-native'; +import { useNavigation } from '@react-navigation/native'; +import type { NativeStackNavigationProp } from '@react-navigation/native-stack'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; +import { Icon, useTheme } from '../theme/system'; +import type { IconName } from '../theme/system/icons'; +import { ROUTES } from '../navigation/routes'; +import type { RootStackParamList } from '../navigation/navigation.types'; -type Props = NativeStackScreenProps; - -type MoreItem = { - route: Exclude; - title: string; - subtitle: string; - icon: string; +type MoreEntry = { + label: string; + description: string; + icon: IconName; + route: keyof RootStackParamList | null; }; -const ITEMS: MoreItem[] = [ +const ENTRIES: MoreEntry[] = [ + { + label: 'Settings', + description: 'Generation and storage', + icon: 'settings', + route: ROUTES.Settings, + }, + { + label: 'Tool Calling', + description: 'Let the LLM use registered tools', + icon: 'sliders', + route: null, + }, + { + label: 'Text to Speech', + description: 'Read text aloud', + icon: 'speak', + route: ROUTES.Speak, + }, + { + label: 'Speech to Text', + description: 'Transcribe audio on-device', + icon: 'transcribe', + route: ROUTES.Transcribe, + }, { - route: 'STT', - title: 'Transcribe', - subtitle: 'Speech-to-text', - icon: 'pulse-outline', + label: 'Voice Detection', + description: 'Detect speech activity in real-time', + icon: 'vad', + route: ROUTES.Vad, }, { - route: 'TTS', - title: 'Speak', - subtitle: 'Text-to-speech', - icon: 'volume-high-outline', + label: 'Vision', + description: 'Describe images with a VLM', + icon: 'vision', + route: ROUTES.Vlm, }, { - route: 'RAG', - title: 'RAG', - subtitle: 'Document question answering', - icon: 'search-outline', + label: 'Documents', + description: 'Chat with your files (RAG)', + icon: 'rag', + route: ROUTES.Rag, }, { - route: 'VAD', - title: 'Voice Activity', - subtitle: 'Speech detection stream', - icon: 'mic-circle-outline', + label: 'Solutions', + description: 'Run prepackaged pipelines from YAML', + icon: 'solutions', + route: ROUTES.Solutions, }, { - route: 'Storage', - title: 'Storage', - subtitle: 'Cache and model storage', - icon: 'folder-outline', + label: 'Cloud Providers', + description: 'Register cloud STT backends', + icon: 'cloud', + route: null, }, { - route: 'Solutions', - title: 'Solutions', - subtitle: 'YAML pipeline demos', - icon: 'layers-outline', + label: 'Benchmarks', + description: 'Measure model performance', + icon: 'benchmarks', + route: ROUTES.Benchmarks, }, ]; -export const MoreScreen: React.FC = ({ navigation }) => ( - - - More - - - {ITEMS.map((item) => ( - navigation.navigate(item.route)} - > - - - - - {item.title} - {item.subtitle} - - - - ))} - - -); +export const MoreScreen: React.FC = () => { + const navigation = + useNavigation>(); + const { colors, typography } = useTheme(); + const insets = useSafeAreaInsets(); + + return ( + + {ENTRIES.map((entry) => { + const enabled = entry.route !== null; + const labelColor = enabled + ? colors.onSurface + : colors.onSurfaceVariant; + const descColor = colors.onSurfaceVariant; + + return ( + { + if (enabled && entry.route) { + navigation.navigate(entry.route as keyof RootStackParamList); + } + }} + > + + + + {entry.label} + + + {entry.description} + + + {enabled ? ( + + ) : ( + + Soon + + )} + + ); + })} + + ); +}; const styles = StyleSheet.create({ - container: { + root: { flex: 1, - backgroundColor: Colors.backgroundPrimary, - }, - header: { - paddingHorizontal: Padding.padding16, - paddingVertical: Padding.padding12, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - title: { - ...Typography.title2, - color: Colors.textPrimary, }, - list: { - padding: Padding.padding16, - gap: Spacing.smallMedium, + content: { + paddingHorizontal: 16, + gap: 8, }, - row: { - minHeight: 72, + card: { flexDirection: 'row', alignItems: 'center', - padding: Padding.padding16, - borderRadius: BorderRadius.regular, - backgroundColor: Colors.backgroundSecondary, - gap: Spacing.medium, + gap: 14, + borderRadius: 16, + paddingHorizontal: 18, + paddingVertical: 16, }, - iconContainer: { - width: 40, - height: 40, - borderRadius: 20, - alignItems: 'center', - justifyContent: 'center', - backgroundColor: Colors.badgeBlue, - }, - rowText: { + textBlock: { flex: 1, - }, - rowTitle: { - ...Typography.headline, - color: Colors.textPrimary, - }, - rowSubtitle: { - ...Typography.footnote, - color: Colors.textSecondary, - marginTop: 2, + gap: 2, }, }); diff --git a/examples/react-native/RunAnywhereAI/src/screens/RAGScreen.tsx b/examples/react-native/RunAnywhereAI/src/screens/RAGScreen.tsx index bbdd460b49..1dd38ea69b 100644 --- a/examples/react-native/RunAnywhereAI/src/screens/RAGScreen.tsx +++ b/examples/react-native/RunAnywhereAI/src/screens/RAGScreen.tsx @@ -22,16 +22,15 @@ import { ScrollView, StyleSheet, ActivityIndicator, - KeyboardAvoidingView, - Platform, } from 'react-native'; -import { SafeAreaView } from 'react-native-safe-area-context'; +import Animated, { + useAnimatedKeyboard, + useAnimatedStyle, +} from 'react-native-reanimated'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { NativeModules } from 'react-native'; -import Icon from 'react-native-vector-icons/Ionicons'; import { pick as documentPick } from '@react-native-documents/picker'; -import { Colors } from '../theme/colors'; -import { Typography, FontWeight } from '../theme/typography'; -import { Spacing, Padding, BorderRadius } from '../theme/spacing'; +import { Icon, useTheme } from '../theme/system'; import { ModelSelectionSheet, ModelSelectionContext, @@ -55,10 +54,6 @@ interface ChatMessage { const { DocumentService: NativeDocumentService } = NativeModules; -/** - * Extract text from a document using native PDFKit (for PDF) or string parsing. - * Mirrors iOS DocumentService.swift - handles PDF, JSON, and plain text. - */ async function extractTextFromFile(filePath: string): Promise { if (NativeDocumentService?.extractText) { return NativeDocumentService.extractText(filePath); @@ -69,6 +64,21 @@ async function extractTextFromFile(filePath: string): Promise { // MARK: - Component export const RAGScreen: React.FC = () => { + const { colors, typography, dimens } = useTheme(); + const insets = useSafeAreaInsets(); + const keyboard = useAnimatedKeyboard(); + + // IME padding: the activity is adjustResize under edge-to-edge, but the window + // isn't lifted for us, so lift the content by the keyboard height (minus the + // bottom inset reanimated already includes). Rests on the bottom safe-area + // inset / screen padding when the keyboard is closed. + const keyboardStyle = useAnimatedStyle(() => ({ + paddingBottom: + keyboard.height.value > 0 + ? Math.max(keyboard.height.value - insets.bottom, dimens.spacing.sm) + : Math.max(insets.bottom, dimens.screenPadding), + })); + // Nitro state const [isNitroReady, setIsNitroReady] = useState(false); const [nitroError, setNitroError] = useState(null); @@ -83,10 +93,12 @@ export const RAGScreen: React.FC = () => { const [showingLLMPicker, setShowingLLMPicker] = useState(false); // Document state - const [documentName, setDocumentName] = useState(null); - const [isDocumentLoaded, setIsDocumentLoaded] = useState(false); + const [documents, setDocuments] = useState([]); const [isLoadingDocument, setIsLoadingDocument] = useState(false); + // Setup card collapse + const [setupExpanded, setSetupExpanded] = useState(true); + // Chat state const [messages, setMessages] = useState([]); const [currentQuestion, setCurrentQuestion] = useState(''); @@ -95,16 +107,23 @@ export const RAGScreen: React.FC = () => { const scrollViewRef = useRef(null); - // D-6: RAG lifecycle is model-id-driven; commons resolves id → path. - // We just need both models selected (id present). const areModelsReady = selectedEmbeddingModel?.id != null && selectedEmbeddingModel.id.length > 0 && selectedLLMModel?.id != null && selectedLLMModel.id.length > 0; + const hasDocuments = documents.length > 0; + const collapsible = areModelsReady && hasDocuments; + const showFullSetup = setupExpanded || !collapsible; + const canAskQuestion = - isDocumentLoaded && !isQuerying && currentQuestion.trim().length > 0; + hasDocuments && !isQuerying && currentQuestion.trim().length > 0; + + // Auto-collapse once ready + useEffect(() => { + if (areModelsReady && hasDocuments) setSetupExpanded(false); + }, [areModelsReady, hasDocuments]); // MARK: - Initialization @@ -126,18 +145,17 @@ export const RAGScreen: React.FC = () => { // Cleanup pipeline on unmount useEffect(() => { return () => { - if (isDocumentLoaded) { + if (hasDocuments) { RunAnywhere.ragDestroyPipeline().catch(console.error); } }; - }, [isDocumentLoaded]); + }, [hasDocuments]); // MARK: - Document Loading const handleSelectDocument = useCallback(async () => { if (!areModelsReady || !isNitroReady) return; - // D-6: RAG accepts model IDs; commons resolves them to artifact paths. const embeddingModelId = selectedEmbeddingModel?.id; const llmModelId = selectedLLMModel?.id; if (!embeddingModelId || !llmModelId) return; @@ -153,36 +171,27 @@ export const RAGScreen: React.FC = () => { setIsLoadingDocument(true); setError(null); - // Extract text from the picked file const text = await extractTextFromFile(fileUri); - // Build RAG configuration using model IDs. commons (via D-6) owns - // id → path resolution inside rac_rag_session_create_proto. - // Canonical defaults do the right thing (matches iOS DocumentRAG): - // commons derives the embedding dimension from the loaded embedding - // model, and the retrieval/chunking values come from idl/rag.proto - // rac_default annotations. const config = RAGConfiguration.fromPartial({ ...rAGConfigurationDefaults(), embeddingModelId, llmModelId, }); - // Create pipeline and ingest document (same as iOS loadDocument) await RunAnywhere.ragCreatePipeline(config); await RunAnywhere.ragIngest(text); - setDocumentName(result.name || 'Document'); - setIsDocumentLoaded(true); + const name = result.name || 'Document'; + setDocuments((prev) => [...prev, name]); } catch (err: unknown) { - // Document picker reports user cancellation via a stable error code. if ( typeof err === 'object' && err !== null && 'code' in err && (err as { code?: unknown }).code === 'OPERATION_CANCELED' ) { - return; // User cancelled + return; } const msg = err instanceof Error ? err.message : 'Failed to load document'; @@ -193,23 +202,20 @@ export const RAGScreen: React.FC = () => { } }, [areModelsReady, isNitroReady, selectedEmbeddingModel, selectedLLMModel]); - const handleChangeDocument = useCallback(async () => { + const handleClearAll = useCallback(async () => { await RunAnywhere.ragDestroyPipeline(); - setDocumentName(null); - setIsDocumentLoaded(false); + setDocuments([]); setMessages([]); setError(null); setCurrentQuestion(''); - - // Re-open document picker - handleSelectDocument(); - }, [handleSelectDocument]); + setSetupExpanded(true); + }, []); // MARK: - Q&A const handleAskQuestion = useCallback(async () => { const question = currentQuestion.trim(); - if (!question || !isDocumentLoaded) return; + if (!question || !hasDocuments) return; setMessages((prev) => [...prev, { role: 'user', text: question }]); setCurrentQuestion(''); @@ -217,11 +223,6 @@ export const RAGScreen: React.FC = () => { setError(null); try { - // Canonical query defaults (matches iOS, which queries with - // RARAGQueryOptions.defaults(question:)) — commons stamps - // maxTokens/temperature/topP and retrieval values when unset. - // Thinking mode mirrors iOS RAGViewModel: suppress thinking output - // for thinking-capable models unless the user enabled it in Settings. const thinkingStr = await AsyncStorage.getItem( GENERATION_SETTINGS_KEYS.THINKING_MODE_ENABLED ); @@ -248,7 +249,7 @@ export const RAGScreen: React.FC = () => { setTimeout(() => { scrollViewRef.current?.scrollToEnd({ animated: true }); }, 100); - }, [currentQuestion, isDocumentLoaded, selectedLLMModel]); + }, [currentQuestion, hasDocuments, selectedLLMModel]); // MARK: - Model Selection Callbacks @@ -265,286 +266,350 @@ export const RAGScreen: React.FC = () => { setShowingLLMPicker(false); }, []); - // MARK: - Error state for NitroModules + // MARK: - Error / Loading state for NitroModules if (nitroError) { return ( - - - Document Q&A - + - - NitroModules Error - {nitroError} + + + NitroModules Error + + + {nitroError} + - + ); } if (!isNitroReady) { return ( - - - Document Q&A - + - - Initializing... + + + Initializing… + - + ); } // MARK: - Render + const addLabel = documents.length === 0 ? 'Add document' : 'Add another'; + return ( - - + - {/* Model Setup Section */} - - setShowingEmbeddingPicker(true)} - > - - Embedding Model - - {selectedEmbeddingModel ? ( - <> - - {selectedEmbeddingModel.name} - - - - ) : ( + + {/* Setup card / compact bar */} + {showFullSetup ? ( + + {/* Collapse header (only when collapsible) */} + {collapsible && ( <> - Not selected - + setSetupExpanded(false)} + activeOpacity={0.7} + > + + Setup + + + + )} - - - - setShowingLLMPicker(true)} - > - - LLM Model - - {selectedLLMModel ? ( - <> - - {selectedLLMModel.name} + + {/* Embedding model row */} + setShowingEmbeddingPicker(true)} + activeOpacity={0.7} + > + + + + Embedding model - - - ) : ( - <> - Not selected - - - )} - - - + + {selectedEmbeddingModel?.name ?? 'Tap to select'} + + + + - {/* Document Status Bar */} - - {isLoadingDocument ? ( - - - Loading document... - - ) : isDocumentLoaded && documentName ? ( - - - - {documentName} - - - Change + + + {/* LLM model row */} + setShowingLLMPicker(true)} + activeOpacity={0.7} + > + + + + Language model + + + {selectedLLMModel?.name ?? 'Tap to select'} + + + + + + + {/* Documents section */} + + + + {documents.length === 0 + ? 'Documents' + : `${documents.length} document${documents.length === 1 ? '' : 's'}`} + + {documents.length > 0 && ( + + Clear + + )} + + + {documents.map((name, i) => ( + + + + {name} + + + ))} + + {/* Add document button */} + + {isLoadingDocument ? ( + <> + + + Reading… + + + ) : ( + <> + + + {areModelsReady ? addLabel : 'Pick models first'} + + + )} + + ) : ( + /* Compact bar */ setSetupExpanded(true)} + activeOpacity={0.7} > - - Select Document + + + {documents.length} document{documents.length === 1 ? '' : 's'} + + {isLoadingDocument ? ( + + ) : ( + + + + )} + )} - - {/* Error Banner */} - {error && ( - - - + {/* Error text */} + {error && ( + {error} - setError(null)}> - - - - )} - - {/* Messages Area */} - - {messages.length === 0 ? ( - - - {isDocumentLoaded ? ( - <> - Document loaded - - Ask a question below to get started - - - ) : !areModelsReady ? ( - <> - - Select models to get started - - - Choose an embedding model and an LLM model above, then pick - a document + )} + + {/* Conversation pane */} + + + {messages.length === 0 ? ( + + + + {!areModelsReady + ? 'Pick an embedding model and an LLM to begin' + : !hasDocuments + ? 'Add a document, then ask a question about it' + : 'Ask a question about your documents'} - + ) : ( <> - No document selected - - Pick a PDF, JSON, or text document to start asking questions - - - )} - - ) : ( - <> - {messages.map((msg, index) => ( - - - ( + - {msg.text} - - - - ))} - {isQuerying && ( - - - Searching document... - + + + {msg.text} + + + + ))} + {isQuerying && ( + + + + Searching your documents… + + + )} + )} - - )} - - - {/* Input Bar */} - - - {isQuerying ? ( - - ) : ( + + + + {/* Input bar */} + + + + - )} - - + + {/* Model Selection Sheets */} { onModelSelected={handleLLMModelSelected} onClose={() => setShowingLLMPicker(false)} /> - + ); }; // MARK: - Styles const styles = StyleSheet.create({ - container: { + fill: { flex: 1, - backgroundColor: Colors.backgroundPrimary, - }, - flex1: { - flex: 1, - }, - header: { - paddingHorizontal: Padding.padding16, - paddingVertical: Padding.padding12, - borderBottomWidth: StyleSheet.hairlineWidth, - borderBottomColor: Colors.borderLight, - }, - title: { - ...Typography.title2, - color: Colors.textPrimary, }, centered: { flex: 1, justifyContent: 'center', alignItems: 'center', - padding: Padding.padding20, - }, - loadingText: { - ...Typography.body, - color: Colors.textSecondary, - marginTop: Spacing.medium, + padding: 24, }, - errorTitle: { - ...Typography.headline, - color: Colors.primaryRed, - marginTop: Spacing.medium, + card: { + overflow: 'hidden', }, - errorHintText: { - ...Typography.body, - color: Colors.textSecondary, - marginTop: Spacing.small, - textAlign: 'center', - }, - - // Model Setup Section - modelSection: { - backgroundColor: Colors.backgroundPrimary, - borderBottomWidth: StyleSheet.hairlineWidth, - borderBottomColor: Colors.borderLight, - paddingHorizontal: Padding.padding16, - paddingVertical: Padding.padding12, - }, - modelRow: { + collapseHeader: { flexDirection: 'row', alignItems: 'center', - paddingVertical: Spacing.smallMedium, - gap: Spacing.mediumLarge, }, - modelLabel: { - ...Typography.subheadline, - color: Colors.textSecondary, + divider: { + height: StyleSheet.hairlineWidth, }, - modelRowRight: { - flex: 1, + setupRow: { flexDirection: 'row', alignItems: 'center', - justifyContent: 'flex-end', - gap: Spacing.xSmall, + gap: 12, }, - modelName: { - ...Typography.subheadline, - fontWeight: FontWeight.medium, - color: Colors.textPrimary, - maxWidth: 180, - }, - modelPlaceholder: { - ...Typography.subheadline, - color: Colors.primaryBlue, + setupRowText: { + flex: 1, + gap: 2, }, - - // Document Status Bar - documentBar: { - backgroundColor: Colors.backgroundPrimary, - borderBottomWidth: StyleSheet.hairlineWidth, - borderBottomColor: Colors.borderLight, - paddingHorizontal: Padding.padding16, - paddingVertical: Padding.padding12, + docsHeader: { + flexDirection: 'row', alignItems: 'center', }, - documentBarInner: { + docRow: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.mediumLarge, - width: '100%', - }, - documentBarText: { - ...Typography.subheadline, - color: Colors.textSecondary, - }, - documentName: { - ...Typography.subheadline, - fontWeight: FontWeight.medium, - color: Colors.textPrimary, - flex: 1, - }, - changeButton: { - ...Typography.caption, - color: Colors.primaryBlue, }, - selectDocButton: { + addDocButton: { flexDirection: 'row', alignItems: 'center', justifyContent: 'center', - backgroundColor: Colors.primaryBlue, - paddingHorizontal: Padding.padding24, - paddingVertical: Padding.padding12, - borderRadius: BorderRadius.large, - gap: Spacing.small, - }, - selectDocButtonDisabled: { - backgroundColor: Colors.textTertiary, }, - selectDocButtonText: { - ...Typography.headline, - color: '#fff', - }, - - // Error Banner - errorBanner: { + compactBar: { flexDirection: 'row', alignItems: 'center', - backgroundColor: `${Colors.primaryRed}15`, - paddingHorizontal: Padding.padding16, - paddingVertical: Spacing.smallMedium, - gap: Spacing.small, - }, - errorBannerText: { - ...Typography.caption, - color: Colors.primaryRed, - flex: 1, - }, - - // Messages Area - messagesArea: { - flex: 1, - backgroundColor: Colors.backgroundSecondary, + gap: 12, }, messagesContent: { - padding: Padding.padding16, flexGrow: 1, }, emptyState: { flex: 1, justifyContent: 'center', alignItems: 'center', - paddingVertical: Padding.padding60, - gap: Spacing.smallMedium, - }, - emptyTitle: { - ...Typography.headline, - color: Colors.textPrimary, - marginTop: Spacing.medium, - }, - emptySubtitle: { - ...Typography.subheadline, - color: Colors.textSecondary, - textAlign: 'center', - paddingHorizontal: Padding.padding40, + paddingVertical: 48, + paddingHorizontal: 32, }, - - // Message Bubbles - messageBubbleRow: { - marginBottom: Spacing.mediumLarge, + bubbleRow: { + flexDirection: 'row', }, - messageBubbleRowUser: { - alignItems: 'flex-end', + bubbleRowUser: { + justifyContent: 'flex-end', }, - messageBubbleRowAssistant: { - alignItems: 'flex-start', + bubbleRowAssistant: { + justifyContent: 'flex-start', }, - messageBubble: { + bubble: { maxWidth: '80%', - paddingHorizontal: Padding.padding14, - paddingVertical: Spacing.smallMedium, - borderRadius: BorderRadius.large, - }, - messageBubbleUser: { - backgroundColor: Colors.primaryBlue, - borderBottomRightRadius: 4, - }, - messageBubbleAssistant: { - backgroundColor: Colors.backgroundPrimary, - borderBottomLeftRadius: 4, - }, - messageBubbleText: { - ...Typography.body, - color: Colors.textPrimary, - lineHeight: 22, - }, - messageBubbleTextUser: { - color: '#fff', }, queryingRow: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.small, - paddingVertical: Spacing.small, - }, - queryingText: { - ...Typography.caption, - color: Colors.textSecondary, }, - - // Input Bar inputBar: { flexDirection: 'row', alignItems: 'flex-end', - backgroundColor: Colors.backgroundPrimary, - borderTopWidth: StyleSheet.hairlineWidth, - borderTopColor: Colors.borderLight, - paddingHorizontal: Padding.padding12, - paddingVertical: Spacing.smallMedium, - gap: Spacing.small, + gap: 8, }, - textInput: { + inputWrapper: { flex: 1, - ...Typography.body, - color: Colors.textPrimary, - minHeight: 36, - maxHeight: 100, - paddingHorizontal: Padding.padding12, - paddingVertical: Spacing.small, + borderRadius: 24, + paddingHorizontal: 16, + paddingVertical: 6, + }, + textInput: { + minHeight: 32, + maxHeight: 120, + paddingTop: 6, + paddingBottom: 6, }, sendButton: { - paddingBottom: 2, + width: 44, + height: 44, + borderRadius: 22, + justifyContent: 'center', + alignItems: 'center', }, }); diff --git a/examples/react-native/RunAnywhereAI/src/screens/STTScreen.tsx b/examples/react-native/RunAnywhereAI/src/screens/STTScreen.tsx index fadb822c67..3ad13a4f49 100644 --- a/examples/react-native/RunAnywhereAI/src/screens/STTScreen.tsx +++ b/examples/react-native/RunAnywhereAI/src/screens/STTScreen.tsx @@ -1,25 +1,3 @@ -/** - * STTScreen - Tab 1: Speech-to-Text - * - * Provides on-device speech recognition with real-time transcription. - * Matches iOS SpeechToTextView architecture and patterns. - * - * Features: - * - Batch mode: Record first, then transcribe - * - Live mode: Real-time transcription (streaming) - * - Model selection sheet - * - Audio level visualization - * - Model status banner - * - * Architecture: - * - Uses the SDK's AudioCaptureManager (16kHz mono Int16 PCM chunks) - * - Model loading via RunAnywhere.loadModel(ModelLoadRequest) - * - Transcription via RunAnywhere.transcribe() with proto-canonical STT options - * - Supports ONNX-based Whisper models - * - * Reference: iOS examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Voice/SpeechToTextView.swift - */ - import React, { useState, useCallback, useEffect, useRef } from 'react'; import { View, @@ -32,25 +10,19 @@ import { Animated, PermissionsAndroid, Linking, + ActivityIndicator, } from 'react-native'; -import Icon from 'react-native-vector-icons/Ionicons'; -import { - SafeAreaView, - useSafeAreaInsets, -} from 'react-native-safe-area-context'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { useFocusEffect } from '@react-navigation/native'; import { check, request, PERMISSIONS, RESULTS } from 'react-native-permissions'; -import { Colors } from '../theme/colors'; -import { Typography } from '../theme/typography'; -import { Spacing, Padding, BorderRadius, ButtonHeight } from '../theme/spacing'; -import { ModelStatusBanner, ModelRequiredOverlay } from '../components/common'; +import { Icon, useTheme } from '../theme/system'; +import { ModelRequiredOverlay } from '../components/common'; import { ModelSelectionSheet, ModelSelectionContext, } from '../components/model'; import { STTMode } from '../types/voice'; -// Import RunAnywhere SDK (Multi-Package Architecture) import { RunAnywhere, AudioCaptureManager, @@ -69,11 +41,10 @@ import { } from '@runanywhere/proto-ts/stt_options'; import { isModelLoadedForCategory } from '../utils/runAnywhereLifecycle'; -/** SDK capture emits 16kHz mono Int16 PCM. */ const CAPTURE_SAMPLE_RATE = 16000; const CAPTURE_BYTES_PER_MS = (CAPTURE_SAMPLE_RATE * 2) / 1000; +const BAR_COUNT = 12; -/** Wrap raw 16kHz mono Int16 PCM bytes in a WAV container. */ function wrapPcm16InWav(pcmChunks: Uint8Array[]): Uint8Array { const dataSize = pcmChunks.reduce((sum, chunk) => sum + chunk.length, 0); const wav = new Uint8Array(44 + dataSize); @@ -89,13 +60,13 @@ function wrapPcm16InWav(pcmChunks: Uint8Array[]): Uint8Array { view.setUint32(4, 36 + dataSize, true); writeString(8, 'WAVE'); writeString(12, 'fmt '); - view.setUint32(16, 16, true); // PCM fmt chunk size - view.setUint16(20, 1, true); // PCM - view.setUint16(22, 1, true); // mono + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); view.setUint32(24, CAPTURE_SAMPLE_RATE, true); - view.setUint32(28, CAPTURE_SAMPLE_RATE * 2, true); // byte rate - view.setUint16(32, 2, true); // block align - view.setUint16(34, 16, true); // bits per sample + view.setUint32(28, CAPTURE_SAMPLE_RATE * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); writeString(36, 'data'); view.setUint32(40, dataSize, true); @@ -115,18 +86,19 @@ async function transcribePcmChunks(pcmChunks: Uint8Array[]) { }); } -// Canonical SDK methods (Swift parity). const listModels = async (): Promise => (await RunAnywhere.listModels()).models?.models ?? []; const loadModelWithRequest = RunAnywhere.loadModel; export const STTScreen: React.FC = () => { - // State + const { colors, typography, dimens } = useTheme(); + const insets = useSafeAreaInsets(); + const [mode, setMode] = useState(STTMode.Batch); const [isRecording, setIsRecording] = useState(false); const [isProcessing, setIsProcessing] = useState(false); const [transcript, setTranscript] = useState(''); - const [partialTranscript, setPartialTranscript] = useState(''); // For live mode - current chunk + const [partialTranscript, setPartialTranscript] = useState(''); const [confidence, setConfidence] = useState(null); const [currentModel, setCurrentModel] = useState(null); const [isModelLoading, setIsModelLoading] = useState(false); @@ -135,10 +107,6 @@ export const STTScreen: React.FC = () => { const [audioLevel, setAudioLevel] = useState(0); const [showModelSelection, setShowModelSelection] = useState(false); - // Safe area insets for header status bar handling - const insets = useSafeAreaInsets(); - - // SDK audio capture manager (one per screen instance) const captureManagerRef = useRef(null); const getCaptureManager = (): AudioCaptureManager => { if (!captureManagerRef.current) { @@ -147,36 +115,21 @@ export const STTScreen: React.FC = () => { return captureManagerRef.current; }; - // Accumulated 16kHz mono Int16 PCM chunks for the in-flight recording const pcmChunksRef = useRef([]); const pcmBytesRef = useRef(0); - - // Live mode accumulated transcript ref const accumulatedTranscriptRef = useRef(''); - - // Live mode streaming refs const liveAudioStreamRef = useRef(null); const liveTranscriptionTaskRef = useRef | null>(null); const isLiveRecordingRef = useRef(false); - // Animation for recording indicator const pulseAnim = useRef(new Animated.Value(1)).current; - // Start pulse animation when recording useEffect(() => { if (isRecording) { const pulse = Animated.loop( Animated.sequence([ - Animated.timing(pulseAnim, { - toValue: 1.3, - duration: 500, - useNativeDriver: true, - }), - Animated.timing(pulseAnim, { - toValue: 1, - duration: 500, - useNativeDriver: true, - }), + Animated.timing(pulseAnim, { toValue: 1.3, duration: 500, useNativeDriver: true }), + Animated.timing(pulseAnim, { toValue: 1, duration: 500, useNativeDriver: true }), ]) ); pulse.start(); @@ -186,112 +139,51 @@ export const STTScreen: React.FC = () => { } }, [isRecording, pulseAnim]); - // Cleanup on unmount useEffect(() => { return () => { - // Stop live recording if active isLiveRecordingRef.current = false; liveAudioStreamRef.current?.close(); liveAudioStreamRef.current = null; - // Stop the SDK capture manager - try { - captureManagerRef.current?.stopRecording(); - } catch { - // Ignore — capture may never have started - } + try { captureManagerRef.current?.stopRecording(); } catch {} pcmChunksRef.current = []; pcmBytesRef.current = 0; }; }, []); - /** - * Load available models and check for loaded model - * Called on mount and when screen comes into focus - */ const loadModels = useCallback(async () => { try { - // Get available STT models from catalog const allModels = await listModels(); - // Filter by category (speech-recognition) matching SDK's ModelCategory const sttModels = allModels.filter( - (m: SDKModelInfo) => - m.category === ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION + (m: SDKModelInfo) => m.category === ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION ); setAvailableModels(sttModels); - - // Log downloaded status for debugging - const downloadedModels = sttModels.filter((m) => m.isDownloaded); - console.warn( - '[STTScreen] Available STT models:', - sttModels.map((m) => `${m.id} (downloaded: ${m.isDownloaded})`) - ); - console.warn( - '[STTScreen] Downloaded STT models:', - downloadedModels.map((m) => m.id) - ); - - // Ask the SDK for the loaded STT model directly. if (!currentModel) { - const loaded = await RunAnywhere.modelInfoForCategory( - ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION - ); - if (loaded) { - setCurrentModel(loaded); - console.warn('[STTScreen] Loaded STT model:', loaded.name); - } + const loaded = await RunAnywhere.modelInfoForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION); + if (loaded) setCurrentModel(loaded); } } catch (error) { console.warn('[STTScreen] Error loading models:', error); } }, [currentModel]); - // Refresh models when screen comes into focus - // This ensures we pick up any models downloaded in the Settings tab useFocusEffect( - useCallback(() => { - console.warn('[STTScreen] Screen focused - refreshing models'); - loadModels(); - }, [loadModels]) + useCallback(() => { loadModels(); }, [loadModels]) ); - /** - * Handle model selection - opens model selection sheet - */ - const handleSelectModel = useCallback(() => { - setShowModelSelection(true); - }, []); + const handleSelectModel = useCallback(() => { setShowModelSelection(true); }, []); - /** - * Handle model selected from the sheet - */ const handleModelSelected = useCallback(async (model: SDKModelInfo) => { - // Close the modal first to prevent UI issues setShowModelSelection(false); - // Then load the model await loadModel(model); }, []); - /** - * Load a model from its info via the canonical lifecycle ABI. - * - * Path-first loading was removed in V2 — model ID is the canonical handle - * and the native registry resolves the artifact path internally. - */ const loadModel = async (model: SDKModelInfo) => { try { setIsModelLoading(true); - console.warn( - `[STTScreen] Loading model: ${model.id} (registry will resolve path)` - ); - if (!model.isDownloaded && !model.localPath) { - Alert.alert( - 'Error', - 'Model has not been downloaded. Open the model picker to download it first.' - ); + Alert.alert('Error', 'Model has not been downloaded. Open the model picker to download it first.'); return; } - const result = await loadModelWithRequest( ModelLoadRequest.fromPartial({ modelId: model.id, @@ -300,44 +192,23 @@ export const STTScreen: React.FC = () => { validateAvailability: true, }) ); - if (result.success) { - const isLoaded = await isModelLoadedForCategory( - ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION - ); + const isLoaded = await isModelLoadedForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION); if (isLoaded) { - const loaded = await RunAnywhere.modelInfoForCategory( - ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION - ); + const loaded = await RunAnywhere.modelInfoForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION); setCurrentModel(loaded ?? model); - console.warn( - `[STTScreen] Model ${model.name} loaded successfully, currentModel set` - ); - } else { - console.warn( - '[STTScreen] Model reported success but lifecycle currentModel() returned no STT model' - ); } } else { - const error = - result.errorMessage || - 'Native model lifecycle returned an unsuccessful load result'; - Alert.alert( - 'Error', - `Failed to load model: ${error || 'Unknown error'}` - ); + const error = result.errorMessage || 'Native model lifecycle returned an unsuccessful load result'; + Alert.alert('Error', `Failed to load model: ${error || 'Unknown error'}`); } } catch (error) { - console.error('[STTScreen] Error loading model:', error); Alert.alert('Error', `Failed to load model: ${error}`); } finally { setIsModelLoading(false); } }; - /** - * Format duration in MM:SS - */ const formatDuration = (ms: number): string => { const totalSeconds = Math.floor(ms / 1000); const minutes = Math.floor(totalSeconds / 60); @@ -345,66 +216,38 @@ export const STTScreen: React.FC = () => { return `${minutes}:${seconds.toString().padStart(2, '0')}`; }; - /** - * Request microphone permission - */ const requestMicrophonePermission = async (): Promise => { try { if (Platform.OS === 'ios') { const status = await check(PERMISSIONS.IOS.MICROPHONE); - console.warn('[STTScreen] iOS microphone permission status:', status); - - if (status === RESULTS.GRANTED) { - return true; - } - + if (status === RESULTS.GRANTED) return true; if (status === RESULTS.DENIED) { const result = await request(PERMISSIONS.IOS.MICROPHONE); - console.warn( - '[STTScreen] iOS microphone permission request result:', - result - ); return result === RESULTS.GRANTED; } - if (status === RESULTS.BLOCKED) { - Alert.alert( - 'Microphone Permission Required', - 'Please enable microphone access in Settings to use speech-to-text.', - [ - { text: 'Cancel', style: 'cancel' }, - { text: 'Open Settings', onPress: () => Linking.openSettings() }, - ] - ); + Alert.alert('Microphone Permission Required', 'Please enable microphone access in Settings.', [ + { text: 'Cancel', style: 'cancel' }, + { text: 'Open Settings', onPress: () => Linking.openSettings() }, + ]); return false; } - return false; } else { - // Android - const granted = await PermissionsAndroid.request( - PermissionsAndroid.PERMISSIONS.RECORD_AUDIO, - { - title: 'Microphone Permission', - message: - 'RunAnywhereAI needs access to your microphone for speech-to-text.', - buttonNeutral: 'Ask Me Later', - buttonNegative: 'Cancel', - buttonPositive: 'OK', - } - ); + const granted = await PermissionsAndroid.request(PermissionsAndroid.PERMISSIONS.RECORD_AUDIO, { + title: 'Microphone Permission', + message: 'RunAnywhereAI needs access to your microphone for speech-to-text.', + buttonNeutral: 'Ask Me Later', + buttonNegative: 'Cancel', + buttonPositive: 'OK', + }); return granted === PermissionsAndroid.RESULTS.GRANTED; } } catch (error) { - console.error('[STTScreen] Permission request error:', error); return false; } }; - /** - * Begin SDK microphone capture, accumulating PCM chunks in memory. - * Audio level + recording duration come straight from the capture stream. - */ const beginCapture = async (onChunk?: (chunk: Uint8Array) => void) => { const capture = getCaptureManager(); pcmChunksRef.current = []; @@ -421,9 +264,6 @@ export const STTScreen: React.FC = () => { }); }; - /** - * Drain the accumulated PCM chunks (resets the buffer). - */ const drainCapturedChunks = (): Uint8Array[] => { const chunks = pcmChunksRef.current; pcmChunksRef.current = []; @@ -431,64 +271,33 @@ export const STTScreen: React.FC = () => { return chunks; }; - /** - * Start recording audio (batch mode) - */ const startRecording = async () => { try { - console.warn('[STTScreen] Starting recording...'); - - // Request microphone permission first const hasPermission = await requestMicrophonePermission(); - if (!hasPermission) { - console.warn('[STTScreen] Microphone permission denied'); - return; - } - - console.warn('[STTScreen] Starting SDK audio capture...'); + if (!hasPermission) return; await beginCapture(); - setIsRecording(true); setTranscript(''); setConfidence(null); } catch (error) { - console.error('[STTScreen] Error starting recording:', error); Alert.alert('Recording Error', `Failed to start recording: ${error}`); } }; - /** - * Stop recording and transcribe the in-memory PCM buffer. - */ const stopRecordingAndTranscribe = async () => { try { - console.warn('[STTScreen] Stopping recording...'); - getCaptureManager().stopRecording(); setIsRecording(false); setIsProcessing(true); const chunks = drainCapturedChunks(); const totalBytes = chunks.reduce((sum, c) => sum + c.length, 0); - console.warn('[STTScreen] Recording size:', totalBytes, 'bytes'); + if (totalBytes < 1000) throw new Error('Recording too short'); - if (totalBytes < 1000) { - throw new Error('Recording too short'); - } - - // Check if model is loaded - const isLoaded = await isModelLoadedForCategory( - ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION - ); - if (!isLoaded) { - throw new Error('STT model not loaded'); - } + const isLoaded = await isModelLoadedForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION); + if (!isLoaded) throw new Error('STT model not loaded'); - console.warn('[STTScreen] Starting transcription...'); const result = await transcribePcmChunks(chunks); - - console.warn('[STTScreen] Transcription result:', result); - if (result.text) { setTranscript(result.text); setConfidence(result.confidence); @@ -496,9 +305,7 @@ export const STTScreen: React.FC = () => { setTranscript('(No speech detected)'); } } catch (error: unknown) { - console.error('[STTScreen] Transcription error:', error); - const errorMessage = - error instanceof Error ? error.message : String(error); + const errorMessage = error instanceof Error ? error.message : String(error); Alert.alert('Transcription Error', errorMessage); setTranscript(''); } finally { @@ -508,28 +315,14 @@ export const STTScreen: React.FC = () => { } }; - /** - * Start live transcription mode - */ const startLiveTranscription = async () => { try { - console.warn('[STTScreen] Starting live transcription stream...'); - const hasPermission = await requestMicrophonePermission(); - if (!hasPermission) { - console.warn('[STTScreen] Microphone permission denied'); - return; - } + if (!hasPermission) return; - const isLoaded = await isModelLoadedForCategory( - ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION - ); - if (!isLoaded) { - Alert.alert('Model Not Loaded', 'Please load an STT model first.'); - return; - } + const isLoaded = await isModelLoadedForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION); + if (!isLoaded) { Alert.alert('Model Not Loaded', 'Please load an STT model first.'); return; } - // Reset state accumulatedTranscriptRef.current = ''; setTranscript(''); setPartialTranscript('Listening...'); @@ -545,25 +338,16 @@ export const STTScreen: React.FC = () => { sampleRate: CAPTURE_SAMPLE_RATE, }); liveTranscriptionTaskRef.current = consumeLiveTranscription(partials); - await beginCapture((chunk) => audioStream.push(chunk)); setIsRecording(true); - - console.warn('[STTScreen] Live transcription started'); } catch (error) { - console.error('[STTScreen] Error starting live transcription:', error); - Alert.alert( - 'Recording Error', - `Failed to start live transcription: ${error}` - ); + Alert.alert('Recording Error', `Failed to start live transcription: ${error}`); isLiveRecordingRef.current = false; liveAudioStreamRef.current?.close(); } }; - const consumeLiveTranscription = async ( - partials: AsyncIterable - ) => { + const consumeLiveTranscription = async (partials: AsyncIterable) => { const iterator = partials[Symbol.asyncIterator](); try { let step = await iterator.next(); @@ -590,28 +374,16 @@ export const STTScreen: React.FC = () => { } }; - /** - * Stop live transcription and transcribe any remaining audio. - */ const stopLiveTranscription = async () => { - console.warn('[STTScreen] Stopping live transcription...'); isLiveRecordingRef.current = false; - try { setIsProcessing(true); setPartialTranscript('Finalizing...'); - getCaptureManager().stopRecording(); liveAudioStreamRef.current?.close(); await liveTranscriptionTaskRef.current; liveAudioStreamRef.current = null; liveTranscriptionTaskRef.current = null; - - console.warn('[STTScreen] Live transcription stopped'); - console.warn( - '[STTScreen] Final transcript:', - accumulatedTranscriptRef.current - ); } catch (error) { console.error('[STTScreen] Error stopping live transcription:', error); } finally { @@ -623,35 +395,18 @@ export const STTScreen: React.FC = () => { } }; - /** - * Toggle recording - */ const handleToggleRecording = useCallback(async () => { if (isRecording) { - // Stop recording based on mode - if (mode === STTMode.Live) { - await stopLiveTranscription(); - } else { - await stopRecordingAndTranscribe(); - } + if (mode === STTMode.Live) await stopLiveTranscription(); + else await stopRecordingAndTranscribe(); } else { - if (!currentModel) { - Alert.alert('Model Required', 'Please select an STT model first.'); - return; - } - // Start recording based on mode - if (mode === STTMode.Live) { - await startLiveTranscription(); - } else { - await startRecording(); - } + if (!currentModel) { Alert.alert('Model Required', 'Please select an STT model first.'); return; } + if (mode === STTMode.Live) await startLiveTranscription(); + else await startRecording(); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [isRecording, currentModel, mode]); - /** - * Clear transcript - */ const handleClear = useCallback(() => { setTranscript(''); setPartialTranscript(''); @@ -659,463 +414,334 @@ export const STTScreen: React.FC = () => { accumulatedTranscriptRef.current = ''; }, []); - /** - * Render mode selector - */ - const renderModeSelector = () => ( - - setMode(STTMode.Batch)} - activeOpacity={0.7} - > - - Batch - - - setMode(STTMode.Live)} - activeOpacity={0.7} - > - - Live - - - - ); + const busy = isRecording || isProcessing; + const recordButtonColor = !currentModel + ? colors.surfaceContainerHighest + : isRecording + ? colors.error + : colors.primary; - /** - * Render mode description - */ - const renderModeDescription = () => ( - - - - {mode === STTMode.Batch - ? 'Record audio, then transcribe all at once for best accuracy.' - : 'Transcribes every few seconds while you speak.'} - - - ); + const modeDescriptionText = + mode === STTMode.Batch + ? 'Tap record, speak, then tap again to transcribe — all on-device.' + : 'Live mode transcribes each phrase as you pause. Tap to start.'; - /** - * Render header - */ - const renderHeader = () => ( - - Speech to Text - {transcript && ( - - - - )} - - ); - - /** - * Render audio level indicator - */ - const renderAudioLevel = () => { - if (!isRecording) return null; - - return ( - - - - - - {formatDuration(recordingDuration)} - - - ); - }; - - // Show model required overlay if no model if (!currentModel && !isModelLoading) { return ( - - {renderHeader()} - - {/* Model Selection Sheet */} + + + + Speech to Text + + + setShowModelSelection(false)} onModelSelected={handleModelSelected} /> - + ); } return ( - - {renderHeader()} - - {/* Model Status Banner */} - - - {/* Mode Selector */} - {renderModeSelector()} - - {/* Mode Description */} - {renderModeDescription()} - - {/* Audio Level Indicator */} - {renderAudioLevel()} - - {/* Transcription Area */} + - {transcript || partialTranscript ? ( - <> - - {transcript} - {partialTranscript ? ( - - {' '} - {partialTranscript} - - ) : null} - - {confidence !== null && !isRecording && ( - - Confidence: - - {Math.round(confidence * 100)}% - - - )} - {isRecording && mode === STTMode.Live && ( - - + + + + + Speech to Text + + + {modeDescriptionText} + + + + {/* Mode Selector */} + + + {([STTMode.Batch, STTMode.Live] as STTMode[]).map((option) => { + const selected = option === mode; + return ( + - Live transcribing... - - )} - - ) : isProcessing ? ( - - - Transcribing audio... + onPress={() => !busy && !selected && setMode(option)} + activeOpacity={0.8} + disabled={busy} + > + + {option === STTMode.Batch ? 'Batch' : 'Live'} + + + ); + })} - ) : isRecording ? ( - - - - {mode === STTMode.Live ? 'Live transcribing...' : 'Listening...'} + + + {/* Model Card */} + + + + Model + + {isModelLoading ? 'Loading…' : (currentModel?.name ?? 'Select a model')} - - {mode === STTMode.Live - ? 'Text will appear as you speak' - : 'Tap the button when done speaking'} + + + + + {/* Record Button */} + + + + + + + {/* Status Line */} + + {isRecording ? ( + + + + {mode === STTMode.Live ? 'Listening — pause to transcribe' : 'Recording…'} + + + {formatDuration(recordingDuration)} + + + ) : isProcessing ? ( + + + + Transcribing… + + + ) : ( + + {currentModel ? 'Tap to record' : 'Select a model to begin'} + )} + + + {/* Transcript Card */} + {(transcript.length > 0 || partialTranscript.length > 0) && ( + + + Transcript + {transcript.length > 0 && ( + + + + )} + + + + {transcript} + {partialTranscript ? ( + + {transcript ? ' ' : ''}{partialTranscript} + + ) : null} + + - ) : ( - - - Tap the microphone to start + )} + + {/* No speech card — show after a completed transcription with empty result */} + {transcript === '' && !isRecording && !isProcessing && confidence !== null && ( + + Transcript + + No speech recognized. + )} - - {/* Record Button */} - - - - - - {isRecording ? 'Tap to stop' : 'Tap to record'} - - + {/* Audio Stats */} + {confidence !== null && !isRecording && !isProcessing && ( + + Audio stats + + + Confidence + + {Math.round(confidence * 100)}% + + + + + )} + + + - {/* Model Selection Sheet */} setShowModelSelection(false)} onModelSelected={handleModelSelected} /> - + + ); +}; + +interface LevelBarsProps { + audioLevel: number; + colors: ReturnType['colors']; + dimens: ReturnType['dimens']; +} + +const LevelBars: React.FC = ({ audioLevel, colors, dimens }) => { + const active = Math.floor(audioLevel * BAR_COUNT); + return ( + + {Array.from({ length: BAR_COUNT }, (_, i) => ( + + ))} + ); }; const styles = StyleSheet.create({ container: { flex: 1, - backgroundColor: Colors.backgroundPrimary, - }, - header: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - paddingHorizontal: Padding.padding16, - paddingTop: 0, - paddingBottom: Padding.padding12, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, }, - title: { - ...Typography.title2, - color: Colors.textPrimary, + scroll: { + flex: 1, }, - clearButton: { - padding: Spacing.small, + content: { + alignItems: 'center', }, - modeSelector: { - flexDirection: 'row', - marginHorizontal: Padding.padding16, - marginTop: Spacing.medium, - backgroundColor: Colors.backgroundSecondary, - borderRadius: BorderRadius.regular, - padding: Spacing.xSmall, + topBar: { + paddingHorizontal: 16, + paddingBottom: 8, }, - modeButton: { - flex: 1, - paddingVertical: Spacing.smallMedium, + headerSection: { alignItems: 'center', - borderRadius: BorderRadius.small, + width: '100%', }, - modeButtonActive: { - backgroundColor: Colors.backgroundPrimary, - shadowColor: '#000', - shadowOffset: { width: 0, height: 1 }, - shadowOpacity: 0.1, - shadowRadius: 2, - elevation: 2, - }, - modeButtonText: { - ...Typography.subheadline, - color: Colors.textSecondary, + iconCircle: { + width: 64, + height: 64, + borderRadius: 32, + alignItems: 'center', + justifyContent: 'center', }, - modeButtonTextActive: { - color: Colors.textPrimary, - fontWeight: '600', + modeContainer: { + width: '100%', + borderRadius: 999, }, - modeDescription: { + modePad: { flexDirection: 'row', - alignItems: 'center', - gap: Spacing.smallMedium, - marginHorizontal: Padding.padding16, - marginTop: Spacing.medium, - padding: Padding.padding12, - backgroundColor: Colors.badgeBlue, - borderRadius: BorderRadius.regular, + padding: 4, }, - modeDescriptionText: { - ...Typography.footnote, - color: Colors.primaryBlue, + modeTab: { flex: 1, + paddingVertical: 8, + borderRadius: 999, + alignItems: 'center', + justifyContent: 'center', }, - audioLevelContainer: { + modelCard: { + width: '100%', flexDirection: 'row', alignItems: 'center', - marginHorizontal: Padding.padding16, - marginTop: Spacing.medium, - gap: Spacing.medium, + gap: 12, + padding: 16, + borderRadius: 20, }, - audioLevelTrack: { + modelCardText: { flex: 1, - height: 4, - backgroundColor: Colors.backgroundGray5, - borderRadius: 2, - overflow: 'hidden', + gap: 2, }, - audioLevelFill: { - height: '100%', - backgroundColor: Colors.primaryGreen, - }, - recordingTime: { - ...Typography.caption, - color: Colors.textSecondary, - minWidth: 40, - textAlign: 'right', - }, - transcriptContainer: { - flex: 1, - marginHorizontal: Padding.padding16, - marginTop: Spacing.large, - }, - transcriptContent: { - flex: 1, - }, - transcriptText: { - ...Typography.body, - color: Colors.textPrimary, - lineHeight: 26, - }, - partialTranscript: { - color: Colors.textSecondary, - fontStyle: 'italic', - }, - liveIndicator: { - flexDirection: 'row', + recordSection: { alignItems: 'center', - gap: Spacing.small, - marginTop: Spacing.medium, - paddingTop: Spacing.medium, - borderTopWidth: 1, - borderTopColor: Colors.borderLight, }, - liveDot: { - width: 8, - height: 8, - borderRadius: 4, - backgroundColor: Colors.primaryRed, - }, - liveText: { - ...Typography.footnote, - color: Colors.textSecondary, - }, - confidenceContainer: { - flexDirection: 'row', + recordButton: { + width: 96, + height: 96, + borderRadius: 48, alignItems: 'center', - gap: Spacing.small, - marginTop: Spacing.medium, - paddingTop: Spacing.medium, - borderTopWidth: 1, - borderTopColor: Colors.borderLight, - }, - confidenceLabel: { - ...Typography.footnote, - color: Colors.textSecondary, - }, - confidenceValue: { - ...Typography.footnote, - color: Colors.primaryGreen, - fontWeight: '600', - }, - processingContainer: { - flex: 1, justifyContent: 'center', - alignItems: 'center', - gap: Spacing.medium, }, - processingText: { - ...Typography.body, - color: Colors.textSecondary, - }, - recordingContainer: { - flex: 1, - justifyContent: 'center', + statusLine: { + width: '100%', alignItems: 'center', - gap: Spacing.medium, + minHeight: 40, }, - recordingIndicator: { - width: 20, - height: 20, - borderRadius: 10, - backgroundColor: Colors.primaryRed, - }, - recordingText: { - ...Typography.body, - color: Colors.textPrimary, + statusCenter: { + alignItems: 'center', }, - recordingHint: { - ...Typography.footnote, - color: Colors.textSecondary, + statusRow: { + flexDirection: 'row', + alignItems: 'center', }, - emptyState: { - flex: 1, - justifyContent: 'center', + levelBars: { + flexDirection: 'row', alignItems: 'center', - gap: Spacing.medium, }, - emptyText: { - ...Typography.body, - color: Colors.textSecondary, + levelBar: { + width: 5, }, - controlsContainer: { - alignItems: 'center', - paddingVertical: Padding.padding20, - paddingBottom: Padding.padding40, + labeledSection: { + width: '100%', }, - recordButton: { - width: ButtonHeight.large, - height: ButtonHeight.large, - borderRadius: ButtonHeight.large / 2, - backgroundColor: Colors.primaryBlue, - justifyContent: 'center', + labelRow: { + flexDirection: 'row', alignItems: 'center', - shadowColor: Colors.primaryBlue, - shadowOffset: { width: 0, height: 4 }, - shadowOpacity: 0.3, - shadowRadius: 8, - elevation: 8, + justifyContent: 'space-between', }, - recordButtonActive: { - backgroundColor: Colors.primaryRed, - shadowColor: Colors.primaryRed, + card: { + borderRadius: 20, + padding: 16, }, - recordButtonLabel: { - ...Typography.footnote, - color: Colors.textSecondary, - marginTop: Spacing.smallMedium, + statRow: { + flexDirection: 'row', + justifyContent: 'space-between', + alignItems: 'center', }, }); diff --git a/examples/react-native/RunAnywhereAI/src/screens/SettingsScreen.tsx b/examples/react-native/RunAnywhereAI/src/screens/SettingsScreen.tsx index 06fa8526a3..df15cfe60b 100644 --- a/examples/react-native/RunAnywhereAI/src/screens/SettingsScreen.tsx +++ b/examples/react-native/RunAnywhereAI/src/screens/SettingsScreen.tsx @@ -1,26 +1,3 @@ -/** - * SettingsScreen - Tab 4: Settings & Storage - * - * Provides SDK configuration, model management, and storage overview. - * Matches iOS CombinedSettingsView architecture and patterns. - * - * Features: - * - Generation settings (temperature, max tokens) - * - API configuration - * - Storage overview (total usage, available space, models storage) - * - Downloaded models list with delete functionality - * - Storage management (clear cache, clean temp files) - * - SDK info (version, capabilities, loaded models) - * - * Architecture: - * - Fetches SDK state via RunAnywhere methods - * - Shows available vs downloaded models - * - Manages model downloads and deletions - * - Displays backend info and capabilities - * - * Reference: iOS examples/ios/RunAnywhereAI/RunAnywhereAI/Features/Settings/CombinedSettingsView.swift - */ - import React, { useState, useCallback, useEffect, useRef } from 'react'; import { View, @@ -31,19 +8,16 @@ import { Alert, TextInput, Modal, + Switch, + ActivityIndicator, + type DimensionValue, } from 'react-native'; import AsyncStorage from '@react-native-async-storage/async-storage'; -import Icon from 'react-native-vector-icons/Ionicons'; -import { - SafeAreaView, - useSafeAreaInsets, -} from 'react-native-safe-area-context'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { useNavigation } from '@react-navigation/native'; import type { NativeStackNavigationProp } from '@react-navigation/native-stack'; -import type { SettingsStackParamList } from '../types'; -import { Colors } from '../theme/colors'; -import { Typography } from '../theme/typography'; -import { Spacing, Padding, BorderRadius } from '../theme/spacing'; +import { ROUTES } from '../navigation/routes'; +import type { RootStackParamList } from '../navigation/navigation.types'; import type { StorageInfo } from '../types/settings'; import { RoutingPolicy, @@ -59,8 +33,8 @@ import { getPrimaryFramework, } from '../utils/modelDisplay'; import { registerDemoTools as registerSharedDemoTools } from '../utils/chatSampleTools'; +import { Icon, useTheme } from '../theme/system'; -// Import RunAnywhere SDK (Multi-Package Architecture) import { RunAnywhere } from '@runanywhere/core'; import { ModelCategory, @@ -72,14 +46,12 @@ import { unloadModelsForCategory, } from '../utils/runAnywhereLifecycle'; -// Canonical SDK methods (Swift parity). const downloadModelStreamHelper = RunAnywhere.downloadModelStream; const listModels = async (): Promise => (await RunAnywhere.listModels()).models?.models ?? []; const listDownloadedModels = async (): Promise => (await RunAnywhere.downloadedModels()).models?.models ?? []; -// Storage keys for API configuration const STORAGE_KEYS = APP_STORAGE_KEYS; function hasUsableBackendConfig(options: { @@ -92,9 +64,6 @@ function hasUsableBackendConfig(options: { return baseURL.startsWith('http://') || baseURL.startsWith('https://'); } -/** - * Get stored API key (for use at app launch) - */ export const getStoredApiKey = async (): Promise => { try { return await AsyncStorage.getItem(STORAGE_KEYS.API_KEY); @@ -103,10 +72,6 @@ export const getStoredApiKey = async (): Promise => { } }; -/** - * Get stored base URL (for use at app launch) - * Automatically adds https:// if no scheme is present - */ export const getStoredBaseURL = async (): Promise => { try { const value = await AsyncStorage.getItem(STORAGE_KEYS.BASE_URL); @@ -121,16 +86,12 @@ export const getStoredBaseURL = async (): Promise => { } }; -/** - * Check if custom configuration is set - */ export const hasCustomConfiguration = async (): Promise => { const apiKey = await getStoredApiKey(); const baseURL = await getStoredBaseURL(); return hasUsableBackendConfig({ apiKey, baseURL }); }; -// Default storage info const DEFAULT_STORAGE_INFO: StorageInfo = { totalStorage: 256 * 1024 * 1024 * 1024, appStorage: 0, @@ -139,9 +100,6 @@ const DEFAULT_STORAGE_INFO: StorageInfo = { freeSpace: 100 * 1024 * 1024 * 1024, }; -/** - * Format bytes to human readable - */ const formatBytes = (bytes: number): string => { if (bytes === 0) return '0 B'; const k = 1024; @@ -150,16 +108,269 @@ const formatBytes = (bytes: number): string => { return `${parseFloat((bytes / Math.pow(k, i)).toFixed(2))} ${sizes[i]}`; }; +// --------------------------------------------------------------------------- +// Sub-components +// --------------------------------------------------------------------------- + +interface SectionProps { + title: string; + children: React.ReactNode; +} + +const Section: React.FC = ({ title, children }) => { + const { colors, typography, dimens } = useTheme(); + return ( + + + {title} + + + {children} + + + ); +}; + +interface SliderRowProps { + label: string; + value: number; + min: number; + max: number; + step: number; + formatValue: (v: number) => string; + onChange: (v: number) => void; +} + +const SliderRow: React.FC = ({ + label, + value, + min, + max, + step, + formatValue, + onChange, +}) => { + const { colors, typography, dimens } = useTheme(); + const fillPct = `${Math.round( + ((value - min) / (max - min)) * 100 + )}%` as DimensionValue; + return ( + + + {label} + + {formatValue(value)} + + + + onChange(Math.max(min, Math.round((value - step) * 100) / 100))} + style={[styles.sliderBtn, { backgroundColor: colors.surfaceContainerHighest }]} + > + − + + + + + onChange(Math.min(max, Math.round((value + step) * 100) / 100))} + style={[styles.sliderBtn, { backgroundColor: colors.surfaceContainerHighest }]} + > + + + + + + ); +}; + +interface ToggleRowProps { + label: string; + description: string; + value: boolean; + onChange: (v: boolean) => void; +} + +const ToggleRow: React.FC = ({ label, description, value, onChange }) => { + const { colors, typography } = useTheme(); + return ( + + + {label} + {description} + + + + ); +}; + +interface InfoRowProps { + label: string; + value: string; +} + +const InfoRow: React.FC = ({ label, value }) => { + const { colors, typography } = useTheme(); + return ( + + {label} + {value} + + ); +}; + +interface DownloadedModelRowProps { + model: ModelInfo; + busy: boolean; + onDelete: () => void; +} + +const DownloadedModelRow: React.FC = ({ model, busy, onDelete }) => { + const { colors, typography, dimens } = useTheme(); + const sizeBytes = getModelDownloadSizeBytes(model); + return ( + + + + + {model.name} + + + {formatBytes(sizeBytes)} + + + {busy ? ( + + ) : ( + + + + )} + + + ); +}; + +interface CatalogModelRowProps { + model: ModelInfo; + isDownloaded: boolean; + isDownloading: boolean; + downloadProgress: number; + onAction: () => void; +} + +const CatalogModelRow: React.FC = ({ + model, + isDownloaded, + isDownloading, + downloadProgress, + onAction, +}) => { + const { colors, typography, dimens } = useTheme(); + const frameworkName = getFrameworkDisplayName(getPrimaryFramework(model)); + const sizeBytes = getModelDownloadSizeBytes(model); + + return ( + + + + {model.name} + + {!!model.metadata?.description && ( + + {model.metadata.description} + + )} + + + {formatBytes(sizeBytes)} + + {!!frameworkName && ( + + {frameworkName} + + )} + + {isDownloading && ( + + + + + + {(downloadProgress * 100).toFixed(0)}% + + + )} + + + {isDownloaded ? ( + + ) : isDownloading ? ( + + ) : ( + + )} + + + ); +}; + +// --------------------------------------------------------------------------- +// Main screen +// --------------------------------------------------------------------------- + export const SettingsScreen: React.FC = () => { - // Safe area insets for header status bar handling const insets = useSafeAreaInsets(); - const navigation = - useNavigation>(); + const navigation = useNavigation>(); + const { colors, typography, dimens } = useTheme(); - // Settings state - // NOTE: several state hooks below are intentionally retained for upcoming - // settings UI (routing policy, capability flags, etc.). Prefixed with `_` - // to silence unused-vars warnings until the UI consumes them. const [_routingPolicy, setRoutingPolicy] = useState( RoutingPolicy.ROUTING_POLICY_UNSPECIFIED ); @@ -167,40 +378,28 @@ export const SettingsScreen: React.FC = () => { const [maxTokens, setMaxTokens] = useState(1000); const [systemPrompt, setSystemPrompt] = useState(''); const [thinkingModeEnabled, setThinkingModeEnabled] = useState(false); - const [apiKeyConfigured, setApiKeyConfigured] = useState(false); // API Configuration state + const [apiKeyConfigured, setApiKeyConfigured] = useState(false); const [apiKey, setApiKey] = useState(''); const [baseURL, setBaseURL] = useState(''); const [isBaseURLConfigured, setIsBaseURLConfigured] = useState(false); const [showApiConfigModal, setShowApiConfigModal] = useState(false); - const [showPassword, setShowPassword] = useState(false); // Storage state + const [showPassword, setShowPassword] = useState(false); - const [storageInfo, setStorageInfo] = - useState(DEFAULT_STORAGE_INFO); + const [storageInfo, setStorageInfo] = useState(DEFAULT_STORAGE_INFO); const [_isRefreshing, setIsRefreshing] = useState(false); - const [sdkVersion, setSdkVersion] = useState('0.1.0'); // SDK State - - // Model catalog state + const [sdkVersion, setSdkVersion] = useState('0.1.0'); const [availableModels, setAvailableModels] = useState([]); - const [downloadingModels, setDownloadingModels] = useState< - Record - >({}); - const [downloadedModels, setDownloadedModels] = useState([]); // Tool calling state - const downloadIteratorsRef = useRef< - Record> - >({}); + const [downloadingModels, setDownloadingModels] = useState>({}); + const [downloadedModels, setDownloadedModels] = useState([]); + const downloadIteratorsRef = useRef>>({}); const [toolCallingEnabled, setToolCallingEnabled] = useState(false); const [registeredTools, setRegisteredTools] = useState< - Array<{ - name: string; - description: string; - parameters: Array<{ name: string }>; - }> - >([]); // Capability names mapping - - // Kept for reference; not yet rendered in the capabilities grid below. + Array<{ name: string; description: string; parameters: Array<{ name: string }> }> + >([]); + const _capabilityNames: Record = { 0: 'STT (Speech-to-Text)', 1: 'TTS (Text-to-Speech)', @@ -208,7 +407,7 @@ export const SettingsScreen: React.FC = () => { 3: 'Embeddings', 4: 'VAD (Voice Activity)', 5: 'Diarization', - }; // Load data on mount + }; useEffect(() => { loadData(); @@ -218,8 +417,6 @@ export const SettingsScreen: React.FC = () => { // eslint-disable-next-line react-hooks/exhaustive-deps }, []); - // Cancel any in-progress downloads when the screen unmounts to avoid - // draining native callbacks and bandwidth with no UI handle to track them. useEffect(() => { return () => { for (const iter of Object.values(downloadIteratorsRef.current)) { @@ -227,15 +424,12 @@ export const SettingsScreen: React.FC = () => { } downloadIteratorsRef.current = {}; }; - }, []); /** - * Load API configuration from AsyncStorage - */ + }, []); const loadApiConfiguration = async () => { try { const storedApiKey = await AsyncStorage.getItem(STORAGE_KEYS.API_KEY); const storedBaseURL = await AsyncStorage.getItem(STORAGE_KEYS.BASE_URL); - setApiKey(storedApiKey || ''); setBaseURL(storedBaseURL || ''); setApiKeyConfigured(!!storedApiKey && storedApiKey !== ''); @@ -243,32 +437,20 @@ export const SettingsScreen: React.FC = () => { } catch (error) { console.error('[Settings] Failed to load API configuration:', error); } - }; /** - * Load generation settings from AsyncStorage - */ + }; const loadGenerationSettings = async () => { try { - const tempStr = await AsyncStorage.getItem( - GENERATION_SETTINGS_KEYS.TEMPERATURE - ); - const maxStr = await AsyncStorage.getItem( - GENERATION_SETTINGS_KEYS.MAX_TOKENS - ); - const sysStr = await AsyncStorage.getItem( - GENERATION_SETTINGS_KEYS.SYSTEM_PROMPT - ); - const thinkingStr = await AsyncStorage.getItem( - GENERATION_SETTINGS_KEYS.THINKING_MODE_ENABLED - ); - + const tempStr = await AsyncStorage.getItem(GENERATION_SETTINGS_KEYS.TEMPERATURE); + const maxStr = await AsyncStorage.getItem(GENERATION_SETTINGS_KEYS.MAX_TOKENS); + const sysStr = await AsyncStorage.getItem(GENERATION_SETTINGS_KEYS.SYSTEM_PROMPT); + const thinkingStr = await AsyncStorage.getItem(GENERATION_SETTINGS_KEYS.THINKING_MODE_ENABLED); const loadedTemperature = tempStr !== null ? parseFloat(tempStr) : 0.7; setTemperature(loadedTemperature); if (maxStr) setMaxTokens(parseInt(maxStr, 10)); if (sysStr) setSystemPrompt(sysStr); setThinkingModeEnabled(thinkingStr === 'true'); - - // eslint-disable-next-line no-console -- demo settings diagnostic + // eslint-disable-next-line no-console console.log('[Settings] Loaded generation settings:', { temperature: loadedTemperature, maxTokens, @@ -278,63 +460,41 @@ export const SettingsScreen: React.FC = () => { } catch (error) { console.error('[Settings] Failed to load generation settings:', error); } - }; /** - * Save generation settings to AsyncStorage - */ + }; const saveGenerationSettings = async () => { try { - await AsyncStorage.setItem( - GENERATION_SETTINGS_KEYS.TEMPERATURE, - temperature.toString() - ); - await AsyncStorage.setItem( - GENERATION_SETTINGS_KEYS.MAX_TOKENS, - maxTokens.toString() - ); - await AsyncStorage.setItem( - GENERATION_SETTINGS_KEYS.SYSTEM_PROMPT, - systemPrompt - ); + await AsyncStorage.setItem(GENERATION_SETTINGS_KEYS.TEMPERATURE, temperature.toString()); + await AsyncStorage.setItem(GENERATION_SETTINGS_KEYS.MAX_TOKENS, maxTokens.toString()); + await AsyncStorage.setItem(GENERATION_SETTINGS_KEYS.SYSTEM_PROMPT, systemPrompt); await AsyncStorage.setItem( GENERATION_SETTINGS_KEYS.THINKING_MODE_ENABLED, thinkingModeEnabled ? 'true' : 'false' ); - - // eslint-disable-next-line no-console -- demo settings diagnostic + // eslint-disable-next-line no-console console.log('[Settings] Saved generation settings:', { temperature, maxTokens, - systemPrompt: systemPrompt - ? `set(${systemPrompt.length} chars)` - : 'empty', + systemPrompt: systemPrompt ? `set(${systemPrompt.length} chars)` : 'empty', thinkingModeEnabled, }); - Alert.alert('Saved', 'Generation settings have been saved successfully.'); } catch (error) { console.error('[Settings] Failed to save generation settings:', error); Alert.alert('Error', `Failed to save settings: ${error}`); } - }; /** - * Load tool calling settings from AsyncStorage - */ + }; const loadToolCallingSettings = async () => { try { - const enabled = await AsyncStorage.getItem( - STORAGE_KEYS.TOOL_CALLING_ENABLED - ); + const enabled = await AsyncStorage.getItem(STORAGE_KEYS.TOOL_CALLING_ENABLED); setToolCallingEnabled(enabled === 'true'); - // void: deliberate fire-and-forget refresh; we don't block load on it. // eslint-disable-next-line no-void void refreshRegisteredTools(); } catch (error) { console.error('[Settings] Failed to load tool calling settings:', error); } - }; /** - * Refresh the list of registered tools from SDK - */ + }; const refreshRegisteredTools = async () => { const tools = await RunAnywhere.getRegisteredTools(); @@ -345,17 +505,12 @@ export const SettingsScreen: React.FC = () => { parameters: t.parameters || [], })) ); - }; /** - * Toggle tool calling enabled state - */ + }; const handleToggleToolCalling = async (enabled: boolean) => { setToolCallingEnabled(enabled); try { - await AsyncStorage.setItem( - STORAGE_KEYS.TOOL_CALLING_ENABLED, - enabled ? 'true' : 'false' - ); + await AsyncStorage.setItem(STORAGE_KEYS.TOOL_CALLING_ENABLED, enabled ? 'true' : 'false'); if (enabled) { await registerSharedDemoTools(); } else { @@ -365,56 +520,41 @@ export const SettingsScreen: React.FC = () => { } catch (error) { console.error('[Settings] Failed to save tool calling setting:', error); } - }; /** - * Clear all registered tools - */ + }; const clearAllTools = () => { - Alert.alert( - 'Clear All Tools', - 'Are you sure you want to remove all registered tools?', - [ - { text: 'Cancel', style: 'cancel' }, - { - text: 'Clear', - style: 'destructive', - onPress: () => { - // void: Alert.onPress is sync; wrap async work in fire-and-forget IIFE. - // eslint-disable-next-line no-void - void (async () => { - await RunAnywhere.clearTools(); - await refreshRegisteredTools(); - })(); - }, + Alert.alert('Clear All Tools', 'Are you sure you want to remove all registered tools?', [ + { text: 'Cancel', style: 'cancel' }, + { + text: 'Clear', + style: 'destructive', + onPress: () => { + // eslint-disable-next-line no-void + void (async () => { + await RunAnywhere.clearTools(); + await refreshRegisteredTools(); + })(); }, - ] - ); - }; /** - * Normalize base URL by adding https:// if no scheme is present - */ + }, + ]); + }; const normalizeBaseURL = (url: string): string => { const trimmed = url.trim(); if (!trimmed) return trimmed; - if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) { - return trimmed; - } + if (trimmed.startsWith('http://') || trimmed.startsWith('https://')) return trimmed; return `https://${trimmed}`; - }; /** - * Save API configuration to AsyncStorage - */ + }; const saveApiConfiguration = async () => { try { const normalizedURL = normalizeBaseURL(baseURL); await AsyncStorage.setItem(STORAGE_KEYS.API_KEY, apiKey); await AsyncStorage.setItem(STORAGE_KEYS.BASE_URL, normalizedURL); - setBaseURL(normalizedURL); setApiKeyConfigured(!!apiKey); setIsBaseURLConfigured(!!normalizedURL); setShowApiConfigModal(false); - Alert.alert( 'Restart Required', 'API configuration has been updated. Please restart the app for changes to take effect.', @@ -423,9 +563,7 @@ export const SettingsScreen: React.FC = () => { } catch (error) { Alert.alert('Error', `Failed to save API configuration: ${error}`); } - }; /** - * Clear API configuration from AsyncStorage - */ + }; const clearApiConfiguration = async () => { try { @@ -434,12 +572,10 @@ export const SettingsScreen: React.FC = () => { STORAGE_KEYS.BASE_URL, STORAGE_KEYS.DEVICE_REGISTERED, ]); - setApiKey(''); setBaseURL(''); setApiKeyConfigured(false); setIsBaseURLConfigured(false); - Alert.alert( 'Restart Required', 'API configuration has been cleared. Please restart the app for changes to take effect.', @@ -453,61 +589,36 @@ export const SettingsScreen: React.FC = () => { const loadData = async () => { setIsRefreshing(true); try { - // Get SDK version const version = RunAnywhere.version; - setSdkVersion(version); // Check if SDK is initialized first - + setSdkVersion(version); const isInit = await RunAnywhere.isInitialized; - // eslint-disable-next-line no-console -- demo settings diagnostic - console.log('[Settings] SDK isInitialized:', isInit); // Get backend info for storage data - + // eslint-disable-next-line no-console + console.log('[Settings] SDK isInitialized:', isInit); const backendInfo = { environment: RunAnywhere.environment, servicesReady: RunAnywhere.areServicesReady, }; - // eslint-disable-next-line no-console -- demo settings diagnostic - console.log('[Settings] Backend info:', backendInfo); // Override name with actual init status - - const sttLoaded = await isModelLoadedForCategory( - ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION - ); - const ttsLoaded = await isModelLoadedForCategory( - ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS - ); - const textLoaded = await isModelLoadedForCategory( - ModelCategory.MODEL_CATEGORY_LANGUAGE - ); - const vadLoaded = await isModelLoadedForCategory( - ModelCategory.MODEL_CATEGORY_VOICE_ACTIVITY_DETECTION - ); - - console.warn( - '[Settings] Models loaded - STT:', - sttLoaded, - 'TTS:', - ttsLoaded, - 'Text:', - textLoaded, - 'VAD:', - vadLoaded - ); // Get available models from catalog - + // eslint-disable-next-line no-console + console.log('[Settings] Backend info:', backendInfo); + const sttLoaded = await isModelLoadedForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION); + const ttsLoaded = await isModelLoadedForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS); + const textLoaded = await isModelLoadedForCategory(ModelCategory.MODEL_CATEGORY_LANGUAGE); + const vadLoaded = await isModelLoadedForCategory(ModelCategory.MODEL_CATEGORY_VOICE_ACTIVITY_DETECTION); + console.warn('[Settings] Models loaded - STT:', sttLoaded, 'TTS:', ttsLoaded, 'Text:', textLoaded, 'VAD:', vadLoaded); try { const available = await listModels(); console.warn('[Settings] Available models:', available); setAvailableModels(available); } catch (err) { console.warn('[Settings] Failed to get available models:', err); - } // Get downloaded models - + } try { const downloaded = await listDownloadedModels(); console.warn('[Settings] Downloaded models:', downloaded); setDownloadedModels(downloaded); } catch (err) { console.warn('[Settings] Failed to get downloaded models:', err); - } // Get storage info using new SDK API - + } try { const storage = await RunAnywhere.getStorageInfo(); console.warn('[Settings] Storage info:', storage); @@ -528,38 +639,27 @@ export const SettingsScreen: React.FC = () => { } finally { setIsRefreshing(false); } - }; /** - * Handle routing policy change - */ + }; - // Kept for upcoming routing-policy UI; not rendered yet in the settings screen. const _handleRoutingPolicyChange = useCallback(() => { Alert.alert( 'Routing Policy', 'Choose how requests are routed', ROUTING_POLICY_OPTIONS.map((policy) => ({ text: RoutingPolicyDisplayNames[policy], - onPress: () => { - setRoutingPolicy(policy); - }, + onPress: () => { setRoutingPolicy(policy); }, })) ); - }, []); /** - * Handle API key configuration - open modal - */ + }, []); const handleConfigureApiKey = useCallback(() => { setShowApiConfigModal(true); - }, []); /** - * Cancel API configuration modal - */ + }, []); const handleCancelApiConfig = useCallback(() => { - loadApiConfiguration(); // Reset to stored values + loadApiConfiguration(); setShowApiConfigModal(false); - }, []); /** - * Handle clear cache - */ + }, []); const handleClearCache = useCallback(() => { Alert.alert( @@ -572,7 +672,6 @@ export const SettingsScreen: React.FC = () => { style: 'destructive', onPress: async () => { try { - // Clear SDK cache using new Storage API await RunAnywhere.clearCache(); await RunAnywhere.cleanTempFiles(); Alert.alert('Success', 'Cache cleared successfully'); @@ -585,14 +684,11 @@ export const SettingsScreen: React.FC = () => { }, ] ); - }, []); /** - * Handle model download - */ + }, []); const handleDownloadModel = useCallback( async (model: ModelInfo) => { if (downloadingModels[model.id] !== undefined) { - // Already downloading, cancel it try { await downloadIteratorsRef.current[model.id]?.return?.(); delete downloadIteratorsRef.current[model.id]; @@ -605,36 +701,26 @@ export const SettingsScreen: React.FC = () => { console.error('Failed to cancel download:', err); } return; - } // Start download with progress tracking - + } setDownloadingModels((prev) => ({ ...prev, [model.id]: 0 })); - try { - // Manual async iteration — Hermes doesn't recognise NitroModules async iterables with for-await const dlIter = downloadModelStreamHelper(model)[Symbol.asyncIterator](); downloadIteratorsRef.current[model.id] = dlIter; let dlResult = await dlIter.next(); while (!dlResult.done) { const progress = dlResult.value; - console.warn( - `[Settings] Download progress for ${model.id}: ${((progress.stageProgress ?? 0) * 100).toFixed(1)}%` - ); - setDownloadingModels((prev) => ({ - ...prev, - [model.id]: progress.stageProgress ?? 0, - })); + console.warn(`[Settings] Download progress for ${model.id}: ${((progress.stageProgress ?? 0) * 100).toFixed(1)}%`); + setDownloadingModels((prev) => ({ ...prev, [model.id]: progress.stageProgress ?? 0 })); dlResult = await dlIter.next(); - } // Download complete - + } setDownloadingModels((prev) => { const updated = { ...prev }; delete updated[model.id]; return updated; }); delete downloadIteratorsRef.current[model.id]; - Alert.alert('Success', `${model.name} downloaded successfully!`); - loadData(); // Refresh to show downloaded model + loadData(); } catch (err) { setDownloadingModels((prev) => { const updated = { ...prev }; @@ -642,24 +728,16 @@ export const SettingsScreen: React.FC = () => { return updated; }); delete downloadIteratorsRef.current[model.id]; - Alert.alert( - 'Download Failed', - `Failed to download ${model.name}: ${err}` - ); + Alert.alert('Download Failed', `Failed to download ${model.name}: ${err}`); } }, [downloadingModels] - ); /** - * Handle delete downloaded model - */ + ); const handleDeleteDownloadedModel = useCallback( async (model: ModelInfo) => { const downloadedModel = downloadedModels.find((m) => m.id === model.id); - // Prefer the downloaded model's size (reported by the SDK after download) - // over the catalog's expected downloadSize. const freedSize = getModelDownloadSizeBytes(downloadedModel ?? model); - Alert.alert( 'Delete Model', `Are you sure you want to delete ${model.name}? This will free up ${formatBytes(freedSize)}.`, @@ -671,13 +749,9 @@ export const SettingsScreen: React.FC = () => { onPress: async () => { try { const result = await RunAnywhere.deleteModel(model.id); - if (!result.success) { - throw new Error( - result.errorMessage || 'Storage delete failed' - ); - } + if (!result.success) throw new Error(result.errorMessage || 'Storage delete failed'); Alert.alert('Deleted', `${model.name} has been deleted.`); - loadData(); // Refresh list + loadData(); } catch (err) { Alert.alert('Error', `Failed to delete: ${err}`); } @@ -687,9 +761,7 @@ export const SettingsScreen: React.FC = () => { ); }, [downloadedModels] - ); /** - * Handle clear all data - */ + ); const handleClearAllData = useCallback(() => { Alert.alert( @@ -702,16 +774,9 @@ export const SettingsScreen: React.FC = () => { style: 'destructive', onPress: async () => { try { - // Unload all models - await unloadModelsForCategory( - ModelCategory.MODEL_CATEGORY_LANGUAGE - ); - await unloadModelsForCategory( - ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION - ); - await unloadModelsForCategory( - ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS - ); + await unloadModelsForCategory(ModelCategory.MODEL_CATEGORY_LANGUAGE); + await unloadModelsForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_RECOGNITION); + await unloadModelsForCategory(ModelCategory.MODEL_CATEGORY_SPEECH_SYNTHESIS); await RunAnywhere.reset(); Alert.alert('Success', 'All data cleared'); } catch (error) { @@ -722,427 +787,241 @@ export const SettingsScreen: React.FC = () => { }, ] ); - }, []); /** - * Render section header - */ - - const renderSectionHeader = (title: string) => ( - {title} - ); /** - * Render setting row - */ - - // Reusable row renderer kept for future settings UI; currently not invoked. - const _renderSettingRow = ( - icon: string, - title: string, - value: string, - onPress?: () => void, - showChevron: boolean = true - ) => ( - - - - {title} - - - {value} - {showChevron && onPress && ( - - )} - - - ); /** - * Render slider setting - */ - - const renderSliderSetting = ( - title: string, - value: number, - onChange: (value: number) => void, - min: number, - max: number, - step: number, - formatValue: (v: number) => string - ) => ( - - - {title} - {formatValue(value)} - - - onChange(Math.max(min, value - step))} - > - - - - - - onChange(Math.min(max, value + step))} - > - - - - - ); /** - * Render storage bar - * Matches iOS: shows app storage usage relative to device free space - */ - - const renderStorageBar = () => { - // Show app storage as portion of (app storage + free space) - const totalAvailable = storageInfo.appStorage + storageInfo.freeSpace; - const usedPercent = - totalAvailable > 0 ? (storageInfo.appStorage / totalAvailable) * 100 : 0; - return ( - - - - - -           {formatBytes(storageInfo.appStorage)} of          {' '} - {formatBytes(storageInfo.freeSpace)} available        {' '} - - - ); - }; /** - * Render catalog model row - */ - - const renderCatalogModelRow = (model: ModelInfo) => { - const isDownloading = downloadingModels[model.id] !== undefined; - const downloadProgress = downloadingModels[model.id] || 0; - const isDownloaded = downloadedModels.some((m) => m.id === model.id); - const frameworkName = getFrameworkDisplayName(getPrimaryFramework(model)); - - const downloadedModel = downloadedModels.find((m) => m.id === model.id); - const modelSize = getModelDownloadSizeBytes(downloadedModel ?? model); - return ( - - - - {model.name} - - {model.category} - - - {model.metadata?.description && ( - -               {model.metadata.description}           {' '} - - )} - - -               {formatBytes(modelSize)}           {' '} - - {frameworkName} - - {isDownloading && ( - - - - - -                 {(downloadProgress * 100).toFixed(0)}%           - - - )} - - - isDownloaded - ? handleDeleteDownloadedModel(model) - : handleDownloadModel(model) - } - > - - - - ); - }; + }, []); + + // Storage summary text — matches Android "Models X · Y free" + const storageSummary = `Models ${formatBytes(storageInfo.modelsStorage)} · ${formatBytes(storageInfo.freeSpace)} free`; return ( - + {/* Header */} - - Settings - - - + + + Settings + - - {/* Generation Settings - Matches iOS CombinedSettingsView order */} - {renderSectionHeader('Generation Settings')} - - {renderSliderSetting( - 'Temperature', - temperature, - setTemperature, - SETTINGS_CONSTRAINTS.temperature.min, - SETTINGS_CONSTRAINTS.temperature.max, - SETTINGS_CONSTRAINTS.temperature.step, - (v) => v.toFixed(1) - )} - {renderSliderSetting( - 'Max Tokens', - maxTokens, - setMaxTokens, - SETTINGS_CONSTRAINTS.maxTokens.min, - SETTINGS_CONSTRAINTS.maxTokens.max, - SETTINGS_CONSTRAINTS.maxTokens.step, - (v) => v.toLocaleString() - )} - - - Thinking Mode - - Model will use its default thinking/reasoning mode. - - - + {/* Generation */} +
+ v.toFixed(1)} + onChange={setTemperature} + /> + v.toLocaleString()} + onChange={(v) => setMaxTokens(Math.round(v))} + /> + + System prompt + setThinkingModeEnabled(!thinkingModeEnabled)} - > - - - - {/* System Prompt Input */} - - System Prompt - - {/* Save Settings Button */} + - - Save Settings + + Save Settings - - {/* API Configuration (Testing) */} - {renderSectionHeader('API Configuration (Testing)')} - - - API Key +
+ + {/* Storage */} +
+ + {storageSummary} + + {downloadedModels.length === 0 ? ( + + No downloaded models + + ) : ( + downloadedModels.map((model) => ( + handleDeleteDownloadedModel(model)} + /> + )) + )} + + + Clear cache + + + Clear all data + + +
+ + {/* API Configuration */} +
+ + API Key -               {apiKeyConfigured ? 'Configured' : 'Not Set'}        + {apiKeyConfigured ? 'Configured' : 'Not Set'} - - - Base URL + + + Base URL -               {isBaseURLConfigured ? 'Configured' : 'Not Set'}    + {isBaseURLConfigured ? 'Configured' : 'Not Set'} - - + + - Configure + Configure {apiKeyConfigured && isBaseURLConfigured && ( - - Clear - + Clear )} - -             Configure custom API key and base URL for testing. - Requires app restart.          {' '} + + Configure custom API key and base URL for testing. Requires app restart. - - {/* Tool Settings - Matches iOS ToolSettingsView */} - {renderSectionHeader('Tool Settings')} - - {/* Enable Tool Calling Toggle */} - - - Enable Tool Calling - -                 Allow LLMs to call tools (APIs, functions)       - - - handleToggleToolCalling(!toolCallingEnabled)} - > - - - +
+ + {/* Tool Settings */} +
+ {toolCallingEnabled && ( <> - - {/* Registered Tools Count */} - - Registered Tools + + + + Registered Tools + 0 - ? Colors.primaryGreen - : Colors.textSecondary, - }, + typography.metric, + { color: registeredTools.length > 0 ? colors.success : colors.onSurfaceVariant }, ]} > -                   {registeredTools.length}{' '} - {registeredTools.length === 1 ? 'tool' : 'tools'}              + {registeredTools.length} {registeredTools.length === 1 ? 'tool' : 'tools'} - {/* Registered Tools List */} {registeredTools.length > 0 && ( <> - - {registeredTools.map((tool, index) => ( - - - - {tool.name} - - {tool.description} + + {registeredTools.map((tool) => ( + + + + + {tool.name} - {tool.parameters.length > 0 && ( - - {tool.parameters.map((p) => ( - - - {p.name} - - - ))} - - )} - {index < registeredTools.length - 1 && ( - + + {tool.description} + + {tool.parameters.length > 0 && ( + + {tool.parameters.map((p) => ( + + + {p.name} + + + ))} + )} ))} - {/* Clear All Tools Button */} - - - - + + + + Clear All Tools @@ -1150,197 +1029,201 @@ export const SettingsScreen: React.FC = () => { )} )} - -             Tools allow the LLM to call external APIs and functions - to get real-time data.          {' '} + + Tools allow the LLM to call external APIs and functions to get real-time data. - - {/* Performance - Matches iOS CombinedSettingsView "Performance" section */} - {renderSectionHeader('Performance')} - +
+ + {/* Performance */} +
navigation.navigate('Benchmarks')} + style={[styles.row, { alignItems: 'center' }]} + onPress={() => navigation.navigate(ROUTES.Benchmarks)} activeOpacity={0.7} > - - - Benchmarks - - - + + + Benchmarks + - - {/* Storage Overview - Matches iOS CombinedSettingsView */} - {renderSectionHeader('Storage Overview')} - - {renderStorageBar()} - - {/* Total Storage - App's total storage usage */} - - Total Storage - -                 {formatBytes(storageInfo.appStorage)}            - - - {/* Models Storage - Downloaded models size */} - - Models - -                 {formatBytes(storageInfo.modelsStorage)}        - - - {/* Cache Size */} - - Cache - -                 {formatBytes(storageInfo.cacheSize)}            - - - {/* Available - Device free space */} - - Available - -                 {formatBytes(storageInfo.freeSpace)}            - - - - +
+ {/* Model Catalog */} - {renderSectionHeader('Model Catalog')} - +
{availableModels.length === 0 ? ( - Loading models... + + Loading models… + ) : ( - availableModels.map(renderCatalogModelRow) + availableModels.map((model, i) => ( + + {i > 0 && ( + + )} + m.id === model.id)} + isDownloading={downloadingModels[model.id] !== undefined} + downloadProgress={downloadingModels[model.id] ?? 0} + onAction={() => + downloadedModels.some((m) => m.id === model.id) + ? handleDeleteDownloadedModel(model) + : handleDownloadModel(model) + } + /> + + )) )} - - {/* Storage Management */} - {renderSectionHeader('Storage Management')} - - - - Clear Cache - +
+ + {/* About */} +
+ + { + /* open docs URL */ + }} > - - -               Clear All Data            {' '} - + Documentation - - {/* Version Info */} - - RunAnywhere AI - SDK v{sdkVersion} - +
+ {/* API Configuration Modal */} - - API Configuration - {/* API Key Input */} - - API Key - + + + API Configuration + + + + API Key + setShowPassword(!showPassword)} > - -                 Your API key for authenticating with the backend + + Your API key for authenticating with the backend - {/* Base URL Input */} - - Base URL + + + Base URL - -                 The backend API URL (https:// added - automatically if missing)              {' '} + + The backend API URL (https:// added automatically if missing) - {/* Warning */} - - - -                 After saving, you must restart the app for - changes to take effect. The SDK will reinitialize with your - custom configuration.              {' '} + + + + + After saving, you must restart the app for changes to take effect. - {/* Buttons */} - + + - Cancel + Cancel Save @@ -1350,686 +1233,135 @@ export const SettingsScreen: React.FC = () => { - + ); }; const styles = StyleSheet.create({ - container: { + root: { flex: 1, - backgroundColor: Colors.backgroundGrouped, }, header: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - paddingHorizontal: Padding.padding16, - paddingTop: 0, - paddingBottom: Padding.padding12, - backgroundColor: Colors.backgroundPrimary, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - title: { - ...Typography.title2, - color: Colors.textPrimary, - }, - refreshButton: { - padding: Spacing.small, + paddingHorizontal: 16, + paddingBottom: 8, }, - content: { - flex: 1, - }, - sectionHeader: { - ...Typography.footnote, - color: Colors.textSecondary, - textTransform: 'uppercase', - marginTop: Spacing.xLarge, - marginBottom: Spacing.small, - marginHorizontal: Padding.padding16, + scrollContent: { + gap: 16, + paddingTop: 8, }, - section: { - backgroundColor: Colors.backgroundPrimary, - borderRadius: BorderRadius.medium, - marginHorizontal: Padding.padding16, + sectionCard: { overflow: 'hidden', }, - settingRow: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - padding: Padding.padding16, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - settingRowLeft: { - flexDirection: 'row', - alignItems: 'center', - gap: Spacing.medium, - }, - settingRowRight: { - flexDirection: 'row', - alignItems: 'center', - gap: Spacing.small, - }, - settingLabel: { - ...Typography.body, - color: Colors.textPrimary, - }, - settingValue: { - ...Typography.body, - color: Colors.textSecondary, - }, - sliderSetting: { - padding: Padding.padding16, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - sliderHeader: { + row: { flexDirection: 'row', justifyContent: 'space-between', alignItems: 'center', - marginBottom: Spacing.medium, }, - sliderValue: { - ...Typography.body, - color: Colors.primaryBlue, - fontWeight: '600', + divider: { + height: StyleSheet.hairlineWidth, }, - sliderControls: { + sliderRow: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.medium, + gap: 8, }, - sliderButton: { - width: 36, - height: 36, - borderRadius: 18, - backgroundColor: Colors.backgroundSecondary, + sliderBtn: { + width: 32, + height: 32, + borderRadius: 16, justifyContent: 'center', alignItems: 'center', }, sliderTrack: { flex: 1, height: 6, - backgroundColor: Colors.backgroundGray5, borderRadius: 3, + overflow: 'hidden', }, sliderFill: { height: '100%', - backgroundColor: Colors.primaryBlue, borderRadius: 3, }, - storageBar: { - padding: Padding.padding16, - }, - storageBarTrack: { - height: 8, - backgroundColor: Colors.backgroundGray5, - borderRadius: 4, - overflow: 'hidden', - }, - storageBarFill: { - height: '100%', - backgroundColor: Colors.primaryBlue, - borderRadius: 4, - }, - storageText: { - ...Typography.footnote, - color: Colors.textSecondary, - marginTop: Spacing.small, - textAlign: 'center', - }, - storageDetails: { - borderTopWidth: 1, - borderTopColor: Colors.borderLight, - padding: Padding.padding16, - gap: Spacing.small, - }, - storageDetailRow: { - flexDirection: 'row', - justifyContent: 'space-between', - }, - storageDetailLabel: { - ...Typography.subheadline, - color: Colors.textSecondary, - }, - storageDetailValue: { - ...Typography.subheadline, - color: Colors.textPrimary, - }, - modelRow: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - padding: Padding.padding16, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - modelInfo: { - flex: 1, - }, - modelName: { - ...Typography.body, - color: Colors.textPrimary, - }, - modelMeta: { - flexDirection: 'row', - alignItems: 'center', - gap: Spacing.small, - marginTop: Spacing.xSmall, - }, - frameworkBadge: { - backgroundColor: Colors.badgeBlue, - paddingHorizontal: Spacing.small, - paddingVertical: Spacing.xxSmall, - borderRadius: BorderRadius.small, - }, - frameworkBadgeText: { - ...Typography.caption2, - color: Colors.primaryBlue, - fontWeight: '600', - }, - modelSize: { - ...Typography.caption, - color: Colors.textSecondary, - }, - deleteButton: { - padding: Spacing.small, - }, - emptyText: { - ...Typography.body, - color: Colors.textSecondary, - textAlign: 'center', - padding: Padding.padding24, + textArea: { + borderWidth: 1, + padding: 12, + minHeight: 72, }, - dangerButton: { + saveBtn: { flexDirection: 'row', alignItems: 'center', justifyContent: 'center', - gap: Spacing.smallMedium, - padding: Padding.padding16, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - dangerButtonRed: { - borderBottomWidth: 0, - }, - dangerButtonText: { - ...Typography.body, - color: Colors.primaryOrange, - }, - dangerButtonTextRed: { - color: Colors.primaryRed, - }, - versionContainer: { - alignItems: 'center', - padding: Padding.padding24, - marginTop: Spacing.large, - marginBottom: Spacing.xxxLarge, - }, - versionText: { - ...Typography.footnote, - color: Colors.textTertiary, - }, - versionSubtext: { - ...Typography.caption, - color: Colors.textTertiary, - marginTop: Spacing.xSmall, - }, // SDK Status styles - statusRow: { - flexDirection: 'row', - justifyContent: 'space-between', - alignItems: 'center', - padding: Padding.padding16, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - statusLabel: { - ...Typography.body, - color: Colors.textPrimary, - }, - statusValue: { - ...Typography.body, - color: Colors.primaryBlue, - fontWeight: '600', - }, - capabilitiesContainer: { - padding: Padding.padding16, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - capabilitiesLabel: { - ...Typography.subheadline, - color: Colors.textSecondary, - marginBottom: Spacing.small, - }, - capabilitiesList: { - flexDirection: 'row', - flexWrap: 'wrap', - gap: Spacing.small, - }, - capabilityBadge: { - flexDirection: 'row', - alignItems: 'center', - gap: Spacing.xSmall, - backgroundColor: Colors.badgeGreen, - paddingHorizontal: Spacing.smallMedium, - paddingVertical: Spacing.xSmall, - borderRadius: BorderRadius.small, - }, - capabilityText: { - ...Typography.caption, - color: Colors.primaryGreen, - fontWeight: '600', - }, - noCapabilities: { - ...Typography.body, - color: Colors.textTertiary, - fontStyle: 'italic', - }, - modelStatusContainer: { - padding: Padding.padding16, - }, - modelStatusGrid: { - flexDirection: 'row', - flexWrap: 'wrap', - gap: Spacing.small, - marginTop: Spacing.small, - }, - modelStatusItem: { - flexDirection: 'row', - alignItems: 'center', - gap: Spacing.xSmall, - backgroundColor: Colors.backgroundSecondary, - paddingHorizontal: Spacing.medium, - paddingVertical: Spacing.small, - borderRadius: BorderRadius.small, - }, - modelStatusItemLoaded: { - backgroundColor: Colors.badgeGreen, - }, - modelStatusText: { - ...Typography.footnote, - color: Colors.textSecondary, - fontWeight: '600', - }, // Model catalog styles - catalogModelRow: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'space-between', - padding: Padding.padding16, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - catalogModelInfo: { - flex: 1, - marginRight: Spacing.medium, - }, - catalogModelHeader: { - flexDirection: 'row', - alignItems: 'center', - gap: Spacing.small, - marginBottom: Spacing.xSmall, - }, - catalogModelName: { - ...Typography.body, - color: Colors.textPrimary, - fontWeight: '600', - }, - catalogModelBadge: { - backgroundColor: Colors.badgeBlue, - paddingHorizontal: Spacing.small, - paddingVertical: Spacing.xxSmall, - borderRadius: BorderRadius.small, + gap: 8, + paddingVertical: 12, + paddingHorizontal: 16, }, - catalogModelBadgeText: { - ...Typography.caption2, - color: Colors.primaryBlue, - fontWeight: '600', - textTransform: 'uppercase', - }, - catalogModelDescription: { - ...Typography.footnote, - color: Colors.textSecondary, - marginBottom: Spacing.xSmall, + outlineBtn: { + paddingHorizontal: 16, + paddingVertical: 8, + borderWidth: 1, }, - catalogModelMeta: { + centerRow: { flexDirection: 'row', alignItems: 'center', - gap: Spacing.small, - }, - catalogModelSize: { - ...Typography.caption, - color: Colors.textTertiary, - }, - catalogModelFormat: { - ...Typography.caption, - color: Colors.textTertiary, - }, - catalogModelButton: { - width: 44, - height: 44, - borderRadius: 22, - backgroundColor: Colors.backgroundSecondary, justifyContent: 'center', - alignItems: 'center', - }, - catalogModelButtonDownloaded: { - backgroundColor: Colors.badgeGreen, - }, - catalogModelButtonDownloading: { - backgroundColor: Colors.badgeOrange, + gap: 8, }, - downloadProgressContainer: { - flexDirection: 'row', - alignItems: 'center', - gap: Spacing.small, - marginTop: Spacing.small, - }, - downloadProgressTrack: { - flex: 1, - height: 4, - backgroundColor: Colors.backgroundGray5, - borderRadius: 2, + modelCard: { overflow: 'hidden', }, - downloadProgressFill: { - height: '100%', - backgroundColor: Colors.primaryBlue, - borderRadius: 2, - }, - downloadProgressText: { - ...Typography.caption, - color: Colors.primaryBlue, - fontWeight: '600', - minWidth: 40, - textAlign: 'right', - }, // API Configuration styles - apiConfigRow: { + modelCardInner: { flexDirection: 'row', - justifyContent: 'space-between', alignItems: 'center', - padding: Padding.padding16, - }, - apiConfigLabel: { - ...Typography.body, - color: Colors.textPrimary, }, - apiConfigValue: { - ...Typography.body, - fontWeight: '500', - }, - apiConfigDivider: { - height: 1, - backgroundColor: Colors.borderLight, - marginHorizontal: Padding.padding16, - }, - apiConfigButtons: { + toolCard: {}, + chipRow: { flexDirection: 'row', - padding: Padding.padding16, - gap: Spacing.small, - }, - apiConfigButton: { - paddingHorizontal: Padding.padding16, - paddingVertical: Spacing.small, - borderRadius: BorderRadius.small, - borderWidth: 1, - borderColor: Colors.primaryBlue, - }, - apiConfigButtonClear: { - borderColor: Colors.primaryRed, - }, - apiConfigButtonText: { - ...Typography.subheadline, - color: Colors.primaryBlue, - fontWeight: '600', + flexWrap: 'wrap', + gap: 6, }, - apiConfigButtonTextClear: { - color: Colors.primaryRed, + chip: { + paddingHorizontal: 8, + paddingVertical: 2, }, - apiConfigHint: { - ...Typography.footnote, - color: Colors.textSecondary, - paddingHorizontal: Padding.padding16, - paddingBottom: Padding.padding16, - }, // Modal styles modalOverlay: { flex: 1, - backgroundColor: 'rgba(0, 0, 0, 0.5)', + backgroundColor: 'rgba(0,0,0,0.5)', justifyContent: 'center', alignItems: 'center', - padding: Padding.padding24, + padding: 24, }, modalContent: { - backgroundColor: Colors.backgroundPrimary, - borderRadius: BorderRadius.large, - padding: Padding.padding24, width: '100%', maxWidth: 400, + padding: 24, }, - modalTitle: { - ...Typography.title2, - color: Colors.textPrimary, - marginBottom: Spacing.large, - textAlign: 'center', - }, - inputGroup: { - marginBottom: Spacing.large, - }, - inputLabel: { - ...Typography.subheadline, - color: Colors.textSecondary, - marginBottom: Spacing.small, - }, - input: { - backgroundColor: Colors.backgroundSecondary, - borderRadius: BorderRadius.small, - padding: Padding.padding12, - ...Typography.body, - color: Colors.textPrimary, - borderWidth: 1, - borderColor: Colors.borderLight, - }, - passwordInputContainer: { + inputRow: { flexDirection: 'row', alignItems: 'center', - backgroundColor: Colors.backgroundSecondary, - borderRadius: BorderRadius.small, borderWidth: 1, - borderColor: Colors.borderLight, - }, - passwordInput: { - flex: 1, - padding: Padding.padding12, - ...Typography.body, - color: Colors.textPrimary, - }, - passwordToggle: { - padding: Padding.padding12, - }, - inputHint: { - ...Typography.caption, - color: Colors.textTertiary, - marginTop: Spacing.xSmall, }, - warningBox: { - flexDirection: 'row', - backgroundColor: Colors.badgeOrange, - borderRadius: BorderRadius.small, - padding: Padding.padding12, - gap: Spacing.small, - marginBottom: Spacing.large, - }, - warningText: { - ...Typography.footnote, - color: Colors.textSecondary, + inputFlex: { flex: 1, + padding: 12, }, - modalButtons: { - flexDirection: 'row', - justifyContent: 'flex-end', - gap: Spacing.medium, - }, - modalButton: { - paddingHorizontal: Padding.padding16, - paddingVertical: Spacing.smallMedium, - borderRadius: BorderRadius.small, - minWidth: 80, - alignItems: 'center', - }, - modalButtonCancel: { - backgroundColor: 'transparent', - }, - modalButtonSave: { - backgroundColor: Colors.primaryBlue, - }, - modalButtonDisabled: { - backgroundColor: Colors.backgroundGray5, - }, - modalButtonTextCancel: { - ...Typography.body, - color: Colors.textSecondary, - }, - modalButtonTextSave: { - ...Typography.body, - color: Colors.textWhite, - fontWeight: '600', - }, - modalButtonTextDisabled: { - color: Colors.textTertiary, - }, // System Prompt styles - systemPromptContainer: { - padding: Padding.padding16, - borderBottomWidth: 1, - borderBottomColor: Colors.borderLight, - }, - systemPromptLabel: { - ...Typography.subheadline, - color: Colors.textPrimary, - marginBottom: Spacing.small, - }, - systemPromptInput: { - backgroundColor: Colors.backgroundSecondary, - borderRadius: BorderRadius.small, - padding: Padding.padding12, - ...Typography.body, - color: Colors.textPrimary, + inputField: { borderWidth: 1, - borderColor: Colors.borderLight, - minHeight: 80, + padding: 12, }, - saveSettingsButton: { - flexDirection: 'row', - alignItems: 'center', - justifyContent: 'center', - gap: Spacing.small, - backgroundColor: Colors.primaryBlue, - padding: Padding.padding16, - margin: Padding.padding16, - borderRadius: BorderRadius.small, - }, - saveSettingsButtonText: { - ...Typography.body, - color: Colors.textWhite, - fontWeight: '600', - }, // Tool Settings styles - toolSettingRow: { - flexDirection: 'row', - justifyContent: 'space-between', - alignItems: 'center', - padding: Padding.padding16, - }, - toolSettingInfo: { - flex: 1, - marginRight: Spacing.medium, - }, - toolSettingLabel: { - ...Typography.body, - color: Colors.textPrimary, - fontWeight: '500', - }, - toolSettingDescription: { - ...Typography.footnote, - color: Colors.textSecondary, - marginTop: Spacing.xxSmall, - }, - toolSettingValue: { - ...Typography.body, - fontWeight: '600', - }, - toggleButton: { - width: 51, - height: 31, - borderRadius: 15.5, - backgroundColor: Colors.backgroundGray5, - padding: 2, - justifyContent: 'center', - }, - toggleButtonActive: { - backgroundColor: Colors.primaryBlue, - }, - toggleKnob: { - width: 27, - height: 27, - borderRadius: 13.5, - backgroundColor: Colors.textWhite, - shadowColor: '#000', - shadowOffset: { width: 0, height: 2 }, - shadowOpacity: 0.2, - shadowRadius: 2, - elevation: 2, - }, - toggleKnobActive: { - alignSelf: 'flex-end', - }, - toolRow: { + warningBox: { flexDirection: 'row', + gap: 8, alignItems: 'flex-start', - padding: Padding.padding16, - gap: Spacing.medium, - }, - toolInfo: { - flex: 1, - }, - toolName: { - ...Typography.subheadline, - color: Colors.textPrimary, - fontWeight: '600', }, - toolDescription: { - ...Typography.footnote, - color: Colors.textSecondary, - marginTop: Spacing.xxSmall, - }, - toolParams: { + modalBtns: { flexDirection: 'row', - flexWrap: 'wrap', - gap: Spacing.xSmall, - marginTop: Spacing.small, - }, - toolParamChip: { - backgroundColor: Colors.badgeBlue, - paddingHorizontal: Spacing.small, - paddingVertical: Spacing.xxSmall, - borderRadius: BorderRadius.small, - }, - toolParamText: { - ...Typography.caption2, - color: Colors.primaryBlue, - fontWeight: '500', + justifyContent: 'flex-end', }, - clearToolsButton: { - flexDirection: 'row', + modalBtn: { + paddingHorizontal: 16, + paddingVertical: 10, alignItems: 'center', - justifyContent: 'center', - gap: Spacing.small, - padding: Padding.padding16, - }, - clearToolsButtonText: { - ...Typography.body, - color: Colors.statusRed, - fontWeight: '600', + minWidth: 80, }, }); diff --git a/examples/react-native/RunAnywhereAI/src/screens/SolutionsScreen.tsx b/examples/react-native/RunAnywhereAI/src/screens/SolutionsScreen.tsx index 13b09d2794..fa3105f4b7 100644 --- a/examples/react-native/RunAnywhereAI/src/screens/SolutionsScreen.tsx +++ b/examples/react-native/RunAnywhereAI/src/screens/SolutionsScreen.tsx @@ -9,12 +9,22 @@ * simple scrolling log. */ import React, { useState, useCallback } from 'react'; -import { View, Text, Button, ScrollView, StyleSheet } from 'react-native'; -import { SafeAreaView } from 'react-native-safe-area-context'; +import { + ScrollView, + StyleSheet, + Text, + TouchableOpacity, + View, +} from 'react-native'; +import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { RunAnywhere } from '@runanywhere/core'; +import { Icon, useTheme } from '../theme/system'; +import type { IconName } from '../theme/system/icons'; import { VOICE_AGENT_YAML, RAG_YAML } from '../generated/solutionsYaml'; export const SolutionsScreen: React.FC = () => { + const { colors, typography, dimens } = useTheme(); + const insets = useSafeAreaInsets(); const [log, setLog] = useState([]); const [isRunning, setIsRunning] = useState(false); @@ -43,50 +53,138 @@ export const SolutionsScreen: React.FC = () => { [isRunning, append] ); + const s = styles(colors, typography, dimens); + return ( - - + + Run a prepackaged pipeline (voice agent or RAG) by handing a YAML config to RunAnywhere.solutions.run. - - - + +

Indexed documents

@@ -104,10 +107,87 @@ export function initDocumentsTab(el: HTMLElement): TabLifecycle { container.querySelector('#docs-ask-btn')!.addEventListener('click', () => { void askQuestion(); }); + container.querySelector('#docs-embedding-download-btn')!.addEventListener('click', () => { + void downloadSelectedModel(selectedEmbeddingModelId, 'embedding'); + }); + container.querySelector('#docs-llm-download-btn')!.addEventListener('click', () => { + void downloadSelectedModel(selectedLlmModelId, 'LLM'); + }); + refreshModelButtons(); return {}; } +// --------------------------------------------------------------------------- +// Model download +// --------------------------------------------------------------------------- + +function setModelStatus(msg: string): void { + const el = container.querySelector('#docs-model-status'); + if (el) el.textContent = msg; +} + +/** Reflect downloaded state on the two download buttons. */ +function refreshModelButtons(): void { + const pairs: Array<['embedding' | 'llm', string]> = [ + ['embedding', selectedEmbeddingModelId], + ['llm', selectedLlmModelId], + ]; + for (const [kind, modelId] of pairs) { + const btn = container.querySelector(`#docs-${kind}-download-btn`); + if (!btn) continue; + const model = modelId ? RunAnywhere.getModel(modelId) : null; + const downloaded = !!(model?.isDownloaded || model?.localPath); + btn.disabled = isBusy || !modelId || downloaded; + btn.textContent = downloaded ? 'Downloaded' : 'Download'; + } +} + +async function downloadSelectedModel( + modelId: string, + label: string, +): Promise { + if (!modelId) { + setModelStatus(`Select a ${label} model first.`); + return; + } + const model = RunAnywhere.getModel(modelId); + if (!model) { + setModelStatus(`${label} model '${modelId}' is not registered.`); + return; + } + isBusy = true; + refreshModelButtons(); + setModelStatus(`Downloading ${label} model ${model.name || modelId}…`); + try { + await RunAnywhere.downloadModel({ + modelId, + model, + allowMeteredNetwork: true, + resumeExisting: true, + verifyChecksums: false, + validateExistingBytes: false, + updateRegistryOnCompletion: true, + storageNamespace: '', + availableStorageBytes: 0, + requiredFreeBytesAfterDownload: 0, + pollIntervalMs: 500, + onProgress: (next) => { + const pct = next.totalBytes > 0 + ? Math.round((Number(next.bytesDownloaded) / Number(next.totalBytes)) * 100) + : 0; + setModelStatus(`Downloading ${label} model… ${pct}%`); + }, + }); + setModelStatus(`${label} model ready: ${model.name || modelId}.`); + } catch (err) { + setModelStatus(`${label} model download failed: ${formatError(err)}`); + } finally { + isBusy = false; + refreshModelButtons(); + } +} + // --------------------------------------------------------------------------- // Model pickers // --------------------------------------------------------------------------- @@ -136,9 +216,11 @@ function populateModelPickers(): void { embeddingSelect.addEventListener('change', () => { selectedEmbeddingModelId = embeddingSelect.value; + refreshModelButtons(); }); llmSelect.addEventListener('change', () => { selectedLlmModelId = llmSelect.value; + refreshModelButtons(); }); } diff --git a/examples/web/RunAnywhereAI/src/views/vision.ts b/examples/web/RunAnywhereAI/src/views/vision.ts index 88745f9e9e..4934b94f32 100644 --- a/examples/web/RunAnywhereAI/src/views/vision.ts +++ b/examples/web/RunAnywhereAI/src/views/vision.ts @@ -43,6 +43,9 @@ const CAPTURE_DIMENSION = 384; let container: HTMLElement; let camera: VideoCapture | null = null; let latestFrame: { rgbPixels: Uint8Array; width: number; height: number } | null = null; +// Data URL preview for an image loaded from disk (null when the source is the +// live camera). Lets the preview survive innerHTML re-renders without a camera. +let loadedPreviewUrl: string | null = null; let lastResult: string | null = null; let status = ''; let isBusy = false; @@ -90,7 +93,7 @@ export function initVisionTab(el: HTMLElement): TabLifecycle { function renderView(): void { const modelLoaded = isVLMModelLoaded(); const captureReady = camera?.isCapturing ?? false; - const canAnalyze = modelLoaded && captureReady && !isBusy; + const canAnalyze = modelLoaded && (captureReady || latestFrame !== null) && !isBusy; container.innerHTML = `
@@ -120,6 +123,10 @@ function renderView(): void { + +
${frameMetaLabel()}
@@ -165,6 +172,11 @@ function renderView(): void { container .querySelector('#vision-capture-btn')! .addEventListener('click', () => captureFrame()); + const imageInput = container.querySelector('#vision-image-input')!; + container + .querySelector('#vision-load-image-btn')! + .addEventListener('click', () => imageInput.click()); + imageInput.addEventListener('change', () => void onImageFileSelected(imageInput)); container .querySelector('#vision-analyze-btn')! .addEventListener('click', () => void onAnalyze()); @@ -175,9 +187,20 @@ function renderView(): void { function reattachCameraPreview(): void { const host = container.querySelector('#vision-preview'); - if (!host || !camera) return; + if (!host) return; host.innerHTML = ''; - host.appendChild(camera.videoElement); + if (camera) { + host.appendChild(camera.videoElement); + return; + } + if (loadedPreviewUrl) { + const img = document.createElement('img'); + img.src = loadedPreviewUrl; + img.alt = 'Loaded image'; + img.style.maxWidth = '100%'; + img.style.borderRadius = '8px'; + host.appendChild(img); + } } function frameMetaLabel(): string { @@ -234,21 +257,94 @@ function captureFrame(): void { return; } latestFrame = frame; + loadedPreviewUrl = null; setStatus(`Captured ${frame.width}×${frame.height} frame.`); renderView(); } // --------------------------------------------------------------------------- -// Analyze +// Image from disk // --------------------------------------------------------------------------- -async function onAnalyze(): Promise { - if (!camera?.isCapturing) { - setStatus('Start the camera first.'); +async function onImageFileSelected(input: HTMLInputElement): Promise { + const file = input.files?.[0]; + // Reset so re-selecting the same file fires `change` again. + input.value = ''; + if (!file) return; + + isBusy = true; + setStatus('Loading image…'); + renderView(); + try { + const decoded = await decodeImageToRgbFrame(file, CAPTURE_DIMENSION); + // A loaded image is an alternative frame source — drop the live camera so + // the preview and analysis operate on the picked image. + stopCamera(); + latestFrame = { + rgbPixels: decoded.rgbPixels, + width: decoded.width, + height: decoded.height, + }; + loadedPreviewUrl = decoded.previewUrl; + setStatus(`Loaded ${decoded.width}×${decoded.height} image from ${file.name}.`); + } catch (err) { + setStatus(`Failed to load image: ${formatError(err)}`); + } finally { + isBusy = false; renderView(); - return; } +} +/** + * Decode an image file into the same raw-RGB frame shape the camera produces: + * aspect-preserving downscale so the longest side is `maxDim`, alpha stripped. + */ +async function decodeImageToRgbFrame( + file: File, + maxDim: number, +): Promise<{ rgbPixels: Uint8Array; width: number; height: number; previewUrl: string }> { + const objectUrl = URL.createObjectURL(file); + try { + const img = await loadImageElement(objectUrl); + const longest = Math.max(img.naturalWidth, img.naturalHeight) || 1; + const scale = Math.min(1, maxDim / longest); + const width = Math.max(1, Math.round(img.naturalWidth * scale)); + const height = Math.max(1, Math.round(img.naturalHeight * scale)); + + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d', { willReadFrequently: true }); + if (!ctx) throw new Error('2D canvas context unavailable'); + ctx.drawImage(img, 0, 0, width, height); + + const { data } = ctx.getImageData(0, 0, width, height); // RGBA + const rgbPixels = new Uint8Array(width * height * 3); + for (let src = 0, dst = 0; src < data.length; src += 4, dst += 3) { + rgbPixels[dst] = data[src]; + rgbPixels[dst + 1] = data[src + 1]; + rgbPixels[dst + 2] = data[src + 2]; + } + return { rgbPixels, width, height, previewUrl: canvas.toDataURL('image/png') }; + } finally { + URL.revokeObjectURL(objectUrl); + } +} + +function loadImageElement(src: string): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = () => reject(new Error('Could not decode the selected image')); + img.src = src; + }); +} + +// --------------------------------------------------------------------------- +// Analyze +// --------------------------------------------------------------------------- + +async function onAnalyze(): Promise { // Gate on the lifecycle's loaded multimodal model — iOS parity: // VLMViewModel.swift:58-62 checkModelStatus() (currentModel(.multimodal)). if (!isVLMModelLoaded()) { @@ -259,9 +355,11 @@ async function onAnalyze(): Promise { return; } - const frame = latestFrame ?? camera.captureFrame(CAPTURE_DIMENSION); + // Frame source: an already-captured/loaded frame, or a fresh camera grab. + const frame = + latestFrame ?? (camera?.isCapturing ? camera.captureFrame(CAPTURE_DIMENSION) : null); if (!frame) { - setStatus('Failed to capture a frame for analysis.'); + setStatus('Capture a camera frame or load an image first.'); renderView(); return; } diff --git a/package.json b/package.json index 341d7e312d..6daaebdafb 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,6 @@ ], "packageManager": "yarn@3.6.1", "resolutions": { - "react": "19.2.3" + "react": "19.2.7" } } diff --git a/sdk/runanywhere-commons/CMakeLists.txt b/sdk/runanywhere-commons/CMakeLists.txt index 5710e9b21a..51e16a6978 100644 --- a/sdk/runanywhere-commons/CMakeLists.txt +++ b/sdk/runanywhere-commons/CMakeLists.txt @@ -947,12 +947,14 @@ set(RAC_FEATURES_SOURCES # voice_agent.cpp — lifecycle (create/destroy/init) # voice_agent_proto_abi.cpp — synchronous proto C ABI # voice_agent_d7_abi.cpp — full-session proto ABI + # voice_agent_feed_abi.cpp — streaming raw-frame ingress + segmentation # voice_agent_audio_pipeline_state.cpp — audio pipeline state helpers # voice_agent_internal_helpers.cpp — shared emit/state helpers src/features/voice_agent/voice_agent.cpp src/features/voice_agent/voice_agent_internal_helpers.cpp src/features/voice_agent/voice_agent_proto_abi.cpp src/features/voice_agent/voice_agent_d7_abi.cpp + src/features/voice_agent/voice_agent_feed_abi.cpp src/features/voice_agent/voice_agent_audio_pipeline_state.cpp # Proto-byte event ABI for VoiceEvent streaming. src/features/voice_agent/rac_voice_event_abi.cpp diff --git a/sdk/runanywhere-commons/exports/RACommons.exports b/sdk/runanywhere-commons/exports/RACommons.exports index c5efd26a8b..713035d025 100644 --- a/sdk/runanywhere-commons/exports/RACommons.exports +++ b/sdk/runanywhere-commons/exports/RACommons.exports @@ -319,6 +319,7 @@ _rac_voice_agent_component_states_proto _rac_voice_agent_process_voice_turn_proto # Full-session voice-agent ABI + per-helper proto wrappers. _rac_voice_agent_process_turn_proto +_rac_voice_agent_feed_audio_proto _rac_voice_agent_transcribe_proto _rac_voice_agent_synthesize_speech_proto _rac_voice_agent_component_create_proto diff --git a/sdk/runanywhere-commons/include/rac/features/voice_agent/rac_voice_agent.h b/sdk/runanywhere-commons/include/rac/features/voice_agent/rac_voice_agent.h index b16d78916a..128d49f33c 100644 --- a/sdk/runanywhere-commons/include/rac/features/voice_agent/rac_voice_agent.h +++ b/sdk/runanywhere-commons/include/rac/features/voice_agent/rac_voice_agent.h @@ -447,7 +447,19 @@ RAC_API rac_result_t rac_voice_agent_generate_response(rac_voice_agent_handle_t * through these APIs MUST call them once per detected utterance — the * C core will not transcribe anything until the SDK pushes a buffer. * - * 2. Continuous streaming (per-modality, no voice-agent aggregation): + * 2. Streaming raw-frame ingress (in-core segmentation): + * rac_voice_agent_feed_audio_proto(handle, pcm, n, sample_rate, + * channels, encoding, is_final, + * &result). + * The SDK pushes raw mic frames continuously (16 kHz mono PCM16); the + * C core performs energy-based utterance segmentation internally and, + * once an utterance closes, runs the same VAD -> STT -> LLM -> TTS + * pipeline as the per-turn path, fanning out VoiceEvents through the + * registered proto callback and returning the synthesized reply inline + * via `out_result`. This is the recommended ingress: the SDK driver + * reduces to "capture -> feed -> play" with no SDK-side VAD. + * + * 3. Continuous per-modality streaming (no voice-agent aggregation): * rac_stt_stream_feed_audio_proto(session, audio_bytes, n) and * rac_vad_stream_feed_audio_proto(session, audio_bytes, n). * SDKs that prefer per-frame VAD/STT events bypass the voice-agent @@ -455,17 +467,10 @@ RAC_API rac_result_t rac_voice_agent_generate_response(rac_voice_agent_handle_t * rac_voice_agent_transcribe_proto / synthesize_speech_proto / * generate-LLM helpers for the remaining stages. * - * Planned (NOT yet wired): a streaming - * `rac_voice_agent_feed_audio_proto(handle, samples, n, sample_rate, - * is_final)` entry point that internally tees the audio into the STT/VAD - * streams and emits an aggregated VoiceEvent stream identical to the - * per-turn path. Until that lands, SDK frontends MUST use one of the two - * modes above; the voice-agent will NOT silently consume mic audio. - * * Frontend authors: if a voice-screen view-model only calls - * rac_voice_agent_set_proto_callback / rac_voice_agent_process_turn_proto - * without ever pushing an audio buffer, expect dead-air. Either drive the - * per-turn API per utterance, or attach a parallel STT/VAD stream session. + * rac_voice_agent_set_proto_callback without ever pushing audio, expect + * dead-air. Either feed raw frames (mode 2), drive the per-turn API per + * utterance (mode 1), or attach a parallel STT/VAD stream session (mode 3). */ /** @@ -524,6 +529,45 @@ RAC_API rac_result_t rac_voice_agent_process_turn_proto( rac_voice_agent_handle_t handle, const uint8_t* request_bytes, size_t request_size, rac_voice_agent_turn_event_callback_fn event_callback, void* user_data); +/** + * @brief Streaming raw-frame audio ingress with in-core segmentation. + * + * Ingress mode 2 (see the @ref voice_agent_audio_ingress section). The SDK + * pushes raw mic frames as they are captured — 16 kHz mono signed-16-bit + * little-endian PCM (UNSPECIFIED encoding is treated as PCM_S16_LE). The C + * core accumulates them, performs energy-based utterance endpointing, and on + * each completed utterance runs the full VAD -> STT -> LLM -> TTS turn + * pipeline, emitting the same VoiceEvents as + * `rac_voice_agent_process_turn_proto` through the proto callback registered + * via `rac_voice_agent_set_proto_callback`. + * + * When a turn completes during a call, @p out_result is filled with a + * serialized `runanywhere.v1.VoiceAgentResult` carrying the transcript, + * assistant response, and synthesized reply as WAV bytes (for inline + * playback). When no utterance closes this call, @p out_result is an empty + * success buffer. Pass @p is_final = RAC_TRUE to flush any in-progress + * utterance (e.g. when the session is stopping). + * + * The call may block for the duration of a turn when an utterance closes; do + * not call it from a real-time audio callback — feed from a dedicated + * consumer loop. @p out_result must be released with rac_proto_buffer_free(). + * + * @param handle Voice agent handle. + * @param audio_data Raw PCM frame bytes (may be NULL only when size == 0). + * @param audio_size Size of @p audio_data in bytes. + * @param sample_rate_hz Sample rate hint (16000 expected). + * @param channels Channel count hint (1 expected). + * @param encoding AudioEncoding value (0/UNSPECIFIED or PCM_S16_LE). + * @param is_final RAC_TRUE to flush the in-progress utterance. + * @param out_result Output: serialized VoiceAgentResult (owned). + * @return RAC_SUCCESS or error code. + */ +RAC_API rac_result_t rac_voice_agent_feed_audio_proto(rac_voice_agent_handle_t handle, + const void* audio_data, size_t audio_size, + int32_t sample_rate_hz, int32_t channels, + int32_t encoding, rac_bool_t is_final, + rac_proto_buffer_t* out_result); + RAC_API rac_result_t rac_voice_agent_transcribe_proto(rac_voice_agent_handle_t handle, const uint8_t* request_bytes, size_t request_size, diff --git a/sdk/runanywhere-commons/include/rac/infrastructure/telemetry/rac_telemetry_manager.h b/sdk/runanywhere-commons/include/rac/infrastructure/telemetry/rac_telemetry_manager.h index 1634685861..4ca547ca58 100644 --- a/sdk/runanywhere-commons/include/rac/infrastructure/telemetry/rac_telemetry_manager.h +++ b/sdk/runanywhere-commons/include/rac/infrastructure/telemetry/rac_telemetry_manager.h @@ -19,6 +19,7 @@ #include "rac/core/rac_error.h" #include "rac/core/rac_types.h" +#include "rac/foundation/rac_proto_buffer.h" #include "rac/infrastructure/network/rac_environment.h" #include "rac/infrastructure/telemetry/rac_telemetry_types.h" @@ -137,6 +138,42 @@ RAC_API rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* */ RAC_API rac_result_t rac_telemetry_manager_flush(rac_telemetry_manager_t* manager); +// ============================================================================= +// ISOLATE-SAFE HTTP DELIVERY (poll-queue) +// ============================================================================= + +/** + * @brief Wake-up callback signalling that HTTP request(s) are queued. + * + * Carries no request data, so it is safe to invoke from any thread/isolate + * (Dart FFI synchronous callbacks are isolate-bound; this lets Flutter register + * a cross-isolate `NativeCallable.listener`). On wake-up the platform drains the + * queue via rac_telemetry_manager_poll_http_request(). SDKs that can invoke a + * data-carrying callback from any thread (JNI, C function pointer) keep using + * rac_telemetry_manager_set_http_callback() instead. + */ +typedef void (*rac_telemetry_http_wakeup_callback_t)(void* user_data); + +/** + * @brief Register the wake-up callback (selects poll-queue HTTP delivery). + * + * When set, flush enqueues each request and invokes [callback] instead of the + * data-carrying rac_telemetry_http_callback_t. + */ +RAC_API void rac_telemetry_manager_set_http_wakeup(rac_telemetry_manager_t* manager, + rac_telemetry_http_wakeup_callback_t callback, + void* user_data); + +/** + * @brief Pop the next queued HTTP request into [out] (owned; free with + * rac_proto_buffer_free). + * + * Framing: [u8 requires_auth][u32 LE endpoint_len][endpoint utf8][json utf8]. + * Returns RAC_ERROR_NOT_FOUND when the queue is empty. + */ +RAC_API rac_result_t rac_telemetry_manager_poll_http_request(rac_telemetry_manager_t* manager, + rac_proto_buffer_t* out); + // ============================================================================= // JSON SERIALIZATION // ============================================================================= diff --git a/sdk/runanywhere-commons/include/rac/infrastructure/telemetry/rac_telemetry_types.h b/sdk/runanywhere-commons/include/rac/infrastructure/telemetry/rac_telemetry_types.h index af06e2bd7b..ec1a0cb582 100644 --- a/sdk/runanywhere-commons/include/rac/infrastructure/telemetry/rac_telemetry_types.h +++ b/sdk/runanywhere-commons/include/rac/infrastructure/telemetry/rac_telemetry_types.h @@ -106,13 +106,41 @@ typedef struct rac_telemetry_payload { // VLM-specific fields (VLM also carries the LLM token fields above) int32_t image_count; + int32_t vision_tokens; // image/vision tokens (via properties carrier) + double vision_encode_time_ms; // vision encode time (via properties carrier) + const char* image_resolution; // "WxH" (via properties carrier; string → dup'd/freed) // RAG-specific fields int32_t retrieved_docs_count; // Embeddings-specific fields (embedding_model is read from model_id) - int32_t input_count; // texts embedded in the op - int32_t vectors_produced; // vectors returned + int32_t input_count; // texts embedded in the op + int32_t vectors_produced; // vectors returned + int32_t embedding_dimension; // vector dimension (via properties carrier) + + // RAG-specific extras (via properties carrier; retrieved_docs_count above) + int32_t top_k; + double retrieval_time_ms; + const char* embedding_model; // RAG embedding model (string → dup'd/freed) + + // LoRA-specific fields. Strings → must be dup'd/freed in track + payload_free. + const char* adapter_id; + const char* operation; // attach/detach/failed (derived from capability kind) + int64_t adapter_size_bytes; // adapter file size in bytes (via properties carrier) + + // ImageGen / diffusion fields (all via properties carrier). Strings (scheduler, + // output_format) → must be dup'd/freed in track + payload_free. + int32_t imagegen_prompt_length; + int32_t imagegen_negative_prompt_length; + int32_t image_width; + int32_t image_height; + int32_t num_images; + int32_t num_inference_steps; + double guidance_scale; + int64_t seed; + int64_t output_size_bytes; + const char* scheduler; + const char* output_format; // Voice-agent per-turn pipeline fields (from MetricsEvent) double voice_stt_ms; @@ -120,9 +148,14 @@ typedef struct rac_telemetry_payload { double voice_tts_ms; double voice_total_ms; - // VAD fields + // VAD fields (silence_duration_ms + segment_count via properties carrier) double speech_duration_ms; double silence_duration_ms; + int32_t segment_count; + + // Voice-agent per-turn fields (via properties carrier; voice_* timing above) + int32_t transcript_chars; + int32_t response_chars; // SDK lifecycle fields int32_t count; diff --git a/sdk/runanywhere-commons/src/core/events.cpp b/sdk/runanywhere-commons/src/core/events.cpp index 484022f642..5efe0917ed 100644 --- a/sdk/runanywhere-commons/src/core/events.cpp +++ b/sdk/runanywhere-commons/src/core/events.cpp @@ -383,13 +383,18 @@ void emit_sdk_models_loaded(int32_t count, double duration_ms) { void emit_model_download_started(const char* model_id, int64_t total_bytes, const char* archive_type) { - (void)archive_type; v1::ModelEvent m; m.set_kind(v1::MODEL_EVENT_KIND_DOWNLOAD_STARTED); if (model_id) m.set_model_id(model_id); m.set_total_bytes(total_bytes); - publish(v1::SDK_COMPONENT_UNSPECIFIED, v1::EVENT_CATEGORY_DOWNLOAD, std::move(m)); + // ModelEvent has no archive_type field → carry it on the envelope + // properties (read into payload.archive_type by the kModel extraction). + v1::SDKEvent event; + *event.mutable_model() = std::move(m); + if (archive_type != nullptr && archive_type[0] != '\0') + (*event.mutable_properties())["archive_type"] = archive_type; + publish(event, v1::SDK_COMPONENT_UNSPECIFIED, v1::EVENT_CATEGORY_DOWNLOAD); } void emit_model_download_progress(const char* model_id, double progress, int64_t bytes_downloaded, @@ -410,7 +415,6 @@ void emit_model_download_progress(const char* model_id, double progress, int64_t void emit_model_download_completed(const char* model_id, int64_t size_bytes, double duration_ms, const char* archive_type) { - (void)archive_type; v1::ModelEvent m; m.set_kind(v1::MODEL_EVENT_KIND_DOWNLOAD_COMPLETED); if (model_id) @@ -418,7 +422,11 @@ void emit_model_download_completed(const char* model_id, int64_t size_bytes, dou m.set_model_size_bytes(size_bytes); m.set_duration_ms(static_cast(duration_ms)); m.set_progress(1.0f); - publish(v1::SDK_COMPONENT_UNSPECIFIED, v1::EVENT_CATEGORY_DOWNLOAD, std::move(m)); + v1::SDKEvent event; + *event.mutable_model() = std::move(m); + if (archive_type != nullptr && archive_type[0] != '\0') + (*event.mutable_properties())["archive_type"] = archive_type; + publish(event, v1::SDK_COMPONENT_UNSPECIFIED, v1::EVENT_CATEGORY_DOWNLOAD); } void emit_model_download_failed(const char* model_id, rac_result_t error_code, diff --git a/sdk/runanywhere-commons/src/features/diffusion/diffusion_module.cpp b/sdk/runanywhere-commons/src/features/diffusion/diffusion_module.cpp index e99543e3e6..c7f3ff6321 100644 --- a/sdk/runanywhere-commons/src/features/diffusion/diffusion_module.cpp +++ b/sdk/runanywhere-commons/src/features/diffusion/diffusion_module.cpp @@ -769,7 +769,11 @@ void publish_event(const runanywhere::v1::SDKEvent& event) { void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, const char* operation, float progress, const char* error, double duration_ms = 0.0, - const char* model_id = nullptr) { + const char* model_id = nullptr, int32_t prompt_length = 0, + int32_t negative_prompt_length = 0, int32_t image_width = 0, + int32_t image_height = 0, int32_t num_inference_steps = 0, + double guidance_scale = 0.0, int64_t seed = 0, + int64_t output_size_bytes = 0) { runanywhere::v1::SDKEvent event; event.set_id(event_id()); event.set_timestamp_ms(now_ms()); @@ -797,6 +801,28 @@ void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, cons if (duration_ms > 0.0) { (*event.mutable_properties())["duration_ms"] = std::to_string(duration_ms); } + // ImageGen detail fields ride the properties carrier (extracted in the + // telemetry kCapability SDK_COMPONENT_DIFFUSION branch). Gated on >0 so the + // started/failed emits (which pass defaults) don't carry them. + if (prompt_length > 0) + (*event.mutable_properties())["prompt_length"] = std::to_string(prompt_length); + if (negative_prompt_length > 0) + (*event.mutable_properties())["negative_prompt_length"] = + std::to_string(negative_prompt_length); + if (image_width > 0) + (*event.mutable_properties())["image_width"] = std::to_string(image_width); + if (image_height > 0) + (*event.mutable_properties())["image_height"] = std::to_string(image_height); + if (num_inference_steps > 0) + (*event.mutable_properties())["num_inference_steps"] = std::to_string(num_inference_steps); + if (guidance_scale > 0.0) + (*event.mutable_properties())["guidance_scale"] = std::to_string(guidance_scale); + if (output_size_bytes > 0) { + (*event.mutable_properties())["output_size_bytes"] = std::to_string(output_size_bytes); + // seed is meaningful (incl. 0) but only emit it on a real completed + // generation — keyed off output_size_bytes being present. + (*event.mutable_properties())["seed"] = std::to_string(seed); + } publish_event(event); } @@ -926,9 +952,14 @@ rac_result_t rac_diffusion_generate_proto(rac_handle_t handle, const uint8_t* op rc = copy_proto(proto, out_result); } if (rc == RAC_SUCCESS) { - publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_DIFFUSION_COMPLETED, - "diffusion.generate", 1.0f, nullptr, - static_cast(result.generation_time_ms), model_id); + publish_capability( + runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_DIFFUSION_COMPLETED, + "diffusion.generate", 1.0f, nullptr, static_cast(result.generation_time_ms), + model_id, options.prompt ? static_cast(strlen(options.prompt)) : 0, + options.negative_prompt ? static_cast(strlen(options.negative_prompt)) : 0, + result.width, result.height, options.steps, + static_cast(options.guidance_scale), result.seed_used, + static_cast(result.image_size)); } else { publish_failure(rc, "diffusion.generate", rac_error_message(rc)); } @@ -989,9 +1020,14 @@ rac_result_t rac_diffusion_generate_with_progress_proto( rc = copy_proto(proto, out_result); } if (rc == RAC_SUCCESS) { - publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_DIFFUSION_COMPLETED, - "diffusion.generate", 1.0f, nullptr, - static_cast(result.generation_time_ms), model_id); + publish_capability( + runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_DIFFUSION_COMPLETED, + "diffusion.generate", 1.0f, nullptr, static_cast(result.generation_time_ms), + model_id, options.prompt ? static_cast(strlen(options.prompt)) : 0, + options.negative_prompt ? static_cast(strlen(options.negative_prompt)) : 0, + result.width, result.height, options.steps, + static_cast(options.guidance_scale), result.seed_used, + static_cast(result.image_size)); } else { publish_failure(rc, "diffusion.generate", rac_error_message(rc)); } diff --git a/sdk/runanywhere-commons/src/features/embeddings/embeddings_module.cpp b/sdk/runanywhere-commons/src/features/embeddings/embeddings_module.cpp index d7457e2ecc..506eafb311 100644 --- a/sdk/runanywhere-commons/src/features/embeddings/embeddings_module.cpp +++ b/sdk/runanywhere-commons/src/features/embeddings/embeddings_module.cpp @@ -376,7 +376,9 @@ void publish_event(const runanywhere::v1::SDKEvent& event) { void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, const char* operation, float progress, int64_t input_count, int64_t output_count, - const char* error, double duration_ms = 0.0) { + const char* error, double duration_ms = 0.0, + int64_t embedding_dimension = 0, const char* model_id = nullptr, + const char* framework = nullptr) { runanywhere::v1::SDKEvent event; event.set_id(event_id()); event.set_timestamp_ms(now_ms()); @@ -390,6 +392,15 @@ void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, cons auto* cap = event.mutable_capability(); cap->set_kind(kind); cap->set_component(runanywhere::v1::SDK_COMPONENT_EMBEDDINGS); + // model_id → telemetry base model_id + embeddings embedding_model column; + // framework rides the properties carrier (CapabilityOperationEvent has no + // framework field). + if (model_id != nullptr && model_id[0] != '\0') { + cap->set_model_id(model_id); + } + if (framework != nullptr && framework[0] != '\0') { + (*event.mutable_properties())["framework"] = framework; + } if (operation) { event.set_operation_id(operation); cap->set_operation(operation); @@ -404,6 +415,10 @@ void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, cons if (duration_ms > 0.0) { (*event.mutable_properties())["duration_ms"] = std::to_string(duration_ms); } + if (embedding_dimension > 0) { + (*event.mutable_properties())["embedding_dimension"] = + std::to_string(embedding_dimension); + } publish_event(event); } @@ -636,6 +651,14 @@ rac_result_t rac_embeddings_embed_batch_lifecycle_proto(const uint8_t* request_p "EmbeddingsRequest.texts is required"); } + // Telemetry: the lifecycle embed path is the one platform SDKs call, so it + // must publish the embeddings capability events (the component-handle path's + // publishes never fire for them). input_count = texts; output_count = + // vectors produced (extracted into the embeddings V2 row). + const int64_t embed_start_ms = now_ms(); + publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_EMBEDDINGS_STARTED, + "embeddings.embed", 0.0f, static_cast(texts.size()), 0, nullptr); + rac_embeddings_options_t options = RAC_EMBEDDINGS_OPTIONS_DEFAULT; if (request.has_options() && !rac::foundation::rac_embeddings_options_from_proto(request.options(), &options)) { @@ -654,6 +677,7 @@ rac_result_t rac_embeddings_embed_batch_lifecycle_proto(const uint8_t* request_p rac_embeddings_result_t raw = {}; rc = rac_embeddings_embed_batch(&service, c_texts.data(), c_texts.size(), &options, &raw); if (rc != RAC_SUCCESS) { + publish_failure(rc, "embeddings.embed", rac_error_message(rc)); rac::lifecycle::release_lifecycle_embeddings(&ref); return rac_proto_buffer_set_error(out_result, rc, rac_error_message(rc)); } @@ -671,6 +695,12 @@ rac_result_t rac_embeddings_embed_batch_lifecycle_proto(const uint8_t* request_p } result.set_model_id(ref.model_id ? ref.model_id : ""); result.set_request_id(request.request_id()); + publish_capability( + runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_EMBEDDINGS_COMPLETED, "embeddings.embed", + 1.0f, static_cast(texts.size()), static_cast(result.vectors_size()), + nullptr, static_cast(now_ms() - embed_start_ms), + raw.num_embeddings > 0 ? static_cast(raw.embeddings[0].dimension) : 0, ref.model_id, + ref.framework_name); rc = copy_proto(result, out_result); rac_embeddings_result_free(&raw); rac::lifecycle::release_lifecycle_embeddings(&ref); diff --git a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp index 037e5290d3..d3284813d1 100644 --- a/sdk/runanywhere-commons/src/features/llm/llm_module.cpp +++ b/sdk/runanywhere-commons/src/features/llm/llm_module.cpp @@ -1293,13 +1293,24 @@ rac_result_t publish_sdk_event(const SDKEvent& event) { void publish_generation_event(GenerationEventKind kind, const char* prompt, const char* token, const char* response, const char* error, const char* model_id, - int32_t token_count, int64_t latency_ms, int32_t input_tokens = 0) { + int32_t token_count, int64_t latency_ms, int32_t input_tokens = 0, + const char* framework_name = nullptr, + double tokens_per_second = 0.0, double ttft_ms = 0.0, + float temperature = -1.0f, int32_t max_tokens = 0, + int32_t context_length = 0, bool is_streaming = false) { SDKEvent event; const bool failed = kind == runanywhere::v1::GENERATION_EVENT_KIND_FAILED; populate_event_envelope(&event, runanywhere::v1::EVENT_CATEGORY_LLM, failed ? runanywhere::v1::ERROR_SEVERITY_ERROR : runanywhere::v1::ERROR_SEVERITY_INFO); event.set_operation_id("llm.generate"); + // This proto-path emitter has no framework proto field wired; carry the + // lifecycle ref's framework_name on the properties map (the kGeneration + // telemetry extraction normalizes it via clean_framework). Without this, + // LLM rows show no framework. + if (framework_name != nullptr && framework_name[0] != '\0') { + (*event.mutable_properties())["framework"] = framework_name; + } auto* generation = event.mutable_generation(); generation->set_kind(kind); if ((prompt != nullptr) && prompt[0] != '\0') { @@ -1327,9 +1338,43 @@ void publish_generation_event(GenerationEventKind kind, const char* prompt, cons if (input_tokens > 0) { generation->set_input_tokens(input_tokens); } + // Completion metrics (proto-path parity with the component path's + // emit_llm_generation_completed). All use existing GenerationEvent proto + // fields; the kGeneration telemetry extraction already reads them. + if (tokens_per_second > 0.0) { + generation->set_tokens_per_second(tokens_per_second); + } + if (ttft_ms > 0.0) { + generation->set_time_to_first_token_ms(static_cast(ttft_ms)); + } + // temperature 0.0 is a valid (greedy) setting, so the sentinel for "unset" + // is a negative default — emit any non-negative value. + if (temperature >= 0.0f) { + generation->set_temperature(temperature); + } + if (max_tokens > 0) { + generation->set_max_tokens(max_tokens); + } + if (context_length > 0) { + generation->set_context_length(context_length); + } + generation->set_is_streaming(is_streaming); (void)publish_sdk_event(event); } +// Best-effort context-length lookup for the handle-less proto path (the +// component path reads it from config; here we query the engine ops vtable). +int32_t lifecycle_context_length(const rac::llm::LifecycleLlmRef& ref) { + if (ref.ops == nullptr || ref.ops->get_info == nullptr) { + return 0; + } + rac_llm_info_t info{}; + if (ref.ops->get_info(ref.impl, &info) != RAC_SUCCESS) { + return 0; + } + return info.context_length; +} + SDKEvent make_cancellation_event(CancellationEventKind kind, const char* reason, rac_bool_t user_initiated, ErrorSeverity severity) { SDKEvent event; @@ -1855,8 +1900,8 @@ rac_result_t rac_llm_generate_proto(const uint8_t* request_proto_bytes, size_t r rac::llm::clear_lifecycle_llm_cancel(&ref); publish_generation_event(runanywhere::v1::GENERATION_EVENT_KIND_STARTED, - request.prompt().c_str(), nullptr, nullptr, nullptr, ref.model_id, 0, - 0); + request.prompt().c_str(), nullptr, nullptr, nullptr, ref.model_id, 0, 0, + 0, ref.framework_name); const std::string system_prompt = system_prompt_from_request(request); std::vector stop_storage; @@ -1881,7 +1926,7 @@ rac_result_t rac_llm_generate_proto(const uint8_t* request_proto_bytes, size_t r if (rc != RAC_SUCCESS) { publish_generation_event(runanywhere::v1::GENERATION_EVENT_KIND_FAILED, request.prompt().c_str(), nullptr, nullptr, rac_error_message(rc), - ref.model_id, 0, elapsed); + ref.model_id, 0, elapsed, 0, ref.framework_name); rac::llm::release_lifecycle_llm(&ref); return rac_proto_buffer_set_error(out_result, rc, rac_error_message(rc)); } @@ -1906,7 +1951,11 @@ rac_result_t rac_llm_generate_proto(const uint8_t* request_proto_bytes, size_t r publish_generation_event( runanywhere::v1::GENERATION_EVENT_KIND_COMPLETED, request.prompt().c_str(), nullptr, response, nullptr, ref.model_id, raw.completion_tokens, - raw.total_time_ms > 0 ? raw.total_time_ms : elapsed, raw.prompt_tokens); + raw.total_time_ms > 0 ? raw.total_time_ms : elapsed, + raw.prompt_tokens > 0 ? raw.prompt_tokens : estimate_tokens(request.prompt().c_str()), + ref.framework_name, static_cast(raw.tokens_per_second), + static_cast(raw.time_to_first_token_ms), options.temperature, options.max_tokens, + lifecycle_context_length(ref), /*is_streaming=*/false); rac_llm_result_free(&raw); rac::llm::release_lifecycle_llm(&ref); @@ -1953,8 +2002,8 @@ rac_result_t rac_llm_generate_stream_proto(const uint8_t* request_proto_bytes, rac::llm::clear_lifecycle_llm_cancel(&ref); publish_generation_event(runanywhere::v1::GENERATION_EVENT_KIND_STARTED, - request.prompt().c_str(), nullptr, nullptr, nullptr, ref.model_id, 0, - 0); + request.prompt().c_str(), nullptr, nullptr, nullptr, ref.model_id, 0, 0, + 0, ref.framework_name); const std::string system_prompt = system_prompt_from_request(request); std::vector stop_storage; @@ -2000,14 +2049,15 @@ rac_result_t rac_llm_generate_stream_proto(const uint8_t* request_proto_bytes, dispatch_terminal_once(&ctx, "cancelled", nullptr); publish_generation_event(runanywhere::v1::GENERATION_EVENT_KIND_CANCELLED, request.prompt().c_str(), nullptr, ctx.response_text.c_str(), - nullptr, ref.model_id, ctx.token_count, now_ms() - ctx.started_ms); + nullptr, ref.model_id, ctx.token_count, now_ms() - ctx.started_ms, + 0, ref.framework_name); rc = RAC_SUCCESS; } else if (rc != RAC_SUCCESS) { dispatch_terminal_once(&ctx, "error", rac_error_message(rc)); publish_generation_event(runanywhere::v1::GENERATION_EVENT_KIND_FAILED, request.prompt().c_str(), nullptr, ctx.response_text.c_str(), rac_error_message(rc), ref.model_id, ctx.token_count, - now_ms() - ctx.started_ms); + now_ms() - ctx.started_ms, 0, ref.framework_name); } else { // Mirror the OpenAI-style finish_reason // contract from llm_component.cpp:867-884 and rac_llm_generate_proto's @@ -2020,9 +2070,19 @@ rac_result_t rac_llm_generate_stream_proto(const uint8_t* request_proto_bytes, const char* finish_reason = (options.max_tokens > 0 && ctx.token_count >= options.max_tokens) ? "length" : "stop"; dispatch_terminal_once(&ctx, finish_reason, nullptr); - publish_generation_event(runanywhere::v1::GENERATION_EVENT_KIND_STREAM_COMPLETED, - request.prompt().c_str(), nullptr, ctx.response_text.c_str(), - nullptr, ref.model_id, ctx.token_count, now_ms() - ctx.started_ms); + const int64_t stream_elapsed = now_ms() - ctx.started_ms; + publish_generation_event( + runanywhere::v1::GENERATION_EVENT_KIND_STREAM_COMPLETED, request.prompt().c_str(), + nullptr, ctx.response_text.c_str(), nullptr, ref.model_id, ctx.token_count, + stream_elapsed, ctx.prompt_tokens, ref.framework_name, + (ctx.token_count > 0 && stream_elapsed > 0) + ? ctx.token_count * 1000.0 / static_cast(stream_elapsed) + : 0.0, + ctx.first_token_ms > ctx.started_ms + ? static_cast(ctx.first_token_ms - ctx.started_ms) + : 0.0, + options.temperature, options.max_tokens, lifecycle_context_length(ref), + /*is_streaming=*/true); } rac::llm::release_lifecycle_llm(&ref); diff --git a/sdk/runanywhere-commons/src/features/llm/tool_calling_session.cpp b/sdk/runanywhere-commons/src/features/llm/tool_calling_session.cpp index cc8a2ec57c..ab643aa38d 100644 --- a/sdk/runanywhere-commons/src/features/llm/tool_calling_session.cpp +++ b/sdk/runanywhere-commons/src/features/llm/tool_calling_session.cpp @@ -42,6 +42,9 @@ #if defined(RAC_HAVE_PROTOBUF) #include "errors.pb.h" #include "llm_service.pb.h" +#include "sdk_events.pb.h" + +#include "infrastructure/events/sdk_event_publish.h" #include "tool_calling.pb.h" #endif @@ -541,6 +544,35 @@ bool run_generate_once(ToolCallingSession& session, const std::string& prompt, if (out_response) { *out_response = raw.text ? raw.text : ""; } + + // Telemetry: each tool-calling turn IS an LLM generation, but the session + // only emits ToolCallingSessionEvents — so without this the "llm" V2 table + // never fills for tools-enabled chats. Publish a routed GenerationEvent so + // the telemetry sink records the turn (component=LLM → modality "llm"). + { + runanywhere::v1::SDKEvent llm_event; + auto* gen = llm_event.mutable_generation(); + gen->set_kind(runanywhere::v1::GENERATION_EVENT_KIND_COMPLETED); + if (ref.model_id != nullptr && ref.model_id[0] != '\0') { + gen->set_model_id(ref.model_id); + } + if (raw.completion_tokens > 0) { + gen->set_tokens_count(raw.completion_tokens); + gen->set_tokens_used(raw.completion_tokens); + } + if (raw.prompt_tokens > 0) { + gen->set_input_tokens(raw.prompt_tokens); + } + if (raw.tokens_per_second > 0.0f) { + gen->set_tokens_per_second(raw.tokens_per_second); + } + if (raw.time_to_first_token_ms > 0) { + gen->set_time_to_first_token_ms(raw.time_to_first_token_ms); + } + (void)rac::events::publish(llm_event, runanywhere::v1::SDK_COMPONENT_LLM, + runanywhere::v1::EVENT_CATEGORY_LLM); + } + rac_llm_result_free(&raw); rac::llm::release_lifecycle_llm(&ref); if (out_rc) diff --git a/sdk/runanywhere-commons/src/features/lora/rac_lora_service.cpp b/sdk/runanywhere-commons/src/features/lora/rac_lora_service.cpp index 5a166275cd..0da663c6e9 100644 --- a/sdk/runanywhere-commons/src/features/lora/rac_lora_service.cpp +++ b/sdk/runanywhere-commons/src/features/lora/rac_lora_service.cpp @@ -210,7 +210,8 @@ void publish_event(const runanywhere::v1::SDKEvent& event) { } void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, const char* operation, - const char* error) { + const char* error, const char* model_id = nullptr, + const char* adapter_id = nullptr, int64_t adapter_size_bytes = 0) { runanywhere::v1::SDKEvent event; event.set_id(event_id()); event.set_timestamp_ms(now_ms()); @@ -224,12 +225,21 @@ void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, cons auto* cap = event.mutable_capability(); cap->set_kind(kind); cap->set_component(runanywhere::v1::SDK_COMPONENT_LLM); + if (model_id != nullptr && model_id[0] != '\0') { + cap->set_model_id(model_id); // base model — telemetry "base_model_id" + } if (operation) { event.set_operation_id(operation); cap->set_operation(operation); } if (error) cap->set_error(error); + if (adapter_id != nullptr && adapter_id[0] != '\0') { + (*event.mutable_properties())["adapter_id"] = adapter_id; + } + if (adapter_size_bytes > 0) { + (*event.mutable_properties())["adapter_size_bytes"] = std::to_string(adapter_size_bytes); + } publish_event(event); } @@ -487,7 +497,7 @@ rac_result_t rac_lora_apply_proto(const uint8_t* request_proto_bytes, size_t req } track_lora_cleared(backend_impl, base_model_id); publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_DETACHED, - "lora.apply.replaceExisting", nullptr); + "lora.apply.replaceExisting", nullptr, base_model_id.c_str()); } for (const auto& config : request.adapters()) { @@ -518,8 +528,18 @@ rac_result_t rac_lora_apply_proto(const uint8_t* request_proto_bytes, size_t req track_lora_applied(backend_impl, base_model_id, applied_info); auto* info = result.add_adapters(); *info = applied_info; + // Adapter file size for telemetry — read via ifstream (portable; no + // dependency). Best-effort: 0 if the path can't be opened. + int64_t adapter_size = 0; + { + std::ifstream sz(config.adapter_path(), std::ios::binary | std::ios::ate); + if (sz) { + adapter_size = static_cast(sz.tellg()); + } + } publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_ATTACHED, - "lora.apply", nullptr); + "lora.apply", nullptr, base_model_id.c_str(), + config.adapter_id().c_str(), adapter_size); } result.set_success(true); @@ -580,7 +600,7 @@ rac_result_t rac_lora_remove_proto(const uint8_t* request_proto_bytes, size_t re track_lora_cleared(backend_impl, base_model_id); populate_tracked_state(backend_impl, base_model_id, &state); publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_DETACHED, - "lora.remove", nullptr); + "lora.remove", nullptr, base_model_id.c_str()); return finish(copy_proto(state, out_state)); } @@ -636,7 +656,7 @@ rac_result_t rac_lora_remove_proto(const uint8_t* request_proto_bytes, size_t re } track_lora_removed_path(backend_impl, base_model_id, adapter_path); publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_DETACHED, - "lora.remove", nullptr); + "lora.remove", nullptr, base_model_id.c_str()); } populate_tracked_state(backend_impl, base_model_id, &state); diff --git a/sdk/runanywhere-commons/src/features/rag/rac_rag_proto_abi.cpp b/sdk/runanywhere-commons/src/features/rag/rac_rag_proto_abi.cpp index 9c7e89ca32..023cc58103 100644 --- a/sdk/runanywhere-commons/src/features/rag/rac_rag_proto_abi.cpp +++ b/sdk/runanywhere-commons/src/features/rag/rac_rag_proto_abi.cpp @@ -91,7 +91,8 @@ void publish_event(const runanywhere::v1::SDKEvent& event) { void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, const char* operation, float progress, int64_t input_count, int64_t output_count, const char* error, double duration_ms = 0.0, - const char* model_id = nullptr) { + const char* model_id = nullptr, int64_t top_k = 0, + double retrieval_time_ms = 0.0, const char* embedding_model = nullptr) { runanywhere::v1::SDKEvent event; event.set_id(event_id()); event.set_timestamp_ms(now_ms()); @@ -121,6 +122,15 @@ void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, cons if (duration_ms > 0.0) { (*event.mutable_properties())["duration_ms"] = std::to_string(duration_ms); } + if (top_k > 0) { + (*event.mutable_properties())["top_k"] = std::to_string(top_k); + } + if (retrieval_time_ms > 0.0) { + (*event.mutable_properties())["retrieval_time_ms"] = std::to_string(retrieval_time_ms); + } + if (embedding_model != nullptr && embedding_model[0] != '\0') { + (*event.mutable_properties())["embedding_model"] = embedding_model; + } publish_event(event); } @@ -533,6 +543,7 @@ rac_result_t rac_rag_ingest_proto(rac_handle_t session, const uint8_t* document_ .count(); publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_RAG_INGESTION_COMPLETED, "rag.ingest", 1.0f, 1, stats.indexed_chunks(), nullptr, ingest_ms, + s->embedding_model_id.c_str(), /*top_k=*/0, /*retrieval_time_ms=*/0.0, s->embedding_model_id.c_str()); return rc; #endif @@ -667,7 +678,8 @@ rac_result_t rac_rag_query_proto(rac_handle_t session, const uint8_t* query_prot publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_RAG_QUERY_COMPLETED, "rag.query", 1.0f, 1, proto.retrieved_chunks_size(), nullptr, total_ms, s->llm_model_id.empty() ? s->embedding_model_id.c_str() - : s->llm_model_id.c_str()); + : s->llm_model_id.c_str(), + query_proto.top_k(), retrieval_ms, s->embedding_model_id.c_str()); rac_llm_result_free(&llm_result); return rc; #endif diff --git a/sdk/runanywhere-commons/src/features/stt/stt_module.cpp b/sdk/runanywhere-commons/src/features/stt/stt_module.cpp index 836910a8c5..dc588cd4ba 100644 --- a/sdk/runanywhere-commons/src/features/stt/stt_module.cpp +++ b/sdk/runanywhere-commons/src/features/stt/stt_module.cpp @@ -325,7 +325,8 @@ void publish_stt_lifecycle_event(runanywhere::v1::VoiceEventKind kind, const cha int64_t processing_ms, int64_t audio_length_ms, int32_t audio_size_bytes, int32_t word_count, double real_time_factor, const char* language, int32_t sample_rate, - const char* error) { + const char* error, const char* framework_name = nullptr, + bool is_streaming = false) { runanywhere::v1::VoiceLifecycleEvent voice; voice.set_kind(kind); if (model_id != nullptr && model_id[0] != '\0') { @@ -361,6 +362,16 @@ void publish_stt_lifecycle_event(runanywhere::v1::VoiceEventKind kind, const cha if (error != nullptr && error[0] != '\0') { voice.set_error(error); } + // Framework + streaming flag — the proto path otherwise leaves these unset, + // so STT rows showed no framework and is_streaming defaulted. Convert the + // lifecycle ref's framework name to the proto enum the same way the + // component path does (track reads v.framework() via framework_proto_to_string). + if (framework_name != nullptr && framework_name[0] != '\0') { + rac_inference_framework_t fw = RAC_FRAMEWORK_UNKNOWN; + (void)rac_inference_framework_from_string(framework_name, &fw); + voice.set_framework(rac::events::framework_to_proto_int(fw)); + } + voice.set_is_streaming(is_streaming); rac::events::publish_with_session(runanywhere::v1::SDK_COMPONENT_STT, runanywhere::v1::EVENT_CATEGORY_STT, std::move(voice), transcription_id); @@ -1452,7 +1463,8 @@ rac_result_t rac_stt_transcribe_lifecycle_proto(const uint8_t* request_proto_byt const std::string transcription_id = generate_unique_id(); publish_stt_lifecycle_event(runanywhere::v1::VOICE_EVENT_KIND_TRANSCRIPTION_STARTED, transcription_id.c_str(), ref.model_id, nullptr, 0.0f, 0, 0, 0, 0, - 0.0, options.language, options.sample_rate, nullptr); + 0.0, options.language, options.sample_rate, nullptr, + ref.framework_name, /*is_streaming=*/false); const auto transcribe_start = std::chrono::steady_clock::now(); rc = rac_stt_transcribe(&service, audio.data(), audio.size(), &options, &raw); @@ -1463,7 +1475,8 @@ rac_result_t rac_stt_transcribe_lifecycle_proto(const uint8_t* request_proto_byt publish_stt_lifecycle_event(runanywhere::v1::VOICE_EVENT_KIND_STT_FAILED, transcription_id.c_str(), ref.model_id, nullptr, 0.0f, processing_ms, 0, 0, 0, 0.0, options.language, - options.sample_rate, rac_error_message(rc)); + options.sample_rate, rac_error_message(rc), + ref.framework_name, /*is_streaming=*/false); rac::lifecycle::release_lifecycle_stt(&ref); return rac_proto_buffer_set_error(out_result, rc, rac_error_message(rc)); } @@ -1503,7 +1516,7 @@ rac_result_t rac_stt_transcribe_lifecycle_proto(const uint8_t* request_proto_byt runanywhere::v1::VOICE_EVENT_KIND_STT_COMPLETED, transcription_id.c_str(), ref.model_id, raw.text, raw.confidence, processing_ms, duration_ms, static_cast(audio.size()), word_count, real_time_factor, options.language, - options.sample_rate, nullptr); + options.sample_rate, nullptr, ref.framework_name, /*is_streaming=*/false); rc = copy_proto(output, out_result); rac_stt_result_free(&raw); diff --git a/sdk/runanywhere-commons/src/features/tts/tts_module.cpp b/sdk/runanywhere-commons/src/features/tts/tts_module.cpp index b228c8d0c2..53c93b4551 100644 --- a/sdk/runanywhere-commons/src/features/tts/tts_module.cpp +++ b/sdk/runanywhere-commons/src/features/tts/tts_module.cpp @@ -229,7 +229,8 @@ void publish_tts_voice_event(runanywhere::v1::VoiceEventKind kind, int64_t durat void publish_tts_lifecycle_event(runanywhere::v1::VoiceEventKind kind, const char* synthesis_id, const char* model_id, int32_t char_count, int64_t audio_duration_ms, int32_t audio_size_bytes, - int64_t processing_ms, int32_t sample_rate, const char* error) { + int64_t processing_ms, int32_t sample_rate, const char* error, + const char* framework_name = nullptr) { runanywhere::v1::VoiceLifecycleEvent voice; voice.set_kind(kind); if (model_id != nullptr && model_id[0] != '\0') { @@ -257,6 +258,14 @@ void publish_tts_lifecycle_event(runanywhere::v1::VoiceEventKind kind, const cha if (error != nullptr && error[0] != '\0') { voice.set_error(error); } + // Framework — proto path otherwise leaves it unset (track reads + // v.framework() via framework_proto_to_string). Convert the lifecycle ref's + // framework name to the proto enum like the component path does. + if (framework_name != nullptr && framework_name[0] != '\0') { + rac_inference_framework_t fw = RAC_FRAMEWORK_UNKNOWN; + (void)rac_inference_framework_from_string(framework_name, &fw); + voice.set_framework(rac::events::framework_to_proto_int(fw)); + } rac::events::publish_with_session(runanywhere::v1::SDK_COMPONENT_TTS, runanywhere::v1::EVENT_CATEGORY_TTS, std::move(voice), synthesis_id); @@ -1156,7 +1165,8 @@ rac_result_t rac_tts_synthesize_lifecycle_proto(const uint8_t* request_proto_byt const std::string synthesis_id = generate_uuid_v4(); const int32_t char_count = static_cast(text.size()); publish_tts_lifecycle_event(runanywhere::v1::VOICE_EVENT_KIND_SYNTHESIS_STARTED, - synthesis_id.c_str(), ref.model_id, char_count, 0, 0, 0, 0, nullptr); + synthesis_id.c_str(), ref.model_id, char_count, 0, 0, 0, 0, nullptr, + ref.framework_name); const auto synth_start = std::chrono::steady_clock::now(); rc = rac_tts_synthesize(&service, text.c_str(), &options, &raw); @@ -1166,7 +1176,7 @@ rac_result_t rac_tts_synthesize_lifecycle_proto(const uint8_t* request_proto_byt if (rc != RAC_SUCCESS) { publish_tts_lifecycle_event(runanywhere::v1::VOICE_EVENT_KIND_SYNTHESIS_FAILED, synthesis_id.c_str(), ref.model_id, char_count, 0, 0, - processing_ms, 0, rac_error_message(rc)); + processing_ms, 0, rac_error_message(rc), ref.framework_name); free_tts_options(&options); rac::lifecycle::release_lifecycle_tts(&ref); return rac_proto_buffer_set_error(out_result, rc, rac_error_message(rc)); @@ -1176,7 +1186,7 @@ rac_result_t rac_tts_synthesize_lifecycle_proto(const uint8_t* request_proto_byt synthesis_id.c_str(), ref.model_id, char_count, static_cast(raw.duration_ms), static_cast(raw.audio_size), processing_ms, - static_cast(raw.sample_rate), nullptr); + static_cast(raw.sample_rate), nullptr, ref.framework_name); runanywhere::v1::TTSOutput output; if (!rac::foundation::rac_tts_result_to_proto(&raw, &output)) { diff --git a/sdk/runanywhere-commons/src/features/vad/vad_module.cpp b/sdk/runanywhere-commons/src/features/vad/vad_module.cpp index 4554b85907..250a3f5207 100644 --- a/sdk/runanywhere-commons/src/features/vad/vad_module.cpp +++ b/sdk/runanywhere-commons/src/features/vad/vad_module.cpp @@ -116,6 +116,107 @@ struct rac_vad_component { namespace { +// VAD utterance accumulator for telemetry enrichment. Keyed by an opaque +// pointer so both the component activity path (key = component) and the +// handle-less lifecycle process path (key = a static sentinel) share it. Tracks +// per-utterance speech/silence duration + segment count, plus prev-frame speech +// state for edge detection on the lifecycle path (whose backend returns raw +// per-frame is_speech, not debounced transitions). Cleared on destroy/reset. +struct VadUtteranceState { + int64_t utterance_start_ms = 0; // when the current speech segment began + int64_t last_transition_ms = 0; // last STARTED/ENDED time (silence base) + int32_t segment_count = 0; // speech segments observed since reset + int32_t sample_rate = 0; // last seen sample rate (for end-flush on reset) + std::string model_id; // last seen VAD model id (for end-flush on reset) + bool prev_is_speech = false; // last frame verdict (lifecycle edge detect) + bool has_prev = false; // whether prev_is_speech is meaningful yet +}; + +// Per-utterance metrics returned by a transition; published to telemetry. +struct VadMetrics { + int64_t speech_ms = 0; + int64_t silence_ms = 0; + int32_t segment_count = 0; +}; + +std::mutex& vad_utterance_mutex() { + static std::mutex m; + return m; +} + +std::unordered_map& vad_utterance_states() { + static std::unordered_map m; + return m; +} + +// Stable sentinel key for the process-wide handle-less lifecycle VAD (singleton +// model → one logical session). +const void* lifecycle_vad_key() { + static const int sentinel = 0; + return &sentinel; +} + +void forget_vad_utterance_state(const void* key) { + std::lock_guard lock(vad_utterance_mutex()); + vad_utterance_states().erase(key); +} + +// Update durations/segment count for a STARTED/ENDED transition on @p st. +// Caller holds vad_utterance_mutex(). +VadMetrics apply_vad_transition(VadUtteranceState& st, bool started, int64_t now_ms) { + VadMetrics m; + if (started) { + if (st.last_transition_ms > 0) { + m.silence_ms = now_ms - st.last_transition_ms; + } + st.utterance_start_ms = now_ms; + st.last_transition_ms = now_ms; + st.segment_count += 1; + } else { + if (st.utterance_start_ms > 0) { + m.speech_ms = now_ms - st.utterance_start_ms; + } + st.last_transition_ms = now_ms; + } + m.segment_count = st.segment_count; + return m; +} + +// Component path: explicit STARTED/ENDED from the energy VAD's debounced state +// machine — just record the transition. +VadMetrics record_vad_transition(const void* key, bool started) { + std::lock_guard lock(vad_utterance_mutex()); + return apply_vad_transition(vad_utterance_states()[key], started, rac_get_current_time_ms()); +} + +// Lifecycle path: raw per-frame is_speech → detect the edge ourselves (treating +// pre-history as silence) and record the transition. fire==false → no edge. +struct VadEdge { + bool fire = false; + bool started = false; + VadMetrics metrics; +}; + +VadEdge step_lifecycle_vad(bool is_speech_now, int32_t sample_rate, const char* model_id) { + VadEdge edge; + std::lock_guard lock(vad_utterance_mutex()); + VadUtteranceState& st = vad_utterance_states()[lifecycle_vad_key()]; + st.sample_rate = sample_rate; + if (model_id != nullptr && model_id[0] != '\0') { + st.model_id = model_id; + } + const bool prev = st.has_prev && st.prev_is_speech; + st.prev_is_speech = is_speech_now; + st.has_prev = true; + if (is_speech_now == prev) { + return edge; // no transition + } + edge.fire = true; + edge.started = is_speech_now; + edge.metrics = apply_vad_transition(st, is_speech_now, rac_get_current_time_ms()); + return edge; +} + #if defined(RAC_HAVE_PROTOBUF) struct ProtoStreamSlot { @@ -260,6 +361,76 @@ void publish_vad_pipeline_event(bool is_speech, float confidence, float energy, (void)rac::events::publish_prebuilt(sdk_event); } +// Build + publish the per-utterance VAD speech-activity telemetry event for a +// transition. Shared by the component activity callback and the handle-less +// lifecycle process path. duration_ms(7) is read as speech_duration_ms by the +// telemetry extractor; silence_duration_ms / segment_count ride the envelope +// properties carrier (no VoiceLifecycleEvent fields). Telemetry-only destination. +void publish_vad_speech_telemetry(bool started, const VadMetrics& metrics, int32_t sample_rate, + const char* model_id) { + runanywhere::v1::SDKEvent event; + auto* voice = event.mutable_voice(); + voice->set_kind(started ? runanywhere::v1::VOICE_EVENT_KIND_SPEECH_STARTED + : runanywhere::v1::VOICE_EVENT_KIND_SPEECH_ENDED); + if (model_id != nullptr && model_id[0] != '\0') { + voice->set_model_id(model_id); + } + if (!started && metrics.speech_ms > 0) { + voice->set_duration_ms(metrics.speech_ms); + } + if (sample_rate > 0) { + voice->set_sample_rate(sample_rate); + } + if (metrics.silence_ms > 0) { + (*event.mutable_properties())["silence_duration_ms"] = std::to_string(metrics.silence_ms); + } + if (metrics.segment_count > 0) { + (*event.mutable_properties())["segment_count"] = std::to_string(metrics.segment_count); + } + event.set_destination(rac::events::legacy_destination_telemetry()); + (void)rac::events::publish(event, runanywhere::v1::SDK_COMPONENT_VAD, + runanywhere::v1::EVENT_CATEGORY_VAD); +} + +// Handle-less lifecycle process telemetry: per-frame VAD event for the in-app +// event stream + edge-detected speech-activity telemetry rows. The handle-less +// path (rac_vad_process_lifecycle_proto) otherwise publishes nothing, so +// standalone VAD never reached the telemetry dashboard. Mirrors the component +// path's two layers (event stream + per-utterance telemetry). +void emit_lifecycle_vad_telemetry(bool is_speech_now, int32_t sample_rate, float confidence, + float energy, int32_t duration_ms, const char* model_id) { + publish_vad_pipeline_event(is_speech_now, confidence, energy, duration_ms); + const VadEdge edge = step_lifecycle_vad(is_speech_now, sample_rate, model_id); + if (edge.fire) { + publish_vad_speech_telemetry(edge.started, edge.metrics, sample_rate, model_id); + } +} + +// On session reset, if a speech utterance is still open (the stream stopped +// mid-speech before a trailing silence frame arrived), emit its SPEECH_ENDED so +// every STARTED has a matching ENDED row in telemetry. +void flush_lifecycle_vad_end() { + bool emit = false; + VadMetrics metrics; + int32_t sample_rate = 0; + std::string model_id; + { + std::lock_guard lock(vad_utterance_mutex()); + VadUtteranceState& st = vad_utterance_states()[lifecycle_vad_key()]; + if (st.has_prev && st.prev_is_speech) { + sample_rate = st.sample_rate; + model_id = st.model_id; + metrics = apply_vad_transition(st, /*started=*/false, rac_get_current_time_ms()); + st.prev_is_speech = false; + emit = true; + } + } + if (emit) { + publish_vad_speech_telemetry(/*started=*/false, metrics, sample_rate, + model_id.empty() ? nullptr : model_id.c_str()); + } +} + void proto_activity_trampoline(rac_speech_activity_t activity, void* user_data) { const rac_handle_t handle = reinterpret_cast(user_data); ProtoStreamSlot slot; @@ -303,19 +474,16 @@ static void vad_speech_activity_callback(rac_speech_activity_event_t event, void if (!component) return; - // Emit telemetry-only voice-lifecycle event for speech activity. + // Emit telemetry-only voice-lifecycle event for speech activity, enriched + // with per-utterance metrics from the accumulator so each transition row + // reports speech/silence duration, segment count, and sample rate instead + // of a bare start/end with no data. #if defined(RAC_HAVE_PROTOBUF) { - runanywhere::v1::VoiceLifecycleEvent voice; - voice.set_kind(event == RAC_SPEECH_ACTIVITY_STARTED - ? runanywhere::v1::VOICE_EVENT_KIND_SPEECH_STARTED - : runanywhere::v1::VOICE_EVENT_KIND_SPEECH_ENDED); - // SPEECH_ENDED telemetry reads speech_duration_ms (= duration_ms); the - // legacy call site passed the default (0) here, preserved exactly. - rac::events::publish_with_session(runanywhere::v1::SDK_COMPONENT_VAD, - runanywhere::v1::EVENT_CATEGORY_VAD, std::move(voice), - /*session_id=*/nullptr, - rac::events::legacy_destination_telemetry()); + const bool started = (event == RAC_SPEECH_ACTIVITY_STARTED); + const VadMetrics metrics = record_vad_transition(component, started); + publish_vad_speech_telemetry(started, metrics, component->config.sample_rate, + component->loaded_model_id); } #endif @@ -511,6 +679,7 @@ extern "C" void rac_vad_component_destroy(rac_handle_t handle) { #if defined(RAC_HAVE_PROTOBUF) clear_proto_activity_slot(handle); #endif + forget_vad_utterance_state(component); // Cleanup first rac_vad_component_cleanup(handle); @@ -640,6 +809,9 @@ extern "C" rac_result_t rac_vad_component_reset(rac_handle_t handle) { auto* component = reinterpret_cast(handle); std::lock_guard lock(component->mtx); + // New session → restart the per-utterance accumulator (segment count etc.). + forget_vad_utterance_state(component); + if (!component->vad_service) { return RAC_ERROR_NOT_INITIALIZED; } @@ -1327,6 +1499,10 @@ rac_result_t rac_vad_process_lifecycle_proto(const uint8_t* request_proto_bytes, } rc = ref.ops->process(ref.impl, samples.data(), samples.size(), &is_speech); if (rc != RAC_SUCCESS) { + // Record the failure to telemetry — the handle-less path otherwise + // dropped failures silently (publish_vad_pipeline_event marks the event + // EVENT_CATEGORY_FAILURE with the error code/message). + publish_vad_pipeline_event(false, 0.0f, 0.0f, 0, rc); rac::lifecycle::release_lifecycle_vad(&ref); return rac_proto_buffer_set_error(out_result, rc, rac_error_message(rc)); } @@ -1346,6 +1522,14 @@ rac_result_t rac_vad_process_lifecycle_proto(const uint8_t* request_proto_bytes, } result.set_duration_ms(duration_ms); result.set_timestamp_ms(rac_get_current_time_ms()); + + // Emit telemetry for the standalone (handle-less) path: a per-frame VAD + // event for the in-app event stream + edge-detected speech-activity rows + // (started/ended with speech/silence duration + segment count). Without + // this, standalone VAD via processLifecycle never reached telemetry. + emit_lifecycle_vad_telemetry(is_speech == RAC_TRUE, sample_rate, result.confidence(), energy, + duration_ms, ref.model_id); + rc = copy_proto(result, out_result); rac::lifecycle::release_lifecycle_vad(&ref); return rc; @@ -1473,6 +1657,11 @@ rac_result_t rac_vad_reset_lifecycle_proto(rac_proto_buffer_t* out_result) { #if !defined(RAC_HAVE_PROTOBUF) return feature_unavailable_lifecycle(out_result); #else + // Close any utterance still open at stop (no trailing silence frame), then + // restart the lifecycle accumulator so durations don't bleed across sessions. + flush_lifecycle_vad_end(); + forget_vad_utterance_state(lifecycle_vad_key()); + rac::lifecycle::LifecycleVadRef ref; rac_result_t rc = rac::lifecycle::acquire_lifecycle_vad(&ref); if (rc != RAC_SUCCESS) { diff --git a/sdk/runanywhere-commons/src/features/vlm/vlm_module.cpp b/sdk/runanywhere-commons/src/features/vlm/vlm_module.cpp index 1819e4dfcf..ee5165981d 100644 --- a/sdk/runanywhere-commons/src/features/vlm/vlm_module.cpp +++ b/sdk/runanywhere-commons/src/features/vlm/vlm_module.cpp @@ -990,7 +990,12 @@ void publish_event(const runanywhere::v1::SDKEvent& event) { void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, const char* operation, float progress, int64_t input_count, int64_t output_count, const char* error, double duration_ms = 0.0, - const char* model_id = nullptr) { + const char* model_id = nullptr, int64_t input_tokens = 0, + int64_t total_tokens = 0, double tokens_per_second = 0.0, + double ttft_ms = 0.0, const char* framework = nullptr, + double temperature = -1.0, int32_t max_tokens = 0, + int64_t vision_tokens = 0, double vision_encode_ms = 0.0, + const char* image_resolution = nullptr) { runanywhere::v1::SDKEvent event; populate_envelope(&event, (error != nullptr && error[0] != '\0') ? runanywhere::v1::ERROR_SEVERITY_ERROR @@ -1001,6 +1006,9 @@ void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, cons if (model_id != nullptr && model_id[0] != '\0') { cap->set_model_id(model_id); } + if (framework != nullptr && framework[0] != '\0') { + (*event.mutable_properties())["framework"] = framework; + } if (operation) { event.set_operation_id(operation); cap->set_operation(operation); @@ -1015,6 +1023,38 @@ void publish_capability(runanywhere::v1::CapabilityOperationEventKind kind, cons if (duration_ms > 0.0) { (*event.mutable_properties())["duration_ms"] = std::to_string(duration_ms); } + // VLM token metrics ride the properties carrier (the VLM V2 row carries the + // LLM-style token fields; output_tokens comes from output_count above). + if (input_tokens > 0) { + (*event.mutable_properties())["input_tokens"] = std::to_string(input_tokens); + } + if (total_tokens > 0) { + (*event.mutable_properties())["total_tokens"] = std::to_string(total_tokens); + } + if (tokens_per_second > 0.0) { + (*event.mutable_properties())["tokens_per_second"] = std::to_string(tokens_per_second); + } + if (ttft_ms > 0.0) { + (*event.mutable_properties())["time_to_first_token_ms"] = std::to_string(ttft_ms); + } + // temperature=0.0 is a valid (greedy) setting, so a -1.0 sentinel marks + // "not provided"; max_tokens 0 means unset. + if (temperature >= 0.0) { + (*event.mutable_properties())["temperature"] = std::to_string(temperature); + } + if (max_tokens > 0) { + (*event.mutable_properties())["max_tokens"] = std::to_string(max_tokens); + } + // Vision-specific metrics (0 when the engine doesn't surface them → skipped). + if (vision_tokens > 0) { + (*event.mutable_properties())["vision_tokens"] = std::to_string(vision_tokens); + } + if (vision_encode_ms > 0.0) { + (*event.mutable_properties())["vision_encode_time_ms"] = std::to_string(vision_encode_ms); + } + if (image_resolution != nullptr && image_resolution[0] != '\0') { + (*event.mutable_properties())["image_resolution"] = image_resolution; + } publish_event(event); } @@ -1386,10 +1426,19 @@ rac_result_t rac_vlm_process_proto(rac_handle_t handle, const uint8_t* image_pro } else { rc = copy_proto(proto, out_result); } + const std::string vlm_res = (image.width > 0 && image.height > 0) + ? std::to_string(image.width) + "x" + std::to_string(image.height) + : std::string(); publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_VLM_COMPLETED, "vlm.process", 1.0f, 1, proto.completion_tokens(), nullptr, static_cast(proto.processing_time_ms()), - have_lifecycle ? lifecycle_ref.model_id : nullptr); + have_lifecycle ? lifecycle_ref.model_id : nullptr, proto.prompt_tokens(), + proto.total_tokens(), static_cast(proto.tokens_per_second()), + static_cast(proto.time_to_first_token_ms()), + have_lifecycle ? lifecycle_ref.framework_name : nullptr, + static_cast(options.temperature), options.max_tokens, + proto.image_tokens(), static_cast(proto.image_encode_time_ms()), + vlm_res.empty() ? nullptr : vlm_res.c_str()); rac_vlm_result_free(&result); free_vlm_image(&image); rac_free(const_cast(prompt)); @@ -1521,9 +1570,19 @@ rac_result_t rac_vlm_generate_proto(const uint8_t* request_proto_bytes, size_t r } else { rc = copy_proto(result, out_result); } + const std::string vlm_gen_res = + (image.width > 0 && image.height > 0) + ? std::to_string(image.width) + "x" + std::to_string(image.height) + : std::string(); publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_VLM_COMPLETED, "vlm.generate", 1.0f, 1, result.completion_tokens(), nullptr, - static_cast(result.processing_time_ms()), ref.model_id); + static_cast(result.processing_time_ms()), ref.model_id, + result.prompt_tokens(), result.total_tokens(), + static_cast(result.tokens_per_second()), + static_cast(result.time_to_first_token_ms()), ref.framework_name, + static_cast(options.temperature), options.max_tokens, + result.image_tokens(), static_cast(result.image_encode_time_ms()), + vlm_gen_res.empty() ? nullptr : vlm_gen_res.c_str()); rac_vlm_result_free(&raw); free_vlm_image(&image); rac_free(const_cast(prompt)); @@ -1624,9 +1683,18 @@ rac_result_t rac_vlm_stream_proto(const uint8_t* request_proto_bytes, size_t req elapsed_ms, &result); dispatch_vlm_terminal_once(&ctx, runanywhere::v1::VLM_STREAM_EVENT_KIND_COMPLETED, &result, nullptr, 0); + const std::string vlm_stream_res = + (image.width > 0 && image.height > 0) + ? std::to_string(image.width) + "x" + std::to_string(image.height) + : std::string(); publish_capability(runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_VLM_COMPLETED, "vlm.stream", 1.0f, 1, ctx.token_count, nullptr, - static_cast(elapsed_ms), ref.model_id); + static_cast(elapsed_ms), ref.model_id, result.prompt_tokens(), + result.total_tokens(), static_cast(result.tokens_per_second()), + static_cast(result.time_to_first_token_ms()), ref.framework_name, + static_cast(options.temperature), options.max_tokens, + result.image_tokens(), static_cast(result.image_encode_time_ms()), + vlm_stream_res.empty() ? nullptr : vlm_stream_res.c_str()); } free_vlm_image(&image); diff --git a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_d7_abi.cpp b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_d7_abi.cpp index 7d383e3c96..fe5b437c47 100644 --- a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_d7_abi.cpp +++ b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_d7_abi.cpp @@ -16,6 +16,7 @@ * global SDKEvent publisher. */ +#include #include #include #include @@ -232,55 +233,18 @@ std::string d7_pick_turn_id(const std::string& request_id) { } // namespace -#endif // RAC_HAVE_PROTOBUF +namespace rac::voice_agent::detail { -extern "C" rac_result_t rac_voice_agent_process_turn_proto( - rac_voice_agent_handle_t handle, const uint8_t* request_bytes, size_t request_size, - rac_voice_agent_turn_event_callback_fn event_callback, void* user_data) { -#if !defined(RAC_HAVE_PROTOBUF) - (void)handle; - (void)request_bytes; - (void)request_size; - (void)event_callback; - (void)user_data; - return RAC_ERROR_FEATURE_NOT_AVAILABLE; -#else - using namespace rac::voice_agent::detail; - if (!handle || !event_callback) +rac_result_t d7_process_utterance(rac_voice_agent_handle_t handle, const std::string& audio, + const std::string& session_id, const std::string& turn_id, + const std::string& request_id, const std::string& language_code, + rac_voice_agent_turn_event_callback_fn event_callback, + void* user_data, runanywhere::v1::VoiceAgentResult* out_result) { + if (audio.empty()) { + d7_emit_error(handle, RAC_ERROR_INVALID_ARGUMENT, "voice_agent", + "voice turn buffer is empty", session_id, turn_id, request_id, event_callback, + user_data); return RAC_ERROR_INVALID_ARGUMENT; - if (!proto_bytes_valid(request_bytes, request_size)) - return RAC_ERROR_DECODING_ERROR; - - runanywhere::v1::VoiceAgentTurnRequest request; - if (!request.ParseFromArray(proto_parse_data(request_bytes, request_size), - static_cast(request_size))) { - return RAC_ERROR_DECODING_ERROR; - } - - const std::string session_id = request.session_id(); - const std::string request_id = request.request_id(); - const std::string turn_id = d7_pick_turn_id(request_id); - - // Admit under the in-flight barrier so rac_voice_agent_destroy's - // drain loop covers this full STT+LLM+TTS turn. The d7 path reads - // is_configured below outside handle->mutex, so without the barrier a - // concurrent destroy could flip is_shutting_down after that read and tear - // the agent down mid-turn while this thread still emits events on it. - InFlightGuard guard(handle); - if (!guard.admitted()) { - d7_emit_error(handle, RAC_ERROR_INVALID_STATE, "voice_agent", - "voice agent is shutting down", session_id, turn_id, request_id, - event_callback, user_data); - return RAC_ERROR_INVALID_STATE; - } - - if (!handle->is_configured.load(std::memory_order_acquire)) { - d7_emit_error(handle, RAC_ERROR_NOT_INITIALIZED, "voice_agent", - "voice agent is not initialized", session_id, turn_id, request_id, - event_callback, user_data); - emit_component_failure(handle, "voice_agent", RAC_ERROR_NOT_INITIALIZED, - "voice agent is not initialized"); - return RAC_ERROR_NOT_INITIALIZED; } runanywhere::v1::VoiceAgentComponentStates component_states; @@ -312,18 +276,47 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( } } - const std::string& audio = request.audio_data(); - if (audio.empty()) { - d7_emit_error(handle, RAC_ERROR_INVALID_ARGUMENT, "voice_agent", - "voice turn request is missing audio_data", session_id, turn_id, request_id, - event_callback, user_data); - return RAC_ERROR_INVALID_ARGUMENT; - } + // Per-turn telemetry: publish a MetricsEvent on every exit (success or any + // failure) so the turn lands under the "voice" modality. This path uses + // early `return rc` at each stage, so an RAII guard is the clean way to + // cover all exits. Declared BEFORE the lock so it destructs (and publishes) + // AFTER the handle mutex is released. `armed` gates it to turns that + // actually started (not the pre-flight config rejections above). + struct TurnMetricsGuard { + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + double stt_ms = 0.0; + double llm_ms = 0.0; + double tts_ms = 0.0; + int64_t tokens = 0; + std::string session_id; + std::string model_id; + std::string framework; + int32_t transcript_chars = 0; + int32_t response_chars = 0; + rac_result_t error_code = RAC_SUCCESS; + std::string error_message; + bool armed = false; + ~TurnMetricsGuard() { + if (!armed) + return; + const double e2e_ms = std::chrono::duration( + std::chrono::steady_clock::now() - start) + .count(); + rac::voice_agent::detail::publish_voice_turn_metrics( + stt_ms, llm_ms, tts_ms, e2e_ms, tokens, + session_id.empty() ? nullptr : session_id.c_str(), + model_id.empty() ? nullptr : model_id.c_str(), + framework.empty() ? nullptr : framework.c_str(), transcript_chars, response_chars, + error_code, error_message.empty() ? nullptr : error_message.c_str()); + } + } turn_metrics; + turn_metrics.session_id = session_id; std::lock_guard lock(handle->mutex); emit_component_states(handle); emit_turn_lifecycle(handle, runanywhere::v1::TURN_LIFECYCLE_EVENT_KIND_STARTED); + turn_metrics.armed = true; d7_emit_state(handle, runanywhere::v1::PIPELINE_STATE_IDLE, runanywhere::v1::PIPELINE_STATE_LISTENING, session_id, turn_id, request_id, event_callback, user_data); @@ -377,6 +370,7 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( rac_stt_result_t stt = {}; rac_result_t rc; + const auto t_stt = std::chrono::steady_clock::now(); if (have_lifecycle_stt) { rac_stt_service_t stt_service{stt_ref.ops, stt_ref.impl, stt_ref.model_id}; rc = rac_stt_transcribe(&stt_service, audio.data(), audio.size(), nullptr, &stt); @@ -384,10 +378,14 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( rc = rac_stt_component_transcribe(handle->stt_handle, audio.data(), audio.size(), nullptr, &stt); } + turn_metrics.stt_ms = + std::chrono::duration(std::chrono::steady_clock::now() - t_stt).count(); if (rc != RAC_SUCCESS) { if (have_lifecycle_stt) { rac::lifecycle::release_lifecycle_stt(&stt_ref); } + turn_metrics.error_code = rc; + turn_metrics.error_message = "STT transcription failed"; d7_emit_error(handle, rc, "stt", "STT transcription failed", session_id, turn_id, request_id, event_callback, user_data); emit_component_failure(handle, "stt", rc, "STT transcription failed"); @@ -398,6 +396,8 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( if (have_lifecycle_stt) { rac::lifecycle::release_lifecycle_stt(&stt_ref); } + turn_metrics.error_code = RAC_ERROR_INVALID_STATE; + turn_metrics.error_message = "STT transcription was empty"; d7_emit_error(handle, RAC_ERROR_INVALID_STATE, "stt", "STT transcription was empty", session_id, turn_id, request_id, event_callback, user_data); emit_component_failure(handle, "stt", RAC_ERROR_INVALID_STATE, @@ -413,11 +413,9 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( /*is_speech=*/false, session_id, turn_id, request_id, event_callback, user_data); } - d7_emit_user_said(handle, stt.text, - request.session_config().has_language_code() - ? request.session_config().language_code() - : std::string(), - session_id, turn_id, request_id, event_callback, user_data); + turn_metrics.transcript_chars = stt.text ? static_cast(std::strlen(stt.text)) : 0; + d7_emit_user_said(handle, stt.text, language_code, session_id, turn_id, request_id, + event_callback, user_data); d7_emit_state(handle, runanywhere::v1::PIPELINE_STATE_PROCESSING_SPEECH, runanywhere::v1::PIPELINE_STATE_GENERATING_RESPONSE, session_id, turn_id, @@ -425,13 +423,23 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( rac::llm::LifecycleLlmRef llm_ref{}; const bool have_lifecycle_llm = rac::llm::acquire_lifecycle_llm(&llm_ref) == RAC_SUCCESS; + if (have_lifecycle_llm) { + if (llm_ref.model_id != nullptr) + turn_metrics.model_id = llm_ref.model_id; + if (llm_ref.framework_name != nullptr) + turn_metrics.framework = llm_ref.framework_name; + } rac_llm_result_t llm = {}; + const auto t_llm = std::chrono::steady_clock::now(); if (have_lifecycle_llm) { rac_llm_service_t llm_service{llm_ref.ops, llm_ref.impl, llm_ref.model_id}; rc = rac_llm_generate(&llm_service, stt.text, nullptr, &llm); } else { rc = rac_llm_component_generate(handle->llm_handle, stt.text, nullptr, &llm); } + turn_metrics.llm_ms = + std::chrono::duration(std::chrono::steady_clock::now() - t_llm).count(); + turn_metrics.tokens = llm.completion_tokens; if (rc != RAC_SUCCESS) { if (have_lifecycle_llm) { rac::llm::release_lifecycle_llm(&llm_ref); @@ -440,11 +448,14 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( if (have_lifecycle_stt) { rac::lifecycle::release_lifecycle_stt(&stt_ref); } + turn_metrics.error_code = rc; + turn_metrics.error_message = "LLM generation failed"; d7_emit_error(handle, rc, "llm", "LLM generation failed", session_id, turn_id, request_id, event_callback, user_data); emit_component_failure(handle, "llm", rc, "LLM generation failed"); return rc; } + turn_metrics.response_chars = llm.text ? static_cast(std::strlen(llm.text)) : 0; d7_emit_assistant_token(handle, llm.text, true, session_id, turn_id, request_id, event_callback, user_data); @@ -455,12 +466,15 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( rac::lifecycle::LifecycleTtsRef tts_ref{}; const bool have_lifecycle_tts = rac::lifecycle::acquire_lifecycle_tts(&tts_ref) == RAC_SUCCESS; rac_tts_result_t tts = {}; + const auto t_tts = std::chrono::steady_clock::now(); if (have_lifecycle_tts) { rac_tts_service_t tts_service{tts_ref.ops, tts_ref.impl, tts_ref.model_id}; rc = rac_tts_synthesize(&tts_service, llm.text, nullptr, &tts); } else { rc = rac_tts_component_synthesize(handle->tts_handle, llm.text, nullptr, &tts); } + turn_metrics.tts_ms = + std::chrono::duration(std::chrono::steady_clock::now() - t_tts).count(); if (rc != RAC_SUCCESS) { if (have_lifecycle_tts) { rac::lifecycle::release_lifecycle_tts(&tts_ref); @@ -473,6 +487,8 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( if (have_lifecycle_stt) { rac::lifecycle::release_lifecycle_stt(&stt_ref); } + turn_metrics.error_code = rc; + turn_metrics.error_message = "TTS synthesis failed"; d7_emit_error(handle, rc, "tts", "TTS synthesis failed", session_id, turn_id, request_id, event_callback, user_data); emit_component_failure(handle, "tts", rc, "TTS synthesis failed"); @@ -484,6 +500,31 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( tts.sample_rate > 0 ? tts.sample_rate : RAC_TTS_DEFAULT_SAMPLE_RATE, true, session_id, turn_id, request_id, event_callback, user_data); + // When the caller wants the synthesized reply inline (the streaming + // feed-audio ingress path), package transcript + response + TTS audio as + // a VoiceAgentResult. The audio is converted to a self-describing WAV so + // the SDK plays it directly without tracking raw float32 rate/encoding. + if (out_result) { + out_result->set_speech_detected(turn_has_speech); + out_result->set_transcription(stt.text); + if (llm.text) { + out_result->set_assistant_response(llm.text); + } + if (tts.audio_data && tts.audio_size > 0) { + void* wav_data = nullptr; + size_t wav_size = 0; + if (rac_audio_float32_to_wav( + tts.audio_data, tts.audio_size, + tts.sample_rate > 0 ? tts.sample_rate : RAC_TTS_DEFAULT_SAMPLE_RATE, &wav_data, + &wav_size) == RAC_SUCCESS && + wav_data && wav_size > 0) { + out_result->set_synthesized_audio(wav_data, wav_size); + std::free(wav_data); + } + } + fill_component_states(handle, out_result->mutable_final_state()); + } + // Honor the documented PLAYING_TTS -> COOLDOWN -> IDLE // pathway so frontends gating the microphone on // rac_audio_pipeline_can_activate_microphone() get the 800ms feedback- @@ -511,6 +552,69 @@ extern "C" rac_result_t rac_voice_agent_process_turn_proto( rac::lifecycle::release_lifecycle_stt(&stt_ref); } return RAC_SUCCESS; +} + +} // namespace rac::voice_agent::detail + +#endif // RAC_HAVE_PROTOBUF + +extern "C" rac_result_t rac_voice_agent_process_turn_proto( + rac_voice_agent_handle_t handle, const uint8_t* request_bytes, size_t request_size, + rac_voice_agent_turn_event_callback_fn event_callback, void* user_data) { +#if !defined(RAC_HAVE_PROTOBUF) + (void)handle; + (void)request_bytes; + (void)request_size; + (void)event_callback; + (void)user_data; + return RAC_ERROR_FEATURE_NOT_AVAILABLE; +#else + using namespace rac::voice_agent::detail; + if (!handle || !event_callback) + return RAC_ERROR_INVALID_ARGUMENT; + if (!proto_bytes_valid(request_bytes, request_size)) + return RAC_ERROR_DECODING_ERROR; + + runanywhere::v1::VoiceAgentTurnRequest request; + if (!request.ParseFromArray(proto_parse_data(request_bytes, request_size), + static_cast(request_size))) { + return RAC_ERROR_DECODING_ERROR; + } + + const std::string session_id = request.session_id(); + const std::string request_id = request.request_id(); + const std::string turn_id = d7_pick_turn_id(request_id); + + // Admit under the in-flight barrier so rac_voice_agent_destroy's + // drain loop covers this full STT+LLM+TTS turn. The d7 path reads + // is_configured below outside handle->mutex, so without the barrier a + // concurrent destroy could flip is_shutting_down after that read and tear + // the agent down mid-turn while this thread still emits events on it. + InFlightGuard guard(handle); + if (!guard.admitted()) { + d7_emit_error(handle, RAC_ERROR_INVALID_STATE, "voice_agent", + "voice agent is shutting down", session_id, turn_id, request_id, + event_callback, user_data); + return RAC_ERROR_INVALID_STATE; + } + + if (!handle->is_configured.load(std::memory_order_acquire)) { + d7_emit_error(handle, RAC_ERROR_NOT_INITIALIZED, "voice_agent", + "voice agent is not initialized", session_id, turn_id, request_id, + event_callback, user_data); + emit_component_failure(handle, "voice_agent", RAC_ERROR_NOT_INITIALIZED, + "voice agent is not initialized"); + return RAC_ERROR_NOT_INITIALIZED; + } + + // The VAD -> STT -> LLM -> TTS pipeline + event emission is shared with + // the streaming feed-audio ingress path (rac_voice_agent_feed_audio_proto). + const std::string language_code = + request.session_config().has_language_code() ? request.session_config().language_code() + : std::string(); + return rac::voice_agent::detail::d7_process_utterance( + handle, request.audio_data(), session_id, turn_id, request_id, language_code, + event_callback, user_data, /*out_result=*/nullptr); #endif } diff --git a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_feed_abi.cpp b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_feed_abi.cpp new file mode 100644 index 0000000000..6dc858ef84 --- /dev/null +++ b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_feed_abi.cpp @@ -0,0 +1,253 @@ +/** + * @file voice_agent_feed_abi.cpp + * @brief Streaming audio-ingress voice-agent C ABI — + * `rac_voice_agent_feed_audio_proto`. + * + * The C core owns no microphone (see the "Audio-Ingress Contract" in + * rac_voice_agent.h). Platform SDKs capture raw mic frames and push them + * here continuously; this TU performs energy-based utterance segmentation + * in-core (the logic that previously lived duplicated in every SDK's + * VoiceAgentMicDriver) and, once an utterance closes, runs the shared + * VAD -> STT -> LLM -> TTS pipeline (`d7_process_utterance`). The + * synthesized reply is returned inline as a `VoiceAgentResult` so the SDK + * driver collapses to "capture -> feed -> play", while the per-stage + * VoiceEvents still fan out through the registered proto callback. + * + * PCM contract: 16 kHz mono signed-16-bit little-endian (the format every + * SDK's AudioCaptureManager already produces). + */ + +#include +#include +#include +#include +#include +#include + +#include "rac/core/rac_error.h" +#include "rac/core/rac_types.h" +#include "rac/features/voice_agent/rac_voice_agent.h" +#include "rac/features/voice_agent/rac_voice_event_abi.h" +#include "rac/foundation/rac_proto_buffer.h" + +#if defined(RAC_HAVE_PROTOBUF) +#include "voice_agent_service.pb.h" +#include "voice_events.pb.h" +#endif + +#include "voice_agent_internal.h" +#include "voice_agent_internal_helpers.h" + +#if defined(RAC_HAVE_PROTOBUF) + +namespace { + +// Energy-VAD / endpointing constants. Ported verbatim from the Swift and +// Kotlin VoiceAgentMicDriver segmenters so on-device behavior is unchanged +// now that segmentation lives in one place. +constexpr int kSampleRateHz = 16000; +constexpr int kBytesPerSample = 2; +constexpr int kFrameMs = 100; +constexpr size_t kFrameBytes = + static_cast(kSampleRateHz * kFrameMs / 1000) * kBytesPerSample; // 3200 bytes +constexpr float kSpeechRmsThreshold = 0.015f; +constexpr float kSpeechFloorMultiplier = 2.2f; +constexpr float kNoiseFloorRise = 0.05f; +constexpr int kEndOfUtteranceSilenceMs = 800; +constexpr int kMinSpeechMs = 300; +constexpr int kMaxUtteranceMs = 15000; +constexpr size_t kPreRollFrames = 3; + +// Normalized RMS of one PCM16 frame (matches the SDK drivers: divide by +// Int16.max so the threshold constants carry over unchanged). +float frame_rms(const uint8_t* data, size_t bytes) { + const size_t samples = bytes / kBytesPerSample; + if (samples == 0) + return 0.0f; + const int16_t* pcm = reinterpret_cast(data); + double sum = 0.0; + for (size_t i = 0; i < samples; ++i) { + const double sample = static_cast(pcm[i]); + sum += sample * sample; + } + return static_cast(std::sqrt(sum / static_cast(samples)) / 32767.0); +} + +// Accumulate fed audio into fixed analysis frames and run energy endpointing. +// Returns true and moves the completed utterance into @p out_utterance when an +// utterance closes this call (silence tail or max-duration cap). At most one +// utterance is reported per call; any buffered backlog is dropped so the +// device's own TTS playout is not folded into the next turn (mirrors the SDK's +// former discard-pending-chunks behavior). The adaptive noise floor persists +// across turns; only transient state resets. +bool feed_segment(rac_voice_agent_feed_state& s, const void* data, size_t size, bool is_final, + std::string* out_utterance) { + if (data && size > 0) { + const uint8_t* bytes = static_cast(data); + s.frame_accum.insert(s.frame_accum.end(), bytes, bytes + size); + } + + bool completed = false; + while (s.frame_accum.size() >= kFrameBytes) { + std::vector frame(s.frame_accum.begin(), s.frame_accum.begin() + kFrameBytes); + s.frame_accum.erase(s.frame_accum.begin(), s.frame_accum.begin() + kFrameBytes); + + const float level = frame_rms(frame.data(), frame.size()); + const float threshold = + std::max(kSpeechRmsThreshold, s.noise_floor * kSpeechFloorMultiplier); + const bool is_speech = level >= threshold; + // Only adapt the floor while idle (between utterances). Adapting + // mid-utterance lets inter-word pauses inflate the floor and lock out + // the next turn. Drop instantly to any quieter ambient; creep up slowly + // otherwise. + if (!s.in_speech) { + if (level < s.noise_floor) { + s.noise_floor = level; + } else if (!is_speech) { + s.noise_floor += (level - s.noise_floor) * kNoiseFloorRise; + } + } + + if (!s.in_speech) { + s.pre_roll.push_back(std::move(frame)); + if (s.pre_roll.size() > kPreRollFrames) + s.pre_roll.pop_front(); + if (is_speech) { + s.in_speech = true; + s.speech_ms = kFrameMs; + s.silence_ms = 0; + s.utterance.clear(); + for (const auto& buffered : s.pre_roll) + s.utterance.append(reinterpret_cast(buffered.data()), + buffered.size()); + s.pre_roll.clear(); + } + continue; + } + + s.utterance.append(reinterpret_cast(frame.data()), frame.size()); + if (is_speech) { + s.speech_ms += kFrameMs; + s.silence_ms = 0; + } else { + s.silence_ms += kFrameMs; + } + + const int utterance_ms = static_cast((s.utterance.size() / kBytesPerSample) * 1000 / + kSampleRateHz); + if (s.silence_ms >= kEndOfUtteranceSilenceMs || utterance_ms >= kMaxUtteranceMs) { + const bool ok = s.speech_ms >= kMinSpeechMs; + std::string audio = std::move(s.utterance); + s.in_speech = false; + s.utterance.clear(); + s.speech_ms = 0; + s.silence_ms = 0; + if (ok) { + *out_utterance = std::move(audio); + completed = true; + // Drop any backlog captured while this utterance ran so the + // upcoming turn + TTS playout is not re-segmented. + s.frame_accum.clear(); + break; + } + } + } + + // Explicit flush (stream stopping): close an in-progress utterance if it + // already holds enough speech. + if (!completed && is_final && s.in_speech && s.speech_ms >= kMinSpeechMs && + !s.utterance.empty()) { + *out_utterance = std::move(s.utterance); + completed = true; + } + if (is_final) { + s.in_speech = false; + s.utterance.clear(); + s.speech_ms = 0; + s.silence_ms = 0; + s.pre_roll.clear(); + s.frame_accum.clear(); + } + return completed; +} + +} // namespace + +#endif // RAC_HAVE_PROTOBUF + +extern "C" rac_result_t rac_voice_agent_feed_audio_proto(rac_voice_agent_handle_t handle, + const void* audio_data, size_t audio_size, + int32_t sample_rate_hz, int32_t channels, + int32_t encoding, rac_bool_t is_final, + rac_proto_buffer_t* out_result) { + if (!out_result) + return RAC_ERROR_INVALID_ARGUMENT; +#if !defined(RAC_HAVE_PROTOBUF) + (void)handle; + (void)audio_data; + (void)audio_size; + (void)sample_rate_hz; + (void)channels; + (void)encoding; + (void)is_final; + return rac_proto_buffer_set_error(out_result, RAC_ERROR_FEATURE_NOT_AVAILABLE, + "protobuf support is not available"); +#else + using namespace rac::voice_agent::detail; + if (!handle) { + return rac_proto_buffer_set_error(out_result, RAC_ERROR_INVALID_HANDLE, + "voice-agent handle is required"); + } + // The in-core segmenter operates on 16 kHz mono PCM16 — the format every + // SDK's AudioCaptureManager already produces. Treat UNSPECIFIED as PCM16. + (void)sample_rate_hz; + (void)channels; + if (encoding != 0 && + encoding != static_cast(runanywhere::v1::AUDIO_ENCODING_PCM_S16_LE)) { + return rac_proto_buffer_set_error(out_result, RAC_ERROR_INVALID_ARGUMENT, + "feed_audio expects PCM_S16_LE mono @ 16 kHz"); + } + + // Admit under the in-flight barrier so destroy()'s drain covers any turn + // this feed call triggers. + InFlightGuard guard(handle); + if (!guard.admitted()) { + return rac_proto_buffer_set_error(out_result, RAC_ERROR_INVALID_STATE, + "voice agent is shutting down"); + } + if (!handle->is_configured.load(std::memory_order_acquire)) { + emit_component_failure(handle, "voice_agent", RAC_ERROR_NOT_INITIALIZED, + "voice agent is not initialized"); + return rac_proto_buffer_set_error(out_result, RAC_ERROR_NOT_INITIALIZED, + "voice agent is not initialized"); + } + + // Segment under the feed lock only; the multi-second turn pipeline runs + // outside it so a slow turn never blocks buffering of the next frame. + std::string utterance; + bool have_utterance = false; + { + std::lock_guard seg_lock(handle->feed.mutex); + have_utterance = + feed_segment(handle->feed, audio_data, audio_size, is_final == RAC_TRUE, &utterance); + } + + if (!have_utterance) { + // No utterance closed this call: return an empty (default) result so + // the SDK sees a valid buffer with no audio to play. + runanywhere::v1::VoiceAgentResult empty; + return copy_proto_message(empty, out_result); + } + + const std::string turn_id = event_id("turn"); + runanywhere::v1::VoiceAgentResult result; + const rac_result_t rc = d7_process_utterance( + handle, utterance, /*session_id=*/std::string(), turn_id, /*request_id=*/std::string(), + /*language_code=*/std::string(), /*event_callback=*/nullptr, /*user_data=*/nullptr, + &result); + if (rc != RAC_SUCCESS) { + return rac_proto_buffer_set_error(out_result, rc, "voice turn failed"); + } + return copy_proto_message(result, out_result); +#endif +} diff --git a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal.h b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal.h index d8514c48d4..482028fb68 100644 --- a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal.h +++ b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal.h @@ -12,10 +12,38 @@ #define RAC_FEATURES_VOICE_AGENT_VOICE_AGENT_INTERNAL_H #include +#include #include +#include +#include #include "rac/core/rac_types.h" +/// Energy-VAD utterance segmenter state for the streaming +/// `rac_voice_agent_feed_audio_proto` ingress path. The SDK feeds raw mic +/// frames; this state accumulates them into utterances using the same +/// energy/noise-floor endpointing the Swift/Kotlin mic drivers used to run +/// per-SDK. PCM is 16 kHz mono S16LE (bytes are little-endian int16). +struct rac_voice_agent_feed_state { + /// Leftover bytes that did not fill a whole analysis frame; prepended to + /// the next feed call's audio. + std::vector frame_accum; + /// Recent pre-speech frames retained so an utterance's onset is not + /// clipped (mirrors the SDK pre-roll). + std::deque> pre_roll; + /// Accumulated PCM16 bytes for the in-progress utterance. + std::string utterance; + bool in_speech{false}; + int speech_ms{0}; + int silence_ms{0}; + /// Adaptive ambient floor; seeded to the absolute speech threshold and + /// never reset across turns (only adapted while idle). + float noise_floor{0.015f}; + /// Serializes feed-call segmentation; the heavy turn pipeline runs + /// outside this lock so concurrent feeds only contend on buffering. + std::mutex mutex; +}; + struct rac_voice_agent { /// Set true when initialize* has run successfully. Atomic so /// `is_ready()` checks don't need the mutex. @@ -38,6 +66,9 @@ struct rac_voice_agent { /// Protects mutable operations (load, process, cleanup). std::mutex mutex; + + /// Streaming-ingress segmenter state (rac_voice_agent_feed_audio_proto). + rac_voice_agent_feed_state feed; }; #endif // RAC_FEATURES_VOICE_AGENT_VOICE_AGENT_INTERNAL_H diff --git a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal_helpers.cpp b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal_helpers.cpp index 08aca43b49..68d8482832 100644 --- a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal_helpers.cpp +++ b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal_helpers.cpp @@ -182,6 +182,72 @@ void publish_voice_pipeline_sdk_event(const runanywhere::v1::VoiceEvent& voice_e (void)rac::events::publish_prebuilt(sdk_event); } +void publish_voice_turn_metrics(double stt_ms, double llm_ms, double tts_ms, double end_to_end_ms, + int64_t tokens_generated, const char* session_id, + const char* model_id, const char* framework, + int32_t transcript_chars, int32_t response_chars, + rac_result_t error_code, const char* error_message) { + const bool failed = (error_code != RAC_SUCCESS); + runanywhere::v1::SDKEvent sdk_event; + sdk_event.set_timestamp_ms(rac_get_current_time_ms()); + sdk_event.set_id(event_id("voice")); + sdk_event.set_category(failed ? runanywhere::v1::EVENT_CATEGORY_ERROR + : runanywhere::v1::EVENT_CATEGORY_VOICE_AGENT); + sdk_event.set_component(runanywhere::v1::SDK_COMPONENT_VOICE_AGENT); + sdk_event.set_severity(failed ? runanywhere::v1::ERROR_SEVERITY_ERROR + : runanywhere::v1::ERROR_SEVERITY_INFO); + sdk_event.set_destination(runanywhere::v1::EVENT_DESTINATION_ALL); + sdk_event.set_source("cpp"); + sdk_event.set_operation_id("voice_agent.turn"); + // Common columns. session_id is a native envelope field; the MetricsEvent + // proto has no model/framework/char fields, so those ride the properties + // carrier (read back in the telemetry kMetrics extraction). + if (session_id != nullptr && session_id[0] != '\0') { + sdk_event.set_session_id(session_id); + } + if (model_id != nullptr && model_id[0] != '\0') { + (*sdk_event.mutable_properties())["model_id"] = model_id; + } + if (framework != nullptr && framework[0] != '\0') { + (*sdk_event.mutable_properties())["framework"] = framework; + } + if (transcript_chars > 0) { + (*sdk_event.mutable_properties())["transcript_chars"] = std::to_string(transcript_chars); + } + if (response_chars > 0) { + (*sdk_event.mutable_properties())["response_chars"] = std::to_string(response_chars); + } + + auto* vp = sdk_event.mutable_voice_pipeline(); + vp->set_timestamp_us(rac_get_current_time_ms() * 1000); + vp->set_severity(failed ? runanywhere::v1::ERROR_SEVERITY_ERROR + : runanywhere::v1::ERROR_SEVERITY_INFO); + vp->set_component(runanywhere::v1::VOICE_PIPELINE_COMPONENT_AGENT); + auto* metrics = vp->mutable_metrics(); + if (stt_ms > 0.0) + metrics->set_stt_final_ms(stt_ms); + if (llm_ms > 0.0) + metrics->set_llm_total_ms(llm_ms); + if (tts_ms > 0.0) + metrics->set_tts_total_ms(tts_ms); + if (end_to_end_ms > 0.0) + metrics->set_end_to_end_ms(end_to_end_ms); + if (tokens_generated > 0) + metrics->set_tokens_generated(tokens_generated); + + // On failure set the envelope SDKError so the telemetry extractor marks the + // row Failed with the message/code (c_abi_code preferred for error_code). + if (failed) { + auto* err = sdk_event.mutable_error(); + err->set_c_abi_code(static_cast(error_code)); + err->set_message(error_message != nullptr && error_message[0] != '\0' ? error_message + : "voice turn failed"); + err->set_component("voice"); + err->set_severity(runanywhere::v1::ERROR_SEVERITY_ERROR); + } + (void)rac::events::publish_prebuilt(sdk_event); +} + void emit_generated_voice_event(rac_voice_agent_handle_t handle, const runanywhere::v1::VoiceEvent& event, runanywhere::v1::ErrorSeverity sdk_severity) { diff --git a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal_helpers.h b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal_helpers.h index 2db40f684e..c15adf31e9 100644 --- a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal_helpers.h +++ b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_internal_helpers.h @@ -109,11 +109,42 @@ void emit_turn_lifecycle(rac_voice_agent_handle_t handle, void emit_component_failure(rac_voice_agent_handle_t handle, const char* component, rac_result_t code, const char* message); +// Publish a per-turn MetricsEvent (kMetrics) so the turn is recorded to +// telemetry under the "voice" modality. telemetry_records() only records +// voice-pipeline events whose payload is kMetrics, so turn lifecycle events +// alone never reach telemetry — this is the row the dashboard shows. On +// failure (error_code != RAC_SUCCESS) the envelope SDKError is set so the row +// is marked Failed with the message/code. Pass 0 for any unmeasured stage. +void publish_voice_turn_metrics(double stt_ms, double llm_ms, double tts_ms, double end_to_end_ms, + int64_t tokens_generated, const char* session_id, + const char* model_id, const char* framework, + int32_t transcript_chars, int32_t response_chars, + rac_result_t error_code, const char* error_message); + // Translate a proto `VoiceAgentComposeConfig` into the C ABI // `rac_voice_agent_config_t`. The returned config aliases string pointers // in `proto`; caller must keep `proto` alive across the use. rac_voice_agent_config_t config_from_proto(const runanywhere::v1::VoiceAgentComposeConfig& proto); +// Run one complete VAD -> STT -> LLM -> TTS turn over a pre-segmented +// `audio` buffer (16 kHz mono PCM16). This is the shared pipeline core of +// the full-session ABI: `rac_voice_agent_process_turn_proto` calls it with +// the parsed turn request, and `rac_voice_agent_feed_audio_proto` calls it +// once the in-core segmenter closes an utterance. Emits the same d7 +// VoiceEvents (component states, turn lifecycle, pipeline state, VAD, STT, +// LLM token, TTS audio) through @p event_callback (may be NULL — the +// per-handle proto callback still receives every event) and, when +// @p out_result is non-NULL, also fills a `VoiceAgentResult` carrying the +// transcript, response, and synthesized WAV for inline playback. +// +// Acquires `handle->mutex` internally. The caller owns admission +// (InFlightGuard) and the `is_configured` gate. +rac_result_t d7_process_utterance(rac_voice_agent_handle_t handle, const std::string& audio, + const std::string& session_id, const std::string& turn_id, + const std::string& request_id, const std::string& language_code, + rac_voice_agent_turn_event_callback_fn event_callback, + void* user_data, runanywhere::v1::VoiceAgentResult* out_result); + #endif // RAC_HAVE_PROTOBUF // Validate all four voice-agent modalities are READY (lifecycle preferred, diff --git a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_proto_abi.cpp b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_proto_abi.cpp index 7e58cef1a7..b76aac751d 100644 --- a/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_proto_abi.cpp +++ b/sdk/runanywhere-commons/src/features/voice_agent/voice_agent_proto_abi.cpp @@ -8,6 +8,7 @@ * `voice_agent_internal_helpers.h`. */ +#include #include #include #include @@ -300,6 +301,23 @@ rac_result_t rac_voice_agent_process_voice_turn_proto(rac_voice_agent_handle_t h rac_result_t rc = RAC_SUCCESS; std::string error_message; rac_result_t error_code = RAC_SUCCESS; + + // Per-turn timing for the telemetry MetricsEvent, read at both exits below. + // Declared here (function scope) so the cleanup_and_return path can publish + // partial metrics on failure. + double stt_ms = 0.0; + double llm_ms = 0.0; + double tts_ms = 0.0; + int64_t turn_tokens = 0; + std::string turn_model_id; + std::string turn_framework; + int32_t turn_transcript_chars = 0; + int32_t turn_response_chars = 0; + const auto turn_start = std::chrono::steady_clock::now(); + auto ms_since = [](std::chrono::steady_clock::time_point t) { + return std::chrono::duration(std::chrono::steady_clock::now() - t) + .count(); + }; { std::lock_guard lock(handle->mutex); @@ -322,6 +340,7 @@ rac_result_t rac_voice_agent_process_voice_turn_proto(rac_voice_agent_handle_t h rac::lifecycle::acquire_lifecycle_stt(&stt_ref) == RAC_SUCCESS; rac_stt_result_t stt = {}; + const auto t_stt = std::chrono::steady_clock::now(); if (have_lifecycle_stt) { rac_stt_service_t stt_service{stt_ref.ops, stt_ref.impl, stt_ref.model_id}; rc = rac_stt_transcribe(&stt_service, audio_data, audio_size, nullptr, &stt); @@ -329,6 +348,8 @@ rac_result_t rac_voice_agent_process_voice_turn_proto(rac_voice_agent_handle_t h rc = rac_stt_component_transcribe(handle->stt_handle, audio_data, audio_size, nullptr, &stt); } + stt_ms = ms_since(t_stt); + turn_transcript_chars = stt.text ? static_cast(std::strlen(stt.text)) : 0; if (rc != RAC_SUCCESS) { if (have_lifecycle_stt) { rac::lifecycle::release_lifecycle_stt(&stt_ref); @@ -373,12 +394,22 @@ rac_result_t rac_voice_agent_process_voice_turn_proto(rac_voice_agent_handle_t h const bool have_lifecycle_llm = rac::llm::acquire_lifecycle_llm(&llm_ref) == RAC_SUCCESS; rac_llm_result_t llm = {}; + const auto t_llm = std::chrono::steady_clock::now(); if (have_lifecycle_llm) { rac_llm_service_t llm_service{llm_ref.ops, llm_ref.impl, llm_ref.model_id}; rc = rac_llm_generate(&llm_service, stt.text, nullptr, &llm); } else { rc = rac_llm_component_generate(handle->llm_handle, stt.text, nullptr, &llm); } + llm_ms = ms_since(t_llm); + turn_tokens = llm.completion_tokens; + turn_response_chars = llm.text ? static_cast(std::strlen(llm.text)) : 0; + if (have_lifecycle_llm) { + if (llm_ref.model_id != nullptr) + turn_model_id = llm_ref.model_id; + if (llm_ref.framework_name != nullptr) + turn_framework = llm_ref.framework_name; + } if (rc != RAC_SUCCESS) { if (have_lifecycle_llm) { rac::llm::release_lifecycle_llm(&llm_ref); @@ -397,7 +428,7 @@ rac_result_t rac_voice_agent_process_voice_turn_proto(rac_voice_agent_handle_t h } { const std::string stt_text(stt.text); - const std::string llm_text(llm.text); + const std::string llm_text(llm.text ? llm.text : ""); pending_emits.emplace_back([handle, stt_text, llm_text]() { emit_turn_lifecycle( handle, runanywhere::v1::TURN_LIFECYCLE_EVENT_KIND_AGENT_RESPONSE_COMPLETED, @@ -410,12 +441,14 @@ rac_result_t rac_voice_agent_process_voice_turn_proto(rac_voice_agent_handle_t h rac::lifecycle::acquire_lifecycle_tts(&tts_ref) == RAC_SUCCESS; rac_tts_result_t tts = {}; + const auto t_tts = std::chrono::steady_clock::now(); if (have_lifecycle_tts) { rac_tts_service_t tts_service{tts_ref.ops, tts_ref.impl, tts_ref.model_id}; rc = rac_tts_synthesize(&tts_service, llm.text, nullptr, &tts); } else { rc = rac_tts_component_synthesize(handle->tts_handle, llm.text, nullptr, &tts); } + tts_ms = ms_since(t_tts); if (rc != RAC_SUCCESS) { if (have_lifecycle_tts) { rac::lifecycle::release_lifecycle_tts(&tts_ref); @@ -503,10 +536,21 @@ rac_result_t rac_voice_agent_process_voice_turn_proto(rac_voice_agent_handle_t h } // lock released here flush_emits(); + publish_voice_turn_metrics(stt_ms, llm_ms, tts_ms, ms_since(turn_start), turn_tokens, + /*session_id=*/nullptr, + turn_model_id.empty() ? nullptr : turn_model_id.c_str(), + turn_framework.empty() ? nullptr : turn_framework.c_str(), + turn_transcript_chars, turn_response_chars, RAC_SUCCESS, nullptr); return copy_proto_message(result, out_result); cleanup_and_return: flush_emits(); + publish_voice_turn_metrics(stt_ms, llm_ms, tts_ms, ms_since(turn_start), turn_tokens, + /*session_id=*/nullptr, + turn_model_id.empty() ? nullptr : turn_model_id.c_str(), + turn_framework.empty() ? nullptr : turn_framework.c_str(), + turn_transcript_chars, turn_response_chars, error_code, + error_message.c_str()); return rac_proto_buffer_set_error(out_result, error_code, error_message.c_str()); #endif } diff --git a/sdk/runanywhere-commons/src/infrastructure/device/rac_device_manager.cpp b/sdk/runanywhere-commons/src/infrastructure/device/rac_device_manager.cpp index 9b0bc24b9e..3298e67ada 100644 --- a/sdk/runanywhere-commons/src/infrastructure/device/rac_device_manager.cpp +++ b/sdk/runanywhere-commons/src/infrastructure/device/rac_device_manager.cpp @@ -104,6 +104,13 @@ rac_result_t rac_device_manager_register_if_needed(rac_environment_t env, const state.callbacks.is_registered(state.callbacks.user_data) == RAC_TRUE; if (was_registered && env != RAC_ENV_DEVELOPMENT) { RAC_LOG_DEBUG(LOG_CAT, "Device already registered, skipping (production mode)"); + // Skip the network round-trip, but still emit the device.registered + // telemetry so the dashboard reflects the active device on every launch + // (not only the first-ever registration). + const char* registered_device_id = state.callbacks.get_device_id(state.callbacks.user_data); + if (registered_device_id != nullptr && registered_device_id[0] != '\0') { + emit_device_registered(registered_device_id); + } return RAC_SUCCESS; } diff --git a/sdk/runanywhere-commons/src/infrastructure/download/download_orchestrator.cpp b/sdk/runanywhere-commons/src/infrastructure/download/download_orchestrator.cpp index a902f72065..a424cb337e 100644 --- a/sdk/runanywhere-commons/src/infrastructure/download/download_orchestrator.cpp +++ b/sdk/runanywhere-commons/src/infrastructure/download/download_orchestrator.cpp @@ -71,6 +71,7 @@ #include "rac/core/rac_platform_adapter.h" #include "rac/foundation/rac_proto_buffer.h" #include "rac/infrastructure/download/rac_download_orchestrator.h" +#include "rac/infrastructure/events/rac_sdk_emit.h" #include "rac/infrastructure/extraction/rac_extraction.h" #include "rac/infrastructure/http/rac_http_transport.h" #include "rac/infrastructure/model_management/rac_model_paths.h" @@ -292,6 +293,8 @@ struct proto_download_task { int64_t last_partial_bytes = 0; int64_t last_deleted_bytes = 0; int64_t started_at_unix_ms = 0; + bool download_telemetry_started = false; // model.download.started emitted once + bool download_telemetry_finished = false; // terminal model.download.* emitted once // Framework + archive structure preserved from the registry at task // creation so the worker can (a) extract into the canonical per-model // folder and (b) resolve the post-extraction nested subdirectory for @@ -786,11 +789,51 @@ void mark_task_stopped(const std::shared_ptr& task) { if (!task) { return; } + // Snapshot the terminal state under the lock; emit telemetry after releasing + // it (the emit feeds the event bus / telemetry sink, never the task mutex). + rav1::DownloadState final_state = rav1::DOWNLOAD_STATE_UNSPECIFIED; + std::string model_id; + std::string error_message; + int64_t bytes = 0; + double duration_ms = 0.0; + bool emit_terminal = false; { std::lock_guard lock(task->mutex); task->running = false; + final_state = task->progress.state(); + if (!task->download_telemetry_finished && + (final_state == rav1::DOWNLOAD_STATE_COMPLETED || + final_state == rav1::DOWNLOAD_STATE_FAILED || + final_state == rav1::DOWNLOAD_STATE_CANCELLED)) { + task->download_telemetry_finished = true; + emit_terminal = true; + model_id = task->model_id; + error_message = task->progress.error_message(); + bytes = task->progress.bytes_downloaded(); + if (task->started_at_unix_ms > 0) { + duration_ms = static_cast(now_unix_ms() - task->started_at_unix_ms); + } + } } task->cv.notify_all(); + if (emit_terminal) { + switch (final_state) { + case rav1::DOWNLOAD_STATE_COMPLETED: + rac::events::emit_model_download_completed(model_id.c_str(), bytes, duration_ms, + nullptr); + break; + case rav1::DOWNLOAD_STATE_FAILED: + rac::events::emit_model_download_failed( + model_id.c_str(), RAC_ERROR_DOWNLOAD_FAILED, + error_message.empty() ? "download failed" : error_message.c_str()); + break; + case rav1::DOWNLOAD_STATE_CANCELLED: + rac::events::emit_model_download_cancelled(model_id.c_str()); + break; + default: + break; + } + } } void set_task_progress(const std::shared_ptr& task, rav1::DownloadState state, @@ -1190,9 +1233,20 @@ void run_proto_download_worker(const std::shared_ptr& task, return; } + std::string started_model_id; + int64_t started_total_bytes = 0; { std::lock_guard lock(task->mutex); task->running = true; + if (!task->download_telemetry_started) { + task->download_telemetry_started = true; + started_model_id = task->model_id; + started_total_bytes = task->progress.total_bytes(); + } + } + if (!started_model_id.empty()) { + rac::events::emit_model_download_started(started_model_id.c_str(), started_total_bytes, + nullptr); } const int64_t total_expected = plan_total_expected(task->files); @@ -1266,8 +1320,330 @@ void run_proto_download_worker_async(void* user_data) { run_proto_download_worker(args->task, args->resume_from); } +// ============================================================================= +// EVENT-DRIVEN ASYNC DOWNLOAD DRIVER (Emscripten / Web) +// ============================================================================= +// The synchronous worker above buffers the whole body on the main thread when +// the only transport is FetchHttpTransport's sync XHR — the UI freezes and the +// poll loop never ticks until 100%. When the platform supplies the async +// `http_download` adapter slot (the Web SDK implements it with streaming +// fetch + ReadableStream), drive the plan event-driven instead: start a file, +// report each chunk via the progress callback, and on completion advance to the +// next file or finalize. Nothing blocks the main thread, so progress is live. +// +// This path reuses every leaf helper the synchronous worker uses +// (set_task_progress / emit_progress / extraction / validate_downloaded_sizes / +// resolve_completion_local_path / self_heal_registry). Only the *sequencing* +// differs (callback continuation vs a blocking loop). The synchronous worker +// and all native code are left untouched and remain the fallback when the slot +// is absent. +// +// Integrity: the slot delivers bytes to the platform (MEMFS), never to C++, so +// the streaming SHA-256 the synchronous path computes cannot run here. The +// post-download size guard (validate_downloaded_sizes) plus HTTPS transport +// integrity cover truncation/stub responses; per-byte checksum is a Web-path +// tradeoff (the native path keeps full SHA-256). +struct web_download_driver { + std::shared_ptr task; + int64_t total_expected = 0; + int64_t completed_before_file = 0; + size_t file_index = 0; + std::string final_path; + char* current_task_id = nullptr; // owned C string from http_download out_task_id + bool cancel_abort_sent = false; +}; + +// Keeps each driver alive across the async callbacks (the callbacks carry only a +// raw driver* as user_data). Erasing drops the last owner and frees the driver. +std::map>& web_download_registry() { + static std::map> registry; + return registry; +} + +void web_download_start_file(web_download_driver* drv); +void web_download_finalize_all(web_download_driver* drv); + +// MUST be the final action on a driver in any terminal branch — frees the +// driver, so nothing may touch `drv` afterwards. +void web_download_finish(web_download_driver* drv) { + if (!drv) { + return; + } + if (drv->current_task_id) { + free(drv->current_task_id); + drv->current_task_id = nullptr; + } + web_download_registry().erase(drv); +} + +// rac_http_progress_callback_fn: void (int64 bytes, int64 total, void* user_data). +void web_download_on_progress(int64_t bytes_downloaded, int64_t /*total_bytes*/, void* user_data) { + auto* drv = static_cast(user_data); + if (!drv) { + return; + } + const std::shared_ptr& task = drv->task; + + // Cooperative cancel: abort the in-flight fetch once; on_complete then fires + // with RAC_ERROR_CANCELLED and emits the terminal event. + if (task->cancel_requested.load() && !drv->cancel_abort_sent) { + drv->cancel_abort_sent = true; + const rac_platform_adapter_t* adapter = rac_get_platform_adapter(); + if (adapter && adapter->http_download_cancel && drv->current_task_id) { + adapter->http_download_cancel(drv->current_task_id, adapter->user_data); + } + return; + } + + const int64_t downloaded = drv->completed_before_file + bytes_downloaded; + const proto_plan_file& file = task->files[drv->file_index]; + set_task_progress(task, rav1::DOWNLOAD_STATE_DOWNLOADING, rav1::DOWNLOAD_STAGE_DOWNLOADING, + downloaded, drv->total_expected, static_cast(drv->file_index), + file.storage_key, "", ""); + emit_progress(task); +} + +// rac_http_complete_callback_fn: void (rac_result_t, const char* path, void* user_data). +void web_download_on_complete(rac_result_t result, const char* /*downloaded_path*/, void* user_data) { + auto* drv = static_cast(user_data); + if (!drv) { + return; + } + // Keep the task alive independently of the driver (web_download_finish frees + // the driver at the end of the terminal branches). + std::shared_ptr task = drv->task; + if (drv->current_task_id) { + free(drv->current_task_id); + drv->current_task_id = nullptr; + } + + const size_t i = drv->file_index; + const proto_plan_file& file = task->files[i]; + + const bool cancelled = task->cancel_requested.load() || result == RAC_ERROR_CANCELLED; + if (cancelled) { + int64_t file_partial = file_size_or_zero(file.destination_path); + int64_t deleted = 0; + bool delete_partial = false; + { + std::lock_guard lock(task->mutex); + delete_partial = task->delete_partial_on_cancel; + } + if (delete_partial) { + deleted = delete_partial_file(file.destination_path); + file_partial = 0; + } + { + std::lock_guard lock(task->mutex); + task->last_deleted_bytes = deleted; + task->last_partial_bytes = drv->completed_before_file + file_partial; + } + set_task_progress(task, rav1::DOWNLOAD_STATE_CANCELLED, rav1::DOWNLOAD_STAGE_DOWNLOADING, + drv->completed_before_file + file_partial, drv->total_expected, + static_cast(i), file.storage_key, "", "download cancelled"); + mark_task_stopped(task); + emit_progress(task); + web_download_finish(drv); + return; + } + + if (result != RAC_SUCCESS) { + int64_t partial = drv->completed_before_file + file_size_or_zero(file.destination_path); + { + std::lock_guard lock(task->mutex); + task->last_partial_bytes = partial; + } + set_task_progress(task, rav1::DOWNLOAD_STATE_FAILED, rav1::DOWNLOAD_STAGE_DOWNLOADING, + partial, drv->total_expected, static_cast(i), file.storage_key, + "", "download failed"); + mark_task_stopped(task); + emit_progress(task); + web_download_finish(drv); + return; + } + + // Success: extract (if archive) and advance the cumulative counter — mirrors + // the post-download tail of process_plan_file. + if (file.requires_extraction) { + set_task_progress(task, rav1::DOWNLOAD_STATE_EXTRACTING, rav1::DOWNLOAD_STAGE_EXTRACTING, + drv->total_expected > 0 ? drv->completed_before_file + file.expected_bytes + : 0, + drv->total_expected, static_cast(i), file.storage_key, "", ""); + emit_progress(task); + + mkdir_p(task->model_folder_path.c_str()); + + rac_extraction_result_t extraction_result{}; + rac_result_t extract_rc = rac_extract_archive_native( + file.destination_path.c_str(), task->model_folder_path.c_str(), nullptr, nullptr, + nullptr, &extraction_result); + if (extract_rc != RAC_SUCCESS) { + set_task_progress(task, rav1::DOWNLOAD_STATE_FAILED, rav1::DOWNLOAD_STAGE_EXTRACTING, + drv->total_expected > 0 + ? drv->completed_before_file + file.expected_bytes + : 0, + drv->total_expected, static_cast(i), file.storage_key, "", + "archive extraction failed"); + mark_task_stopped(task); + emit_progress(task); + web_download_finish(drv); + return; + } + + delete_file(file.destination_path.c_str()); + + char resolved_path[4096]; + rac_result_t find_rc = rac_find_model_path_after_extraction( + task->model_folder_path.c_str(), task->archive_structure, task->framework, task->format, + resolved_path, sizeof(resolved_path)); + drv->final_path = (find_rc == RAC_SUCCESS && resolved_path[0] != '\0') + ? std::string(resolved_path) + : task->model_folder_path; + } else { + drv->final_path = file.destination_path; + } + + if (drv->total_expected > 0) { + drv->completed_before_file += std::max(file.expected_bytes, 0); + } else { + drv->completed_before_file += file_size_or_zero(drv->final_path); + } + + drv->file_index += 1; + if (drv->file_index < task->files.size()) { + web_download_start_file(drv); + } else { + web_download_finalize_all(drv); + } +} + +void web_download_start_file(web_download_driver* drv) { + const std::shared_ptr& task = drv->task; + const size_t i = drv->file_index; + const proto_plan_file& file = task->files[i]; + + if (task->cancel_requested.load()) { + set_task_progress(task, rav1::DOWNLOAD_STATE_CANCELLED, rav1::DOWNLOAD_STAGE_DOWNLOADING, + drv->completed_before_file, drv->total_expected, static_cast(i), + file.storage_key, "", "download cancelled"); + mark_task_stopped(task); + emit_progress(task); + web_download_finish(drv); + return; + } + + // A previous attempt may already have landed this file completely; skip the + // fetch and run the success tail directly (extraction + advance). + const bool already_complete = + file.expected_bytes > 0 && file_size_or_zero(file.destination_path) == file.expected_bytes; + if (already_complete) { + web_download_on_complete(RAC_SUCCESS, file.destination_path.c_str(), drv); + return; + } + + set_task_progress(task, rav1::DOWNLOAD_STATE_DOWNLOADING, rav1::DOWNLOAD_STAGE_DOWNLOADING, + drv->completed_before_file, drv->total_expected, static_cast(i), + file.storage_key, "", ""); + emit_progress(task); + + const rac_platform_adapter_t* adapter = rac_get_platform_adapter(); + char* task_id = nullptr; + rac_result_t rc = adapter->http_download(file.url.c_str(), file.destination_path.c_str(), + web_download_on_progress, web_download_on_complete, drv, + &task_id, adapter->user_data); + drv->current_task_id = task_id; + if (rc != RAC_SUCCESS) { + set_task_progress(task, rav1::DOWNLOAD_STATE_FAILED, rav1::DOWNLOAD_STAGE_DOWNLOADING, + drv->completed_before_file, drv->total_expected, static_cast(i), + file.storage_key, "", "failed to start download"); + mark_task_stopped(task); + emit_progress(task); + web_download_finish(drv); + } +} + +void web_download_finalize_all(web_download_driver* drv) { + const std::shared_ptr& task = drv->task; + + if (task->cancel_requested.load()) { + set_task_progress(task, rav1::DOWNLOAD_STATE_CANCELLED, rav1::DOWNLOAD_STAGE_DOWNLOADING, + drv->completed_before_file, drv->total_expected, 0, "", "", + "download cancelled"); + mark_task_stopped(task); + emit_progress(task); + web_download_finish(drv); + return; + } + + const int64_t completed_bytes = + drv->total_expected > 0 ? drv->total_expected : drv->completed_before_file; + + size_t failing_index = 0; + std::string sanity_error; + if (!validate_downloaded_sizes(task, failing_index, sanity_error)) { + int64_t failed_bytes = drv->total_expected > 0 ? drv->completed_before_file : 0; + set_task_progress(task, rav1::DOWNLOAD_STATE_FAILED, rav1::DOWNLOAD_STAGE_DOWNLOADING, + failed_bytes, drv->total_expected, static_cast(failing_index), + task->files[failing_index].storage_key, "", sanity_error); + mark_task_stopped(task); + emit_progress(task); + web_download_finish(drv); + return; + } + + std::string completion_local_path = resolve_completion_local_path(task, drv->final_path); + set_task_progress(task, rav1::DOWNLOAD_STATE_COMPLETED, rav1::DOWNLOAD_STAGE_COMPLETED, + completed_bytes, drv->total_expected, + static_cast(task->files.size() - 1), + task->files.empty() ? "" : task->files.back().storage_key, + completion_local_path, ""); + mark_task_stopped(task); + emit_progress(task); + self_heal_registry(task, completion_local_path); + web_download_finish(drv); +} + +// Non-blocking entry: kick off file 0; the adapter's async callbacks drive the +// rest. Mirrors run_proto_download_worker's telemetry-start preamble. +void web_download_start(const std::shared_ptr& task, int64_t /*resume_from*/) { + auto drv = std::make_shared(); + drv->task = task; + drv->total_expected = plan_total_expected(task->files); + web_download_registry()[drv.get()] = drv; + + std::string started_model_id; + int64_t started_total_bytes = 0; + { + std::lock_guard lock(task->mutex); + task->running = true; + if (!task->download_telemetry_started) { + task->download_telemetry_started = true; + started_model_id = task->model_id; + started_total_bytes = task->progress.total_bytes(); + } + } + if (!started_model_id.empty()) { + rac::events::emit_model_download_started(started_model_id.c_str(), started_total_bytes, + nullptr); + } + + if (task->files.empty()) { + web_download_finalize_all(drv.get()); + return; + } + web_download_start_file(drv.get()); +} + void start_proto_download_worker(const std::shared_ptr& task, int64_t resume_from) { + // Prefer the async streaming slot when the platform supplies it (Web): it + // reports progress per chunk without blocking the main thread. Otherwise + // fall back to the synchronous worker on a deferred tick. + const rac_platform_adapter_t* adapter = rac_get_platform_adapter(); + if (adapter && adapter->http_download) { + web_download_start(task, resume_from); + return; + } auto* args = new emscripten_proto_download_args{task, resume_from}; emscripten_async_call(run_proto_download_worker_async, args, 0); } diff --git a/sdk/runanywhere-commons/src/infrastructure/network/environment.cpp b/sdk/runanywhere-commons/src/infrastructure/network/environment.cpp index d3e9ac0304..b55f0cc428 100644 --- a/sdk/runanywhere-commons/src/infrastructure/network/environment.cpp +++ b/sdk/runanywhere-commons/src/infrastructure/network/environment.cpp @@ -58,7 +58,12 @@ rac_log_level_t rac_env_default_log_level(rac_environment_t env) { } bool rac_env_should_send_telemetry(rac_environment_t env) { - return env == RAC_ENV_PRODUCTION; + // Telemetry is sent in every environment — development flushes immediately to + // the local backend, staging and production batch + send with auth. This must + // agree with the actual send gate (rac_env_requires_auth, also !=DEVELOPMENT); + // returning production-only here previously contradicted that and mislabeled + // staging as "no telemetry" even though staging does send. + return env != RAC_ENV_DEVELOPMENT; } bool rac_env_should_sync_with_backend(rac_environment_t env) { diff --git a/sdk/runanywhere-commons/src/infrastructure/telemetry/telemetry_json.cpp b/sdk/runanywhere-commons/src/infrastructure/telemetry/telemetry_json.cpp index 8f0479d68f..811c519c07 100644 --- a/sdk/runanywhere-commons/src/infrastructure/telemetry/telemetry_json.cpp +++ b/sdk/runanywhere-commons/src/infrastructure/telemetry/telemetry_json.cpp @@ -99,6 +99,13 @@ class JsonBuilder { ss_ << "\"" << key << "\":" << value; } + // Emit even when 0 — for fields where 0 is a meaningful measurement + // (e.g. temperature=0.0 greedy decode) rather than "unset". + void add_double_always(const char* key, double value) { + comma(); + ss_ << "\"" << key << "\":" << value; + } + void add_bool(const char* key, rac_bool_t value, rac_bool_t has_value) { if (has_value == RAC_FALSE) return; @@ -246,7 +253,7 @@ rac_result_t rac_telemetry_manager_payload_to_json(const rac_telemetry_payload_t json.add_double("prompt_eval_time_ms", payload->prompt_eval_time_ms); json.add_double("generation_time_ms", payload->generation_time_ms); json.add_int("context_length", payload->context_length); - json.add_double("temperature", payload->temperature); + json.add_double_always("temperature", payload->temperature); json.add_int("max_tokens", payload->max_tokens); } else if (strcmp(modality, "stt") == 0) { json.add_double("audio_duration_ms", payload->audio_duration_ms); @@ -263,8 +270,10 @@ rac_result_t rac_telemetry_manager_payload_to_json(const rac_telemetry_payload_t json.add_string("voice", payload->voice); json.add_double("output_duration_ms", payload->output_duration_ms); } else if (strcmp(modality, "vlm") == 0) { - // VLM = LLM token fields PLUS vision fields. Only image_count has a data - // source today; the other vision fields are optional and omitted. + // VLM = LLM token fields PLUS vision fields. The token fields are now + // populated via the properties carrier (input/total tokens, tps, ttft, + // generation_time). The vision-specific fields (vision_tokens, + // vision_encode_time_ms, image_resolution) still need carriers. json.add_int("input_tokens", payload->input_tokens); json.add_int("output_tokens", payload->output_tokens); json.add_int("total_tokens", payload->total_tokens); @@ -273,18 +282,27 @@ rac_result_t rac_telemetry_manager_payload_to_json(const rac_telemetry_payload_t json.add_double("prompt_eval_time_ms", payload->prompt_eval_time_ms); json.add_double("generation_time_ms", payload->generation_time_ms); json.add_int("context_length", payload->context_length); - json.add_double("temperature", payload->temperature); + json.add_double_always("temperature", payload->temperature); json.add_int("max_tokens", payload->max_tokens); json.add_int("image_count", payload->image_count); + json.add_int("vision_tokens", payload->vision_tokens); + json.add_double("vision_encode_time_ms", payload->vision_encode_time_ms); + json.add_string("image_resolution", payload->image_resolution); } else if (strcmp(modality, "rag") == 0) { - // Only retrieved_docs_count has a data source today; the other RAG fields - // (query_token_count, embedding_model, top_k, …) are optional and omitted. + // retrieved_docs_count / top_k / retrieval_time_ms / embedding_model have + // sources today (via the properties carrier). query_token_count / + // reranker_used / context_tokens still need carriers. json.add_int("retrieved_docs_count", payload->retrieved_docs_count); + json.add_int("top_k", payload->top_k); + json.add_double("retrieval_time_ms", payload->retrieval_time_ms); + json.add_string("embedding_model", payload->embedding_model); } else if (strcmp(modality, "embeddings") == 0) { - // input_count / vectors_produced / embedding_model have sources today; - // embedding_dimension / total_tokens / batch_size need a proto carrier. + // input_count / vectors_produced / embedding_model / embedding_dimension + // have sources today (dimension via the properties carrier). + // total_tokens / batch_size still need a carrier. json.add_int("input_count", payload->input_count); json.add_int("vectors_produced", payload->vectors_produced); + json.add_int("embedding_dimension", payload->embedding_dimension); json.add_string("embedding_model", payload->model_id); } else if (strcmp(modality, "voice") == 0) { // Per-turn voice-agent pipeline summary (from MetricsEvent). @@ -292,25 +310,43 @@ rac_result_t rac_telemetry_manager_payload_to_json(const rac_telemetry_payload_t json.add_double("llm_ms", payload->voice_llm_ms); json.add_double("tts_ms", payload->voice_tts_ms); json.add_double("total_ms", payload->voice_total_ms); + json.add_int("transcript_chars", payload->transcript_chars); + json.add_int("response_chars", payload->response_chars); } else if (strcmp(modality, "vad") == 0) { json.add_double("speech_duration_ms", payload->speech_duration_ms); json.add_double("silence_duration_ms", payload->silence_duration_ms); + json.add_int("segment_count", payload->segment_count); json.add_int("sample_rate", payload->sample_rate); } else if (strcmp(modality, "lora") == 0) { - // operation is encoded in event_type; base model rides on model_id. + // base model rides on model_id; adapter_id + operation via the carrier. + // adapter_size_bytes still needs a source (would require stat-ing the file). + json.add_string("operation", payload->operation); json.add_string("base_model_id", payload->model_id); + json.add_string("adapter_id", payload->adapter_id); + json.add_int("adapter_size_bytes", payload->adapter_size_bytes); } else if (strcmp(modality, "imagegen") == 0) { - // Diffusion capability events carry only progress, which the imagegen - // schema does not accept — emit base fields only until diffusion supplies - // real metrics (num_images, image_width, …). + // Diffusion detail fields ride the properties carrier (extracted in the + // kCapability SDK_COMPONENT_DIFFUSION branch). seed=0 / steps=0 are + // meaningful, so those use add_int_always. + json.add_int("prompt_length", payload->imagegen_prompt_length); + json.add_int("negative_prompt_length", payload->imagegen_negative_prompt_length); + json.add_int("image_width", payload->image_width); + json.add_int("image_height", payload->image_height); + json.add_int("num_images", payload->num_images); + json.add_int("num_inference_steps", payload->num_inference_steps); + json.add_double("guidance_scale", payload->guidance_scale); + json.add_int_always("seed", payload->seed); + json.add_int("output_size_bytes", payload->output_size_bytes); + json.add_string("scheduler", payload->scheduler); + json.add_string("output_format", payload->output_format); } else if (strcmp(modality, "model") == 0) { json.add_int("model_size_bytes", payload->model_size_bytes); json.add_string("archive_type", payload->archive_type); json.add_double("progress", payload->progress); } else { // "system": SDK lifecycle / storage / network. - json.add_int("count", payload->count); - json.add_int("freed_bytes", payload->freed_bytes); + json.add_int_always("count", payload->count); + json.add_int_always("freed_bytes", payload->freed_bytes); json.add_bool("is_online", payload->is_online, payload->has_is_online); } diff --git a/sdk/runanywhere-commons/src/infrastructure/telemetry/telemetry_manager.cpp b/sdk/runanywhere-commons/src/infrastructure/telemetry/telemetry_manager.cpp index 42b304e914..5b4ce777b2 100644 --- a/sdk/runanywhere-commons/src/infrastructure/telemetry/telemetry_manager.cpp +++ b/sdk/runanywhere-commons/src/infrastructure/telemetry/telemetry_manager.cpp @@ -6,8 +6,10 @@ */ #include +#include #include #include +#include #include #include #include @@ -37,11 +39,28 @@ struct rac_telemetry_manager { std::string sdk_version; std::string device_model; std::string os_version; + // One id per SDK run; stamped on every event that doesn't carry its own + // session/operation id, so the dashboard can group a run's telemetry and no + // row has a blank Session Id. Generated at manager creation. + std::string sdk_session_id; // HTTP callback rac_telemetry_http_callback_t http_callback; void* http_user_data; + // Isolate-safe HTTP delivery (poll-queue). When http_wakeup is set, flush + // enqueues each request here and signals http_wakeup instead of invoking + // http_callback directly — see rac_telemetry_manager_poll_http_request. + struct PendingHttpRequest { + std::string endpoint; + std::string json; + bool requires_auth; + }; + std::deque http_queue; + std::mutex http_queue_mutex; + rac_telemetry_http_wakeup_callback_t http_wakeup = nullptr; + void* http_wakeup_user_data = nullptr; + // Event queue std::vector queue; std::mutex queue_mutex; @@ -115,6 +134,12 @@ void free_payload_strings(rac_telemetry_payload_t& event) { free((void*)event.language); free((void*)event.voice); free((void*)event.archive_type); + free((void*)event.adapter_id); + free((void*)event.operation); + free((void*)event.embedding_model); + free((void*)event.image_resolution); + free((void*)event.scheduler); + free((void*)event.output_format); } #if defined(RAC_HAVE_PROTOBUF) @@ -151,6 +176,19 @@ const char* framework_proto_to_string(int32_t framework) { } } +// Normalize a framework string carried on the properties map to the same clean +// lowercase form the proto paths use. The carrier value is typically the proto +// enum NAME ("INFERENCE_FRAMEWORK_LLAMA_CPP") from a lifecycle ref's +// framework_name; convert it via the enum so "llamacpp" is emitted consistently +// across every modality. Returns the raw string if it isn't a known enum name. +const char* clean_framework(const std::string& raw) { + runanywhere::v1::InferenceFramework fw; + if (!raw.empty() && runanywhere::v1::InferenceFramework_Parse(raw, &fw)) { + return framework_proto_to_string(static_cast(fw)); + } + return raw.c_str(); +} + // Component → modality string for the V2 telemetry table grouping. One string // per backend V2 endpoint (POST /api/v2/sdk/telemetry/{modality}): llm, stt, // tts, vlm, rag, imagegen, system, model. Model events override the component @@ -203,6 +241,7 @@ rac_telemetry_manager_t* rac_telemetry_manager_create(rac_environment_t env, con manager->device_id = device_id ? device_id : ""; manager->platform = platform ? platform : ""; manager->sdk_version = sdk_version ? sdk_version : ""; + manager->sdk_session_id = generate_uuid(); manager->http_callback = nullptr; manager->http_user_data = nullptr; // Start the batch timer at creation. Flushing the very first tracked event @@ -262,6 +301,47 @@ void rac_telemetry_manager_set_http_callback(rac_telemetry_manager_t* manager, manager->http_user_data = user_data; } +void rac_telemetry_manager_set_http_wakeup(rac_telemetry_manager_t* manager, + rac_telemetry_http_wakeup_callback_t callback, + void* user_data) { + if (!manager) + return; + + manager->http_wakeup = callback; + manager->http_wakeup_user_data = user_data; +} + +rac_result_t rac_telemetry_manager_poll_http_request(rac_telemetry_manager_t* manager, + rac_proto_buffer_t* out) { + if (!manager || !out) { + return RAC_ERROR_INVALID_ARGUMENT; + } + + rac_telemetry_manager::PendingHttpRequest req; + { + std::lock_guard lock(manager->http_queue_mutex); + if (manager->http_queue.empty()) { + return RAC_ERROR_NOT_FOUND; + } + req = std::move(manager->http_queue.front()); + manager->http_queue.pop_front(); + } + + // Framing: [u8 requires_auth][u32 LE endpoint_len][endpoint utf8][json utf8]. + const uint32_t endpoint_len = static_cast(req.endpoint.size()); + std::vector framed; + framed.reserve(5 + req.endpoint.size() + req.json.size()); + framed.push_back(req.requires_auth ? 1u : 0u); + framed.push_back(static_cast(endpoint_len & 0xFFu)); + framed.push_back(static_cast((endpoint_len >> 8) & 0xFFu)); + framed.push_back(static_cast((endpoint_len >> 16) & 0xFFu)); + framed.push_back(static_cast((endpoint_len >> 24) & 0xFFu)); + framed.insert(framed.end(), req.endpoint.begin(), req.endpoint.end()); + framed.insert(framed.end(), req.json.begin(), req.json.end()); + + return rac_proto_buffer_copy(framed.data(), framed.size(), out); +} + // ============================================================================= // EVENT TRACKING // ============================================================================= @@ -291,6 +371,12 @@ rac_result_t rac_telemetry_manager_track(rac_telemetry_manager_t* manager, copy.language = dup_string(payload->language); copy.voice = dup_string(payload->voice); copy.archive_type = dup_string(payload->archive_type); + copy.adapter_id = dup_string(payload->adapter_id); + copy.operation = dup_string(payload->operation); + copy.embedding_model = dup_string(payload->embedding_model); + copy.image_resolution = dup_string(payload->image_resolution); + copy.scheduler = dup_string(payload->scheduler); + copy.output_format = dup_string(payload->output_format); { std::lock_guard lock(manager->queue_mutex); @@ -307,8 +393,8 @@ rac_result_t rac_telemetry_manager_track(rac_telemetry_manager_t* manager, RAC_LOG_DEBUG("Telemetry", "Telemetry event queued: %s", payload->event_type); // Auto-flush logic - if (!manager->http_callback) { - RAC_LOG_DEBUG("Telemetry", "HTTP callback not set, skipping auto-flush"); + if (!manager->http_callback && !manager->http_wakeup) { + RAC_LOG_DEBUG("Telemetry", "HTTP delivery not set, skipping auto-flush"); return RAC_SUCCESS; } @@ -702,6 +788,11 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, // Common: session id from the envelope. if (!ev.session_id().empty()) { payload.session_id = ev.session_id().c_str(); + } else if (!manager->sdk_session_id.empty()) { + // No per-operation session/request id on this event — fall back to the + // per-run SDK session id so no row has a blank Session Id and a run's + // events can be grouped. + payload.session_id = manager->sdk_session_id.c_str(); } // Error → success=false + error_message. Read the envelope SDKError first, @@ -709,6 +800,7 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, // populate the payload error field are still recorded as failures (parity // with the legacy union path, which carried error on the per-event struct). const std::string* payload_error = nullptr; + int payload_error_code = 0; // from a per-payload error arm that carries a code switch (ev.event_case()) { case SDKEvent::kGeneration: payload_error = &ev.generation().error(); @@ -725,17 +817,39 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, case SDKEvent::kCapability: payload_error = &ev.capability().error(); break; + case SDKEvent::kVoicePipeline: + // VAD / voice-agent failures ride the VoiceEvent ErrorEvent arm, not + // the envelope SDKError — without this they were recorded as failures + // (category=FAILURE) but with no error_message/error_code. + if (ev.voice_pipeline().payload_case() == runanywhere::v1::VoiceEvent::kError) { + payload_error = &ev.voice_pipeline().error().message(); + payload_error_code = ev.voice_pipeline().error().code(); + } + break; default: break; } + // error_code (string column on every modality row) — must outlive the + // track() deep-copy below, so keep the backing string in function scope. + std::string error_code_str; + int error_code_num = 0; if (ev.has_error() && !ev.error().message().empty()) { payload.success = RAC_FALSE; payload.has_success = RAC_TRUE; payload.error_message = ev.error().message().c_str(); + // Prefer the negative rac_result_t (c_abi_code) when present; else the + // ErrorCode enum value. + error_code_num = ev.error().has_c_abi_code() ? ev.error().c_abi_code() + : static_cast(ev.error().code()); } else if (payload_error != nullptr && !payload_error->empty()) { payload.success = RAC_FALSE; payload.has_success = RAC_TRUE; payload.error_message = payload_error->c_str(); + error_code_num = payload_error_code; + } + if (error_code_num != 0) { + error_code_str = std::to_string(error_code_num); + payload.error_code = error_code_str.c_str(); } // Strings referenced by the payload must outlive the track() copy below; keep @@ -765,6 +879,14 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, payload.has_is_streaming = RAC_TRUE; framework_str = framework_proto_to_string(g.framework()); payload.framework = framework_str.c_str(); + // The handle-less generate path carries framework on the properties + // map (proto framework is 0 there → "unknown"); prefer it when set. + { + auto fw_it = ev.properties().find("framework"); + if (fw_it != ev.properties().end() && !fw_it->second.empty()) { + payload.framework = clean_framework(fw_it->second); + } + } payload.temperature = g.temperature(); payload.max_tokens = g.max_tokens(); payload.context_length = g.context_length(); @@ -787,6 +909,11 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, payload.processing_time_ms = static_cast(m.duration_ms()); framework_str = framework_proto_to_string(m.framework()); payload.framework = framework_str.c_str(); + // archive_type has no ModelEvent field; rides the properties carrier. + auto at_it = ev.properties().find("archive_type"); + if (at_it != ev.properties().end() && !at_it->second.empty()) { + payload.archive_type = at_it->second.c_str(); + } // ModelEvent.progress is 0..1; the backend model endpoint wants 0..100. // Only emit when present so non-progress events don't send progress=0. if (m.progress() > 0.0f) { @@ -849,8 +976,24 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, payload.has_success = RAC_TRUE; } } else { - // VAD — telemetry reads only speech_duration_ms (= duration_ms(7)). + // VAD — speech_duration_ms = duration_ms(7); sample_rate is a + // native field; silence_duration_ms / segment_count ride the + // envelope properties carrier (no VoiceLifecycleEvent fields). + if (!v.model_id().empty()) { + payload.model_id = v.model_id().c_str(); + payload.model_name = !v.model_name().empty() ? v.model_name().c_str() + : v.model_id().c_str(); + } payload.speech_duration_ms = static_cast(v.duration_ms()); + payload.sample_rate = v.sample_rate(); + auto sil_it = ev.properties().find("silence_duration_ms"); + if (sil_it != ev.properties().end()) { + payload.silence_duration_ms = std::atof(sil_it->second.c_str()); + } + auto seg_it = ev.properties().find("segment_count"); + if (seg_it != ev.properties().end()) { + payload.segment_count = std::atoi(seg_it->second.c_str()); + } } break; } @@ -890,33 +1033,163 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, } } } + // CapabilityOperationEvent has no framework field; emitters that know + // it (embeddings/vlm/rag) carry it in the properties map. + { + auto fw_it = ev.properties().find("framework"); + if (fw_it != ev.properties().end() && !fw_it->second.empty()) { + payload.framework = clean_framework(fw_it->second); + } + } // LoRA capability events ride on SDK_COMPONENT_LLM, so component_to_modality // mapped them to "llm". Override to "lora" by capability kind. model_id // carries the base model; the operation is encoded in event_type. switch (c.kind()) { case runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_ATTACHED: case runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_DETACHED: - case runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_FAILED: + case runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_FAILED: { payload.modality = "lora"; + // operation column (backend aggregates lora by_operation) — + // derived from the capability kind. String literals; dup'd in + // track() like the other payload strings. + payload.operation = + c.kind() == runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_ATTACHED + ? "attach" + : (c.kind() == + runanywhere::v1::CAPABILITY_OPERATION_EVENT_KIND_LORA_DETACHED + ? "detach" + : "failed"); + // adapter_id rides the properties carrier; points into `ev`, + // dup'd when the payload is queued (rac_telemetry_manager_track). + auto ad_it = ev.properties().find("adapter_id"); + if (ad_it != ev.properties().end()) { + payload.adapter_id = ad_it->second.c_str(); + } + auto asz_it = ev.properties().find("adapter_size_bytes"); + if (asz_it != ev.properties().end()) { + payload.adapter_size_bytes = std::atoll(asz_it->second.c_str()); + } break; + } default: break; } switch (ev.component()) { - case runanywhere::v1::SDK_COMPONENT_VLM: + case runanywhere::v1::SDK_COMPONENT_VLM: { payload.image_count = static_cast(c.input_count()); payload.output_tokens = static_cast(c.output_count()); - payload.total_tokens = payload.input_tokens + payload.output_tokens; + // LLM-style token metrics ride the properties carrier; the VLM + // V2 row carries them alongside image_count. + auto in_it = ev.properties().find("input_tokens"); + if (in_it != ev.properties().end()) { + payload.input_tokens = static_cast(std::atoi(in_it->second.c_str())); + } + auto tot_it = ev.properties().find("total_tokens"); + payload.total_tokens = + tot_it != ev.properties().end() + ? static_cast(std::atoi(tot_it->second.c_str())) + : payload.input_tokens + payload.output_tokens; + auto tps_it = ev.properties().find("tokens_per_second"); + if (tps_it != ev.properties().end()) { + payload.tokens_per_second = std::atof(tps_it->second.c_str()); + } + auto ttft_it = ev.properties().find("time_to_first_token_ms"); + if (ttft_it != ev.properties().end()) { + payload.time_to_first_token_ms = std::atof(ttft_it->second.c_str()); + } + auto temp_it = ev.properties().find("temperature"); + if (temp_it != ev.properties().end()) { + payload.temperature = std::atof(temp_it->second.c_str()); + } + auto mt_it = ev.properties().find("max_tokens"); + if (mt_it != ev.properties().end()) { + payload.max_tokens = std::atoi(mt_it->second.c_str()); + } + // Vision-specific metrics ride the properties carrier (no proto fields). + auto vt_it = ev.properties().find("vision_tokens"); + if (vt_it != ev.properties().end()) { + payload.vision_tokens = static_cast(std::atoi(vt_it->second.c_str())); + } + auto vet_it = ev.properties().find("vision_encode_time_ms"); + if (vet_it != ev.properties().end()) { + payload.vision_encode_time_ms = std::atof(vet_it->second.c_str()); + } + auto ir_it = ev.properties().find("image_resolution"); + if (ir_it != ev.properties().end() && !ir_it->second.empty()) { + payload.image_resolution = ir_it->second.c_str(); + } + // generation_time ≈ processing_time (set from duration_ms above). + payload.generation_time_ms = payload.processing_time_ms; break; - case runanywhere::v1::SDK_COMPONENT_RAG: + } + case runanywhere::v1::SDK_COMPONENT_RAG: { payload.retrieved_docs_count = static_cast(c.output_count()); + // top_k / retrieval_time_ms ride the properties carrier. + auto tk_it = ev.properties().find("top_k"); + if (tk_it != ev.properties().end()) { + payload.top_k = static_cast(std::atoi(tk_it->second.c_str())); + } + auto rt_it = ev.properties().find("retrieval_time_ms"); + if (rt_it != ev.properties().end()) { + payload.retrieval_time_ms = std::atof(rt_it->second.c_str()); + } + auto em_it = ev.properties().find("embedding_model"); + if (em_it != ev.properties().end() && !em_it->second.empty()) { + payload.embedding_model = em_it->second.c_str(); + } break; - case runanywhere::v1::SDK_COMPONENT_EMBEDDINGS: + } + case runanywhere::v1::SDK_COMPONENT_EMBEDDINGS: { // input_count = texts embedded, output_count = vectors produced. // embedding_model is read from model_id (set above) in the JSON. payload.input_count = static_cast(c.input_count()); payload.vectors_produced = static_cast(c.output_count()); + // embedding_dimension rides the properties carrier (no proto field). + auto dim_it = ev.properties().find("embedding_dimension"); + if (dim_it != ev.properties().end()) { + payload.embedding_dimension = + static_cast(std::atoi(dim_it->second.c_str())); + } break; + } + case runanywhere::v1::SDK_COMPONENT_DIFFUSION: { + // ImageGen detail fields all ride the properties carrier + // (CapabilityOperationEvent has no diffusion fields). + auto pl_it = ev.properties().find("prompt_length"); + if (pl_it != ev.properties().end()) + payload.imagegen_prompt_length = std::atoi(pl_it->second.c_str()); + auto npl_it = ev.properties().find("negative_prompt_length"); + if (npl_it != ev.properties().end()) + payload.imagegen_negative_prompt_length = std::atoi(npl_it->second.c_str()); + auto iw_it = ev.properties().find("image_width"); + if (iw_it != ev.properties().end()) + payload.image_width = std::atoi(iw_it->second.c_str()); + auto ih_it = ev.properties().find("image_height"); + if (ih_it != ev.properties().end()) + payload.image_height = std::atoi(ih_it->second.c_str()); + auto ni_it = ev.properties().find("num_images"); + if (ni_it != ev.properties().end()) + payload.num_images = std::atoi(ni_it->second.c_str()); + auto ns_it = ev.properties().find("num_inference_steps"); + if (ns_it != ev.properties().end()) + payload.num_inference_steps = std::atoi(ns_it->second.c_str()); + auto gs_it = ev.properties().find("guidance_scale"); + if (gs_it != ev.properties().end()) + payload.guidance_scale = std::atof(gs_it->second.c_str()); + auto seed_it = ev.properties().find("seed"); + if (seed_it != ev.properties().end()) + payload.seed = std::atoll(seed_it->second.c_str()); + auto osz_it = ev.properties().find("output_size_bytes"); + if (osz_it != ev.properties().end()) + payload.output_size_bytes = std::atoll(osz_it->second.c_str()); + auto sch_it = ev.properties().find("scheduler"); + if (sch_it != ev.properties().end() && !sch_it->second.empty()) + payload.scheduler = sch_it->second.c_str(); + auto of_it = ev.properties().find("output_format"); + if (of_it != ev.properties().end() && !of_it->second.empty()) + payload.output_format = of_it->second.c_str(); + break; + } default: break; } @@ -955,6 +1228,26 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, payload.voice_tts_ms = m.tts_total_ms(); payload.voice_total_ms = m.end_to_end_ms(); payload.processing_time_ms = m.end_to_end_ms(); + // MetricsEvent has no model/framework/char fields — read them + // from the envelope properties carrier set by + // publish_voice_turn_metrics. (session_id is read at L761.) + auto mid_it = ev.properties().find("model_id"); + if (mid_it != ev.properties().end() && !mid_it->second.empty()) { + payload.model_id = mid_it->second.c_str(); + payload.model_name = mid_it->second.c_str(); // id-as-name fallback + } + auto fw_it = ev.properties().find("framework"); + if (fw_it != ev.properties().end() && !fw_it->second.empty()) { + payload.framework = clean_framework(fw_it->second); + } + auto tc_it = ev.properties().find("transcript_chars"); + if (tc_it != ev.properties().end()) { + payload.transcript_chars = std::atoi(tc_it->second.c_str()); + } + auto rc_it = ev.properties().find("response_chars"); + if (rc_it != ev.properties().end()) { + payload.response_chars = std::atoi(rc_it->second.c_str()); + } if (!ev.has_error()) { payload.success = RAC_TRUE; payload.has_success = RAC_TRUE; @@ -962,6 +1255,20 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, } break; } + case SDKEvent::kInitialization: { + // sdk.models.loaded carries its count + duration in the envelope + // properties map (InitializationEvent has no numeric fields), so + // read them here or SystemIngestEvent.count stays blank. + auto cnt_it = ev.properties().find("model_count"); + if (cnt_it != ev.properties().end()) { + payload.count = std::atoi(cnt_it->second.c_str()); + } + auto dur_it = ev.properties().find("duration_ms"); + if (dur_it != ev.properties().end()) { + payload.processing_time_ms = std::atof(dur_it->second.c_str()); + } + break; + } default: break; } @@ -993,7 +1300,7 @@ rac_result_t rac_telemetry_manager_track_proto(rac_telemetry_manager_t* manager, rac_result_t result = rac_telemetry_manager_track(manager, &payload); if (result == RAC_SUCCESS && manager->environment != RAC_ENV_DEVELOPMENT && is_completion && - manager->http_callback) { + (manager->http_callback || manager->http_wakeup)) { RAC_LOG_DEBUG("Telemetry", "Completion event detected, triggering immediate flush"); rac_telemetry_manager_flush(manager); } @@ -1019,8 +1326,8 @@ rac_result_t rac_telemetry_manager_flush(rac_telemetry_manager_t* manager) { return RAC_ERROR_INVALID_ARGUMENT; } - if (!manager->http_callback) { - RAC_LOG_DEBUG("Telemetry", "No HTTP callback registered, cannot flush telemetry"); + if (!manager->http_callback && !manager->http_wakeup) { + RAC_LOG_DEBUG("Telemetry", "No HTTP delivery registered, cannot flush telemetry"); return RAC_ERROR_NOT_INITIALIZED; } @@ -1094,7 +1401,19 @@ rac_result_t rac_telemetry_manager_flush(rac_telemetry_manager_t* manager) { const std::string endpoint = std::string(RAC_ENDPOINT_TELEMETRY_V2_PREFIX) + modality; RAC_LOG_DEBUG("Telemetry", "POST %s (%zu bytes): %.500s", endpoint.c_str(), json_len, json); - manager->http_callback(manager->http_user_data, endpoint.c_str(), json, json_len, RAC_TRUE); + if (manager->http_wakeup) { + // Isolate-safe path: enqueue an owned copy and signal the platform to + // drain it from its own thread/isolate (see poll_http_request). Used + // by Flutter, whose Dart FFI data callbacks are isolate-bound. + { + std::lock_guard lock(manager->http_queue_mutex); + manager->http_queue.push_back({endpoint, std::string(json, json_len), true}); + } + manager->http_wakeup(manager->http_wakeup_user_data); + } else if (manager->http_callback) { + manager->http_callback(manager->http_user_data, endpoint.c_str(), json, json_len, + RAC_TRUE); + } free(json); } diff --git a/sdk/runanywhere-commons/src/jni/runanywhere_commons_jni.cpp b/sdk/runanywhere-commons/src/jni/runanywhere_commons_jni.cpp index b6a4b4de6e..992d9fc356 100644 --- a/sdk/runanywhere-commons/src/jni/runanywhere_commons_jni.cpp +++ b/sdk/runanywhere-commons/src/jni/runanywhere_commons_jni.cpp @@ -5704,6 +5704,27 @@ Java_com_runanywhere_sdk_native_bridge_RunAnywhereBridge_racVoiceAgentProcessTur return static_cast(rc); } +// Streaming raw-frame ingress: the core segments utterances and runs the turn +// pipeline, returning a VoiceAgentResult inline when one completes (empty +// otherwise). VoiceEvents fan out to the handle callback (no per-call +// listener). +JNIEXPORT jbyteArray JNICALL +Java_com_runanywhere_sdk_native_bridge_RunAnywhereBridge_racVoiceAgentFeedAudioProto( + JNIEnv* env, jclass clazz, jlong handle, jbyteArray audioData, jint sampleRateHz, jint channels, + jint encoding, jboolean isFinal) { + (void)clazz; + JByteArrayView audio(env, audioData); + if (handle == 0L || !audio.ok) + return nullptr; + rac_proto_buffer_t result = {}; + rac_proto_buffer_init(&result); + rac_result_t rc = rac_voice_agent_feed_audio_proto( + reinterpret_cast(handle), audio.data(), audio.size(), + static_cast(sampleRateHz), static_cast(channels), + static_cast(encoding), isFinal == JNI_TRUE ? RAC_TRUE : RAC_FALSE, &result); + return makeProtoCallResult(env, rc, &result, "racVoiceAgentFeedAudioProto"); +} + JNIEXPORT jbyteArray JNICALL Java_com_runanywhere_sdk_native_bridge_RunAnywhereBridge_racVoiceAgentTranscribeProto( JNIEnv* env, jclass clazz, jlong handle, jbyteArray requestBytes) { diff --git a/sdk/runanywhere-flutter/packages/runanywhere/android/src/main/kotlin/com/runanywhere/sdk/httptransport/OkHttpHttpTransport.kt b/sdk/runanywhere-flutter/packages/runanywhere/android/src/main/kotlin/com/runanywhere/sdk/httptransport/OkHttpHttpTransport.kt index e4a43c4306..0ce1710cf7 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/android/src/main/kotlin/com/runanywhere/sdk/httptransport/OkHttpHttpTransport.kt +++ b/sdk/runanywhere-flutter/packages/runanywhere/android/src/main/kotlin/com/runanywhere/sdk/httptransport/OkHttpHttpTransport.kt @@ -25,6 +25,7 @@ package com.runanywhere.sdk.httptransport +import android.os.Looper import android.util.Log import com.runanywhere.sdk.native.bridge.RunAnywhereBridge import okhttp3.Call @@ -36,7 +37,11 @@ import okhttp3.RequestBody import okhttp3.RequestBody.Companion.toRequestBody import java.io.IOException import java.time.Duration +import java.util.concurrent.Callable import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutionException +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicReference @@ -233,6 +238,34 @@ object OkHttpHttpTransport { // `cRequestSend` / `cRequestStream` / `cRequestResume` trampolines) // --------------------------------------------------------------------- + /** + * Dedicated daemon IO pool used to keep blocking requests off the main + * thread (see [runOffMainThread]). + */ + private val ioExecutor: ExecutorService = Executors.newCachedThreadPool { r -> + Thread(r, "rac-http-io").apply { isDaemon = true } + } + + /** + * Run a blocking network [block] off the Android main thread. The C ABI is + * synchronous — the caller blocks for the result either way — but this + * transport is invoked on Flutter's main isolate thread (unlike the Kotlin + * SDK's Dispatchers.IO caller), so executing the socket I/O inline triggers + * NetworkOnMainThreadException. Hop to the IO pool and block for the result; + * when already off the main thread the work runs inline (no extra hop). + */ + private fun runOffMainThread(block: () -> T): T { + if (Looper.myLooper() != Looper.getMainLooper()) { + return block() + } + val future = ioExecutor.submit(Callable { block() }) + return try { + future.get() + } catch (e: ExecutionException) { + throw e.cause ?: e + } + } + /** * `request_send` vtable slot — blocking single-shot request. Invoked * from JNI via `CallStaticObjectMethod`. Returns a [HttpResponse]. @@ -246,18 +279,20 @@ object OkHttpHttpTransport { timeoutMs: Long, ): HttpResponse { return try { - val request = buildRequest(method, url, headersFlat, bodyBytes, resumeFromByte = 0L) - val clientForCall = resolveClient(timeoutMs) - - clientForCall.newCall(request).execute().use { resp -> - val headerPairs = flattenHeaders(resp.headers) - val responseBody = resp.body?.bytes() ?: ByteArray(0) - HttpResponse( - statusCode = resp.code, - headers = headerPairs, - bodyBytes = responseBody, - errorMessage = null, - ) + runOffMainThread { + val request = buildRequest(method, url, headersFlat, bodyBytes, resumeFromByte = 0L) + val clientForCall = resolveClient(timeoutMs) + + clientForCall.newCall(request).execute().use { resp -> + val headerPairs = flattenHeaders(resp.headers) + val responseBody = resp.body?.bytes() ?: ByteArray(0) + HttpResponse( + statusCode = resp.code, + headers = headerPairs, + bodyBytes = responseBody, + errorMessage = null, + ) + } } } catch (e: Throwable) { HttpResponse( diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/features/tts/services/audio_playback_manager.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/features/tts/services/audio_playback_manager.dart index 57d29b7187..3bb6bf3e1f 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/features/tts/services/audio_playback_manager.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/features/tts/services/audio_playback_manager.dart @@ -66,6 +66,16 @@ class AudioPlaybackManager { final AudioPlayer _player = AudioPlayer(); + /// When `true` (default) this manager configures playback with the + /// output-only `.playback` audio session category — correct for standalone + /// `RunAnywhere.speak(...)`. Set to `false` when the caller owns a live + /// full-duplex session — e.g. the voice agent keeps a single `.playAndRecord` + /// session active across capture and playback. Switching to `.playback` while + /// the mic session is live trips AVAudioSessionErrorInsufficientPriority + /// ('!pri', OSStatus 561017449) and the reply never plays. Mirrors Swift + /// `AudioPlaybackManager.managesAudioSession`. + bool managesAudioSession = true; + bool _isPlaying = false; Duration _duration = Duration.zero; Duration _position = Duration.zero; @@ -118,6 +128,45 @@ class AudioPlaybackManager { _currentTempFile = tempFile; await tempFile.writeAsBytes(wavData); + // Mix with other audio instead of taking exclusive audio focus. On Android + // this maps to AndroidAudioFocus.none, so playing a TTS reply does NOT + // evict the voice-agent mic recorder (which otherwise receives + // AUDIOFOCUS_LOSS and stops — leaving the pipeline deaf after the first + // turn). The voice-agent mic driver gates capture during playback, so + // coexisting record + playback never self-transcribes. + if (managesAudioSession) { + await _player.setAudioContext( + AudioContextConfig(focus: AudioContextConfigFocus.mixWithOthers) + .build(), + ); + } else { + // The voice agent owns a live `.playAndRecord` mic session (the `record` + // plugin's default category). Match that category here instead of the + // default `.playback`: switching to an output-only category while the mic + // session is active trips AVAudioSessionErrorInsufficientPriority ('!pri', + // OSStatus 561017449) and the reply is silently dropped. `defaultToSpeaker` + // forces the loud speaker route (under `.playAndRecord` the output can + // otherwise fall back to the quiet receiver). Android keeps focus=none so + // playback does not evict the recorder. Mirrors the iOS Swift driver's + // configureVoiceAudioSession() + managesAudioSession=false. + await _player.setAudioContext( + AudioContext( + iOS: AudioContextIOS( + category: AVAudioSessionCategory.playAndRecord, + options: const { + AVAudioSessionOptions.defaultToSpeaker, + AVAudioSessionOptions.mixWithOthers, + AVAudioSessionOptions.allowBluetooth, + AVAudioSessionOptions.allowBluetoothA2DP, + }, + ), + android: const AudioContextAndroid( + audioFocus: AndroidAudioFocus.none, + ), + ), + ); + } + await _player.setVolume(volume.clamp(0.0, 1.0)); await _player.setPlaybackRate(rate.clamp(0.5, 2.0)); diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/features/voice_agent/services/voice_agent_mic_driver.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/features/voice_agent/services/voice_agent_mic_driver.dart new file mode 100644 index 0000000000..a49b870503 --- /dev/null +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/features/voice_agent/services/voice_agent_mic_driver.dart @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// voice_agent_mic_driver.dart — audio ingress for the Flutter voice agent. +// +// The C ABI owns NO microphone (rac_voice_agent.h "Audio-Ingress Contract"): +// the platform SDK must capture mic audio and push complete utterances into +// the C core, or the session is dead air. Without this driver the Flutter +// voice agent only subscribed to the output event stream and never fed any +// PCM, so VAD/STT never saw audio → the LLM got no input → no reply. +// +// Mirrors Kotlin `VoiceAgentMicDriver`: capture 16 kHz mono PCM16 via +// [AudioCaptureManager], segment utterances with energy-based endpointing, +// and drive each utterance through `rac_voice_agent_process_turn_proto` +// (DartBridgeVoiceAgent.processTurnStream). The turn's VoiceEvents are +// forwarded to [events] (so the public eventStream observes them) and the +// synthesized TTS reply is played through [AudioPlaybackManager]. +// +// The endpointing is intentionally simple — the C++ pipeline re-runs its own +// VAD over each submitted buffer; the only job here is deciding where one +// utterance ends. Mic chunks that arrive while a turn is processing are +// dropped: the pipeline is strictly turn-taking (no barge-in), which also +// avoids transcribing the device's own TTS output. + +import 'dart:async'; +import 'dart:math'; +import 'dart:typed_data'; + +import 'package:runanywhere/features/stt/services/audio_capture_manager.dart'; +import 'package:runanywhere/features/tts/services/audio_playback_manager.dart'; +import 'package:runanywhere/foundation/logging/sdk_logger.dart'; +import 'package:runanywhere/generated/voice_agent_service.pb.dart' + as voice_agent_pb; +import 'package:runanywhere/generated/voice_events.pb.dart' as voice_events_pb; +import 'package:runanywhere/native/dart_bridge.dart'; +import 'package:runanywhere/native/dart_bridge_audio.dart'; + +/// Captures mic audio and drives per-utterance voice-agent turns. [start] +/// begins capture; [events] streams every turn's VoiceEvents; [stop] tears +/// capture + playback down. Mirrors Kotlin `VoiceAgentMicDriver`. +class VoiceAgentMicDriver { + VoiceAgentMicDriver(); + + static final _logger = SDKLogger('VoiceAgentMic'); + + static const int _sampleRateHz = 16000; + static const int _bytesPerSample = 2; + + /// Absolute floor for the adaptive speech threshold (normalized RMS). + static const double _speechRmsThreshold = 0.015; + + /// Speech must exceed this multiple of the tracked ambient noise floor. + static const double _speechFloorMultiplier = 2.2; + + /// Per-chunk rate at which the ambient floor creeps up toward louder ambient. + static const double _noiseFloorRise = 0.05; + + /// Trailing silence that closes an utterance. + static const int _endOfUtteranceSilenceMs = 800; + + /// Utterances with less accumulated speech than this are noise. + static const int _minSpeechMs = 300; + + /// Hard cap so a noisy room cannot grow an unbounded buffer. + static const int _maxUtteranceMs = 15000; + + /// Leading chunks kept so the utterance onset is not clipped. + static const int _preRollChunks = 3; + + /// Piper's native rate; used when an audio frame omits sample_rate_hz. + static const int _defaultTtsSampleRateHz = 22050; + + final AudioCaptureManager _capture = AudioCaptureManager(); + final AudioPlaybackManager _playback = AudioPlaybackManager(); + final StreamController _out = + StreamController(); + + StreamSubscription? _micSub; + bool _stopped = false; + bool _processing = false; + + // Segmentation state. + final List _preRoll = []; + final BytesBuilder _utterance = BytesBuilder(); + bool _inSpeech = false; + int _speechMs = 0; + int _silenceMs = 0; + double _noiseFloor = _speechRmsThreshold; + + /// VoiceEvents produced by each turn (userSaid, llm tokens, audio, pipeline + /// state). The public eventStream yields from this. + Stream get events => _out.stream; + + /// Begin mic capture. On permission/capture failure the [events] stream is + /// closed with an error so the collector can surface it. + Future start() async { + // The voice agent runs a single full-duplex (.playAndRecord) session for the + // whole turn-taking loop — the `record` plugin configures it on + // startRecording (defaultToSpeaker by default). Playback must NOT switch the + // session to the output-only `.playback` category, or it trips + // AVAudioSessionErrorInsufficientPriority ('!pri', OSStatus 561017449) and + // the reply is dropped. Mirrors the iOS Swift driver + // (playback.managesAudioSession = false). + _playback.managesAudioSession = false; + final stream = + await _capture.startRecording(sampleRate: _sampleRateHz, numChannels: 1); + if (stream == null) { + _out.addError(StateError( + 'Microphone capture unavailable (permission denied or busy)')); + unawaited(_out.close()); + return; + } + _logger.info('Voice-agent mic capture started'); + _micSub = stream.listen( + _onChunk, + onError: (Object e, StackTrace st) => _logger.warning('Mic error: $e'), + cancelOnError: false, + ); + } + + /// Stop capture + playback and close the event stream. + Future stop() async { + if (_stopped) return; + _stopped = true; + await _micSub?.cancel(); + _micSub = null; + await _capture.stopRecording(); + await _playback.stop(); + if (!_out.isClosed) { + await _out.close(); + } + _logger.info('Voice-agent mic capture stopped'); + } + + void _onChunk(Uint8List chunk) { + // Drop chunks while a turn is processing (turn-taking, no barge-in). + if (_stopped || _processing || chunk.isEmpty) return; + + final chunkMs = chunk.length * 1000 ~/ (_sampleRateHz * _bytesPerSample); + final level = _rms(chunk); + // Adaptive endpointing: track the ambient floor — drop instantly to any + // quieter level, creep up only while not in speech — and require a chunk + // to rise clearly above that floor to count as speech. + final speechThreshold = + max(_speechRmsThreshold, _noiseFloor * _speechFloorMultiplier); + final isSpeech = level >= speechThreshold; + if (level < _noiseFloor) { + _noiseFloor = level; + } else if (!isSpeech) { + _noiseFloor += (level - _noiseFloor) * _noiseFloorRise; + } + + if (!_inSpeech) { + _preRoll.add(chunk); + while (_preRoll.length > _preRollChunks) { + _preRoll.removeAt(0); + } + if (isSpeech) { + _inSpeech = true; + _speechMs = chunkMs; + _silenceMs = 0; + _utterance.clear(); + for (final pre in _preRoll) { + _utterance.add(pre); + } + _preRoll.clear(); + } + return; + } + + _utterance.add(chunk); + if (isSpeech) { + _speechMs += chunkMs; + _silenceMs = 0; + } else { + _silenceMs += chunkMs; + } + + final utteranceMs = + _utterance.length * 1000 ~/ (_sampleRateHz * _bytesPerSample); + if (_silenceMs >= _endOfUtteranceSilenceMs || utteranceMs >= _maxUtteranceMs) { + final audio = _utterance.toBytes(); + final hadSpeech = _speechMs >= _minSpeechMs; + _resetSegmentation(); + if (hadSpeech) { + unawaited(_processTurn(audio)); + } else { + _logger.debug('Utterance discarded (${_speechMs}ms speech < ' + '${_minSpeechMs}ms)'); + } + } + } + + void _resetSegmentation() { + _inSpeech = false; + _speechMs = 0; + _silenceMs = 0; + _utterance.clear(); + _preRoll.clear(); + } + + Future _processTurn(Uint8List audio) async { + _processing = true; + final ttsPcm = BytesBuilder(); + var ttsSampleRate = 0; + var ttsEncoding = voice_events_pb.AudioEncoding.AUDIO_ENCODING_UNSPECIFIED; + try { + final request = voice_agent_pb.VoiceAgentTurnRequest( + requestId: 'turn-${DateTime.now().microsecondsSinceEpoch}', + audioData: audio, + sampleRateHz: _sampleRateHz, + channels: 1, + encoding: voice_events_pb.AudioEncoding.AUDIO_ENCODING_PCM_S16_LE, + ); + _logger.info('Submitting voice turn (${audio.length} bytes)'); + + await for (final ev in DartBridge.voiceAgent.processTurnStream(request)) { + if (!_out.isClosed) { + _out.add(ev); + } + if (ev.hasAudio() && ev.audio.pcm.isNotEmpty) { + ttsPcm.add(ev.audio.pcm is Uint8List + ? ev.audio.pcm as Uint8List + : Uint8List.fromList(ev.audio.pcm)); + if (ev.audio.sampleRateHz > 0) { + ttsSampleRate = ev.audio.sampleRateHz; + } + if (ev.audio.encoding != + voice_events_pb.AudioEncoding.AUDIO_ENCODING_UNSPECIFIED) { + ttsEncoding = ev.audio.encoding; + } + } + } + } catch (e) { + _logger.warning('Voice turn failed: $e'); + if (!_out.isClosed) { + _out.addError(e); + } + } finally { + _processing = false; + _resetSegmentation(); + } + + await _playTts(ttsPcm.toBytes(), ttsSampleRate, ttsEncoding); + } + + // Play the turn's synthesized reply through the TTS sink. Runs before the + // next mic chunk is processed (the _processing gate is cleared above but the + // playback await keeps the turn logically open), so the mic stays gated + // while the device speaks — no self-transcription. + Future _playTts( + Uint8List pcm, + int sampleRateHz, + voice_events_pb.AudioEncoding encoding, + ) async { + if (pcm.isEmpty || _stopped) return; + final sampleRate = sampleRateHz > 0 ? sampleRateHz : _defaultTtsSampleRateHz; + // TTS backends emit f32 LE by default (AudioFrameEvent contract); only + // convert as PCM16 when the frame explicitly says so. + final wav = + encoding == voice_events_pb.AudioEncoding.AUDIO_ENCODING_PCM_S16_LE + ? DartBridgeAudio.int16ToWav(pcm, sampleRate) + : DartBridgeAudio.float32ToWav(pcm, sampleRate); + if (wav == null || wav.isEmpty) { + _logger.warning('TTS audio conversion failed (${pcm.length} bytes, ' + '${sampleRate}Hz, $encoding)'); + return; + } + _processing = true; // keep mic gated while speaking + try { + await _playback.play(wav); + } catch (e) { + _logger.warning('Agent reply playback failed: $e'); + } finally { + _processing = false; + _resetSegmentation(); + } + } + + double _rms(Uint8List chunk) { + final samples = chunk.length ~/ _bytesPerSample; + if (samples == 0) return 0.0; + final data = ByteData.sublistView(chunk); + var sum = 0.0; + for (var i = 0; i < samples; i++) { + final sample = data.getInt16(i * _bytesPerSample, Endian.little).toDouble(); + sum += sample * sample; + } + return sqrt(sum / samples) / 32767.0; + } +} diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge.dart index 3cba8321f9..6373db1832 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge.dart @@ -298,6 +298,7 @@ class DartBridge { // Phase 2 now owns assignment fetch through rac_http_transport. await DartBridgeModelAssignment.register( environment: _environment, + baseURL: baseURL, autoFetch: false, ); _logger.debug('Model assignment callbacks registered (autoFetch: false)'); diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_auth.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_auth.dart index 07a961cd02..e0bf0cf2cc 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_auth.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_auth.dart @@ -272,6 +272,14 @@ class DartBridgeAuth { final refreshed = await _retryHTTPViaCommons(); if (refreshed != null && refreshed.isNotEmpty) return refreshed; + // A proactive refresh just failed (init race / transient network). Prefer a + // still-usable live token over giving up — returning empty here makes the + // adapter fall back to `Bearer `, which is a guaranteed 401 on the + // JWT-only V2 telemetry endpoints. The adapter's 401-retry path + // (_refreshForAdapter) still handles the case where this token is truly + // expired, so we never strand an auth'd request that had a valid token. + if (current != null && current.isNotEmpty) return current; + // Last-resort cached access token (may still be stale; the server // will reject it and the 401 retry path will refresh again). return _secureCache['com.runanywhere.sdk.accessToken']; diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_device.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_device.dart index a170e5d102..44bffddc1b 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_device.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_device.dart @@ -8,6 +8,7 @@ import 'dart:io'; import 'package:device_info_plus/device_info_plus.dart'; import 'package:ffi/ffi.dart'; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; +import 'package:runanywhere/adapters/http_client_adapter.dart'; import 'package:runanywhere/foundation/logging/sdk_logger.dart'; import 'package:runanywhere/native/dart_bridge_auth.dart'; import 'package:runanywhere/native/platform_loader.dart'; @@ -282,8 +283,18 @@ class DartBridgeDevice { final accessToken = _accessToken ?? DartBridgeAuth.instance.getAccessToken(); - if (requiresAuth && accessToken != null && accessToken.isNotEmpty) { - headers['Authorization'] = 'Bearer $accessToken'; + if (requiresAuth) { + // Fall back to the SDK API key when no access token has been issued yet. + // Device registration is the first authenticated call (before any JWT + // exists), so without this fallback the Authorization header is omitted + // and the backend returns 401 "Authorization header missing". Kotlin + // parity: HTTPClientAdapter.resolveToken() resolves token -> apiKey. + final token = (accessToken != null && accessToken.isNotEmpty) + ? accessToken + : HTTPClientAdapter.shared.apiKey; + if (token.isNotEmpty) { + headers['Authorization'] = 'Bearer $token'; + } } logger.debug('Device registration POST to: $fullUrl'); @@ -522,8 +533,15 @@ void _getDeviceInfoCallback( _cachedDeviceInfoPtrs.add(appIdPtr); outInfo.ref.appIdentifier = appIdPtr; - // Platform - final platformPtr = 'flutter'.toNativeUtf8(); + // Platform = OS family (matches iOS/Kotlin + backend contract), not binding. + final platformName = Platform.isAndroid + ? 'android' + : Platform.isIOS + ? 'ios' + : Platform.isMacOS + ? 'macos' + : 'flutter'; + final platformPtr = platformName.toNativeUtf8(); _cachedDeviceInfoPtrs.add(platformPtr); outInfo.ref.platform = platformPtr; } catch (e) { diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_embeddings.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_embeddings.dart index 0c40606df9..6b43365675 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_embeddings.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_embeddings.dart @@ -4,6 +4,11 @@ // embeddings service; Dart passes generated request bytes and receives // generated result bytes. +import 'dart:ffi'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:ffi/ffi.dart'; import 'package:runanywhere/core/native/rac_native.dart'; import 'package:runanywhere/generated/embeddings_options.pb.dart' show EmbeddingsRequest, EmbeddingsResult; @@ -22,6 +27,9 @@ class DartBridgeEmbeddings { _embedBatchLifecycleProtoForTesting = override; } + /// Synchronous variant retained for the unit-test harness (which drives it + /// with [setEmbedBatchLifecycleProtoForTesting]). Production callers use + /// [embedBatchAsync], which runs the blocking native call off the UI isolate. EmbeddingsResult embedBatch(EmbeddingsRequest request) { _validateRequest(request); @@ -45,9 +53,74 @@ class DartBridgeEmbeddings { ); } + /// Embed a batch through the lifecycle-owned generated-proto ABI, running the + /// blocking native call in a short-lived worker isolate (`Isolate.run`) so the + /// calling isolate — usually the Flutter UI isolate — stays responsive. + /// Embedding a large batch (e.g. RAG ingest) is a long synchronous block; on + /// the UI isolate it freezes frames. Mirrors `dart_bridge_stt.dart`'s + /// `transcribeLifecycleProtoAsync`: this ABI is lifecycle-owned (no Dart-held + /// handle), so the worker re-resolves the engine via the commons model + /// lifecycle — nothing isolate-bound crosses. + Future embedBatchAsync(EmbeddingsRequest request) async { + _validateRequest(request); + + final override = _embedBatchLifecycleProtoForTesting; + if (override != null) { + return override(request); + } + + if (RacNative.bindings.rac_embeddings_embed_batch_lifecycle_proto == null) { + throw UnsupportedError( + 'rac_embeddings_embed_batch_lifecycle_proto is unavailable', + ); + } + + final requestBytes = request.writeToBuffer(); + final resultBytes = + await Isolate.run(() => _embeddingsEmbedBatchWorker(requestBytes)); + return EmbeddingsResult.fromBuffer(resultBytes); + } + void _validateRequest(EmbeddingsRequest request) { if (request.texts.where((text) => text.isNotEmpty).isEmpty) { throw ArgumentError('EmbeddingsRequest.texts is required'); } } } + +/// Blocking body of [DartBridgeEmbeddings.embedBatchAsync]: a plain +/// request→response proto call with no callbacks. Top-level so the +/// `Isolate.run` closure captures only its sendable `Uint8List` argument. +/// `RacNative.bindings` is a per-isolate static — the worker re-resolves the +/// dylib symbols on first access (idempotent `PlatformLoader.loadCommons()`, +/// same convention as the LLM/STT/TTS workers). Returns the serialized +/// EmbeddingsResult so the main isolate owns the decode. +Uint8List _embeddingsEmbedBatchWorker(Uint8List requestBytes) { + final bindings = RacNative.bindings; + final fn = bindings.rac_embeddings_embed_batch_lifecycle_proto; + if (fn == null) { + throw UnsupportedError( + 'rac_embeddings_embed_batch_lifecycle_proto is unavailable', + ); + } + + final requestPtr = DartBridgeProtoUtils.copyBytes(requestBytes); + final out = calloc(); + try { + bindings.rac_proto_buffer_init(out); + final code = fn(requestPtr, requestBytes.length, out); + DartBridgeProtoUtils.ensureSuccess( + out, + code, + 'rac_embeddings_embed_batch_lifecycle_proto', + ); + if (out.ref.data == nullptr || out.ref.size == 0) { + return Uint8List(0); + } + return Uint8List.fromList(out.ref.data.asTypedList(out.ref.size)); + } finally { + bindings.rac_proto_buffer_free(out); + calloc.free(out); + calloc.free(requestPtr); + } +} diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_llm.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_llm.dart index e3275903d3..ef7e1476d1 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_llm.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_llm.dart @@ -243,7 +243,7 @@ class DartBridgeLLM { final requestBytes = request.writeToBuffer(); final sendPort = receivePort.sendPort; unawaited( - Isolate.run(() => _llmStreamWorker(requestBytes, sendPort)) + _runLlmStreamWorker(requestBytes, sendPort) .catchError((Object e, StackTrace st) { // Worker isolate crashed (RemoteError) before the rc sentinel. if (!controller.isClosed) { @@ -318,6 +318,15 @@ class DartBridgeLLM { // that callback to `.listener` — do not move the blocking call back onto the // main isolate. +/// Runs [_llmStreamWorker] in a worker isolate. Hoisted to top level so the +/// `Isolate.run` closure captures ONLY its two sendable parameters +/// (`Uint8List` + `SendPort`). Inlined in [DartBridgeLLM.generateStreamProto], +/// the closure captured that method's whole local context — including the +/// unsendable `ReceivePort`/`StreamController` — which fails the isolate spawn +/// with "object is unsendable". +Future _runLlmStreamWorker(Uint8List requestBytes, SendPort port) => + Isolate.run(() => _llmStreamWorker(requestBytes, port)); + /// Blocking body of [DartBridgeLLM.generateStreamProto]. Runs the single-call /// streaming ABI on the worker isolate; the worker-owned `isolateLocal` /// callback fires synchronously per token (commons requires a synchronous diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_rag.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_rag.dart index 88db068f55..41f1032d45 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_rag.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_rag.dart @@ -3,6 +3,8 @@ // Generated-proto RAG session bridge. import 'dart:ffi' as ffi; +import 'dart:isolate'; +import 'dart:typed_data'; import 'package:ffi/ffi.dart'; import 'package:runanywhere/core/native/rac_native.dart'; @@ -110,6 +112,25 @@ class DartBridgeRAG { ); } + /// Ingest a document, running the blocking chunk→embed→index work in a + /// short-lived worker isolate (`Isolate.run`) so the calling isolate — usually + /// the Flutter UI isolate — stays responsive. The RAG session is process- + /// global C++ state (in-memory, mutex-protected vector store + ONNX + /// embeddings + llama.cpp, no file I/O on this path), so its address is valid + /// on the worker; this is the same cross-isolate engine use that + /// `dart_bridge_llm.dart`'s worker already relies on. + Future ingestDocumentAsync(RAGDocument document) async { + final session = _requireSession(); + if (RacNative.bindings.rac_rag_ingest_proto == null) { + throw UnsupportedError('rac_rag_ingest_proto is unavailable'); + } + final sessionAddr = session.address; + final requestBytes = document.writeToBuffer(); + final resultBytes = + await Isolate.run(() => _ragIngestWorker(sessionAddr, requestBytes)); + return RAGStatistics.fromBuffer(resultBytes); + } + RAGStatistics clearDocuments() { final session = _requireSession(); final fn = RacNative.bindings.rac_rag_clear_proto; @@ -143,7 +164,25 @@ class DartBridgeRAG { ); } - Future queryAsync(RAGQueryOptions options) async => query(options); + /// Query the pipeline, running the blocking embed→retrieve→LLM-generate work + /// in a short-lived worker isolate (`Isolate.run`) so the calling isolate — + /// usually the Flutter UI isolate — stays responsive for the whole answer. + /// `rac_rag_query_proto` runs the LLM generation inline (max_tokens/ + /// temperature/system_prompt), so this is the heavy chat call. Same + /// cross-isolate engine use as `dart_bridge_llm.dart` (see + /// [ingestDocumentAsync]); the synchronous [query] is retained for callers + /// that need it. + Future queryAsync(RAGQueryOptions options) async { + final session = _requireSession(); + if (RacNative.bindings.rac_rag_query_proto == null) { + throw UnsupportedError('rac_rag_query_proto is unavailable'); + } + final sessionAddr = session.address; + final requestBytes = options.writeToBuffer(); + final resultBytes = + await Isolate.run(() => _ragQueryWorker(sessionAddr, requestBytes)); + return RAGResult.fromBuffer(resultBytes); + } RAGStatistics getStatistics() { final session = _requireSession(); @@ -167,3 +206,63 @@ class DartBridgeRAG { return session; } } + +// MARK: - Worker-isolate entry points +// +// Top-level so the `Isolate.run` closures capture only sendable values (the +// session address as an int + the request `Uint8List`). The RAG session is +// process-global C++ state, so `Pointer.fromAddress` reconstitutes a valid +// pointer on the worker isolate; the session's ONNX embeddings + llama.cpp LLM +// are invoked cross-isolate exactly as `dart_bridge_llm.dart`'s worker does, +// and the vector store is in-memory + mutex-protected (no file I/O on these +// paths). `RacNative.bindings` re-resolves the dylib symbols on first access +// (idempotent `PlatformLoader.loadCommons()`). Each returns the serialized +// proto result so the calling isolate owns the decode. + +Uint8List _ragIngestWorker(int sessionAddr, Uint8List requestBytes) { + final bindings = RacNative.bindings; + final fn = bindings.rac_rag_ingest_proto; + if (fn == null) { + throw UnsupportedError('rac_rag_ingest_proto is unavailable'); + } + final session = ffi.Pointer.fromAddress(sessionAddr); + final reqPtr = DartBridgeProtoUtils.copyBytes(requestBytes); + final out = calloc(); + try { + bindings.rac_proto_buffer_init(out); + final code = fn(session, reqPtr, requestBytes.length, out); + DartBridgeProtoUtils.ensureSuccess(out, code, 'rac_rag_ingest_proto'); + if (out.ref.data == ffi.nullptr || out.ref.size == 0) { + return Uint8List(0); + } + return Uint8List.fromList(out.ref.data.asTypedList(out.ref.size)); + } finally { + bindings.rac_proto_buffer_free(out); + calloc.free(reqPtr); + calloc.free(out); + } +} + +Uint8List _ragQueryWorker(int sessionAddr, Uint8List requestBytes) { + final bindings = RacNative.bindings; + final fn = bindings.rac_rag_query_proto; + if (fn == null) { + throw UnsupportedError('rac_rag_query_proto is unavailable'); + } + final session = ffi.Pointer.fromAddress(sessionAddr); + final reqPtr = DartBridgeProtoUtils.copyBytes(requestBytes); + final out = calloc(); + try { + bindings.rac_proto_buffer_init(out); + final code = fn(session, reqPtr, requestBytes.length, out); + DartBridgeProtoUtils.ensureSuccess(out, code, 'rac_rag_query_proto'); + if (out.ref.data == ffi.nullptr || out.ref.size == 0) { + return Uint8List(0); + } + return Uint8List.fromList(out.ref.data.asTypedList(out.ref.size)); + } finally { + bindings.rac_proto_buffer_free(out); + calloc.free(reqPtr); + calloc.free(out); + } +} diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_stt.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_stt.dart index 66ab697c6e..b770a4b070 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_stt.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_stt.dart @@ -6,6 +6,7 @@ library; import 'dart:async'; import 'dart:ffi'; +import 'dart:isolate'; import 'dart:typed_data'; import 'package:ffi/ffi.dart'; @@ -112,6 +113,11 @@ class DartBridgeSTT { // MARK: - Transcription /// Transcribe audio through the lifecycle-owned generated-proto STT ABI. + /// + /// Synchronous variant retained for the unit-test harness (which drives it + /// with [setTranscribeLifecycleProtoForTesting]). Production callers use + /// [transcribeLifecycleProtoAsync], which runs the blocking native call off + /// the UI isolate. STTOutput transcribeLifecycleProto(STTTranscriptionRequest request) { _validateLifecycleRequest(request); @@ -135,6 +141,36 @@ class DartBridgeSTT { ); } + /// Transcribe audio through the lifecycle-owned generated-proto STT ABI, + /// running the blocking native call in a short-lived worker isolate + /// (`Isolate.run`) so the calling isolate — usually the Flutter UI isolate — + /// stays responsive for the whole transcription. Whisper decode of a batch + /// buffer is a long synchronous block; on the UI isolate it freezes frames. + /// Mirrors `dart_bridge_llm.dart`'s `generateProto`: this ABI is + /// lifecycle-owned (no Dart-held handle), so the worker re-resolves the + /// engine via the commons model lifecycle — nothing isolate-bound crosses. + Future transcribeLifecycleProtoAsync( + STTTranscriptionRequest request, + ) async { + _validateLifecycleRequest(request); + + final override = _transcribeLifecycleProtoForTesting; + if (override != null) { + return override(request); + } + + if (RacNative.bindings.rac_stt_transcribe_lifecycle_proto == null) { + throw UnsupportedError( + 'rac_stt_transcribe_lifecycle_proto is unavailable', + ); + } + + final requestBytes = request.writeToBuffer(); + final resultBytes = + await Isolate.run(() => _sttTranscribeWorker(requestBytes)); + return STTOutput.fromBuffer(resultBytes); + } + /// Transcribe audio with serialized runanywhere.v1.STTOptions. Future transcribeProto( Uint8List audioData, @@ -443,3 +479,38 @@ class DartBridgeSTT { } } } + +/// Blocking body of [DartBridgeSTT.transcribeLifecycleProtoAsync]: a plain +/// request→response proto call with no callbacks. Top-level so the +/// `Isolate.run` closure captures only its sendable `Uint8List` argument. +/// `RacNative.bindings` is a per-isolate static — the worker re-resolves the +/// dylib symbols on first access (idempotent `PlatformLoader.loadCommons()`, +/// same convention as the LLM/VLM workers). Returns the serialized STTOutput +/// so the main isolate owns the decode. +Uint8List _sttTranscribeWorker(Uint8List requestBytes) { + final bindings = RacNative.bindings; + final fn = bindings.rac_stt_transcribe_lifecycle_proto; + if (fn == null) { + throw UnsupportedError('rac_stt_transcribe_lifecycle_proto is unavailable'); + } + + final requestPtr = DartBridgeProtoUtils.copyBytes(requestBytes); + final out = calloc(); + try { + bindings.rac_proto_buffer_init(out); + final code = fn(requestPtr, requestBytes.length, out); + DartBridgeProtoUtils.ensureSuccess( + out, + code, + 'rac_stt_transcribe_lifecycle_proto', + ); + if (out.ref.data == nullptr || out.ref.size == 0) { + return Uint8List(0); + } + return Uint8List.fromList(out.ref.data.asTypedList(out.ref.size)); + } finally { + bindings.rac_proto_buffer_free(out); + calloc.free(out); + calloc.free(requestPtr); + } +} diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_telemetry.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_telemetry.dart index 3c81870b5d..415b6eee1f 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_telemetry.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_telemetry.dart @@ -5,12 +5,11 @@ import 'dart:convert'; import 'dart:ffi'; import 'dart:io'; -import 'dart:typed_data'; - import 'package:device_info_plus/device_info_plus.dart'; import 'package:ffi/ffi.dart'; import 'package:runanywhere/adapters/http_client_adapter.dart'; +import 'package:runanywhere/core/native/rac_native.dart'; import 'package:runanywhere/foundation/logging/sdk_logger.dart'; import 'package:runanywhere/native/dart_bridge_environment.dart'; import 'package:runanywhere/native/platform_loader.dart'; @@ -42,11 +41,8 @@ class DartBridgeTelemetry { static bool _isInitialized = false; // ignore: unused_field static SDKEnvironment? _environment; - static String? _baseURL; - static String? _accessToken; static Pointer? _managerPtr; - static Pointer>? - _httpCallbackPtr; + static NativeCallable)>? _httpWakeup; // ============================================================================ // Lifecycle @@ -77,98 +73,125 @@ class DartBridgeTelemetry { } } - /// Initialize telemetry manager with device info (full async init) + /// Create the telemetry manager and attach it as the C++ event router's + /// telemetry sink, in Phase 1 — BEFORE commons emits the + /// INITIALIZATION_STAGE_STARTED/COMPLETED events from rac_sdk_init_phase1_proto. + /// Without this the "system" modality never lands: those init events fire + /// during Phase-1 core init, and if the sink is only attached in Phase 2 they + /// hit a null sink and are dropped. The manager queues events immediately; + /// the actual flush still waits for Phase 2 (HTTP layer configured). Device + /// info (async model lookup) is filled in later by [initialize]. + static void attachSinkPhase1({ + required SDKEnvironment environment, + required String deviceId, + }) { + if (_managerPtr != null) return; + _environment = environment; + try { + _createManagerAndAttach(PlatformLoader.loadCommons(), environment, deviceId); + _logger.debug('Telemetry sink attached in Phase 1'); + } catch (e) { + _logger.debug('Phase-1 telemetry attach failed: $e'); + } + } + + /// Create the manager (env/device/platform/sdk), register the HTTP wake-up, + /// and attach the telemetry sink. Shared by [attachSinkPhase1] (Phase 1) and + /// the [initialize] fallback (when Phase-1 attach didn't run). Does NOT set + /// device info or mark initialized. + static void _createManagerAndAttach( + DynamicLibrary lib, SDKEnvironment environment, String deviceId) { + const sdkVersion = '0.19.13'; + // platform is the OS family (matches iOS "ios"/"macos", Kotlin "android", + // and the backend telemetry_events.platform contract), NOT the binding. + final platform = Platform.isAndroid + ? 'android' + : Platform.isIOS + ? 'ios' + : Platform.isMacOS + ? 'macos' + : 'flutter'; + + final createManager = lib.lookupFunction< + Pointer Function(Int32, Pointer, Pointer, Pointer), + Pointer Function(int, Pointer, Pointer, + Pointer)>('rac_telemetry_manager_create'); + + final deviceIdPtr = deviceId.toNativeUtf8(); + final platformPtr = platform.toNativeUtf8(); + final sdkVersionPtr = sdkVersion.toNativeUtf8(); + try { + final ptr = createManager( + _environmentToInt(environment), deviceIdPtr, platformPtr, sdkVersionPtr); + if (ptr == nullptr) { + _logger.warning('Failed to create telemetry manager'); + return; + } + _managerPtr = ptr; + // Register the HTTP wake-up + attach the sink so the router feeds events + // into the manager (queued until Phase-2 flush). + _registerHttpCallback(); + _setTelemetrySink(ptr); + } finally { + calloc.free(deviceIdPtr); + calloc.free(platformPtr); + calloc.free(sdkVersionPtr); + } + } + + /// Complete telemetry init (Phase 2): fill in device info. Reuses the manager + /// created in Phase 1 via [attachSinkPhase1]; only creates one here as a + /// fallback (gated on a real baseURL) if Phase-1 attach never ran. static Future initialize({ required SDKEnvironment environment, required String deviceId, String? baseURL, - String? accessToken, }) async { if (_isInitialized) { _logger.debug('Telemetry already initialized'); return; } - - // Bail out if the example app forwarded an unfilled - // .env / dart-define placeholder. We don't want to POST telemetry - // to a literal "YOUR_SUPABASE_PROJECT_URL" string. - if (!DartBridgeDevConfig.isUsableCredential(baseURL) || - !DartBridgeDevConfig.isUsableCredential(accessToken)) { - _logger.warning( - 'Telemetry skipped — baseURL/accessToken looks like a placeholder. ' - 'Set real values via dart-define or runtime config.', - ); - _isInitialized = true; // Suppress retry. - return; - } - _environment = environment; - _baseURL = baseURL; - _accessToken = accessToken; try { final lib = PlatformLoader.loadCommons(); - // Get device info - final deviceModel = await _getDeviceModel(); - final osVersion = Platform.operatingSystemVersion; - const sdkVersion = '0.19.13'; - const platform = 'flutter'; - - // Create telemetry manager - final createManager = lib.lookupFunction< - Pointer Function( - Int32, Pointer, Pointer, Pointer), - Pointer Function(int, Pointer, Pointer, - Pointer)>('rac_telemetry_manager_create'); - - final envValue = _environmentToInt(environment); - final deviceIdPtr = deviceId.toNativeUtf8(); - final platformPtr = platform.toNativeUtf8(); - final sdkVersionPtr = sdkVersion.toNativeUtf8(); - - try { - _managerPtr = - createManager(envValue, deviceIdPtr, platformPtr, sdkVersionPtr); - - if (_managerPtr == nullptr || - _managerPtr == Pointer.fromAddress(0)) { - _logger.warning('Failed to create telemetry manager'); + if (_managerPtr == null) { + // Fallback: Phase-1 attach didn't run. The auth token isn't available + // yet (applied at send time — Kotlin parity); only the baseURL must be + // real, else creating a manager that can never flush is wasteful. + if (!DartBridgeDevConfig.isUsableCredential(baseURL)) { + _logger.warning( + 'Telemetry skipped — baseURL looks like a placeholder. ' + 'Set a real base URL via dart-define or runtime config.', + ); + _isInitialized = true; // Suppress retry. return; } + _createManagerAndAttach(lib, environment, deviceId); + } - // Set device info - final setDeviceInfo = lib.lookupFunction< - Void Function(Pointer, Pointer, Pointer), - void Function(Pointer, Pointer, - Pointer)>('rac_telemetry_manager_set_device_info'); - - final deviceModelPtr = deviceModel.toNativeUtf8(); - final osVersionPtr = osVersion.toNativeUtf8(); - - setDeviceInfo(_managerPtr!, deviceModelPtr, osVersionPtr); - - calloc.free(deviceModelPtr); - calloc.free(osVersionPtr); - - // Register HTTP callback - _registerHttpCallback(); - - // Attach this manager as the C++ event router's telemetry sink. The - // router (`rac::events::route`) forwards every event whose destination - // carries the TELEMETRY bit into the manager via - // `rac_telemetry_manager_track_proto` and does the per-event translation - // internally — Dart no longer forwards per-event analytics. Mirrors - // Swift's `rac_events_set_telemetry_sink(mgr.ptr)`. - _setTelemetrySink(_managerPtr!); - + if (_managerPtr == null || _managerPtr == nullptr) { _isInitialized = true; - _logger.debug('Telemetry manager initialized'); - } finally { - calloc.free(deviceIdPtr); - calloc.free(platformPtr); - calloc.free(sdkVersionPtr); + return; } + + // Device info — the model lookup is async, so it only happens here in + // Phase 2 (the Phase-1 attach creates the manager without it). + final deviceModel = await _getDeviceModel(); + final osVersion = Platform.operatingSystemVersion; + final setDeviceInfo = lib.lookupFunction< + Void Function(Pointer, Pointer, Pointer), + void Function(Pointer, Pointer, + Pointer)>('rac_telemetry_manager_set_device_info'); + final deviceModelPtr = deviceModel.toNativeUtf8(); + final osVersionPtr = osVersion.toNativeUtf8(); + setDeviceInfo(_managerPtr!, deviceModelPtr, osVersionPtr); + calloc.free(deviceModelPtr); + calloc.free(osVersionPtr); + + _isInitialized = true; + _logger.debug('Telemetry manager initialized'); } catch (e, stack) { _logger.debug('Telemetry initialization error: $e', metadata: { 'stack': stack.toString(), @@ -188,6 +211,10 @@ class DartBridgeTelemetry { // into a manager we are about to destroy. _setTelemetrySink(nullptr); + // Close the cross-isolate wake-up callable. + _httpWakeup?.close(); + _httpWakeup = null; + final destroy = lib.lookupFunction), void Function(Pointer)>('rac_telemetry_manager_destroy'); @@ -216,11 +243,6 @@ class DartBridgeTelemetry { } } - /// Update access token - static void setAccessToken(String? token) { - _accessToken = token; - } - // ============================================================================ // HTTP Callback Registration // ============================================================================ @@ -230,23 +252,72 @@ class DartBridgeTelemetry { try { final lib = PlatformLoader.loadCommons(); - final setCallback = lib.lookupFunction< - Void Function( - Pointer, - Pointer>, - Pointer), + final setWakeup = lib.lookupFunction< + Void Function(Pointer, + Pointer)>>, Pointer), void Function( Pointer, - Pointer>, - Pointer)>('rac_telemetry_manager_set_http_callback'); + Pointer)>>, + Pointer)>('rac_telemetry_manager_set_http_wakeup'); + + // NativeCallable.listener is cross-isolate-safe: commons may signal this + // from the LLM worker isolate during a generation-completion flush, and + // Dart dispatches it back to this (main) isolate. The wake-up carries no + // request data, so it avoids the "native callback from a different + // isolate" abort a data-carrying Pointer.fromFunction hit. On wake-up we + // drain commons' owned request queue via rac_telemetry_manager_poll_*. + final wakeup = + NativeCallable)>.listener(_telemetryHttpWakeup); + _httpWakeup = wakeup; + setWakeup(_managerPtr!, wakeup.nativeFunction, nullptr); + _logger.debug('Telemetry HTTP wake-up registered'); + } catch (e) { + _logger.debug('Failed to register HTTP wake-up: $e'); + } + } - _httpCallbackPtr = Pointer.fromFunction( - _telemetryHttpCallback); + /// Drain commons' queued telemetry HTTP requests (signalled by the wake-up). + /// Each request is an owned buffer framed as + /// [u8 requiresAuth][u32 LE endpointLen][endpoint][json]. + static void drainHttpQueue() { + final managerPtr = _managerPtr; + if (managerPtr == null) return; - setCallback(_managerPtr!, _httpCallbackPtr!, nullptr); - _logger.debug('Telemetry HTTP callback registered'); + try { + final lib = PlatformLoader.loadCommons(); + final pollFn = lib.lookupFunction< + Int32 Function(Pointer, Pointer), + int Function(Pointer, Pointer)>( + 'rac_telemetry_manager_poll_http_request', + ); + final bindings = RacNative.bindings; + + // Bounded so a misbehaving producer can't spin forever; the loop also + // exits as soon as poll reports an empty queue (rc != 0). + for (var i = 0; i < 256; i++) { + final out = calloc(); + try { + bindings.rac_proto_buffer_init(out); + final code = pollFn(managerPtr, out); + // RAC_SUCCESS == 0; anything else (e.g. RAC_ERROR_NOT_FOUND) = drained. + if (code != 0 || out.ref.data == nullptr || out.ref.size < 5) { + return; + } + final bytes = + out.ref.data.asTypedList(out.ref.size).toList(growable: false); + final requiresAuth = bytes[0] != 0; + final endpointLen = + bytes[1] | (bytes[2] << 8) | (bytes[3] << 16) | (bytes[4] << 24); + final endpoint = utf8.decode(bytes.sublist(5, 5 + endpointLen)); + final body = utf8.decode(bytes.sublist(5 + endpointLen)); + unawaited(_sendTelemetryHttp(endpoint, body, requiresAuth)); + } finally { + bindings.rac_proto_buffer_free(out); + calloc.free(out); + } + } } catch (e) { - _logger.debug('Failed to register HTTP callback: $e'); + _logger.debug('Telemetry HTTP queue drain error: $e'); } } @@ -289,59 +360,54 @@ class DartBridgeTelemetry { } // ============================================================================= -// HTTP Callback Function +// HTTP Wake-up Function // ============================================================================= -/// HTTP callback invoked by C++ when telemetry needs to be sent -void _telemetryHttpCallback( - Pointer userData, - Pointer endpoint, - Pointer jsonBody, - int jsonLength, - int requiresAuth, -) { - if (endpoint == nullptr || jsonBody == nullptr) return; - - try { - final endpointStr = endpoint.toDartString(); - final bodyStr = jsonBody.toDartString(); - final needsAuth = requiresAuth != 0; - - // Fire and forget HTTP call - unawaited(_sendTelemetryHttp(endpointStr, bodyStr, needsAuth)); - } catch (e) { - SDKLogger('DartBridge.Telemetry').error('HTTP callback error: $e'); - } +/// Wake-up from commons (may be signalled from any isolate, e.g. the LLM +/// streaming worker isolate). Registered as a `NativeCallable.listener` so the +/// drain runs on this (main) isolate regardless of the caller. No request data +/// is passed here — it is pulled from commons' owned queue in [drainHttpQueue]. +void _telemetryHttpWakeup(Pointer userData) { + DartBridgeTelemetry.drainHttpQueue(); } -/// Send telemetry via HTTP +/// Send telemetry via HTTP. +/// +/// Routes through the adapter's authenticated [HTTPClientAdapter.send] so the +/// request carries the resolved SDK access token (with 401-refresh) and the +/// canonical SDK headers. The backend telemetry endpoints require a valid +/// bearer token (`validate_sdk_bearer_token`); the previous `rawRequest` path +/// only attached `_accessToken`, which is never populated here, so every +/// telemetry POST was unauthenticated. Kotlin parity: telemetry goes through +/// the token-managed adapter, not a hand-built request. Future _sendTelemetryHttp( String endpoint, String body, bool requiresAuth) async { try { - final baseURL = - DartBridgeTelemetry._baseURL ?? 'https://api.runanywhere.ai'; - final url = '$baseURL$endpoint'; - - final headers = { - 'Content-Type': 'application/json', - 'Accept': 'application/json', - }; - - if (requiresAuth && DartBridgeTelemetry._accessToken != null) { - headers['Authorization'] = 'Bearer ${DartBridgeTelemetry._accessToken}'; - } - - final response = await HTTPClientAdapter.shared.rawRequest( + final response = await HTTPClientAdapter.shared.send( method: 'POST', - url: url, - headers: headers, - body: Uint8List.fromList(utf8.encode(body)), + path: endpoint, + body: body, + requiresAuth: requiresAuth, ); + if (response.isSuccess) { + DartBridgeTelemetry._logger + .info('Telemetry POST $endpoint -> ${response.statusCode} OK'); + } else { + // Surface the real reason — commons only logs the error string, and a + // non-2xx carries its detail in the body (e.g. a 422 extra="forbid" + // field, a 404 missing endpoint). Truncated to keep logs readable. + final detail = response.body; + DartBridgeTelemetry._logger.warning( + 'Telemetry POST $endpoint -> ${response.statusCode}: ' + '${detail.length > 400 ? detail.substring(0, 400) : detail}', + ); + } + _notifyHttpComplete( response.isSuccess, response.body, - null, + response.isSuccess ? null : 'HTTP ${response.statusCode}', ); } catch (e) { _notifyHttpComplete(false, null, e.toString()); @@ -378,10 +444,3 @@ void _notifyHttpComplete(bool success, String? responseJson, String? error) { } } -// ============================================================================= -// FFI Types -// ============================================================================= - -/// HTTP callback type: void (*callback)(void*, const char*, const char*, size_t, rac_bool_t) -typedef RacTelemetryHttpCallbackNative = Void Function( - Pointer, Pointer, Pointer, IntPtr, Int32); diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_tool_calling.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_tool_calling.dart index c5cb05fa7a..628cfa1ce7 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_tool_calling.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_tool_calling.dart @@ -10,9 +10,11 @@ library; import 'dart:async'; import 'dart:ffi' as ffi; +import 'dart:isolate'; import 'dart:typed_data'; import 'package:ffi/ffi.dart'; +import 'package:fixnum/fixnum.dart' show Int64; import 'package:runanywhere/core/native/rac_native.dart'; import 'package:runanywhere/foundation/logging/sdk_logger.dart'; @@ -116,96 +118,77 @@ class DartBridgeToolCalling { ); } - /// Create a native tool-calling session + install a callback that decodes - /// [ToolCallingSessionEvent] bytes onto the returned broadcast stream. - /// The returned [ToolCallingSessionHandle] wraps the session handle + the - /// live NativeCallable so callers can destroy both together. + /// Create a tool-calling session that runs entirely in a dedicated worker + /// isolate, so the inline llama.cpp generation loop (create + every step) + /// never blocks the calling isolate. Events are copied SYNCHRONOUSLY inside a + /// worker-owned `NativeCallable.isolateLocal` (commons' dispatcher hands out a + /// stack-local buffer valid only for the call — see tool_calling_session.cpp + /// `dispatch_pending`) and forwarded to this isolate over a port, where they + /// are decoded onto the returned broadcast stream. Tool results are sent back + /// to the worker via [ToolCallingSessionHandle.stepWithResult]; cancellation + /// uses the thread-safe `rac_tool_calling_session_cancel_proto` directly. ToolCallingSessionHandle createSession( ToolCallingSessionCreateRequest request, ) { - final createFn = RacNative.bindings.rac_tool_calling_session_create_proto; - final destroyFn = RacNative.bindings.rac_tool_calling_session_destroy_proto; - if (createFn == null || destroyFn == null) { + if (RacNative.bindings.rac_tool_calling_session_create_proto == null || + RacNative.bindings.rac_tool_calling_session_step_with_result_proto == + null || + RacNative.bindings.rac_tool_calling_session_destroy_proto == null) { throw UnsupportedError( 'rac_tool_calling_session_* proto APIs are unavailable', ); } final controller = StreamController.broadcast(); - final nativeCb = - ffi.NativeCallable.listener(( - ffi.Pointer bytesPtr, - int bytesLen, - ffi.Pointer _, - ) { - if (bytesLen <= 0 || bytesPtr == ffi.nullptr) return; - final copy = Uint8List.fromList(bytesPtr.asTypedList(bytesLen)); - try { - controller.add(ToolCallingSessionEvent.fromBuffer(copy)); - } catch (e, st) { - controller.addError(e, st); - } - }); - - final bytes = request.writeToBuffer(); - final reqPtr = DartBridgeProtoUtils.copyBytes(bytes); - final handleOut = calloc(); + final fromWorker = ReceivePort(); + final idCompleter = Completer(); + final handle = ToolCallingSessionHandle._( + sessionId: idCompleter.future, + events: controller, + fromWorker: fromWorker, + ); - try { - final code = createFn( - reqPtr, - bytes.length, - nativeCb.nativeFunction, - ffi.nullptr, - handleOut, - ); - if (code != 0) { - nativeCb.close(); - unawaited(controller.close()); - throw StateError( - 'rac_tool_calling_session_create_proto failed: code=$code', + fromWorker.listen((Object? message) { + if (message is SendPort) { + handle._toWorker = message; + } else if (message is Uint8List) { + if (controller.isClosed) return; + try { + controller.add(ToolCallingSessionEvent.fromBuffer(message)); + } catch (e, st) { + controller.addError(e, st); + } + } else if (message is int) { + _logger.debug('Tool calling session created: handle=$message'); + if (!idCompleter.isCompleted) idCompleter.complete(message); + } else if (message is List && message.isNotEmpty && message[0] == 'err') { + final err = StateError( + message.length > 1 ? '${message[1]}' : 'tool-calling worker error', ); + if (!controller.isClosed) controller.addError(err); + if (!idCompleter.isCompleted) idCompleter.completeError(err); } - final sessionHandle = handleOut.value; - _logger.debug('Tool calling session created: handle=$sessionHandle'); - return ToolCallingSessionHandle._( - sessionHandle: sessionHandle, - events: controller, - nativeCb: nativeCb, - ); - } finally { - calloc.free(reqPtr); - calloc.free(handleOut); - } - } + }); - /// Forward a tool-result into an in-flight session so commons can continue - /// the orchestration loop. - void sessionStepWithResult( - ToolCallingSessionStepWithResultRequest request, - ) { - final fn = - RacNative.bindings.rac_tool_calling_session_step_with_result_proto; - if (fn == null) { - throw UnsupportedError( - 'rac_tool_calling_session_step_with_result_proto is unavailable', - ); - } final bytes = request.writeToBuffer(); - final ptr = DartBridgeProtoUtils.copyBytes(bytes); - try { - final code = fn(ptr, bytes.length); - if (code != 0) { - throw StateError( - 'rac_tool_calling_session_step_with_result_proto failed: code=$code', + unawaited(() async { + try { + final iso = await Isolate.spawn( + _toolSessionWorkerEntry, + [fromWorker.sendPort, bytes], ); + handle._isolate = iso; + if (handle._closed) iso.kill(priority: Isolate.immediate); + } catch (e, st) { + if (!controller.isClosed) controller.addError(e, st); + if (!idCompleter.isCompleted) idCompleter.completeError(e, st); } - } finally { - calloc.free(ptr); - } + }()); + + return handle; } - /// Teardown a session created via [createSession]. + /// Teardown a session handle (called by the session worker, not directly). void destroySession(int sessionHandle) { final fn = RacNative.bindings.rac_tool_calling_session_destroy_proto; if (fn == null) return; @@ -243,49 +226,181 @@ class DartBridgeToolCalling { /// must be closed when the session ends. class ToolCallingSessionHandle { ToolCallingSessionHandle._({ - required this.sessionHandle, + required Future sessionId, required StreamController events, - required ffi.NativeCallable - nativeCb, - }) : _events = events, - _nativeCb = nativeCb; + required ReceivePort fromWorker, + }) : _sessionId = sessionId, + _events = events, + _fromWorker = fromWorker { + // Cache the resolved id for synchronous cancel. Errors are swallowed here — + // a failed create surfaces through the events stream and to [sessionId]. + _sessionId.then((id) => _resolvedId = id).catchError((Object _) => 0); + } - /// C-side session handle (also carried on `ToolCallingSessionCreateResult`). - final int sessionHandle; + final Future _sessionId; + int _resolvedId = 0; final StreamController _events; - final ffi.NativeCallable _nativeCb; + final ReceivePort _fromWorker; + SendPort? _toWorker; + Isolate? _isolate; bool _closed = false; - /// Stream of `ToolCallingSessionEvent`s emitted by commons. + /// Future of the C-side session id, resolved when the worker reports it back + /// (a turn after the first event). Only needed for [cancel]. + Future get sessionId => _sessionId; + + /// Resolved C-side session id, or 0 if the worker has not reported it yet. + int get resolvedSessionId => _resolvedId; + + /// Stream of `ToolCallingSessionEvent`s emitted by commons (forwarded from + /// the session worker over a port). Stream get events => _events.stream; - /// Destroy the native session, close the callback, and complete the stream. + /// Forward a tool result so commons continues the loop. Runs the next + /// generation turn IN THE SESSION WORKER — never blocks the calling isolate. + /// The worker fills in its own session id, so callers need not await it. + void stepWithResult({ + required String toolCallId, + required String resultJson, + required String error, + }) { + _toWorker?.send(['step', toolCallId, resultJson, error]); + } + + /// Tear the session down: ask the worker to destroy the native session, + /// quiesce, and close its callback (the worker owns the teardown ordering so + /// commons never invokes a freed trampoline), then drop our port. When the + /// worker is idle (the usual case — close runs after the final event) it + /// processes this immediately and exits on its own. Future close() async { if (_closed) return; _closed = true; - DartBridgeToolCalling.shared.destroySession(sessionHandle); - // Teardown ordering: (1) destroy the session above so commons stops - // accepting new dispatches into this NativeCallable, (2) quiesce so the - // dispatcher returns from any in-flight callback whose `user_data` slot - // was snapshotted under the commons mutex before the destroy landed - // (see `rac_tool_calling.h:642` + `tool_calling_session.cpp:841`), - // (3) close the NativeCallable backing that `user_data`. Skipping the - // quiesce step lets the dispatcher invoke the trampoline after Dart frees - // the user_data pointer (UAF). Mirrors the same ordering used by every - // other Flutter stream wrapper (LLM/STT/TTS/VLM/voice-agent) and Swift's - // `HandleStreamAdapter.tearDown()`. - RacNative.bindings.rac_tool_calling_session_proto_quiesce?.call(); - _nativeCb.close(); + final worker = _toWorker; + if (worker != null) { + worker.send('close'); + } else { + // Worker not live yet — kill it once it spawns so it cannot leak. + _isolate?.kill(priority: Isolate.immediate); + } + _toWorker = null; + _fromWorker.close(); await _events.close(); } - /// Cancel the in-flight native loop. Distinct from [close]: - /// cancel interrupts the underlying LLM generate from another isolate, - /// while [close] tears the session down. The recommended pattern is to - /// wire this into a `StreamSubscription.onCancel`, fanning consumer-side - /// cancellation into the native loop. + /// Cancel the in-flight native loop. `rac_tool_calling_session_cancel_proto` + /// is thread-safe and idempotent, so it is invoked directly from this isolate + /// (it latches a cancel the worker's in-flight generate checks) rather than + /// queued behind the worker's blocking turn. bool cancel() { - if (_closed) return false; - return DartBridgeToolCalling.shared.cancelSession(sessionHandle); + if (_closed || _resolvedId == 0) return false; + return DartBridgeToolCalling.shared.cancelSession(_resolvedId); + } +} + +// MARK: - Session worker isolate +// +// `Isolate.spawn` entry (top-level; the spawn message is a sendable +// `[SendPort, Uint8List]` list — no closure capture). The whole session lives +// here: create + every step run the blocking llama.cpp generation loop inline +// without touching the UI isolate, and the event callback is a WORKER-owned +// `NativeCallable.isolateLocal` that fires SYNCHRONOUSLY on this worker thread +// during create/step. That synchronous timing is mandatory: commons' +// `dispatch_pending` hands the callback a pointer into a stack-local buffer +// that is freed the moment the call returns (tool_calling_session.cpp), so the +// bytes must be copied here-and-now — a deferred `.listener` read on another +// isolate is a use-after-free. The copy is what crosses back to the main +// isolate over `mainPort`. `RacNative.bindings` re-resolves the dylib symbols +// on first access (idempotent `PlatformLoader.loadCommons()`). +// +// Protocol — worker → main: the control `SendPort` (first), then serialized +// `ToolCallingSessionEvent` bytes per emission, then the `int` session id once +// create returns; `['err', msg]` on a hard create/step failure. main → worker: +// `['step', toolCallId, resultJson, error]` to continue the loop; `'close'` to +// destroy + quiesce + close the callback and exit. Cancellation is NOT routed +// here — the thread-safe cancel ABI is called directly from the main isolate. +void _toolSessionWorkerEntry(List args) { + final mainPort = args[0] as SendPort; + final createRequestBytes = args[1] as Uint8List; + + final bindings = RacNative.bindings; + final createFn = bindings.rac_tool_calling_session_create_proto; + final stepFn = bindings.rac_tool_calling_session_step_with_result_proto; + final destroyFn = bindings.rac_tool_calling_session_destroy_proto; + if (createFn == null || stepFn == null || destroyFn == null) { + mainPort.send(['err', 'tool-calling session ABI unavailable']); + return; + } + + final control = ReceivePort(); + var sessionId = 0; + var closed = false; + + final nativeCb = + ffi.NativeCallable.isolateLocal( + (ffi.Pointer bytesPtr, int bytesLen, + ffi.Pointer _) { + if (bytesLen <= 0 || bytesPtr == ffi.nullptr) return; + // Copy synchronously — commons frees this buffer when the call returns. + mainPort.send(Uint8List.fromList(bytesPtr.asTypedList(bytesLen))); + }); + + void cleanup() { + if (closed) return; + closed = true; + if (sessionId != 0) destroyFn(sessionId); + bindings.rac_tool_calling_session_proto_quiesce?.call(); + nativeCb.close(); + control.close(); + } + + // Hand the control port back before generating so main can send steps/close. + mainPort.send(control.sendPort); + + control.listen((Object? message) { + if (closed) return; + if (message is List && message.isNotEmpty && message[0] == 'step') { + final req = ToolCallingSessionStepWithResultRequest( + sessionHandle: Int64(sessionId), + toolCallId: message[1] as String, + resultJson: message[2] as String, + error: message[3] as String, + ); + final bytes = req.writeToBuffer(); + final ptr = DartBridgeProtoUtils.copyBytes(bytes); + try { + final code = stepFn(ptr, bytes.length); + if (code != 0) { + mainPort.send(['err', 'step failed: code=$code']); + } + } finally { + calloc.free(ptr); + } + } else if (message == 'close') { + cleanup(); + } + }); + + // First turn — blocks here; events fire via nativeCb → mainPort. Commons + // returns once it needs a tool result (or on the final answer). + final reqPtr = DartBridgeProtoUtils.copyBytes(createRequestBytes); + final handleOut = calloc(); + try { + final code = createFn( + reqPtr, + createRequestBytes.length, + nativeCb.nativeFunction, + ffi.nullptr, + handleOut, + ); + if (code != 0) { + mainPort.send(['err', 'create failed: code=$code']); + cleanup(); + return; + } + sessionId = handleOut.value; + mainPort.send(sessionId); + } finally { + calloc.free(reqPtr); + calloc.free(handleOut); } } diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_tts.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_tts.dart index 9e28cac061..be56284af2 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_tts.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_tts.dart @@ -6,6 +6,7 @@ library; import 'dart:async'; import 'dart:ffi'; +import 'dart:isolate'; import 'dart:typed_data'; import 'package:ffi/ffi.dart'; @@ -167,6 +168,37 @@ class DartBridgeTTS { ); } + /// Synthesize speech through the lifecycle-owned generated-proto TTS ABI, + /// running the blocking native call in a short-lived worker isolate + /// (`Isolate.run`) so the calling isolate — usually the Flutter UI isolate — + /// stays responsive for the whole synthesis. Vocoder synthesis of a full + /// utterance is a long synchronous block; on the UI isolate it freezes + /// frames. Mirrors `dart_bridge_stt.dart`'s `transcribeLifecycleProtoAsync` + /// and `dart_bridge_llm.dart`'s `generateProto`: this ABI is lifecycle-owned + /// (no Dart-held handle), so the worker re-resolves the engine via the + /// commons model lifecycle — nothing isolate-bound crosses. + Future synthesizeLifecycleProtoAsync( + TTSSynthesisRequest request, + ) async { + _validateLifecycleRequest(request); + + final override = _synthesizeLifecycleProtoForTesting; + if (override != null) { + return override(request); + } + + if (RacNative.bindings.rac_tts_synthesize_lifecycle_proto == null) { + throw UnsupportedError( + 'rac_tts_synthesize_lifecycle_proto is unavailable', + ); + } + + final requestBytes = request.writeToBuffer(); + final resultBytes = + await Isolate.run(() => _ttsSynthesizeWorker(requestBytes)); + return TTSOutput.fromBuffer(resultBytes); + } + /// Stream TTSStreamEvent chunks via the lifecycle-owned generated-proto ABI. /// /// Mirrors STT's `transcribeStreamLifecycleProto`. Requires commons to have @@ -176,16 +208,20 @@ class DartBridgeTTS { ) { _validateLifecycleRequest(request); - final streamOverride = _synthesizeStreamLifecycleProtoForTesting; - // Defer the FFI lookup when a test seam is installed — accessing + // Test seam: drive the wrapper without the FFI. Accessing // `RacNative.bindings` triggers a `dlopen` of librac_commons, which fails - // in the unit-test harness where no native library is staged. The test - // group `DartBridgeTTS.synthesizeStreamLifecycleProto — real wrapper, fake - // FFI` covers the production wrapper without the FFI. - final fn = streamOverride == null - ? RacNative.bindings.rac_tts_synthesize_stream_lifecycle_proto - : null; - if (streamOverride == null && fn == null) { + // in the unit-test harness where no native library is staged. The seam + // path runs entirely on the calling isolate so the test group + // `DartBridgeTTS.synthesizeStreamLifecycleProto — real wrapper, fake FFI` + // exercises the production drain loop + dispatch closure without spawning + // a worker. + final streamOverride = _synthesizeStreamLifecycleProtoForTesting; + if (streamOverride != null) { + return _synthesizeStreamViaTestSeam(request, streamOverride); + } + + final fn = RacNative.bindings.rac_tts_synthesize_stream_lifecycle_proto; + if (fn == null) { return Stream.error( UnsupportedError( 'rac_tts_synthesize_stream_lifecycle_proto is unavailable', @@ -193,128 +229,145 @@ class DartBridgeTTS { ); } + // Production path: the blocking `rac_tts_synthesize_stream_lifecycle_proto` + // call runs inside a short-lived worker isolate (`Isolate.run`), where a + // worker-owned `NativeCallable.isolateLocal` fires synchronously per chunk + // (commons serializes each event into a `thread_local` scratch buffer that + // is reused on the next emission, so the callback must copy eagerly and + // same-thread) and sends the copy over a `SendPort`. The calling isolate — + // usually the Flutter UI isolate — decodes and emits each event as it + // arrives and is never blocked for the synthesis duration. This is the + // Dart equivalent of `dart_bridge_llm.dart`'s `generateStreamProto`. final controller = StreamController(sync: false); - NativeCallable? callback; + final receivePort = ReceivePort(); var sawTerminalEvent = false; + var tornDown = false; - // Shared dispatch closure — used by both the real NativeCallable.listener - // and the test-injected fake FFI. Centralizing this guarantees the test - // path exercises the same listener-body behavior (closed-controller - // guard, terminal-kind tracking) as production. - void dispatchEvent(TTSStreamEvent event) { - if (controller.isClosed) return; - sawTerminalEvent = sawTerminalEvent || - event.kind == TTSStreamEventKind.TTS_STREAM_EVENT_KIND_COMPLETED || - event.kind == TTSStreamEventKind.TTS_STREAM_EVENT_KIND_ERROR; - controller.add(event); + void teardown() { + if (tornDown) return; + tornDown = true; + receivePort.close(); } - Future run() async { - // Test seam: skip FFI entirely; let the fake drive `dispatchEvent` - // synchronously then return an rc. Same drain + close semantics as - // the real path. - if (streamOverride != null) { + receivePort.listen((Object? message) { + if (message is Uint8List) { + // One serialized TTSStreamEvent, already copied in the worker's + // synchronous callback, delivered over the port in emission order. + if (controller.isClosed) return; try { - final rc = await streamOverride( - request, - dispatchEvent, - () => sawTerminalEvent, - ); - await drainPendingStreamCallbacks(() => sawTerminalEvent); - if (rc != RAC_SUCCESS && !controller.isClosed) { - controller.addError(StateError( - 'rac_tts_synthesize_stream_lifecycle_proto (test fake) failed: ' - '${RacResultCode.getMessage(rc)}', - )); - } - if (!controller.isClosed) { - await controller.close(); + final event = TTSStreamEvent.fromBuffer(message); + final isTerminal = event.kind == + TTSStreamEventKind.TTS_STREAM_EVENT_KIND_COMPLETED || + event.kind == TTSStreamEventKind.TTS_STREAM_EVENT_KIND_ERROR; + sawTerminalEvent = sawTerminalEvent || isTerminal; + controller.add(event); + if (isTerminal) { + unawaited(controller.close()); } } catch (e, st) { - if (!controller.isClosed) { - controller.addError(e, st); - await controller.close(); - } + controller.addError(e, st); + unawaited(controller.close()); + } + } else if (message is int) { + // rc sentinel — always the LAST message (same port as the chunks, so + // FIFO ordering is guaranteed). Early-return rcs (parse / no-model + // errors) produce no terminal event, so surface them. + if (message != RAC_SUCCESS && + !sawTerminalEvent && + !controller.isClosed) { + controller.addError( + StateError( + 'rac_tts_synthesize_stream_lifecycle_proto failed: ' + '${RacResultCode.getMessage(message)}', + ), + ); } - return; + if (!controller.isClosed) { + unawaited(controller.close()); + } + teardown(); + } + }); + + final requestBytes = request.writeToBuffer(); + final sendPort = receivePort.sendPort; + unawaited( + _runTtsStreamWorker(requestBytes, sendPort) + .catchError((Object e, StackTrace st) { + // Worker isolate crashed (RemoteError) before the rc sentinel. + if (!controller.isClosed) { + controller.addError(e, st); + unawaited(controller.close()); + } + teardown(); + return RAC_SUCCESS; + }), + ); + + controller.onCancel = () { + // Best-effort: ask commons to stop lifecycle synthesis so native CPU + // isn't burned for a Dart subscriber that has already gone away. + // RunAnywhereTTS.stopSynthesis() routes through the same ABI; mirror its + // semantics here. The worker's blocking call returns shortly after, emits + // a terminal event (dropped — the controller is closing) and the rc + // sentinel closes the port. Errors are swallowed so cancellation stays + // best-effort. + try { + stopLifecycleProto(); + } catch (e) { + _logger.debug('stopLifecycleProto on stream cancel failed: $e'); } + teardown(); + }; + + return controller.stream; + } + + /// Test-only streaming path: drives the production drain loop + dispatch + /// closure with a Dart-side fake instead of the FFI. Runs entirely on the + /// calling isolate (the unit-test harness has no native library to `dlopen` + /// and no worker to spawn). + Stream _synthesizeStreamViaTestSeam( + TTSSynthesisRequest request, + TTSStreamFakeFFI streamOverride, + ) { + final controller = StreamController(sync: false); + var sawTerminalEvent = false; - final bytes = request.writeToBuffer(); - final requestPtr = DartBridgeProtoUtils.copyBytes(bytes); + void dispatchEvent(TTSStreamEvent event) { + if (controller.isClosed) return; + sawTerminalEvent = sawTerminalEvent || + event.kind == TTSStreamEventKind.TTS_STREAM_EVENT_KIND_COMPLETED || + event.kind == TTSStreamEventKind.TTS_STREAM_EVENT_KIND_ERROR; + controller.add(event); + } + Future run() async { try { - // Main-isolate single-call ABI (NOTE: DartBridgeLLM.generateStreamProto - // has since moved its blocking call into a worker isolate — that is - // the template for converting this path; TTS still blocks the calling - // isolate for the duration of synthesis): - // use `isolateLocal` (not `.listener`) so the callback runs - // synchronously on the same thread that invoked - // `rac_tts_synthesize_stream_lifecycle_proto`. The commons producer - // (`emit_event` / `chunk_bridge` in rac_nonllm_lifecycle_proto_abi.cpp) - // serializes each event into a stack-local `std::vector` and - // calls the callback with that buffer's pointer inline. With - // `.listener` the callback is queued onto the isolate's event loop and - // runs ASYNCHRONOUSLY — by then the producing call has returned and the - // stack buffer is gone, so `bytesPtr.asTypedList` reads freed memory - // (use-after-free → malformed/empty TTSStreamEvent bytes). - // `isolateLocal` is safe because the ABI runs synchronously on the - // Dart isolate that created the callback, so it always fires on that - // isolate and cannot be re-entered after `fn()` returns. - callback = NativeCallable.isolateLocal(( - Pointer bytesPtr, - int bytesLen, - Pointer _, - ) { - if (bytesPtr == nullptr || bytesLen <= 0) return; - try { - final copy = Uint8List.fromList(bytesPtr.asTypedList(bytesLen)); - dispatchEvent(TTSStreamEvent.fromBuffer(copy)); - } catch (e, st) { - controller.addError(e, st); - unawaited(controller.close()); - } - }); - final rc = fn!( - requestPtr, - bytes.length, - callback!.nativeFunction, - nullptr, + final rc = await streamOverride( + request, + dispatchEvent, + () => sawTerminalEvent, ); + await drainPendingStreamCallbacks(() => sawTerminalEvent); if (rc != RAC_SUCCESS && !controller.isClosed) { controller.addError(StateError( - 'rac_tts_synthesize_stream_lifecycle_proto failed: ' + 'rac_tts_synthesize_stream_lifecycle_proto (test fake) failed: ' '${RacResultCode.getMessage(rc)}', )); } if (!controller.isClosed) { await controller.close(); } - } finally { - calloc.free(requestPtr); - // Drain in-flight TTS chunk dispatches before - // closing the NativeCallable. `rac_tts_synthesize_stream_lifecycle_proto` - // may post the terminal COMPLETED/ERROR callback from a worker - // thread that copies the user_data slot under commons' internal - // mutex and releases it BEFORE invoking the callback (see - // `rac/features/tts/rac_tts_stream.h` warning). Without - // `rac_tts_proto_quiesce()` the C side can invoke the trampoline - // backed by NativeCallable user_data after `callback.close()` — - // UAF on the proto scratch buffer. Best-effort — if the commons - // binary predates the export (or, in unit-test harnesses, the - // native library is not staged) the call is skipped. - _quiesceBestEffort(); - callback?.close(); - callback = null; + } catch (e, st) { + if (!controller.isClosed) { + controller.addError(e, st); + await controller.close(); + } } } controller.onCancel = () { - // Best-effort: ask commons to stop lifecycle synthesis so native CPU - // isn't burned for a Dart subscriber that has already gone away. - // RunAnywhereTTS.stopSynthesis() routes through the same ABI; mirror - // its semantics here so cancelling the public stream subscription - // also stops the underlying lifecycle work. Errors are swallowed so - // cancellation remains best-effort. try { final stopOverride = _stopLifecycleProtoForTesting; if (stopOverride != null) { @@ -325,10 +378,6 @@ class DartBridgeTTS { } catch (e) { _logger.debug('stopLifecycleProto on stream cancel failed: $e'); } - // Same ordering as the run() teardown — quiesce first. - _quiesceBestEffort(); - callback?.close(); - callback = null; }; unawaited(run()); @@ -567,3 +616,98 @@ typedef TTSStreamFakeFFI = Future Function( void Function(TTSStreamEvent) dispatch, bool Function() terminalObserved, ); + +// MARK: - Worker-isolate entry points +// +// Top-level so `Isolate.run` closures capture only sendable values +// (Uint8List / SendPort). `RacNative.bindings` is a per-isolate static — the +// worker re-resolves the dylib symbols on first access (idempotent +// `PlatformLoader.loadCommons()`, same convention as the LLM/STT/VLM workers). + +/// Blocking body of [DartBridgeTTS.synthesizeLifecycleProtoAsync]: a plain +/// request→response proto call with no callbacks. Returns the serialized +/// TTSOutput so the main isolate owns the decode. +Uint8List _ttsSynthesizeWorker(Uint8List requestBytes) { + final bindings = RacNative.bindings; + final fn = bindings.rac_tts_synthesize_lifecycle_proto; + if (fn == null) { + throw UnsupportedError('rac_tts_synthesize_lifecycle_proto is unavailable'); + } + + final requestPtr = DartBridgeProtoUtils.copyBytes(requestBytes); + final out = calloc(); + try { + bindings.rac_proto_buffer_init(out); + final code = fn(requestPtr, requestBytes.length, out); + DartBridgeProtoUtils.ensureSuccess( + out, + code, + 'rac_tts_synthesize_lifecycle_proto', + ); + if (out.ref.data == nullptr || out.ref.size == 0) { + return Uint8List(0); + } + return Uint8List.fromList(out.ref.data.asTypedList(out.ref.size)); + } finally { + bindings.rac_proto_buffer_free(out); + calloc.free(out); + calloc.free(requestPtr); + } +} + +/// Runs [_ttsStreamWorker] in a worker isolate. Hoisted to top level so the +/// `Isolate.run` closure captures ONLY its two sendable parameters +/// (`Uint8List` + `SendPort`) — inlined, the closure would capture the +/// unsendable `ReceivePort`/`StreamController` and fail the isolate spawn. +Future _runTtsStreamWorker(Uint8List requestBytes, SendPort port) => + Isolate.run(() => _ttsStreamWorker(requestBytes, port)); + +/// Blocking body of [DartBridgeTTS.synthesizeStreamLifecycleProto]. Runs the +/// single-call streaming ABI on the worker isolate; the worker-owned +/// `isolateLocal` callback fires synchronously per chunk (commons passes a +/// pointer into a `thread_local` scratch buffer, so the callback must copy +/// eagerly and same-thread), copies the bytes, and forwards the copy to the +/// main isolate. The rc is sent LAST on the same port so it is FIFO-ordered +/// after every chunk. +int _ttsStreamWorker(Uint8List requestBytes, SendPort port) { + final bindings = RacNative.bindings; + final fn = bindings.rac_tts_synthesize_stream_lifecycle_proto; + if (fn == null) { + throw UnsupportedError( + 'rac_tts_synthesize_stream_lifecycle_proto is unavailable', + ); + } + + final requestPtr = DartBridgeProtoUtils.copyBytes(requestBytes); + NativeCallable? callback; + try { + callback = NativeCallable.isolateLocal( + (Pointer bytesPtr, int bytesLen, Pointer _) { + if (bytesPtr == nullptr || bytesLen <= 0) return; + // Copy INSIDE the synchronous callback — commons reuses the scratch + // buffer the moment we return. The copy is what crosses isolates. + port.send(Uint8List.fromList(bytesPtr.asTypedList(bytesLen))); + }, + ); + + final rc = fn( + requestPtr, + requestBytes.length, + callback.nativeFunction, + nullptr, + ); + port.send(rc); + return rc; + } finally { + // Quiesce in-flight TTS chunk dispatches before closing the NativeCallable. + // `rac_tts_synthesize_stream_lifecycle_proto` may post the terminal + // callback from a worker thread that copies the user_data slot under + // commons' internal mutex and releases it BEFORE invoking the callback + // (see `rac/features/tts/rac_tts_stream.h`). Without `rac_tts_proto_quiesce` + // the C side can invoke the trampoline after `callback.close()` — UAF on + // the proto scratch buffer. Best-effort: skipped if the export is absent. + bindings.rac_tts_proto_quiesce?.call(); + callback?.close(); + calloc.free(requestPtr); + } +} diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_vlm.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_vlm.dart index db27904b71..85557231cb 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_vlm.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_vlm.dart @@ -7,6 +7,7 @@ library; import 'dart:async'; import 'dart:ffi' as ffi; +import 'dart:isolate'; import 'dart:typed_data'; import 'package:ffi/ffi.dart'; @@ -32,11 +33,18 @@ typedef _RacVlmGenerateProtoDart = int Function( ); /// Stream-event callback signature used with `NativeCallable.isolateLocal`. -/// The underlying C type returns `rac_bool_t`, but the callback fires -/// synchronously on the Dart isolate that invoked `rac_vlm_stream_proto`, so -/// no value is returned — matching the canonical Flutter stream bridge (LLM, -/// voice-agent). -typedef _RacVlmStreamEventProtoCallbackNative = ffi.Void Function( +/// +/// Returns `rac_bool_t` (RAC_TRUE = keep streaming, RAC_FALSE = stop). The VLM +/// engine consults this return value on EVERY token and breaks its decode loop +/// on RAC_FALSE (rac_vlm_llamacpp.cpp — "Callback requested stop"). This is +/// UNLIKE the LLM stream callback, whose C type is `void` (rac_llm_stream.h:54) +/// and whose engine ignores any return — so the LLM bridge can use a `Void` +/// trampoline, but VLM CANNOT. A `Void` trampoline leaves a garbage/zero value +/// in the return register; the engine reads it as RAC_FALSE and truncates +/// generation after the first token. Declare the real rac_bool_t (int32) return +/// and emit RAC_TRUE so the engine keeps decoding. Cancellation flows through +/// the lifecycle cancel flag the engine also checks each iteration, not here. +typedef _RacVlmStreamEventProtoCallbackNative = ffi.Int32 Function( ffi.Pointer, ffi.Size, ffi.Pointer, @@ -86,96 +94,68 @@ class DartBridgeVLM { VLMGenerationRequest request, ) { final controller = StreamController(sync: false); - ffi.NativeCallable<_RacVlmStreamEventProtoCallbackNative>? callback; - + final receivePort = ReceivePort(); var sawTerminalEvent = false; + var tornDown = false; + + void teardown() { + if (tornDown) return; + tornDown = true; + receivePort.close(); + } - Future run() async { - final bytes = request.writeToBuffer(); - final requestPtr = DartBridgeProtoUtils.copyBytes(bytes); - - try { - // Mirrors dart_bridge_llm.dart: use `isolateLocal` - // (not `.listener`) so the callback fires SYNCHRONOUSLY on the Dart - // isolate that invokes `rac_vlm_stream_proto`. Commons serializes each - // VLMStreamEvent into a thread-local scratch vector and calls the - // callback with `scratch.data()` inline; under `.listener` the callback - // is queued onto the event loop and runs after a later token has - // already resized/overwritten that scratch slot, so the captured - // pointer decodes partially-overwritten bytes (use-after-free). The - // engine vtable iterates tokens on this same calling thread, so the - // callback always fires on the isolate that created it — the exact - // precondition `isolateLocal` requires. - callback = ffi.NativeCallable< - _RacVlmStreamEventProtoCallbackNative>.isolateLocal( - ( - ffi.Pointer bytesPtr, - int bytesLen, - ffi.Pointer _, - ) { - if (controller.isClosed || - bytesPtr == ffi.nullptr || - bytesLen <= 0) { - return; - } - try { - final copy = Uint8List.fromList(bytesPtr.asTypedList(bytesLen)); - final event = VLMStreamEvent.fromBuffer(copy); - sawTerminalEvent = sawTerminalEvent || event.isFinal; - controller.add(event); - if (event.isFinal) { - unawaited(controller.close()); - } - } catch (e, st) { - controller.addError(e, st); - unawaited(controller.close()); - } - }, - ); - - final fn = _lookupStreamProto(); - final code = fn( - requestPtr, - bytes.length, - callback!.nativeFunction, - ffi.nullptr, - ); - if (code != RacResultCode.success && !controller.isClosed) { + receivePort.listen((Object? message) { + if (message is Uint8List) { + // One serialized VLMStreamEvent, already copied in the worker's + // synchronous callback, delivered over the port in emission order. + if (controller.isClosed) return; + try { + final event = VLMStreamEvent.fromBuffer(message); + sawTerminalEvent = sawTerminalEvent || event.isFinal; + controller.add(event); + if (event.isFinal) { + unawaited(controller.close()); + } + } catch (e, st) { + controller.addError(e, st); + unawaited(controller.close()); + } + } else if (message is int) { + // rc sentinel — always the LAST message on this port (FIFO after every + // event). Early-return rcs (parse / no-model errors) produce no + // terminal event, so surface them. + if (message != RacResultCode.success && + !sawTerminalEvent && + !controller.isClosed) { controller.addError(StateError( - 'rac_vlm_stream_proto failed: ${RacResultCode.getMessage(code)}', + 'rac_vlm_stream_proto failed: ${RacResultCode.getMessage(message)}', )); - await controller.close(); - } else if (!sawTerminalEvent && !controller.isClosed) { - await controller.close(); } - } catch (e, st) { if (!controller.isClosed) { - controller.addError(e, st); - await controller.close(); + unawaited(controller.close()); } - } finally { - calloc.free(requestPtr); - // Teardown: with `isolateLocal` every event - // has already drained by the time `fn` returns, but - // `rac_vlm_proto_quiesce()` is still invoked as a defensive barrier in - // case a future commons revision posts a late callback from a worker - // thread (see `rac/features/vlm/rac_vlm_service.h`) — closing the - // NativeCallable while that worker is mid-dispatch would be a UAF. - RacNative.bindings.rac_vlm_proto_quiesce?.call(); - callback?.close(); - callback = null; + teardown(); } - } + }); - controller.onListen = () { - unawaited(run()); - }; - controller.onCancel = () { - cancel(); - RacNative.bindings.rac_vlm_proto_quiesce?.call(); - callback?.close(); - callback = null; - }; + final requestBytes = request.writeToBuffer(); + final sendPort = receivePort.sendPort; + unawaited( + _runVlmStreamWorker(requestBytes, sendPort) + .catchError((Object e, StackTrace st) { + // Worker isolate crashed (RemoteError) before the rc sentinel. + if (!controller.isClosed) { + controller.addError(e, st); + unawaited(controller.close()); + } + teardown(); + return RacResultCode.success; + }), + ); + + // Cancel sets the lifecycle cancel flag; the worker's blocking call returns + // shortly after and the rc sentinel closes the port. + controller.onCancel = cancel; return controller.stream; } @@ -250,3 +230,76 @@ class DartBridgeVLM { } } } + +// MARK: - Worker-isolate entry points +// +// VLM inference (image encode + prefill + decode) is a long synchronous block. +// Run on the calling isolate — the Flutter UI isolate — it freezes the UI for +// the whole generation (unlike token-by-token LLM, a single VLM frame is one +// uninterrupted FFI call). The blocking `rac_vlm_stream_proto` therefore runs +// in a short-lived worker isolate (`Isolate.run`), exactly like +// `dart_bridge_llm.dart`'s streaming path; the worker-owned `isolateLocal` +// callback fires synchronously per event, copies the proto bytes eagerly +// (commons reuses a `thread_local` scratch buffer on the next emission) and +// forwards the copy over a `SendPort`. Now that telemetry HTTP is drained via a +// cross-isolate-safe wakeup poll-queue, the VLM completion event published from +// the worker no longer trips the previous cross-isolate telemetry SIGABRT. +// +// Top-level so the `Isolate.run` closure captures ONLY its two sendable +// parameters (`Uint8List` + `SendPort`) — never the method's unsendable +// `ReceivePort`/`StreamController`. `RacNative.bindings` and +// `PlatformLoader.loadCommons()` are per-isolate and re-resolve the dylib +// symbols on first access in the worker (idempotent). + +/// Runs [_vlmStreamWorker] in a worker isolate. Hoisted to top level so the +/// `Isolate.run` closure captures only its sendable parameters. +Future _runVlmStreamWorker(Uint8List requestBytes, SendPort port) => + Isolate.run(() => _vlmStreamWorker(requestBytes, port)); + +/// Blocking body of [DartBridgeVLM.processImageStreamProto]. Runs the +/// single-call streaming ABI on the worker isolate; the worker-owned +/// `isolateLocal` callback fires synchronously per event (commons requires a +/// synchronous same-thread callback because it passes a pointer into a +/// `thread_local` scratch buffer), copies the bytes eagerly, and forwards the +/// copy to the main isolate. The rc is sent LAST on the same port so it is +/// FIFO-ordered after every event. +int _vlmStreamWorker(Uint8List requestBytes, SendPort port) { + final fn = DartBridgeVLM.shared._lookupStreamProto(); + final requestPtr = DartBridgeProtoUtils.copyBytes(requestBytes); + ffi.NativeCallable<_RacVlmStreamEventProtoCallbackNative>? callback; + try { + callback = + ffi.NativeCallable<_RacVlmStreamEventProtoCallbackNative>.isolateLocal( + ( + ffi.Pointer bytesPtr, + int bytesLen, + ffi.Pointer _, + ) { + if (bytesPtr == ffi.nullptr || bytesLen <= 0) return RAC_TRUE; + // Copy INSIDE the synchronous callback — commons reuses the scratch + // buffer the moment we return. The copy is what crosses isolates. + port.send(Uint8List.fromList(bytesPtr.asTypedList(bytesLen))); + // RAC_TRUE = keep decoding. The VLM engine breaks its loop on RAC_FALSE, + // so a void/zero return truncates generation after the first token. + return RAC_TRUE; + }, + // Value returned to C if the Dart callback throws: stop the stream. + exceptionalReturn: RAC_FALSE, + ); + + final rc = fn( + requestPtr, + requestBytes.length, + callback.nativeFunction, + ffi.nullptr, + ); + port.send(rc); + return rc; + } finally { + // Defensive quiesce, then close the callable on its owning isolate AFTER + // the blocking call has returned — no emission can occur past this point. + RacNative.bindings.rac_vlm_proto_quiesce?.call(); + callback?.close(); + calloc.free(requestPtr); + } +} diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_voice_agent.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_voice_agent.dart index 778c451c10..34819a5056 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_voice_agent.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/native/dart_bridge_voice_agent.dart @@ -16,6 +16,7 @@ library; import 'dart:async'; import 'dart:ffi'; +import 'dart:isolate'; import 'dart:typed_data'; import 'package:ffi/ffi.dart'; @@ -233,95 +234,93 @@ class DartBridgeVoiceAgent { } } - /// Streaming turn processing. Invokes - /// `rac_voice_agent_process_turn_proto` and pipes decoded `VoiceEvent` - /// bytes onto the returned broadcast stream. + /// Streaming turn processing. Invokes `rac_voice_agent_process_turn_proto` + /// on a short-lived WORKER isolate and pipes each decoded `VoiceEvent` onto + /// the returned stream as it is emitted. + /// + /// Why a worker isolate: commons runs the ENTIRE turn (STT → LLM → TTS) + /// synchronously on the calling thread under `handle->mutex`, invoking the + /// event callback inline per VoiceEvent. Run on the MAIN isolate that blocks + /// the UI for the whole turn AND — because Dart can't pump the stream's + /// listener until the blocking FFI call returns — every event (including the + /// early `userSaid` transcript) is only delivered AFTER the LLM+TTS finish, + /// so the transcript appears seconds late. Running on a worker isolate (the + /// canonical pattern from `dart_bridge_llm.generateStreamProto`) lets the + /// worker-owned `isolateLocal` callback fire synchronously per event, copy + /// the bytes, and forward them over a `SendPort`; the main isolate decodes + /// and emits each as it arrives — transcript shows the instant STT finishes, + /// then LLM tokens stream, then audio. Mirrors Kotlin's `Dispatchers.IO` + /// placement of the same single-call ABI. Stream processTurnStream( voice_agent_pb.VoiceAgentTurnRequest request, ) { - final controller = StreamController(); - NativeCallable? nativeCb; + if (RacNative.bindings.rac_voice_agent_process_turn_proto == null) { + return Stream.error(UnsupportedError( + 'rac_voice_agent_process_turn_proto is unavailable')); + } + + final controller = StreamController(sync: false); + final receivePort = ReceivePort(); + var sawError = false; + var tornDown = false; + + void teardown() { + if (tornDown) return; + tornDown = true; + receivePort.close(); + } + + receivePort.listen((Object? message) { + if (message is Uint8List) { + // One serialized VoiceEvent, copied in the worker's synchronous + // callback, delivered over the port in emission order. + if (controller.isClosed) return; + try { + controller.add(voice_events_pb.VoiceEvent.fromBuffer(message)); + } catch (e, st) { + sawError = true; + controller.addError(e, st); + } + } else if (message is int) { + // rc sentinel — always LAST (same port as events ⇒ FIFO). + if (message != 0 && !sawError && !controller.isClosed) { + controller.addError(StateError( + 'rac_voice_agent_process_turn_proto failed: code=$message')); + } + if (!controller.isClosed) { + unawaited(controller.close()); + } + teardown(); + } + }); controller ..onListen = () async { try { final handle = await getHandle(); - final fn = RacNative.bindings.rac_voice_agent_process_turn_proto; - if (fn == null) { - controller.addError(UnsupportedError( - 'rac_voice_agent_process_turn_proto is unavailable')); + final requestBytes = request.writeToBuffer(); + unawaited( + _runVoiceTurnWorker( + handle.address, requestBytes, receivePort.sendPort) + .catchError((Object e, StackTrace st) { + // Worker isolate crashed before the rc sentinel. + if (!controller.isClosed) { + controller.addError(e, st); + unawaited(controller.close()); + } + teardown(); + return 0; + }), + ); + } catch (e, st) { + if (!controller.isClosed) { + controller.addError(e, st); unawaited(controller.close()); - return; - } - // Use `isolateLocal` (not `.listener`) so the - // callback fires SYNCHRONOUSLY on the same Dart isolate that - // invokes `rac_voice_agent_process_turn_proto`. The commons - // implementation in `voice_agent_d7_abi.cpp` runs the entire turn - // (STT → LLM → TTS) on the calling thread under `handle->mutex` - // and invokes `event_callback` for each VoiceEvent inline. With - // `.listener` mode the callbacks are queued onto the isolate's - // event loop and the `finally` below closes the controller before - // any of them drain — every event is silently dropped at - // `controller.add(...)` on a closed controller. `isolateLocal` - // ensures every emission lands on the still-open controller - // before `fn(...)` returns. This mirrors the canonical pattern in - // `dart_bridge_llm.dart` (`_generateStreamProto`). - nativeCb = NativeCallable< - RacVoiceAgentProtoEventCallbackNative>.isolateLocal(( - Pointer bytesPtr, - int bytesLen, - Pointer _, - ) { - if (controller.isClosed || bytesLen <= 0 || bytesPtr == nullptr) { - return; - } - final copy = Uint8List.fromList(bytesPtr.asTypedList(bytesLen)); - try { - controller.add(voice_events_pb.VoiceEvent.fromBuffer(copy)); - } catch (e, st) { - controller.addError(e, st); - } - }); - final bytes = request.writeToBuffer(); - final reqPtr = DartBridgeProtoUtils.copyBytes(bytes); - try { - final code = fn( - handle, - reqPtr, - bytes.length, - nativeCb!.nativeFunction, - nullptr, - ); - if (code != 0) { - controller.addError( - StateError( - 'rac_voice_agent_process_turn_proto failed: code=$code', - ), - ); - } - } finally { - calloc.free(reqPtr); } - } catch (e, st) { - controller.addError(e, st); - } finally { - // Teardown: with `isolateLocal` - // all events have already drained by the time `fn` returns, but - // `rac_voice_agent_proto_quiesce()` is still invoked as a defensive - // barrier in case a future commons revision posts late events from - // a worker thread. - RacNative.bindings.rac_voice_agent_proto_quiesce?.call(); - nativeCb?.close(); - nativeCb = null; - unawaited(controller.close()); + teardown(); } } - ..onCancel = () { - // Same ordering as the run() teardown above. - RacNative.bindings.rac_voice_agent_proto_quiesce?.call(); - nativeCb?.close(); - nativeCb = null; - }; + ..onCancel = teardown; return controller.stream; } @@ -484,3 +483,65 @@ void _safeRacFree(Pointer ptr) { // rac_free may not exist in some native builds } } + +// MARK: - Voice-turn worker isolate +// +// Top-level so the `Isolate.run` closure captures ONLY sendable values +// (int handle address + Uint8List + SendPort). Mirrors +// `dart_bridge_llm._runLlmStreamWorker` / `_llmStreamWorker`. The voice turn +// (STT → LLM → TTS) is inference over already-loaded models — the same class +// of work STT/VLM/RAG/embeddings already run on worker isolates here — and +// its only Dart-bound callbacks (SDK events, logging, telemetry HTTP wakeup) +// are `.listener` (cross-isolate safe). The `Pointer.fromFunction` +// platform-adapter trampolines that SIGABRT under `Isolate.run` (model LOAD) +// are not invoked during a turn; if a future commons change adds one to this +// path, make that callback `.listener` rather than moving the turn back to +// the main isolate. + +/// Runs [_voiceTurnWorker] in a worker isolate. Hoisted to top level so the +/// closure captures only its three sendable parameters. +Future _runVoiceTurnWorker( + int handleAddress, Uint8List requestBytes, SendPort port) => + Isolate.run(() => _voiceTurnWorker(handleAddress, requestBytes, port)); + +/// Blocking body of [DartBridgeVoiceAgent.processTurnStream]. The worker-owned +/// `isolateLocal` callback fires synchronously per VoiceEvent (commons passes +/// a pointer into a reused scratch buffer, so the bytes are copied INSIDE the +/// callback before they cross the isolate). The rc is sent LAST on the same +/// port, FIFO-ordered after every event. +int _voiceTurnWorker(int handleAddress, Uint8List requestBytes, SendPort port) { + final bindings = RacNative.bindings; + final fn = bindings.rac_voice_agent_process_turn_proto; + if (fn == null) { + throw UnsupportedError('rac_voice_agent_process_turn_proto is unavailable'); + } + + final handle = Pointer.fromAddress(handleAddress); + final requestPtr = DartBridgeProtoUtils.copyBytes(requestBytes); + NativeCallable? callback; + try { + callback = + NativeCallable.isolateLocal( + (Pointer bytesPtr, int bytesLen, Pointer _) { + if (bytesPtr == nullptr || bytesLen <= 0) return; + // Copy inside the synchronous callback — commons reuses the scratch + // buffer the moment we return. The copy is what crosses isolates. + port.send(Uint8List.fromList(bytesPtr.asTypedList(bytesLen))); + }, + ); + + final rc = fn( + handle, + requestPtr, + requestBytes.length, + callback.nativeFunction, + nullptr, + ); + port.send(rc); + return rc; + } finally { + bindings.rac_voice_agent_proto_quiesce?.call(); + callback?.close(); + calloc.free(requestPtr); + } +} diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_embeddings.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_embeddings.dart index ae33ef208c..d69bdb68f8 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_embeddings.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_embeddings.dart @@ -80,7 +80,7 @@ class RunAnywhereEmbeddings { ); } lifecycleRequest.modelId = modelId; - return DartBridgeEmbeddings.shared.embedBatch(lifecycleRequest); + return DartBridgeEmbeddings.shared.embedBatchAsync(lifecycleRequest); } Future unload() async { diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_llm.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_llm.dart index 9f088db028..6dca1b57e6 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_llm.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_llm.dart @@ -336,9 +336,11 @@ class RunAnywhereLLM { await controller.close(); } + // Start the worker only once a listener attaches (canonical lazy pattern), + // so generation can't begin — and tokens can't be produced — before the + // subscriber is ready. Mirrors the VLM bridge's onListen deferral. + controller.onListen = () => unawaited(run()); controller.onCancel = _cancelProto; - - unawaited(run()); return controller.stream; } diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_rag.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_rag.dart index 3d81e4a416..d9f749e5b5 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_rag.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_rag.dart @@ -112,7 +112,7 @@ class RunAnywhereRAG { } try { - return DartBridgeRAG.shared.ingestDocument(document); + return await DartBridgeRAG.shared.ingestDocumentAsync(document); } catch (e) { throw SDKException.invalidState('RAG ingestion failed: $e'); } @@ -140,7 +140,7 @@ class RunAnywhereRAG { try { for (final document in documents) { - DartBridgeRAG.shared.ingestDocument(document); + await DartBridgeRAG.shared.ingestDocumentAsync(document); } } catch (e) { throw SDKException.invalidState('RAG batch ingestion failed: $e'); diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_stt.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_stt.dart index 888d2f43c3..a69e186bce 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_stt.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_stt.dart @@ -300,7 +300,7 @@ class RunAnywhereSTT { metadata: {'model_id': modelId}.entries, ); - return DartBridgeSTT.shared.transcribeLifecycleProto(request); + return DartBridgeSTT.shared.transcribeLifecycleProtoAsync(request); } Future _requireLoadedModelId() async { diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_tools.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_tools.dart index 530636f450..67984a6e2c 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_tools.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_tools.dart @@ -15,7 +15,6 @@ import 'dart:convert'; import 'dart:ffi' show nullptr; import 'package:ffi/ffi.dart' show Utf8Pointer; -import 'package:fixnum/fixnum.dart' show Int64; import 'package:protobuf/protobuf.dart' show GeneratedMessageGenericExtensions; import 'package:runanywhere/core/native/rac_native.dart' show RacNative; @@ -31,7 +30,6 @@ import 'package:runanywhere/generated/tool_calling.pb.dart' ToolCallingSessionCreateRequest, ToolCallingSessionEvent, ToolCallingSessionEvent_Kind, - ToolCallingSessionStepWithResultRequest, ToolChoiceMode, ToolDefinition, ToolResult, @@ -263,8 +261,12 @@ class RunAnywhereTools { final session = DartBridgeToolCalling.shared.createSession(request); // Publish the active session handle so consumers can // call `RunAnywhereTools.shared.cancelGeneration()` to interrupt the - // in-flight loop (mirrors RunAnywhereLLM.cancelGeneration). - _activeSessionHandle = session.sessionHandle; + // in-flight loop (mirrors RunAnywhereLLM.cancelGeneration). The native + // session id resolves a turn after create starts (it now runs off the UI + // isolate), so publish it once known. + unawaited(session.sessionId + .then((id) => _activeSessionHandle = id) + .catchError((Object _) => _activeSessionHandle)); final collectedCalls = []; final collectedResults = []; final completer = Completer(); @@ -291,26 +293,23 @@ class RunAnywhereTools { await sub.cancel(); return; } + // Forward the result to the session worker, which fills in its own + // session id and runs the next turn off the UI isolate; the turn's + // events arrive back on this stream. try { final result = await execute(call); collectedResults.add(result); - DartBridgeToolCalling.shared.sessionStepWithResult( - ToolCallingSessionStepWithResultRequest( - sessionHandle: _toFixnum(session.sessionHandle), - toolCallId: call.id, - resultJson: result.resultJson, - error: result.error, - ), + session.stepWithResult( + toolCallId: call.id, + resultJson: result.resultJson, + error: result.error, ); } catch (e) { _logger.error('Tool executor threw: $e'); - DartBridgeToolCalling.shared.sessionStepWithResult( - ToolCallingSessionStepWithResultRequest( - sessionHandle: _toFixnum(session.sessionHandle), - toolCallId: call.id, - resultJson: '', - error: e.toString(), - ), + session.stepWithResult( + toolCallId: call.id, + resultJson: '', + error: e.toString(), ); } break; @@ -347,7 +346,7 @@ class RunAnywhereTools { // Clear the published handle BEFORE close — once close // returns, any pending cancelGeneration() call would race a freshly // started session. - if (_activeSessionHandle == session.sessionHandle) { + if (_activeSessionHandle == session.resolvedSessionId) { _activeSessionHandle = 0; } await session.close(); @@ -404,8 +403,6 @@ class RunAnywhereTools { ); } -Int64 _toFixnum(int value) => Int64(value); - /// Run-loop knobs derived from [ToolCallingOptions]. Mirrors Swift's /// `RAToolCallingOptions` extension (ToolCallingTypes.swift:157-171). extension ToolCallingOptionsRunLoop on ToolCallingOptions { diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_tts.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_tts.dart index 642c718794..ad9b99e402 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_tts.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_tts.dart @@ -160,7 +160,7 @@ class RunAnywhereTTS { options: opts, metadata: {'voice_id': voiceId}.entries, ); - return DartBridgeTTS.shared.synthesizeLifecycleProto(request); + return DartBridgeTTS.shared.synthesizeLifecycleProtoAsync(request); } /// Stream generated [TTSOutput] chunks as they are produced. diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_voice.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_voice.dart index 7ba7d97b98..0e10df0b32 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_voice.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/capabilities/runanywhere_voice.dart @@ -13,7 +13,7 @@ import 'dart:typed_data'; -import 'package:runanywhere/adapters/voice_agent_stream_adapter.dart'; +import 'package:runanywhere/features/voice_agent/services/voice_agent_mic_driver.dart'; import 'package:runanywhere/foundation/errors/sdk_exception.dart'; import 'package:runanywhere/foundation/logging/sdk_logger.dart'; import 'package:runanywhere/generated/errors.pbenum.dart' show ErrorCode; @@ -239,20 +239,27 @@ class RunAnywhereVoice { /// Subscribe to canonical voice-agent events. /// - /// Symmetric with `RunAnywhere.llm.generateStream(...)`: - /// the capability owns adapter construction so callers never touch - /// `VoiceAgentStreamAdapter` directly. The handle is fetched from - /// the internal `DartBridgeVoiceAgent` singleton — call - /// [initializeWithLoadedModels] first. + /// The C ABI owns NO microphone (rac_voice_agent.h audio-ingress contract): + /// subscribing to the handle callback alone is dead air. While this stream is + /// collected, a [VoiceAgentMicDriver] captures mic audio, segments utterances + /// by energy endpointing, and drives per-utterance turns through + /// `rac_voice_agent_process_turn_proto`; their VoiceEvents are forwarded here + /// and the synthesized reply is played back. Mirrors Kotlin + /// `RunAnywhere.streamVoiceAgent()` (mic driver + event fan-out). /// - /// Cancellation propagates: cancelling the returned stream's - /// subscription tears down the underlying C-side proto callback. + /// Call [initializeWithLoadedModels] first. Cancelling the subscription stops + /// capture/playback and tears the turn pipeline down. /// - /// Advanced callers needing multiple fan-out subscriptions or a - /// custom handle can still construct `VoiceAgentStreamAdapter` - /// directly (exported from `package:runanywhere/runanywhere.dart`). + /// Advanced callers needing the raw handle-callback fan-out (no mic ingress) + /// can still construct `VoiceAgentStreamAdapter(handle)` directly (exported + /// from `package:runanywhere/runanywhere.dart`). Stream eventStream() async* { - final handle = await DartBridge.voiceAgent.getHandle(); - yield* VoiceAgentStreamAdapter(handle).stream(); + final driver = VoiceAgentMicDriver(); + await driver.start(); + try { + yield* driver.events; + } finally { + await driver.stop(); + } } } diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/extensions/runanywhere_storage.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/extensions/runanywhere_storage.dart index 3d03b4d6c3..cc7c7ba972 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/extensions/runanywhere_storage.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/extensions/runanywhere_storage.dart @@ -70,8 +70,12 @@ class RunAnywhereStorage { } if (memoryRequirement != null) { request.memoryRequiredBytes = Int64(memoryRequirement); - request.downloadSizeBytes = Int64(memoryRequirement); } + // Intentionally NOT setting downloadSizeBytes from memoryRequirement: that + // value gates the post-finalize download-size check, and the RAM estimate + // is usually a round placeholder (e.g. 500 MB for a real 397 MB file), + // which leaves is_downloaded=false forever. Leaving it unset lets commons + // validate against the actual transfer — matches Kotlin's catalog. final model = await DartBridgeModelRegistry.instance.registerModelFromUrl( request, @@ -203,8 +207,10 @@ class RunAnywhereStorage { ); if (memoryRequirement != null) { request.memoryRequiredBytes = Int64(memoryRequirement); - request.downloadSizeBytes = Int64(memoryRequirement); } + // See registerModel: downloadSizeBytes is intentionally left unset so the + // post-finalize size guard validates against the actual transfer rather + // than the RAM-estimate placeholder. final resolvedContextLength = contextLength ?? (modality.requiresContextLength ? 2048 : null); if (resolvedContextLength != null) { diff --git a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/runanywhere.dart b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/runanywhere.dart index 7814dcf337..871cdf7a64 100644 --- a/sdk/runanywhere-flutter/packages/runanywhere/lib/public/runanywhere.dart +++ b/sdk/runanywhere-flutter/packages/runanywhere/lib/public/runanywhere.dart @@ -484,6 +484,16 @@ abstract final class RunAnywhere { final phase1DeviceId = await DartBridgeDevice.instance.getDeviceId(); + // Attach the telemetry sink BEFORE DartBridge.initialize below: commons + // emits INITIALIZATION_STAGE_STARTED/COMPLETED ("system" modality) during + // Phase-1 core init, so the sink must already be registered or those + // events hit a null sink and are dropped (the manager queues them; Phase 2 + // owns the flush). Without this, the "system" telemetry table never fills. + DartBridgeTelemetry.attachSinkPhase1( + environment: params.environment, + deviceId: phase1DeviceId, + ); + // --- Phase 1: Core init (sync after Flutter async device-id lookup) --- // Phase-1 failures (invalid env, library load) propagate to the // caller via the surrounding try / rethrow. diff --git a/sdk/runanywhere-kotlin/src/main/kotlin/com/runanywhere/sdk/features/VoiceAgent/Services/VoiceAgentMicDriver.kt b/sdk/runanywhere-kotlin/src/main/kotlin/com/runanywhere/sdk/features/VoiceAgent/Services/VoiceAgentMicDriver.kt index be56ecc42c..330f935cc2 100644 --- a/sdk/runanywhere-kotlin/src/main/kotlin/com/runanywhere/sdk/features/VoiceAgent/Services/VoiceAgentMicDriver.kt +++ b/sdk/runanywhere-kotlin/src/main/kotlin/com/runanywhere/sdk/features/VoiceAgent/Services/VoiceAgentMicDriver.kt @@ -5,46 +5,39 @@ * VoiceAgentMicDriver.kt * * Audio ingress for the voice agent. The C ABI owns NO microphone access - * (rac_voice_agent.h "Audio-Ingress Contract"): the platform SDK must - * capture mic audio and push complete utterances into the C core, or the - * session is dead-air. This driver implements ingress mode 1 (per-utterance - * turns): capture 16 kHz mono PCM16 via [AudioCaptureManager], segment - * utterances with energy-based endpointing, and feed each utterance through - * `rac_voice_agent_process_turn_proto`. Turn VoiceEvents fan out to the - * handle callback, so `RunAnywhere.streamVoiceAgent()` collectors observe - * them without extra wiring. + * (rac_voice_agent.h "Audio-Ingress Contract"): the platform SDK captures raw + * mic frames and pushes them continuously into the C core via + * `rac_voice_agent_feed_audio_proto`. The core performs energy-based utterance + * segmentation and runs the STT -> LLM -> TTS turn pipeline itself, returning + * the synthesized reply inline for playback. This driver is therefore a thin + * capture -> feed -> play loop with NO SDK-side VAD; turn VoiceEvents fan out + * to the handle callback, so `RunAnywhere.streamVoiceAgent()` collectors + * observe them without extra wiring. */ package com.runanywhere.sdk.features.VoiceAgent.Services import ai.runanywhere.proto.v1.AudioEncoding -import ai.runanywhere.proto.v1.VoiceAgentTurnRequest -import ai.runanywhere.proto.v1.VoiceEvent +import ai.runanywhere.proto.v1.VoiceAgentResult import com.runanywhere.sdk.features.STT.Services.AudioCaptureManager import com.runanywhere.sdk.features.TTS.Services.AudioPlaybackManager -import com.runanywhere.sdk.foundation.errors.SDKException import com.runanywhere.sdk.infrastructure.logging.SDKLogger import com.runanywhere.sdk.native.bridge.RunAnywhereBridge -import java.io.ByteArrayOutputStream -import java.util.UUID import kotlin.coroutines.cancellation.CancellationException -import kotlin.math.sqrt import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.isActive -import okio.ByteString.Companion.toByteString /** - * Captures mic audio and drives per-utterance voice-agent turns against - * [handle]. [run] suspends until the calling coroutine is cancelled; cancel - * it to stop the session (capture teardown is handled in a finally block). + * Captures mic audio and feeds raw frames to the in-core voice agent bound to + * [handle]. [run] suspends until the calling coroutine is cancelled; cancel it + * to stop the session (capture teardown is handled in a finally block). * - * Endpointing is energy-based and intentionally simple: the C++ pipeline - * re-runs its own VAD over each submitted buffer, so the only job here is - * deciding where one utterance ends. Mic chunks that arrive while a turn is - * processing are discarded — the pipeline is strictly turn-taking (no - * barge-in), which also avoids transcribing the device's own TTS output. + * Segmentation/endpointing lives in the C core, which re-runs its own VAD over + * each utterance and is strictly turn-taking (no barge-in). Mic frames that + * arrive while a turn is processing are dropped by the bounded channel, which + * also avoids transcribing the device's own TTS output. */ internal class VoiceAgentMicDriver(private val handle: Long) { @@ -61,7 +54,7 @@ internal class VoiceAgentMicDriver(private val handle: Long) { capture.startRecording { chunk -> chunks.trySend(chunk) } logger.info("Voice-agent mic capture started") try { - segmentLoop(chunks) + feedLoop(chunks) } finally { capture.stopRecording() playback.stop() @@ -70,151 +63,55 @@ internal class VoiceAgentMicDriver(private val handle: Long) { } } - private suspend fun segmentLoop(chunks: Channel) { - val preRoll = ArrayDeque() - val utterance = ByteArrayOutputStream() - var inSpeech = false - var speechMs = 0 - var silenceMs = 0 - var noiseFloor = SPEECH_RMS_THRESHOLD - + private suspend fun feedLoop(chunks: Channel) { while (currentCoroutineContext().isActive) { val chunk = chunks.receive() - val chunkMs = chunk.size * 1000 / (SAMPLE_RATE_HZ * BYTES_PER_SAMPLE) - // Adaptive endpointing. A fixed RMS threshold misses the end-of- - // utterance pause on devices whose mic noise floor sits above the - // constant: silence is never seen, so an utterance only ends at the - // MAX_UTTERANCE_MS cap (a ~1s turn then waits ~15s). Track the ambient - // floor — drop instantly to any quieter level, and creep up only while - // not in speech so loud speech can't inflate it — and require a chunk - // to rise clearly above that floor to count as speech. - val level = rms(chunk) - val speechThreshold = maxOf(SPEECH_RMS_THRESHOLD, noiseFloor * SPEECH_FLOOR_MULTIPLIER) - val speech = level >= speechThreshold - noiseFloor = when { - level < noiseFloor -> level - !speech -> noiseFloor + (level - noiseFloor) * NOISE_FLOOR_RISE - else -> noiseFloor - } - if (!inSpeech) { - preRoll.addLast(chunk) - while (preRoll.size > PRE_ROLL_CHUNKS) preRoll.removeFirst() - if (speech) { - inSpeech = true - speechMs = chunkMs - silenceMs = 0 - utterance.reset() - preRoll.forEach(utterance::write) - preRoll.clear() - } - continue - } - - utterance.write(chunk) - if (speech) { - speechMs += chunkMs - silenceMs = 0 - } else { - silenceMs += chunkMs - } - - val utteranceMs = utterance.size() * 1000 / (SAMPLE_RATE_HZ * BYTES_PER_SAMPLE) - if (silenceMs >= END_OF_UTTERANCE_SILENCE_MS || utteranceMs >= MAX_UTTERANCE_MS) { - val audio = utterance.toByteArray() - inSpeech = false - utterance.reset() - if (speechMs >= MIN_SPEECH_MS) { - processTurn(audio) - // Drop chunks captured while the turn ran (agent thinking / - // speaking) so stale audio is not folded into the next turn. - while (chunks.tryReceive().isSuccess) Unit - } else { - logger.debug("Utterance discarded (${speechMs}ms speech < ${MIN_SPEECH_MS}ms)") - } - speechMs = 0 - silenceMs = 0 + val resultBytes = + try { + RunAnywhereBridge.racVoiceAgentFeedAudioProto( + handle, + chunk, + SAMPLE_RATE_HZ, + 1, + AudioEncoding.AUDIO_ENCODING_PCM_S16_LE.value, + false, + ) + } catch (e: CancellationException) { + throw e + } catch (e: Throwable) { + // Never swallow JVM Errors (OOM, …). A recoverable failure + // means this utterance's turn failed (e.g. empty STT) or the + // agent was torn down; the session cancels this coroutine on + // teardown, so log and keep feeding rather than killing the + // loop on a single bad turn. + if (e is Error) throw e + logger.warning("Voice feed failed: ${e.message}") + null + } ?: continue + + val result = + try { + VoiceAgentResult.ADAPTER.decode(resultBytes) + } catch (_: Exception) { + null + } ?: continue + + // A non-empty reply means the core closed an utterance and ran a full + // turn this call. synthesized_audio is self-describing WAV. + val reply = result.synthesized_audio + if (reply != null && reply.size > 0) { + logger.info("Playing agent reply (${reply.size} WAV bytes)") + playReply(reply.toByteArray()) + // Drop frames captured while the turn ran / the device spoke so + // stale audio is not folded into the next turn. + while (chunks.tryReceive().isSuccess) Unit } } } - private suspend fun processTurn(audio: ByteArray) { - val request = - VoiceAgentTurnRequest( - request_id = UUID.randomUUID().toString(), - audio_data = audio.toByteString(), - sample_rate_hz = SAMPLE_RATE_HZ, - channels = 1, - encoding = AudioEncoding.AUDIO_ENCODING_PCM_S16_LE, - ) - logger.info("Submitting voice turn (${audio.size} bytes)") - - // Accumulate synthesized TTS frames from the turn's event stream; - // played after the native call returns. Events also reach - // streamVoiceAgent() collectors via the handle callback fan-out. - val ttsPcm = ByteArrayOutputStream() - var ttsSampleRate = 0 - var ttsEncoding = AudioEncoding.AUDIO_ENCODING_UNSPECIFIED - var rc = RunAnywhereBridge.RAC_SUCCESS - try { - rc = - RunAnywhereBridge.racVoiceAgentProcessTurnProto( - handle, - VoiceAgentTurnRequest.ADAPTER.encode(request), - ) { bytes -> - try { - val frame = VoiceEvent.ADAPTER.decode(bytes).audio - if (frame != null && frame.pcm.size > 0) { - ttsPcm.write(frame.pcm.toByteArray()) - if (frame.sample_rate_hz > 0) ttsSampleRate = frame.sample_rate_hz - if (frame.encoding != AudioEncoding.AUDIO_ENCODING_UNSPECIFIED) { - ttsEncoding = frame.encoding - } - } - } catch (_: Exception) { - // Non-VoiceEvent or undecodable payload — ignore. - } - true - } - } catch (e: CancellationException) { - throw e - } catch (e: Throwable) { - // Never swallow JVM Errors (OOM, StackOverflow, …) — rethrow so they - // propagate; only a recoverable exception counts as a failed turn. - if (e is Error) throw e - logger.error("Voice turn threw: ${e.message}") - } - - if (rc == RunAnywhereBridge.RAC_ERROR_NOT_INITIALIZED) { - // The agent was torn down (session cleaned up) while this driver was - // still capturing. Stop instead of re-submitting every utterance to a - // dead agent, which spams "voice agent is not initialized" failures. - throw SDKException.voiceAgent("Voice agent is no longer initialized") - } - if (rc != RunAnywhereBridge.RAC_SUCCESS) { - logger.warning("Voice turn failed: rc=$rc") - } - - playTtsAudio(ttsPcm.toByteArray(), ttsSampleRate, ttsEncoding) - } - - // Play the turn's synthesized reply through the shared TTS sink. Runs - // before segmentLoop drains stale mic chunks, so the microphone stays - // gated while the device speaks (no self-transcription). - private suspend fun playTtsAudio(pcm: ByteArray, sampleRateHz: Int, encoding: AudioEncoding) { - if (pcm.isEmpty()) return - val sampleRate = if (sampleRateHz > 0) sampleRateHz else DEFAULT_TTS_SAMPLE_RATE_HZ - val wav = - when (encoding) { - AudioEncoding.AUDIO_ENCODING_PCM_S16_LE -> pcmS16ToWav(pcm, sampleRate) - // TTS backends emit f32 LE (AudioFrameEvent contract default). - else -> RunAnywhereBridge.racAudioFloat32ToWav(pcm, sampleRate) - } - if (wav == null || wav.isEmpty()) { - logger.warning("TTS audio conversion failed (${pcm.size} bytes, ${sampleRate}Hz, $encoding)") - return - } - logger.info("Playing agent reply (${pcm.size} PCM bytes @ ${sampleRate}Hz)") + private suspend fun playReply(wav: ByteArray) { + if (wav.isEmpty()) return try { playback.play(wav) } catch (e: CancellationException) { @@ -225,84 +122,15 @@ internal class VoiceAgentMicDriver(private val handle: Long) { } } - private fun pcmS16ToWav(pcm: ByteArray, sampleRate: Int): ByteArray { - val channels = 1 - val bitsPerSample = 16 - val byteRate = sampleRate * channels * bitsPerSample / 8 - val header = ByteArray(44) - fun putInt(offset: Int, value: Int) { - header[offset] = value.toByte() - header[offset + 1] = (value shr 8).toByte() - header[offset + 2] = (value shr 16).toByte() - header[offset + 3] = (value shr 24).toByte() - } - fun putShort(offset: Int, value: Int) { - header[offset] = value.toByte() - header[offset + 1] = (value shr 8).toByte() - } - "RIFF".toByteArray().copyInto(header, 0) - putInt(4, 36 + pcm.size) - "WAVE".toByteArray().copyInto(header, 8) - "fmt ".toByteArray().copyInto(header, 12) - putInt(16, 16) - putShort(20, 1) - putShort(22, channels) - putInt(24, sampleRate) - putInt(28, byteRate) - putShort(32, channels * bitsPerSample / 8) - putShort(34, bitsPerSample) - "data".toByteArray().copyInto(header, 36) - putInt(40, pcm.size) - return header + pcm - } - - private fun rms(chunk: ByteArray): Double { - val samples = chunk.size / BYTES_PER_SAMPLE - if (samples == 0) return 0.0 - var sum = 0.0 - for (i in 0 until samples) { - val lo = chunk[2 * i].toInt() and 0xff - val hi = chunk[2 * i + 1].toInt() - val sample = ((hi shl 8) or lo).toDouble() - sum += sample * sample - } - return sqrt(sum / samples) / Short.MAX_VALUE - } - private companion object { const val SAMPLE_RATE_HZ = 16_000 - const val BYTES_PER_SAMPLE = 2 /** * Bounded mic ingress buffer. The capture callback trySends while the * consumer pauses for the duration of each turn, so an unbounded channel * could grow without limit on long turns. DROP_OLDEST bounds memory; - * chunks captured mid-turn are discarded anyway (no barge-in). + * frames captured mid-turn are discarded anyway (no barge-in). */ const val MIC_CHANNEL_CAPACITY = 128 - - /** Absolute floor for the adaptive speech threshold (normalized RMS). */ - const val SPEECH_RMS_THRESHOLD = 0.015 - - /** Speech must exceed this multiple of the tracked ambient noise floor. */ - const val SPEECH_FLOOR_MULTIPLIER = 2.2 - - /** Per-chunk rate at which the ambient floor creeps up toward louder ambient. */ - const val NOISE_FLOOR_RISE = 0.05 - - /** Trailing silence that closes an utterance. */ - const val END_OF_UTTERANCE_SILENCE_MS = 800 - - /** Utterances with less accumulated speech than this are noise. */ - const val MIN_SPEECH_MS = 300 - - /** Hard cap so a noisy room cannot grow an unbounded buffer. */ - const val MAX_UTTERANCE_MS = 15_000 - - /** Leading chunks kept so the utterance onset is not clipped. */ - const val PRE_ROLL_CHUNKS = 3 - - /** Piper's native rate; used when the audio frame omits sample_rate_hz. */ - const val DEFAULT_TTS_SAMPLE_RATE_HZ = 22_050 } } diff --git a/sdk/runanywhere-kotlin/src/main/kotlin/com/runanywhere/sdk/native/bridge/RunAnywhereBridge.kt b/sdk/runanywhere-kotlin/src/main/kotlin/com/runanywhere/sdk/native/bridge/RunAnywhereBridge.kt index c40d221076..4d934805f9 100644 --- a/sdk/runanywhere-kotlin/src/main/kotlin/com/runanywhere/sdk/native/bridge/RunAnywhereBridge.kt +++ b/sdk/runanywhere-kotlin/src/main/kotlin/com/runanywhere/sdk/native/bridge/RunAnywhereBridge.kt @@ -1092,6 +1092,27 @@ object RunAnywhereBridge { listener: NativeProtoProgressListener, ): Int + /** + * Feed raw mic frames (16 kHz mono PCM16) into the in-core segmenter. The + * core accumulates frames, performs energy-based utterance endpointing, and + * on each completed utterance runs the full VAD→STT→LLM→TTS turn pipeline. + * Returns serialized VoiceAgentResult bytes — carrying the synthesized + * reply (WAV) when a turn completed this call, or an empty result + * otherwise. Per-stage VoiceEvents fan out to the handle callback (so + * streamVoiceAgent() collectors observe them). Throws a native-proto + * failure on error. Pass [isFinal] = true to flush an in-progress + * utterance. + */ + @JvmStatic + external fun racVoiceAgentFeedAudioProto( + handle: Long, + audioData: ByteArray, + sampleRateHz: Int, + channels: Int, + encoding: Int, + isFinal: Boolean, + ): ByteArray? + // TOOL-CALLING SESSION (rac_tool_calling.h) // // Native-owned state machine for generate → parse → execute → loop. The diff --git a/sdk/runanywhere-react-native/package.json b/sdk/runanywhere-react-native/package.json index bf7990bc7a..948d91e728 100644 --- a/sdk/runanywhere-react-native/package.json +++ b/sdk/runanywhere-react-native/package.json @@ -55,7 +55,7 @@ "eslint-plugin-prettier": "^5.5.5", "lerna": "^8.2.4", "prettier": "^3.8.3", - "react": "19.2.3", + "react": "19.2.7", "react-native": "0.85.3", "typescript": "^5.9.3" }, diff --git a/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore+ProtoCompat.hpp b/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore+ProtoCompat.hpp index baadaa08d7..d6863903e7 100644 --- a/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore+ProtoCompat.hpp +++ b/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore+ProtoCompat.hpp @@ -294,6 +294,17 @@ using VoiceAgentProcessTurnProtoFn = rac_result_t (*)( const void*, size_t, rac_proto_buffer_t*); +// Streaming feed-audio ingress (rac_voice_agent_feed_audio_proto): raw frames +// in, serialized VoiceAgentResult out (empty until the core closes a turn). +using VoiceAgentFeedAudioProtoFn = rac_result_t (*)( + void*, + const void*, + size_t, + int32_t, + int32_t, + int32_t, + rac_bool_t, + rac_proto_buffer_t*); // D-7 helper-level proto wrappers for the voice-agent sub-components. using VoiceAgentTranscribeProtoFn = rac_result_t (*)( diff --git a/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore+Voice.cpp b/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore+Voice.cpp index 1e6b673cc3..c4dd27a8d9 100644 --- a/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore+Voice.cpp +++ b/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore+Voice.cpp @@ -1429,6 +1429,37 @@ HybridRunAnywhereCore::voiceAgentProcessTurnProto( }); } +std::shared_ptr>> +HybridRunAnywhereCore::voiceAgentFeedAudioProto( + const std::shared_ptr& audioBytes, double sampleRateHz, + double channels, double encoding, bool isFinal) { + auto audio = copyVoiceArrayBufferBytes(audioBytes); + return Promise>::async( + [audio = std::move(audio), sampleRateHz, channels, encoding, isFinal]() { + rac_voice_agent_handle_t handle = getGlobalVoiceAgentHandle(); + auto fn = proto_compat::symbol( + "rac_voice_agent_feed_audio_proto"); + if (!handle || !fn) { + LOGE("voiceAgentFeedAudioProto: handle or proto ABI unavailable"); + return emptyVoiceProtoBuffer(); + } + rac_proto_buffer_t out; + proto_compat::initBuffer(&out); + const void* data = audio.empty() ? nullptr : audio.data(); + rac_result_t rc = fn(static_cast(handle), data, audio.size(), + static_cast(sampleRateHz), + static_cast(channels), + static_cast(encoding), + isFinal ? RAC_TRUE : RAC_FALSE, &out); + if (rc != RAC_SUCCESS && out.status == RAC_SUCCESS) { + LOGE("voiceAgentFeedAudioProto: rc=%d", rc); + proto_compat::freeBuffer(&out); + return emptyVoiceProtoBuffer(); + } + return copyVoiceProtoBuffer(out, "voiceAgentFeedAudioProto"); + }); +} + // ============================================================================ // Global component teardown // ============================================================================ diff --git a/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore.hpp b/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore.hpp index 2ec4f3b1e8..3bfa706adc 100644 --- a/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore.hpp +++ b/sdk/runanywhere-react-native/packages/core/cpp/HybridRunAnywhereCore.hpp @@ -412,6 +412,10 @@ class HybridRunAnywhereCore : public HybridRunAnywhereCoreSpec { std::shared_ptr>> voiceAgentProcessTurnProto( const std::shared_ptr &audioBytes) override; + std::shared_ptr>> + voiceAgentFeedAudioProto(const std::shared_ptr &audioBytes, + double sampleRateHz, double channels, double encoding, + bool isFinal) override; // ============================================================================ // Tool Calling - delegates generated proto bytes to commons C ABI. diff --git a/sdk/runanywhere-react-native/packages/core/ios/HybridAudioPlayback.swift b/sdk/runanywhere-react-native/packages/core/ios/HybridAudioPlayback.swift index 507762d1e6..931574b78b 100644 --- a/sdk/runanywhere-react-native/packages/core/ios/HybridAudioPlayback.swift +++ b/sdk/runanywhere-react-native/packages/core/ios/HybridAudioPlayback.swift @@ -180,9 +180,16 @@ class HybridAudioPlayback: HybridAudioPlaybackSpec { // on cleanup. The TTS-only screen (no capture session) still gets a // dedicated `.playback` session. if session.category == .playAndRecord { + // Reuse the voice agent's live session untouched, but force the loud + // speaker route: under `.playAndRecord` the output can fall back to the + // quiet receiver/earpiece, which presents as "no audio" even though + // playback succeeded. + try? session.overrideOutputAudioPort(.speaker) + logger.info("[playback-v2] reusing active .playAndRecord session (speaker route, no setCategory)") lock.withLock { $0.ownsSession = false } return } + logger.info("[playback-v2] configuring dedicated .playback session (category was \(session.category.rawValue))") try session.setCategory(.playback, mode: .default, options: [.duckOthers]) try session.setActive(true) lock.withLock { $0.ownsSession = true } diff --git a/sdk/runanywhere-react-native/packages/core/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.cpp b/sdk/runanywhere-react-native/packages/core/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.cpp index 9a54baa163..e27b182761 100644 --- a/sdk/runanywhere-react-native/packages/core/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.cpp +++ b/sdk/runanywhere-react-native/packages/core/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.cpp @@ -130,6 +130,7 @@ namespace margelo::nitro::runanywhere { prototype.registerHybridMethod("voiceAgentInitializeProto", &HybridRunAnywhereCoreSpec::voiceAgentInitializeProto); prototype.registerHybridMethod("voiceAgentComponentStatesProto", &HybridRunAnywhereCoreSpec::voiceAgentComponentStatesProto); prototype.registerHybridMethod("voiceAgentProcessTurnProto", &HybridRunAnywhereCoreSpec::voiceAgentProcessTurnProto); + prototype.registerHybridMethod("voiceAgentFeedAudioProto", &HybridRunAnywhereCoreSpec::voiceAgentFeedAudioProto); prototype.registerHybridMethod("toolParseProto", &HybridRunAnywhereCoreSpec::toolParseProto); prototype.registerHybridMethod("toolFormatPromptProto", &HybridRunAnywhereCoreSpec::toolFormatPromptProto); prototype.registerHybridMethod("toolValidateProto", &HybridRunAnywhereCoreSpec::toolValidateProto); diff --git a/sdk/runanywhere-react-native/packages/core/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.hpp b/sdk/runanywhere-react-native/packages/core/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.hpp index a918ed8840..43c9a2fcc5 100644 --- a/sdk/runanywhere-react-native/packages/core/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.hpp +++ b/sdk/runanywhere-react-native/packages/core/nitrogen/generated/shared/c++/HybridRunAnywhereCoreSpec.hpp @@ -47,7 +47,7 @@ namespace margelo::nitro::runanywhere { public: // Properties - + public: // Methods @@ -167,6 +167,7 @@ namespace margelo::nitro::runanywhere { virtual std::shared_ptr>> voiceAgentInitializeProto(const std::shared_ptr& configBytes) = 0; virtual std::shared_ptr>> voiceAgentComponentStatesProto() = 0; virtual std::shared_ptr>> voiceAgentProcessTurnProto(const std::shared_ptr& audioBytes) = 0; + virtual std::shared_ptr>> voiceAgentFeedAudioProto(const std::shared_ptr& audioBytes, double sampleRateHz, double channels, double encoding, bool isFinal) = 0; virtual std::shared_ptr>> toolParseProto(const std::shared_ptr& requestBytes) = 0; virtual std::shared_ptr>> toolFormatPromptProto(const std::shared_ptr& requestBytes) = 0; virtual std::shared_ptr>> toolValidateProto(const std::shared_ptr& requestBytes) = 0; diff --git a/sdk/runanywhere-react-native/packages/core/src/Features/VoiceAgent/VoiceAgentMicDriver.ts b/sdk/runanywhere-react-native/packages/core/src/Features/VoiceAgent/VoiceAgentMicDriver.ts index fa98e366e2..89fe90b62a 100644 --- a/sdk/runanywhere-react-native/packages/core/src/Features/VoiceAgent/VoiceAgentMicDriver.ts +++ b/sdk/runanywhere-react-native/packages/core/src/Features/VoiceAgent/VoiceAgentMicDriver.ts @@ -2,20 +2,20 @@ * VoiceAgentMicDriver.ts * * Audio ingress for the voice agent. The C ABI owns NO microphone access - * (rac_voice_agent.h "Audio-Ingress Contract"): the platform SDK must capture - * mic audio and push complete utterances into the C core, or the session is - * dead-air. This driver implements ingress mode 1 (per-utterance turns): - * capture 16 kHz mono PCM16 via {@link AudioCaptureManager}, segment utterances - * with energy-based endpointing, and submit each utterance through - * `rac_voice_agent_process_voice_turn_proto` (`voiceAgentProcessTurnProto`). - * Turn VoiceEvents fan out to the handle callback, so + * (rac_voice_agent.h "Audio-Ingress Contract"): the platform SDK captures raw + * mic frames and pushes them continuously into the C core via + * `rac_voice_agent_feed_audio_proto` (`voiceAgentFeedAudioProto`). The core + * performs energy-based utterance segmentation and runs the STT -> LLM -> TTS + * turn pipeline itself, returning the synthesized reply inline for playback. + * This driver is therefore a thin capture -> feed -> play loop with NO SDK-side + * VAD. Turn VoiceEvents fan out through the handle callback, so * `RunAnywhere.streamVoiceAgent()` collectors observe them without extra wiring. * * Mirrors `sdk/runanywhere-swift/.../VoiceAgentMicDriver.swift` and - * `sdk/runanywhere-kotlin/.../VoiceAgentMicDriver.kt`. Endpointing is - * energy-based; mic chunks that arrive while a turn is processing are discarded - * (strict turn-taking, no barge-in — also avoids transcribing the device's own - * TTS output). + * `sdk/runanywhere-kotlin/.../VoiceAgentMicDriver.kt`. Segmentation/endpointing + * lives in the C core; frames captured while a turn is processing are dropped + * by the core (and the bounded queue here) so the device's own TTS playout is + * not re-fed (strict turn-taking, no barge-in). */ import { AudioCaptureManager } from '../VoiceSession/AudioCaptureManager'; @@ -24,30 +24,22 @@ import { SDKLogger } from '../../Foundation/Logging/Logger/SDKLogger'; import { requireNativeModule } from '../../native'; import { arrayBufferToBytes } from '../../services/ProtoBytes'; import { VoiceAgentResult as VoiceAgentResultMessage } from '@runanywhere/proto-ts/voice_agent_service'; +import { AudioEncoding } from '@runanywhere/proto-ts/voice_events'; const SAMPLE_RATE_HZ = 16_000; -const BYTES_PER_SAMPLE = 2; -/** Absolute floor for the adaptive speech threshold (normalized RMS). */ -const SPEECH_RMS_THRESHOLD = 0.015; -/** Speech must exceed this multiple of the tracked ambient noise floor. */ -const SPEECH_FLOOR_MULTIPLIER = 2.2; -/** Per-chunk rate at which the ambient floor creeps up toward louder ambient. */ -const NOISE_FLOOR_RISE = 0.05; -/** Trailing silence that closes an utterance. */ -const END_OF_UTTERANCE_SILENCE_MS = 800; -/** Utterances with less accumulated speech than this are noise. */ -const MIN_SPEECH_MS = 300; -/** Hard cap so a noisy room cannot grow an unbounded buffer. */ -const MAX_UTTERANCE_MS = 15_000; -/** Leading chunks kept so the utterance onset is not clipped. */ -const PRE_ROLL_CHUNKS = 3; +const CHANNELS = 1; +/** Bounded backlog so a slow turn cannot grow the queue without limit. */ +const CHANNEL_CAPACITY = 128; +/** Poll interval when no captured frames are pending. */ +const FEED_IDLE_SLEEP_MS = 20; /** - * Captures mic audio and drives per-utterance voice-agent turns. {@link start} - * runs until {@link stop} is called. Segmentation runs synchronously inside the - * capture callback; an utterance end kicks off an async turn, during which - * incoming chunks are dropped (the `processing` gate) so the mic stays gated - * while the device thinks / speaks. + * Captures mic audio and feeds raw frames to the in-core voice agent. + * {@link start} runs until {@link stop} is called. The capture callback only + * enqueues frames; a single async feed loop drains them and calls the core, + * which blocks for the duration of a turn when an utterance closes and returns + * the synthesized reply inline. We play it and drop any backlog captured during + * the turn so the device's own playout is not re-fed. */ export class VoiceAgentMicDriver { private readonly logger = new SDKLogger('VoiceAgentMic'); @@ -55,35 +47,29 @@ export class VoiceAgentMicDriver { private readonly playback = new AudioPlaybackManager(); private stopped = false; - private processing = false; + private queue: Uint8Array[] = []; - // Segmentation state. - private preRoll: Uint8Array[] = []; - private utterance: Uint8Array[] = []; - private inSpeech = false; - private speechMs = 0; - private silenceMs = 0; - private noiseFloor = SPEECH_RMS_THRESHOLD; - - /** Begin mic capture + segmentation. Resolves once capture has started. */ + /** Begin mic capture + feed loop. Resolves once capture has started. */ async start(): Promise { const granted = await this.capture.requestPermission(); if (!granted) { throw new Error('Microphone permission denied'); } this.stopped = false; + this.queue = []; // The voice agent runs a single full-duplex (.playAndRecord) session for the - // whole turn-taking loop so TTS replies can play through the speaker while the - // mic stays live. Activate it BEFORE startRecording so capture reuses it + // whole turn-taking loop so TTS replies can play through the speaker while + // the mic stays live. Activate it BEFORE startRecording so capture reuses it // instead of forcing the output-only .record session (which silences replies // and trips cannotStartPlaying). Mirrors the iOS Swift driver's // configureVoiceAudioSession(); no-op on Android. await this.capture.activateAudioSession(); - await this.capture.startRecording((chunk) => this.onChunk(chunk)); + await this.capture.startRecording((chunk) => this.enqueueChunk(chunk)); this.logger.info('Voice-agent mic capture started'); + void this.feedLoop(); } - /** Stop capture + playback and reset segmentation state. */ + /** Stop capture + playback and reset state. */ async stop(): Promise { if (this.stopped) return; this.stopped = true; @@ -97,126 +83,82 @@ export class VoiceAgentMicDriver { } catch { /* noop */ } - this.preRoll = []; - this.utterance = []; - this.inSpeech = false; + this.queue = []; this.logger.info('Voice-agent mic capture stopped'); } - private onChunk(chunk: Uint8Array): void { - // Drop chunks while a turn is in flight (no barge-in) or after stop(). - if (this.stopped || this.processing || chunk.byteLength === 0) return; - - const chunkMs = - (chunk.byteLength * 1000) / (SAMPLE_RATE_HZ * BYTES_PER_SAMPLE); - // Adaptive endpointing: a fixed RMS threshold misses the end-of-utterance - // pause on devices whose mic noise floor sits above the constant. Track the - // ambient floor — drop instantly to any quieter level, creep up only while - // not in speech — and require a chunk to rise clearly above it. - const level = VoiceAgentMicDriver.rms(chunk); - const speechThreshold = Math.max( - SPEECH_RMS_THRESHOLD, - this.noiseFloor * SPEECH_FLOOR_MULTIPLIER - ); - const isSpeech = level >= speechThreshold; - if (!this.inSpeech) { - if (level < this.noiseFloor) { - this.noiseFloor = level; - } else if (!isSpeech) { - this.noiseFloor += (level - this.noiseFloor) * NOISE_FLOOR_RISE; - } - } - - if (!this.inSpeech) { - this.preRoll.push(chunk); - if (this.preRoll.length > PRE_ROLL_CHUNKS) { - this.preRoll.shift(); - } - if (isSpeech) { - this.inSpeech = true; - this.speechMs = chunkMs; - this.silenceMs = 0; - this.utterance = [...this.preRoll]; - this.preRoll = []; - } - return; - } - - this.utterance.push(chunk); - if (isSpeech) { - this.speechMs += chunkMs; - this.silenceMs = 0; - } else { - this.silenceMs += chunkMs; + private enqueueChunk(chunk: Uint8Array): void { + if (this.stopped || chunk.byteLength === 0) return; + this.queue.push(chunk); + if (this.queue.length > CHANNEL_CAPACITY) { + this.queue.splice(0, this.queue.length - CHANNEL_CAPACITY); } + } - const utteranceBytes = this.utterance.reduce((n, c) => n + c.byteLength, 0); - const utteranceMs = - (utteranceBytes * 1000) / (SAMPLE_RATE_HZ * BYTES_PER_SAMPLE); - if ( - this.silenceMs >= END_OF_UTTERANCE_SILENCE_MS || - utteranceMs >= MAX_UTTERANCE_MS - ) { - const speechMs = this.speechMs; - const audio = VoiceAgentMicDriver.concat(this.utterance); - this.inSpeech = false; - this.utterance = []; - this.speechMs = 0; - this.silenceMs = 0; - if (speechMs >= MIN_SPEECH_MS) { - // Gate the mic for the duration of the turn (and TTS playback) so the - // device does not transcribe its own reply. `processing` is cleared - // once the turn + playback finish. - this.processing = true; - void this.processTurn(audio).finally(() => { - this.processing = false; - }); - } else { - this.logger.debug( - `Utterance discarded (${Math.round(speechMs)}ms speech < ${MIN_SPEECH_MS}ms)` - ); - } - } + private drainChunks(): Uint8Array[] { + if (this.queue.length === 0) return []; + const drained = this.queue; + this.queue = []; + return drained; } - private async processTurn(audio: Uint8Array): Promise { - if (this.stopped || audio.byteLength === 0) return; - this.logger.info(`Submitting voice turn (${audio.byteLength} bytes)`); - try { - const native = requireNativeModule(); - const resultBytes = await native.voiceAgentProcessTurnProto( - VoiceAgentMicDriver.toArrayBuffer(audio) - ); - if (this.stopped) return; - const bytes = arrayBufferToBytes(resultBytes); - if (bytes.byteLength === 0) { - this.logger.warning('Voice turn returned an empty result'); - return; + /** + * Drains captured frames and feeds them to the core. A non-empty + * `synthesizedAudio` in the returned result means the core closed an + * utterance and ran a full turn this call; we play the reply and drop any + * backlog captured during the turn so the device's own playout is not re-fed. + */ + private async feedLoop(): Promise { + const native = requireNativeModule(); + while (!this.stopped) { + const chunks = this.drainChunks(); + if (chunks.length === 0) { + await VoiceAgentMicDriver.sleep(FEED_IDLE_SLEEP_MS); + continue; } - const result = VoiceAgentResultMessage.decode(bytes); - if (result.errorMessage && result.errorMessage.length > 0) { - this.logger.warning(`Voice turn failed: ${result.errorMessage}`); + + for (const chunk of chunks) { + if (this.stopped) return; + try { + const resultBytes = await native.voiceAgentFeedAudioProto( + VoiceAgentMicDriver.toArrayBuffer(chunk), + SAMPLE_RATE_HZ, + CHANNELS, + AudioEncoding.AUDIO_ENCODING_PCM_S16_LE, + false + ); + if (this.stopped) return; + const bytes = arrayBufferToBytes(resultBytes); + if (bytes.byteLength === 0) continue; // utterance still open + + const result = VoiceAgentResultMessage.decode(bytes); + if (result.errorMessage && result.errorMessage.length > 0) { + this.logger.warning(`Voice turn failed: ${result.errorMessage}`); + } + if (await this.playReply(result)) { + // Drop frames captured during the turn + playback. + this.queue = []; + } + } catch (error) { + this.logger.warning( + `Voice feed threw: ${error instanceof Error ? error.message : String(error)}` + ); + } } - await this.playReply(result); - } catch (error) { - this.logger.warning( - `Voice turn threw: ${error instanceof Error ? error.message : String(error)}` - ); } } /** - * Play the turn's synthesized reply through the shared playback sink. Runs - * while `processing` is still set, so the mic stays gated while the device - * speaks (no self-transcription). Commons returns `synthesized_audio` as a - * complete WAV (`rac_audio_float32_to_wav`), so the bytes are handed to the - * native player as-is — not re-encoded. + * Play the turn's synthesized reply through the shared playback sink. + * Commons returns `synthesized_audio` as a complete WAV + * (`rac_audio_float32_to_wav`), so the bytes are handed to the native player + * as-is — not re-encoded. Returns true when a reply was played. */ private async playReply( result: ReturnType - ): Promise { + ): Promise { const audio = result.synthesizedAudio; - if (!audio || audio.byteLength === 0) return; + if (!audio || audio.byteLength === 0) return false; const wav = VoiceAgentMicDriver.toArrayBuffer(audio); this.logger.info(`Playing agent reply (${audio.byteLength} WAV bytes)`); @@ -227,34 +169,11 @@ export class VoiceAgentMicDriver { `Agent reply playback failed: ${error instanceof Error ? error.message : String(error)}` ); } + return true; } - /** Normalized RMS of a 16 kHz mono Int16-LE chunk (0..1). */ - private static rms(chunk: Uint8Array): number { - const sampleCount = Math.floor(chunk.byteLength / BYTES_PER_SAMPLE); - if (sampleCount === 0) return 0; - const view = new DataView( - chunk.buffer, - chunk.byteOffset, - sampleCount * BYTES_PER_SAMPLE - ); - let sum = 0; - for (let i = 0; i < sampleCount; i++) { - const sample = view.getInt16(i * BYTES_PER_SAMPLE, true); - sum += sample * sample; - } - return Math.sqrt(sum / sampleCount) / 32767; - } - - private static concat(chunks: Uint8Array[]): Uint8Array { - const total = chunks.reduce((n, c) => n + c.byteLength, 0); - const out = new Uint8Array(total); - let offset = 0; - for (const c of chunks) { - out.set(c, offset); - offset += c.byteLength; - } - return out; + private static sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); } private static toArrayBuffer(bytes: Uint8Array): ArrayBuffer { diff --git a/sdk/runanywhere-react-native/packages/core/src/Features/VoiceSession/AudioPlaybackManager.ts b/sdk/runanywhere-react-native/packages/core/src/Features/VoiceSession/AudioPlaybackManager.ts index 36a1911e19..51979a9761 100644 --- a/sdk/runanywhere-react-native/packages/core/src/Features/VoiceSession/AudioPlaybackManager.ts +++ b/sdk/runanywhere-react-native/packages/core/src/Features/VoiceSession/AudioPlaybackManager.ts @@ -59,30 +59,28 @@ export class AudioPlaybackManager { } /** - * Play already-encoded in-memory WAV audio. Unlike {@link play}, the bytes - * are handed to the native player as-is — no float32→WAV re-encoding. Used - * for the voice agent's synthesized reply, which commons returns as a - * complete WAV (`rac_audio_float32_to_wav`), not raw PCM. + * Play a ready-made WAV buffer (RIFF header + samples) directly, without + * re-encoding. Use this when the source is already a complete WAV — e.g. the + * voice agent's `VoiceAgentResult.synthesizedAudio`, which commons emits as a + * full WAV blob. Passing such a blob to {@link play} would misinterpret the + * RIFF header + int16 samples as raw float32 PCM and wrap it in a second WAV + * header → fast/noisy garbage. */ async playWav(wavData: ArrayBuffer): Promise { if (this.state === 'playing') { this.stop(); } - this.state = 'playing'; - logger.info('Playing in-memory WAV...'); - try { await AudioPlayback.play(wavData); this.state = 'idle'; } catch (error) { if ((this.state as PlaybackState) === 'stopped') { - // stop() interrupts the in-flight play() — not an error. return; } this.state = 'error'; logger.error( - `Playback failed: ${error instanceof Error ? error.message : String(error)}` + `WAV playback failed: ${error instanceof Error ? error.message : String(error)}` ); throw error; } diff --git a/sdk/runanywhere-react-native/packages/core/src/Features/VoiceSession/VoiceAgentMicDriver.ts b/sdk/runanywhere-react-native/packages/core/src/Features/VoiceSession/VoiceAgentMicDriver.ts new file mode 100644 index 0000000000..833ce7801f --- /dev/null +++ b/sdk/runanywhere-react-native/packages/core/src/Features/VoiceSession/VoiceAgentMicDriver.ts @@ -0,0 +1,266 @@ +/** + * VoiceAgentMicDriver.ts — microphone audio ingress for the RN voice agent. + * + * The commons C ABI owns NO microphone (rac_voice_agent.h "audio-ingress + * contract"): subscribing to `streamVoiceAgent()` alone is dead air. The + * platform SDK must capture mic audio and push utterances into the core, or + * the pipeline never sees PCM → VAD/STT get nothing → the LLM has no input → + * no reply. This driver closes that gap for RN. + * + * Mirrors the Kotlin/Flutter `VoiceAgentMicDriver`: capture 16 kHz mono PCM16 + * via {@link AudioCaptureManager}, segment utterances with energy-based + * endpointing, and run each utterance through `processVoiceTurn` (the one-shot + * VAD→STT→LLM→TTS C entry, which on this build also re-runs the loaded VAD + * model over the submitted buffer). The transcription + assistant response are + * surfaced via {@link VoiceAgentMicCallbacks.onTurn}, and the synthesized TTS + * reply is played through {@link AudioPlaybackManager}. + * + * The endpointing is intentionally coarse — it only decides where one + * utterance ends; the C++ pipeline VADs each submitted buffer. Mic chunks that + * arrive while a turn is processing (or while the reply is playing) are + * dropped: the pipeline is strictly turn-taking (no barge-in), which also + * avoids transcribing the device's own TTS output. + */ + +import { AudioCaptureManager } from './AudioCaptureManager'; +import { AudioPlaybackManager } from './AudioPlaybackManager'; +import { processVoiceTurn } from '../../Public/Extensions/VoiceAgent/RunAnywhere+VoiceAgent'; +import { SDKLogger } from '../../Foundation/Logging/Logger/SDKLogger'; + +const logger = new SDKLogger('VoiceAgentMicDriver'); + +/** A completed turn: what the user said and how the agent replied. */ +export interface VoiceAgentMicTurn { + userText: string; + assistantText: string; +} + +/** Coarse pipeline phase, for UI status. */ +export type VoiceAgentMicPhase = 'listening' | 'processing' | 'speaking'; + +export interface VoiceAgentMicCallbacks { + /** A turn finished: user transcription + assistant reply. */ + onTurn?: (turn: VoiceAgentMicTurn) => void; + /** Coarse phase transitions (listening / processing / speaking). */ + onPhase?: (phase: VoiceAgentMicPhase) => void; + /** A non-fatal turn error (capture continues). */ + onError?: (error: Error) => void; +} + +const SAMPLE_RATE_HZ = 16000; +const BYTES_PER_SAMPLE = 2; + +/** Absolute floor for the adaptive speech threshold (normalized RMS). */ +const SPEECH_RMS_THRESHOLD = 0.015; +/** Speech must exceed this multiple of the tracked ambient noise floor. */ +const SPEECH_FLOOR_MULTIPLIER = 2.2; +/** Per-chunk rate at which the ambient floor creeps toward louder ambient. */ +const NOISE_FLOOR_RISE = 0.05; +/** Trailing silence that closes an utterance. */ +const END_OF_UTTERANCE_SILENCE_MS = 800; +/** Utterances with less accumulated speech than this are discarded as noise. */ +const MIN_SPEECH_MS = 300; +/** Hard cap so a noisy room cannot grow an unbounded buffer. */ +const MAX_UTTERANCE_MS = 15000; +/** Leading chunks kept so the utterance onset is not clipped. */ +const PRE_ROLL_CHUNKS = 3; + +export class VoiceAgentMicDriver { + private readonly capture = new AudioCaptureManager(); + private readonly playback = new AudioPlaybackManager(); + private callbacks: VoiceAgentMicCallbacks = {}; + + private stopped = false; + private processing = false; + + // Segmentation state. + private preRoll: Uint8Array[] = []; + private utterance: Uint8Array[] = []; + private utteranceBytes = 0; + private inSpeech = false; + private speechMs = 0; + private silenceMs = 0; + private noiseFloor = SPEECH_RMS_THRESHOLD; + + /** + * Request mic permission and begin capture. Returns `false` if the + * permission was denied (caller should surface it); throws on capture + * failure. + */ + async start(callbacks: VoiceAgentMicCallbacks = {}): Promise { + this.callbacks = callbacks; + this.stopped = false; + + const granted = await this.capture.requestPermission(); + if (!granted) { + return false; + } + + // The voice agent runs a single full-duplex (.playAndRecord) session for the + // whole turn-taking loop so TTS replies can play through the speaker while + // the mic session stays live. Activate it BEFORE startRecording so capture + // reuses it instead of forcing the output-only .record session — under + // .record, playback cannot claim the output and trips + // AVAudioSessionErrorInsufficientPriority ('!pri', OSStatus 561017449). + // Mirrors the iOS Swift driver's configureVoiceAudioSession(); no-op on + // Android. + await this.capture.activateAudioSession(); + await this.capture.startRecording((chunk) => this.onChunk(chunk)); + this.callbacks.onPhase?.('listening'); + logger.info('Voice-agent mic capture started'); + return true; + } + + /** Stop capture + playback and reset segmentation. */ + stop(): void { + if (this.stopped) return; + this.stopped = true; + try { + this.capture.stopRecording(); + } catch (e) { + logger.warning(`stopRecording failed: ${String(e)}`); + } + this.playback.stop(); + this.resetSegmentation(); + logger.info('Voice-agent mic capture stopped'); + } + + private onChunk(chunk: Uint8Array): void { + // Drop chunks while a turn is processing or the reply is playing + // (turn-taking, no barge-in, no self-transcription). + if (this.stopped || this.processing || chunk.length === 0) return; + + const chunkMs = (chunk.length * 1000) / (SAMPLE_RATE_HZ * BYTES_PER_SAMPLE); + const level = this.rms(chunk); + + // Adaptive endpointing: track the ambient floor (drop instantly to any + // quieter level, creep up only while not in speech) and require a chunk to + // rise clearly above that floor to count as speech. + const speechThreshold = Math.max( + SPEECH_RMS_THRESHOLD, + this.noiseFloor * SPEECH_FLOOR_MULTIPLIER + ); + const isSpeech = level >= speechThreshold; + if (level < this.noiseFloor) { + this.noiseFloor = level; + } else if (!isSpeech) { + this.noiseFloor += (level - this.noiseFloor) * NOISE_FLOOR_RISE; + } + + if (!this.inSpeech) { + this.preRoll.push(chunk); + while (this.preRoll.length > PRE_ROLL_CHUNKS) { + this.preRoll.shift(); + } + if (isSpeech) { + this.inSpeech = true; + this.speechMs = chunkMs; + this.silenceMs = 0; + this.utterance = [...this.preRoll]; + this.utteranceBytes = this.preRoll.reduce((n, c) => n + c.length, 0); + this.preRoll = []; + } + return; + } + + this.utterance.push(chunk); + this.utteranceBytes += chunk.length; + if (isSpeech) { + this.speechMs += chunkMs; + this.silenceMs = 0; + } else { + this.silenceMs += chunkMs; + } + + const utteranceMs = + (this.utteranceBytes * 1000) / (SAMPLE_RATE_HZ * BYTES_PER_SAMPLE); + if ( + this.silenceMs >= END_OF_UTTERANCE_SILENCE_MS || + utteranceMs >= MAX_UTTERANCE_MS + ) { + const audio = this.concatUtterance(); + const hadSpeech = this.speechMs >= MIN_SPEECH_MS; + this.resetSegmentation(); + if (hadSpeech) { + void this.processTurn(audio); + } else { + logger.debug( + `Utterance discarded (${this.speechMs}ms speech < ${MIN_SPEECH_MS}ms)` + ); + } + } + } + + private concatUtterance(): Uint8Array { + const out = new Uint8Array(this.utteranceBytes); + let offset = 0; + for (const c of this.utterance) { + out.set(c, offset); + offset += c.length; + } + return out; + } + + private resetSegmentation(): void { + this.inSpeech = false; + this.speechMs = 0; + this.silenceMs = 0; + this.utterance = []; + this.utteranceBytes = 0; + this.preRoll = []; + } + + private async processTurn(audio: Uint8Array): Promise { + this.processing = true; + this.callbacks.onPhase?.('processing'); + logger.info(`Submitting voice turn (${audio.length} bytes)`); + try { + const result = await processVoiceTurn(audio); + + const userText = (result.transcription ?? '').trim(); + const assistantText = (result.assistantResponse ?? '').trim(); + if (userText.length > 0 || assistantText.length > 0) { + this.callbacks.onTurn?.({ userText, assistantText }); + } + + const wav = result.synthesizedAudio; + if (wav && wav.byteLength > 0 && !this.stopped) { + this.callbacks.onPhase?.('speaking'); + // `synthesizedAudio` is a complete WAV blob (commons emits WAV, header + // included), so play it directly — NOT through play(), which would + // treat the RIFF header + int16 samples as raw float32 PCM and re-wrap + // it (→ fast/noisy garbage). Copy into a fresh ArrayBuffer-backed view + // in case the source spans a larger or shared buffer. + const copy = new Uint8Array(wav.byteLength); + copy.set(wav); + try { + await this.playback.playWav(copy.buffer); + } catch (e) { + logger.warning(`Agent reply playback failed: ${String(e)}`); + } + } + } catch (e) { + const err = e instanceof Error ? e : new Error(String(e)); + logger.warning(`Voice turn failed: ${err.message}`); + this.callbacks.onError?.(err); + } finally { + this.processing = false; + this.resetSegmentation(); + if (!this.stopped) { + this.callbacks.onPhase?.('listening'); + } + } + } + + private rms(chunk: Uint8Array): number { + const samples = Math.floor(chunk.length / BYTES_PER_SAMPLE); + if (samples === 0) return 0; + const view = new DataView(chunk.buffer, chunk.byteOffset, chunk.byteLength); + let sum = 0; + for (let i = 0; i < samples; i++) { + const sample = view.getInt16(i * BYTES_PER_SAMPLE, true); + sum += sample * sample; + } + return Math.sqrt(sum / samples) / 32767; + } +} diff --git a/sdk/runanywhere-react-native/packages/core/src/Foundation/Constants/SDKConstants.ts b/sdk/runanywhere-react-native/packages/core/src/Foundation/Constants/SDKConstants.ts index b94c75610d..ddd1f8710a 100644 --- a/sdk/runanywhere-react-native/packages/core/src/Foundation/Constants/SDKConstants.ts +++ b/sdk/runanywhere-react-native/packages/core/src/Foundation/Constants/SDKConstants.ts @@ -4,6 +4,22 @@ * Mirrors `sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Constants/SDKConstants.swift`. */ +import { Platform } from 'react-native'; + +/** + * Backend `DevicePlatform` enum value for the current OS family. The backend + * auth/device contract only accepts the OS family ("ios", "android", "macos", + * "windows", "web") — not the binding name ("react_native"), which 422s the + * SDK auth exchange and leaves every request unauthenticated (telemetry + + * device registration both fail). Mirrors the Flutter SDK's + * `SDKConstants.platform` getter and Kotlin's "android". + */ +function osPlatform(): string { + // Platform.OS is already the OS family the backend expects + // ("ios"/"android"/"macos"/"windows"/"web"), never the binding name. + return Platform.OS; +} + export const SDKConstants = { /** * SDK version - must stay in sync with package.json `version`. @@ -21,9 +37,10 @@ export const SDKConstants = { }, /** - * SDK platform identifier used by backend auth/device metadata. + * SDK platform identifier used by backend auth/device metadata. Must be the + * OS family (backend `DevicePlatform` enum), not the binding name. */ - platform: 'react_native', + platform: osPlatform(), /** * Minimum log level in production. diff --git a/sdk/runanywhere-react-native/packages/core/src/Public/Extensions/LLM/RunAnywhere+TextGeneration.ts b/sdk/runanywhere-react-native/packages/core/src/Public/Extensions/LLM/RunAnywhere+TextGeneration.ts index 1b1e6d2d1d..74d40e0e9c 100644 --- a/sdk/runanywhere-react-native/packages/core/src/Public/Extensions/LLM/RunAnywhere+TextGeneration.ts +++ b/sdk/runanywhere-react-native/packages/core/src/Public/Extensions/LLM/RunAnywhere+TextGeneration.ts @@ -32,7 +32,7 @@ import { } from '@runanywhere/proto-ts/llm_options'; import { LLMGenerateRequest, - LLMStreamEvent, + LLMStreamEvent as LLMStreamEventMessage, type LLMStreamEvent as LLMStreamEventType, } from '@runanywhere/proto-ts/llm_service'; import { inferenceFrameworkToJSON, ModelCategory } from '@runanywhere/proto-ts/model_types'; @@ -150,22 +150,11 @@ export async function generate( * * Matches Swift SDK: `RunAnywhere.generateStream(_ request: RALLMGenerateRequest)`. * - * Wire-up (iOS parity): a single atomic `native.llmGenerateStreamProto` - * call — the RN binding for `rac_llm_generate_stream_proto` — wires the - * per-request stream callback and runs generation together, exactly like - * Swift's `CppBridge.LLM.generateStream` / `ProtoStreamContext.runRequestStream`. - * Each invocation of the callback delivers one serialized - * `runanywhere.v1.LLMStreamEvent`; the stream finishes on `event.isFinal`. - * - * This replaces the previous handle-subscription approach - * (`NitroLLM.subscribeProtoEvents` + `native.llmGenerateProto`), which - * registered a callback the *non-streaming* buffered generate path never - * feeds — so no events were ever delivered and the UI hung "thinking - * forever". `rac_llm_set_stream_proto_callback` is fed by the component - * `generate_stream` path, not by the handleless `rac_llm_generate_proto`. - * - * Cancellation propagates through `for-await break` → `iterator.return()` - * → `native.llmCancelProto()` → C++ generation observes the cancel signal. + * Wire-up: events arrive via `native.llmGenerateStreamProto(bytes, onEvent)` + * — the dedicated callback-streaming method (same path as STT/TTS/VLM). Each + * `onEvent` delivers one proto-encoded `LLMStreamEvent`; the wrapper buffers + * them into this AsyncIterable. Cancellation propagates through + * `iterator.return()` → `native.llmCancelProto()`. */ export function generateStream( request: LLMGenerateRequest, @@ -188,10 +177,15 @@ export function generateStream( const llmRequest = normalizeLLMGenerateRequest(requestOrPrompt, options, true); const requestBytes = encodeLLMGenerateRequest(llmRequest); + // Stream via the dedicated callback method `llmGenerateStreamProto(bytes, cb)` + // — the same path STT/TTS/VLM use. The earlier handle-subscribe variant paired + // with the BLOCKING `llmGenerateProto` never fed the subscription (native + // generated, returned the aggregate, and emitted no per-token events), so the + // UI hung "generating" forever. Mirrors RunAnywhere+VisionLanguage.processStream. return { [Symbol.asyncIterator](): AsyncIterator { const queue: LLMStreamEventType[] = []; - let resolver: ((value: IteratorResult) => void) | null = null; + let resolver: ((v: IteratorResult) => void) | null = null; let done = false; let started = false; let streamError: Error | null = null; @@ -213,28 +207,29 @@ export function generateStream( } }; - const start = (): void => { + const start = async (): Promise => { if (started) return; started = true; - // iOS parity: one atomic streaming call wires the callback + runs - // generation together (no register/generate race). The callback - // receives one serialized LLMStreamEvent per token; `isFinal` ends it. - ensureServicesReady() - .then(() => - native.llmGenerateStreamProto(requestBytes, (eventBytes: ArrayBuffer) => { - try { - const event = LLMStreamEvent.decode(arrayBufferToBytes(eventBytes)); - push(event); - if (event.isFinal) { - finish(); - } - } catch (error) { - streamError = - error instanceof Error ? error : new Error(String(error)); + await ensureServicesReady(); + native + .llmGenerateStreamProto(requestBytes, (eventBytes: ArrayBuffer) => { + try { + const event = LLMStreamEventMessage.decode( + arrayBufferToBytes(eventBytes) + ); + if (event.errorMessage) { + streamError = new Error(event.errorMessage); + } + push(event); + if (event.isFinal) { finish(); } - }), - ) + } catch (error) { + streamError = + error instanceof Error ? error : new Error(String(error)); + finish(); + } + }) .then(() => { if (!done) finish(); }) @@ -246,29 +241,24 @@ export function generateStream( return { async next(): Promise> { - start(); + await start(); if (queue.length > 0) { return { value: queue.shift()!, done: false }; } - if (streamError) { - throw streamError; - } + if (streamError) throw streamError; if (done) { return { value: undefined as unknown as LLMStreamEventType, done: true }; } return new Promise>((resolve) => { resolver = resolve; }).then((result) => { - if (streamError) { - throw streamError; - } + if (streamError) throw streamError; return result; }); }, async return(): Promise> { // Await the native cancel before resolving so back-to-back - // cancel → generate sequences are race-free. Matches Swift - // cancelGeneration() which awaits CppBridge.LLM.shared.cancelProto(). + // cancel → generate sequences are race-free. try { await native.llmCancelProto(); } catch { /* noop */ } finish(); return { value: undefined as unknown as LLMStreamEventType, done: true }; @@ -327,23 +317,36 @@ export async function aggregateStream( let terminalError = ''; let finalEvent: LLMStreamEventType | undefined; - for await (const event of iterable) { - if (event.token && event.token.length > 0) { - if (firstTokenTimeMs === undefined) { - firstTokenTimeMs = Date.now(); + // Drive the stream with a manual `iterator.next()` loop, NOT `for await...of`. + // Hermes does not support `for await...of` over the NitroModules-backed LLM + // stream — it silently fails to iterate, so tokens never arrive and the UI + // hangs "thinking" forever. `iterator.return()` in the finally tears the + // native subscription down (cancel) when we break early or throw. + const iterator = iterable[Symbol.asyncIterator](); + try { + for (;;) { + const next = await iterator.next(); + if (next.done) break; + const event = next.value; + if (event.token && event.token.length > 0) { + if (firstTokenTimeMs === undefined) { + firstTokenTimeMs = Date.now(); + } + fullResponse += event.token; + tokenCount += 1; + if (onToken) { + await onToken(fullResponse); + } } - fullResponse += event.token; - tokenCount += 1; - if (onToken) { - await onToken(fullResponse); + if (event.isFinal) { + finalEvent = event; + finishReason = event.finishReason ?? ''; + terminalError = event.errorMessage ?? ''; + break; } } - if (event.isFinal) { - finalEvent = event; - finishReason = event.finishReason ?? ''; - terminalError = event.errorMessage ?? ''; - break; - } + } finally { + await iterator.return?.(); } const totalLatencyMs = Date.now() - startTimeMs; diff --git a/sdk/runanywhere-react-native/packages/core/src/Public/Extensions/VoiceAgent/RunAnywhere+VoiceAgent.ts b/sdk/runanywhere-react-native/packages/core/src/Public/Extensions/VoiceAgent/RunAnywhere+VoiceAgent.ts index a7ac110b53..363c4195bb 100644 --- a/sdk/runanywhere-react-native/packages/core/src/Public/Extensions/VoiceAgent/RunAnywhere+VoiceAgent.ts +++ b/sdk/runanywhere-react-native/packages/core/src/Public/Extensions/VoiceAgent/RunAnywhere+VoiceAgent.ts @@ -327,8 +327,9 @@ export function streamVoiceAgent(): AsyncIterable { const adapter = new VoiceAgentStreamAdapter(handle); // Swift parity (RunAnywhere+VoiceAgent.swift:225-236): while the event - // stream is consumed, a platform mic driver captures audio, segments - // utterances, and submits each via `voiceAgentProcessTurnProto`. The C + // stream is consumed, a platform mic driver captures audio and feeds raw + // frames to the core via `voiceAgentFeedAudioProto`; the core segments + // utterances and runs the turn pipeline itself (no SDK-side VAD). The C // ABI owns no microphone access — without this driver the pipeline gets // no audio buffer and stays "listening" forever (dead-air), per the // rac_voice_agent.h Audio-Ingress Contract. Events emitted during each diff --git a/sdk/runanywhere-react-native/packages/core/src/index.ts b/sdk/runanywhere-react-native/packages/core/src/index.ts index 0ea3cbd0db..171b769986 100644 --- a/sdk/runanywhere-react-native/packages/core/src/index.ts +++ b/sdk/runanywhere-react-native/packages/core/src/index.ts @@ -112,6 +112,17 @@ export type { ErrorContext } from './Foundation/Errors'; // Features/TTS/Services/AudioPlaybackManager.swift. export { AudioCaptureManager } from './Features/VoiceSession/AudioCaptureManager'; export { AudioPlaybackManager } from './Features/VoiceSession/AudioPlaybackManager'; +// Mic-driven voice-agent ingress (capture → endpoint → processVoiceTurn → TTS +// playback). Mirrors the Kotlin/Flutter VoiceAgentMicDriver; without it the +// voice agent only observes output events and never receives audio. +export { + VoiceAgentMicDriver, +} from './Features/VoiceSession/VoiceAgentMicDriver'; +export type { + VoiceAgentMicTurn, + VoiceAgentMicPhase, + VoiceAgentMicCallbacks, +} from './Features/VoiceSession/VoiceAgentMicDriver'; export { EventBus, modelLifecycleChange } from './Public/Events/EventBus'; export type { diff --git a/sdk/runanywhere-react-native/packages/core/src/native/NitroModulesGlobalInit.ts b/sdk/runanywhere-react-native/packages/core/src/native/NitroModulesGlobalInit.ts index 6834cd5756..062c44edf7 100644 --- a/sdk/runanywhere-react-native/packages/core/src/native/NitroModulesGlobalInit.ts +++ b/sdk/runanywhere-react-native/packages/core/src/native/NitroModulesGlobalInit.ts @@ -73,16 +73,14 @@ export async function initializeNitroModulesGlobally(): Promise { /** * Get the NitroModules proxy synchronously. * - * The `react-native-nitro-modules` import (top of this file) installs Nitro into - * the runtime at module-load time, so the proxy is available immediately — well - * before the async `initializeNitroModulesGlobally()` runs. If the cache has not - * been populated yet (e.g. a backend's `register()` runs before - * `RunAnywhere.initialize()`), resolve and cache it from the imported singleton - * instead of returning null. This keeps backend registration order-independent - * so callers never have to sequence init before use. + * Falls back to the static `react-native-nitro-modules` import (and caches it) + * when `initializeNitroModulesGlobally()` has not run yet. The proxy is the + * synchronous import object — the same value the async initializer assigns — so + * this fallback is equivalent and idempotent. Without it, callers that run + * before `RunAnywhere.initialize()` (e.g. backend `register()` during app + * bootstrap) incorrectly see "native module not available". * - * @returns NitroModules proxy, or null only if the native module is genuinely - * unavailable (not installed/linked). + * @returns NitroModules proxy, or null only if the import itself is unavailable */ export function getNitroModulesProxySync(): NitroProxy | null { if (_nitroModulesProxy === null && NitroModulesNamed) { diff --git a/sdk/runanywhere-react-native/packages/core/src/specs/RunAnywhereCore.nitro.ts b/sdk/runanywhere-react-native/packages/core/src/specs/RunAnywhereCore.nitro.ts index e793ba4723..af2c80b401 100644 --- a/sdk/runanywhere-react-native/packages/core/src/specs/RunAnywhereCore.nitro.ts +++ b/sdk/runanywhere-react-native/packages/core/src/specs/RunAnywhereCore.nitro.ts @@ -921,6 +921,24 @@ export interface RunAnywhereCore extends HybridObject<{ voiceAgentComponentStatesProto(): Promise; voiceAgentProcessTurnProto(audioBytes: ArrayBuffer): Promise; + /** + * Stream raw mic frames into the in-core voice agent via the commons + * `rac_voice_agent_feed_audio_proto` ABI. The core performs energy-based + * utterance segmentation and runs the STT -> LLM -> TTS turn pipeline itself; + * there is NO SDK-side VAD. Each call returns a serialized + * `runanywhere.v1.VoiceAgentResult`: empty (zero-length) while the utterance + * is still open, non-empty (with `synthesizedAudio`) on the call that closes a + * turn. `isFinal` flushes the in-progress utterance. Mirrors the iOS Swift / + * Kotlin drivers' feed loop. + */ + voiceAgentFeedAudioProto( + audioBytes: ArrayBuffer, + sampleRateHz: number, + channels: number, + encoding: number, + isFinal: boolean + ): Promise; + // ============================================================================ // Tool Calling Capability // diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Features/VoiceAgent/Services/VoiceAgentMicDriver.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Features/VoiceAgent/Services/VoiceAgentMicDriver.swift index 3dceb90f22..3fb0d4b6a7 100644 --- a/sdk/runanywhere-swift/Sources/RunAnywhere/Features/VoiceAgent/Services/VoiceAgentMicDriver.swift +++ b/sdk/runanywhere-swift/Sources/RunAnywhere/Features/VoiceAgent/Services/VoiceAgentMicDriver.swift @@ -3,8 +3,11 @@ // RunAnywhere SDK // // Audio ingress for the voice agent. The C ABI owns no microphone access; -// the platform SDK captures mic audio and pushes complete utterances into -// the C core via rac_voice_agent_process_turn_proto. +// the platform SDK captures raw mic frames and pushes them continuously into +// the C core via rac_voice_agent_feed_audio_proto. The core performs energy- +// based utterance segmentation and runs the STT -> LLM -> TTS turn pipeline +// itself, returning the synthesized reply inline for playback. This driver is +// therefore a thin capture -> feed -> play loop with NO SDK-side VAD. // import AVFoundation @@ -12,10 +15,11 @@ import CRACommons import Foundation import os -/// Captures mic audio and drives per-utterance voice-agent turns. +/// Captures mic audio and feeds raw frames to the in-core voice agent. /// -/// Mirrors Kotlin `VoiceAgentMicDriver.kt`. Endpointing is energy-based; -/// mic chunks that arrive while a turn is processing are discarded. +/// Mirrors Kotlin `VoiceAgentMicDriver.kt`. Segmentation/endpointing lives in +/// the C core (`rac_voice_agent_feed_audio_proto`); frames captured while a +/// turn is processing are dropped by the bounded queue. final class VoiceAgentMicDriver: @unchecked Sendable { private let handle: rac_voice_agent_handle_t private let capture = AudioCaptureManager() @@ -23,7 +27,6 @@ final class VoiceAgentMicDriver: @unchecked Sendable { private let logger = SDKLogger(category: "VoiceAgentMic") private let chunkLock = OSAllocatedUnfairLock<[Data]>(initialState: []) - private let processingLock = OSAllocatedUnfairLock(initialState: false) init(handle: rac_voice_agent_handle_t) { self.handle = handle @@ -58,7 +61,7 @@ final class VoiceAgentMicDriver: @unchecked Sendable { logger.info("Voice-agent mic capture stopped") } - try await segmentLoop() + try await feedLoop() } // MARK: - Audio session @@ -96,7 +99,6 @@ final class VoiceAgentMicDriver: @unchecked Sendable { // MARK: - Chunk queue private func enqueueChunk(_ chunk: Data) { - if processingLock.withLock({ $0 }) { return } chunkLock.withLock { queue in queue.append(chunk) if queue.count > MicConstants.channelCapacity { @@ -117,16 +119,13 @@ final class VoiceAgentMicDriver: @unchecked Sendable { chunkLock.withLock { $0.removeAll() } } - // MARK: - Segmentation - - private func segmentLoop() async throws { - var preRoll: [Data] = [] - var utterance = Data() - var inSpeech = false - var speechMs = 0 - var silenceMs = 0 - var noiseFloor = MicConstants.speechRMSThreshold + // MARK: - Feed loop + /// Drains captured frames and feeds them to the core. The core blocks the + /// feed call for the duration of a turn when an utterance closes and + /// returns the synthesized reply inline; we play it and drop any backlog + /// captured during the turn so the device's own playout is not re-fed. + private func feedLoop() async throws { while !Task.isCancelled { let chunks = drainChunks() if chunks.isEmpty { @@ -137,214 +136,40 @@ final class VoiceAgentMicDriver: @unchecked Sendable { for chunk in chunks { if Task.isCancelled { return } - let chunkMs = Self.chunkDurationMs(chunk) - // Adaptive endpointing. A fixed RMS threshold misses the end-of- - // utterance pause on devices whose mic noise floor sits above the - // constant. Track the ambient floor and require a chunk to rise - // clearly above it. - let level = Self.rms(chunk) - let threshold = max(MicConstants.speechRMSThreshold, noiseFloor * MicConstants.speechFloorMultiplier) - let isSpeech = level >= threshold - // Only adapt the floor while idle (between utterances). Adapting - // mid-utterance lets inter-word pauses and the end-of-utterance - // silence tail inflate the floor; it then stays high (it's never - // reset across turns) and locks out the next turn's speech. Drop - // instantly to any quieter ambient; creep up slowly otherwise. - if !inSpeech { - if level < noiseFloor { - noiseFloor = level - } else if !isSpeech { - noiseFloor += (level - noiseFloor) * MicConstants.noiseFloorRise - } + let (status, result) = try CppBridge.VoiceAgent.feedAudioProto( + handle: handle, + audio: chunk, + sampleRateHz: Int32(MicConstants.sampleRateHz), + channels: 1, + encoding: Int32(RAAudioEncoding.pcmS16Le.rawValue), + isFinal: false + ) + + if status == RAC_ERROR_NOT_INITIALIZED { + throw SDKException( + code: .notInitialized, + message: "Voice agent is no longer initialized", + category: .component + ) } - - if !inSpeech { - preRoll.append(chunk) - if preRoll.count > MicConstants.preRollChunks { - preRoll.removeFirst() - } - if isSpeech { - inSpeech = true - speechMs = chunkMs - silenceMs = 0 - utterance = Data() - for buffered in preRoll { utterance.append(buffered) } - preRoll.removeAll() - } + if status != RAC_SUCCESS { + logger.warning("Voice feed failed: rc=\(status)") continue } - utterance.append(chunk) - if isSpeech { - speechMs += chunkMs - silenceMs = 0 - } else { - silenceMs += chunkMs - } - - let utteranceMs = (utterance.count / MicConstants.bytesPerSample) * 1000 / MicConstants.sampleRateHz - if silenceMs >= MicConstants.endOfUtteranceSilenceMs || utteranceMs >= MicConstants.maxUtteranceMs { - let audio = utterance - inSpeech = false - utterance = Data() - if speechMs >= MicConstants.minSpeechMs { - try await processTurn(audio: audio) - // Drop chunks captured while the turn ran (agent thinking / - // speaking) so stale audio is not folded into the next turn. - discardPendingChunks() - } else { - logger.debug("Utterance discarded (\(speechMs)ms speech < \(MicConstants.minSpeechMs)ms)") - } - speechMs = 0 - silenceMs = 0 + // A non-empty reply means the core closed an utterance and ran a + // full turn this call. `synthesizedAudio` is self-describing WAV. + if let reply = result?.synthesizedAudio, !reply.isEmpty { + logger.info("Playing agent reply (\(reply.count) WAV bytes)") + try await playback.play(reply) + discardPendingChunks() } } } } - - // MARK: - Turn processing - - private func processTurn(audio: Data) async throws { - processingLock.withLock { $0 = true } - defer { processingLock.withLock { $0 = false } } - - var request = RAVoiceAgentTurnRequest() - request.requestID = UUID().uuidString - request.audioData = audio - request.sampleRateHz = Int32(MicConstants.sampleRateHz) - request.channels = 1 - request.encoding = .pcmS16Le - - logger.info("Submitting voice turn (\(audio.count) bytes)") - - var ttsPCM = Data() - var ttsSampleRate: Int32 = 0 - var ttsEncoding: RAAudioEncoding = .unspecified - - let rc = try CppBridge.VoiceAgent.processTurnProto(handle: handle, request: request) { event in - guard case let .audio(frame) = event.payload else { return } - guard !frame.pcm.isEmpty else { return } - ttsPCM.append(frame.pcm) - if frame.sampleRateHz > 0 { ttsSampleRate = frame.sampleRateHz } - if frame.encoding != .unspecified { ttsEncoding = frame.encoding } - } - - if rc == RAC_ERROR_NOT_INITIALIZED { - throw SDKException( - code: .notInitialized, - message: "Voice agent is no longer initialized", - category: .component - ) - } - if rc != RAC_SUCCESS { - logger.warning("Voice turn failed: rc=\(rc)") - } - - try await playTTSReply(pcm: ttsPCM, sampleRateHz: ttsSampleRate, encoding: ttsEncoding) - } - - private func playTTSReply(pcm: Data, sampleRateHz: Int32, encoding: RAAudioEncoding) async throws { - guard !pcm.isEmpty else { return } - - let sampleRate = sampleRateHz > 0 ? sampleRateHz : MicConstants.defaultTTSSampleRateHz - let wav: Data - switch encoding { - case .pcmS16Le: - wav = try Self.pcmS16ToWAV(pcm: pcm, sampleRate: sampleRate) - default: - wav = try Self.float32PCMToWAV(pcm: pcm, sampleRate: sampleRate) - } - - guard !wav.isEmpty else { - logger.warning("TTS audio conversion failed (\(pcm.count) bytes, \(sampleRate)Hz, \(encoding))") - return - } - - logger.info("Playing agent reply (\(pcm.count) PCM bytes @ \(sampleRate)Hz)") - try await playback.play(wav) - } - - // MARK: - Audio helpers - - private static func rms(_ chunk: Data) -> Double { - let sampleCount = chunk.count / MicConstants.bytesPerSample - guard sampleCount > 0 else { return 0 } - - var sum = 0.0 - chunk.withUnsafeBytes { raw in - let bytes = raw.bindMemory(to: UInt8.self) - for index in 0.. Int { - let samples = chunk.count / MicConstants.bytesPerSample - return (samples * 1000) / MicConstants.sampleRateHz - } - - private static func pcmS16ToWAV(pcm: Data, sampleRate: Int32) throws -> Data { - var wavDataPtr: UnsafeMutableRawPointer? - var wavSize = 0 - let rc = pcm.withUnsafeBytes { raw in - rac_audio_int16_to_wav( - raw.baseAddress, - pcm.count, - sampleRate, - &wavDataPtr, - &wavSize - ) - } - guard rc == RAC_SUCCESS, let ptr = wavDataPtr, wavSize > 0 else { - throw SDKException( - code: .processingFailed, - message: "Failed to convert Int16 PCM to WAV: \(rc)", - category: .component - ) - } - defer { rac_free(ptr) } - return Data(bytes: ptr, count: wavSize) - } - - private static func float32PCMToWAV(pcm: Data, sampleRate: Int32) throws -> Data { - var wavDataPtr: UnsafeMutableRawPointer? - var wavSize = 0 - let rc = pcm.withUnsafeBytes { raw in - rac_audio_float32_to_wav( - raw.baseAddress, - pcm.count, - sampleRate, - &wavDataPtr, - &wavSize - ) - } - guard rc == RAC_SUCCESS, let ptr = wavDataPtr, wavSize > 0 else { - throw SDKException( - code: .processingFailed, - message: "Failed to convert Float32 PCM to WAV: \(rc)", - category: .component - ) - } - defer { rac_free(ptr) } - return Data(bytes: ptr, count: wavSize) - } } private enum MicConstants { static let sampleRateHz = 16_000 - static let bytesPerSample = 2 static let channelCapacity = 128 - static let speechRMSThreshold = 0.015 - static let speechFloorMultiplier = 2.2 - static let noiseFloorRise = 0.05 - static let endOfUtteranceSilenceMs = 800 - static let minSpeechMs = 300 - static let maxUtteranceMs = 15_000 - static let preRollChunks = 3 - static let defaultTTSSampleRateHz: Int32 = 22_050 } diff --git a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+ModalityProtoABI.swift b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+ModalityProtoABI.swift index 8f25cdb783..b2d9ae2706 100644 --- a/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+ModalityProtoABI.swift +++ b/sdk/runanywhere-swift/Sources/RunAnywhere/Foundation/Bridge/Extensions/CppBridge+ModalityProtoABI.swift @@ -76,14 +76,29 @@ private enum VoiceAgentStateProtoABI { UnsafeMutableRawPointer? ) -> rac_result_t + // Streaming raw-frame ingress: the core segments utterances and runs the + // turn pipeline, returning a VoiceAgentResult inline when one completes. + typealias FeedAudio = @convention(c) ( + rac_voice_agent_handle_t?, + UnsafeRawPointer?, + Int, + Int32, + Int32, + Int32, + rac_bool_t, + UnsafeMutablePointer? + ) -> rac_result_t + static let processTurnName = "rac_voice_agent_process_voice_turn_proto" static let processTurnWithEventsName = "rac_voice_agent_process_turn_proto" + static let feedAudioName = "rac_voice_agent_feed_audio_proto" static let processTurn = NativeProtoABI.load(processTurnName, as: ProcessTurn.self) static let processTurnWithEvents = NativeProtoABI.load( processTurnWithEventsName, as: ProcessTurnWithEvents.self ) + static let feedAudio = NativeProtoABI.load(feedAudioName, as: FeedAudio.self) } private final class VoiceTurnEventCollector: @unchecked Sendable { @@ -518,6 +533,58 @@ extension CppBridge.VoiceAgent { ) } } + + /// Push raw mic frames (16 kHz mono PCM16) into the core. The C core + /// segments utterances itself and runs the full turn pipeline; when an + /// utterance completes this call, the returned `RAVoiceAgentResult` + /// carries the synthesized reply (WAV) for inline playback. Otherwise the + /// result is empty. Per-stage VoiceEvents still fan out to the handle + /// callback. Pass `isFinal` to flush an in-progress utterance. + /// + /// Returns the native status alongside the decoded result so the caller + /// can distinguish a fatal condition (e.g. the agent is no longer + /// initialized) from a recoverable per-turn failure (e.g. empty STT), + /// which should be logged but not stop the capture loop. Throws only when + /// the native symbol is unavailable or the proto cannot be decoded. + nonisolated static func feedAudioProto( + handle: rac_voice_agent_handle_t, + audio: Data, + sampleRateHz: Int32, + channels: Int32, + encoding: Int32, + isFinal: Bool + ) throws -> (status: rac_result_t, result: RAVoiceAgentResult?) { + let feed = try NativeProtoABI.require( + VoiceAgentStateProtoABI.feedAudio, + named: VoiceAgentStateProtoABI.feedAudioName + ) + guard NativeProtoABI.canReceiveProtoBuffer else { + throw SDKException( + code: .notSupported, + message: NativeProtoABI.missingSymbolMessage(VoiceAgentStateProtoABI.feedAudioName), + category: .internal + ) + } + var outBuffer = rac_proto_buffer_t() + defer { NativeProtoABI.free(&outBuffer) } + let status = audio.withUnsafeBytes { audioBytes in + feed( + handle, + audioBytes.baseAddress, + audio.count, + sampleRateHz, + channels, + encoding, + isFinal ? RAC_TRUE : RAC_FALSE, + &outBuffer + ) + } + guard status == RAC_SUCCESS else { + return (status, nil) + } + let result = try NativeProtoABI.decode(RAVoiceAgentResult.self, from: outBuffer) + return (status, result) + } } // MARK: - VLM custom diff --git a/sdk/runanywhere-web/packages/core/src/runtime/PlatformAdapter.ts b/sdk/runanywhere-web/packages/core/src/runtime/PlatformAdapter.ts index 00a236fceb..7a2b223aaf 100644 --- a/sdk/runanywhere-web/packages/core/src/runtime/PlatformAdapter.ts +++ b/sdk/runanywhere-web/packages/core/src/runtime/PlatformAdapter.ts @@ -31,8 +31,14 @@ const RAC_ERROR_FILE_NOT_FOUND = -183; const RAC_ERROR_FILE_WRITE_FAILED = -185; const RAC_ERROR_PLATFORM = -180; const RAC_ERROR_INVALID_ARGUMENT = -259; +const RAC_ERROR_CANCELLED = -380; const RAC_DIRECTORY_ENTRY_NAME_MAX = 512; +// In-flight async downloads started through the `http_download` adapter slot, +// keyed by the task id handed back to C++. Used by `http_download_cancel`. +let httpDownloadCounter = 0; +const httpDownloadTasks = new Map(); + /** * Structural Emscripten-module surface the adapter needs. Any RACommons * WASM module (core, llamacpp, onnx) satisfies this. @@ -66,6 +72,8 @@ interface CallbackPtrs { fileListDirectory: number; isNonEmptyDirectory: number; getVendorId: number; + httpDownload: number; + httpDownloadCancel: number; } export class PlatformAdapter { @@ -113,6 +121,8 @@ export class PlatformAdapter { fileListDirectory: this.registerFileListDirectory(), isNonEmptyDirectory: this.registerIsNonEmptyDirectory(), getVendorId: this.registerGetVendorId(), + httpDownload: this.registerHttpDownload(), + httpDownloadCancel: this.registerHttpDownloadCancel(), }; // Runtime struct offsets — each helper must be exported by @@ -147,10 +157,12 @@ export class PlatformAdapter { m.setValue(this.adapterPtr + getOffset('log'), this.callbacks.log, '*'); m.setValue(this.adapterPtr + getOffset('now_ms'), this.callbacks.nowMs, '*'); m.setValue(this.adapterPtr + getOffset('get_memory_info'), this.callbacks.getMemoryInfo, '*'); - // http_download (optional) → null. The HTTPAdapter / FetchHttpTransport - // path takes over once setRunanywhereModule installs the module. - m.setValue(this.adapterPtr + getOffset('http_download'), 0, '*'); - m.setValue(this.adapterPtr + getOffset('http_download_cancel'), 0, '*'); + // http_download (async streaming slot). The C++ download orchestrator + // prefers this slot on Emscripten (event-driven, non-blocking) so progress + // ticks live; without it, the synchronous FetchHttpTransport path buffers + // the whole body on the main thread and the UI freezes at 0% until 100%. + m.setValue(this.adapterPtr + getOffset('http_download'), this.callbacks.httpDownload, '*'); + m.setValue(this.adapterPtr + getOffset('http_download_cancel'), this.callbacks.httpDownloadCancel, '*'); // extract_archive — native libarchive is compiled into WASM. m.setValue(this.adapterPtr + getOffset('extract_archive'), 0, '*'); m.setValue(this.adapterPtr + getOffset('file_list_directory'), this.callbacks.fileListDirectory, '*'); @@ -427,6 +439,84 @@ export class PlatformAdapter { } }, 'iiii'); } + + /** + * rac_result_t (*http_download)(const char* url, const char* destination_path, + * rac_http_progress_callback_fn progress_callback, + * rac_http_complete_callback_fn complete_callback, + * void* callback_user_data, char** out_task_id, void* user_data) + * + * Async by contract: start the transfer, return RAC_OK immediately with a + * task id, then stream bytes to MEMFS while calling progress_callback per + * chunk and complete_callback at the end. Because it returns immediately and + * the fetch runs on the event loop, the main thread is never blocked — the + * C++ download orchestrator's poll loop sees live byte counts and the UI does + * not freeze (unlike the synchronous FetchHttpTransport fallback). + */ + private registerHttpDownload(): number { + const m = this.m; + return m.addFunction(( + urlPtr: number, + destPtr: number, + progressCbPtr: number, + completeCbPtr: number, + cbUserData: number, + outTaskIdPtr: number, + _userData: number, + ) => { + try { + const url = m.UTF8ToString(urlPtr); + const dest = m.UTF8ToString(destPtr); + const taskId = `webdl_${++httpDownloadCounter}`; + + // Hand the task id back to C++ as an owned C string (orchestrator frees). + if (outTaskIdPtr) { + const len = m.lengthBytesUTF8(taskId) + 1; + const idPtr = m._malloc(len); + m.stringToUTF8(taskId, idPtr, len); + m.setValue(outTaskIdPtr, idPtr, '*'); + } + + const controller = new AbortController(); + httpDownloadTasks.set(taskId, controller); + + // Fire-and-forget: the transfer drives itself on the event loop and + // reports back through the C callbacks. Errors are surfaced via + // complete_callback, never thrown out of this synchronous start call. + void runHttpDownload(m, { + url, + dest, + progressCbPtr, + completeCbPtr, + cbUserData, + controller, + }).finally(() => httpDownloadTasks.delete(taskId)); + + return RAC_OK; + } catch (error) { + logger.warning(`http_download start failed: ${error instanceof Error ? error.message : String(error)}`); + return RAC_ERROR_PLATFORM; + } + }, 'iiiiiiii'); + } + + /** rac_result_t (*http_download_cancel)(const char* task_id, void* user_data) */ + private registerHttpDownloadCancel(): number { + const m = this.m; + return m.addFunction((taskIdPtr: number, _userData: number) => { + try { + const taskId = m.UTF8ToString(taskIdPtr); + const controller = httpDownloadTasks.get(taskId); + if (controller) { + controller.abort(); + httpDownloadTasks.delete(taskId); + } + return RAC_OK; + } catch { + return RAC_ERROR_PLATFORM; + } + }, 'iii'); + } } // --------------------------------------------------------------------------- @@ -449,6 +539,194 @@ function readBytes(m: PlatformAdapterModule, srcPtr: number, length: number): Ui return out; } +// --------------------------------------------------------------------------- +// http_download slot — async streaming fetch → MEMFS with progress callbacks. +// --------------------------------------------------------------------------- + +/** Emscripten FS stream-write surface needed to write chunks incrementally. */ +interface StreamingFS { + open(path: string, flags: string): unknown; + write(stream: unknown, buffer: ArrayBufferView, offset: number, length: number, position?: number): number; + close(stream: unknown): void; + mkdirTree?(path: string): void; + analyzePath?(path: string): { exists: boolean }; + stat?(path: string): { size?: number }; +} + +function streamingFsOf(m: PlatformAdapterModule): StreamingFS | null { + const fs = (m as { FS?: unknown }).FS as Partial | undefined; + if (fs && typeof fs.open === 'function' && typeof fs.write === 'function' + && typeof fs.close === 'function') { + return fs as StreamingFS; + } + return null; +} + +function memfsFileSize(fs: StreamingFS, path: string): number { + try { + if (fs.analyzePath && !fs.analyzePath(path)?.exists) return 0; + return fs.stat?.(path)?.size ?? 0; + } catch { + return 0; + } +} + +/** Resolve a WASM-table entry as a callable (i64 args are passed as BigInt). */ +function wasmCallable( + m: PlatformAdapterModule, + ptr: number, +): ((...args: Array) => number) | null { + if (ptr === 0) return null; + const tbl = m as unknown as { + getWasmTableEntry?: (p: number) => (...args: Array) => number; + wasmTable?: { get(p: number): (...args: Array) => number }; + }; + if (typeof tbl.getWasmTableEntry === 'function') return tbl.getWasmTableEntry(ptr); + if (tbl.wasmTable && typeof tbl.wasmTable.get === 'function') return tbl.wasmTable.get(ptr); + return null; +} + +/** Invoke rac_http_progress_callback_fn — void (int64, int64, void*). */ +function invokeProgressCallback( + m: PlatformAdapterModule, + cbPtr: number, + bytesDownloaded: number, + totalBytes: number, + userData: number, +): void { + const callable = wasmCallable(m, cbPtr); + if (!callable) return; + try { + callable(BigInt(bytesDownloaded), BigInt(totalBytes), userData); + } catch (error) { + logger.warning(`http_download progress callback threw: ${error instanceof Error ? error.message : String(error)}`); + } +} + +/** Invoke rac_http_complete_callback_fn — void (rac_result_t, const char*, void*). */ +function invokeCompleteCallback( + m: PlatformAdapterModule, + cbPtr: number, + result: number, + downloadedPath: string | null, + userData: number, +): void { + const callable = wasmCallable(m, cbPtr); + if (!callable) return; + let pathPtr = 0; + try { + if (downloadedPath) { + const len = m.lengthBytesUTF8(downloadedPath) + 1; + pathPtr = m._malloc(len); + m.stringToUTF8(downloadedPath, pathPtr, len); + } + callable(result, pathPtr, userData); + } catch (error) { + logger.warning(`http_download complete callback threw: ${error instanceof Error ? error.message : String(error)}`); + } finally { + if (pathPtr) { + try { m._free(pathPtr); } catch { /* noop */ } + } + } +} + +interface HttpDownloadArgs { + url: string; + dest: string; + progressCbPtr: number; + completeCbPtr: number; + cbUserData: number; + controller: AbortController; +} + +/** + * Stream `url` into the MEMFS file at `dest`, reporting incremental progress. + * Resumes from any bytes already on disk via a Range request when the server + * honours it (HTTP 206); otherwise restarts from zero. Always finishes by + * invoking the C complete callback (success, cancel, or error) exactly once. + */ +async function runHttpDownload(m: PlatformAdapterModule, args: HttpDownloadArgs): Promise { + const { url, dest, progressCbPtr, completeCbPtr, cbUserData, controller } = args; + const fs = streamingFsOf(m); + if (!fs) { + invokeCompleteCallback(m, completeCbPtr, RAC_ERROR_PLATFORM, null, cbUserData); + return; + } + + let stream: unknown = null; + try { + const parent = dest.slice(0, dest.lastIndexOf('/')) || '/'; + try { fs.mkdirTree?.(parent); } catch { /* dir may already exist */ } + + const existing = memfsFileSize(fs, dest); + const headers: Record = {}; + if (existing > 0) headers.Range = `bytes=${existing}-`; + + const response = await fetch(url, { headers, signal: controller.signal }); + + // 416 Range Not Satisfiable on a resume request means the file on disk is + // already at/past the requested offset — i.e. the download is complete. + // Report success without rewriting so a re-trigger of an already-present + // model is a no-op rather than a hard failure. + if (existing > 0 && response.status === 416) { + invokeCompleteCallback(m, completeCbPtr, RAC_OK, dest, cbUserData); + return; + } + + let received = 0; + let position = 0; + if (existing > 0 && response.status === 206) { + // Server honoured the range — append to the partial file. + received = existing; + position = existing; + stream = fs.open(dest, 'r+'); + } else { + if (!response.ok) throw new Error(`HTTP ${response.status}`); + // Fresh download (or server ignored Range) — truncate and restart. + stream = fs.open(dest, 'w'); + } + + const contentLength = Number(response.headers.get('Content-Length') ?? 0); + const totalBytes = contentLength > 0 ? received + contentLength : 0; + + if (!response.body) throw new Error('response has no readable body'); + const reader = response.body.getReader(); + + invokeProgressCallback(m, progressCbPtr, received, totalBytes, cbUserData); + + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (value && value.length > 0) { + fs.write(stream, value, 0, value.length, position); + position += value.length; + received += value.length; + invokeProgressCallback(m, progressCbPtr, received, totalBytes, cbUserData); + } + } + + fs.close(stream); + stream = null; + invokeCompleteCallback(m, completeCbPtr, RAC_OK, dest, cbUserData); + } catch (error) { + if (stream) { + try { fs.close(stream); } catch { /* noop */ } + } + const aborted = controller.signal.aborted + || (error instanceof DOMException && error.name === 'AbortError'); + if (!aborted) { + logger.warning(`http_download '${url}' failed: ${error instanceof Error ? error.message : String(error)}`); + } + invokeCompleteCallback( + m, + completeCbPtr, + aborted ? RAC_ERROR_CANCELLED : RAC_ERROR_PLATFORM, + null, + cbUserData, + ); + } +} + interface DirectoryEntryInfo { name: string; isDir: boolean; diff --git a/sdk/runanywhere-web/wasm/CMakeLists.txt b/sdk/runanywhere-web/wasm/CMakeLists.txt index 8e448de66e..e74030b01a 100644 --- a/sdk/runanywhere-web/wasm/CMakeLists.txt +++ b/sdk/runanywhere-web/wasm/CMakeLists.txt @@ -457,6 +457,13 @@ set(RAC_EXPORTED_FUNCTIONS_BASE "_rac_framework_supports_tts" "_rac_model_format_extension" + # Framework <-> OPFS directory resolution. FrameworkOPFSPaths.ts needs both + # to compute the per-framework model folder (e.g. "LlamaCpp") on the JS side + # for cold-start hydration; without them frameworkOPFSDir() returns null and + # hydrateModelRegistry() can never re-detect downloaded models after reload. + "_rac_inference_framework_from_proto" + "_rac_framework_raw_value" + # Model paths — base-dir configuration. Web has no native filesystem root, # but the C++ download orchestrator rejects an empty base via # `rac_model_paths_get_model_folder`. The TypeScript bridge installs a @@ -895,6 +902,7 @@ set(RAC_EXPORTED_FUNCTIONS_BASE "_rac_auth_is_authenticated" "_rac_auth_get_user_id" "_rac_auth_get_organization_id" + "_rac_auth_get_access_token" "_rac_state_is_device_registered" # Telemetry @@ -1223,7 +1231,14 @@ function(rac_wasm_add_target) if(RAC_WASM_PTHREADS) list(APPEND _link_flags "-pthread" - "-sPTHREAD_POOL_SIZE=4" + # Pre-spawn enough workers to cover the engine's max compute thread + # count (get_num_threads caps at 8). VLM inference runs synchronously + # on the main thread; if ggml's threadpool needs to spawn a worker + # on-demand mid-encode, the blocked main thread can never service the + # Worker creation → mtmd_helper_eval_chunks deadlocks. Sizing the + # pool to 8 means all compute threads are already live, so no + # on-demand spawn happens during the blocked call. + "-sPTHREAD_POOL_SIZE=8" "-sPTHREAD_POOL_SIZE_STRICT=0" ) target_compile_options(${RWT_NAME} PRIVATE "-pthread") diff --git a/sdk/runanywhere-web/wasm/scripts/vendor-onnxruntime-wasm.sh b/sdk/runanywhere-web/wasm/scripts/vendor-onnxruntime-wasm.sh index eb1762a434..f6eba570d0 100755 --- a/sdk/runanywhere-web/wasm/scripts/vendor-onnxruntime-wasm.sh +++ b/sdk/runanywhere-web/wasm/scripts/vendor-onnxruntime-wasm.sh @@ -20,10 +20,51 @@ ONNX_RUNTIME_VERSION="${ONNX_RUNTIME_VERSION:-${ONNX_VERSION_LINUX}}" SRC_DIR="${ONNX_RUNTIME_SRC_DIR:-${WASM_DIR}/third_party/onnxruntime}" DEST_DIR="${REPO_ROOT}/sdk/runanywhere-commons/third_party/onnxruntime-wasm" BUILD_CONFIG="${ONNX_RUNTIME_BUILD_CONFIG:-Release}" -ORT_BUILD_DIR="${SRC_DIR}/build/MacOS/${BUILD_CONFIG}" +case "$(uname -s)" in + Darwin) _ORT_OS_DIR="MacOS" ;; + *) _ORT_OS_DIR="Linux" ;; +esac +ORT_BUILD_DIR="${SRC_DIR}/build/${_ORT_OS_DIR}/${BUILD_CONFIG}" mkdir -p "$(dirname "${SRC_DIR}")" "${DEST_DIR}/lib" "${DEST_DIR}/include" +# --- Prebuilt WASM bundle (download-first; mirrors the Android prebuilt .so) --- +# The matched ORT+sherpa WASM static libs are published on the sherpa-onnx-rac +# release. Download + extract instead of the ~30-60 min from-source build. +# Force a source build with RAC_WASM_BUILD_FROM_SOURCE=1; override the source +# repo/tag with RAC_WASM_PREBUILT_REPO / RAC_WASM_PREBUILT_TAG. +if [ "${RAC_WASM_BUILD_FROM_SOURCE:-0}" != "1" ]; then + if [ -f "${DEST_DIR}/lib/libonnxruntime.a" ]; then + echo "ONNX Runtime WASM already vendored: ${DEST_DIR}/lib/libonnxruntime.a" + exit 0 + fi + _RAC_TP="${REPO_ROOT}/sdk/runanywhere-commons/third_party" + _RAC_REPO="${RAC_WASM_PREBUILT_REPO:-${SHERPA_ONNX_REPO_ANDROID:-Siddhesh2377/sherpa-onnx-rac}}" + _RAC_TAG="${RAC_WASM_PREBUILT_TAG:-v${SHERPA_ONNX_VERSION_LINUX}}" + _RAC_TARBALL="sherpa-onnx-${_RAC_TAG}-wasm.tar.bz2" + _RAC_URL="https://github.com/${_RAC_REPO}/releases/download/${_RAC_TAG}/${_RAC_TARBALL}" + _RAC_CACHE="${WASM_DIR}/third_party/${_RAC_TARBALL}" + if [ ! -f "${_RAC_CACHE}" ]; then + echo "Downloading prebuilt WASM bundle: ${_RAC_URL}" + if curl -fL --retry 3 -o "${_RAC_CACHE}.part" "${_RAC_URL}"; then + mv "${_RAC_CACHE}.part" "${_RAC_CACHE}" + else + echo "Prebuilt download failed; falling back to from-source build." + rm -f "${_RAC_CACHE}.part" + fi + fi + if [ -f "${_RAC_CACHE}" ]; then + mkdir -p "${_RAC_TP}" + tar -xjf "${_RAC_CACHE}" -C "${_RAC_TP}" onnxruntime-wasm sherpa-onnx-wasm + if [ -f "${DEST_DIR}/lib/libonnxruntime.a" ]; then + echo "Vendored ONNX Runtime WASM from prebuilt bundle: ${DEST_DIR}/lib/libonnxruntime.a" + exit 0 + fi + echo "Prebuilt extract did not produce libonnxruntime.a; falling back to from-source build." + fi +fi +# --- from-source build (reached if RAC_WASM_BUILD_FROM_SOURCE=1 or download failed) --- + if [ ! -d "${SRC_DIR}/.git" ]; then rm -rf "${SRC_DIR}" git clone --depth 1 --branch "v${ONNX_RUNTIME_VERSION}" \ @@ -66,6 +107,7 @@ set +e --enable_wasm_simd \ --skip_tests \ --disable_rtti \ + --parallel "${CMAKE_BUILD_PARALLEL_LEVEL:-12}" \ --cmake_extra_defines CMAKE_POLICY_VERSION_MINIMUM=3.5 BUILD_RC=$? set -e diff --git a/sdk/runanywhere-web/wasm/scripts/vendor-sherpa-onnx-wasm.sh b/sdk/runanywhere-web/wasm/scripts/vendor-sherpa-onnx-wasm.sh index d75d759daf..0c72e84a87 100755 --- a/sdk/runanywhere-web/wasm/scripts/vendor-sherpa-onnx-wasm.sh +++ b/sdk/runanywhere-web/wasm/scripts/vendor-sherpa-onnx-wasm.sh @@ -18,6 +18,43 @@ DEST_DIR="${REPO_ROOT}/sdk/runanywhere-commons/third_party/sherpa-onnx-wasm" ORT_DIR="${REPO_ROOT}/sdk/runanywhere-commons/third_party/onnxruntime-wasm" BUILD_DIR="${SRC_DIR}/build-wasm-static" +# --- Prebuilt WASM bundle (download-first; mirrors the Android prebuilt .so) --- +# Download + extract the matched ORT+sherpa WASM static libs from the +# sherpa-onnx-rac release instead of building from source. Force a source build +# with RAC_WASM_BUILD_FROM_SOURCE=1; override the source repo/tag with +# RAC_WASM_PREBUILT_REPO / RAC_WASM_PREBUILT_TAG. +if [ "${RAC_WASM_BUILD_FROM_SOURCE:-0}" != "1" ]; then + if [ -f "${DEST_DIR}/lib/libsherpa-onnx-c-api.a" ]; then + echo "Sherpa-ONNX WASM already vendored: ${DEST_DIR}/lib/libsherpa-onnx-c-api.a" + exit 0 + fi + _RAC_TP="${REPO_ROOT}/sdk/runanywhere-commons/third_party" + _RAC_REPO="${RAC_WASM_PREBUILT_REPO:-${SHERPA_ONNX_REPO_ANDROID:-Siddhesh2377/sherpa-onnx-rac}}" + _RAC_TAG="${RAC_WASM_PREBUILT_TAG:-v${SHERPA_ONNX_VERSION_LINUX}}" + _RAC_TARBALL="sherpa-onnx-${_RAC_TAG}-wasm.tar.bz2" + _RAC_URL="https://github.com/${_RAC_REPO}/releases/download/${_RAC_TAG}/${_RAC_TARBALL}" + _RAC_CACHE="${WASM_DIR}/third_party/${_RAC_TARBALL}" + if [ ! -f "${_RAC_CACHE}" ]; then + echo "Downloading prebuilt WASM bundle: ${_RAC_URL}" + if curl -fL --retry 3 -o "${_RAC_CACHE}.part" "${_RAC_URL}"; then + mv "${_RAC_CACHE}.part" "${_RAC_CACHE}" + else + echo "Prebuilt download failed; falling back to from-source build." + rm -f "${_RAC_CACHE}.part" + fi + fi + if [ -f "${_RAC_CACHE}" ]; then + mkdir -p "${_RAC_TP}" + tar -xjf "${_RAC_CACHE}" -C "${_RAC_TP}" onnxruntime-wasm sherpa-onnx-wasm + if [ -f "${DEST_DIR}/lib/libsherpa-onnx-c-api.a" ]; then + echo "Vendored Sherpa-ONNX WASM from prebuilt bundle: ${DEST_DIR}/lib/libsherpa-onnx-c-api.a" + exit 0 + fi + echo "Prebuilt extract did not produce libsherpa-onnx-c-api.a; falling back to from-source build." + fi +fi +# --- from-source build (reached if RAC_WASM_BUILD_FROM_SOURCE=1 or download failed) --- + if [ ! -f "${ORT_DIR}/lib/libonnxruntime.a" ]; then echo "ERROR: ${ORT_DIR}/lib/libonnxruntime.a is required before building Sherpa-ONNX WASM." >&2 echo "Run: sdk/runanywhere-web/wasm/scripts/vendor-onnxruntime-wasm.sh" >&2 diff --git a/yarn.lock b/yarn.lock index ee315d6d10..a8abec6c27 100644 --- a/yarn.lock +++ b/yarn.lock @@ -136,6 +136,28 @@ __metadata: languageName: node linkType: hard +"@babel/helper-annotate-as-pure@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/helper-annotate-as-pure@npm:7.29.7" + dependencies: + "@babel/types": ^7.29.7 + checksum: acd9e128de634a5144b5d622357d018fa616de45f64c74e42007c048dd15d0a0be213f4d5a2bf02307bdaddf053791b87900a99d183de828c08dc3b556329009 + languageName: node + linkType: hard + +"@babel/helper-compilation-targets@npm:^7.27.2, @babel/helper-compilation-targets@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/helper-compilation-targets@npm:7.29.7" + dependencies: + "@babel/compat-data": ^7.29.7 + "@babel/helper-validator-option": ^7.29.7 + browserslist: ^4.24.0 + lru-cache: ^5.1.1 + semver: ^6.3.1 + checksum: f60a943937f4eba0e671aa28551cb45569fd081c1e30a52ede167860475dc0417f3dbdf2a0fa3f086965595c7070aa76308da60cc0319860de05db4ed2a431f7 + languageName: node + linkType: hard + "@babel/helper-compilation-targets@npm:^7.28.6": version: 7.28.6 resolution: "@babel/helper-compilation-targets@npm:7.28.6" @@ -149,16 +171,20 @@ __metadata: languageName: node linkType: hard -"@babel/helper-compilation-targets@npm:^7.29.7": +"@babel/helper-create-class-features-plugin@npm:^7.27.1, @babel/helper-create-class-features-plugin@npm:^7.29.7": version: 7.29.7 - resolution: "@babel/helper-compilation-targets@npm:7.29.7" + resolution: "@babel/helper-create-class-features-plugin@npm:7.29.7" dependencies: - "@babel/compat-data": ^7.29.7 - "@babel/helper-validator-option": ^7.29.7 - browserslist: ^4.24.0 - lru-cache: ^5.1.1 + "@babel/helper-annotate-as-pure": ^7.29.7 + "@babel/helper-member-expression-to-functions": ^7.29.7 + "@babel/helper-optimise-call-expression": ^7.29.7 + "@babel/helper-replace-supers": ^7.29.7 + "@babel/helper-skip-transparent-expression-wrappers": ^7.29.7 + "@babel/traverse": ^7.29.7 semver: ^6.3.1 - checksum: f60a943937f4eba0e671aa28551cb45569fd081c1e30a52ede167860475dc0417f3dbdf2a0fa3f086965595c7070aa76308da60cc0319860de05db4ed2a431f7 + peerDependencies: + "@babel/core": ^7.0.0 + checksum: c954e4bfe423a277cdcbad64344637cf696ddcd80085fce5f284b02a0c700af0d2d7b61468a06d9e296e948de60ece138921b65564aec023a9d9594f5d9fe18d languageName: node linkType: hard @@ -192,6 +218,19 @@ __metadata: languageName: node linkType: hard +"@babel/helper-create-regexp-features-plugin@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/helper-create-regexp-features-plugin@npm:7.29.7" + dependencies: + "@babel/helper-annotate-as-pure": ^7.29.7 + regexpu-core: ^6.3.1 + semver: ^6.3.1 + peerDependencies: + "@babel/core": ^7.0.0 + checksum: 702a34db6c064a2c26675b717b3af88b8acaeb5341e2285792a873a67a16ec4cbe987ba72e055db198b2e03ead05600d8c9c0c1e7436708e95865bbfb3516026 + languageName: node + linkType: hard + "@babel/helper-define-polyfill-provider@npm:^0.6.5, @babel/helper-define-polyfill-provider@npm:^0.6.8": version: 0.6.8 resolution: "@babel/helper-define-polyfill-provider@npm:0.6.8" @@ -231,6 +270,16 @@ __metadata: languageName: node linkType: hard +"@babel/helper-member-expression-to-functions@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/helper-member-expression-to-functions@npm:7.29.7" + dependencies: + "@babel/traverse": ^7.29.7 + "@babel/types": ^7.29.7 + checksum: 79d5f095b4bafadff3d1ec316d9f17ec85940fece957db62dd523b08e142da73c53180d1bce2fdc4d523e3889bb01b39bea70ad968b6e2f4dbdc66ebd1055b8a + languageName: node + linkType: hard + "@babel/helper-module-imports@npm:^7.28.6": version: 7.28.6 resolution: "@babel/helper-module-imports@npm:7.28.6" @@ -286,6 +335,15 @@ __metadata: languageName: node linkType: hard +"@babel/helper-optimise-call-expression@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/helper-optimise-call-expression@npm:7.29.7" + dependencies: + "@babel/types": ^7.29.7 + checksum: 6b477e01b403fd48349336cb1d94722bff4fa54af2841b5fa950c557b796f4ecc14724052252ed1362ccfc23d1c09c54dc03e182fea59d3dc5bd69f8c626ba25 + languageName: node + linkType: hard + "@babel/helper-plugin-utils@npm:^7.0.0, @babel/helper-plugin-utils@npm:^7.10.4, @babel/helper-plugin-utils@npm:^7.12.13, @babel/helper-plugin-utils@npm:^7.14.5, @babel/helper-plugin-utils@npm:^7.29.7": version: 7.29.7 resolution: "@babel/helper-plugin-utils@npm:7.29.7" @@ -313,6 +371,19 @@ __metadata: languageName: node linkType: hard +"@babel/helper-replace-supers@npm:^7.27.1, @babel/helper-replace-supers@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/helper-replace-supers@npm:7.29.7" + dependencies: + "@babel/helper-member-expression-to-functions": ^7.29.7 + "@babel/helper-optimise-call-expression": ^7.29.7 + "@babel/traverse": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0 + checksum: f7eb9a6b035d9d45250c880eb09605f95998998e828ec90759ed45764fe0abeee583419969de8b8dee551163dff914c9fc6ace90c1d819c56c23146f8df525db + languageName: node + linkType: hard + "@babel/helper-replace-supers@npm:^7.28.6": version: 7.28.6 resolution: "@babel/helper-replace-supers@npm:7.28.6" @@ -336,6 +407,16 @@ __metadata: languageName: node linkType: hard +"@babel/helper-skip-transparent-expression-wrappers@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/helper-skip-transparent-expression-wrappers@npm:7.29.7" + dependencies: + "@babel/traverse": ^7.29.7 + "@babel/types": ^7.29.7 + checksum: a5800bfcdca6cef7f6fe33ac02a0f05ff33da9746f97806553f249733f7ba8400290a17f3831d7faa5d91656f254ab749931f53c8a29f301d958d7dd00499637 + languageName: node + linkType: hard + "@babel/helper-string-parser@npm:^7.27.1": version: 7.27.1 resolution: "@babel/helper-string-parser@npm:7.27.1" @@ -552,25 +633,25 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-syntax-jsx@npm:^7.28.6": - version: 7.28.6 - resolution: "@babel/plugin-syntax-jsx@npm:7.28.6" +"@babel/plugin-syntax-jsx@npm:^7.27.1, @babel/plugin-syntax-jsx@npm:^7.29.7, @babel/plugin-syntax-jsx@npm:^7.7.2": + version: 7.29.7 + resolution: "@babel/plugin-syntax-jsx@npm:7.29.7" dependencies: - "@babel/helper-plugin-utils": ^7.28.6 + "@babel/helper-plugin-utils": ^7.29.7 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 572e38f5c1bb4b8124300e7e3dd13e82ae84a21f90d3f0786c98cd05e63c78ca1f32d1cfe462dfbaf5e7d5102fa7cd8fd741dfe4f3afc2e01a3b2877dcc8c866 + checksum: 84150d27c553a1d3d921354437f6725ca1d63b49514c25591bfcaaafa6ea4d6c10715b66fe7245e4ad7ab7c6cf4b6e1de7373defd3df00877ab12638170d7772 languageName: node linkType: hard -"@babel/plugin-syntax-jsx@npm:^7.7.2": - version: 7.29.7 - resolution: "@babel/plugin-syntax-jsx@npm:7.29.7" +"@babel/plugin-syntax-jsx@npm:^7.28.6": + version: 7.28.6 + resolution: "@babel/plugin-syntax-jsx@npm:7.28.6" dependencies: - "@babel/helper-plugin-utils": ^7.29.7 + "@babel/helper-plugin-utils": ^7.28.6 peerDependencies: "@babel/core": ^7.0.0-0 - checksum: 84150d27c553a1d3d921354437f6725ca1d63b49514c25591bfcaaafa6ea4d6c10715b66fe7245e4ad7ab7c6cf4b6e1de7373defd3df00877ab12638170d7772 + checksum: 572e38f5c1bb4b8124300e7e3dd13e82ae84a21f90d3f0786c98cd05e63c78ca1f32d1cfe462dfbaf5e7d5102fa7cd8fd741dfe4f3afc2e01a3b2877dcc8c866 languageName: node linkType: hard @@ -673,7 +754,7 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-syntax-typescript@npm:^7.7.2": +"@babel/plugin-syntax-typescript@npm:^7.29.7, @babel/plugin-syntax-typescript@npm:^7.7.2": version: 7.29.7 resolution: "@babel/plugin-syntax-typescript@npm:7.29.7" dependencies: @@ -684,6 +765,28 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-arrow-functions@npm:7.27.1": + version: 7.27.1 + resolution: "@babel/plugin-transform-arrow-functions@npm:7.27.1" + dependencies: + "@babel/helper-plugin-utils": ^7.27.1 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 62c2cc0ae2093336b1aa1376741c5ed245c0987d9e4b4c5313da4a38155509a7098b5acce582b6781cc0699381420010da2e3086353344abe0a6a0ec38961eb7 + languageName: node + linkType: hard + +"@babel/plugin-transform-arrow-functions@npm:^7.27.1": + version: 7.29.7 + resolution: "@babel/plugin-transform-arrow-functions@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 0037fd7563c7c91cddb8ce104e270bc260190d29c7a297df65e4306471010b4343366de13fab4602cf8ee6c672d3b313b34a01b62f27a333cad16908c83368d8 + languageName: node + linkType: hard + "@babel/plugin-transform-async-generator-functions@npm:^7.25.4": version: 7.29.0 resolution: "@babel/plugin-transform-async-generator-functions@npm:7.29.0" @@ -721,6 +824,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-class-properties@npm:7.27.1": + version: 7.27.1 + resolution: "@babel/plugin-transform-class-properties@npm:7.27.1" + dependencies: + "@babel/helper-create-class-features-plugin": ^7.27.1 + "@babel/helper-plugin-utils": ^7.27.1 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 475a6e5a9454912fe1bdc171941976ca10ea4e707675d671cdb5ce6b6761d84d1791ac61b6bca81a2e5f6430cb7b9d8e4b2392404110e69c28207a754e196294 + languageName: node + linkType: hard + "@babel/plugin-transform-class-properties@npm:^7.25.4": version: 7.28.6 resolution: "@babel/plugin-transform-class-properties@npm:7.28.6" @@ -733,6 +848,34 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-class-properties@npm:^7.28.6": + version: 7.29.7 + resolution: "@babel/plugin-transform-class-properties@npm:7.29.7" + dependencies: + "@babel/helper-create-class-features-plugin": ^7.29.7 + "@babel/helper-plugin-utils": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 0cc5e7a882e29eead360f02ef79f6b2ec3b3813213b1513d8fdaa931d1d1361fccc92fbacc9b399e42495953d9d6fc722f283b5f3aa272fe016a0b5fe1e6a130 + languageName: node + linkType: hard + +"@babel/plugin-transform-classes@npm:7.28.4": + version: 7.28.4 + resolution: "@babel/plugin-transform-classes@npm:7.28.4" + dependencies: + "@babel/helper-annotate-as-pure": ^7.27.3 + "@babel/helper-compilation-targets": ^7.27.2 + "@babel/helper-globals": ^7.28.0 + "@babel/helper-plugin-utils": ^7.27.1 + "@babel/helper-replace-supers": ^7.27.1 + "@babel/traverse": ^7.28.4 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: f412e00c86584a9094cc0a2f3dd181b8108a4dced477d609c5406beddd5bf79d05a7ea74db508dc4dcb37172f042d5ef98d3d6311ade61c7ea6fbbbb70f5ec29 + languageName: node + linkType: hard + "@babel/plugin-transform-classes@npm:^7.25.4": version: 7.28.6 resolution: "@babel/plugin-transform-classes@npm:7.28.6" @@ -749,6 +892,22 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-classes@npm:^7.28.6": + version: 7.29.7 + resolution: "@babel/plugin-transform-classes@npm:7.29.7" + dependencies: + "@babel/helper-annotate-as-pure": ^7.29.7 + "@babel/helper-compilation-targets": ^7.29.7 + "@babel/helper-globals": ^7.29.7 + "@babel/helper-plugin-utils": ^7.29.7 + "@babel/helper-replace-supers": ^7.29.7 + "@babel/traverse": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: cc45ff5b5b063339131cd701266af90506db8640e56494de0c78f882da6317f057951bf6507c5b48a0278cf5dad21691baf2af05e24b8c26b0725d677088edbf + languageName: node + linkType: hard + "@babel/plugin-transform-destructuring@npm:^7.24.8": version: 7.28.5 resolution: "@babel/plugin-transform-destructuring@npm:7.28.5" @@ -797,6 +956,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-modules-commonjs@npm:^7.27.1, @babel/plugin-transform-modules-commonjs@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-modules-commonjs@npm:7.29.7" + dependencies: + "@babel/helper-module-transforms": ^7.29.7 + "@babel/helper-plugin-utils": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: e9d8102deca4c2f06507a8d071ca0e161f01cec0e108151142edb39995bd830e312d55d21fac0ceb390ac79a1fee5fb768b719413b8fc5adda5f06ecd8845cd6 + languageName: node + linkType: hard + "@babel/plugin-transform-named-capturing-groups-regex@npm:^7.24.7": version: 7.29.0 resolution: "@babel/plugin-transform-named-capturing-groups-regex@npm:7.29.0" @@ -809,6 +980,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-nullish-coalescing-operator@npm:7.27.1": + version: 7.27.1 + resolution: "@babel/plugin-transform-nullish-coalescing-operator@npm:7.27.1" + dependencies: + "@babel/helper-plugin-utils": ^7.27.1 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 1c6b3730748782d2178cc30f5cc37be7d7666148260f3f2dfc43999908bdd319bdfebaaf19cf04ac1f9dee0f7081093d3fa730cda5ae1b34bcd73ce406a78be7 + languageName: node + linkType: hard + "@babel/plugin-transform-nullish-coalescing-operator@npm:^7.24.7": version: 7.28.6 resolution: "@babel/plugin-transform-nullish-coalescing-operator@npm:7.28.6" @@ -820,6 +1002,17 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-nullish-coalescing-operator@npm:^7.28.6": + version: 7.29.7 + resolution: "@babel/plugin-transform-nullish-coalescing-operator@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: f486e737ddcec3a88e2e0dc004c28e15156dd255f3b2d944903f3d75666257c96ee7ebf57d80d2cf42eda2bee497db8134a26d657ddc3defcc34b9583ecbd119 + languageName: node + linkType: hard + "@babel/plugin-transform-optional-catch-binding@npm:^7.24.7": version: 7.28.6 resolution: "@babel/plugin-transform-optional-catch-binding@npm:7.28.6" @@ -831,6 +1024,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-optional-chaining@npm:7.27.1": + version: 7.27.1 + resolution: "@babel/plugin-transform-optional-chaining@npm:7.27.1" + dependencies: + "@babel/helper-plugin-utils": ^7.27.1 + "@babel/helper-skip-transparent-expression-wrappers": ^7.27.1 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: c4428d31f182d724db6f10575669aad3dbccceb0dea26aa9071fa89f11b3456278da3097fcc78937639a13c105a82cd452dc0218ce51abdbcf7626a013b928a5 + languageName: node + linkType: hard + "@babel/plugin-transform-optional-chaining@npm:^7.24.8": version: 7.28.6 resolution: "@babel/plugin-transform-optional-chaining@npm:7.28.6" @@ -843,6 +1048,18 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-optional-chaining@npm:^7.28.6": + version: 7.29.7 + resolution: "@babel/plugin-transform-optional-chaining@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": ^7.29.7 + "@babel/helper-skip-transparent-expression-wrappers": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 6255d259bb0143f7938278ebb382aba22311c260919d3f08476509c76335a111b479b3ad7363e86de064f87af85ca4d7f03f08195f0646c525ea70fb587c0a2d + languageName: node + linkType: hard + "@babel/plugin-transform-private-methods@npm:^7.24.7": version: 7.28.6 resolution: "@babel/plugin-transform-private-methods@npm:7.28.6" @@ -943,6 +1160,50 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-shorthand-properties@npm:7.27.1": + version: 7.27.1 + resolution: "@babel/plugin-transform-shorthand-properties@npm:7.27.1" + dependencies: + "@babel/helper-plugin-utils": ^7.27.1 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: fbba6e2aef0b69681acb68202aa249c0598e470cc0853d7ff5bd0171fd6a7ec31d77cfabcce9df6360fc8349eded7e4a65218c32551bd3fc0caaa1ac899ac6d4 + languageName: node + linkType: hard + +"@babel/plugin-transform-shorthand-properties@npm:^7.27.1": + version: 7.29.7 + resolution: "@babel/plugin-transform-shorthand-properties@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: c57ef27853f334a6147da9aa00f8a8f4c3a1c217eb2efa73cba2e118edda10754fa23cec2c0c7f7408279ad28fef92c1f663dfec137a7503813331569c3e02f9 + languageName: node + linkType: hard + +"@babel/plugin-transform-template-literals@npm:7.27.1": + version: 7.27.1 + resolution: "@babel/plugin-transform-template-literals@npm:7.27.1" + dependencies: + "@babel/helper-plugin-utils": ^7.27.1 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 93aad782503b691faef7c0893372d5243df3219b07f1f22cfc32c104af6a2e7acd6102c128439eab15336d048f1b214ca134b87b0630d8cd568bf447f78b25ce + languageName: node + linkType: hard + +"@babel/plugin-transform-template-literals@npm:^7.27.1": + version: 7.29.7 + resolution: "@babel/plugin-transform-template-literals@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: d1014dab020f0f802089de17bba82d929eda6ac87fde5f58fb9763885b8d645ce63fc1df97055c06a12d95a8788334a93b559fcaf1da6d7777a191e5f9c5646e + languageName: node + linkType: hard + "@babel/plugin-transform-typescript@npm:^7.25.2": version: 7.28.6 resolution: "@babel/plugin-transform-typescript@npm:7.28.6" @@ -958,7 +1219,22 @@ __metadata: languageName: node linkType: hard -"@babel/plugin-transform-unicode-regex@npm:^7.24.7": +"@babel/plugin-transform-typescript@npm:^7.27.1, @babel/plugin-transform-typescript@npm:^7.29.7": + version: 7.29.7 + resolution: "@babel/plugin-transform-typescript@npm:7.29.7" + dependencies: + "@babel/helper-annotate-as-pure": ^7.29.7 + "@babel/helper-create-class-features-plugin": ^7.29.7 + "@babel/helper-plugin-utils": ^7.29.7 + "@babel/helper-skip-transparent-expression-wrappers": ^7.29.7 + "@babel/plugin-syntax-typescript": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: e95bce53fa2add836eec5ef5221e260cfa4ab889a146f7ba5e29cbd42bfe3183cb94e40b49bfb0d14a75f233982723903d3efad0f528b835ce771e38bd365440 + languageName: node + linkType: hard + +"@babel/plugin-transform-unicode-regex@npm:7.27.1, @babel/plugin-transform-unicode-regex@npm:^7.24.7": version: 7.27.1 resolution: "@babel/plugin-transform-unicode-regex@npm:7.27.1" dependencies: @@ -970,6 +1246,48 @@ __metadata: languageName: node linkType: hard +"@babel/plugin-transform-unicode-regex@npm:^7.27.1": + version: 7.29.7 + resolution: "@babel/plugin-transform-unicode-regex@npm:7.29.7" + dependencies: + "@babel/helper-create-regexp-features-plugin": ^7.29.7 + "@babel/helper-plugin-utils": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 1ade0672ae5bbbf2ec1ea0a8de1b5d804ae414283215620097ab21cf7f05dae8916f5b0548a18c6f080ec17135018f5edd2d38f8fa9ca052af570cab5c712786 + languageName: node + linkType: hard + +"@babel/preset-typescript@npm:7.27.1": + version: 7.27.1 + resolution: "@babel/preset-typescript@npm:7.27.1" + dependencies: + "@babel/helper-plugin-utils": ^7.27.1 + "@babel/helper-validator-option": ^7.27.1 + "@babel/plugin-syntax-jsx": ^7.27.1 + "@babel/plugin-transform-modules-commonjs": ^7.27.1 + "@babel/plugin-transform-typescript": ^7.27.1 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: 38020f1b23e88ec4fbffd5737da455d8939244bddfb48a2516aef93fb5947bd9163fb807ce6eff3e43fa5ffe9113aa131305fef0fb5053998410bbfcfe6ce0ec + languageName: node + linkType: hard + +"@babel/preset-typescript@npm:^7.28.5": + version: 7.29.7 + resolution: "@babel/preset-typescript@npm:7.29.7" + dependencies: + "@babel/helper-plugin-utils": ^7.29.7 + "@babel/helper-validator-option": ^7.29.7 + "@babel/plugin-syntax-jsx": ^7.29.7 + "@babel/plugin-transform-modules-commonjs": ^7.29.7 + "@babel/plugin-transform-typescript": ^7.29.7 + peerDependencies: + "@babel/core": ^7.0.0-0 + checksum: f2f58cbbbdb84f6b27e20c7835fbd1e2474e7e6075c97a6609c606139bc782c995f0c5eb5b64f5b613997f8a463f3b4cea10cd1f0d706a66bf1aa41fea666725 + languageName: node + linkType: hard + "@babel/runtime@npm:^7.25.0, @babel/runtime@npm:^7.28.6": version: 7.29.2 resolution: "@babel/runtime@npm:7.29.2" @@ -1014,7 +1332,7 @@ __metadata: languageName: node linkType: hard -"@babel/traverse@npm:^7.29.7": +"@babel/traverse@npm:^7.28.4, @babel/traverse@npm:^7.29.7": version: 7.29.7 resolution: "@babel/traverse@npm:7.29.7" dependencies: @@ -1151,6 +1469,40 @@ __metadata: languageName: node linkType: hard +"@gorhom/bottom-sheet@npm:^5.2.14": + version: 5.2.14 + resolution: "@gorhom/bottom-sheet@npm:5.2.14" + dependencies: + "@gorhom/portal": 1.0.14 + invariant: ^2.2.4 + peerDependencies: + "@types/react": "*" + "@types/react-native": "*" + react: "*" + react-native: "*" + react-native-gesture-handler: ">=2.16.1" + react-native-reanimated: "*" + peerDependenciesMeta: + "@types/react": + optional: true + "@types/react-native": + optional: true + checksum: c4387fa66b06e9c5dac713af7cdc9f9f5da42743f7dd6a2117d8c956e9410573c24e552fdb20232e251de5890ee83569894312eb5d4531f933f512b673ffeb53 + languageName: node + linkType: hard + +"@gorhom/portal@npm:1.0.14": + version: 1.0.14 + resolution: "@gorhom/portal@npm:1.0.14" + dependencies: + nanoid: ^3.3.1 + peerDependencies: + react: "*" + react-native: "*" + checksum: 227bb96a2db854ab29bb9da8d4f3823c7f7448358de459709dd1b78522110da564c9a8734c6bc7d7153ed7c99320e0fb5d60b420c2ebb75ecaf2f0d757f410f9 + languageName: node + linkType: hard + "@hapi/hoek@npm:^9.0.0, @hapi/hoek@npm:^9.3.0": version: 9.3.0 resolution: "@hapi/hoek@npm:9.3.0" @@ -2397,18 +2749,6 @@ __metadata: languageName: unknown linkType: soft -"@runanywhere/genie@npm:^0.1.1": - version: 0.1.1 - resolution: "@runanywhere/genie@npm:0.1.1" - peerDependencies: - "@runanywhere/core": ">=0.16.0" - react: ">=18.0.0" - react-native: ">=0.74.0" - react-native-nitro-modules: ">=0.31.3" - checksum: d4f24df74106cbb1677b0db478f6f912cfb813893ab274f9bf09f4803b95d71c83a4de69ce0c387c0a9921dbfae2050a5bdfb0b68c5ff71acc63d69ce545648d - languageName: node - linkType: hard - "@runanywhere/llamacpp@workspace:*, @runanywhere/llamacpp@workspace:sdk/runanywhere-react-native/packages/llamacpp": version: 0.0.0-use.local resolution: "@runanywhere/llamacpp@workspace:sdk/runanywhere-react-native/packages/llamacpp" @@ -2645,6 +2985,15 @@ __metadata: languageName: node linkType: hard +"@types/react-test-renderer@npm:^19.1.0": + version: 19.1.0 + resolution: "@types/react-test-renderer@npm:19.1.0" + dependencies: + "@types/react": "*" + checksum: 2ef3aec0f2fd638902cda606d70c8531d66f8e8944334427986b99dcac9755ee60b700c5c3a19ac354680f9c45669e98077b84f79cac60e950bdb7d38aebffde + languageName: node + linkType: hard + "@types/react@npm:*, @types/react@npm:~19.2.14": version: 19.2.14 resolution: "@types/react@npm:19.2.14" @@ -3481,6 +3830,13 @@ __metadata: languageName: node linkType: hard +"boolbase@npm:^1.0.0": + version: 1.0.0 + resolution: "boolbase@npm:1.0.0" + checksum: 3e25c80ef626c3a3487c73dbfc70ac322ec830666c9ad915d11b701142fab25ec1e63eff2c450c74347acfd2de854ccde865cd79ef4db1683f7c7b046ea43bb0 + languageName: node + linkType: hard + "brace-expansion@npm:^1.1.7": version: 1.1.14 resolution: "brace-expansion@npm:1.1.14" @@ -3930,7 +4286,7 @@ __metadata: languageName: node linkType: hard -"convert-source-map@npm:^2.0.0": +"convert-source-map@npm:2.0.0, convert-source-map@npm:^2.0.0": version: 2.0.0 resolution: "convert-source-map@npm:2.0.0" checksum: 63ae9933be5a2b8d4509daca5124e20c14d023c820258e484e32dc324d34c2754e71297c94a05784064ad27615037ef677e3f0c00469fb55f409d2bb21261035 @@ -3991,6 +4347,36 @@ __metadata: languageName: node linkType: hard +"css-select@npm:^5.1.0": + version: 5.2.2 + resolution: "css-select@npm:5.2.2" + dependencies: + boolbase: ^1.0.0 + css-what: ^6.1.0 + domhandler: ^5.0.2 + domutils: ^3.0.1 + nth-check: ^2.0.1 + checksum: 0ab672620c6bdfe4129dfecf202f6b90f92018b24a1a93cfbb295c24026d0163130ba4b98d7443f87246a2c1d67413798a7a5920cd102b0cfecfbc89896515aa + languageName: node + linkType: hard + +"css-tree@npm:^1.1.3": + version: 1.1.3 + resolution: "css-tree@npm:1.1.3" + dependencies: + mdn-data: 2.0.14 + source-map: ^0.6.1 + checksum: 79f9b81803991b6977b7fcb1588799270438274d89066ce08f117f5cdb5e20019b446d766c61506dd772c839df84caa16042d6076f20c97187f5abe3b50e7d1f + languageName: node + linkType: hard + +"css-what@npm:^6.1.0": + version: 6.2.2 + resolution: "css-what@npm:6.2.2" + checksum: 4d1f07b348a638e1f8b4c72804a1e93881f35e0f541256aec5ac0497c5855df7db7ab02da030de950d4813044f6d029a14ca657e0f92c3987e4b604246235b2b + languageName: node + linkType: hard + "csstype@npm:^3.2.2": version: 3.2.3 resolution: "csstype@npm:3.2.3" @@ -4167,6 +4553,44 @@ __metadata: languageName: node linkType: hard +"dom-serializer@npm:^2.0.0": + version: 2.0.0 + resolution: "dom-serializer@npm:2.0.0" + dependencies: + domelementtype: ^2.3.0 + domhandler: ^5.0.2 + entities: ^4.2.0 + checksum: cd1810544fd8cdfbd51fa2c0c1128ec3a13ba92f14e61b7650b5de421b88205fd2e3f0cc6ace82f13334114addb90ed1c2f23074a51770a8e9c1273acbc7f3e6 + languageName: node + linkType: hard + +"domelementtype@npm:^2.3.0": + version: 2.3.0 + resolution: "domelementtype@npm:2.3.0" + checksum: ee837a318ff702622f383409d1f5b25dd1024b692ef64d3096ff702e26339f8e345820f29a68bcdcea8cfee3531776b3382651232fbeae95612d6f0a75efb4f6 + languageName: node + linkType: hard + +"domhandler@npm:^5.0.2, domhandler@npm:^5.0.3": + version: 5.0.3 + resolution: "domhandler@npm:5.0.3" + dependencies: + domelementtype: ^2.3.0 + checksum: 0f58f4a6af63e6f3a4320aa446d28b5790a009018707bce2859dcb1d21144c7876482b5188395a188dfa974238c019e0a1e610d2fc269a12b2c192ea2b0b131c + languageName: node + linkType: hard + +"domutils@npm:^3.0.1": + version: 3.2.2 + resolution: "domutils@npm:3.2.2" + dependencies: + dom-serializer: ^2.0.0 + domelementtype: ^2.3.0 + domhandler: ^5.0.3 + checksum: ae941d56f03d857077d55dde9297e960a625229fc2b933187cc4123084d7c2d2517f58283a7336567127029f1e008449bac8ac8506d44341e29e3bb18e02f906 + languageName: node + linkType: hard + "dunder-proto@npm:^1.0.0, dunder-proto@npm:^1.0.1": version: 1.0.1 resolution: "dunder-proto@npm:1.0.1" @@ -4227,6 +4651,13 @@ __metadata: languageName: node linkType: hard +"entities@npm:^4.2.0": + version: 4.5.0 + resolution: "entities@npm:4.5.0" + checksum: 853f8ebd5b425d350bffa97dd6958143179a5938352ccae092c62d1267c4e392a039be1bae7d51b6e4ffad25f51f9617531fedf5237f15df302ccfb452cbf2d7 + languageName: node + linkType: hard + "env-paths@npm:^2.2.0, env-paths@npm:^2.2.1": version: 2.2.1 resolution: "env-paths@npm:2.2.1" @@ -6781,6 +7212,17 @@ __metadata: languageName: node linkType: hard +"lucide-react-native@npm:^1.21.0": + version: 1.21.0 + resolution: "lucide-react-native@npm:1.21.0" + peerDependencies: + react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-native: "*" + react-native-svg: ^12.0.0 || ^13.0.0 || ^14.0.0 || ^15.0.0 + checksum: b0fe09b47336a2839d82cb827c0629a2411c611f160a9e8be4f9a5260643e8edd249fc5e8e943d5fbd372839204a3b2a79584ea287c4ae2cffcb0a09fcbd1087 + languageName: node + linkType: hard + "make-dir@npm:^4.0.0": version: 4.0.0 resolution: "make-dir@npm:4.0.0" @@ -6820,6 +7262,13 @@ __metadata: languageName: node linkType: hard +"mdn-data@npm:2.0.14": + version: 2.0.14 + resolution: "mdn-data@npm:2.0.14" + checksum: 9d0128ed425a89f4cba8f787dca27ad9408b5cb1b220af2d938e2a0629d17d879a34d2cb19318bdb26c3f14c77dd5dfbae67211f5caaf07b61b1f2c5c8c7dc16 + languageName: node + linkType: hard + "media-typer@npm:^1.1.0": version: 1.1.0 resolution: "media-typer@npm:1.1.0" @@ -7212,6 +7661,15 @@ __metadata: languageName: node linkType: hard +"nanoid@npm:^3.3.1": + version: 3.3.13 + resolution: "nanoid@npm:3.3.13" + bin: + nanoid: bin/nanoid.cjs + checksum: 9e20048d50392eebe70ad3802c5e9396cfe6f70c990cdb18390acc038bae8d93c28f79b5d4680886b8f835af65154d5feca15d9fc00fc2b4cb899ee00594fef0 + languageName: node + linkType: hard + "nanoid@npm:^3.3.11": version: 3.3.11 resolution: "nanoid@npm:3.3.11" @@ -7358,6 +7816,15 @@ __metadata: languageName: node linkType: hard +"nth-check@npm:^2.0.1": + version: 2.1.1 + resolution: "nth-check@npm:2.1.1" + dependencies: + boolbase: ^1.0.0 + checksum: 5afc3dafcd1573b08877ca8e6148c52abd565f1d06b1eb08caf982e3fa289a82f2cae697ffb55b5021e146d60443f1590a5d6b944844e944714a5b549675bcd3 + languageName: node + linkType: hard + "nullthrows@npm:^1.1.1": version: 1.1.1 resolution: "nullthrows@npm:1.1.1" @@ -8002,6 +8469,20 @@ __metadata: languageName: node linkType: hard +"react-native-actions-sheet@npm:^10.1.2": + version: 10.1.2 + resolution: "react-native-actions-sheet@npm:10.1.2" + dependencies: + react-native-worklets: ^0.7.1 + peerDependencies: + react-native: "*" + react-native-gesture-handler: "*" + react-native-reanimated: "*" + react-native-safe-area-context: "*" + checksum: f7246035e6b3d534d05ea5309b109ace280f4cfa1ef00eef5c36cc37405834d11c0cd9b13ae9d83f30c9571bb24b46112d8f8e3a260431180a8a65559aa31186 + languageName: node + linkType: hard + "react-native-fs@npm:^2.20.0": version: 2.20.0 resolution: "react-native-fs@npm:2.20.0" @@ -8018,6 +8499,19 @@ __metadata: languageName: node linkType: hard +"react-native-gesture-handler@npm:^3.0.2": + version: 3.0.2 + resolution: "react-native-gesture-handler@npm:3.0.2" + dependencies: + "@types/react-test-renderer": ^19.1.0 + invariant: ^2.2.4 + peerDependencies: + react: "*" + react-native: "*" + checksum: 28505ddc9f30ad81952af033f49c66969332ab848421c27eb3161f3245a89d58afbb56acd21c9fcf85b561c09d63b3950c3401a43be19f970bcf990b90b17ad2 + languageName: node + linkType: hard + "react-native-image-picker@npm:^8.2.1": version: 8.2.1 resolution: "react-native-image-picker@npm:8.2.1" @@ -8028,6 +8522,16 @@ __metadata: languageName: node linkType: hard +"react-native-is-edge-to-edge@npm:^1.3.1": + version: 1.3.1 + resolution: "react-native-is-edge-to-edge@npm:1.3.1" + peerDependencies: + react: "*" + react-native: "*" + checksum: dc82d54e0bf8f89208a538bb0d14e4891af6efae27ed5b7b21be683a72c38c5219ab9be1ea9bd40aa1c905d481174e649d0b71aeceaa9946e6c707f251568282 + languageName: node + linkType: hard + "react-native-monorepo-config@npm:^0.3.0": version: 0.3.3 resolution: "react-native-monorepo-config@npm:0.3.3" @@ -8072,6 +8576,20 @@ __metadata: languageName: node linkType: hard +"react-native-reanimated@npm:^4.4.1": + version: 4.4.1 + resolution: "react-native-reanimated@npm:4.4.1" + dependencies: + react-native-is-edge-to-edge: ^1.3.1 + semver: ^7.7.3 + peerDependencies: + react: "*" + react-native: 0.83 - 0.86 + react-native-worklets: 0.9.x + checksum: 670c78009531fc572356739d9dad486e561faed52a3783fb27c08872cc7e774c625d8021d3fa545f5397e7e9a780b6192f26861a9a19818cb12bafb0216a1e66 + languageName: node + linkType: hard + "react-native-safe-area-context@npm:^5.6.2": version: 5.7.0 resolution: "react-native-safe-area-context@npm:5.7.0" @@ -8095,6 +8613,19 @@ __metadata: languageName: node linkType: hard +"react-native-svg@npm:^15.15.5": + version: 15.15.5 + resolution: "react-native-svg@npm:15.15.5" + dependencies: + css-select: ^5.1.0 + css-tree: ^1.1.3 + peerDependencies: + react: "*" + react-native: "*" + checksum: 78f5aa54150be40075af77bea0d01f0f25c942bb3a8114c25386426f6af114d3602e7ccae94da4ba6ff0f85f6087b544c7067ab3415dd600924995c730aa4a60 + languageName: node + linkType: hard + "react-native-vector-icons@npm:^10.3.0": version: 10.3.0 resolution: "react-native-vector-icons@npm:10.3.0" @@ -8130,6 +8661,54 @@ __metadata: languageName: node linkType: hard +"react-native-worklets@npm:^0.7.1": + version: 0.7.4 + resolution: "react-native-worklets@npm:0.7.4" + dependencies: + "@babel/plugin-transform-arrow-functions": 7.27.1 + "@babel/plugin-transform-class-properties": 7.27.1 + "@babel/plugin-transform-classes": 7.28.4 + "@babel/plugin-transform-nullish-coalescing-operator": 7.27.1 + "@babel/plugin-transform-optional-chaining": 7.27.1 + "@babel/plugin-transform-shorthand-properties": 7.27.1 + "@babel/plugin-transform-template-literals": 7.27.1 + "@babel/plugin-transform-unicode-regex": 7.27.1 + "@babel/preset-typescript": 7.27.1 + convert-source-map: 2.0.0 + semver: 7.7.3 + peerDependencies: + "@babel/core": "*" + react: "*" + react-native: "*" + checksum: eb92247e208c6ee641dd639319628213aa2f22249b2fb09d5b9b0ea8a13e7bbb906eabc9dfebfe00404eec5d3e233378cc4991b9a43565338a4f083a68b7bd5f + languageName: node + linkType: hard + +"react-native-worklets@npm:^0.9.2": + version: 0.9.2 + resolution: "react-native-worklets@npm:0.9.2" + dependencies: + "@babel/plugin-transform-arrow-functions": ^7.27.1 + "@babel/plugin-transform-class-properties": ^7.28.6 + "@babel/plugin-transform-classes": ^7.28.6 + "@babel/plugin-transform-nullish-coalescing-operator": ^7.28.6 + "@babel/plugin-transform-optional-chaining": ^7.28.6 + "@babel/plugin-transform-shorthand-properties": ^7.27.1 + "@babel/plugin-transform-template-literals": ^7.27.1 + "@babel/plugin-transform-unicode-regex": ^7.27.1 + "@babel/preset-typescript": ^7.28.5 + "@babel/types": ^7.27.1 + convert-source-map: ^2.0.0 + semver: ^7.7.4 + peerDependencies: + "@babel/core": "*" + "@react-native/metro-config": "*" + react: "*" + react-native: 0.83 - 0.86 + checksum: 98225e012aba6923c8f39c3c10cbac92de9b6e6dc30be247a8c473aeb8585eb725e51a5547d9d59a0232c652bef2a132401af543c9207de1d8325af77a5c76ce + languageName: node + linkType: hard + "react-native@npm:0.85.3": version: 0.85.3 resolution: "react-native@npm:0.85.3" @@ -8188,10 +8767,10 @@ __metadata: languageName: node linkType: hard -"react@npm:19.2.3": - version: 19.2.3 - resolution: "react@npm:19.2.3" - checksum: 506e369ae13cb46b7f303c0201aadf856642f482cdf5b1c3730c3a6d1762fd5a3ae1dd31196a4686bfbbe56456dcd0c48a4656c75cbcb45620e3028c54789ae9 +"react@npm:19.2.7": + version: 19.2.7 + resolution: "react@npm:19.2.7" + checksum: be94ec2f1779b27cac328ddae18e52933c2977a2425b67039229ead45c62d0363b66e894076236616c8d98821a9a760fcbab8a9da8ae9c4b2849dfd12cc32a46 languageName: node linkType: hard @@ -8428,6 +9007,7 @@ __metadata: "@babel/core": ^7.25.2 "@babel/runtime": ^7.28.6 "@bufbuild/protobuf": ^2.12.0 + "@gorhom/bottom-sheet": ^5.2.14 "@react-native-async-storage/async-storage": ^2.2.0 "@react-native-clipboard/clipboard": ^1.16.3 "@react-native-community/cli": ^20.1.1 @@ -8443,7 +9023,6 @@ __metadata: "@react-navigation/native": ^7.2.4 "@react-navigation/native-stack": ^7.12.0 "@runanywhere/core": "workspace:*" - "@runanywhere/genie": ^0.1.1 "@runanywhere/llamacpp": "workspace:*" "@runanywhere/onnx": "workspace:*" "@runanywhere/proto-ts": "workspace:*" @@ -8457,19 +9036,25 @@ __metadata: eslint-plugin-prettier: ^5.0.1 eslint-plugin-unused-imports: ^4.3.0 knip: ^5.76.0 + lucide-react-native: ^1.21.0 patch-package: ^8.0.1 prettier: ^3.8.3 - react: 19.2.3 + react: 19.2.7 react-native: 0.85.3 + react-native-actions-sheet: ^10.1.2 react-native-fs: ^2.20.0 + react-native-gesture-handler: ^3.0.2 react-native-image-picker: ^8.2.1 react-native-monorepo-config: ^0.3.0 react-native-nitro-modules: ^0.33.9 react-native-permissions: ^5.4.4 + react-native-reanimated: ^4.4.1 react-native-safe-area-context: ^5.6.2 react-native-screens: ~4.18.0 + react-native-svg: ^15.15.5 react-native-vector-icons: ^10.3.0 react-native-vision-camera: ^4.7.3 + react-native-worklets: ^0.9.2 typescript: ^5.9.3 zustand: ^5.0.13 languageName: unknown @@ -8536,6 +9121,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:7.7.3": + version: 7.7.3 + resolution: "semver@npm:7.7.3" + bin: + semver: bin/semver.js + checksum: f013a3ee4607857bcd3503b6ac1d80165f7f8ea94f5d55e2d3e33df82fce487aa3313b987abf9b39e0793c83c9fc67b76c36c067625141a9f6f704ae0ea18db2 + languageName: node + linkType: hard + "semver@npm:^6.3.0, semver@npm:^6.3.1": version: 6.3.1 resolution: "semver@npm:6.3.1" @@ -8563,6 +9157,15 @@ __metadata: languageName: node linkType: hard +"semver@npm:^7.7.4": + version: 7.8.4 + resolution: "semver@npm:7.8.4" + bin: + semver: bin/semver.js + checksum: 42404642f4b892bd95859c6e2c5777a2916d6e4f0d924441593dea0911cadf5d8761d41b4ca3701793818ecdc72982df5c6d6328cba88252a1f0ebf93e375463 + languageName: node + linkType: hard + "send@npm:~0.19.1": version: 0.19.2 resolution: "send@npm:0.19.2"