[RNE Rewrite] feat: add voice activity detection pipeline (#1249)

Port the VAD feature to the rewrite as a pure-TypeScript pipeline on top of
the core model.execute primitive (no new C++):

- src/extensions/speech/tasks/vad.ts: createVAD runner replicating the native
  FSMN-VAD algorithm (framing + Hann window + pre-emphasis, chunked inference,
  thresholding / min-duration / padding / merge). Segments are returned in
  seconds. Relies on the get_dynamic_dims relaxed input validation for the
  dynamic frame dimension; the fsmn-vad model is re-exported with it.
- src/extensions/speech/vadStreamer.ts: pure streaming state machine driving
  onSpeechBegin / onSpeechEnd over an accumulating buffer.
- src/hooks/useVAD.ts: hook wrapping createVAD + streamer lifecycle.
- Register models.vad.FSMN_VAD and export the speech extension.
- apps/speech: expo-router demo (mirrors apps/nlp) with a real-time mic VAD
  screen via react-native-audio-api.

Author: msluszniak

.cspell-wordlist.txt

@@ -278,3 +278,6 @@ binarization
 bugprone
 NOLINTNEXTLINE
 
+hann
+preemphasis
+coeff

apps/speech/app.json

@@ -0,0 +1,51 @@
+{
+  "expo": {
+    "name": "speech",
+    "slug": "speech",
+    "version": "1.0.0",
+    "orientation": "portrait",
+    "icon": "./assets/icons/icon.png",
+    "userInterfaceStyle": "light",
+    "newArchEnabled": true,
+    "scheme": "rne-speech",
+    "splash": {
+      "image": "./assets/icons/splash.png",
+      "resizeMode": "contain",
+      "backgroundColor": "#ffffff"
+    },
+    "ios": {
+      "supportsTablet": true,
+      "bundleIdentifier": "com.anonymous.speech",
+      "infoPlist": {
+        "NSMicrophoneUsageDescription": "This app uses the microphone to detect voice activity."
+      }
+    },
+    "android": {
+      "adaptiveIcon": {
+        "foregroundImage": "./assets/icons/adaptive-icon.png",
+        "backgroundColor": "#ffffff"
+      },
+      "package": "com.anonymous.speech",
+      "permissions": [
+        "android.permission.RECORD_AUDIO"
+      ]
+    },
+    "web": {
+      "favicon": "./assets/icons/favicon.png"
+    },
+    "plugins": [
+      "expo-router",
+      [
+        "expo-build-properties",
+        {
+          "android": {
+            "minSdkVersion": 26
+          },
+          "ios": {
+            "deploymentTarget": "17.0"
+          }
+        }
+      ]
+    ]
+  }
+}

apps/speech/app/_layout.tsx

@@ -0,0 +1,32 @@
+import { Drawer } from 'expo-router/drawer';
+import { ColorPalette } from '../theme';
+import React from 'react';
+
+export default function Layout() {
+  return (
+    <Drawer
+      screenOptions={{
+        drawerActiveTintColor: ColorPalette.primary,
+        drawerInactiveTintColor: '#888',
+        headerTintColor: ColorPalette.primary,
+        headerTitleStyle: { color: ColorPalette.primary },
+      }}
+    >
+      <Drawer.Screen
+        name="index"
+        options={{
+          drawerLabel: () => null,
+          title: 'Main Menu',
+          drawerItemStyle: { display: 'none' },
+        }}
+      />
+      <Drawer.Screen
+        name="vad/index"
+        options={{
+          drawerLabel: 'Voice Activity Detection',
+          title: 'Voice Activity Detection',
+        }}
+      />
+    </Drawer>
+  );
+}

apps/speech/app/index.tsx

@@ -0,0 +1,51 @@
+import { useRouter } from 'expo-router';
+import { View, Text, StyleSheet, TouchableOpacity } from 'react-native';
+import { ColorPalette } from '../theme';
+import ExecutorchLogo from '../assets/icons/executorch.svg';
+
+export default function Home() {
+  const router = useRouter();
+
+  return (
+    <View style={styles.container}>
+      <ExecutorchLogo width={64} height={64} />
+      <Text style={styles.headerText}>Select a demo</Text>
+      <View style={styles.buttonContainer}>
+        <TouchableOpacity style={styles.button} onPress={() => router.navigate('vad/')}>
+          <Text style={styles.buttonText}>Voice Activity Detection</Text>
+        </TouchableOpacity>
+      </View>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    justifyContent: 'center',
+    alignItems: 'center',
+    backgroundColor: '#fff',
+  },
+  headerText: {
+    fontSize: 18,
+    color: ColorPalette.strongPrimary,
+    margin: 20,
+  },
+  buttonContainer: {
+    width: '80%',
+    justifyContent: 'space-evenly',
+    marginBottom: 20,
+  },
+  button: {
+    backgroundColor: ColorPalette.strongPrimary,
+    borderRadius: 8,
+    padding: 14,
+    alignItems: 'center',
+    marginBottom: 12,
+  },
+  buttonText: {
+    color: 'white',
+    fontSize: 16,
+    fontWeight: '600',
+  },
+});

apps/speech/app/vad/index.tsx

@@ -0,0 +1,227 @@
+import React, { useEffect, useRef, useState } from 'react';
+import { View, Text, StyleSheet, ScrollView, Platform } from 'react-native';
+import { useVAD, models } from 'react-native-executorch';
+import { AudioManager, AudioRecorder } from 'react-native-audio-api';
+import DeviceInfo from 'react-native-device-info';
+
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { ModelStatus } from '../../components/ModelStatus';
+import { Button } from '../../components/Button';
+import { theme } from '../../theme';
+
+const SAMPLE_RATE = 16000;
+const isSimulator = DeviceInfo.isEmulatorSync();
+
+function VADContent() {
+  const { isReady, downloadProgress, error, stream, streamInsert, streamStop } = useVAD(
+    models.vad.FSMN_VAD
+  );
+
+  const [isStreaming, setIsStreaming] = useState(false);
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const [hasMicPermission, setHasMicPermission] = useState(false);
+  const [runError, setRunError] = useState<string | null>(null);
+  const [logs, setLogs] = useState<string[]>([]);
+
+  const recorder = useRef(new AudioRecorder());
+  const logScrollRef = useRef<ScrollView>(null);
+
+  const addLog = (message: string) => {
+    setLogs((prev) => [...prev, `${new Date().toLocaleTimeString()}: ${message}`]);
+  };
+
+  useEffect(() => {
+    AudioManager.setAudioSessionOptions({
+      iosCategory: 'playAndRecord',
+      iosMode: 'spokenAudio',
+      iosOptions: ['allowBluetoothHFP', 'defaultToSpeaker'],
+    });
+    AudioManager.requestRecordingPermissions().then((status) =>
+      setHasMicPermission(status === 'Granted')
+    );
+  }, []);
+
+  const handleStart = async () => {
+    if (isStreaming || !isReady) return;
+
+    if (!hasMicPermission) {
+      setRunError('Microphone permission denied. Please enable it in Settings.');
+      return;
+    }
+
+    setRunError(null);
+    setLogs([]);
+    setIsStreaming(true);
+    addLog('Starting VAD stream…');
+
+    recorder.current.onAudioReady(
+      { sampleRate: SAMPLE_RATE, bufferLength: 1600, channelCount: 1 },
+      ({ buffer }) => streamInsert(buffer.getChannelData(0))
+    );
+
+    try {
+      await AudioManager.setAudioSessionActivity(true);
+      const started = await recorder.current.start();
+      if (started.status === 'error') {
+        throw new Error(started.message);
+      }
+
+      await stream({
+        onSpeechBegin: () => {
+          setIsSpeaking(true);
+          addLog('Speech detected (begin)');
+        },
+        onSpeechEnd: () => {
+          setIsSpeaking(false);
+          addLog('Silence detected (end)');
+        },
+        options: { timeout: 100, detectionMargin: 300 },
+      });
+    } catch (e) {
+      setRunError(e instanceof Error ? e.message : String(e));
+      setIsStreaming(false);
+    }
+  };
+
+  const handleStop = async () => {
+    await recorder.current.stop();
+    streamStop();
+    setIsStreaming(false);
+    setIsSpeaking(false);
+    addLog('VAD stream stopped');
+  };
+
+  const streamDisabled = isSimulator || !isReady;
+
+  return (
+    <ScrollView style={styles.container} contentContainerStyle={styles.content}>
+      <View style={styles.card}>
+        <Text style={styles.cardTitle}>Voice Activity Detection</Text>
+        <Text style={styles.cardDescription}>
+          Streams microphone audio through the FSMN-VAD model and reports when speech begins and
+          ends in real time.
+        </Text>
+        <ModelStatus
+          isReady={isReady}
+          downloadProgress={downloadProgress}
+          error={error ? error.message : null}
+          modelTypeLabel="VAD model"
+        />
+      </View>
+
+      {runError && (
+        <View style={styles.errorContainer}>
+          <Text style={styles.errorText}>{runError}</Text>
+        </View>
+      )}
+
+      <View style={styles.card}>
+        <View style={styles.visualizer}>
+          <View style={[styles.indicator, isSpeaking ? styles.speaking : styles.silent]} />
+          <Text
+            style={[styles.visualizerText, isSpeaking ? styles.speakingText : styles.silentText]}
+          >
+            {isSpeaking ? 'SPEAKING' : 'SILENT'}
+          </Text>
+        </View>
+
+        {isStreaming ? (
+          <Button title="Stop VAD stream" variant="accent" onPress={handleStop} />
+        ) : (
+          <Button
+            title={isSimulator ? 'Recording not available on simulator' : 'Start VAD stream'}
+            onPress={handleStart}
+            disabled={streamDisabled}
+          />
+        )}
+      </View>
+
+      <View style={styles.card}>
+        <Text style={styles.sectionTitle}>VAD events</Text>
+        <ScrollView
+          ref={logScrollRef}
+          style={styles.logScroll}
+          onContentSizeChange={() => logScrollRef.current?.scrollToEnd({ animated: true })}
+        >
+          {logs.length > 0 ? (
+            logs.map((log, i) => (
+              <Text key={i} style={styles.logText}>
+                {log}
+              </Text>
+            ))
+          ) : (
+            <Text style={styles.emptyText}>No events logged yet…</Text>
+          )}
+        </ScrollView>
+      </View>
+    </ScrollView>
+  );
+}
+
+export default function VADScreen() {
+  return (
+    <ScreenWrapper>
+      <VADContent />
+    </ScreenWrapper>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: { flex: 1, backgroundColor: theme.colors.background },
+  content: { padding: theme.spacing.large, paddingBottom: 40 },
+  card: {
+    backgroundColor: theme.colors.cardBackground,
+    borderRadius: theme.radius.large,
+    padding: 20,
+    marginBottom: 20,
+    borderWidth: 1,
+    borderColor: theme.colors.lightBorder,
+  },
+  cardTitle: {
+    fontSize: theme.typography.title.fontSize,
+    fontWeight: theme.typography.title.fontWeight,
+    color: theme.colors.strongPrimary,
+    marginBottom: 8,
+  },
+  cardDescription: {
+    fontSize: 14,
+    color: theme.colors.textMuted,
+    lineHeight: 20,
+    marginBottom: 16,
+  },
+  visualizer: { alignItems: 'center', marginBottom: 20 },
+  indicator: {
+    width: 96,
+    height: 96,
+    borderRadius: 48,
+    marginBottom: 16,
+  },
+  speaking: { backgroundColor: '#22c55e' },
+  silent: { backgroundColor: '#e9ecef' },
+  visualizerText: { fontSize: 22, fontWeight: '800', letterSpacing: 2 },
+  speakingText: { color: '#22c55e' },
+  silentText: { color: theme.colors.textPlaceholder },
+  sectionTitle: { fontSize: 16, fontWeight: '700', color: '#212529', marginBottom: 10 },
+  logScroll: {
+    maxHeight: 180,
+    backgroundColor: '#f8fafc',
+    borderRadius: theme.radius.small,
+    borderWidth: 1,
+    borderColor: theme.colors.lightBorder,
+    padding: 12,
+  },
+  logText: {
+    fontSize: 12,
+    fontFamily: Platform.OS === 'ios' ? 'Menlo' : 'monospace',
+    color: '#334155',
+    marginBottom: 2,
+  },
+  emptyText: { color: theme.colors.textPlaceholder, fontStyle: 'italic' },
+  errorContainer: {
+    backgroundColor: theme.colors.errorBackground,
+    padding: 12,
+    borderRadius: theme.radius.small,
+    marginBottom: 20,
+  },
+  errorText: { color: theme.colors.errorText, fontSize: 14, textAlign: 'center' },
+});