diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index f1d55b4c98..e90a9b0d50 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -183,3 +183,8 @@ Synchronizable stringifying hɛloʊ wɜːld +webrtc +fishjam +Fishjam +deinitialize +Deinitialize diff --git a/apps/computer-vision/app.json b/apps/computer-vision/app.json index 4fcbca2ce3..c2cfc3e45a 100644 --- a/apps/computer-vision/app.json +++ b/apps/computer-vision/app.json @@ -17,7 +17,8 @@ "supportsTablet": true, "bundleIdentifier": "com.anonymous.computervision", "infoPlist": { - "NSCameraUsageDescription": "Process photo from camera" + "NSCameraUsageDescription": "Process photo from camera", + "NSMicrophoneUsageDescription": "Required for audio processing" } }, "android": { @@ -26,7 +27,12 @@ "backgroundColor": "#ffffff" }, "package": "com.anonymous.computervision", - "permissions": ["android.permission.CAMERA"] + "permissions": [ + "android.permission.CAMERA", + "android.permission.INTERNET", + "android.permission.RECORD_AUDIO", + "android.permission.ACCESS_NETWORK_STATE" + ] }, "web": { "favicon": "./assets/icons/favicon.png" diff --git a/packages/react-native-executorch-webrtc/README.md b/packages/react-native-executorch-webrtc/README.md new file mode 100644 index 0000000000..711fa74522 --- /dev/null +++ b/packages/react-native-executorch-webrtc/README.md @@ -0,0 +1,91 @@ +# react-native-executorch-webrtc + +Real-time background blur for Fishjam WebRTC applications, powered by ExecuTorch segmentation models. + +This package provides GPU-accelerated background blur effects using on-device ExecuTorch models for foreground / background segmentation. + +## Requirements + +- iOS 13.0+ +- Android SDK 26+ +- Peer dependencies: +- `@fishjam-cloud/react-native-client` +- `@fishjam-cloud/react-native-webrtc` +- `react-native-executorch` + +## Installation + +```bash +yarn add react-native-executorch-webrtc +``` + +For iOS: +```bash +cd ios && pod install +``` + +## Usage + +### With Fishjam SDK + +```tsx +import { useBackgroundBlur } from 'react-native-executorch-webrtc'; +import { useCamera } from '@fishjam-cloud/react-native-client'; + +function VideoCall() { + const [blurEnabled, setBlurEnabled] = useState(true); + + const { blurMiddleware } = useBackgroundBlur({ + // NOTE: you can use React Native Executorch's Resource Fetcher to download model files + modelUri: 'file:///path/to/selfie_segmenter.pte', + blurRadius: 15, + }); + + const { toggleCamera } = useCamera({ + cameraTrackMiddleware: blurEnabled ? blurMiddleware : undefined, + }); + + return ( +