feat: add Fishjam background blur integration

.cspell-wordlist.txt

@@ -183,3 +183,8 @@ Synchronizable
 stringifying
 hɛloʊ
 wɜːld
+webrtc
+fishjam
+Fishjam
+deinitialize
+Deinitialize

apps/computer-vision/app.json

@@ -17,7 +17,8 @@
       "supportsTablet": true,
       "bundleIdentifier": "com.anonymous.computervision",
       "infoPlist": {
-        "NSCameraUsageDescription": "Process photo from camera"
+        "NSCameraUsageDescription": "Process photo from camera",
+        "NSMicrophoneUsageDescription": "Required for audio processing"
       }
     },
     "android": {
@@ -26,7 +27,12 @@
         "backgroundColor": "#ffffff"
       },
       "package": "com.anonymous.computervision",
-      "permissions": ["android.permission.CAMERA"]
+      "permissions": [
+        "android.permission.CAMERA",
+        "android.permission.INTERNET",
+        "android.permission.RECORD_AUDIO",
+        "android.permission.ACCESS_NETWORK_STATE"
+      ]
     },
     "web": {
       "favicon": "./assets/icons/favicon.png"

packages/react-native-executorch-webrtc/README.md

@@ -0,0 +1,91 @@
+# react-native-executorch-webrtc
+
+Real-time background blur for Fishjam WebRTC applications, powered by ExecuTorch segmentation models.
+
+This package provides GPU-accelerated background blur effects using on-device ExecuTorch models for foreground / background segmentation.
+
+## Requirements
+
+- iOS 13.0+
+- Android SDK 26+
+- Peer dependencies:
+- `@fishjam-cloud/react-native-client`
+- `@fishjam-cloud/react-native-webrtc`
+- `react-native-executorch`
+
+## Installation
+
+```bash
+yarn add react-native-executorch-webrtc
+```
+
+For iOS:
+```bash
+cd ios && pod install
+```
+
+## Usage
+
+### With Fishjam SDK
+
+```tsx
+import { useBackgroundBlur } from 'react-native-executorch-webrtc';
+import { useCamera } from '@fishjam-cloud/react-native-client';
+
+function VideoCall() {
+  const [blurEnabled, setBlurEnabled] = useState(true);
+
+  const { blurMiddleware } = useBackgroundBlur({
+    // NOTE: you can use React Native Executorch's Resource Fetcher to download model files
+    modelUri: 'file:///path/to/selfie_segmenter.pte',
+    blurRadius: 15,
+  });
+
+  const { toggleCamera } = useCamera({
+    cameraTrackMiddleware: blurEnabled ? blurMiddleware : undefined,
+  });
+
+  return (
+    <Button
+      title={blurEnabled ? 'Disable Blur' : 'Enable Blur'}
+      onPress={() => setBlurEnabled(!blurEnabled)}
+    />
+  );
+}
+```
+
+## API
+
+### `useBackgroundBlur(options)`
+
+React hook that provides camera track middleware for background blur.
+
+**Options:**
+- `modelUri`: `string` - Path to the ExecuTorch segmentation model (.pte file)
+- `blurRadius`: `number` (optional, default: 12) - Blur intensity
+
+**Returns:**
+- `blurMiddleware`: `TrackMiddleware` - Middleware function for use with Fishjam's `useCamera`
+
+### Blur Radius
+
+The `blurRadius` can be updated dynamically without reinitializing:
+
+```tsx
+const { blurMiddleware } = useBackgroundBlur({
+  modelUri: modelPath,
+  blurRadius: intensity,
+});
+```
+
+## Platform Notes
+
+### iOS
+Uses Core Image for GPU-accelerated blur compositing.
+
+### Android
+Uses OpenGL ES shaders for blur and compositing. OpenMP is used for parallel CPU operations.
+
+## License
+
+MIT

packages/react-native-executorch-webrtc/android/CMakeLists.txt

@@ -0,0 +1,78 @@
+cmake_minimum_required(VERSION 3.13)
+project(react-native-executorch-webrtc)
+
+set(CMAKE_VERBOSE_MAKEFILE ON)
+set(CMAKE_CXX_STANDARD 20)
+
+# Paths to react-native-executorch
+set(RN_EXECUTORCH_DIR "${CMAKE_SOURCE_DIR}/../../react-native-executorch")
+set(RN_EXECUTORCH_THIRD_PARTY "${RN_EXECUTORCH_DIR}/third-party/include")
+
+# Add our JNI bridge source files
+file(GLOB SOURCES "src/main/cpp/*.cpp")
+
+add_library(
+  ${CMAKE_PROJECT_NAME}
+  SHARED
+  ${SOURCES}
+)
+
+# Find packages - ReactAndroid provides JSI headers
+find_package(ReactAndroid REQUIRED CONFIG)
+find_package(fbjni REQUIRED CONFIG)
+find_package(react-native-executorch REQUIRED CONFIG)
+
+# Include headers
+target_include_directories(
+  ${CMAKE_PROJECT_NAME}
+  PRIVATE
+  "${RN_EXECUTORCH_DIR}/common"
+  "${RN_EXECUTORCH_THIRD_PARTY}"
+)
+
+# Find prebuilt libraries
+find_library(LOG_LIB log)
+find_library(ANDROID_LIB android)
+
+# Import ExecuTorch library
+set(LIBS_DIR "${RN_EXECUTORCH_DIR}/third-party/android/libs")
+add_library(executorch SHARED IMPORTED)
+set_target_properties(executorch PROPERTIES
+  IMPORTED_LOCATION "${LIBS_DIR}/executorch/${ANDROID_ABI}/libexecutorch.so")
+
+# OpenCV libraries
+set(OPENCV_LIBS
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_core.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_imgproc.a"
+)
+
+# OpenCV third-party libraries (arm64 specific)
+if(ANDROID_ABI STREQUAL "arm64-v8a")
+  set(OPENCV_THIRD_PARTY_LIBS
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_hal.a"
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_thread.a"
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv.a"
+  )
+else()
+  set(OPENCV_THIRD_PARTY_LIBS "")
+endif()
+
+# OpenMP for OpenCV parallel operations
+target_link_options(${CMAKE_PROJECT_NAME} PRIVATE -fopenmp -static-openmp)
+
+# Link against libraries
+target_link_libraries(
+  ${CMAKE_PROJECT_NAME}
+  ${LOG_LIB}
+  ${ANDROID_LIB}
+  ReactAndroid::jsi
+  ReactAndroid::reactnative
+  fbjni::fbjni
+  react-native-executorch::react-native-executorch
+  ${OPENCV_LIBS}
+  ${OPENCV_THIRD_PARTY_LIBS}
+  executorch
+)
+
+# Ensure react-native-executorch symbols are exported (needed for BaseModel/BaseSemanticSegmentation)
+target_link_options(${CMAKE_PROJECT_NAME} PRIVATE -Wl,--no-as-needed)

packages/react-native-executorch-webrtc/android/build.gradle

@@ -0,0 +1,72 @@
+buildscript {
+    ext.kotlin_version = '1.9.0'
+
+    repositories {
+        google()
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath 'com.android.tools.build:gradle:8.2.2'
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+    }
+}
+
+apply plugin: 'com.android.library'
+apply plugin: 'kotlin-android'
+
+android {
+    namespace 'com.executorch.webrtc'
+    compileSdkVersion 34
+
+    buildFeatures {
+        prefab true
+    }
+
+    defaultConfig {
+        minSdkVersion 26
+        targetSdkVersion 34
+
+        externalNativeBuild {
+            cmake {
+                cppFlags "-std=c++20"
+                arguments "-DANDROID_STL=c++_shared"
+            }
+        }
+
+        ndk {
+            // Only build for architectures where OpenCV/ExecuTorch libraries exist
+            abiFilters 'arm64-v8a'
+        }
+    }
+
+    externalNativeBuild {
+        cmake {
+            path "CMakeLists.txt"
+        }
+    }
+
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_17
+        targetCompatibility JavaVersion.VERSION_17
+    }
+
+    kotlinOptions {
+        jvmTarget = '17'
+    }
+}
+
+repositories {
+    google()
+    mavenCentral()
+}
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    implementation 'com.facebook.react:react-native:+'
+
+    // WebRTC classes - provided by app via autolinking
+    compileOnly project(':fishjam-cloud_react-native-webrtc')
+    // ExecuTorch for vision model processing
+    compileOnly project(':react-native-executorch')
+}

packages/react-native-executorch-webrtc/android/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.0-milestone-1-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists