feat: add Fishjam background blur integration (#1080)

## Description

Adds react-native-executorch-webrtc package for real-time background
blur in Fishjam WebRTC video calls using on-device ExecuTorch
segmentation models.

  Key features:
- useBackgroundBlur hook providing blurMiddleware for Fishjam's
useCamera
  - blur compositing (OpenGL ES on Android, Core Image on iOS)
  - Morphological mask cleaning + EMA temporal smoothing (C++/OpenCV)

  Architecture:
- Reuses BaseSemanticSegmentation from react-native-executorch for
inference
  - Registers custom VideoFrameProcessor with Fishjam's WebRTC pipeline
  - All heavy processing in native (C++/Objective-C++) for performance

### Introduces a breaking change?

- [ ] Yes
- [x] No

### Type of change

- [ ] Bug fix (change which fixes an issue)
- [x] New feature (change which adds functionality)
- [ ] Documentation update (improves or adds clarity to existing
documentation)
- [ ] Other (chores, tests, code style improvements etc.)

### Tested on

- [x] iOS
- [x] Android

### Testing instructions

You'll need to setup your fishjam account, and verify this example works
properly:

```import { useEffect, useState } from 'react';
import { StatusBar } from 'expo-status-bar';
import { StyleSheet, Text, View, TouchableOpacity } from 'react-native';
import {
  FishjamProvider,
  useConnection,
  useCamera,
  useInitializeDevices,
  useSandbox,
  RTCView,
} from '@fishjam-cloud/react-native-client';
import { useBackgroundBlur } from 'react-native-executorch-webrtc';
import { ResourceFetcher, SELFIE_SEGMENTATION, initExecutorch } from 'react-native-executorch';
import { ExpoResourceFetcher } from 'react-native-executorch-expo-resource-fetcher';

initExecutorch({ resourceFetcher: ExpoResourceFetcher });

const FISHJAM_ID = 'your-id';

function CameraScreen() {
  const { initializeDevices } = useInitializeDevices();
  const { cameraStream, cameraDevices, currentCamera, selectCamera, setCameraTrackMiddleware } = useCamera();
  const { joinRoom, leaveRoom, peerStatus } = useConnection();
  const { getSandboxPeerToken } = useSandbox();
  const [isJoining, setIsJoining] = useState(false);
  const [modelPath, setModelPath] = useState<string | null>(null);
  const [downloadProgress, setDownloadProgress] = useState(0);
  const [blurEnabled, setBlurEnabled] = useState(false);

  const { blurMiddleware } = useBackgroundBlur({
    modelUri: modelPath || '',
    blurRadius: 12,
  });

  // Download the selfie segmentation model
  useEffect(() => {
    const downloadModel = async () => {
      try {
        const paths = await ResourceFetcher.fetch(
          (progress) => setDownloadProgress(progress),
          SELFIE_SEGMENTATION.modelSource
        );
        if (paths?.[0]) {
          setModelPath(paths[0]);
        }
      } catch (error) {
        console.error('Failed to download model:', error);
      }
    };
    downloadModel();
  }, []);

  const handleFlipCamera = async () => {
    if (cameraDevices.length < 2) return;
    const currentIndex = cameraDevices.findIndex(
      (device) => device.deviceId === currentCamera?.deviceId
    );
    const nextIndex = (currentIndex + 1) % cameraDevices.length;
    await selectCamera(cameraDevices[nextIndex].deviceId);
  };

  const handleToggleBlur = async () => {
    if (!modelPath) return;
    if (blurEnabled) {
      await setCameraTrackMiddleware(null);
      setBlurEnabled(false);
    } else {
      await setCameraTrackMiddleware(blurMiddleware);
      setBlurEnabled(true);
    }
  };

  useEffect(() => {
    initializeDevices();
  }, []);

  const handleJoinRoom = async () => {
    setIsJoining(true);
    try {
      const roomName = 'demo-room';
      const peerName = `user_${Date.now()}`;
      const peerToken = await getSandboxPeerToken(roomName, peerName);
      await joinRoom({ peerToken });
    } catch (error) {
      console.error('Failed to join room:', error);
    } finally {
      setIsJoining(false);
    }
  };

  return (
    <View style={styles.container}>
      <StatusBar style="light" />

      <View style={styles.videoContainer}>
        {cameraStream ? (
          <RTCView
            mediaStream={cameraStream}
            style={styles.video}
            objectFit="cover"
            mirror={true}
          />
        ) : (
          <View style={styles.placeholder}>
            <Text style={styles.placeholderText}>Starting camera...</Text>
          </View>
        )}
      </View>

      <View style={styles.controls}>
        <Text style={styles.status}>Status: {peerStatus}</Text>
        {downloadProgress > 0 && downloadProgress < 1 && (
          <Text style={styles.status}>
            Downloading model: {(downloadProgress * 100).toFixed(0)}%
          </Text>
        )}
        <View style={styles.buttons}>
          <TouchableOpacity style={styles.flipButton} onPress={handleFlipCamera}>
            <Text style={styles.buttonText}>Flip</Text>
          </TouchableOpacity>
          <TouchableOpacity
            style={[styles.blurButton, blurEnabled && styles.blurButtonActive, !modelPath && styles.buttonDisabled]}
            onPress={handleToggleBlur}
            disabled={!modelPath}
          >
            <Text style={styles.buttonText}>{blurEnabled ? 'Blur On' : 'Blur'}</Text>
          </TouchableOpacity>
          {peerStatus === 'connected' ? (
            <TouchableOpacity style={styles.leaveButton} onPress={leaveRoom}>
              <Text style={styles.buttonText}>Leave Room</Text>
            </TouchableOpacity>
          ) : (
            <TouchableOpacity
              style={[styles.button, isJoining && styles.buttonDisabled]}
              onPress={handleJoinRoom}
              disabled={isJoining}
            >
              <Text style={styles.buttonText}>
                {isJoining ? 'Joining...' : 'Join Room'}
              </Text>
            </TouchableOpacity>
          )}
        </View>
      </View>
    </View>
  );
}

export default function App() {
  return (
    <FishjamProvider fishjamId={FISHJAM_ID}>
      <CameraScreen />
    </FishjamProvider>
  );
}

const styles = StyleSheet.create({
  container: {
    flex: 1,
    backgroundColor: '#000',
  },
  videoContainer: {
    flex: 1,
  },
  video: {
    flex: 1,
  },
  placeholder: {
    flex: 1,
    alignItems: 'center',
    justifyContent: 'center',
    backgroundColor: '#1a1a1a',
  },
  placeholderText: {
    color: '#666',
    fontSize: 18,
  },
  controls: {
    padding: 20,
    paddingBottom: 40,
    alignItems: 'center',
    gap: 12,
  },
  status: {
    color: '#888',
    fontSize: 14,
  },
  buttons: {
    flexDirection: 'row',
    gap: 12,
  },
  button: {
    backgroundColor: '#007AFF',
    paddingHorizontal: 32,
    paddingVertical: 14,
    borderRadius: 12,
  },
  flipButton: {
    backgroundColor: '#333',
    paddingHorizontal: 24,
    paddingVertical: 14,
    borderRadius: 12,
  },
  leaveButton: {
    backgroundColor: '#FF3B30',
    paddingHorizontal: 32,
    paddingVertical: 14,
    borderRadius: 12,
  },
  blurButton: {
    backgroundColor: '#5856D6',
    paddingHorizontal: 24,
    paddingVertical: 14,
    borderRadius: 12,
  },
  blurButtonActive: {
    backgroundColor: '#34C759',
  },
  buttonDisabled: {
    backgroundColor: '#444',
  },
  buttonText: {
    color: '#fff',
    fontSize: 16,
    fontWeight: '600',
  },
});
```

### Screenshots

<!-- Add screenshots here, if applicable -->

### Related issues

<!-- Link related issues here using #issue-number -->

### Checklist

- [ ] I have performed a self-review of my code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have updated the documentation accordingly
- [ ] My changes generate no new warnings

### Additional notes

<!-- Include any additional information, assumptions, or context that
reviewers might need to understand this PR. -->

Author: chmjkb

.cspell-wordlist.txt

@@ -198,3 +198,8 @@ keypoints
 Keypoint
 Keypoints
 letterboxing
+webrtc
+fishjam
+Fishjam
+deinitialize
+Deinitialize

docs/docs/05-utilities/01-webrtc-integration.md

@@ -0,0 +1,117 @@
+---
+title: Fishjam Usage
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+:::danger
+This integration is currently in beta.
+:::
+
+## Overview
+
+Starting from v0.9.0, you can use `react-native-executorch` powered background blur integration in your [Fishjam](https://fishjam.io) applications. The package `react-native-executorch-webrtc` exposes a hook, which returns a middleware for your camera streams. We plan to extend this to other models, such as text to speech, speech to text, or other vision models in the future.
+
+## Installation
+
+Install the package with your package manager of choice. Make sure to also have `react-native-executorch` and a resource fetcher adapter installed (see [Getting Started](../01-fundamentals/01-getting-started.md)).
+
+<Tabs>
+  <TabItem value="npm" label="NPM">
+
+    ```bash
+    npm install react-native-executorch-webrtc
+    ```
+
+  </TabItem>
+  <TabItem value="pnpm" label="PNPM">
+
+    ```bash
+    pnpm install react-native-executorch-webrtc
+    ```
+
+  </TabItem>
+  <TabItem value="yarn" label="YARN">
+
+    ```bash
+    yarn add react-native-executorch-webrtc
+    ```
+
+  </TabItem>
+</Tabs>
+
+The following peer dependencies must also be installed in your app:
+
+- `@fishjam-cloud/react-native-client`
+- `@fishjam-cloud/react-native-webrtc`
+- `react-native-executorch`
+
+## Usage
+
+The integration is built around the `SELFIE_SEGMENTATION` model that we export from `react-native-executorch`. It's the only model we currently support and tune for — the blur pipeline expects its specific input shape and output classes, so other segmentation models will not work correctly.
+
+Use `ResourceFetcher` together with `SELFIE_SEGMENTATION.modelSource` to download (and cache) the model, then pass the resulting path to `useBackgroundBlur`. The returned `blurMiddleware` plugs into Fishjam's `cameraTrackMiddleware`.
+
+```tsx
+import { useEffect, useState } from 'react';
+import { Button, Text } from 'react-native';
+import { ResourceFetcher, SELFIE_SEGMENTATION } from 'react-native-executorch';
+import { useBackgroundBlur } from 'react-native-executorch-webrtc';
+import { useCamera } from '@fishjam-cloud/react-native-client';
+
+function VideoCall() {
+  const [modelUri, setModelUri] = useState<string | null>(null);
+
+  useEffect(() => {
+    ResourceFetcher.fetch(() => {}, SELFIE_SEGMENTATION.modelSource).then(
+      (paths) => paths?.[0] && setModelUri(paths[0])
+    );
+  }, []);
+
+  // Wait for the model to be available before mounting the hook —
+  // useBackgroundBlur expects a real path, not an empty string.
+  if (!modelUri) {
+    return <Text>Downloading model…</Text>;
+  }
+
+  return <VideoCallWithBlur modelUri={modelUri} />;
+}
+
+function VideoCallWithBlur({ modelUri }: { modelUri: string }) {
+  const [blurEnabled, setBlurEnabled] = useState(true);
+
+  const { blurMiddleware } = useBackgroundBlur({
+    modelUri,
+    blurRadius: 15,
+  });
+
+  useCamera({
+    cameraTrackMiddleware: blurEnabled ? blurMiddleware : undefined,
+  });
+
+  return (
+    <Button
+      title={blurEnabled ? 'Disable Blur' : 'Enable Blur'}
+      onPress={() => setBlurEnabled(!blurEnabled)}
+    />
+  );
+}
+```
+
+## API
+
+### `useBackgroundBlur(options)`
+
+| Option       | Type     | Description                                                                |
+| ------------ | -------- | -------------------------------------------------------------------------- |
+| `modelUri`   | `string` | **Required.** Path or `file://` URI to the segmentation `.pte` model.      |
+| `blurRadius` | `number` | Optional. Gaussian blur sigma applied to the background. Defaults to `12`. |
+
+Returns:
+
+| Field            | Type              | Description                                                    |
+| ---------------- | ----------------- | -------------------------------------------------------------- |
+| `blurMiddleware` | `TrackMiddleware` | Pass to `useCamera({ cameraTrackMiddleware })` to enable blur. |
+
+The hook initializes the native processor on mount and releases it on unmount, so you don't need to manage its lifecycle manually.

packages/react-native-executorch-webrtc/README.md

@@ -0,0 +1,91 @@
+# react-native-executorch-webrtc
+
+Real-time background blur for Fishjam WebRTC applications, powered by ExecuTorch segmentation models.
+
+This package provides GPU-accelerated background blur effects using on-device ExecuTorch models for foreground / background segmentation.
+
+## Requirements
+
+- iOS 13.0+
+- Android SDK 26+
+- Peer dependencies:
+- `@fishjam-cloud/react-native-client`
+- `@fishjam-cloud/react-native-webrtc`
+- `react-native-executorch`
+
+## Installation
+
+```bash
+yarn add react-native-executorch-webrtc
+```
+
+For iOS:
+```bash
+cd ios && pod install
+```
+
+## Usage
+
+### With Fishjam SDK
+
+```tsx
+import { useBackgroundBlur } from 'react-native-executorch-webrtc';
+import { useCamera } from '@fishjam-cloud/react-native-client';
+
+function VideoCall() {
+  const [blurEnabled, setBlurEnabled] = useState(true);
+
+  const { blurMiddleware } = useBackgroundBlur({
+    // NOTE: you can use React Native Executorch's Resource Fetcher to download model files
+    modelUri: 'file:///path/to/selfie_segmenter.pte',
+    blurRadius: 15,
+  });
+
+  const { toggleCamera } = useCamera({
+    cameraTrackMiddleware: blurEnabled ? blurMiddleware : undefined,
+  });
+
+  return (
+    <Button
+      title={blurEnabled ? 'Disable Blur' : 'Enable Blur'}
+      onPress={() => setBlurEnabled(!blurEnabled)}
+    />
+  );
+}
+```
+
+## API
+
+### `useBackgroundBlur(options)`
+
+React hook that provides camera track middleware for background blur.
+
+**Options:**
+- `modelUri`: `string` - Path to the ExecuTorch segmentation model (.pte file)
+- `blurRadius`: `number` (optional, default: 12) - Blur intensity
+
+**Returns:**
+- `blurMiddleware`: `TrackMiddleware` - Middleware function for use with Fishjam's `useCamera`
+
+### Blur Radius
+
+The `blurRadius` can be updated dynamically without reinitializing:
+
+```tsx
+const { blurMiddleware } = useBackgroundBlur({
+  modelUri: modelPath,
+  blurRadius: intensity,
+});
+```
+
+## Platform Notes
+
+### iOS
+Uses Core Image for GPU-accelerated blur compositing.
+
+### Android
+Uses OpenGL ES shaders for blur and compositing. OpenMP is used for parallel CPU operations.
+
+## License
+
+MIT

packages/react-native-executorch-webrtc/android/CMakeLists.txt

@@ -0,0 +1,78 @@
+cmake_minimum_required(VERSION 3.13)
+project(react-native-executorch-webrtc)
+
+set(CMAKE_VERBOSE_MAKEFILE ON)
+set(CMAKE_CXX_STANDARD 20)
+
+# Paths to react-native-executorch
+set(RN_EXECUTORCH_DIR "${CMAKE_SOURCE_DIR}/../../react-native-executorch")
+set(RN_EXECUTORCH_THIRD_PARTY "${RN_EXECUTORCH_DIR}/third-party/include")
+
+# Add our JNI bridge source files
+file(GLOB SOURCES "src/main/cpp/*.cpp")
+
+add_library(
+  ${CMAKE_PROJECT_NAME}
+  SHARED
+  ${SOURCES}
+)
+
+# Find packages - ReactAndroid provides JSI headers
+find_package(ReactAndroid REQUIRED CONFIG)
+find_package(fbjni REQUIRED CONFIG)
+find_package(react-native-executorch REQUIRED CONFIG)
+
+# Include headers
+target_include_directories(
+  ${CMAKE_PROJECT_NAME}
+  PRIVATE
+  "${RN_EXECUTORCH_DIR}/common"
+  "${RN_EXECUTORCH_THIRD_PARTY}"
+)
+
+# Find prebuilt libraries
+find_library(LOG_LIB log)
+find_library(ANDROID_LIB android)
+
+# Import ExecuTorch library
+set(LIBS_DIR "${RN_EXECUTORCH_DIR}/third-party/android/libs")
+add_library(executorch SHARED IMPORTED)
+set_target_properties(executorch PROPERTIES
+  IMPORTED_LOCATION "${LIBS_DIR}/executorch/${ANDROID_ABI}/libexecutorch.so")
+
+# OpenCV libraries
+set(OPENCV_LIBS
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_core.a"
+  "${LIBS_DIR}/opencv/${ANDROID_ABI}/libopencv_imgproc.a"
+)
+
+# OpenCV third-party libraries (arm64 specific)
+if(ANDROID_ABI STREQUAL "arm64-v8a")
+  set(OPENCV_THIRD_PARTY_LIBS
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_hal.a"
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv_thread.a"
+    "${LIBS_DIR}/opencv-third-party/${ANDROID_ABI}/libkleidicv.a"
+  )
+else()
+  set(OPENCV_THIRD_PARTY_LIBS "")
+endif()
+
+# OpenMP for OpenCV parallel operations
+target_link_options(${CMAKE_PROJECT_NAME} PRIVATE -fopenmp -static-openmp)
+
+# Link against libraries
+target_link_libraries(
+  ${CMAKE_PROJECT_NAME}
+  ${LOG_LIB}
+  ${ANDROID_LIB}
+  ReactAndroid::jsi
+  ReactAndroid::reactnative
+  fbjni::fbjni
+  react-native-executorch::react-native-executorch
+  ${OPENCV_LIBS}
+  ${OPENCV_THIRD_PARTY_LIBS}
+  executorch
+)
+
+# Ensure react-native-executorch symbols are exported (needed for BaseModel/BaseSemanticSegmentation)
+target_link_options(${CMAKE_PROJECT_NAME} PRIVATE -Wl,--no-as-needed)

packages/react-native-executorch-webrtc/android/build.gradle

@@ -0,0 +1,82 @@
+buildscript {
+    ext.kotlin_version = '1.9.0'
+
+    repositories {
+        google()
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath 'com.android.tools.build:gradle:8.2.2'
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+    }
+}
+
+apply plugin: 'com.android.library'
+apply plugin: 'kotlin-android'
+
+android {
+    namespace 'com.executorch.webrtc'
+    compileSdkVersion 34
+
+    buildFeatures {
+        prefab true
+    }
+
+    defaultConfig {
+        minSdkVersion 26
+        targetSdkVersion 34
+
+        externalNativeBuild {
+            cmake {
+                cppFlags "-std=c++20"
+                arguments "-DANDROID_STL=c++_shared"
+            }
+        }
+
+        ndk {
+            // Only build for architectures where OpenCV/ExecuTorch libraries exist
+            abiFilters 'arm64-v8a'
+        }
+    }
+
+    externalNativeBuild {
+        cmake {
+            path "CMakeLists.txt"
+        }
+    }
+
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_17
+        targetCompatibility JavaVersion.VERSION_17
+    }
+
+    kotlinOptions {
+        jvmTarget = '17'
+    }
+
+    packaging {
+        jniLibs {
+            // Don't bundle react-native-executorch.so - the app gets it from the main package
+            excludes += ['**/libreact-native-executorch.so']
+        }
+    }
+}
+
+repositories {
+    google()
+    mavenCentral()
+}
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    implementation 'com.facebook.react:react-native:+'
+
+    // WebRTC classes - provided by app via autolinking
+    compileOnly project(':fishjam-cloud_react-native-webrtc')
+    // ExecuTorch for vision model processing.
+    // Must be `implementation` (not compileOnly) so AGP exposes the producer's
+    // prefab .so to our CMake link step. The packaging.jniLibs.excludes block
+    // above prevents double-bundling — the app gets the .so from the main pkg.
+    implementation project(':react-native-executorch')
+}

packages/react-native-executorch-webrtc/android/gradle/wrapper/gradle-wrapper.jar

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.0-milestone-1-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists