Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 1 addition & 19 deletions apps/computer-vision/app/vision_camera/index.tsx
Original file line number Diff line number Diff line change
@@ -1,10 +1,4 @@
import React, {
useCallback,
useContext,
useEffect,
useMemo,
useState,
} from 'react';
import React, { useCallback, useContext, useEffect, useState } from 'react';
import {
ScrollView,
StatusBar,
Expand All @@ -17,8 +11,6 @@ import { useSafeAreaInsets } from 'react-native-safe-area-context';
import { useIsFocused } from '@react-navigation/native';
import {
Camera,
getCameraFormat,
Templates,
useCameraDevices,
useCameraPermission,
useFrameOutput,
Expand Down Expand Up @@ -147,15 +139,6 @@ export default function VisionCameraScreen() {
const devices = useCameraDevices();
const device =
devices.find((d) => d.position === cameraPosition) ?? devices[0];
const format = useMemo(() => {
if (device == null) return undefined;
try {
return getCameraFormat(device, { ...Templates.FrameProcessing });
} catch {
return undefined;
}
}, [device]);

useEffect(() => {
frameKillSwitch.setBlocking(true);
setError(null);
Expand Down Expand Up @@ -238,7 +221,6 @@ export default function VisionCameraScreen() {
device={device}
outputs={frameOutput ? [frameOutput] : []}
isActive={isFocused}
format={format}
orientationSource="device"
/>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { StyleSheet, Text, View } from 'react-native';
import { Frame, useFrameOutput } from 'react-native-vision-camera';
import { scheduleOnRN } from 'react-native-worklets';
import { EFFICIENTNET_V2_S, useClassification } from 'react-native-executorch';
import { TaskProps } from './types';
import { FRAME_TARGET_RESOLUTION, TaskProps } from './types';

type Props = Omit<
TaskProps,
Expand Down Expand Up @@ -53,6 +53,7 @@ export default function ClassificationTask({
);

const frameOutput = useFrameOutput({
targetResolution: FRAME_TARGET_RESOLUTION,
pixelFormat: 'rgb',
dropFramesWhileBusy: true,
enablePreviewSizedOutputBuffers: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
} from 'react-native-executorch';
import { Canvas, Image as SkiaImage } from '@shopify/react-native-skia';
import { labelColor, labelColorBg } from '../../utils/colors';
import { TaskProps } from './types';
import { FRAME_TARGET_RESOLUTION, TaskProps } from './types';
import {
buildDisplayInstances,
DisplayInstance,
Expand Down Expand Up @@ -99,6 +99,7 @@ export default function InstanceSegmentationTask({
);

const frameOutput = useFrameOutput({
targetResolution: FRAME_TARGET_RESOLUTION,
pixelFormat: 'rgb',
dropFramesWhileBusy: true,
enablePreviewSizedOutputBuffers: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { Frame, useFrameOutput } from 'react-native-vision-camera';
import { scheduleOnRN } from 'react-native-worklets';
import { OCR_ENGLISH, OCRDetection, useOCR } from 'react-native-executorch';
import Svg, { Polygon, Text as SvgText } from 'react-native-svg';
import { TaskProps } from './types';
import { FRAME_TARGET_RESOLUTION, TaskProps } from './types';

type Props = Omit<TaskProps, 'activeModel'>;

Expand Down Expand Up @@ -55,6 +55,7 @@ export default function OCRTask({
);

const frameOutput = useFrameOutput({
targetResolution: FRAME_TARGET_RESOLUTION,
pixelFormat: 'rgb',
dropFramesWhileBusy: true,
enablePreviewSizedOutputBuffers: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
CocoLabelYolo,
} from 'react-native-executorch';
import BoundingBoxes from '../../BoundingBoxes';
import { TaskProps } from './types';
import { FRAME_TARGET_RESOLUTION, TaskProps } from './types';

type ObjModelId =
| 'objectDetectionSsdlite'
Expand Down Expand Up @@ -101,6 +101,7 @@ export default function ObjectDetectionTask({
);

const frameOutput = useFrameOutput({
targetResolution: FRAME_TARGET_RESOLUTION,
pixelFormat: 'rgb',
dropFramesWhileBusy: true,
enablePreviewSizedOutputBuffers: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import {
SkImage,
} from '@shopify/react-native-skia';
import { CLASS_COLORS } from '../../utils/colors';
import { TaskProps } from './types';
import { FRAME_TARGET_RESOLUTION, TaskProps } from './types';

type SegModelId =
| 'segmentationDeeplabResnet50'
Expand Down Expand Up @@ -142,6 +142,7 @@ export default function SegmentationTask({
const colors = CLASS_COLORS;

const frameOutput = useFrameOutput({
targetResolution: FRAME_TARGET_RESOLUTION,
pixelFormat: 'rgb',
dropFramesWhileBusy: true,
enablePreviewSizedOutputBuffers: true,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import {
Skia,
SkImage,
} from '@shopify/react-native-skia';
import { TaskProps } from './types';
import { FRAME_TARGET_RESOLUTION, TaskProps } from './types';

type StyleModelId = 'styleTransferCandy' | 'styleTransferMosaic';

Expand Down Expand Up @@ -96,6 +96,7 @@ export default function StyleTransferTask({
);

const frameOutput = useFrameOutput({
targetResolution: FRAME_TARGET_RESOLUTION,
pixelFormat: 'rgb',
dropFramesWhileBusy: true,
enablePreviewSizedOutputBuffers: true,
Expand Down
2 changes: 2 additions & 0 deletions apps/computer-vision/components/vision_camera/tasks/types.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { useFrameOutput } from 'react-native-vision-camera';
import { createSynchronizable } from 'react-native-worklets';

export const FRAME_TARGET_RESOLUTION = { width: 1280, height: 720 } as const;

export type TaskProps = {
activeModel: string;
canvasSize: { width: number; height: number };
Expand Down
11 changes: 6 additions & 5 deletions apps/computer-vision/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,16 @@
"react-native-gesture-handler": "~2.28.0",
"react-native-image-picker": "^7.2.2",
"react-native-loading-spinner-overlay": "^3.0.1",
"react-native-nitro-image": "0.13.0",
"react-native-nitro-modules": "0.35.2",
"react-native-reanimated": "~4.2.2",
"react-native-nitro-image": "0.13.1",
"react-native-nitro-modules": "0.35.4",
"react-native-reanimated": "4.3.0",
"react-native-safe-area-context": "~5.6.0",
"react-native-screens": "~4.16.0",
"react-native-svg": "15.15.3",
"react-native-svg-transformer": "^1.5.3",
"react-native-vision-camera": "5.0.0-beta.7",
"react-native-worklets": "0.7.4"
"react-native-vision-camera": "^5.0.6",
"react-native-vision-camera-worklets": "^5.0.6",
"react-native-worklets": "0.8.1"
},
"devDependencies": {
"@babel/core": "^7.29.0",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
title: VisionCamera Integration
---

React Native ExecuTorch vision models support real-time frame processing via [VisionCamera v5](https://react-native-vision-camera-v5-docs.vercel.app) using the `runOnFrame` worklet. This page explains how to set it up and what to watch out for.
React Native ExecuTorch vision models support real-time frame processing via [VisionCamera v5](https://visioncamera.margelo.com) using the `runOnFrame` worklet. This page explains how to set it up and what to watch out for.

## Prerequisites

Make sure you have the following packages installed:

- [`react-native-vision-camera`](https://react-native-vision-camera-v5-docs.vercel.app) v5
- [`react-native-vision-camera`](https://visioncamera.margelo.com) v5
- [`react-native-worklets`](https://docs.swmansion.com/react-native-worklets/)

## Which models support runOnFrame?
Expand Down Expand Up @@ -37,14 +37,14 @@ Use `runOnFrame` when you need to process every camera frame. Use `forward` for

## How it works

VisionCamera v5 delivers frames via [`useFrameOutput`](https://react-native-vision-camera-v5-docs.vercel.app/docs/frame-output). Inside the `onFrame` worklet you call `runOnFrame(frame, isFrontCamera)` synchronously, then use `scheduleOnRN` from `react-native-worklets` to post the result back to React state on the main thread.
VisionCamera v5 delivers frames via [`useFrameOutput`](https://visioncamera.margelo.com/docs/frame-output). Inside the `onFrame` worklet you call `runOnFrame(frame, isFrontCamera)` synchronously, then use `scheduleOnRN` from `react-native-worklets` to post the result back to React state on the main thread.

The `isFrontCamera` parameter tells the native side whether the front camera is active so it can correctly mirror the results. The library handles all device orientation rotation internally — results are always returned in screen-space coordinates regardless of how the user holds their device.

:::warning

- You **must** set `pixelFormat: 'rgb'` in `useFrameOutput`. Our extraction pipeline expects RGB pixel data — any other format (e.g. the default `yuv`) will produce incorrect results.
- `runOnFrame` is synchronous and runs on the JS worklet thread. For models with longer inference times, use `dropFramesWhileBusy: true` to skip frames and avoid blocking the camera pipeline. For more control, see VisionCamera's [async frame processing guide](https://react-native-vision-camera-v5-docs.vercel.app/docs/async-frame-processing).
- `runOnFrame` is synchronous and runs on the JS worklet thread. For models with longer inference times, use `dropFramesWhileBusy: true` to skip frames and avoid blocking the camera pipeline. For more control, see VisionCamera's [async frame processing guide](https://visioncamera.margelo.com/docs/async-frame-processing).
- Always call `frame.dispose()` after processing to release the frame buffer. Wrap your inference in a `try/finally` to ensure it's always called even if `runOnFrame` throws.
:::

Expand Down Expand Up @@ -304,7 +304,7 @@ You are not passing `isFrontCamera: true` when using the front camera. See [Hand

#### App freezes or camera drops frames

Your model's inference time exceeds the frame interval. Enable `dropFramesWhileBusy: true` in `useFrameOutput`, or move inference off the worklet thread using VisionCamera's [async frame processing](https://react-native-vision-camera-v5-docs.vercel.app/docs/async-frame-processing).
Your model's inference time exceeds the frame interval. Enable `dropFramesWhileBusy: true` in `useFrameOutput`, or move inference off the worklet thread using VisionCamera's [async frame processing](https://visioncamera.margelo.com/docs/async-frame-processing).

#### Memory leak / crash after many frames

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
title: VisionCamera Integration
---

React Native ExecuTorch vision models support real-time frame processing via [VisionCamera v5](https://react-native-vision-camera-v5-docs.vercel.app) using the `runOnFrame` worklet. This page explains how to set it up and what to watch out for.
React Native ExecuTorch vision models support real-time frame processing via [VisionCamera v5](https://visioncamera.margelo.com) using the `runOnFrame` worklet. This page explains how to set it up and what to watch out for.

## Prerequisites

Make sure you have the following packages installed:

- [`react-native-vision-camera`](https://react-native-vision-camera-v5-docs.vercel.app) v5
- [`react-native-vision-camera`](https://visioncamera.margelo.com) v5
- [`react-native-worklets`](https://docs.swmansion.com/react-native-worklets/)

## Which models support runOnFrame?
Expand Down Expand Up @@ -37,14 +37,14 @@ Use `runOnFrame` when you need to process every camera frame. Use `forward` for

## How it works

VisionCamera v5 delivers frames via [`useFrameOutput`](https://react-native-vision-camera-v5-docs.vercel.app/docs/frame-output). Inside the `onFrame` worklet you call `runOnFrame(frame, isFrontCamera)` synchronously, then use `scheduleOnRN` from `react-native-worklets` to post the result back to React state on the main thread.
VisionCamera v5 delivers frames via [`useFrameOutput`](https://visioncamera.margelo.com/docs/frame-output). Inside the `onFrame` worklet you call `runOnFrame(frame, isFrontCamera)` synchronously, then use `scheduleOnRN` from `react-native-worklets` to post the result back to React state on the main thread.

The `isFrontCamera` parameter tells the native side whether the front camera is active so it can correctly mirror the results. The library handles all device orientation rotation internally — results are always returned in screen-space coordinates regardless of how the user holds their device.

:::warning

- You **must** set `pixelFormat: 'rgb'` in `useFrameOutput`. Our extraction pipeline expects RGB pixel data — any other format (e.g. the default `yuv`) will produce incorrect results.
- `runOnFrame` is synchronous and runs on the JS worklet thread. For models with longer inference times, use `dropFramesWhileBusy: true` to skip frames and avoid blocking the camera pipeline. For more control, see VisionCamera's [async frame processing guide](https://react-native-vision-camera-v5-docs.vercel.app/docs/async-frame-processing).
- `runOnFrame` is synchronous and runs on the JS worklet thread. For models with longer inference times, use `dropFramesWhileBusy: true` to skip frames and avoid blocking the camera pipeline. For more control, see VisionCamera's [async frame processing guide](https://visioncamera.margelo.com/docs/async-frame-processing).
- Always call `frame.dispose()` after processing to release the frame buffer. Wrap your inference in a `try/finally` to ensure it's always called even if `runOnFrame` throws.
:::

Expand Down Expand Up @@ -304,7 +304,7 @@ You are not passing `isFrontCamera: true` when using the front camera. See [Hand

#### App freezes or camera drops frames

Your model's inference time exceeds the frame interval. Enable `dropFramesWhileBusy: true` in `useFrameOutput`, or move inference off the worklet thread using VisionCamera's [async frame processing](https://react-native-vision-camera-v5-docs.vercel.app/docs/async-frame-processing).
Your model's inference time exceeds the frame interval. Enable `dropFramesWhileBusy: true` in `useFrameOutput`, or move inference off the worklet thread using VisionCamera's [async frame processing](https://visioncamera.margelo.com/docs/async-frame-processing).

#### Memory leak / crash after many frames

Expand Down
Loading
Loading