Skip to content

Commit 44676fc

Browse files
NorbertKlockiewiczNorbertKlockiewicz
committed
feat: add example screen with vision camera to computer vision app
1 parent ffcf72f commit 44676fc

8 files changed

Lines changed: 995 additions & 15 deletions

File tree

‎.yarn/patches/react-native-vision-camera@npm-5.0.0-beta.1.patch‎

Lines changed: 713 additions & 0 deletions
Large diffs are not rendered by default.

‎apps/computer-vision/app.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525
"foregroundImage": "./assets/icons/adaptive-icon.png",
2626
"backgroundColor": "#ffffff"
2727
},
28-
"package": "com.anonymous.computervision"
28+
"package": "com.anonymous.computervision",
29+
"permissions": ["android.permission.CAMERA"]
2930
},
3031
"web": {
3132
"favicon": "./assets/icons/favicon.png"

‎apps/computer-vision/app/_layout.tsx‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,14 @@ export default function _layout() {
8383
headerTitleStyle: { color: ColorPalette.primary },
8484
}}
8585
/>
86+
<Drawer.Screen
87+
name="object_detection_live/index"
88+
options={{
89+
drawerLabel: 'Object Detection (Live)',
90+
title: 'Object Detection (Live)',
91+
headerTitleStyle: { color: ColorPalette.primary },
92+
}}
93+
/>
8694
<Drawer.Screen
8795
name="ocr/index"
8896
options={{

‎apps/computer-vision/app/index.tsx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@ export default function Home() {
2929
>
3030
<Text style={styles.buttonText}>Object Detection</Text>
3131
</TouchableOpacity>
32+
<TouchableOpacity
33+
style={styles.button}
34+
onPress={() => router.navigate('object_detection_live/')}
35+
>
36+
<Text style={styles.buttonText}>Object Detection Live</Text>
37+
</TouchableOpacity>
3238
<TouchableOpacity
3339
style={styles.button}
3440
onPress={() => router.navigate('ocr/')}

‎apps/computer-vision/app/object_detection_live/index.tsx‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
import React, {
2+
useCallback,
3+
useContext,
4+
useEffect,
5+
useMemo,
6+
useRef,
7+
useState,
8+
} from 'react';
9+
import {
10+
StatusBar,
11+
StyleSheet,
12+
Text,
13+
TouchableOpacity,
14+
View,
15+
} from 'react-native';
16+
import { useSafeAreaInsets } from 'react-native-safe-area-context';
17+
18+
import {
19+
Camera,
20+
getCameraFormat,
21+
Templates,
22+
useCameraDevices,
23+
useCameraPermission,
24+
useFrameOutput,
25+
} from 'react-native-vision-camera';
26+
import { scheduleOnRN } from 'react-native-worklets';
27+
import {
28+
Detection,
29+
SSDLITE_320_MOBILENET_V3_LARGE,
30+
useObjectDetection,
31+
} from 'react-native-executorch';
32+
import { GeneratingContext } from '../../context';
33+
import Spinner from '../../components/Spinner';
34+
import ColorPalette from '../../colors';
35+
36+
export default function ObjectDetectionLiveScreen() {
37+
const insets = useSafeAreaInsets();
38+
39+
const model = useObjectDetection({ model: SSDLITE_320_MOBILENET_V3_LARGE });
40+
const { setGlobalGenerating } = useContext(GeneratingContext);
41+
42+
useEffect(() => {
43+
setGlobalGenerating(model.isGenerating);
44+
}, [model.isGenerating, setGlobalGenerating]);
45+
46+
const [detectionCount, setDetectionCount] = useState(0);
47+
const [fps, setFps] = useState(0);
48+
const lastFrameTimeRef = useRef(Date.now());
49+
50+
const cameraPermission = useCameraPermission();
51+
const devices = useCameraDevices();
52+
const device = devices.find((d) => d.position === 'back') ?? devices[0];
53+
54+
const format = useMemo(() => {
55+
if (device == null) return undefined;
56+
try {
57+
return getCameraFormat(device, Templates.Video);
58+
} catch {
59+
return undefined;
60+
}
61+
}, [device]);
62+
63+
const updateStats = useCallback((results: Detection[]) => {
64+
setDetectionCount(results.length);
65+
const now = Date.now();
66+
const timeDiff = now - lastFrameTimeRef.current;
67+
if (timeDiff > 0) {
68+
setFps(Math.round(1000 / timeDiff));
69+
}
70+
lastFrameTimeRef.current = now;
71+
}, []);
72+
73+
const frameOutput = useFrameOutput({
74+
pixelFormat: 'rgb',
75+
onFrame(frame) {
76+
'worklet';
77+
if (!model.runOnFrame) {
78+
frame.dispose();
79+
return;
80+
}
81+
try {
82+
const result = model.runOnFrame(frame, 0.5);
83+
if (result) {
84+
scheduleOnRN(updateStats, result);
85+
}
86+
} catch {
87+
// ignore frame errors
88+
} finally {
89+
frame.dispose();
90+
}
91+
},
92+
});
93+
94+
if (!model.isReady) {
95+
return (
96+
<Spinner
97+
visible={!model.isReady}
98+
textContent={`Loading the model ${(model.downloadProgress * 100).toFixed(0)} %`}
99+
/>
100+
);
101+
}
102+
103+
if (!cameraPermission.hasPermission) {
104+
return (
105+
<View style={styles.centered}>
106+
<Text style={styles.message}>Camera access needed</Text>
107+
<TouchableOpacity
108+
onPress={() => cameraPermission.requestPermission()}
109+
style={styles.button}
110+
>
111+
<Text style={styles.buttonText}>Grant Permission</Text>
112+
</TouchableOpacity>
113+
</View>
114+
);
115+
}
116+
117+
if (device == null) {
118+
return (
119+
<View style={styles.centered}>
120+
<Text style={styles.message}>No camera device found</Text>
121+
</View>
122+
);
123+
}
124+
125+
return (
126+
<View style={styles.container}>
127+
<StatusBar barStyle="light-content" translucent />
128+
129+
<Camera
130+
style={StyleSheet.absoluteFill}
131+
device={device}
132+
outputs={[frameOutput]}
133+
isActive={true}
134+
format={format}
135+
/>
136+
137+
<View
138+
style={[styles.bottomBarWrapper, { paddingBottom: insets.bottom + 12 }]}
139+
pointerEvents="none"
140+
>
141+
<View style={styles.bottomBar}>
142+
<View style={styles.statItem}>
143+
<Text style={styles.statValue}>{detectionCount}</Text>
144+
<Text style={styles.statLabel}>objects</Text>
145+
</View>
146+
<View style={styles.statDivider} />
147+
<View style={styles.statItem}>
148+
<Text style={styles.statValue}>{fps}</Text>
149+
<Text style={styles.statLabel}>fps</Text>
150+
</View>
151+
</View>
152+
</View>
153+
</View>
154+
);
155+
}
156+
157+
const styles = StyleSheet.create({
158+
container: {
159+
flex: 1,
160+
backgroundColor: 'black',
161+
},
162+
centered: {
163+
flex: 1,
164+
backgroundColor: 'black',
165+
justifyContent: 'center',
166+
alignItems: 'center',
167+
gap: 16,
168+
},
169+
message: {
170+
color: 'white',
171+
fontSize: 18,
172+
},
173+
button: {
174+
paddingHorizontal: 24,
175+
paddingVertical: 12,
176+
backgroundColor: ColorPalette.primary,
177+
borderRadius: 24,
178+
},
179+
buttonText: {
180+
color: 'white',
181+
fontSize: 15,
182+
fontWeight: '600',
183+
letterSpacing: 0.3,
184+
},
185+
186+
// Bottom stats bar
187+
bottomBarWrapper: {
188+
position: 'absolute',
189+
bottom: 0,
190+
left: 0,
191+
right: 0,
192+
alignItems: 'center',
193+
},
194+
bottomBar: {
195+
flexDirection: 'row',
196+
alignItems: 'center',
197+
backgroundColor: 'rgba(0, 0, 0, 0.55)',
198+
borderRadius: 24,
199+
paddingHorizontal: 28,
200+
paddingVertical: 10,
201+
gap: 24,
202+
},
203+
statItem: {
204+
alignItems: 'center',
205+
},
206+
statValue: {
207+
color: 'white',
208+
fontSize: 22,
209+
fontWeight: '700',
210+
letterSpacing: -0.5,
211+
},
212+
statLabel: {
213+
color: 'rgba(255,255,255,0.55)',
214+
fontSize: 11,
215+
fontWeight: '500',
216+
textTransform: 'uppercase',
217+
letterSpacing: 0.8,
218+
},
219+
statDivider: {
220+
width: 1,
221+
height: 32,
222+
backgroundColor: 'rgba(255,255,255,0.2)',
223+
},
224+
});

‎apps/computer-vision/package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,14 @@
3131
"react-native-gesture-handler": "~2.28.0",
3232
"react-native-image-picker": "^7.2.2",
3333
"react-native-loading-spinner-overlay": "^3.0.1",
34-
"react-native-nitro-image": "0.10.2",
35-
"react-native-nitro-modules": "0.33.4",
34+
"react-native-nitro-image": "^0.12.0",
35+
"react-native-nitro-modules": "^0.33.9",
3636
"react-native-reanimated": "~4.2.1",
3737
"react-native-safe-area-context": "~5.6.0",
3838
"react-native-screens": "~4.16.0",
3939
"react-native-svg": "15.12.1",
4040
"react-native-svg-transformer": "^1.5.0",
41+
"react-native-vision-camera": "patch:react-native-vision-camera@npm%3A5.0.0-beta.1#~/.yarn/patches/react-native-vision-camera@npm-5.0.0-beta.1.patch",
4142
"react-native-worklets": "^0.7.2"
4243
},
4344
"devDependencies": {

‎packages/react-native-executorch/src/types/objectDetection.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,10 +225,10 @@ export interface ObjectDetectionType {
225225
* ```
226226
*
227227
* @param frame - VisionCamera Frame object
228-
* @param detectionThreshold - The threshold for detection sensitivity. Default is 0.5.
228+
* @param detectionThreshold - The threshold for detection sensitivity.
229229
* @returns Array of Detection objects representing detected items in the frame.
230230
*/
231231
runOnFrame:
232-
| ((frame: Frame, detectionThreshold?: number) => Detection[])
232+
| ((frame: Frame, detectionThreshold: number) => Detection[])
233233
| null;
234234
}

‎yarn.lock‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6585,13 +6585,14 @@ __metadata:
65856585
react-native-gesture-handler: "npm:~2.28.0"
65866586
react-native-image-picker: "npm:^7.2.2"
65876587
react-native-loading-spinner-overlay: "npm:^3.0.1"
6588-
react-native-nitro-image: "npm:0.10.2"
6589-
react-native-nitro-modules: "npm:0.33.4"
6588+
react-native-nitro-image: "npm:^0.12.0"
6589+
react-native-nitro-modules: "npm:^0.33.9"
65906590
react-native-reanimated: "npm:~4.2.1"
65916591
react-native-safe-area-context: "npm:~5.6.0"
65926592
react-native-screens: "npm:~4.16.0"
65936593
react-native-svg: "npm:15.12.1"
65946594
react-native-svg-transformer: "npm:^1.5.0"
6595+
react-native-vision-camera: "patch:react-native-vision-camera@npm%3A5.0.0-beta.1#~/.yarn/patches/react-native-vision-camera@npm-5.0.0-beta.1.patch"
65956596
react-native-worklets: "npm:^0.7.2"
65966597
languageName: unknown
65976598
linkType: soft
@@ -13394,24 +13395,24 @@ __metadata:
1339413395
languageName: node
1339513396
linkType: hard
1339613397

13397-
"react-native-nitro-image@npm:0.10.2":
13398-
version: 0.10.2
13399-
resolution: "react-native-nitro-image@npm:0.10.2"
13398+
"react-native-nitro-image@npm:^0.12.0":
13399+
version: 0.12.0
13400+
resolution: "react-native-nitro-image@npm:0.12.0"
1340013401
peerDependencies:
1340113402
react: "*"
1340213403
react-native: "*"
1340313404
react-native-nitro-modules: "*"
13404-
checksum: 10/3be75e93da369adfe00441dae78171572dec38d3d7e75e5d4cb302b81479be9686c8d8dc0ea4b331514b8725099bf3eb069ab9933f7029627d12a72d71766cb4
13405+
checksum: 10/03f165381c35e060d4d05eae3ce029b32a4009482f327e9526840f306181ca87a862b335e12667c55d4ee9f2069542ca93dd112feb7f1822bf7d2ddc38fe58f0
1340513406
languageName: node
1340613407
linkType: hard
1340713408

13408-
"react-native-nitro-modules@npm:0.33.4":
13409-
version: 0.33.4
13410-
resolution: "react-native-nitro-modules@npm:0.33.4"
13409+
"react-native-nitro-modules@npm:^0.33.9":
13410+
version: 0.33.9
13411+
resolution: "react-native-nitro-modules@npm:0.33.9"
1341113412
peerDependencies:
1341213413
react: "*"
1341313414
react-native: "*"
13414-
checksum: 10/a737ff6b142c55821688612305245fd10a7cff36f0ee66cad0956c6815a60cdd4ba64cdfba6137a6dbfe815645763ce5d406cf488876edd47dab7f8d0031e01a
13415+
checksum: 10/4ebf4db46d1e4987a0e52054724081aa9712bcd1d505a6dbdd47aebc6afe72a7abaa0e947651d9f3cc594e4eb3dba47fc6f59db27c5a5ed383946e40d96543a0
1341513416
languageName: node
1341613417
linkType: hard
1341713418

@@ -13497,6 +13498,32 @@ __metadata:
1349713498
languageName: node
1349813499
linkType: hard
1349913500

13501+
"react-native-vision-camera@npm:5.0.0-beta.1":
13502+
version: 5.0.0-beta.1
13503+
resolution: "react-native-vision-camera@npm:5.0.0-beta.1"
13504+
peerDependencies:
13505+
react: "*"
13506+
react-native: "*"
13507+
react-native-nitro-image: "*"
13508+
react-native-nitro-modules: "*"
13509+
react-native-worklets: "*"
13510+
checksum: 10/873410a33e33d68b162b6524997480133ef9b6469dce3f87253c371bba1643d326e835891b0c9f75018d376faf4aec23daba5ab729f431c718ecf901601a8d12
13511+
languageName: node
13512+
linkType: hard
13513+
13514+
"react-native-vision-camera@patch:react-native-vision-camera@npm%3A5.0.0-beta.1#~/.yarn/patches/react-native-vision-camera@npm-5.0.0-beta.1.patch":
13515+
version: 5.0.0-beta.1
13516+
resolution: "react-native-vision-camera@patch:react-native-vision-camera@npm%3A5.0.0-beta.1#~/.yarn/patches/react-native-vision-camera@npm-5.0.0-beta.1.patch::version=5.0.0-beta.1&hash=b52326"
13517+
peerDependencies:
13518+
react: "*"
13519+
react-native: "*"
13520+
react-native-nitro-image: "*"
13521+
react-native-nitro-modules: "*"
13522+
react-native-worklets: "*"
13523+
checksum: 10/4ddf9325752243c92c5104b2fe8520d91072d4c359c52708872909b2bb85d136db59215bac1c6f902f04eee683a9d3d8ff11f7729e0468b00dee5aa3bb8f1944
13524+
languageName: node
13525+
linkType: hard
13526+
1350013527
"react-native-worklets@npm:0.5.1":
1350113528
version: 0.5.1
1350213529
resolution: "react-native-worklets@npm:0.5.1"

0 commit comments

Comments
 (0)