Skip to content

Commit 66e358e

Browse files
NorbertKlockiewiczNorbertKlockiewicz
committed
feat: make all cv models compatible with Vision Camera
1 parent 73fbdab commit 66e358e

File tree

47 files changed

+3239
-315
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+3239
-315
lines changed

‎apps/computer-vision/app/_layout.tsx‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,46 @@ export default function _layout() {
9191
headerTitleStyle: { color: ColorPalette.primary },
9292
}}
9393
/>
94+
<Drawer.Screen
95+
name="classification_live/index"
96+
options={{
97+
drawerLabel: 'Classification (Live)',
98+
title: 'Classification (Live)',
99+
headerTitleStyle: { color: ColorPalette.primary },
100+
}}
101+
/>
102+
<Drawer.Screen
103+
name="image_segmentation_live/index"
104+
options={{
105+
drawerLabel: 'Image Segmentation (Live)',
106+
title: 'Image Segmentation (Live)',
107+
headerTitleStyle: { color: ColorPalette.primary },
108+
}}
109+
/>
110+
<Drawer.Screen
111+
name="style_transfer_live/index"
112+
options={{
113+
drawerLabel: 'Style Transfer (Live)',
114+
title: 'Style Transfer (Live)',
115+
headerTitleStyle: { color: ColorPalette.primary },
116+
}}
117+
/>
118+
<Drawer.Screen
119+
name="ocr_live/index"
120+
options={{
121+
drawerLabel: 'OCR (Live)',
122+
title: 'OCR (Live)',
123+
headerTitleStyle: { color: ColorPalette.primary },
124+
}}
125+
/>
126+
<Drawer.Screen
127+
name="vision_camera_live/index"
128+
options={{
129+
drawerLabel: 'Vision Camera (Live)',
130+
title: 'Vision Camera',
131+
headerTitleStyle: { color: ColorPalette.primary },
132+
}}
133+
/>
94134
<Drawer.Screen
95135
name="ocr/index"
96136
options={{

‎apps/computer-vision/app/classification_live/index.tsx‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
import React, {
2+
useCallback,
3+
useContext,
4+
useEffect,
5+
useMemo,
6+
useRef,
7+
useState,
8+
} from 'react';
9+
import {
10+
StatusBar,
11+
StyleSheet,
12+
Text,
13+
TouchableOpacity,
14+
View,
15+
} from 'react-native';
16+
import { useSafeAreaInsets } from 'react-native-safe-area-context';
17+
18+
import {
19+
Camera,
20+
getCameraFormat,
21+
Templates,
22+
useCameraDevices,
23+
useCameraPermission,
24+
useFrameOutput,
25+
} from 'react-native-vision-camera';
26+
import { scheduleOnRN } from 'react-native-worklets';
27+
import { EFFICIENTNET_V2_S, useClassification } from 'react-native-executorch';
28+
import { GeneratingContext } from '../../context';
29+
import Spinner from '../../components/Spinner';
30+
import ColorPalette from '../../colors';
31+
32+
export default function ClassificationLiveScreen() {
33+
const insets = useSafeAreaInsets();
34+
35+
const { isReady, isGenerating, downloadProgress, runOnFrame } =
36+
useClassification({ model: EFFICIENTNET_V2_S });
37+
const { setGlobalGenerating } = useContext(GeneratingContext);
38+
39+
useEffect(() => {
40+
setGlobalGenerating(isGenerating);
41+
}, [isGenerating, setGlobalGenerating]);
42+
43+
const [topLabel, setTopLabel] = useState('');
44+
const [topScore, setTopScore] = useState(0);
45+
const [fps, setFps] = useState(0);
46+
const lastFrameTimeRef = useRef(Date.now());
47+
48+
const cameraPermission = useCameraPermission();
49+
const devices = useCameraDevices();
50+
const device = devices.find((d) => d.position === 'back') ?? devices[0];
51+
52+
const format = useMemo(() => {
53+
if (device == null) return undefined;
54+
try {
55+
return getCameraFormat(device, Templates.FrameProcessing);
56+
} catch {
57+
return undefined;
58+
}
59+
}, [device]);
60+
61+
const updateStats = useCallback(
62+
(result: { label: string; score: number }) => {
63+
setTopLabel(result.label);
64+
setTopScore(result.score);
65+
const now = Date.now();
66+
const timeDiff = now - lastFrameTimeRef.current;
67+
if (timeDiff > 0) {
68+
setFps(Math.round(1000 / timeDiff));
69+
}
70+
lastFrameTimeRef.current = now;
71+
},
72+
[]
73+
);
74+
75+
const frameOutput = useFrameOutput({
76+
pixelFormat: 'rgb',
77+
onFrame(frame) {
78+
'worklet';
79+
if (!runOnFrame) {
80+
frame.dispose();
81+
return;
82+
}
83+
try {
84+
const result = runOnFrame(frame);
85+
if (result) {
86+
// find the top-1 entry
87+
let bestLabel = '';
88+
let bestScore = -1;
89+
const entries = Object.entries(result);
90+
for (let i = 0; i < entries.length; i++) {
91+
const [label, score] = entries[i];
92+
if ((score as number) > bestScore) {
93+
bestScore = score as number;
94+
bestLabel = label;
95+
}
96+
}
97+
scheduleOnRN(updateStats, { label: bestLabel, score: bestScore });
98+
}
99+
} catch {
100+
// ignore frame errors
101+
} finally {
102+
frame.dispose();
103+
}
104+
},
105+
});
106+
107+
if (!isReady) {
108+
return (
109+
<Spinner
110+
visible={!isReady}
111+
textContent={`Loading the model ${(downloadProgress * 100).toFixed(0)} %`}
112+
/>
113+
);
114+
}
115+
116+
if (!cameraPermission.hasPermission) {
117+
return (
118+
<View style={styles.centered}>
119+
<Text style={styles.message}>Camera access needed</Text>
120+
<TouchableOpacity
121+
onPress={() => cameraPermission.requestPermission()}
122+
style={styles.button}
123+
>
124+
<Text style={styles.buttonText}>Grant Permission</Text>
125+
</TouchableOpacity>
126+
</View>
127+
);
128+
}
129+
130+
if (device == null) {
131+
return (
132+
<View style={styles.centered}>
133+
<Text style={styles.message}>No camera device found</Text>
134+
</View>
135+
);
136+
}
137+
138+
return (
139+
<View style={styles.container}>
140+
<StatusBar barStyle="light-content" translucent />
141+
142+
<Camera
143+
style={StyleSheet.absoluteFill}
144+
device={device}
145+
outputs={[frameOutput]}
146+
isActive={true}
147+
format={format}
148+
/>
149+
150+
<View
151+
style={[styles.bottomBarWrapper, { paddingBottom: insets.bottom + 12 }]}
152+
pointerEvents="none"
153+
>
154+
<View style={styles.bottomBar}>
155+
<View style={styles.labelContainer}>
156+
<Text style={styles.labelText} numberOfLines={1}>
157+
{topLabel || '—'}
158+
</Text>
159+
<Text style={styles.scoreText}>
160+
{topLabel ? (topScore * 100).toFixed(1) + '%' : ''}
161+
</Text>
162+
</View>
163+
<View style={styles.statDivider} />
164+
<View style={styles.statItem}>
165+
<Text style={styles.statValue}>{fps}</Text>
166+
<Text style={styles.statLabel}>fps</Text>
167+
</View>
168+
</View>
169+
</View>
170+
</View>
171+
);
172+
}
173+
174+
const styles = StyleSheet.create({
175+
container: {
176+
flex: 1,
177+
backgroundColor: 'black',
178+
},
179+
centered: {
180+
flex: 1,
181+
backgroundColor: 'black',
182+
justifyContent: 'center',
183+
alignItems: 'center',
184+
gap: 16,
185+
},
186+
message: {
187+
color: 'white',
188+
fontSize: 18,
189+
},
190+
button: {
191+
paddingHorizontal: 24,
192+
paddingVertical: 12,
193+
backgroundColor: ColorPalette.primary,
194+
borderRadius: 24,
195+
},
196+
buttonText: {
197+
color: 'white',
198+
fontSize: 15,
199+
fontWeight: '600',
200+
letterSpacing: 0.3,
201+
},
202+
bottomBarWrapper: {
203+
position: 'absolute',
204+
bottom: 0,
205+
left: 0,
206+
right: 0,
207+
alignItems: 'center',
208+
paddingHorizontal: 16,
209+
},
210+
bottomBar: {
211+
flexDirection: 'row',
212+
alignItems: 'center',
213+
backgroundColor: 'rgba(0, 0, 0, 0.55)',
214+
borderRadius: 24,
215+
paddingHorizontal: 28,
216+
paddingVertical: 10,
217+
gap: 24,
218+
maxWidth: '100%',
219+
},
220+
labelContainer: {
221+
flex: 1,
222+
alignItems: 'flex-start',
223+
},
224+
labelText: {
225+
color: 'white',
226+
fontSize: 16,
227+
fontWeight: '700',
228+
},
229+
scoreText: {
230+
color: 'rgba(255,255,255,0.7)',
231+
fontSize: 13,
232+
fontWeight: '500',
233+
},
234+
statItem: {
235+
alignItems: 'center',
236+
},
237+
statValue: {
238+
color: 'white',
239+
fontSize: 22,
240+
fontWeight: '700',
241+
letterSpacing: -0.5,
242+
},
243+
statLabel: {
244+
color: 'rgba(255,255,255,0.55)',
245+
fontSize: 11,
246+
fontWeight: '500',
247+
textTransform: 'uppercase',
248+
letterSpacing: 0.8,
249+
},
250+
statDivider: {
251+
width: 1,
252+
height: 32,
253+
backgroundColor: 'rgba(255,255,255,0.2)',
254+
},
255+
});

0 commit comments

Comments
 (0)