Skip to content

Commit bf00556

Browse files
barhancmsluszniak
barhanc
and
msluszniak
authored
[RNE Rewrite] feat: add keypoint detection task pipeline (#1280)
## Description Adds kepoint detection task pipeline implementation and corresponding example screen in computer-vision app. ### Introduces a breaking change? - [ ] Yes - [x] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [x] New feature (change which adds functionality) - [ ] Documentation update (improves or adds clarity to existing documentation) - [ ] Other (chores, tests, code style improvements etc.) ### Tested on - [x] iOS - [ ] Android ### Testing instructions - [ ] Build computer-vision example app on both Android and iOS. - [ ] Test the newly added keypoint detection screen. - [ ] Check updated HuggingFace repos: - https://huggingface.co/software-mansion/react-native-executorch-blazeface - ~https://huggingface.co/software-mansion/react-native-executorch-yolov8n-pose~ - https://huggingface.co/software-mansion/react-native-executorch-yolo26-pose ### Screenshots <!-- Add screenshots here, if applicable --> ### Related issues Closes #1241 ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have updated the documentation accordingly - [x] My changes generate no new warnings ### Additional notes <!-- Include any additional information, assumptions, or context that reviewers might need to understand this PR. --> --------- Co-authored-by: Mateusz Sluszniak <56299341+msluszniak@users.noreply.github.com>
1 parent 862b77d commit bf00556

17 files changed

Lines changed: 1250 additions & 0 deletions

File tree

‎.cspell-wordlist.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,10 +234,13 @@ Amdahl
234234
Amdahl's
235235
xyxy
236236
xywh
237+
cxcywh
237238
subfolders
238239
podspec
239240
logcat
240241
modelname
241242
optionalsize
242243
pushd
243244
popd
245+
yolov
246+
YOLOV

‎apps/computer-vision/app/_layout.tsx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,13 @@ export default function Layout() {
4141
title: 'Semantic Segmentation',
4242
}}
4343
/>
44+
<Drawer.Screen
45+
name="keypoint/index"
46+
options={{
47+
drawerLabel: 'Keypoint Detection',
48+
title: 'Keypoint Detection',
49+
}}
50+
/>
4451
<Drawer.Screen
4552
name="inspect/index"
4653
options={{

‎apps/computer-vision/app/index.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ export default function Home() {
2020
<TouchableOpacity style={styles.button} onPress={() => router.navigate('segmentation/')}>
2121
<Text style={styles.buttonText}>Semantic Segmentation</Text>
2222
</TouchableOpacity>
23+
<TouchableOpacity style={styles.button} onPress={() => router.navigate('keypoint/')}>
24+
<Text style={styles.buttonText}>Keypoint Detection</Text>
25+
</TouchableOpacity>
2326
<TouchableOpacity style={styles.button} onPress={() => router.navigate('inspect/')}>
2427
<Text style={styles.buttonText}>Model Inspector</Text>
2528
</TouchableOpacity>

‎apps/computer-vision/app/keypoint/index.tsx‎

Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,279 @@
1+
import React, { useState } from 'react';
2+
import { View, Text, StyleSheet, ScrollView, Dimensions, Platform } from 'react-native';
3+
import { useSafeAreaInsets } from 'react-native-safe-area-context';
4+
import { commonStyles, theme } from '../../theme';
5+
import { useImage } from '@shopify/react-native-skia';
6+
import { useKeypointDetector, models, type KeypointDetection } from 'react-native-executorch';
7+
import ScreenWrapper from '../../components/ScreenWrapper';
8+
import { getImage } from '../../utils';
9+
import { ModelPicker, type ModelOption } from '../../components/ModelPicker';
10+
import { ImageViewport } from '../../components/ImageViewport';
11+
import { ModelStatus } from '../../components/ModelStatus';
12+
import { LatencyIndicator } from '../../components/LatencyIndicator';
13+
import { Button } from '../../components/Button';
14+
import { BoundingBox } from '../../components/BoundingBox';
15+
16+
const MODEL_OPTIONS: ModelOption[] = [
17+
{
18+
label: 'BlazeFace (XNNPACK FP32)',
19+
value: models.keypointDetection.BLAZEFACE,
20+
},
21+
{
22+
label: 'YOLO26 Pose (XNNPACK FP32)',
23+
value: models.keypointDetection.YOLO26_POSE.SIZE_384.XNNPACK_FP32,
24+
},
25+
{
26+
label: 'RF-DETR Keypoint (XNNPACK FP32)',
27+
value: models.keypointDetection.RFDETR_KEYPOINT.XNNPACK_FP32,
28+
},
29+
{
30+
label: 'RF-DETR Keypoint (CoreML FP32)',
31+
value: models.keypointDetection.RFDETR_KEYPOINT.COREML_FP32,
32+
disabled: Platform.OS !== 'ios',
33+
},
34+
{
35+
label: 'RF-DETR Keypoint (MLX FP32)',
36+
value: models.keypointDetection.RFDETR_KEYPOINT.MLX_FP32,
37+
disabled: Platform.OS !== 'ios',
38+
},
39+
];
40+
41+
const VIEW_WIDTH = Dimensions.get('window').width - 32;
42+
const VIEW_HEIGHT = Math.round((VIEW_WIDTH * 16) / 9);
43+
44+
function KeypointContent() {
45+
const insets = useSafeAreaInsets();
46+
const [selectedModel, setSelectedModel] = useState<any>(MODEL_OPTIONS[0].value);
47+
const [imageUri, setImageUri] = useState<string | null>(null);
48+
const [isProcessing, setIsProcessing] = useState(false);
49+
const [results, setResults] = useState<KeypointDetection<'xyxy', string>[]>([]);
50+
const [latency, setLatency] = useState<number | null>(null);
51+
const [error, setError] = useState<string | null>(null);
52+
53+
const skiaImage = useImage(imageUri, (err) => setError(err.message || String(err)));
54+
55+
const {
56+
isReady,
57+
downloadProgress,
58+
error: loadError,
59+
detectKeypoints,
60+
detectKeypointsWorklet,
61+
} = useKeypointDetector(selectedModel);
62+
63+
const handlePickImage = async (useCamera: boolean) => {
64+
setError(null);
65+
try {
66+
const uri = await getImage(useCamera);
67+
if (uri) {
68+
setImageUri(uri);
69+
setResults([]);
70+
setLatency(null);
71+
}
72+
} catch (e: any) {
73+
setError(e.message || String(e));
74+
}
75+
};
76+
77+
const runDetection = async (sync: boolean) => {
78+
if (!skiaImage || !detectKeypoints || !detectKeypointsWorklet) return;
79+
if (!sync) setIsProcessing(true);
80+
setError(null);
81+
try {
82+
const pixels = skiaImage.readPixels();
83+
if (!pixels) {
84+
throw new Error('Failed to read pixels from image');
85+
}
86+
if (!(pixels instanceof Uint8Array)) {
87+
throw new Error('Expected Uint8Array from readPixels');
88+
}
89+
const buffer = {
90+
data: pixels,
91+
width: skiaImage.width(),
92+
height: skiaImage.height(),
93+
format: 'rgba' as const,
94+
layout: 'hwc' as const,
95+
};
96+
const start = Date.now();
97+
const output = (
98+
sync ? detectKeypointsWorklet(buffer) : await detectKeypoints(buffer)
99+
) as KeypointDetection<'xyxy', string>[];
100+
101+
setLatency(Date.now() - start);
102+
setResults(output);
103+
} catch (e: any) {
104+
setError(e.message || String(e));
105+
} finally {
106+
if (!sync) setIsProcessing(false);
107+
}
108+
};
109+
110+
let scaleX = 1;
111+
let scaleY = 1;
112+
let offsetX = 0;
113+
let offsetY = 0;
114+
115+
if (skiaImage) {
116+
const imgW = skiaImage.width();
117+
const imgH = skiaImage.height();
118+
const scale = Math.min(VIEW_WIDTH / imgW, VIEW_HEIGHT / imgH);
119+
const displayedW = imgW * scale;
120+
const displayedH = imgH * scale;
121+
offsetX = (VIEW_WIDTH - displayedW) / 2;
122+
offsetY = (VIEW_HEIGHT - displayedH) / 2;
123+
scaleX = scale;
124+
scaleY = scale;
125+
}
126+
127+
const activeError = loadError ? String(loadError) : error;
128+
129+
return (
130+
<ScrollView
131+
style={commonStyles.container}
132+
contentContainerStyle={[
133+
commonStyles.contentContainer,
134+
{ paddingBottom: insets.bottom + theme.spacing.large },
135+
]}
136+
>
137+
<Text style={commonStyles.description}>
138+
Upload or capture an image to run keypoint/pose estimation on it.
139+
</Text>
140+
141+
<ModelPicker
142+
label="Model"
143+
options={MODEL_OPTIONS}
144+
selectedValue={selectedModel}
145+
onValueChange={(model) => {
146+
setSelectedModel(model);
147+
setResults([]);
148+
setLatency(null);
149+
setError(null);
150+
}}
151+
/>
152+
153+
<ModelStatus
154+
isReady={isReady}
155+
downloadProgress={downloadProgress}
156+
error={activeError}
157+
modelTypeLabel="keypoint model"
158+
/>
159+
160+
<ImageViewport skiaImage={skiaImage} onPressPlaceholder={() => handlePickImage(false)}>
161+
{skiaImage && results.length > 0 && (
162+
<View style={styles.overlayContainer} pointerEvents="none">
163+
{results.map((det, index: number) => {
164+
const strokeColor = '#00ff00';
165+
const bgColor = 'rgba(0, 255, 0, 0.15)';
166+
const landmarkColor = '#ff00ff';
167+
168+
const left = offsetX + det.box.xmin * scaleX;
169+
const top = offsetY + det.box.ymin * scaleY;
170+
const width = (det.box.xmax - det.box.xmin) * scaleX;
171+
const height = (det.box.ymax - det.box.ymin) * scaleY;
172+
173+
return (
174+
<React.Fragment key={index}>
175+
{/* Bounding Box */}
176+
<BoundingBox
177+
left={left}
178+
top={top}
179+
width={width}
180+
height={height}
181+
borderColor={strokeColor}
182+
backgroundColor={bgColor}
183+
label={`Det ${Math.round(det.confidence * 100)}%`}
184+
/>
185+
186+
{/* Landmarks */}
187+
{Object.entries(det.landmarks).map(([key, point]) => {
188+
const x = offsetX + point.x * scaleX;
189+
const y = offsetY + point.y * scaleY;
190+
return (
191+
<View
192+
key={key}
193+
style={[
194+
styles.landmarkContainer,
195+
{
196+
left: x - 50,
197+
top: y - 4,
198+
},
199+
]}
200+
>
201+
<View style={[styles.landmarkDot, { backgroundColor: landmarkColor }]} />
202+
<Text style={[styles.landmarkText, { color: landmarkColor }]}>
203+
{key}: {Math.round(point.confidence * 100)}%
204+
</Text>
205+
</View>
206+
);
207+
})}
208+
</React.Fragment>
209+
);
210+
})}
211+
</View>
212+
)}
213+
</ImageViewport>
214+
215+
<View style={commonStyles.buttonRow}>
216+
<Button title="Gallery" onPress={() => handlePickImage(false)} variant="secondary" />
217+
<Button title="Camera" onPress={() => handlePickImage(true)} variant="secondary" />
218+
</View>
219+
220+
<View style={commonStyles.buttonRow}>
221+
<Button
222+
title="Run Async"
223+
onPress={() => runDetection(false)}
224+
disabled={!skiaImage || !isReady || isProcessing}
225+
loading={isProcessing}
226+
/>
227+
<Button
228+
title="Run Sync"
229+
onPress={() => runDetection(true)}
230+
disabled={!skiaImage || !isReady || isProcessing}
231+
variant="accent"
232+
/>
233+
</View>
234+
235+
<LatencyIndicator latency={latency} />
236+
</ScrollView>
237+
);
238+
}
239+
240+
export default function KeypointScreen() {
241+
return (
242+
<ScreenWrapper>
243+
<KeypointContent />
244+
</ScreenWrapper>
245+
);
246+
}
247+
248+
const styles = StyleSheet.create({
249+
overlayContainer: {
250+
position: 'absolute',
251+
left: 0,
252+
right: 0,
253+
top: 0,
254+
bottom: 0,
255+
overflow: 'hidden',
256+
},
257+
landmarkContainer: {
258+
position: 'absolute',
259+
width: 100,
260+
alignItems: 'center',
261+
},
262+
landmarkDot: {
263+
width: 8,
264+
height: 8,
265+
borderRadius: 4,
266+
backgroundColor: '#ff00ff',
267+
borderWidth: 1,
268+
borderColor: '#fff',
269+
},
270+
landmarkText: {
271+
color: '#ff00ff',
272+
fontSize: 8,
273+
fontWeight: 'bold',
274+
textShadowColor: '#000',
275+
textShadowOffset: { width: 1, height: 1 },
276+
textShadowRadius: 1,
277+
textAlign: 'center',
278+
},
279+
});

‎apps/computer-vision/components/BoundingBox.tsx‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import React from 'react';
2+
import { View, Text, StyleSheet } from 'react-native';
3+
4+
export interface BoundingBoxProps {
5+
left: number;
6+
top: number;
7+
width: number;
8+
height: number;
9+
label?: string;
10+
borderColor?: string;
11+
backgroundColor?: string;
12+
labelTextColor?: string;
13+
}
14+
15+
export function BoundingBox({
16+
left,
17+
top,
18+
width,
19+
height,
20+
label,
21+
borderColor = '#00ff00',
22+
backgroundColor = 'rgba(0, 255, 0, 0.15)',
23+
labelTextColor = '#000',
24+
}: BoundingBoxProps) {
25+
return (
26+
<View
27+
style={[
28+
styles.detectionBox,
29+
{
30+
left,
31+
top,
32+
width,
33+
height,
34+
borderColor,
35+
backgroundColor,
36+
},
37+
]}
38+
pointerEvents="none"
39+
>
40+
{label ? (
41+
<View style={[styles.boxLabelBadge, { backgroundColor: borderColor }]}>
42+
<Text style={[styles.boxLabelText, { color: labelTextColor }]}>{label}</Text>
43+
</View>
44+
) : null}
45+
</View>
46+
);
47+
}
48+
49+
const styles = StyleSheet.create({
50+
detectionBox: {
51+
position: 'absolute',
52+
borderWidth: 2,
53+
borderRadius: 4,
54+
},
55+
boxLabelBadge: {
56+
position: 'absolute',
57+
top: -20,
58+
left: -2,
59+
paddingHorizontal: 6,
60+
paddingVertical: 2,
61+
borderTopLeftRadius: 4,
62+
borderTopRightRadius: 4,
63+
},
64+
boxLabelText: {
65+
fontSize: 10,
66+
fontWeight: 'bold',
67+
},
68+
});

0 commit comments

Comments
 (0)