Skip to content

Commit ad0d2f3

Browse files
msluszniakmsluszniak
authored
chore: Fix bounding box annotation (#982)
## Description This PR fixes the problem in demo apps where bounding boxes were utilized. For small objects annotation width was set to width of the bounding box since they were in the same view. Fix is to make these two be in relation child - parent components. ### Introduces a breaking change? - [ ] Yes - [x] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [ ] New feature (change which adds functionality) - [ ] Documentation update (improves or adds clarity to existing documentation) - [x] Other (chores, tests, code style improvements etc.) ### Tested on - [ ] iOS - [x] Android ### Testing instructions - [ ] Compare result of annotation for apps that utilizes bounding boxes for small detected objects. ### Screenshots Now bounding boxes on small objects looks correct (in the #975 was the original result for the same image): <img width="553" height="432" alt="image" src="https://github.com/user-attachments/assets/2598df7e-351e-4441-ac32-7272c9519a89" /> ### Related issues Closes #975 - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have updated the documentation accordingly - [x] My changes generate no new warnings ### Additional notes <!-- Include any additional information, assumptions, or context that reviewers might need to understand this PR. -->
1 parent 3e7242c commit ad0d2f3

File tree

6 files changed

+117
-91
lines changed

6 files changed

+117
-91
lines changed

‎apps/computer-vision/components/BoundingBoxes.tsx‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import React from 'react';
2+
import { StyleSheet, Text, View } from 'react-native';
3+
import { Detection, LabelEnum } from 'react-native-executorch';
4+
import { labelColor, labelColorBg } from './utils/colors';
5+
6+
interface Props<L extends LabelEnum> {
7+
detections: Detection<L>[];
8+
scaleX: number;
9+
scaleY: number;
10+
offsetX: number;
11+
offsetY: number;
12+
mirrorLabels?: boolean;
13+
containerWidth?: number;
14+
}
15+
16+
export default function BoundingBoxes<L extends LabelEnum>({
17+
detections,
18+
scaleX,
19+
scaleY,
20+
offsetX,
21+
offsetY,
22+
mirrorLabels = false,
23+
containerWidth,
24+
}: Props<L>) {
25+
return (
26+
<>
27+
{detections.map((det, i) => {
28+
const left = det.bbox.x1 * scaleX + offsetX;
29+
const top = det.bbox.y1 * scaleY + offsetY;
30+
const width = (det.bbox.x2 - det.bbox.x1) * scaleX;
31+
const height = (det.bbox.y2 - det.bbox.y1) * scaleY;
32+
const labelTop = top < 26 ? top + height + 2 : top - 26;
33+
34+
return (
35+
<React.Fragment key={i}>
36+
<View
37+
style={[
38+
styles.bbox,
39+
{
40+
left,
41+
top,
42+
width,
43+
height,
44+
borderColor: labelColor(det.label as string),
45+
},
46+
]}
47+
/>
48+
<View
49+
style={[
50+
styles.label,
51+
{
52+
left,
53+
top: labelTop,
54+
backgroundColor: labelColorBg(det.label as string),
55+
...(containerWidth !== undefined && {
56+
maxWidth: containerWidth - left,
57+
}),
58+
},
59+
mirrorLabels && { transform: [{ scaleX: -1 }] },
60+
]}
61+
>
62+
<Text style={styles.labelText} numberOfLines={1}>
63+
{String(det.label)} ({(det.score * 100).toFixed(1)}%)
64+
</Text>
65+
</View>
66+
</React.Fragment>
67+
);
68+
})}
69+
</>
70+
);
71+
}
72+
73+
const styles = StyleSheet.create({
74+
bbox: {
75+
position: 'absolute',
76+
borderWidth: 2,
77+
borderRadius: 4,
78+
},
79+
label: {
80+
position: 'absolute',
81+
paddingHorizontal: 6,
82+
paddingVertical: 2,
83+
borderRadius: 4,
84+
},
85+
labelText: {
86+
color: 'white',
87+
fontSize: 11,
88+
fontWeight: '600',
89+
},
90+
});

‎apps/computer-vision/components/ImageWithBboxes.tsx‎

Lines changed: 15 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import React from 'react';
2-
import { Image, StyleSheet, View, Text } from 'react-native';
2+
import { Image, StyleSheet, View } from 'react-native';
33
import { Detection } from 'react-native-executorch';
4+
import BoundingBoxes from './BoundingBoxes';
45

56
interface Props {
67
imageUri: string;
@@ -21,7 +22,7 @@ export default function ImageWithBboxes({
2122
const imageRatio = imageWidth / imageHeight;
2223
const layoutRatio = layout.width / layout.height;
2324

24-
let sx, sy; // Scale in x and y directions
25+
let sx, sy;
2526
if (imageRatio > layoutRatio) {
2627
// image is more "wide"
2728
sx = layout.width / imageWidth;
@@ -35,11 +36,13 @@ export default function ImageWithBboxes({
3536
return {
3637
scaleX: sx,
3738
scaleY: sy,
38-
offsetX: (layout.width - imageWidth * sx) / 2, // Centering the image horizontally
39-
offsetY: (layout.height - imageHeight * sy) / 2, // Centering the image vertically
39+
offsetX: (layout.width - imageWidth * sx) / 2,
40+
offsetY: (layout.height - imageHeight * sy) / 2,
4041
};
4142
};
4243

44+
const { scaleX, scaleY, offsetX, offsetY } = calculateAdjustedDimensions();
45+
4346
return (
4447
<View
4548
style={styles.container}
@@ -57,24 +60,14 @@ export default function ImageWithBboxes({
5760
: require('../assets/icons/executorch_logo.png')
5861
}
5962
/>
60-
{detections.map((detection, index) => {
61-
const { scaleX, scaleY, offsetX, offsetY } =
62-
calculateAdjustedDimensions();
63-
const { x1, y1, x2, y2 } = detection.bbox;
64-
65-
const left = x1 * scaleX + offsetX;
66-
const top = y1 * scaleY + offsetY;
67-
const width = (x2 - x1) * scaleX;
68-
const height = (y2 - y1) * scaleY;
69-
70-
return (
71-
<View key={index} style={[styles.bbox, { left, top, width, height }]}>
72-
<Text style={styles.label}>
73-
{detection.label} ({(detection.score * 100).toFixed(1)}%)
74-
</Text>
75-
</View>
76-
);
77-
})}
63+
<BoundingBoxes
64+
detections={detections}
65+
scaleX={scaleX}
66+
scaleY={scaleY}
67+
offsetX={offsetX}
68+
offsetY={offsetY}
69+
containerWidth={layout.width}
70+
/>
7871
</View>
7972
);
8073
}
@@ -89,19 +82,4 @@ const styles = StyleSheet.create({
8982
width: '100%',
9083
height: '100%',
9184
},
92-
bbox: {
93-
position: 'absolute',
94-
borderWidth: 2,
95-
borderColor: 'red',
96-
},
97-
label: {
98-
position: 'absolute',
99-
top: -20,
100-
left: 0,
101-
backgroundColor: 'rgba(255, 0, 0, 0.7)',
102-
color: 'white',
103-
fontSize: 12,
104-
paddingHorizontal: 4,
105-
borderRadius: 4,
106-
},
10785
});

apps/computer-vision/components/vision_camera/utils/colors.ts renamed to apps/computer-vision/components/utils/colors.ts

File renamed without changes.

‎apps/computer-vision/components/vision_camera/tasks/InstanceSegmentationTask.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import {
1111
CocoLabelYolo,
1212
} from 'react-native-executorch';
1313
import { Canvas, Image as SkiaImage } from '@shopify/react-native-skia';
14-
import { labelColor, labelColorBg } from '../utils/colors';
14+
import { labelColor, labelColorBg } from '../../utils/colors';
1515
import { TaskProps } from './types';
1616
import {
1717
buildDisplayInstances,

‎apps/computer-vision/components/vision_camera/tasks/ObjectDetectionTask.tsx‎

Lines changed: 10 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import React, { useCallback, useEffect, useRef, useState } from 'react';
2-
import { StyleSheet, Text, View } from 'react-native';
2+
import { StyleSheet, View } from 'react-native';
33
import { Frame, useFrameOutput } from 'react-native-vision-camera';
44
import { scheduleOnRN } from 'react-native-worklets';
55
import {
@@ -8,7 +8,7 @@ import {
88
SSDLITE_320_MOBILENET_V3_LARGE,
99
useObjectDetection,
1010
} from 'react-native-executorch';
11-
import { labelColor, labelColorBg } from '../utils/colors';
11+
import BoundingBoxes from '../../BoundingBoxes';
1212
import { TaskProps } from './types';
1313

1414
type ObjModelId = 'objectDetectionSsdlite' | 'objectDetectionRfdetr';
@@ -122,56 +122,14 @@ export default function ObjectDetectionTask({
122122

123123
return (
124124
<View style={[StyleSheet.absoluteFill]} pointerEvents="none">
125-
{detections.map((det, i) => {
126-
const left = det.bbox.x1 * scale + offsetX;
127-
const top = det.bbox.y1 * scale + offsetY;
128-
const w = (det.bbox.x2 - det.bbox.x1) * scale;
129-
const h = (det.bbox.y2 - det.bbox.y1) * scale;
130-
return (
131-
<View
132-
key={i}
133-
style={[
134-
styles.bbox,
135-
{
136-
left,
137-
top,
138-
width: w,
139-
height: h,
140-
borderColor: labelColor(det.label),
141-
},
142-
]}
143-
>
144-
<View
145-
style={[
146-
styles.bboxLabel,
147-
{ backgroundColor: labelColorBg(det.label) },
148-
]}
149-
>
150-
<Text style={styles.bboxLabelText}>
151-
{det.label} {(det.score * 100).toFixed(1)}
152-
</Text>
153-
</View>
154-
</View>
155-
);
156-
})}
125+
<BoundingBoxes
126+
detections={detections}
127+
scaleX={scale}
128+
scaleY={scale}
129+
offsetX={offsetX}
130+
offsetY={offsetY}
131+
containerWidth={canvasSize.width}
132+
/>
157133
</View>
158134
);
159135
}
160-
161-
const styles = StyleSheet.create({
162-
bbox: {
163-
position: 'absolute',
164-
borderWidth: 2,
165-
borderColor: 'cyan',
166-
borderRadius: 4,
167-
},
168-
bboxLabel: {
169-
position: 'absolute',
170-
top: -22,
171-
left: -2,
172-
paddingHorizontal: 6,
173-
paddingVertical: 2,
174-
borderRadius: 4,
175-
},
176-
bboxLabelText: { color: 'white', fontSize: 11, fontWeight: '600' },
177-
});

‎apps/computer-vision/components/vision_camera/tasks/SegmentationTask.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import {
2020
Skia,
2121
SkImage,
2222
} from '@shopify/react-native-skia';
23-
import { CLASS_COLORS } from '../utils/colors';
23+
import { CLASS_COLORS } from '../../utils/colors';
2424
import { TaskProps } from './types';
2525

2626
type SegModelId =

0 commit comments

Comments
 (0)