Skip to content

Commit 9ca8288

Browse files
JakubGoneraJakubGonera
authored
feat: Add image segmentation example (#138)
## Description Add image segmentation example to computer vision app. ### Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Documentation update (improves or adds clarity to existing documentation) ### Tested on - [x] iOS - [x] Android ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have updated the documentation accordingly - [x] My changes generate no new warnings
1 parent 9a97485 commit 9ca8288

File tree

6 files changed

+288
-0
lines changed

6 files changed

+288
-0
lines changed

‎examples/computer-vision/App.tsx‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,15 @@ import { ClassificationScreen } from './screens/ClassificationScreen';
1010
import { ObjectDetectionScreen } from './screens/ObjectDetectionScreen';
1111
import { OCRScreen } from './screens/OCRScreen';
1212
import { VerticalOCRScreen } from './screens/VerticalOCRScreen';
13+
import { ImageSegmentationScreen } from './screens/ImageSegmentationScreen';
1314

1415
enum ModelType {
1516
STYLE_TRANSFER,
1617
OBJECT_DETECTION,
1718
CLASSIFICATION,
1819
OCR,
1920
VERTICAL_OCR,
21+
IMAGE_SEGMENTATION,
2022
}
2123

2224
export default function App() {
@@ -56,6 +58,13 @@ export default function App() {
5658
return (
5759
<VerticalOCRScreen imageUri={imageUri} setImageUri={setImageUri} />
5860
);
61+
case ModelType.IMAGE_SEGMENTATION:
62+
return (
63+
<ImageSegmentationScreen
64+
imageUri={imageUri}
65+
setImageUri={setImageUri}
66+
/>
67+
);
5968
default:
6069
return (
6170
<StyleTransferScreen imageUri={imageUri} setImageUri={setImageUri} />
@@ -76,6 +85,7 @@ export default function App() {
7685
'Classification',
7786
'OCR',
7887
'Vertical OCR',
88+
'Image Segmentation',
7989
]}
8090
onValueChange={(_, selectedIndex) => {
8191
handleModeChange(selectedIndex);

‎examples/computer-vision/ios/Podfile.lock‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1387,6 +1387,29 @@ PODS:
13871387
- ReactCommon/turbomodule/bridging
13881388
- ReactCommon/turbomodule/core
13891389
- Yoga
1390+
- react-native-skia (1.11.18):
1391+
- DoubleConversion
1392+
- glog
1393+
- hermes-engine
1394+
- RCT-Folly (= 2024.01.01.00)
1395+
- RCTRequired
1396+
- RCTTypeSafety
1397+
- React
1398+
- React-callinvoker
1399+
- React-Core
1400+
- React-debug
1401+
- React-Fabric
1402+
- React-featureflags
1403+
- React-graphics
1404+
- React-ImageManager
1405+
- React-NativeModulesApple
1406+
- React-RCTFabric
1407+
- React-rendererdebug
1408+
- React-utils
1409+
- ReactCodegen
1410+
- ReactCommon/turbomodule/bridging
1411+
- ReactCommon/turbomodule/core
1412+
- Yoga
13901413
- React-nativeconfig (0.76.3)
13911414
- React-NativeModulesApple (0.76.3):
13921415
- glog
@@ -1835,6 +1858,7 @@ DEPENDENCIES:
18351858
- react-native-executorch (from `../node_modules/react-native-executorch`)
18361859
- react-native-image-picker (from `../node_modules/react-native-image-picker`)
18371860
- react-native-safe-area-context (from `../node_modules/react-native-safe-area-context`)
1861+
- "react-native-skia (from `../node_modules/@shopify/react-native-skia`)"
18381862
- React-nativeconfig (from `../node_modules/react-native/ReactCommon`)
18391863
- React-NativeModulesApple (from `../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`)
18401864
- React-perflogger (from `../node_modules/react-native/ReactCommon/reactperflogger`)
@@ -1963,6 +1987,8 @@ EXTERNAL SOURCES:
19631987
:path: "../node_modules/react-native-image-picker"
19641988
react-native-safe-area-context:
19651989
:path: "../node_modules/react-native-safe-area-context"
1990+
react-native-skia:
1991+
:path: "../node_modules/@shopify/react-native-skia"
19661992
React-nativeconfig:
19671993
:path: "../node_modules/react-native/ReactCommon"
19681994
React-NativeModulesApple:

‎examples/computer-vision/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
},
1111
"dependencies": {
1212
"@react-native/metro-config": "^0.76.3",
13+
"@shopify/react-native-skia": "^1.11.11",
1314
"expo": "~52.0.17",
1415
"expo-font": "^13.0.1",
1516
"expo-status-bar": "~2.0.0",

‎examples/computer-vision/screens/ImageSegmentationScreen.tsx‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
import Spinner from 'react-native-loading-spinner-overlay';
2+
import { BottomBar } from '../components/BottomBar';
3+
import { getImage } from '../utils';
4+
import {
5+
useImageSegmentation,
6+
DeeplabLabel,
7+
DEEPLABV3_RESNET50,
8+
} from 'react-native-executorch';
9+
import {
10+
Canvas,
11+
Image as SkiaImage,
12+
Skia,
13+
AlphaType,
14+
ColorType,
15+
} from '@shopify/react-native-skia';
16+
import { View, StyleSheet, Image } from 'react-native';
17+
import { useState } from 'react';
18+
19+
const width = 224;
20+
const height = 224;
21+
22+
let pixels = new Uint8Array(width * height * 4);
23+
pixels.fill(255);
24+
25+
let data = Skia.Data.fromBytes(pixels);
26+
let img = Skia.Image.MakeImage(
27+
{
28+
width: width,
29+
height: height,
30+
alphaType: AlphaType.Opaque,
31+
colorType: ColorType.RGBA_8888,
32+
},
33+
data,
34+
width * 4
35+
);
36+
37+
const numberToColor: number[][] = [
38+
[255, 87, 51], // 0 Red
39+
[51, 255, 87], // 1 Green
40+
[51, 87, 255], // 2 Blue
41+
[255, 51, 246], // 3 Magenta
42+
[51, 255, 246], // 4 Cyan
43+
[243, 255, 51], // 5 Yellow
44+
[141, 51, 255], // 6 Purple
45+
[255, 131, 51], // 7 Orange
46+
[51, 255, 131], // 8 Spring Green
47+
[131, 51, 255], // 9 Violet
48+
[255, 255, 51], // 10 Bright Yellow
49+
[51, 255, 255], // 11 Aqua
50+
[255, 51, 143], // 12 Deep Pink
51+
[127, 51, 255], // 13 Dark Orchid
52+
[51, 255, 175], // 14 Medium Spring Green
53+
[255, 175, 51], // 15 Sandy Brown
54+
[179, 255, 51], // 16 Chartreuse
55+
[255, 87, 51], // 17 Red (darker shade)
56+
[255, 51, 162], // 18 Hot Pink
57+
[51, 162, 255], // 19 Sky Blue
58+
[162, 51, 255], // 20 Amethyst
59+
];
60+
61+
export const ImageSegmentationScreen = ({
62+
imageUri,
63+
setImageUri,
64+
}: {
65+
imageUri: string;
66+
setImageUri: (imageUri: string) => void;
67+
}) => {
68+
const model = useImageSegmentation({
69+
modelSource: DEEPLABV3_RESNET50,
70+
});
71+
72+
const handleCameraPress = async (isCamera: boolean) => {
73+
const image = await getImage(isCamera);
74+
const uri = image?.uri;
75+
setImageUri(uri as string);
76+
};
77+
78+
const [resultPresent, setResultPresent] = useState(false);
79+
80+
const runForward = async () => {
81+
if (imageUri) {
82+
try {
83+
const output = await model.forward(imageUri);
84+
pixels = new Uint8Array(width * height * 4);
85+
86+
for (let x = 0; x < width; x++) {
87+
for (let y = 0; y < height; y++) {
88+
for (let i = 0; i < 3; i++) {
89+
pixels[(x * height + y) * 4 + i] =
90+
numberToColor[
91+
(output[DeeplabLabel.ARGMAX] || [])[x * height + y]
92+
][i];
93+
}
94+
pixels[(x * height + y) * 4 + 3] = 255;
95+
}
96+
}
97+
98+
data = Skia.Data.fromBytes(pixels);
99+
img = Skia.Image.MakeImage(
100+
{
101+
width: width,
102+
height: height,
103+
alphaType: AlphaType.Opaque,
104+
colorType: ColorType.RGBA_8888,
105+
},
106+
data,
107+
width * 4
108+
);
109+
setResultPresent(true);
110+
} catch (e) {
111+
console.error(e);
112+
}
113+
}
114+
};
115+
116+
if (!model.isReady) {
117+
return (
118+
<Spinner
119+
visible={!model.isReady}
120+
textContent={`Loading the model ${(model.downloadProgress * 100).toFixed(0)} %`}
121+
/>
122+
);
123+
}
124+
125+
return (
126+
<>
127+
<View style={styles.imageCanvasContainer}>
128+
<View style={styles.imageContainer}>
129+
<Image
130+
style={styles.image}
131+
resizeMode="contain"
132+
source={
133+
imageUri
134+
? { uri: imageUri }
135+
: require('../assets/icons/executorch_logo.png')
136+
}
137+
/>
138+
</View>
139+
{resultPresent && (
140+
<View style={styles.canvasContainer}>
141+
<Canvas style={styles.canvas}>
142+
<SkiaImage
143+
image={img}
144+
fit="contain"
145+
x={0}
146+
y={0}
147+
width={width}
148+
height={height}
149+
/>
150+
</Canvas>
151+
</View>
152+
)}
153+
</View>
154+
<BottomBar
155+
handleCameraPress={handleCameraPress}
156+
runForward={runForward}
157+
/>
158+
</>
159+
);
160+
};
161+
162+
const styles = StyleSheet.create({
163+
imageCanvasContainer: {
164+
flex: 6,
165+
width: '100%',
166+
padding: 16,
167+
},
168+
imageContainer: {
169+
flex: 1,
170+
width: '100%',
171+
},
172+
image: {
173+
flex: 1,
174+
borderRadius: 8,
175+
width: '100%',
176+
},
177+
canvasContainer: {
178+
flex: 1,
179+
justifyContent: 'center',
180+
alignItems: 'center',
181+
gap: 4,
182+
padding: 4,
183+
},
184+
canvas: {
185+
width: width,
186+
height: height,
187+
},
188+
});

‎examples/computer-vision/yarn.lock‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2609,6 +2609,27 @@ __metadata:
26092609
languageName: node
26102610
linkType: hard
26112611

2612+
"@shopify/react-native-skia@npm:^1.11.11":
2613+
version: 1.11.18
2614+
resolution: "@shopify/react-native-skia@npm:1.11.18"
2615+
dependencies:
2616+
canvaskit-wasm: 0.39.1
2617+
react-reconciler: 0.27.0
2618+
peerDependencies:
2619+
react: ">=18.0 <19.0.0"
2620+
react-native: ">=0.64 <0.78.0"
2621+
react-native-reanimated: ">=2.0.0"
2622+
peerDependenciesMeta:
2623+
react-native:
2624+
optional: true
2625+
react-native-reanimated:
2626+
optional: true
2627+
bin:
2628+
setup-skia-web: scripts/setup-canvaskit.js
2629+
checksum: 4aca995cde8453ee6b62dc9daf0eac92b48771073d7da48d506f179364b74dac16f641036cee3832e010b0942fbd2c35cf300cbf4ba7f1000f79a1e29b48b9e8
2630+
languageName: node
2631+
linkType: hard
2632+
26122633
"@sinclair/typebox@npm:^0.27.8":
26132634
version: 0.27.8
26142635
resolution: "@sinclair/typebox@npm:0.27.8"
@@ -2936,6 +2957,13 @@ __metadata:
29362957
languageName: node
29372958
linkType: hard
29382959

2960+
"@webgpu/types@npm:0.1.21":
2961+
version: 0.1.21
2962+
resolution: "@webgpu/types@npm:0.1.21"
2963+
checksum: b16ddc4d46a3c507a7f243f14b08b96ebbdb4495f9ea703441ad3525dd5e7e4a5f743f2879e51d74c78846f950e7d60c1349670f75cb7ca03ac3ddf51d46aabe
2964+
languageName: node
2965+
linkType: hard
2966+
29392967
"@xmldom/xmldom@npm:^0.8.8":
29402968
version: 0.8.10
29412969
resolution: "@xmldom/xmldom@npm:0.8.10"
@@ -3626,6 +3654,15 @@ __metadata:
36263654
languageName: node
36273655
linkType: hard
36283656

3657+
"canvaskit-wasm@npm:0.39.1":
3658+
version: 0.39.1
3659+
resolution: "canvaskit-wasm@npm:0.39.1"
3660+
dependencies:
3661+
"@webgpu/types": 0.1.21
3662+
checksum: da62926fc81f424a781e148b4d76bb5fc9b0188f136090b3b287522dc653cb002bfb406e2eff45b55fcc1cafbc7629f988e20ad6c777bab85c1bb09e1091a5e2
3663+
languageName: node
3664+
linkType: hard
3665+
36293666
"chalk@npm:^2.0.1, chalk@npm:^2.4.2":
36303667
version: 2.4.2
36313668
resolution: "chalk@npm:2.4.2"
@@ -3882,6 +3919,7 @@ __metadata:
38823919
dependencies:
38833920
"@babel/core": ^7.25.2
38843921
"@react-native/metro-config": ^0.76.3
3922+
"@shopify/react-native-skia": ^1.11.11
38853923
"@types/react": ~18.3.12
38863924
expo: ~52.0.17
38873925
expo-font: ^13.0.1
@@ -7704,6 +7742,18 @@ __metadata:
77047742
languageName: node
77057743
linkType: hard
77067744

7745+
"react-reconciler@npm:0.27.0":
7746+
version: 0.27.0
7747+
resolution: "react-reconciler@npm:0.27.0"
7748+
dependencies:
7749+
loose-envify: ^1.1.0
7750+
scheduler: ^0.21.0
7751+
peerDependencies:
7752+
react: ^18.0.0
7753+
checksum: c2ae111f150c2a46970182df12ea8254719fdfec5e26574711b1838fc37863c63671460a351570fd359c088d891e7bb0ff89023c2f7c1582393b57dd517b92c2
7754+
languageName: node
7755+
linkType: hard
7756+
77077757
"react-refresh@npm:^0.14.0, react-refresh@npm:^0.14.2":
77087758
version: 0.14.2
77097759
resolution: "react-refresh@npm:0.14.2"
@@ -8039,6 +8089,15 @@ __metadata:
80398089
languageName: node
80408090
linkType: hard
80418091

8092+
"scheduler@npm:^0.21.0":
8093+
version: 0.21.0
8094+
resolution: "scheduler@npm:0.21.0"
8095+
dependencies:
8096+
loose-envify: ^1.1.0
8097+
checksum: 4f8285076041ed2c81acdd1faa987f1655fdbd30554bc667c1ea64743fc74fb3a04ca7d27454b3d678735df5a230137a3b84756061b43dc5796e80701b66d124
8098+
languageName: node
8099+
linkType: hard
8100+
80428101
"selfsigned@npm:^2.4.1":
80438102
version: 2.4.1
80448103
resolution: "selfsigned@npm:2.4.1"

‎src/constants/modelUrls.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ export const RECOGNIZER_EN_CRNN_128 =
7878
export const RECOGNIZER_EN_CRNN_64 =
7979
'https://huggingface.co/software-mansion/react-native-executorch-recognizer-crnn.en/resolve/v0.3.0/xnnpack/xnnpack_crnn_en_64.pte';
8080

81+
// Image segmentation
82+
export const DEEPLABV3_RESNET50 =
83+
'https://huggingface.co/software-mansion/react-native-executorch-deeplab-v3/resolve/v0.4.0/xnnpack/deeplabV3_xnnpack_fp32.pte';
84+
8185
// Backward compatibility
8286
export const LLAMA3_2_3B_URL = LLAMA3_2_3B;
8387
export const LLAMA3_2_3B_QLORA_URL = LLAMA3_2_3B_QLORA;

0 commit comments

Comments
 (0)