Skip to content

Commit 3fe7c12

Browse files
msluszniakbarhanc
msluszniak
and
barhanc
authored
chore: Make apps more intuitive in use (#1006)
## Description This PR adds: * In LLM app - Adds better phrasing in voice chat example. * In CV app - Blocks buttons when image not selected & add brief description of an example. Additionally, adds model selection list to classification example. * In CLIP embeddings app - Removes text embedding comparison from this example (since it already is in text embeddings example) and make app more intuitive in use (chosen image is displayed). Also adds model selection list to both text and image embeddings. ### Introduces a breaking change? - [ ] Yes - [x] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [ ] New feature (change which adds functionality) - [ ] Documentation update (improves or adds clarity to existing documentation) - [x] Other (chores, tests, code style improvements etc.) ### Tested on - [x] iOS - [x] Android ### Testing instructions Run the following apps: - [ ] LLM - [ ] Computer Vision - [ ] Clip embeddings And check if described changes works correctly and does not introduce any problems. ### Screenshots <!-- Add screenshots here, if applicable --> ### Related issues Closes #351 ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have updated the documentation accordingly - [x] My changes generate no new warnings ### Additional notes <!-- Include any additional information, assumptions, or context that reviewers might need to understand this PR. --> --------- Co-authored-by: Bartosz Hanc <bartosz.hanc02@gmail.com>
1 parent 46ea659 commit 3fe7c12

File tree

22 files changed

+1088
-431
lines changed

22 files changed

+1088
-431
lines changed

‎apps/computer-vision/app/_layout.tsx‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,14 @@ export default function _layout() {
5959
headerTitleStyle: { color: ColorPalette.primary },
6060
}}
6161
>
62+
<Drawer.Screen
63+
name="index"
64+
options={{
65+
drawerLabel: () => null,
66+
title: 'Main Menu',
67+
drawerItemStyle: { display: 'none' },
68+
}}
69+
/>
6270
<Drawer.Screen
6371
name="vision_camera/index"
6472
options={{
@@ -132,14 +140,6 @@ export default function _layout() {
132140
headerTitleStyle: { color: ColorPalette.primary },
133141
}}
134142
/>
135-
<Drawer.Screen
136-
name="index"
137-
options={{
138-
drawerLabel: () => null,
139-
title: 'Main Menu',
140-
drawerItemStyle: { display: 'none' },
141-
}}
142-
/>
143143
</Drawer>
144144
</GeneratingContext>
145145
);

‎apps/computer-vision/app/classification/index.tsx‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,16 @@ import Spinner from '../../components/Spinner';
22
import { getImage } from '../../utils';
33
import {
44
useClassification,
5+
EFFICIENTNET_V2_S,
56
EFFICIENTNET_V2_S_QUANTIZED,
7+
ClassificationModelSources,
68
} from 'react-native-executorch';
9+
import { ModelPicker, ModelOption } from '../../components/ModelPicker';
10+
11+
const MODELS: ModelOption<ClassificationModelSources>[] = [
12+
{ label: 'EfficientNet V2 S Quantized', value: EFFICIENTNET_V2_S_QUANTIZED },
13+
{ label: 'EfficientNet V2 S', value: EFFICIENTNET_V2_S },
14+
];
715
import { View, StyleSheet, Image, Text, ScrollView } from 'react-native';
816
import { BottomBar } from '../../components/BottomBar';
917
import React, { useContext, useEffect, useState } from 'react';
@@ -13,6 +21,8 @@ import { StatsBar } from '../../components/StatsBar';
1321
import ErrorBanner from '../../components/ErrorBanner';
1422

1523
export default function ClassificationScreen() {
24+
const [selectedModel, setSelectedModel] =
25+
useState<ClassificationModelSources>(EFFICIENTNET_V2_S_QUANTIZED);
1626
const [results, setResults] = useState<{ label: string; score: number }[]>(
1727
[]
1828
);
@@ -21,7 +31,7 @@ export default function ClassificationScreen() {
2131

2232
const [error, setError] = useState<string | null>(null);
2333

24-
const model = useClassification({ model: EFFICIENTNET_V2_S_QUANTIZED });
34+
const model = useClassification({ model: selectedModel });
2535
const { setGlobalGenerating } = useContext(GeneratingContext);
2636

2737
useEffect(() => {
@@ -82,6 +92,16 @@ export default function ClassificationScreen() {
8292
: require('../../assets/icons/executorch_logo.png')
8393
}
8494
/>
95+
{!imageUri && (
96+
<View style={styles.infoContainer}>
97+
<Text style={styles.infoTitle}>Image Classification</Text>
98+
<Text style={styles.infoText}>
99+
This model analyzes an image and returns the top 10 most likely
100+
labels with confidence scores. Use the gallery or camera icons
101+
below to pick an image, then tap the button to run the model.
102+
</Text>
103+
</View>
104+
)}
85105
{results.length > 0 && (
86106
<View style={styles.results}>
87107
<Text style={styles.resultHeader}>Results Top 10</Text>
@@ -96,10 +116,21 @@ export default function ClassificationScreen() {
96116
</View>
97117
)}
98118
</View>
119+
<ModelPicker
120+
models={MODELS}
121+
selectedModel={selectedModel}
122+
disabled={model.isGenerating}
123+
onSelect={(m) => {
124+
setSelectedModel(m);
125+
setResults([]);
126+
}}
127+
/>
99128
<StatsBar inferenceTime={inferenceTime} />
100129
<BottomBar
101130
handleCameraPress={handleCameraPress}
102131
runForward={runForward}
132+
hasImage={!!imageUri}
133+
isGenerating={model.isGenerating}
103134
/>
104135
</ScreenWrapper>
105136
);
@@ -141,4 +172,20 @@ const styles = StyleSheet.create({
141172
flex: 1,
142173
marginRight: 4,
143174
},
175+
infoContainer: {
176+
alignItems: 'center',
177+
padding: 16,
178+
gap: 8,
179+
},
180+
infoTitle: {
181+
fontSize: 18,
182+
fontWeight: '600',
183+
color: 'navy',
184+
},
185+
infoText: {
186+
fontSize: 14,
187+
color: '#555',
188+
textAlign: 'center',
189+
lineHeight: 20,
190+
},
144191
});

‎apps/computer-vision/app/instance_segmentation/index.tsx‎

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,19 @@ export default function InstanceSegmentationScreen() {
7070

7171
// Set default input size when model is ready
7272
useEffect(() => {
73-
if (isReady && availableInputSizes && availableInputSizes.length > 0) {
74-
setSelectedInputSize(availableInputSizes[0]);
73+
if (!isReady) return;
74+
75+
if (availableInputSizes && availableInputSizes.length > 0) {
76+
setSelectedInputSize((prev) => {
77+
if (typeof prev === 'number' && availableInputSizes.includes(prev)) {
78+
return prev;
79+
}
80+
return availableInputSizes[0];
81+
});
82+
return;
7583
}
84+
85+
setSelectedInputSize(null);
7686
}, [isReady, availableInputSizes]);
7787

7888
const handleCameraPress = async (isCamera: boolean) => {
@@ -90,14 +100,21 @@ export default function InstanceSegmentationScreen() {
90100
const runForward = async () => {
91101
if (!imageUri || imageSize.width === 0 || imageSize.height === 0) return;
92102

103+
const inputSize =
104+
availableInputSizes &&
105+
typeof selectedInputSize === 'number' &&
106+
availableInputSizes.includes(selectedInputSize)
107+
? selectedInputSize
108+
: undefined;
109+
93110
try {
94111
const start = Date.now();
95112
const output = await forward(imageUri, {
96113
confidenceThreshold: 0.5,
97114
iouThreshold: 0.55,
98115
maxInstances: 20,
99116
returnMaskAtOriginalResolution: true,
100-
inputSize: selectedInputSize ?? undefined,
117+
inputSize,
101118
});
102119

103120
setInferenceTime(Date.now() - start);
@@ -144,6 +161,16 @@ export default function InstanceSegmentationScreen() {
144161
imageWidth={imageSize.width}
145162
imageHeight={imageSize.height}
146163
/>
164+
{!imageUri && (
165+
<View style={styles.infoContainer}>
166+
<Text style={styles.infoTitle}>Instance Segmentation</Text>
167+
<Text style={styles.infoText}>
168+
This model detects individual objects and draws a precise mask
169+
over each one. Pick an image from your gallery or take one with
170+
your camera to get started.
171+
</Text>
172+
</View>
173+
)}
147174
</View>
148175

149176
{imageUri && availableInputSizes && availableInputSizes.length > 0 && (
@@ -215,6 +242,8 @@ export default function InstanceSegmentationScreen() {
215242
<BottomBar
216243
handleCameraPress={handleCameraPress}
217244
runForward={runForward}
245+
hasImage={!!imageUri}
246+
isGenerating={isGenerating}
218247
/>
219248
</ScreenWrapper>
220249
);
@@ -318,4 +347,20 @@ const styles = StyleSheet.create({
318347
color: '#999',
319348
fontFamily: 'Courier',
320349
},
350+
infoContainer: {
351+
alignItems: 'center',
352+
padding: 16,
353+
gap: 8,
354+
},
355+
infoTitle: {
356+
fontSize: 18,
357+
fontWeight: '600',
358+
color: 'navy',
359+
},
360+
infoText: {
361+
fontSize: 14,
362+
color: '#555',
363+
textAlign: 'center',
364+
lineHeight: 20,
365+
},
321366
});

‎apps/computer-vision/app/object_detection/index.tsx‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import {
1414
YOLO26X,
1515
ObjectDetectionModelSources,
1616
} from 'react-native-executorch';
17-
import { View, StyleSheet, Image } from 'react-native';
17+
import { View, StyleSheet, Image, Text } from 'react-native';
1818
import ImageWithBboxes from '../../components/ImageWithBboxes';
1919
import React, { useContext, useEffect, useState } from 'react';
2020
import { GeneratingContext } from '../../context';
@@ -112,6 +112,16 @@ export default function ObjectDetectionScreen() {
112112
/>
113113
)}
114114
</View>
115+
{!imageUri && (
116+
<View style={styles.infoContainer}>
117+
<Text style={styles.infoTitle}>Object Detection</Text>
118+
<Text style={styles.infoText}>
119+
This model detects objects in an image and draws bounding boxes
120+
around them with class labels and confidence scores. Pick an image
121+
from your gallery or take one with your camera to get started.
122+
</Text>
123+
</View>
124+
)}
115125
</View>
116126
<ModelPicker
117127
models={MODELS}
@@ -129,6 +139,8 @@ export default function ObjectDetectionScreen() {
129139
<BottomBar
130140
handleCameraPress={handleCameraPress}
131141
runForward={runForward}
142+
hasImage={!!imageUri}
143+
isGenerating={model.isGenerating}
132144
/>
133145
</ScreenWrapper>
134146
);
@@ -149,4 +161,20 @@ const styles = StyleSheet.create({
149161
width: '100%',
150162
height: '100%',
151163
},
164+
infoContainer: {
165+
alignItems: 'center',
166+
padding: 16,
167+
gap: 8,
168+
},
169+
infoTitle: {
170+
fontSize: 18,
171+
fontWeight: '600',
172+
color: 'navy',
173+
},
174+
infoText: {
175+
fontSize: 14,
176+
color: '#555',
177+
textAlign: 'center',
178+
lineHeight: 20,
179+
},
152180
});

‎apps/computer-vision/app/ocr/index.tsx‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,17 @@ export default function OCRScreen() {
112112
/>
113113
)}
114114
</View>
115+
{!imageUri && (
116+
<View style={styles.infoContainer}>
117+
<Text style={styles.infoTitle}>OCR</Text>
118+
<Text style={styles.infoText}>
119+
This model reads and extracts text from images, returning each
120+
detected text region with its bounding box and confidence score.
121+
Pick an image from your gallery or take one with your camera to
122+
get started.
123+
</Text>
124+
</View>
125+
)}
115126
{results.length > 0 && (
116127
<View style={styles.results}>
117128
<Text style={styles.resultHeader}>Results</Text>
@@ -142,6 +153,8 @@ export default function OCRScreen() {
142153
<BottomBar
143154
handleCameraPress={handleCameraPress}
144155
runForward={runForward}
156+
hasImage={!!imageUri}
157+
isGenerating={model.isGenerating}
145158
/>
146159
</ScreenWrapper>
147160
);
@@ -187,4 +200,20 @@ const styles = StyleSheet.create({
187200
flex: 1,
188201
marginRight: 4,
189202
},
203+
infoContainer: {
204+
alignItems: 'center',
205+
padding: 16,
206+
gap: 8,
207+
},
208+
infoTitle: {
209+
fontSize: 18,
210+
fontWeight: '600',
211+
color: 'navy',
212+
},
213+
infoText: {
214+
fontSize: 14,
215+
color: '#555',
216+
textAlign: 'center',
217+
lineHeight: 20,
218+
},
190219
});

‎apps/computer-vision/app/ocr_vertical/index.tsx‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,26 @@ export default function VerticalOCRScreen() {
9292
/>
9393
)}
9494
</View>
95+
{!imageUri && (
96+
<View style={styles.infoContainer}>
97+
<Text style={styles.infoTitle}>Vertical OCR</Text>
98+
<Text style={styles.infoText}>
99+
This model reads vertical text (e.g. Japanese, Korean, Chinese
100+
columns) from images, returning each detected text region with its
101+
bounding box and confidence score. Pick an image from your gallery
102+
or take one with your camera to get started.
103+
</Text>
104+
</View>
105+
)}
106+
{imageUri && inferenceTime !== null && results.length === 0 && (
107+
<View style={styles.infoContainer}>
108+
<Text style={styles.infoTitle}>No text detected</Text>
109+
<Text style={styles.infoText}>
110+
The model did not find any vertical text in this image. Try an
111+
image containing vertical Japanese, Korean, or Chinese text.
112+
</Text>
113+
</View>
114+
)}
95115
{results.length > 0 && (
96116
<View style={styles.results}>
97117
<Text style={styles.resultHeader}>Results</Text>
@@ -113,6 +133,8 @@ export default function VerticalOCRScreen() {
113133
<BottomBar
114134
handleCameraPress={handleCameraPress}
115135
runForward={runForward}
136+
hasImage={!!imageUri}
137+
isGenerating={model.isGenerating}
116138
/>
117139
</ScreenWrapper>
118140
);
@@ -158,4 +180,20 @@ const styles = StyleSheet.create({
158180
flex: 1,
159181
marginRight: 4,
160182
},
183+
infoContainer: {
184+
alignItems: 'center',
185+
padding: 16,
186+
gap: 8,
187+
},
188+
infoTitle: {
189+
fontSize: 18,
190+
fontWeight: '600',
191+
color: 'navy',
192+
},
193+
infoText: {
194+
fontSize: 14,
195+
color: '#555',
196+
textAlign: 'center',
197+
lineHeight: 20,
198+
},
161199
});

0 commit comments

Comments
 (0)