Skip to content

Commit 380e7d9

Browse files
msluszniakIgorSwatbarhanc
authored
chore: Add basic statistics to demo apps (#985)
## Description Added inference times, plus for some apps TTFT, tokens/second. Fixed T2S streaming for unfinished sentences. ### Introduces a breaking change? - [ ] Yes - [x] No ### Type of change - [ ] Bug fix (change which fixes an issue) - [ ] New feature (change which adds functionality) - [ ] Documentation update (improves or adds clarity to existing documentation) - [x] Other (chores, tests, code style improvements etc.) ### Tested on - [x] iOS - [ ] Android ### Testing instructions - [ ] Run all apps and check if basic statistics are correctly displayed - [x] Check that text without end of sequence sign is processed correctly in text to speech example ### Screenshots <!-- Add screenshots here, if applicable --> ### Related issues Closes #959 ### Checklist - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have updated the documentation accordingly - [x] My changes generate no new warnings ### Additional notes <!-- Include any additional information, assumptions, or context that reviewers might need to understand this PR. --> --------- Co-authored-by: IgorSwat <igorswat2002@o2.pl> Co-authored-by: Bartosz Hanc <bartosz.hanc02@gmail.com>
1 parent bf29217 commit 380e7d9

File tree

22 files changed

+375
-54
lines changed

22 files changed

+375
-54
lines changed

‎apps/computer-vision/app/classification/index.tsx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@ import { BottomBar } from '../../components/BottomBar';
99
import React, { useContext, useEffect, useState } from 'react';
1010
import { GeneratingContext } from '../../context';
1111
import ScreenWrapper from '../../ScreenWrapper';
12+
import { StatsBar } from '../../components/StatsBar';
1213

1314
export default function ClassificationScreen() {
1415
const [results, setResults] = useState<{ label: string; score: number }[]>(
1516
[]
1617
);
1718
const [imageUri, setImageUri] = useState('');
19+
const [inferenceTime, setInferenceTime] = useState<number | null>(null);
1820

1921
const model = useClassification({ model: EFFICIENTNET_V2_S_QUANTIZED });
2022
const { setGlobalGenerating } = useContext(GeneratingContext);
@@ -28,13 +30,16 @@ export default function ClassificationScreen() {
2830
if (typeof uri === 'string') {
2931
setImageUri(uri as string);
3032
setResults([]);
33+
setInferenceTime(null);
3134
}
3235
};
3336

3437
const runForward = async () => {
3538
if (imageUri) {
3639
try {
40+
const start = Date.now();
3741
const output = await model.forward(imageUri);
42+
setInferenceTime(Date.now() - start);
3843
const top10 = Object.entries(output)
3944
.sort(([, a], [, b]) => (b as number) - (a as number))
4045
.slice(0, 10)
@@ -80,6 +85,7 @@ export default function ClassificationScreen() {
8085
</View>
8186
)}
8287
</View>
88+
<StatsBar inferenceTime={inferenceTime} />
8389
<BottomBar
8490
handleCameraPress={handleCameraPress}
8591
runForward={runForward}

‎apps/computer-vision/app/instance_segmentation/index.tsx‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,17 @@
11
import Spinner from '../../components/Spinner';
22
import { BottomBar } from '../../components/BottomBar';
33
import { getImage } from '../../utils';
4-
import { useInstanceSegmentation, YOLO26N_SEG } from 'react-native-executorch';
4+
import { ModelPicker, ModelOption } from '../../components/ModelPicker';
5+
import {
6+
useInstanceSegmentation,
7+
YOLO26N_SEG,
8+
YOLO26S_SEG,
9+
YOLO26M_SEG,
10+
YOLO26L_SEG,
11+
YOLO26X_SEG,
12+
RF_DETR_NANO_SEG,
13+
InstanceSegmentationModelSources,
14+
} from 'react-native-executorch';
515
import {
616
View,
717
StyleSheet,
@@ -16,8 +26,22 @@ import ImageWithMasks, {
1626
buildDisplayInstances,
1727
DisplayInstance,
1828
} from '../../components/ImageWithMasks';
29+
import { StatsBar } from '../../components/StatsBar';
30+
31+
const MODELS: ModelOption<InstanceSegmentationModelSources>[] = [
32+
{ label: 'Yolo26N', value: YOLO26N_SEG },
33+
{ label: 'Yolo26S', value: YOLO26S_SEG },
34+
{ label: 'Yolo26M', value: YOLO26M_SEG },
35+
{ label: 'Yolo26L', value: YOLO26L_SEG },
36+
{ label: 'Yolo26X', value: YOLO26X_SEG },
37+
{ label: 'RF-DeTR Nano', value: RF_DETR_NANO_SEG },
38+
];
1939

2040
export default function InstanceSegmentationScreen() {
41+
const [selectedModel, setSelectedModel] =
42+
useState<InstanceSegmentationModelSources>(YOLO26N_SEG);
43+
const [inferenceTime, setInferenceTime] = useState<number | null>(null);
44+
2145
const { setGlobalGenerating } = useContext(GeneratingContext);
2246

2347
const {
@@ -28,7 +52,7 @@ export default function InstanceSegmentationScreen() {
2852
error,
2953
getAvailableInputSizes,
3054
} = useInstanceSegmentation({
31-
model: YOLO26N_SEG,
55+
model: selectedModel,
3256
});
3357

3458
const [imageUri, setImageUri] = useState('');
@@ -60,12 +84,14 @@ export default function InstanceSegmentationScreen() {
6084
height: image.height ?? 0,
6185
});
6286
setInstances([]);
87+
setInferenceTime(null);
6388
};
6489

6590
const runForward = async () => {
6691
if (!imageUri || imageSize.width === 0 || imageSize.height === 0) return;
6792

6893
try {
94+
const start = Date.now();
6995
const output = await forward(imageUri, {
7096
confidenceThreshold: 0.5,
7197
iouThreshold: 0.55,
@@ -74,6 +100,8 @@ export default function InstanceSegmentationScreen() {
74100
inputSize: selectedInputSize ?? undefined,
75101
});
76102

103+
setInferenceTime(Date.now() - start);
104+
77105
// Convert raw masks → small Skia images immediately.
78106
// Raw Uint8Array mask buffers (backed by native OwningArrayBuffer)
79107
// go out of scope here and become eligible for GC right away.
@@ -168,6 +196,22 @@ export default function InstanceSegmentationScreen() {
168196
)}
169197
</View>
170198

199+
<ModelPicker
200+
models={MODELS}
201+
selectedModel={selectedModel}
202+
disabled={isGenerating}
203+
onSelect={(m) => {
204+
setSelectedModel(m);
205+
setInstances([]);
206+
setInferenceTime(null);
207+
}}
208+
/>
209+
210+
<StatsBar
211+
inferenceTime={inferenceTime}
212+
detectionCount={instances.length > 0 ? instances.length : null}
213+
/>
214+
171215
<BottomBar
172216
handleCameraPress={handleCameraPress}
173217
runForward={runForward}

‎apps/computer-vision/app/object_detection/index.tsx‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import ImageWithBboxes from '../../components/ImageWithBboxes';
1414
import React, { useContext, useEffect, useState } from 'react';
1515
import { GeneratingContext } from '../../context';
1616
import ScreenWrapper from '../../ScreenWrapper';
17+
import { StatsBar } from '../../components/StatsBar';
1718

1819
const MODELS: ModelOption<ObjectDetectionModelSources>[] = [
1920
{ label: 'RF-DeTR Nano', value: RF_DETR_NANO },
@@ -29,6 +30,7 @@ export default function ObjectDetectionScreen() {
2930
}>();
3031
const [selectedModel, setSelectedModel] =
3132
useState<ObjectDetectionModelSources>(RF_DETR_NANO);
33+
const [inferenceTime, setInferenceTime] = useState<number | null>(null);
3234

3335
const model = useObjectDetection({ model: selectedModel });
3436
const { setGlobalGenerating } = useContext(GeneratingContext);
@@ -46,13 +48,16 @@ export default function ObjectDetectionScreen() {
4648
setImageUri(image.uri as string);
4749
setImageDimensions({ width: width as number, height: height as number });
4850
setResults([]);
51+
setInferenceTime(null);
4952
}
5053
};
5154

5255
const runForward = async () => {
5356
if (imageUri) {
5457
try {
58+
const start = Date.now();
5559
const output = await model.forward(imageUri);
60+
setInferenceTime(Date.now() - start);
5661
setResults(output);
5762
} catch (e) {
5863
console.error(e);
@@ -100,6 +105,10 @@ export default function ObjectDetectionScreen() {
100105
setResults([]);
101106
}}
102107
/>
108+
<StatsBar
109+
inferenceTime={inferenceTime}
110+
detectionCount={results.length > 0 ? results.length : null}
111+
/>
103112
<BottomBar
104113
handleCameraPress={handleCameraPress}
105114
runForward={runForward}

‎apps/computer-vision/app/ocr/index.tsx‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import ImageWithBboxes2 from '../../components/ImageWithOCRBboxes';
1818
import React, { useContext, useEffect, useState } from 'react';
1919
import { GeneratingContext } from '../../context';
2020
import ScreenWrapper from '../../ScreenWrapper';
21+
import { StatsBar } from '../../components/StatsBar';
2122

2223
type OCRModelSources = OCRProps['model'];
2324

@@ -40,6 +41,7 @@ export default function OCRScreen() {
4041
}>();
4142
const [selectedModel, setSelectedModel] =
4243
useState<OCRModelSources>(OCR_ENGLISH);
44+
const [inferenceTime, setInferenceTime] = useState<number | null>(null);
4345

4446
const model = useOCR({
4547
model: selectedModel,
@@ -58,12 +60,15 @@ export default function OCRScreen() {
5860
if (typeof uri === 'string') {
5961
setImageUri(uri as string);
6062
setResults([]);
63+
setInferenceTime(null);
6164
}
6265
};
6366

6467
const runForward = async () => {
6568
try {
69+
const start = Date.now();
6670
const output = await model.forward(imageUri);
71+
setInferenceTime(Date.now() - start);
6772
setResults(output);
6873
} catch (e) {
6974
console.error(e);
@@ -123,6 +128,10 @@ export default function OCRScreen() {
123128
setResults([]);
124129
}}
125130
/>
131+
<StatsBar
132+
inferenceTime={inferenceTime}
133+
detectionCount={results.length > 0 ? results.length : null}
134+
/>
126135
<BottomBar
127136
handleCameraPress={handleCameraPress}
128137
runForward={runForward}

‎apps/computer-vision/app/ocr_vertical/index.tsx‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import ImageWithBboxes2 from '../../components/ImageWithOCRBboxes';
77
import React, { useContext, useEffect, useState } from 'react';
88
import { GeneratingContext } from '../../context';
99
import ScreenWrapper from '../../ScreenWrapper';
10+
import { StatsBar } from '../../components/StatsBar';
1011

1112
export default function VerticalOCRScree() {
1213
const [imageUri, setImageUri] = useState('');
@@ -15,6 +16,7 @@ export default function VerticalOCRScree() {
1516
width: number;
1617
height: number;
1718
}>();
19+
const [inferenceTime, setInferenceTime] = useState<number | null>(null);
1820
const model = useVerticalOCR({
1921
model: OCR_ENGLISH,
2022
independentCharacters: true,
@@ -33,12 +35,15 @@ export default function VerticalOCRScree() {
3335
if (typeof uri === 'string') {
3436
setImageUri(uri as string);
3537
setResults([]);
38+
setInferenceTime(null);
3639
}
3740
};
3841

3942
const runForward = async () => {
4043
try {
44+
const start = Date.now();
4145
const output = await model.forward(imageUri);
46+
setInferenceTime(Date.now() - start);
4247
setResults(output);
4348
} catch (e) {
4449
console.error(e);
@@ -89,6 +94,10 @@ export default function VerticalOCRScree() {
8994
</View>
9095
)}
9196
</View>
97+
<StatsBar
98+
inferenceTime={inferenceTime}
99+
detectionCount={results.length > 0 ? results.length : null}
100+
/>
92101
<BottomBar
93102
handleCameraPress={handleCameraPress}
94103
runForward={runForward}

‎apps/computer-vision/app/semantic_segmentation/index.tsx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import { View, StyleSheet, Image } from 'react-native';
2424
import React, { useContext, useEffect, useState } from 'react';
2525
import { GeneratingContext } from '../../context';
2626
import ScreenWrapper from '../../ScreenWrapper';
27+
import { StatsBar } from '../../components/StatsBar';
2728

2829
const numberToColor: number[][] = [
2930
[255, 87, 51], // 0 Red
@@ -75,6 +76,7 @@ export default function SemanticSegmentationScreen() {
7576
const [imageSize, setImageSize] = useState({ width: 0, height: 0 });
7677
const [segImage, setSegImage] = useState<SkImage | null>(null);
7778
const [canvasSize, setCanvasSize] = useState({ width: 0, height: 0 });
79+
const [inferenceTime, setInferenceTime] = useState<number | null>(null);
7880

7981
useEffect(() => {
8082
setGlobalGenerating(isGenerating);
@@ -86,11 +88,13 @@ export default function SemanticSegmentationScreen() {
8688
setImageUri(image.uri);
8789
setImageSize({ width: image.width ?? 0, height: image.height ?? 0 });
8890
setSegImage(null);
91+
setInferenceTime(null);
8992
};
9093

9194
const runForward = async () => {
9295
if (!imageUri || imageSize.width === 0 || imageSize.height === 0) return;
9396
try {
97+
const start = Date.now();
9498
const { width, height } = imageSize;
9599
const output = await forward(imageUri, [], true);
96100
const argmax = output.ARGMAX || [];
@@ -119,6 +123,7 @@ export default function SemanticSegmentationScreen() {
119123
width * 4
120124
);
121125
setSegImage(img);
126+
setInferenceTime(Date.now() - start);
122127
} catch (e) {
123128
console.error(e);
124129
}
@@ -179,6 +184,7 @@ export default function SemanticSegmentationScreen() {
179184
setSegImage(null);
180185
}}
181186
/>
187+
<StatsBar inferenceTime={inferenceTime} />
182188
<BottomBar
183189
handleCameraPress={handleCameraPress}
184190
runForward={runForward}

‎apps/computer-vision/app/style_transfer/index.tsx‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import { View, StyleSheet, Image } from 'react-native';
1616
import React, { useContext, useEffect, useState } from 'react';
1717
import { GeneratingContext } from '../../context';
1818
import ScreenWrapper from '../../ScreenWrapper';
19+
import { StatsBar } from '../../components/StatsBar';
1920

2021
type StyleTransferModelSources = {
2122
modelName: StyleTransferModelName;
@@ -42,20 +43,24 @@ export default function StyleTransferScreen() {
4243

4344
const [imageUri, setImageUri] = useState('');
4445
const [styledUri, setStyledUri] = useState('');
46+
const [inferenceTime, setInferenceTime] = useState<number | null>(null);
4547

4648
const handleCameraPress = async (isCamera: boolean) => {
4749
const image = await getImage(isCamera);
4850
const uri = image?.uri;
4951
if (typeof uri === 'string') {
5052
setImageUri(uri);
5153
setStyledUri('');
54+
setInferenceTime(null);
5255
}
5356
};
5457

5558
const runForward = async () => {
5659
if (imageUri) {
5760
try {
61+
const start = Date.now();
5862
const uri = await model.forward(imageUri, 'url');
63+
setInferenceTime(Date.now() - start);
5964
setStyledUri(uri);
6065
} catch (e) {
6166
console.error(e);
@@ -96,6 +101,7 @@ export default function StyleTransferScreen() {
96101
setStyledUri('');
97102
}}
98103
/>
104+
<StatsBar inferenceTime={inferenceTime} />
99105
<BottomBar
100106
handleCameraPress={handleCameraPress}
101107
runForward={runForward}

0 commit comments

Comments
 (0)