Skip to content

Commit ce799cd

Browse files
authored
feat: benchmark app improvements (#456)
* save * benchmark app * lint scan fix
1 parent 060cba5 commit ce799cd

6 files changed

Lines changed: 436 additions & 101 deletions

File tree

docs/source/guides/performance_metrics.md

Lines changed: 33 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ Performance metrics are automatically collected during model inference and inclu
1616
- **Total frames**: Total number of inferences
1717
- **FPS**: Frames Per Second
1818

19+
All timing statistics are reported in milliseconds (ms).
20+
1921
Each metric provides statistical information including mean, standard deviation, and individual measurements.
2022

2123
## Basic Usage
@@ -59,21 +61,21 @@ This will output detailed performance information:
5961
============================================================
6062

6163
📊 Model Loading:
62-
Load Time: 2.497s
64+
Load Time: 2497.00 ms
6365

6466
⚙️ Processing Times (mean ± std):
65-
Preprocess: 0.001s ± 0.000s
66-
Inference: 0.570s ± 0.020s
67-
Postprocess: 0.001s ± 0.000s
67+
Preprocess: 1.00 ms ± 0.10 ms
68+
Inference: 570.00 ms ± 20.00 ms
69+
Postprocess: 1.00 ms ± 0.10 ms
6870

6971
📈 Total Time Statistics:
70-
Mean: 0.572s ± 0.020s
71-
Min: 0.556s
72-
Max: 0.642s
72+
Mean: 572.00 ms ± 20.00 ms
73+
Min: 556.00 ms
74+
Max: 642.00 ms
7375

7476
🎯 Performance Summary:
75-
Total Frames: 100
76-
FPS: 1.75
77+
Total Frames: 100
78+
FPS: 1.75
7779
============================================================
7880
```
7981

@@ -94,9 +96,9 @@ total_min_time = metrics.get_total_time_min()
9496
total_max_time = metrics.get_total_time_max()
9597

9698
# Access statistical information
97-
print(f"Mean inference time: {inference_time.mean():.3f} seconds")
98-
print(f"Standard deviation: {inference_time.stddev():.3f} seconds")
99-
print(f"Total inference time: {inference_time.time:.3f} seconds")
99+
print(f"Mean inference time: {inference_time.mean():.2f} ms")
100+
print(f"Standard deviation: {inference_time.stddev():.2f} ms")
101+
print(f"Total inference time: {inference_time.time:.2f} ms")
100102
print(f"Number of inferences: {inference_time.count}")
101103
```
102104

@@ -150,7 +152,7 @@ for i in range(100):
150152
if (i + 1) % 10 == 0:
151153
metrics = model.get_performance_metrics()
152154
print(f"After {i + 1} inferences:")
153-
print(f" Mean inference time: {metrics.get_inference_time().mean():.3f}s")
155+
print(f" Mean inference time: {metrics.get_inference_time().mean():.2f} ms")
154156
print(f" Current FPS: {metrics.get_fps():.2f}")
155157
```
156158

@@ -166,13 +168,12 @@ metrics = model.get_performance_metrics()
166168
preprocess_time = metrics.get_preprocess_time().mean()
167169
inference_time = metrics.get_inference_time().mean()
168170
postprocess_time = metrics.get_postprocess_time().mean()
171+
total = preprocess_time + inference_time + postprocess_time
169172

170173
print("Time breakdown:")
171-
print(f" Preprocessing: {preprocess_time:.3f}s ({preprocess_time/total:.1%})")
172-
print(f" Inference: {inference_time:.3f}s ({inference_time/total:.1%})")
173-
print(f" Postprocessing: {postprocess_time:.3f}s ({postprocess_time/total:.1%})")
174-
175-
total = preprocess_time + inference_time + postprocess_time
174+
print(f" Preprocessing: {preprocess_time:.2f} ms ({preprocess_time/total:.1%})")
175+
print(f" Inference: {inference_time:.2f} ms ({inference_time/total:.1%})")
176+
print(f" Postprocessing: {postprocess_time:.2f} ms ({postprocess_time/total:.1%})")
176177
```
177178

178179
### Warm-up Considerations
@@ -184,8 +185,12 @@ The first few inferences may be slower due to system warm-up. Consider excluding
184185
for _ in range(5):
185186
model(image)
186187

187-
# Reset metrics after warm-up
188-
model.get_performance_metrics().reset()
188+
# Reset metrics after warm-up (load time stats are preserved by default)
189+
metrics = model.get_performance_metrics()
190+
metrics.reset()
191+
192+
# If you also need to clear model load measurements
193+
# metrics.reset(include_load_time=True)
189194

190195
# Now measure actual performance
191196
for _ in range(100):
@@ -227,8 +232,9 @@ def analyze_model_performance(model_path, test_images, warmup_runs=5, test_runs=
227232
for _ in range(warmup_runs):
228233
model(image)
229234

230-
# Reset metrics after warm-up
231-
model.get_performance_metrics().reset()
235+
# Reset metrics after warm-up (keeping load time by default)
236+
metrics = model.get_performance_metrics()
237+
metrics.reset()
232238

233239
print(f"Running {test_runs} test inferences...")
234240
# Performance measurement runs
@@ -252,9 +258,11 @@ def analyze_model_performance(model_path, test_images, warmup_runs=5, test_runs=
252258
# Additional analysis
253259
inference_time = metrics.get_inference_time()
254260
print(f"\nInference time analysis:")
255-
print(f" Minimum: {min(inference_time.durations):.3f}s")
256-
print(f" Maximum: {max(inference_time.durations):.3f}s")
257-
print(f" Median: {sorted(inference_time.durations)[len(inference_time.durations)//2]:.3f}s")
261+
print(f" Minimum: {min(inference_time.durations):.2f} ms")
262+
print(f" Maximum: {max(inference_time.durations):.2f} ms")
263+
print(
264+
f" Median: {sorted(inference_time.durations)[len(inference_time.durations)//2]:.2f} ms"
265+
)
258266

259267
return metrics
260268

0 commit comments

Comments
 (0)