Skip to content

Commit 1fdeeee

Browse files
edge-impulse-firmware-uploader[bot]edge-impulse-firmware-uploader[bot]
committed
Release v1.24.3
1 parent be25784 commit 1fdeeee

1 file changed

Lines changed: 51 additions & 39 deletions

File tree

‎cli-common/model-monitor.ts‎

Lines changed: 51 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import { MgmtInterfaceImpulseRecordRawData, MgmtInterfaceInferenceSummary } from
88
export const DEFAULT_MONITOR_SUMMARY_INTERVAL_MS = 60000;
99
const MONITOR_PREFIX = '\x1b[34m[MON]\x1b[0m';
1010

11+
const DEFAULT_THRESHOLD = 0.5;
12+
1113
function checkFileExists(file: string) {
1214
return new Promise(resolve => {
1315
return fs.access(file, fs.constants.F_OK)
@@ -399,7 +401,7 @@ class StorageManager {
399401

400402
interface MetricCalculator {
401403
name: string;
402-
update(classification: { [label: string]: number } | undefined): void;
404+
update(classification: { [label: string]: number } | undefined, threshold: number): void;
403405
getMetric(): { label: string; value: number }[];
404406
}
405407

@@ -413,20 +415,16 @@ type MeanValues = {
413415
class MeanCalculator implements MetricCalculator {
414416
private _mean: MeanValues;
415417
private _name: string = 'mean';
416-
private _confidenceThreshold: number;
417418

418-
constructor(opts: {
419-
confidenceThreshold: number,
420-
}) {
419+
constructor() {
421420
this._mean = { };
422-
this._confidenceThreshold = opts.confidenceThreshold;
423421
}
424422

425423
get name(): string {
426424
return this._name;
427425
}
428426

429-
update(classification: { [label: string]: number } | undefined): void {
427+
update(classification: { [label: string]: number } | undefined, threshold: number): void {
430428
if (classification === undefined) {
431429
return;
432430
}
@@ -443,7 +441,7 @@ class MeanCalculator implements MetricCalculator {
443441

444442
for (const label of Object.keys(classification)) {
445443
const score = Number(classification[label]);
446-
if (score > predictedScore && score >= this._confidenceThreshold) {
444+
if (score > predictedScore && score >= threshold) {
447445
predictedScore = score;
448446
predictedLabel = label;
449447
}
@@ -491,13 +489,9 @@ type StdDevValues = {
491489
class StdDevCalculator implements MetricCalculator {
492490
private _stdDev: StdDevValues;
493491
private _name: string = 'standardDeviation';
494-
private _confidenceThreshold: number;
495492

496-
constructor(opts: {
497-
confidenceThreshold: number,
498-
}) {
493+
constructor() {
499494
this._stdDev = { };
500-
this._confidenceThreshold = opts.confidenceThreshold;
501495
}
502496

503497
get name(): string {
@@ -506,7 +500,7 @@ class StdDevCalculator implements MetricCalculator {
506500

507501
// based on Welford's online algorithm
508502
// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
509-
update(classification: { [label: string]: number } | undefined ): void {
503+
update(classification: { [label: string]: number } | undefined, threshold: number): void {
510504
if (classification === undefined) {
511505
return;
512506
}
@@ -519,7 +513,7 @@ class StdDevCalculator implements MetricCalculator {
519513

520514
for (const label of Object.keys(classification)) {
521515
const score = Number(classification[label]);
522-
if (score > predictedScore && score >= this._confidenceThreshold) {
516+
if (score > predictedScore && score >= threshold) {
523517
predictedScore = score;
524518
predictedLabel = label;
525519
}
@@ -572,8 +566,8 @@ class MetricsCalculator {
572566
private _stdDevCalculator: StdDevCalculator;
573567
private _metrics: InferenceMetrics;
574568
private _model: ModelInformation;
575-
private _confidenceThreshold: number;
576569
private _summaryWindowMs: number;
570+
private _confidenceThresholds = new Map<string, number>();
577571

578572
// parameters and variables used to determine summary period
579573
private _windowStartIndex: number = 0;
@@ -620,29 +614,39 @@ class MetricsCalculator {
620614
if (thresholdObj && thresholdObj.type === 'anomaly_gmm') {
621615
if (typeof thresholdObj.min_anomaly_score === 'number') {
622616
threshold = thresholdObj.min_anomaly_score;
617+
this._confidenceThresholds.set('min_anomaly_score', threshold);
623618
}
624619
}
625620
}
626-
else if (model.modelParameters.model_type === 'constrained_object_detection' ||
627-
model.modelParameters.model_type === 'object_detection'
621+
622+
// An impulse might have both visual AD and classifier/object detection, but classification and object detection
623+
// are mutually exclusive, so we check them in an if-else statement
624+
if (model.modelParameters.model_type === 'constrained_object_detection' ||
625+
model.modelParameters.model_type === 'object_detection'
628626
) {
629627
const thresholdObj = (model.modelParameters.thresholds || []).find(x => x.type === 'object_detection');
630628
if (thresholdObj && thresholdObj.type === 'object_detection') {
631629
if (typeof thresholdObj.min_score === 'number') {
632630
threshold = thresholdObj.min_score;
631+
this._confidenceThresholds.set('min_score', threshold);
633632
}
634633
}
635634
}
636-
637-
if (typeof threshold === 'undefined') {
638-
console.warn(MONITOR_PREFIX, 'Model threshold is not defined, using default value of 0.5');
639-
this._confidenceThreshold = 0.5;
635+
else if (model.modelParameters.model_type === 'classification') {
636+
const thresholdObj = (model.modelParameters.thresholds || []).find(x => x.type === 'classification');
637+
if (thresholdObj && thresholdObj.type === 'classification') {
638+
if (typeof thresholdObj.min_score === 'number') {
639+
threshold = thresholdObj.min_score;
640+
this._confidenceThresholds.set('min_score', threshold);
641+
}
642+
}
640643
}
641-
else {
642-
this._confidenceThreshold = threshold;
644+
645+
if (this._confidenceThresholds.size === 0) {
646+
console.warn(MONITOR_PREFIX, `Model threshold is not defined, using default value of ${DEFAULT_THRESHOLD}`);
643647
}
644-
this._meanCalculator = new MeanCalculator({ confidenceThreshold: this._confidenceThreshold });
645-
this._stdDevCalculator = new StdDevCalculator({ confidenceThreshold: this._confidenceThreshold });
648+
this._meanCalculator = new MeanCalculator();
649+
this._stdDevCalculator = new StdDevCalculator();
646650

647651
// init as empty, will be overwritten by restoreMetrics
648652
this._metrics = {
@@ -656,8 +660,8 @@ class MetricsCalculator {
656660
}
657661

658662
private resetMetrics(): void {
659-
this._meanCalculator = new MeanCalculator({ confidenceThreshold: this._confidenceThreshold });
660-
this._stdDevCalculator = new StdDevCalculator({ confidenceThreshold: this._confidenceThreshold });
663+
this._meanCalculator = new MeanCalculator();
664+
this._stdDevCalculator = new StdDevCalculator();
661665

662666
this._metrics = {
663667
start: { index: this._windowStartIndex, timestamp: this._windowStartTimestamp },
@@ -700,38 +704,46 @@ class MetricsCalculator {
700704
};
701705

702706
if (hasVisualAd(this._model) && record.impulseRecord.result.visual_anomaly_max !== undefined) {
703-
const isAnomaly = record.impulseRecord.result.visual_anomaly_max > this._confidenceThreshold;
707+
const minAnomalyScore = this._confidenceThresholds.get('min_anomaly_score') ?? DEFAULT_THRESHOLD;
708+
709+
const isAnomaly = record.impulseRecord.result.visual_anomaly_max > minAnomalyScore;
704710
const predictedLabel = isAnomaly ? 'anomaly' : 'no anomaly';
705711
incrementClassificationCounter(predictedLabel);
706712

707713
const classifications = { [predictedLabel]: record.impulseRecord.result.visual_anomaly_max };
708714
// update mean and stdev partial values
709-
this._meanCalculator.update(classifications);
710-
this._stdDevCalculator.update(classifications);
715+
this._meanCalculator.update(classifications, minAnomalyScore);
716+
this._stdDevCalculator.update(classifications, minAnomalyScore);
711717
}
718+
712719
// check if property classification exists in record.impulseRecord.result
713-
else if (record.impulseRecord.result.classification !== undefined) {
720+
if (record.impulseRecord.result.classification !== undefined) {
721+
const minScore = this._confidenceThresholds.get('min_score') ?? DEFAULT_THRESHOLD;
722+
714723
// iterate over properites in impulseRecord.result.classification
715724
for (const key in record.impulseRecord.result.classification) {
716725
if (!record.impulseRecord.result.classification.hasOwnProperty(key)) {
717726
continue;
718727
}
719728

720729
// check if value is above threshold
721-
if (Number(record.impulseRecord.result.classification[key]) > this._confidenceThreshold) {
730+
if (Number(record.impulseRecord.result.classification[key]) > minScore) {
722731
incrementClassificationCounter(key);
723732
}
724733
}
725734

726735
// update mean and stdev partial values
727-
this._meanCalculator.update(record.impulseRecord.result.classification);
728-
this._stdDevCalculator.update(record.impulseRecord.result.classification);
736+
this._meanCalculator.update(record.impulseRecord.result.classification, minScore);
737+
this._stdDevCalculator.update(record.impulseRecord.result.classification, minScore);
729738
}
730-
else if (record.impulseRecord.result.bounding_boxes !== undefined) {
739+
740+
if (record.impulseRecord.result.bounding_boxes !== undefined) {
741+
const minScore = this._confidenceThresholds.get('min_score') ?? DEFAULT_THRESHOLD;
742+
731743
// iterate over list of bounding boxes
732744
for (const box of record.impulseRecord.result.bounding_boxes) {
733745
// check if value is above threshold
734-
if (box.value > this._confidenceThreshold) {
746+
if (box.value > minScore) {
735747
incrementClassificationCounter(box.label);
736748
}
737749
}
@@ -745,8 +757,8 @@ class MetricsCalculator {
745757
}, {} as { [k: string]: number });
746758

747759
// update mean and stdev partial values
748-
this._meanCalculator.update(classificationData);
749-
this._stdDevCalculator.update(classificationData);
760+
this._meanCalculator.update(classificationData, minScore);
761+
this._stdDevCalculator.update(classificationData, minScore);
750762
}
751763

752764
// Calculate average inference time, if timing information is present

0 commit comments

Comments
 (0)