Skip to content

Commit ca1c82e

Browse files
mgumowskmgumowsk
authored
test: anomalib models tests (#395)
* precommit, functional tests * accurancy tests
1 parent 301b799 commit ca1c82e

8 files changed

Lines changed: 142 additions & 7 deletions

File tree

‎src/model_api/models/image_model.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,10 +121,12 @@ def parameters(cls) -> dict[str, Any]:
121121
"orig_height": NumericalValue(
122122
int,
123123
description="Model input height before embedding processing",
124+
default_value=None,
124125
),
125126
"orig_width": NumericalValue(
126127
int,
127128
description="Model input width before embedding processing",
129+
default_value=None,
128130
),
129131
"pad_value": NumericalValue(
130132
int,

‎src/model_api/models/types.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,10 @@ def __init__(
6262
self.value_type = value_type
6363

6464
def from_str(self, value: str) -> Any:
65+
if not value and self.default_value is not None:
66+
return self.default_value
67+
if not value and self.default_value is None:
68+
return None
6569
return self.value_type(value)
6670

6771
def validate(self, value):

‎tests/accuracy/prepare_data.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,21 @@ async def download_otx_model(client, otx_models_dir, model_name, format="xml"):
5353
)
5454

5555

56+
async def download_anomalib_model(client, models_dir, model_name):
57+
await asyncio.gather(
58+
stream_file(
59+
client,
60+
f"https://storage.geti.intel.com/geti_predict/test/anomalib_models/{model_name}/openvino.xml",
61+
f"{models_dir}/{model_name}.xml",
62+
),
63+
stream_file(
64+
client,
65+
f"https://storage.geti.intel.com/geti_predict/test/anomalib_models/{model_name}/openvino.bin",
66+
f"{models_dir}/{model_name}.bin",
67+
),
68+
)
69+
70+
5671
async def main():
5772
parser = argparse.ArgumentParser()
5873
parser.add_argument(
@@ -66,6 +81,8 @@ async def main():
6681

6782
otx_models_dir = args.data_dir / "otx_models"
6883
otx_models_dir.mkdir(parents=True, exist_ok=True)
84+
anomalib_models_dir = args.data_dir / "anomalib_models"
85+
anomalib_models_dir.mkdir(parents=True, exist_ok=True)
6986
async with httpx.AsyncClient(timeout=20.0) as client:
7087
await asyncio.gather(
7188
download_images(args.data_dir),
@@ -138,6 +155,9 @@ async def main():
138155
download_otx_model(client, otx_models_dir, "sam_vit_b_zsl_decoder"),
139156
download_otx_model(client, otx_models_dir, "rtmpose_tiny"),
140157
download_otx_model(client, otx_models_dir, "segnext_t_tiling"),
158+
download_anomalib_model(client, anomalib_models_dir, "padim"),
159+
download_anomalib_model(client, anomalib_models_dir, "stfpm"),
160+
download_anomalib_model(client, anomalib_models_dir, "uflow"),
141161
)
142162

143163

‎tests/accuracy/public_scope.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,5 +441,41 @@
441441
]
442442
}
443443
]
444+
},
445+
{
446+
"name": "anomalib_models/padim.xml",
447+
"type": "AnomalyDetection",
448+
"test_data": [
449+
{
450+
"image": "coco128/images/train2017/000000000074.jpg",
451+
"reference": [
452+
"anomaly_map min:153 max:255;pred_score:1.0;pred_label:True;pred_mask min:1 max:1;"
453+
]
454+
}
455+
]
456+
},
457+
{
458+
"name": "anomalib_models/stfpm.xml",
459+
"type": "AnomalyDetection",
460+
"test_data": [
461+
{
462+
"image": "coco128/images/train2017/000000000074.jpg",
463+
"reference": [
464+
"anomaly_map min:127 max:255;pred_score:1.0;pred_label:True;pred_mask min:0 max:1;"
465+
]
466+
}
467+
]
468+
},
469+
{
470+
"name": "anomalib_models/uflow.xml",
471+
"type": "AnomalyDetection",
472+
"test_data": [
473+
{
474+
"image": "coco128/images/train2017/000000000074.jpg",
475+
"reference": [
476+
"anomaly_map min:191 max:191;pred_score:1.0;pred_label:True;pred_mask min:1 max:1;"
477+
]
478+
}
479+
]
444480
}
445481
]

‎tests/accuracy/test_accuracy.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,12 +95,14 @@ def create_models(model_type, model_path, download_dir, force_onnx_adapter=False
9595
model_type.create_model(model_path, device="CPU", download_dir=download_dir),
9696
]
9797
if model_path.endswith(".xml"):
98-
wrapper_type = model_type.get_model_class(
99-
create_core().read_model(model_path).get_rt_info(["model_info", "model_type"]).astype(str),
100-
)
101-
model = wrapper_type(OpenvinoAdapter(create_core(), model_path, device="CPU"))
102-
model.load()
103-
models.append(model)
98+
model = create_core().read_model(model_path)
99+
if model.has_rt_info(["model_info", "model_type"]):
100+
wrapper_type = model_type.get_model_class(
101+
create_core().read_model(model_path).get_rt_info(["model_info", "model_type"]).astype(str),
102+
)
103+
model = wrapper_type(OpenvinoAdapter(create_core(), model_path, device="CPU"))
104+
model.load()
105+
models.append(model)
104106
return models
105107

106108

‎tests/functional/test_save.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,3 +70,45 @@ def test_onnx_save(tmp_path, data):
7070
assert type(cls_model) is type(deserialized)
7171
for attr in cls_model.parameters():
7272
assert getattr(cls_model, attr) == getattr(deserialized, attr)
73+
74+
75+
def test_padim_save(tmp_path, data):
76+
padim_model = Model.create_model(
77+
Path(data) / "anomalib_models/padim.xml",
78+
)
79+
xml_path = str(tmp_path / "a.xml")
80+
padim_model.save(xml_path)
81+
deserialized = Model.create_model(xml_path)
82+
83+
assert not deserialized.get_model().get_rt_info(["model_info", "embedded_processing"]).astype(bool)
84+
assert type(padim_model) is type(deserialized)
85+
for attr in padim_model.parameters():
86+
assert getattr(padim_model, attr) == getattr(deserialized, attr)
87+
88+
89+
def test_stfpm_save(tmp_path, data):
90+
stfpm_model = Model.create_model(
91+
Path(data) / "anomalib_models/stfpm.xml",
92+
)
93+
xml_path = str(tmp_path / "a.xml")
94+
stfpm_model.save(xml_path)
95+
deserialized = Model.create_model(xml_path)
96+
97+
assert not deserialized.get_model().get_rt_info(["model_info", "embedded_processing"]).astype(bool)
98+
assert type(stfpm_model) is type(deserialized)
99+
for attr in stfpm_model.parameters():
100+
assert getattr(stfpm_model, attr) == getattr(deserialized, attr)
101+
102+
103+
def test_uflow_save(tmp_path, data):
104+
uflow_model = Model.create_model(
105+
Path(data) / "anomalib_models/uflow.xml",
106+
)
107+
xml_path = str(tmp_path / "a.xml")
108+
uflow_model.save(xml_path)
109+
deserialized = Model.create_model(xml_path)
110+
111+
assert not deserialized.get_model().get_rt_info(["model_info", "embedded_processing"]).astype(bool)
112+
assert type(uflow_model) is type(deserialized)
113+
for attr in uflow_model.parameters():
114+
assert getattr(uflow_model, attr) == getattr(deserialized, attr)

‎tests/precommit/prepare_data.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,30 @@ def retrieve_otx_model(data_dir, model_name, format="xml"):
2828
)
2929

3030

31+
def retrieve_anomalib_model(data_dir, model_name, format="xml"):
32+
destination_folder = Path(data_dir) / "anomalib_models"
33+
destination_folder.mkdir(parents=True, exist_ok=True)
34+
urlretrieve(
35+
f"https://storage.geti.intel.com/geti_predict/test/anomalib_models/{model_name}/openvino.xml",
36+
destination_folder / f"{model_name}.xml",
37+
)
38+
urlretrieve(
39+
f"https://storage.geti.intel.com/geti_predict/test/anomalib_models/{model_name}/openvino.bin",
40+
f"{destination_folder}/{model_name}.bin",
41+
)
42+
43+
3144
def prepare_model(
3245
data_dir="./data",
3346
public_scope=Path(__file__).resolve().parent / "public_scope.json",
3447
):
3548
# TODO refactor this test so that it does not use eval
3649
# flake8: noqa: F401
37-
from model_api.models import ClassificationModel, DetectionModel, SegmentationModel
50+
from model_api.models import AnomalyDetection, ClassificationModel, DetectionModel, SegmentationModel
3851

3952
# Mapping of model type strings to actual classes for security
4053
MODEL_TYPE_MAPPING = {
54+
"AnomalyDetection": AnomalyDetection,
4155
"ClassificationModel": ClassificationModel,
4256
"DetectionModel": DetectionModel,
4357
"SegmentationModel": SegmentationModel,
@@ -91,3 +105,6 @@ def prepare_data(data_dir="./data"):
91105
retrieve_otx_model(args.data_dir, "Lite-hrnet-18_mod2")
92106
retrieve_otx_model(args.data_dir, "tinynet_imagenet")
93107
retrieve_otx_model(args.data_dir, "cls_mobilenetv3_large_cars", "onnx")
108+
retrieve_anomalib_model(args.data_dir, "padim")
109+
retrieve_anomalib_model(args.data_dir, "stfpm")
110+
retrieve_anomalib_model(args.data_dir, "uflow")

‎tests/precommit/public_scope.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,17 @@
1414
{
1515
"name": "otx_models/tinynet_imagenet.xml",
1616
"type": "ClassificationModel"
17+
},
18+
{
19+
"name": "anomalib_models/padim.xml",
20+
"type": "AnomalyDetection"
21+
},
22+
{
23+
"name": "anomalib_models/stfpm.xml",
24+
"type": "AnomalyDetection"
25+
},
26+
{
27+
"name": "anomalib_models/uflow.xml",
28+
"type": "AnomalyDetection"
1729
}
1830
]

0 commit comments

Comments
 (0)