Skip to content

chore(test): remove --dump argument from accuracy tests #436

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tybulewicz wants to merge 1 commit into from
Closed

chore(test): remove --dump argument from accuracy tests #436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 0 additions & 21 deletions tests/accuracy/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,6 @@
# Copyright (C) 2020-2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
import json
from pathlib import Path

import pytest


def pytest_addoption(parser):
Expand All @@ -22,12 +18,6 @@ def pytest_addoption(parser):
default="CPU",
help="device to run tests on (in case of OpenvinoAdapter)",
)
parser.addoption(
"--dump",
action="store_true",
default=False,
help="whether to dump results into json file",
)
parser.addoption(
"--results-dir",
action="store",
Expand All @@ -38,14 +28,3 @@ def pytest_addoption(parser):

def pytest_configure(config):
config.test_results = []


@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
outcome = yield
result = outcome.get_result()

if result.when == "call":
test_results = item.config.test_results
with Path("test_scope.json").open("w") as outfile:
json.dump(test_results, outfile, indent=4)
65 changes: 1 addition & 64 deletions tests/accuracy/test_accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,6 @@ def device(pytestconfig):
return pytestconfig.getoption("device")


@pytest.fixture(scope="session")
def dump(pytestconfig):
return pytestconfig.getoption("dump")


@pytest.fixture(scope="session")
def result(pytestconfig):
return pytestconfig.test_results
Expand Down Expand Up @@ -183,30 +178,6 @@ def compare_classification_result(outputs: ClassificationResult, reference: dict
assert np.allclose(outputs.raw_scores, expected_scores, rtol=1e-2, atol=1e-1), "raw_scores mismatch"


def create_classification_result_dump(outputs: ClassificationResult) -> dict:
"""Create a JSON-serializable dump of ClassificationResult.

Args:
outputs: The ClassificationResult to serialize

Returns:
Dictionary containing top_labels and raw_scores in JSON-serializable format
"""
return {
"top_labels": [
{
"id": int(label.id) if label.id is not None else None,
"name": label.name,
"confidence": float(label.confidence) if label.confidence is not None else None,
}
for label in outputs.top_labels
]
if outputs.top_labels
else None,
"raw_scores": [float(x) for x in outputs.raw_scores.tolist()] if outputs.raw_scores is not None else None,
}


def compare_detection_result(outputs: DetectionResult, reference: dict) -> None:
"""Compare DetectionResult with reference data.

Expand Down Expand Up @@ -259,24 +230,7 @@ def compare_detection_result(outputs: DetectionResult, reference: dict) -> None:
assert outputs.label_names == reference["label_names"], "label_names mismatch"


def create_detection_result_dump(outputs: DetectionResult) -> dict:
"""Create a JSON-serializable dump of DetectionResult.

Args:
outputs: The DetectionResult to serialize

Returns:
Dictionary containing bboxes, labels, scores, and label_names in JSON-serializable format
"""
return {
"bboxes": outputs.bboxes.tolist() if outputs.bboxes is not None else None,
"labels": outputs.labels.tolist() if outputs.labels is not None else None,
"scores": [float(x) for x in outputs.scores.tolist()] if outputs.scores is not None else None,
"label_names": outputs.label_names if outputs.label_names is not None else None,
}


def test_image_models(data, device, dump, result, model_data, results_dir): # noqa: C901
def test_image_models(data, device, result, model_data, results_dir): # noqa: C901
name = model_data["name"]
if name.endswith((".xml", ".onnx")):
name = f"{data}/{name}"
Expand Down Expand Up @@ -322,10 +276,6 @@ def test_image_models(data, device, dump, result, model_data, results_dir): # n
)
model = MODEL_TYPE_MAPPING[model_data["prompter"]](encoder_model, model)

if dump:
result.append(model_data)
inference_results = []

for test_data in model_data["test_data"]:
image_path = Path(data) / test_data["image"]
image = cv2.imread(str(image_path))
Expand Down Expand Up @@ -377,10 +327,8 @@ def test_image_models(data, device, dump, result, model_data, results_dir): # n

if isinstance(outputs, ClassificationResult):
compare_classification_result(outputs, test_data["reference"])
image_result = create_classification_result_dump(outputs)
elif type(outputs) is DetectionResult:
compare_detection_result(outputs, test_data["reference"])
image_result = create_detection_result_dump(outputs)
elif isinstance(outputs, ImageResultWithSoftPrediction):
assert len(test_data["reference"]) == 1
if hasattr(model, "get_contours"):
Expand All @@ -392,7 +340,6 @@ def test_image_models(data, device, dump, result, model_data, results_dir): # n
contour_str += str(contour) + ", "
output_str = str(outputs) + contour_str
assert test_data["reference"][0] == output_str
image_result = [output_str]
elif type(outputs) is InstanceSegmentationResult:
assert len(test_data["reference"]) == 1
output_str = str(add_rotated_rects(outputs)) + "; "
Expand All @@ -401,22 +348,15 @@ def test_image_models(data, device, dump, result, model_data, results_dir): # n
# That doesn't hold for some models
output_str += "; ".join(str(contour) for contour in get_contours(outputs)) + "; "
assert test_data["reference"][0] == output_str
image_result = [output_str]
elif isinstance(outputs, AnomalyResult):
assert len(test_data["reference"]) == 1
output_str = str(outputs)
assert test_data["reference"][0] == output_str
image_result = [output_str]
elif isinstance(outputs, (ZSLVisualPromptingResult, VisualPromptingResult, DetectedKeypoints)):
output_str = str(outputs)
assert test_data["reference"][0] == output_str
image_result = [output_str]
else:
pytest.fail(f"Unexpected output type: {type(outputs)}")
if dump:
inference_results.append(
{"image": test_data["image"], "reference": image_result},
)
save_name = Path(name).name if name.endswith(".xml") else name + ".xml"

if not model_data.get("force_ort", False):
Expand All @@ -434,9 +374,6 @@ def test_image_models(data, device, dump, result, model_data, results_dir): # n
.astype(str)
)

if dump:
result[-1]["test_data"] = inference_results


def store_outputs(name, image, device, result, results_dir: str) -> None:
if not results_dir:
Expand Down
Loading