From ead83b18691e27fdbb70121416558787e0279f1c Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Mon, 15 Dec 2025 10:00:23 +0100 Subject: [PATCH 1/9] Remove legacy OVMS support --- src/model_api/models/classification.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/model_api/models/classification.py b/src/model_api/models/classification.py index 5e5d0e8e..e12e4fba 100644 --- a/src/model_api/models/classification.py +++ b/src/model_api/models/classification.py @@ -106,16 +106,10 @@ def _setup_single_label(self) -> None: if self.params.output_raw_scores: self.out_layer_names.append(self.raw_scores_name) except (RuntimeError, AttributeError): - # exception means we have a non-ov model - # with already inserted softmax and topk - if self.params.embedded_processing and len(self.outputs) >= 2: - self.embedded_topk = True - self.out_layer_names = ["indices", "scores"] - self.raw_scores_name = _raw_scores_name - else: # likely a non-ov model - self.embedded_topk = False - self.out_layer_names = _get_non_xai_names(self.outputs.keys()) - self.raw_scores_name = self.out_layer_names[0] + # non OV model + self.embedded_topk = False + self.out_layer_names = _get_non_xai_names(self.outputs.keys()) + self.raw_scores_name = self.out_layer_names[0] self.embedded_processing = True From a4c4fe38e9e0206e6d481e1e1c24a5e4a0c4cb4b Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Mon, 15 Dec 2025 10:03:33 +0100 Subject: [PATCH 2/9] Simplify labels selection in get_multiclass_predictions --- src/model_api/models/classification.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/model_api/models/classification.py b/src/model_api/models/classification.py index e12e4fba..7ed09d59 100644 --- a/src/model_api/models/classification.py +++ b/src/model_api/models/classification.py @@ -271,15 +271,16 @@ def get_multilabel_predictions(self, logits: np.ndarray) -> list[Label]: return list(starmap(Label, zip(indices, labels, scores))) def get_multiclass_predictions(self, outputs: dict) -> list[Label]: - labels_list = self.params.labels if self.embedded_topk: indicesTensor = outputs[self.out_layer_names[0]][0] scoresTensor = outputs[self.out_layer_names[1]][0] - labels = [labels_list[i] if labels_list else "" for i in indicesTensor] else: scoresTensor = softmax(outputs[self.out_layer_names[0]][0]) indicesTensor = [int(np.argmax(scoresTensor))] - labels = [labels_list[i] if labels_list else "" for i in indicesTensor] + + labels_list = self.params.labels + labels = [labels_list[i] if labels_list else "" for i in indicesTensor] + return list(starmap(Label, zip(indicesTensor, labels, scoresTensor))) From e63582a8d0f1c1ecf51c7de7388670af8d19517d Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Mon, 15 Dec 2025 14:08:55 +0100 Subject: [PATCH 3/9] Skip graph modification for models without embedded softmax --- src/model_api/models/classification.py | 15 +++++++++------ src/model_api/models/utils.py | 20 ++++++++++++++++++++ tests/accuracy/public_scope.json | 2 +- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/model_api/models/classification.py b/src/model_api/models/classification.py index 7ed09d59..2ce5646d 100644 --- a/src/model_api/models/classification.py +++ b/src/model_api/models/classification.py @@ -19,7 +19,7 @@ from model_api.models.image_model import ImageModel from model_api.models.parameters import ParameterRegistry from model_api.models.result import ClassificationResult, Label -from model_api.models.utils import softmax +from model_api.models.utils import softmax, topk if TYPE_CHECKING: from model_api.adapters.inference_adapter import InferenceAdapter @@ -106,7 +106,7 @@ def _setup_single_label(self) -> None: if self.params.output_raw_scores: self.out_layer_names.append(self.raw_scores_name) except (RuntimeError, AttributeError): - # non OV model + # model does not have embedded topk, will be calculated later in postprocessing self.embedded_topk = False self.out_layer_names = _get_non_xai_names(self.outputs.keys()) self.raw_scores_name = self.out_layer_names[0] @@ -275,8 +275,10 @@ def get_multiclass_predictions(self, outputs: dict) -> list[Label]: indicesTensor = outputs[self.out_layer_names[0]][0] scoresTensor = outputs[self.out_layer_names[1]][0] else: - scoresTensor = softmax(outputs[self.out_layer_names[0]][0]) - indicesTensor = [int(np.argmax(scoresTensor))] + softmaxResult = softmax(outputs[self.out_layer_names[0]], axis=1) + topKResult = topk(softmaxResult, self.params.topk, 1) + scoresTensor = topKResult.values[0] # noqa: PD011 # silecing false positive - it's not pandas code + indicesTensor = topKResult.indices[0] labels_list = self.params.labels labels = [labels_list[i] if labels_list else "" for i in indicesTensor] @@ -294,8 +296,9 @@ def addOrFindSoftmaxAndTopkOutputs(inference_adapter: InferenceAdapter, topk: in return if softmaxNode is None: - logitsNode = inference_adapter.model.get_output_op(0).input(0).get_source_output().get_node() - softmaxNode = opset.softmax(logitsNode.output(0), 1) + # no softmax found, will be calculated in postprocessing + raise RuntimeError + k = opset.constant(topk, np.int32) topkNode = opset.topk(softmaxNode, k, 1, "max", "value") diff --git a/src/model_api/models/utils.py b/src/model_api/models/utils.py index fb7c9cd4..ae34d1b7 100644 --- a/src/model_api/models/utils.py +++ b/src/model_api/models/utils.py @@ -5,6 +5,7 @@ from __future__ import annotations +from collections import namedtuple from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING @@ -14,6 +15,9 @@ from model_api.models.result import Contour, InstanceSegmentationResult, RotatedSegmentationResult +topk_namedtuple = namedtuple("topk_namedtuple", ["values", "indices"]) + + if TYPE_CHECKING: from model_api.models.result.detection import DetectionResult @@ -287,3 +291,19 @@ def multiclass_nms( def softmax(logits: np.ndarray, eps: float = 1e-9, axis=None, keepdims: bool = False) -> np.ndarray: exp = np.exp(logits - np.max(logits)) return exp / (np.sum(exp, axis=axis, keepdims=keepdims) + eps) + + +def topk(array: np.ndarray, k: int, axis: int) -> topk_namedtuple: + """Returns the top k values and their indices along the specified axis.""" + if k <= 0 or k > array.shape[axis]: + message = "k must be in the range 1 to the size of the selected axis" + raise ValueError(message) + # Get indices of the top k elements + indices = np.take(np.argpartition(array, -k, axis=axis), range(-k, 0), axis=axis) + # Gather the top k values + topk_values = np.take_along_axis(array, indices, axis=axis) + # Sort the top k values and indices in descending order + sorted_order = np.argsort(-topk_values, axis=axis) + topk_values = np.take_along_axis(topk_values, sorted_order, axis=axis) + indices = np.take_along_axis(indices, sorted_order, axis=axis) + return topk_namedtuple(values=topk_values, indices=indices) diff --git a/tests/accuracy/public_scope.json b/tests/accuracy/public_scope.json index 5586aec4..4397ee47 100644 --- a/tests/accuracy/public_scope.json +++ b/tests/accuracy/public_scope.json @@ -346,7 +346,7 @@ { "id": 105, "name": "194", - "confidence": 0.06216677650809288 + "confidence": 0.4564049541950226 } ], "raw_scores": [ From b548f58472f4471767c81631128472766d3c094d Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Mon, 15 Dec 2025 14:09:30 +0100 Subject: [PATCH 4/9] Skip not needed model read in tests --- tests/accuracy/test_accuracy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/accuracy/test_accuracy.py b/tests/accuracy/test_accuracy.py index 4c6d32c1..ea161210 100644 --- a/tests/accuracy/test_accuracy.py +++ b/tests/accuracy/test_accuracy.py @@ -99,7 +99,7 @@ def create_models(model_type, model_path, download_dir, force_onnx_adapter=False model = create_core().read_model(model_path) if model.has_rt_info(["model_info", "model_type"]): wrapper_type = model_type.get_model_class( - create_core().read_model(model_path).get_rt_info(["model_info", "model_type"]).astype(str), + model.get_rt_info(["model_info", "model_type"]).astype(str), ) model = wrapper_type(OpenvinoAdapter(create_core(), model_path, device=device)) model.load() From 984fa4de19729784848eb8d0ffda709a4acbbfbd Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Mon, 15 Dec 2025 15:08:54 +0100 Subject: [PATCH 5/9] Calculate TopK if output is already softmaxed --- src/model_api/models/classification.py | 92 +++----------------------- src/model_api/models/utils.py | 10 +++ tests/accuracy/public_scope.json | 6 +- 3 files changed, 22 insertions(+), 86 deletions(-) diff --git a/src/model_api/models/classification.py b/src/model_api/models/classification.py index 2ce5646d..1fef0a0b 100644 --- a/src/model_api/models/classification.py +++ b/src/model_api/models/classification.py @@ -12,14 +12,11 @@ from typing import TYPE_CHECKING import numpy as np -from openvino import Model, Type -from openvino import opset10 as opset -from openvino.preprocess import PrePostProcessor from model_api.models.image_model import ImageModel from model_api.models.parameters import ParameterRegistry from model_api.models.result import ClassificationResult, Label -from model_api.models.utils import softmax, topk +from model_api.models.utils import is_softmaxed, softmax, topk if TYPE_CHECKING: from model_api.adapters.inference_adapter import InferenceAdapter @@ -95,21 +92,8 @@ def _setup_multilabel(self) -> None: def _setup_single_label(self) -> None: """Configure model for single-label classification with TopK.""" - try: - addOrFindSoftmaxAndTopkOutputs( - self.inference_adapter, - self.params.topk, - self.params.output_raw_scores, - ) - self.embedded_topk = True - self.out_layer_names = ["indices", "scores"] - if self.params.output_raw_scores: - self.out_layer_names.append(self.raw_scores_name) - except (RuntimeError, AttributeError): - # model does not have embedded topk, will be calculated later in postprocessing - self.embedded_topk = False - self.out_layer_names = _get_non_xai_names(self.outputs.keys()) - self.raw_scores_name = self.out_layer_names[0] + self.out_layer_names = _get_non_xai_names(self.outputs.keys()) + self.raw_scores_name = self.out_layer_names[0] self.embedded_processing = True @@ -222,8 +206,6 @@ def get_all_probs(self, logits: np.ndarray) -> np.ndarray: if cls_heads_info["num_multilabel_classes"]: logits_begin = cls_heads_info["num_single_label_classes"] probs[logits_begin:] = sigmoid_numpy(logits[logits_begin:]) - elif self.embedded_topk: - probs = logits.reshape(-1) else: probs = softmax(logits.reshape(-1)) return probs @@ -271,14 +253,13 @@ def get_multilabel_predictions(self, logits: np.ndarray) -> list[Label]: return list(starmap(Label, zip(indices, labels, scores))) def get_multiclass_predictions(self, outputs: dict) -> list[Label]: - if self.embedded_topk: - indicesTensor = outputs[self.out_layer_names[0]][0] - scoresTensor = outputs[self.out_layer_names[1]][0] - else: - softmaxResult = softmax(outputs[self.out_layer_names[0]], axis=1) - topKResult = topk(softmaxResult, self.params.topk, 1) - scoresTensor = topKResult.values[0] # noqa: PD011 # silecing false positive - it's not pandas code - indicesTensor = topKResult.indices[0] + axis = 1 + logits = outputs[self.out_layer_names[0]] + if not is_softmaxed(logits, axis=axis): + logits = softmax(logits, axis=axis) + topKResult = topk(logits, self.params.topk, axis=axis) + scoresTensor = topKResult.values[0] # noqa: PD011 # silencing false positive - it's not pandas code + indicesTensor = topKResult.indices[0] labels_list = self.params.labels labels = [labels_list[i] if labels_list else "" for i in indicesTensor] @@ -286,59 +267,6 @@ def get_multiclass_predictions(self, outputs: dict) -> list[Label]: return list(starmap(Label, zip(indicesTensor, labels, scoresTensor))) -def addOrFindSoftmaxAndTopkOutputs(inference_adapter: InferenceAdapter, topk: int, output_raw_scores: bool) -> None: - softmaxNode = None - for i in range(len(inference_adapter.model.outputs)): - output_node = inference_adapter.model.get_output_op(i).input(0).get_source_output().get_node() - if output_node.get_type_name() == "Softmax": - softmaxNode = output_node - elif output_node.get_type_name() == "TopK": - return - - if softmaxNode is None: - # no softmax found, will be calculated in postprocessing - raise RuntimeError - - k = opset.constant(topk, np.int32) - topkNode = opset.topk(softmaxNode, k, 1, "max", "value") - - indices = topkNode.output(0) - scores = topkNode.output(1) - results_descr = [indices, scores] - if output_raw_scores: - raw_scores = softmaxNode.output(0) - results_descr.append(raw_scores) - for output in inference_adapter.model.outputs: - if _saliency_map_name in output.get_names() or _feature_vector_name in output.get_names(): - results_descr.append(output) - - source_rt_info = inference_adapter.get_model().get_rt_info() - inference_adapter.model = Model( - results_descr, - inference_adapter.model.get_parameters(), - "classification", - ) - - if "model_info" in source_rt_info: - source_rt_info = source_rt_info["model_info"] - for k in source_rt_info: - inference_adapter.model.set_rt_info(source_rt_info[k], ["model_info", k]) - - # manually set output tensors name for created topK node - inference_adapter.model.outputs[0].tensor.set_names({"scores"}) - inference_adapter.model.outputs[1].tensor.set_names({"indices"}) - if output_raw_scores: - inference_adapter.model.outputs[2].tensor.set_names({_raw_scores_name}) - - # set output precisions - ppp = PrePostProcessor(inference_adapter.model) - ppp.output("indices").tensor().set_element_type(Type.i32) - ppp.output("scores").tensor().set_element_type(Type.f32) - if output_raw_scores: - ppp.output(_raw_scores_name).tensor().set_element_type(Type.f32) - inference_adapter.model = ppp.build() - - def sigmoid_numpy(x: np.ndarray) -> np.ndarray: return 1.0 / (1.0 + np.exp(-x)) diff --git a/src/model_api/models/utils.py b/src/model_api/models/utils.py index ae34d1b7..18049051 100644 --- a/src/model_api/models/utils.py +++ b/src/model_api/models/utils.py @@ -288,6 +288,16 @@ def multiclass_nms( return det, keep +def is_softmaxed(array: np.ndarray, axis: int, atol: float = 1e-5) -> bool: + """Check if the input array is softmaxed along the specified axis.""" + # Check values are in [0, 1] + if not np.all((array >= 0) & (array <= 1)): + return False + # Check sum along axis is close to 1 + sums = np.sum(array, axis=axis) + return np.allclose(sums, 1.0, atol=atol) + + def softmax(logits: np.ndarray, eps: float = 1e-9, axis=None, keepdims: bool = False) -> np.ndarray: exp = np.exp(logits - np.max(logits)) return exp / (np.sum(exp, axis=axis, keepdims=keepdims) + eps) diff --git a/tests/accuracy/public_scope.json b/tests/accuracy/public_scope.json index 4397ee47..b773251b 100644 --- a/tests/accuracy/public_scope.json +++ b/tests/accuracy/public_scope.json @@ -703,10 +703,8 @@ } ], "raw_scores": [ - 0.027470914646983147, 0.11825039982795715, 0.01633000187575817, - 0.6481003165245056, 0.008704839274287224, 0.11668245494365692, - 0.006007326766848564, 0.003731546690687537, 0.01372036337852478, - 0.0410018190741539 + 0.09112413, 0.09978341, 0.09011459, 0.16950002, 0.08943006, + 0.0996271, 0.08918914, 0.08898639, 0.0898797, 0.09236551 ] } } From f1e918309ed853b32cbab6af8ea9748faac2cf5c Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Tue, 16 Dec 2025 09:02:37 +0100 Subject: [PATCH 6/9] Don't softmax raw_scores if already done --- src/model_api/models/classification.py | 2 +- tests/accuracy/public_scope.json | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/model_api/models/classification.py b/src/model_api/models/classification.py index 1fef0a0b..7e9f5efa 100644 --- a/src/model_api/models/classification.py +++ b/src/model_api/models/classification.py @@ -207,7 +207,7 @@ def get_all_probs(self, logits: np.ndarray) -> np.ndarray: logits_begin = cls_heads_info["num_single_label_classes"] probs[logits_begin:] = sigmoid_numpy(logits[logits_begin:]) else: - probs = softmax(logits.reshape(-1)) + probs = logits.reshape(-1) if is_softmaxed(logits, axis=1) else softmax(logits.reshape(-1)) return probs def get_hierarchical_predictions(self, logits: np.ndarray) -> list[Label]: diff --git a/tests/accuracy/public_scope.json b/tests/accuracy/public_scope.json index b773251b..4397ee47 100644 --- a/tests/accuracy/public_scope.json +++ b/tests/accuracy/public_scope.json @@ -703,8 +703,10 @@ } ], "raw_scores": [ - 0.09112413, 0.09978341, 0.09011459, 0.16950002, 0.08943006, - 0.0996271, 0.08918914, 0.08898639, 0.0898797, 0.09236551 + 0.027470914646983147, 0.11825039982795715, 0.01633000187575817, + 0.6481003165245056, 0.008704839274287224, 0.11668245494365692, + 0.006007326766848564, 0.003731546690687537, 0.01372036337852478, + 0.0410018190741539 ] } } From 2ab578f5a46d79c9d5f1062432c9a124f62b6631 Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Tue, 16 Dec 2025 09:13:03 +0100 Subject: [PATCH 7/9] Rename variables to match python standards --- src/model_api/models/classification.py | 12 ++++++------ src/model_api/models/utils.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/model_api/models/classification.py b/src/model_api/models/classification.py index 7e9f5efa..174f3342 100644 --- a/src/model_api/models/classification.py +++ b/src/model_api/models/classification.py @@ -16,7 +16,7 @@ from model_api.models.image_model import ImageModel from model_api.models.parameters import ParameterRegistry from model_api.models.result import ClassificationResult, Label -from model_api.models.utils import is_softmaxed, softmax, topk +from model_api.models.utils import is_softmaxed, softmax, top_k if TYPE_CHECKING: from model_api.adapters.inference_adapter import InferenceAdapter @@ -257,14 +257,14 @@ def get_multiclass_predictions(self, outputs: dict) -> list[Label]: logits = outputs[self.out_layer_names[0]] if not is_softmaxed(logits, axis=axis): logits = softmax(logits, axis=axis) - topKResult = topk(logits, self.params.topk, axis=axis) - scoresTensor = topKResult.values[0] # noqa: PD011 # silencing false positive - it's not pandas code - indicesTensor = topKResult.indices[0] + top_k_result = top_k(logits, self.params.topk, axis=axis) + scores_tensor = top_k_result.values[0] # noqa: PD011 # silencing false positive - it's not pandas code + indices_tensor = top_k_result.indices[0] labels_list = self.params.labels - labels = [labels_list[i] if labels_list else "" for i in indicesTensor] + labels = [labels_list[i] if labels_list else "" for i in indices_tensor] - return list(starmap(Label, zip(indicesTensor, labels, scoresTensor))) + return list(starmap(Label, zip(indices_tensor, labels, scores_tensor))) def sigmoid_numpy(x: np.ndarray) -> np.ndarray: diff --git a/src/model_api/models/utils.py b/src/model_api/models/utils.py index 18049051..46d40e77 100644 --- a/src/model_api/models/utils.py +++ b/src/model_api/models/utils.py @@ -303,7 +303,7 @@ def softmax(logits: np.ndarray, eps: float = 1e-9, axis=None, keepdims: bool = F return exp / (np.sum(exp, axis=axis, keepdims=keepdims) + eps) -def topk(array: np.ndarray, k: int, axis: int) -> topk_namedtuple: +def top_k(array: np.ndarray, k: int, axis: int) -> topk_namedtuple: """Returns the top k values and their indices along the specified axis.""" if k <= 0 or k > array.shape[axis]: message = "k must be in the range 1 to the size of the selected axis" From 3c3769661f8891c725c1f94eb46befe232ec30b8 Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Tue, 16 Dec 2025 09:31:40 +0100 Subject: [PATCH 8/9] Remove misleading variable names --- src/model_api/models/classification.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/model_api/models/classification.py b/src/model_api/models/classification.py index 174f3342..0df0fed9 100644 --- a/src/model_api/models/classification.py +++ b/src/model_api/models/classification.py @@ -258,13 +258,13 @@ def get_multiclass_predictions(self, outputs: dict) -> list[Label]: if not is_softmaxed(logits, axis=axis): logits = softmax(logits, axis=axis) top_k_result = top_k(logits, self.params.topk, axis=axis) - scores_tensor = top_k_result.values[0] # noqa: PD011 # silencing false positive - it's not pandas code - indices_tensor = top_k_result.indices[0] + scores = top_k_result.values[0] # noqa: PD011 # silencing false positive - it's not pandas code + indices = top_k_result.indices[0] labels_list = self.params.labels - labels = [labels_list[i] if labels_list else "" for i in indices_tensor] + labels = [labels_list[i] if labels_list else "" for i in indices] - return list(starmap(Label, zip(indices_tensor, labels, scores_tensor))) + return list(starmap(Label, zip(indices, labels, scores))) def sigmoid_numpy(x: np.ndarray) -> np.ndarray: From abb53e43dbe72dd160bcad93aa09cb5db18d0608 Mon Sep 17 00:00:00 2001 From: "Tybulewicz, Tomasz" Date: Tue, 16 Dec 2025 11:13:10 +0100 Subject: [PATCH 9/9] Remove handholding from internal function --- src/model_api/models/classification.py | 3 ++- src/model_api/models/utils.py | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/model_api/models/classification.py b/src/model_api/models/classification.py index 0df0fed9..ecb02b25 100644 --- a/src/model_api/models/classification.py +++ b/src/model_api/models/classification.py @@ -207,7 +207,8 @@ def get_all_probs(self, logits: np.ndarray) -> np.ndarray: logits_begin = cls_heads_info["num_single_label_classes"] probs[logits_begin:] = sigmoid_numpy(logits[logits_begin:]) else: - probs = logits.reshape(-1) if is_softmaxed(logits, axis=1) else softmax(logits.reshape(-1)) + logits_flattened = logits.reshape(-1) + probs = logits_flattened if is_softmaxed(logits_flattened, axis=0) else softmax(logits_flattened) return probs def get_hierarchical_predictions(self, logits: np.ndarray) -> list[Label]: diff --git a/src/model_api/models/utils.py b/src/model_api/models/utils.py index 46d40e77..e02b77bb 100644 --- a/src/model_api/models/utils.py +++ b/src/model_api/models/utils.py @@ -305,9 +305,6 @@ def softmax(logits: np.ndarray, eps: float = 1e-9, axis=None, keepdims: bool = F def top_k(array: np.ndarray, k: int, axis: int) -> topk_namedtuple: """Returns the top k values and their indices along the specified axis.""" - if k <= 0 or k > array.shape[axis]: - message = "k must be in the range 1 to the size of the selected axis" - raise ValueError(message) # Get indices of the top k elements indices = np.take(np.argpartition(array, -k, axis=axis), range(-k, 0), axis=axis) # Gather the top k values