diff --git a/perceptionmetrics/evaluation/__init__.py b/perceptionmetrics/evaluation/__init__.py new file mode 100644 index 00000000..4258807c --- /dev/null +++ b/perceptionmetrics/evaluation/__init__.py @@ -0,0 +1,3 @@ +from perceptionmetrics.evaluation.segmentation_evaluator import SegmentationEvaluator + +__all__ = ["SegmentationEvaluator"] diff --git a/perceptionmetrics/evaluation/segmentation_evaluator.py b/perceptionmetrics/evaluation/segmentation_evaluator.py new file mode 100644 index 00000000..66035fab --- /dev/null +++ b/perceptionmetrics/evaluation/segmentation_evaluator.py @@ -0,0 +1,60 @@ +from typing import Any, List + +import numpy as np +from PIL import Image + +import perceptionmetrics.utils.segmentation_metrics as um + + +class SegmentationEvaluator: + """Minimal prototype evaluator for segmentation models. + + This class extracts only the common evaluation loop: + dataset iteration, `model.predict()` invocation, and metric aggregation. + + The current prototype intentionally excludes ontology translation, prediction + saving, per-sample reports, batching, and backend-specific behavior so it can + be integrated safely and extended incrementally in the future. + + Note: + This implementation is intentionally minimal and mirrors only the + default evaluation path to ensure backward compatibility. + """ + + def __init__(self, n_classes: int): + """Initialize metric aggregation. + + :param n_classes: Number of segmentation classes. + :type n_classes: int + """ + self.n_classes = n_classes + self.metrics_factory = um.SegmentationMetricsFactory(n_classes) + + def evaluate(self, model: Any, dataset: Any, splits: List[str]): + """Evaluate a segmentation model over dataset samples. + + :param model: Segmentation model exposing `predict(image)`. + :type model: Any + :param dataset: Dataset object exposing `dataset` and `make_fname_global`. + :type dataset: Any + :param splits: Dataset splits to evaluate. + :type splits: List[str] + :return: Aggregated segmentation metrics factory. + :rtype: um.SegmentationMetricsFactory + """ + if "split" not in dataset.dataset.columns: + raise ValueError("Dataset must contain 'split' column") + + dataset.make_fname_global() + df = dataset.dataset[dataset.dataset["split"].isin(splits)].copy() + + for _, row in df.iterrows(): + image = Image.open(row["image"]).convert("RGB") + label = np.array(Image.open(row["label"])) + pred = model.predict(image) + + pred = np.array(pred) + + self.metrics_factory.update(pred, label, None) + + return self.metrics_factory diff --git a/perceptionmetrics/models/torch_segmentation.py b/perceptionmetrics/models/torch_segmentation.py index 5b71ff78..f935ec33 100644 --- a/perceptionmetrics/models/torch_segmentation.py +++ b/perceptionmetrics/models/torch_segmentation.py @@ -389,6 +389,26 @@ def eval( if predictions_outdir is not None: os.makedirs(predictions_outdir, exist_ok=True) + splits = [split] if isinstance(split, str) else split + ignored_classes = self.model_cfg.get("ignored_classes", []) + use_evaluator = ( + ontology_translation is None + and predictions_outdir is None + and not results_per_sample + and len(ignored_classes) == 0 + ) + + if use_evaluator: + # Use minimal evaluator only for default/simple path to avoid + # modifying advanced evaluation behavior (ontology, saving, etc.). + from perceptionmetrics.evaluation.segmentation_evaluator import ( + SegmentationEvaluator, + ) + + evaluator = SegmentationEvaluator(n_classes=len(self.ontology)) + metrics_factory = evaluator.evaluate(self, dataset, splits) + return um.get_metrics_dataframe(metrics_factory, self.ontology) + # Build a LUT for transforming ontology if needed (aligned with TorchLiDARSegmentationModel.eval) eval_ontology = self.ontology @@ -420,7 +440,7 @@ def eval( dataset, transform=self.transform_input, target_transform=self.transform_label, - splits=[split] if isinstance(split, str) else split, + splits=splits, ) dataloader = DataLoader(