Enable offline image detection evaluation + improvements in resizing/padding (#608)

* Refactor image detection resizing to allow for padding to closest divisor

* Enable offline image detection evaluation and minor fixes in relevant datasets

* Add example for offline image detection evaluation

* Fix bug in new yolo dataset parsing

* Fix missing attribute

* Allow for setting longer side max size alone for image detection

* Update streamlit app to support padding and max side for image detection

Author: dpascualhe

app.py

@@ -171,7 +171,7 @@ def browse_dataset_path():
             if enable_resize:
                 resize_strategy = st.radio(
                     "Resize Strategy",
-                    ["Fixed Dimensions", "Min Side"],
+                    ["Min Side", "Max Side", "Fixed Dimensions"],
                     key="resize_strategy",
                     horizontal=True,
                     label_visibility="collapsed",
@@ -199,7 +199,7 @@ def browse_dataset_path():
                             key="resize_width",
                             help="Width to resize images for inference",
                         )
-                else:
+                elif resize_strategy == "Min Side":
                     st.number_input(
                         "Min Side",
                         min_value=1,
@@ -209,6 +209,34 @@ def browse_dataset_path():
                         key="min_side",
                         help="Minimum size of the shorter side of the image",
                     )
+                elif resize_strategy == "Max Side":
+                    st.number_input(
+                        "Max Side",
+                        min_value=1,
+                        max_value=4096,
+                        value=640,
+                        step=1,
+                        key="max_side",
+                        help="Maximum size of the longer side of the image",
+                    )
+                else:
+                    st.error("Invalid resize strategy selected")
+
+            # Pad to closest multiple
+            enable_pad = st.checkbox(
+                "Enable Padding to Closest Multiple", value=True, key="enable_pad"
+            )
+
+            if enable_pad:
+                st.number_input(
+                    "Divisor",
+                    min_value=1,
+                    max_value=128,
+                    value=32,
+                    step=1,
+                    key="pad_divisor",
+                    help="Pad image dimensions to the closest multiple of this value",
+                )
 
             # Crop Logic
             enable_crop = st.checkbox("Enable Center Crop", key="enable_crop")
@@ -303,9 +331,21 @@ def browse_dataset_path():
                                 "height": resize_height,
                                 "width": resize_width,
                             }
-                        else:
+                        elif resize_strategy == "Min Side":
                             min_side = int(st.session_state.get("min_side", 640))
                             resize_cfg = {"min_side": min_side}
+                        elif resize_strategy == "Max Side":
+                            max_side = int(st.session_state.get("max_side", 640))
+                            resize_cfg = {"max_side": max_side}
+                        else:
+                            st.error("Invalid resize strategy selected")
+
+                    if enable_pad:
+                        pad_divisor = int(st.session_state.get("pad_divisor", 32))
+                        if resize_cfg is not None:
+                            resize_cfg["closest_divisor"] = pad_divisor
+                        else:
+                            resize_cfg = {"closest_divisor": pad_divisor}
 
                     config_data = {
                         "confidence_threshold": confidence_threshold,

examples/eval_detection_preds.py

@@ -0,0 +1,97 @@
+import argparse
+import os
+
+from perceptionmetrics.datasets.yolo import YOLODataset
+from perceptionmetrics.datasets.coco import CocoDataset
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse user input arguments.
+
+    :return: Parsed arguments
+    :rtype: argparse.Namespace
+    """
+    parser = argparse.ArgumentParser(
+        description="Evaluate pre-computed detection predictions against a GT dataset."
+    )
+    parser.add_argument(
+        "--format",
+        type=str,
+        choices=["yolo", "coco"],
+        default="yolo",
+        help="Dataset format (yolo or coco)",
+    )
+    parser.add_argument(
+        "--dataset",
+        type=str,
+        required=True,
+        help="Dataset configuration file (YAML for YOLO, JSON for COCO)",
+    )
+    parser.add_argument(
+        "--dataset_dir",
+        type=str,
+        default=None,
+        help="Path to the directory containing images. For YOLO, inferred if not provided.",
+    )
+    parser.add_argument(
+        "--predictions_dir",
+        type=str,
+        required=True,
+        help="Root directory containing prediction JSON files",
+    )
+    parser.add_argument(
+        "--split",
+        type=str,
+        default="test",
+        help="Name of the split to evaluate (default: test)",
+    )
+    parser.add_argument(
+        "--out_fname",
+        type=str,
+        required=True,
+        help="CSV file where the evaluation results will be stored",
+    )
+    parser.add_argument(
+        "--ignored_classes",
+        type=str,
+        nargs="+",
+        default=None,
+        help="List of class names to ignore during evaluation",
+    )
+    parser.add_argument(
+        "--results_per_sample",
+        action="store_true",
+        help="Store per-sample results as CSV files next to each prediction",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """Main function."""
+    args = parse_args()
+
+    if args.format == "yolo":
+        dataset = YOLODataset(dataset_fname=args.dataset, dataset_dir=args.dataset_dir)
+    elif args.format == "coco":
+        if args.dataset_dir is None:
+            raise ValueError("--dataset_dir is required for COCO format")
+        dataset = CocoDataset(
+            annotation_file=args.dataset, image_dir=args.dataset_dir, split=args.split
+        )
+    else:
+        raise ValueError(f"Unsupported format: {args.format}")
+
+    results = dataset.eval_preds(
+        predictions_dir=args.predictions_dir,
+        split=args.split,
+        ignored_classes=args.ignored_classes,
+        results_per_sample=args.results_per_sample,
+    )
+
+    os.makedirs(os.path.dirname(args.out_fname), exist_ok=True)
+    results.to_csv(args.out_fname)
+    print(f"Results saved to {args.out_fname}")
+
+
+if __name__ == "__main__":
+    main()

perceptionmetrics/datasets/coco.py

@@ -1,3 +1,4 @@
+from typing import OrderedDict
 from pycocotools.coco import COCO
 import os
 import re
@@ -88,18 +89,18 @@ def build_coco_dataset(
         }
 
     # Build dataset DataFrame from COCO image IDs
-    rows = []
+    dataset = OrderedDict()
     for img_id in coco.getImgIds():
         img_info = coco.loadImgs(img_id)[0]
-        rows.append(
-            {
-                "image": img_info["file_name"],
-                "annotation": str(img_id),
-                "split": split,  # Use provided split parameter
-            }
+        sample_name = os.path.basename(img_info["file_name"]).split(".")[0]
+        dataset[sample_name] = (
+            img_info["file_name"],
+            str(img_id),
+            split,  # Use provided split parameter
         )
 
-    dataset = pd.DataFrame(rows)
+    cols = ["image", "annotation", "split"]
+    dataset = pd.DataFrame.from_dict(dataset, orient="index", columns=cols)
     dataset.attrs = {"ontology": ontology}
 
     return dataset, ontology

perceptionmetrics/datasets/detection.py

@@ -1,4 +1,5 @@
 from abc import abstractmethod
+import json
 import os
 from typing import List, Optional, Union
 
@@ -7,6 +8,8 @@
 from tqdm import tqdm
 
 from perceptionmetrics.datasets.perception import PerceptionDataset
+import perceptionmetrics.utils.conversion as uc
+import perceptionmetrics.utils.detection_metrics as um
 
 
 class DetectionDataset(PerceptionDataset):
@@ -100,9 +103,112 @@ def eval_preds(
         :return: DataFrame containing evaluation results
         :rtype: pd.DataFrame
         """
-        raise NotImplementedError(
-            "eval_preds is not yet implemented for ImageDetectionDataset"
-        )
+        splits = [split] if isinstance(split, str) else split
+        self._validate_splits(splits)
+
+        df = self.dataset[self.dataset["split"].isin(splits)]
+
+        # Determine the evaluation ontology and build a LUT if needed
+        eval_ontology = self.ontology
+        lut_ontology = None
+
+        if pred_ontology is None:
+            pred_ontology = self.ontology
+
+        if pred_ontology != self.ontology:
+            if ontology_translation is None:
+                raise ValueError(
+                    "'ontology_translation' must be provided when GT and prediction "
+                    "ontologies differ."
+                )
+            if translation_direction == "dataset_to_model":
+                eval_ontology = pred_ontology
+                lut_ontology = uc.get_ontology_conversion_lut(
+                    self.ontology, pred_ontology, ontology_translation
+                )
+            else:
+                lut_ontology = uc.get_ontology_conversion_lut(
+                    pred_ontology, self.ontology, ontology_translation
+                )
+
+        n_classes = len(eval_ontology)
+
+        # Retrieve ignored label indices
+        ignored_label_indices = []
+        if ignored_classes:
+            for cls_name in ignored_classes:
+                ignored_label_indices.append(self.ontology[cls_name]["idx"])
+
+        # Init metrics
+        metrics_factory = um.DetectionMetricsFactory(num_classes=n_classes)
+
+        pbar = tqdm(df.iterrows(), total=len(df), leave=True)
+        for sample_name, row in pbar:
+            pbar.set_description(f"Evaluating sample: {sample_name}")
+
+            # Read GT annotation
+            gt_ann_fname = row["annotation"]
+            if self.dataset_dir is not None:
+                gt_ann_fname = os.path.join(self.dataset_dir, gt_ann_fname)
+
+            gt_boxes, gt_labels = self.read_annotation(gt_ann_fname)
+            gt_boxes = (
+                np.array(gt_boxes, dtype=np.float32).reshape(-1, 4)
+                if len(gt_boxes) > 0
+                else np.zeros((0, 4), dtype=np.float32)
+            )
+            gt_labels = np.array(gt_labels, dtype=np.int64)
+
+            # Build valid mask from ignored classes
+            if ignored_label_indices:
+                valid_mask = np.ones(len(gt_labels), dtype=bool)
+                for idx in ignored_label_indices:
+                    valid_mask &= gt_labels != idx
+                gt_boxes = gt_boxes[valid_mask]
+                gt_labels = gt_labels[valid_mask]
+
+            # Read predictions
+            pred_file = os.path.join(predictions_dir, f"{sample_name}.json")
+            if not os.path.isfile(pred_file):
+                raise FileNotFoundError(f"Prediction file not found: {pred_file}")
+
+            with open(pred_file, "r") as f:
+                preds = json.load(f)
+
+            pred_boxes = [p["bbox"] for p in preds]
+            pred_labels = [p["label"] for p in preds]
+            pred_scores = [p["score"] for p in preds]
+
+            pred_boxes = (
+                np.array(pred_boxes, dtype=np.float32).reshape(-1, 4)
+                if pred_boxes
+                else np.zeros((0, 4), dtype=np.float32)
+            )
+            pred_labels = np.array(pred_labels, dtype=np.int64)
+            pred_scores = np.array(pred_scores, dtype=np.float32)
+
+            # Apply ontology translation
+            if lut_ontology is not None:
+                if translation_direction == "dataset_to_model":
+                    gt_labels = lut_ontology[gt_labels]
+                else:
+                    pred_labels = lut_ontology[pred_labels]
+
+            metrics_factory.update(
+                gt_boxes, gt_labels, pred_boxes, pred_labels, pred_scores
+            )
+
+            # Per-sample results
+            if results_per_sample:
+                sample_mf = um.DetectionMetricsFactory(num_classes=n_classes)
+                sample_mf.update(
+                    gt_boxes, gt_labels, pred_boxes, pred_labels, pred_scores
+                )
+                sample_df = sample_mf.get_metrics_dataframe(eval_ontology)
+                sample_csv = os.path.join(predictions_dir, f"{sample_name}_metrics.csv")
+                sample_df.to_csv(sample_csv)
+
+        return metrics_factory.get_metrics_dataframe(eval_ontology)
 
 
 class LiDARDetectionDataset(DetectionDataset):