Added ability to customize segmentation

GANDLF/cli/generate_metrics.py

@@ -287,7 +287,7 @@ def generate_metrics_dict(
                 pred_array = pred_tensor.squeeze(0).numpy().astype(int)
 
             overall_stats_dict[current_subject_id] = generate_instance_segmentation(
-                prediction=pred_array, target=label_array
+                prediction=pred_array, target=label_array, parameters=parameters
             )
 
     elif problem_type == "synthesis":

GANDLF/metrics/segmentation_panoptica.py

@@ -1,31 +1,46 @@
 from pathlib import Path
+import tempfile
 
 import numpy as np
 
 from panoptica import Panoptica_Evaluator
 
 
 def generate_instance_segmentation(
-    prediction: np.ndarray, target: np.ndarray, panoptica_config_path: str = None
+    prediction: np.ndarray,
+    target: np.ndarray,
+    parameters: dict = None,
+    panoptica_config_path: str = None,
 ) -> dict:
     """
     Evaluate a single exam using Panoptica.
 
     Args:
         prediction (np.ndarray): The input prediction containing objects.
         label_path (str): The path to the reference label.
+        target (np.ndarray): The input target containing objects.
         panoptica_config_path (str): The path to the Panoptica configuration file.
 
     Returns:
         dict: The evaluation results.
     """
 
     cwd = Path(__file__).parent.absolute()
-    panoptica_config_path = (
-        cwd / "panoptica_config_brats.yaml"
-        if panoptica_config_path is None
-        else panoptica_config_path
-    )
+    # the parameters dict takes precedence over the panoptica_config_path
+    panoptica_config = parameters.get("panoptica_config", None)
+    if panoptica_config is None:
+        panoptica_config_path = (
+            cwd / "panoptica_config_brats.yaml"
+            if panoptica_config_path is None
+            else panoptica_config_path
+        )
+    else:
+        # write the panoptica config to a file
+        panoptica_config_path = tempfile.NamedTemporaryFile(
+            mode="w", delete=False, suffix=".yaml"
+        ).name
+        with open(panoptica_config_path, "w") as f:
+            f.write(panoptica_config)
     evaluator = Panoptica_Evaluator.load_from_config(panoptica_config_path)
 
     # call evaluate

docs/usage.md

@@ -282,7 +282,7 @@ SubjectID,Target,Prediction
 
 ### Special cases 
 
-1. **BraTS Segmentation Metrics**: To generate annotation to annotation metrics for BraTS segmentation tasks [[ref](https://www.synapse.org/brats)], ensure that the config has `problem_type: segmentation_brats`, and the CSV can be in the same format as segmentation:
+- **BraTS Segmentation Metrics**: To generate annotation to annotation metrics for BraTS segmentation tasks [[ref](https://www.synapse.org/brats)], ensure that the config has `problem_type: segmentation_brats`, and the CSV can be in the same format as segmentation:
 
   ```csv
   SubjectID,Target,Prediction
@@ -291,7 +291,9 @@ SubjectID,Target,Prediction
   ...
   ```
 
-2. **BraTS Synthesis Metrics**: To generate image to image metrics for synthesis tasks (including for the BraTS synthesis tasks [[1](https://www.synapse.org/#!Synapse:syn51156910/wiki/622356), [2](https://www.synapse.org/#!Synapse:syn51156910/wiki/622357)]), ensure that the config has `problem_type: synthesis`, and the CSV can be in the same format as segmentation (note that the `Mask` column is optional):
+  This can also be customized using the `panoptica_config` dictionary. See [this sample](https://github.com/mlcommons/GaNDLF/blob/master/samples/config_segmentation_metrics_brats_default.yaml) for an example. Additionally, a more "concise" variant of the config is present [here](https://github.com/mlcommons/GaNDLF/blob/master/samples/config_segmentation_metrics_brats_concise.yaml).
+
+- **BraTS Synthesis Metrics**: To generate image to image metrics for synthesis tasks (including for the BraTS synthesis tasks [[1](https://www.synapse.org/#!Synapse:syn51156910/wiki/622356), [2](https://www.synapse.org/#!Synapse:syn51156910/wiki/622357)]), ensure that the config has `problem_type: synthesis`, and the CSV can be in the same format as segmentation (note that the `Mask` column is optional):
 
   ```csv
   SubjectID,Target,Prediction,Mask

samples/config_segmentation_metrics_brats_concise.yaml

@@ -0,0 +1,150 @@
+# Choose the segmentation model here
+# options: unet, resunet, fcn
+version:
+  {
+    minimum: 0.0.14,
+    maximum: 0.1.5
+  }
+model:
+  {
+    dimension: 3, # the dimension of the model and dataset: defines dimensionality of computations
+    base_filters: 32, # Set base filters: number of filters present in the initial module of the U-Net convolution; for IncU-Net, keep this divisible by 4
+    architecture: resunet, # options: unet, resunet, fcn, uinc
+    final_layer: sigmoid, # can be either sigmoid, softmax or none (none == regression)
+    norm_type: instance, # can be either batch or instance
+    class_list: [0, 255], # Set the list of labels the model should train on and predict
+    amp: False, # Set if you want to use Automatic Mixed Precision for your operations or not - options: True, False
+    # n_channels: 3, # set the input channels - useful when reading RGB or images that have vectored pixel types
+  }
+metrics:
+  - dice
+  - precision
+  - iou
+  - f1
+  - recall: { average: macro }
+problem_type: segmentation_brats
+verbose: True
+inference_mechanism: { grid_aggregator_overlap: average, patch_overlap: 0 }
+modality: rad
+# Patch size during training - 2D patch for breast images since third dimension is not patched
+patch_size: [32, 32, 32]
+# Number of epochs
+num_epochs: 1
+patience: 1
+# Set the batch size
+batch_size: 1
+# Set the initial learning rate
+learning_rate: 0.001
+# Set the learning rate scheduler i.e. the way the initial learning rate must be updated while the training progresses
+# Options: steplr, exponentiallr, cosineannealinglr, reducelronplateau, cycliclr
+scheduler: triangle
+# Set which loss function you want to use - options : 'dc' - for dice only, 'dcce' - for sum of dice and CE and you can guess the next (only lower-case please)
+# options: dc (dice only), ce (), dcce (sume of dice and ce), mse (), ...
+loss_function: dc
+weighted_loss: True
+# Which optimizer do you want to use - adam/sgd
+optimizer: adam
+# the value of 'k' for cross-validation, this is the percentage of total training data to use as validation;
+# randomized split is performed using sklearn's KFold method
+# for single fold run, use '-' before the fold number
+nested_training: {
+    testing: -5, # this controls the holdout data splits for final model evaluation; use '1' if this is to be disabled
+    validation: -5, # this controls the validation data splits for model training
+  }
+# various data augmentation techniques
+# options: affine, elastic, downsample, motion, ghosting, bias, blur, gaussianNoise, swap
+# keep/edit as needed
+# all transforms: https://torchio.readthedocs.io/transforms/transforms.html?highlight=transforms
+data_augmentation: {}
+# 'spatial':{
+#   'probability': 0.5
+# },
+# 'kspace':{
+#   'probability': 0.5
+# },
+# 'bias':{
+#   'probability': 0.5
+# },
+# 'blur':{
+#   'probability': 0.5
+# },
+# 'noise':{
+#   'probability': 0.5
+# },
+# 'swap':{
+#   'probability': 0.5
+# }
+data_preprocessing: {
+    # 'threshold':{
+    #   'min': 10,
+    #   'max': 75
+    # },
+    # 'clip':{
+    #   'min': 10,
+    #   'max': 75
+    # },
+    "normalize",
+    # 'resample':{
+    #   'resolution': [1,2,3]
+    # },
+    #'resize': [128,128], # this is generally not recommended, as it changes image properties in unexpected ways
+  }
+# data postprocessing node
+data_postprocessing: {}
+# 'largest_component',
+# 'hole_filling'
+# parallel training on HPC - here goes the command to prepend to send to a high performance computing
+# cluster for parallel computing during multi-fold training
+# not used for single fold training
+# this gets passed before the training_loop, so ensure enough memory is provided along with other parameters
+# that your HPC would expect
+# ${outputDir} will be changed to the outputDir you pass in CLI + '/${fold_number}'
+#parallel_compute_command: <insert parallel command here>
+
+q_max_length: 1
+
+q_samples_per_volume: 1
+
+q_num_workers: 0
+
+panoptica: !Panoptica_Evaluator
+  decision_metric: null
+  decision_threshold: null
+  edge_case_handler: !EdgeCaseHandler
+    empty_list_std: !EdgeCaseResult NAN
+    listmetric_zeroTP_handling:
+      !Metric DSC: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO,
+        empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN,
+        normal: !EdgeCaseResult ZERO}
+      !Metric NSD: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult INF,
+        empty_reference_result: !EdgeCaseResult INF, no_instances_result: !EdgeCaseResult NAN,
+        normal: !EdgeCaseResult INF}
+  expected_input: !InputType SEMANTIC
+  global_metrics: [!Metric DSC]
+  instance_approximator: !ConnectedComponentsInstanceApproximator {cca_backend: null}
+  instance_matcher: !NaiveThresholdMatching {allow_many_to_one: false, matching_metric: !Metric IOU,
+    matching_threshold: 0.5}
+  instance_metrics: [!Metric DSC, !Metric NSD]
+  log_times: false
+  save_group_times: false
+  segmentation_class_groups: !SegmentationClassGroups
+    groups:
+      snfh: !LabelGroup
+        single_instance: false
+        value_labels: [2]
+      et: !LabelGroup
+        single_instance: false
+        value_labels: [3]
+      netc: !LabelGroup
+        single_instance: false
+        value_labels: [1]
+      rc: !LabelGroup
+        single_instance: false
+        value_labels: [4]
+      tc: !LabelMergeGroup
+        single_instance: false
+        value_labels: [1, 3, 4]
+      wt: !LabelMergeGroup
+        single_instance: false
+        value_labels: [1, 2, 3, 4]
+  verbose: false

samples/config_segmentation_metrics_brats_default.yaml

@@ -0,0 +1,162 @@
+# Choose the segmentation model here
+# options: unet, resunet, fcn
+version:
+  {
+    minimum: 0.0.14,
+    maximum: 0.1.5
+  }
+model:
+  {
+    dimension: 3, # the dimension of the model and dataset: defines dimensionality of computations
+    base_filters: 32, # Set base filters: number of filters present in the initial module of the U-Net convolution; for IncU-Net, keep this divisible by 4
+    architecture: resunet, # options: unet, resunet, fcn, uinc
+    final_layer: sigmoid, # can be either sigmoid, softmax or none (none == regression)
+    norm_type: instance, # can be either batch or instance
+    class_list: [0, 255], # Set the list of labels the model should train on and predict
+    amp: False, # Set if you want to use Automatic Mixed Precision for your operations or not - options: True, False
+    # n_channels: 3, # set the input channels - useful when reading RGB or images that have vectored pixel types
+  }
+metrics:
+  - dice
+  - precision
+  - iou
+  - f1
+  - recall: { average: macro }
+problem_type: segmentation_brats
+verbose: True
+inference_mechanism: { grid_aggregator_overlap: average, patch_overlap: 0 }
+modality: rad
+# Patch size during training - 2D patch for breast images since third dimension is not patched
+patch_size: [32, 32, 32]
+# Number of epochs
+num_epochs: 1
+patience: 1
+# Set the batch size
+batch_size: 1
+# Set the initial learning rate
+learning_rate: 0.001
+# Set the learning rate scheduler i.e. the way the initial learning rate must be updated while the training progresses
+# Options: steplr, exponentiallr, cosineannealinglr, reducelronplateau, cycliclr
+scheduler: triangle
+# Set which loss function you want to use - options : 'dc' - for dice only, 'dcce' - for sum of dice and CE and you can guess the next (only lower-case please)
+# options: dc (dice only), ce (), dcce (sume of dice and ce), mse (), ...
+loss_function: dc
+weighted_loss: True
+# Which optimizer do you want to use - adam/sgd
+optimizer: adam
+# the value of 'k' for cross-validation, this is the percentage of total training data to use as validation;
+# randomized split is performed using sklearn's KFold method
+# for single fold run, use '-' before the fold number
+nested_training: {
+    testing: -5, # this controls the holdout data splits for final model evaluation; use '1' if this is to be disabled
+    validation: -5, # this controls the validation data splits for model training
+  }
+# various data augmentation techniques
+# options: affine, elastic, downsample, motion, ghosting, bias, blur, gaussianNoise, swap
+# keep/edit as needed
+# all transforms: https://torchio.readthedocs.io/transforms/transforms.html?highlight=transforms
+data_augmentation: {}
+# 'spatial':{
+#   'probability': 0.5
+# },
+# 'kspace':{
+#   'probability': 0.5
+# },
+# 'bias':{
+#   'probability': 0.5
+# },
+# 'blur':{
+#   'probability': 0.5
+# },
+# 'noise':{
+#   'probability': 0.5
+# },
+# 'swap':{
+#   'probability': 0.5
+# }
+data_preprocessing: {
+    # 'threshold':{
+    #   'min': 10,
+    #   'max': 75
+    # },
+    # 'clip':{
+    #   'min': 10,
+    #   'max': 75
+    # },
+    "normalize",
+    # 'resample':{
+    #   'resolution': [1,2,3]
+    # },
+    #'resize': [128,128], # this is generally not recommended, as it changes image properties in unexpected ways
+  }
+# data postprocessing node
+data_postprocessing: {}
+# 'largest_component',
+# 'hole_filling'
+# parallel training on HPC - here goes the command to prepend to send to a high performance computing
+# cluster for parallel computing during multi-fold training
+# not used for single fold training
+# this gets passed before the training_loop, so ensure enough memory is provided along with other parameters
+# that your HPC would expect
+# ${outputDir} will be changed to the outputDir you pass in CLI + '/${fold_number}'
+#parallel_compute_command: <insert parallel command here>
+
+q_max_length: 1
+
+q_samples_per_volume: 1
+
+q_num_workers: 0
+
+panoptica: !Panoptica_Evaluator
+  decision_metric: null
+  decision_threshold: null
+  edge_case_handler: !EdgeCaseHandler
+    empty_list_std: !EdgeCaseResult NAN
+    listmetric_zeroTP_handling:
+      !Metric DSC: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO,
+        empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN,
+        normal: !EdgeCaseResult ZERO}
+      !Metric clDSC: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO,
+        empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN,
+        normal: !EdgeCaseResult ZERO}
+      !Metric IOU: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO,
+        empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN,
+        normal: !EdgeCaseResult ZERO}
+      !Metric NSD: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult INF,
+        empty_reference_result: !EdgeCaseResult INF, no_instances_result: !EdgeCaseResult NAN,
+        normal: !EdgeCaseResult INF}
+      !Metric HD95: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult INF,
+        empty_reference_result: !EdgeCaseResult INF, no_instances_result: !EdgeCaseResult NAN,
+        normal: !EdgeCaseResult INF}
+      !Metric RVD: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult NAN,
+        empty_reference_result: !EdgeCaseResult NAN, no_instances_result: !EdgeCaseResult NAN,
+        normal: !EdgeCaseResult NAN}
+      !Metric RVAE: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult NAN,
+        empty_reference_result: !EdgeCaseResult NAN, no_instances_result: !EdgeCaseResult NAN,
+        normal: !EdgeCaseResult NAN}
+  expected_input: !InputType SEMANTIC
+  global_metrics: [!Metric DSC]
+  instance_approximator: !ConnectedComponentsInstanceApproximator {cca_backend: null}
+  instance_matcher: !NaiveThresholdMatching {allow_many_to_one: false, matching_metric: !Metric IOU,
+    matching_threshold: 0.5}
+  instance_metrics: [!Metric DSC, !Metric IOU, !Metric RVD, !Metric NSD, !Metric HD95]
+  log_times: false
+  save_group_times: false
+  segmentation_class_groups: !SegmentationClassGroups
+    groups:
+      ed: !LabelGroup
+        single_instance: false
+        value_labels: [2]
+      et: !LabelGroup
+        single_instance: false
+        value_labels: [3]
+      net: !LabelGroup
+        single_instance: false
+        value_labels: [1]
+      tc: !LabelMergeGroup
+        single_instance: false
+        value_labels: [1, 3]
+      wt: !LabelMergeGroup
+        single_instance: false
+        value_labels: [1, 2, 3, 4]
+  verbose: false