diff --git a/GANDLF/cli/generate_metrics.py b/GANDLF/cli/generate_metrics.py index 4cfdcfac9..4dfb2fa8d 100644 --- a/GANDLF/cli/generate_metrics.py +++ b/GANDLF/cli/generate_metrics.py @@ -287,7 +287,7 @@ def generate_metrics_dict( pred_array = pred_tensor.squeeze(0).numpy().astype(int) overall_stats_dict[current_subject_id] = generate_instance_segmentation( - prediction=pred_array, target=label_array + prediction=pred_array, target=label_array, parameters=parameters ) elif problem_type == "synthesis": diff --git a/GANDLF/metrics/segmentation_panoptica.py b/GANDLF/metrics/segmentation_panoptica.py index 7a9645a1a..9c8eb3319 100644 --- a/GANDLF/metrics/segmentation_panoptica.py +++ b/GANDLF/metrics/segmentation_panoptica.py @@ -1,4 +1,5 @@ from pathlib import Path +import tempfile import numpy as np @@ -6,7 +7,10 @@ def generate_instance_segmentation( - prediction: np.ndarray, target: np.ndarray, panoptica_config_path: str = None + prediction: np.ndarray, + target: np.ndarray, + parameters: dict = None, + panoptica_config_path: str = None, ) -> dict: """ Evaluate a single exam using Panoptica. @@ -14,6 +18,7 @@ def generate_instance_segmentation( Args: prediction (np.ndarray): The input prediction containing objects. label_path (str): The path to the reference label. + target (np.ndarray): The input target containing objects. panoptica_config_path (str): The path to the Panoptica configuration file. Returns: @@ -21,11 +26,21 @@ def generate_instance_segmentation( """ cwd = Path(__file__).parent.absolute() - panoptica_config_path = ( - cwd / "panoptica_config_brats.yaml" - if panoptica_config_path is None - else panoptica_config_path - ) + # the parameters dict takes precedence over the panoptica_config_path + panoptica_config = parameters.get("panoptica_config", None) + if panoptica_config is None: + panoptica_config_path = ( + cwd / "panoptica_config_brats.yaml" + if panoptica_config_path is None + else panoptica_config_path + ) + else: + # write the panoptica config to a file + panoptica_config_path = tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".yaml" + ).name + with open(panoptica_config_path, "w") as f: + f.write(panoptica_config) evaluator = Panoptica_Evaluator.load_from_config(panoptica_config_path) # call evaluate diff --git a/docs/usage.md b/docs/usage.md index d1273fb2b..ba4032035 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -282,7 +282,7 @@ SubjectID,Target,Prediction ### Special cases -1. **BraTS Segmentation Metrics**: To generate annotation to annotation metrics for BraTS segmentation tasks [[ref](https://www.synapse.org/brats)], ensure that the config has `problem_type: segmentation_brats`, and the CSV can be in the same format as segmentation: +- **BraTS Segmentation Metrics**: To generate annotation to annotation metrics for BraTS segmentation tasks [[ref](https://www.synapse.org/brats)], ensure that the config has `problem_type: segmentation_brats`, and the CSV can be in the same format as segmentation: ```csv SubjectID,Target,Prediction @@ -291,7 +291,9 @@ SubjectID,Target,Prediction ... ``` -2. **BraTS Synthesis Metrics**: To generate image to image metrics for synthesis tasks (including for the BraTS synthesis tasks [[1](https://www.synapse.org/#!Synapse:syn51156910/wiki/622356), [2](https://www.synapse.org/#!Synapse:syn51156910/wiki/622357)]), ensure that the config has `problem_type: synthesis`, and the CSV can be in the same format as segmentation (note that the `Mask` column is optional): + This can also be customized using the `panoptica_config` dictionary. See [this sample](https://github.com/mlcommons/GaNDLF/blob/master/samples/config_segmentation_metrics_brats_default.yaml) for an example. Additionally, a more "concise" variant of the config is present [here](https://github.com/mlcommons/GaNDLF/blob/master/samples/config_segmentation_metrics_brats_concise.yaml). + +- **BraTS Synthesis Metrics**: To generate image to image metrics for synthesis tasks (including for the BraTS synthesis tasks [[1](https://www.synapse.org/#!Synapse:syn51156910/wiki/622356), [2](https://www.synapse.org/#!Synapse:syn51156910/wiki/622357)]), ensure that the config has `problem_type: synthesis`, and the CSV can be in the same format as segmentation (note that the `Mask` column is optional): ```csv SubjectID,Target,Prediction,Mask diff --git a/samples/config_segmentation_metrics_brats_concise.yaml b/samples/config_segmentation_metrics_brats_concise.yaml new file mode 100644 index 000000000..2de92b37d --- /dev/null +++ b/samples/config_segmentation_metrics_brats_concise.yaml @@ -0,0 +1,150 @@ +# Choose the segmentation model here +# options: unet, resunet, fcn +version: + { + minimum: 0.0.14, + maximum: 0.1.5 + } +model: + { + dimension: 3, # the dimension of the model and dataset: defines dimensionality of computations + base_filters: 32, # Set base filters: number of filters present in the initial module of the U-Net convolution; for IncU-Net, keep this divisible by 4 + architecture: resunet, # options: unet, resunet, fcn, uinc + final_layer: sigmoid, # can be either sigmoid, softmax or none (none == regression) + norm_type: instance, # can be either batch or instance + class_list: [0, 255], # Set the list of labels the model should train on and predict + amp: False, # Set if you want to use Automatic Mixed Precision for your operations or not - options: True, False + # n_channels: 3, # set the input channels - useful when reading RGB or images that have vectored pixel types + } +metrics: + - dice + - precision + - iou + - f1 + - recall: { average: macro } +problem_type: segmentation_brats +verbose: True +inference_mechanism: { grid_aggregator_overlap: average, patch_overlap: 0 } +modality: rad +# Patch size during training - 2D patch for breast images since third dimension is not patched +patch_size: [32, 32, 32] +# Number of epochs +num_epochs: 1 +patience: 1 +# Set the batch size +batch_size: 1 +# Set the initial learning rate +learning_rate: 0.001 +# Set the learning rate scheduler i.e. the way the initial learning rate must be updated while the training progresses +# Options: steplr, exponentiallr, cosineannealinglr, reducelronplateau, cycliclr +scheduler: triangle +# Set which loss function you want to use - options : 'dc' - for dice only, 'dcce' - for sum of dice and CE and you can guess the next (only lower-case please) +# options: dc (dice only), ce (), dcce (sume of dice and ce), mse (), ... +loss_function: dc +weighted_loss: True +# Which optimizer do you want to use - adam/sgd +optimizer: adam +# the value of 'k' for cross-validation, this is the percentage of total training data to use as validation; +# randomized split is performed using sklearn's KFold method +# for single fold run, use '-' before the fold number +nested_training: { + testing: -5, # this controls the holdout data splits for final model evaluation; use '1' if this is to be disabled + validation: -5, # this controls the validation data splits for model training + } +# various data augmentation techniques +# options: affine, elastic, downsample, motion, ghosting, bias, blur, gaussianNoise, swap +# keep/edit as needed +# all transforms: https://torchio.readthedocs.io/transforms/transforms.html?highlight=transforms +data_augmentation: {} +# 'spatial':{ +# 'probability': 0.5 +# }, +# 'kspace':{ +# 'probability': 0.5 +# }, +# 'bias':{ +# 'probability': 0.5 +# }, +# 'blur':{ +# 'probability': 0.5 +# }, +# 'noise':{ +# 'probability': 0.5 +# }, +# 'swap':{ +# 'probability': 0.5 +# } +data_preprocessing: { + # 'threshold':{ + # 'min': 10, + # 'max': 75 + # }, + # 'clip':{ + # 'min': 10, + # 'max': 75 + # }, + "normalize", + # 'resample':{ + # 'resolution': [1,2,3] + # }, + #'resize': [128,128], # this is generally not recommended, as it changes image properties in unexpected ways + } +# data postprocessing node +data_postprocessing: {} +# 'largest_component', +# 'hole_filling' +# parallel training on HPC - here goes the command to prepend to send to a high performance computing +# cluster for parallel computing during multi-fold training +# not used for single fold training +# this gets passed before the training_loop, so ensure enough memory is provided along with other parameters +# that your HPC would expect +# ${outputDir} will be changed to the outputDir you pass in CLI + '/${fold_number}' +#parallel_compute_command: + +q_max_length: 1 + +q_samples_per_volume: 1 + +q_num_workers: 0 + +panoptica: !Panoptica_Evaluator + decision_metric: null + decision_threshold: null + edge_case_handler: !EdgeCaseHandler + empty_list_std: !EdgeCaseResult NAN + listmetric_zeroTP_handling: + !Metric DSC: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO, + empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN, + normal: !EdgeCaseResult ZERO} + !Metric NSD: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult INF, + empty_reference_result: !EdgeCaseResult INF, no_instances_result: !EdgeCaseResult NAN, + normal: !EdgeCaseResult INF} + expected_input: !InputType SEMANTIC + global_metrics: [!Metric DSC] + instance_approximator: !ConnectedComponentsInstanceApproximator {cca_backend: null} + instance_matcher: !NaiveThresholdMatching {allow_many_to_one: false, matching_metric: !Metric IOU, + matching_threshold: 0.5} + instance_metrics: [!Metric DSC, !Metric NSD] + log_times: false + save_group_times: false + segmentation_class_groups: !SegmentationClassGroups + groups: + snfh: !LabelGroup + single_instance: false + value_labels: [2] + et: !LabelGroup + single_instance: false + value_labels: [3] + netc: !LabelGroup + single_instance: false + value_labels: [1] + rc: !LabelGroup + single_instance: false + value_labels: [4] + tc: !LabelMergeGroup + single_instance: false + value_labels: [1, 3, 4] + wt: !LabelMergeGroup + single_instance: false + value_labels: [1, 2, 3, 4] + verbose: false diff --git a/samples/config_segmentation_metrics_brats_default.yaml b/samples/config_segmentation_metrics_brats_default.yaml new file mode 100644 index 000000000..9a7d3fdd9 --- /dev/null +++ b/samples/config_segmentation_metrics_brats_default.yaml @@ -0,0 +1,162 @@ +# Choose the segmentation model here +# options: unet, resunet, fcn +version: + { + minimum: 0.0.14, + maximum: 0.1.5 + } +model: + { + dimension: 3, # the dimension of the model and dataset: defines dimensionality of computations + base_filters: 32, # Set base filters: number of filters present in the initial module of the U-Net convolution; for IncU-Net, keep this divisible by 4 + architecture: resunet, # options: unet, resunet, fcn, uinc + final_layer: sigmoid, # can be either sigmoid, softmax or none (none == regression) + norm_type: instance, # can be either batch or instance + class_list: [0, 255], # Set the list of labels the model should train on and predict + amp: False, # Set if you want to use Automatic Mixed Precision for your operations or not - options: True, False + # n_channels: 3, # set the input channels - useful when reading RGB or images that have vectored pixel types + } +metrics: + - dice + - precision + - iou + - f1 + - recall: { average: macro } +problem_type: segmentation_brats +verbose: True +inference_mechanism: { grid_aggregator_overlap: average, patch_overlap: 0 } +modality: rad +# Patch size during training - 2D patch for breast images since third dimension is not patched +patch_size: [32, 32, 32] +# Number of epochs +num_epochs: 1 +patience: 1 +# Set the batch size +batch_size: 1 +# Set the initial learning rate +learning_rate: 0.001 +# Set the learning rate scheduler i.e. the way the initial learning rate must be updated while the training progresses +# Options: steplr, exponentiallr, cosineannealinglr, reducelronplateau, cycliclr +scheduler: triangle +# Set which loss function you want to use - options : 'dc' - for dice only, 'dcce' - for sum of dice and CE and you can guess the next (only lower-case please) +# options: dc (dice only), ce (), dcce (sume of dice and ce), mse (), ... +loss_function: dc +weighted_loss: True +# Which optimizer do you want to use - adam/sgd +optimizer: adam +# the value of 'k' for cross-validation, this is the percentage of total training data to use as validation; +# randomized split is performed using sklearn's KFold method +# for single fold run, use '-' before the fold number +nested_training: { + testing: -5, # this controls the holdout data splits for final model evaluation; use '1' if this is to be disabled + validation: -5, # this controls the validation data splits for model training + } +# various data augmentation techniques +# options: affine, elastic, downsample, motion, ghosting, bias, blur, gaussianNoise, swap +# keep/edit as needed +# all transforms: https://torchio.readthedocs.io/transforms/transforms.html?highlight=transforms +data_augmentation: {} +# 'spatial':{ +# 'probability': 0.5 +# }, +# 'kspace':{ +# 'probability': 0.5 +# }, +# 'bias':{ +# 'probability': 0.5 +# }, +# 'blur':{ +# 'probability': 0.5 +# }, +# 'noise':{ +# 'probability': 0.5 +# }, +# 'swap':{ +# 'probability': 0.5 +# } +data_preprocessing: { + # 'threshold':{ + # 'min': 10, + # 'max': 75 + # }, + # 'clip':{ + # 'min': 10, + # 'max': 75 + # }, + "normalize", + # 'resample':{ + # 'resolution': [1,2,3] + # }, + #'resize': [128,128], # this is generally not recommended, as it changes image properties in unexpected ways + } +# data postprocessing node +data_postprocessing: {} +# 'largest_component', +# 'hole_filling' +# parallel training on HPC - here goes the command to prepend to send to a high performance computing +# cluster for parallel computing during multi-fold training +# not used for single fold training +# this gets passed before the training_loop, so ensure enough memory is provided along with other parameters +# that your HPC would expect +# ${outputDir} will be changed to the outputDir you pass in CLI + '/${fold_number}' +#parallel_compute_command: + +q_max_length: 1 + +q_samples_per_volume: 1 + +q_num_workers: 0 + +panoptica: !Panoptica_Evaluator + decision_metric: null + decision_threshold: null + edge_case_handler: !EdgeCaseHandler + empty_list_std: !EdgeCaseResult NAN + listmetric_zeroTP_handling: + !Metric DSC: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO, + empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN, + normal: !EdgeCaseResult ZERO} + !Metric clDSC: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO, + empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN, + normal: !EdgeCaseResult ZERO} + !Metric IOU: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO, + empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN, + normal: !EdgeCaseResult ZERO} + !Metric NSD: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult INF, + empty_reference_result: !EdgeCaseResult INF, no_instances_result: !EdgeCaseResult NAN, + normal: !EdgeCaseResult INF} + !Metric HD95: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult INF, + empty_reference_result: !EdgeCaseResult INF, no_instances_result: !EdgeCaseResult NAN, + normal: !EdgeCaseResult INF} + !Metric RVD: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult NAN, + empty_reference_result: !EdgeCaseResult NAN, no_instances_result: !EdgeCaseResult NAN, + normal: !EdgeCaseResult NAN} + !Metric RVAE: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult NAN, + empty_reference_result: !EdgeCaseResult NAN, no_instances_result: !EdgeCaseResult NAN, + normal: !EdgeCaseResult NAN} + expected_input: !InputType SEMANTIC + global_metrics: [!Metric DSC] + instance_approximator: !ConnectedComponentsInstanceApproximator {cca_backend: null} + instance_matcher: !NaiveThresholdMatching {allow_many_to_one: false, matching_metric: !Metric IOU, + matching_threshold: 0.5} + instance_metrics: [!Metric DSC, !Metric IOU, !Metric RVD, !Metric NSD, !Metric HD95] + log_times: false + save_group_times: false + segmentation_class_groups: !SegmentationClassGroups + groups: + ed: !LabelGroup + single_instance: false + value_labels: [2] + et: !LabelGroup + single_instance: false + value_labels: [3] + net: !LabelGroup + single_instance: false + value_labels: [1] + tc: !LabelMergeGroup + single_instance: false + value_labels: [1, 3] + wt: !LabelMergeGroup + single_instance: false + value_labels: [1, 2, 3, 4] + verbose: false