diff --git a/ignite/metrics/fbeta.py b/ignite/metrics/fbeta.py index f8c165b77903..b4cd78c8293d 100644 --- a/ignite/metrics/fbeta.py +++ b/ignite/metrics/fbeta.py @@ -177,6 +177,16 @@ def thresholded_output_transform(output): elif recall._average: raise ValueError("Input recall metric should have average=False") + if precision._class_names is not None: + + def _fbeta_with_class_names(p: dict, r: dict) -> dict: + p_vals = torch.tensor(list(p.values())) + r_vals = torch.tensor(list(r.values())) + scores = (1.0 + beta**2) * p_vals * r_vals / (beta**2 * p_vals + r_vals + 1e-15) + return dict(zip(precision._class_names, scores.tolist())) + + return MetricsLambda(_fbeta_with_class_names, precision, recall) + fbeta = (1.0 + beta**2) * precision * recall / (beta**2 * precision + recall + 1e-15) if average: diff --git a/ignite/metrics/precision.py b/ignite/metrics/precision.py index be33b268b15e..4e6a00187ad6 100644 --- a/ignite/metrics/precision.py +++ b/ignite/metrics/precision.py @@ -23,7 +23,17 @@ def __init__( is_multilabel: bool = False, device: str | torch.device = torch.device("cpu"), skip_unrolling: bool = False, + class_names: list[str] | None = None, ): + if class_names is not None: + if not isinstance(class_names, (list, tuple)) or not all(isinstance(n, str) for n in class_names): + raise ValueError("class_names must be a list of strings") + if average is not False and average is not None: + raise ValueError( + f"class_names is only applicable when average=False or average=None, got average={average!r}." + ) + self._class_names = class_names + if not (average is None or isinstance(average, bool) or average in ["macro", "micro", "weighted", "samples"]): raise ValueError( "Argument average should be None or a boolean or one of values" @@ -125,7 +135,7 @@ def reset(self) -> None: super().reset() @sync_all_reduce("_numerator", "_denominator") - def compute(self) -> torch.Tensor | float: + def compute(self) -> torch.Tensor | float | dict: r""" Return value of the metric for `average` options `'weighted'` and `'macro'` is computed as follows. @@ -157,6 +167,8 @@ def compute(self) -> torch.Tensor | float: elif self._average == "macro": return cast(torch.Tensor, fraction).mean().item() else: + if self._class_names is not None: + return dict(zip(self._class_names, cast(torch.Tensor, fraction).tolist())) return fraction @@ -246,6 +258,10 @@ class Precision(_BasePrecisionRecall): skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` Alternatively, ``output_transform`` can be used to handle this. + class_names: list of class name strings used to label per-class output when ``average=False`` + or ``average=None``. If provided, ``compute()`` returns a ``dict`` mapping each class + name to its metric value instead of a tensor. Must match the number of classes inferred + from the data. Default: ``None``. Examples: @@ -428,5 +444,9 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) - + if self._class_names is not None and len(self._class_names) != self._numerator.shape[0]: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{self._numerator.shape[0]} classes." + ) self._updated = True diff --git a/ignite/metrics/recall.py b/ignite/metrics/recall.py index 34da36ced48f..95259a93c476 100644 --- a/ignite/metrics/recall.py +++ b/ignite/metrics/recall.py @@ -11,213 +11,220 @@ class Recall(_BasePrecisionRecall): r"""Calculates recall for binary, multiclass and multilabel data. - .. math:: \text{Recall} = \frac{ TP }{ TP + FN } + .. math:: \text{Recall} = \frac{ TP }{ TP + FN } - where :math:`\text{TP}` is true positives and :math:`\text{FN}` is false negatives. + where :math:`\text{TP}` is true positives and :math:`\text{FN}` is false negatives. - - ``update`` must receive output of the form ``(y_pred, y)``. - - `y_pred` must be in the following shape (batch_size, num_categories, ...) or (batch_size, ...). - - `y` must be in the following shape (batch_size, ...). + - ``update`` must receive output of the form ``(y_pred, y)``. + - `y_pred` must be in the following shape (batch_size, num_categories, ...) or (batch_size, ...). + - `y` must be in the following shape (batch_size, ...). - Args: - output_transform: a callable that is used to transform the - :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the - form expected by the metric. This can be useful if, for example, you have a multi-output model and - you want to compute the metric with respect to one of the outputs. - average: available options are + Args: + output_transform: a callable that is used to transform the + :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the + form expected by the metric. This can be useful if, for example, you have a multi-output model and + you want to compute the metric with respect to one of the outputs. + average: available options are + + False + default option. For multiclass and multilabel inputs, per class and per label + metric is returned respectively. - False - default option. For multiclass and multilabel inputs, per class and per label - metric is returned respectively. + None + like `False` option except that per class metric is returned for binary data as well. + For compatibility with Scikit-Learn api. - None - like `False` option except that per class metric is returned for binary data as well. - For compatibility with Scikit-Learn api. + 'micro' + Metric is computed counting stats of classes/labels altogether. - 'micro' - Metric is computed counting stats of classes/labels altogether. + .. math:: + \text{Micro Recall} = \frac{\sum_{k=1}^C TP_k}{\sum_{k=1}^C TP_k+FN_k} - .. math:: - \text{Micro Recall} = \frac{\sum_{k=1}^C TP_k}{\sum_{k=1}^C TP_k+FN_k} + where :math:`C` is the number of classes/labels (2 in binary case). :math:`k` in + :math:`TP_k` and :math:`FN_k` means that the measures are computed for class/label :math:`k` (in + a one-vs-rest sense in multiclass case). - where :math:`C` is the number of classes/labels (2 in binary case). :math:`k` in - :math:`TP_k` and :math:`FN_k` means that the measures are computed for class/label :math:`k` (in - a one-vs-rest sense in multiclass case). + For binary and multiclass inputs, this is equivalent with accuracy, + so use :class:`~ignite.metrics.accuracy.Accuracy`. - For binary and multiclass inputs, this is equivalent with accuracy, - so use :class:`~ignite.metrics.accuracy.Accuracy`. + 'samples' + for multilabel input, at first, recall is computed on a + per sample basis and then average across samples is returned. + + .. math:: + \text{Sample-averaged Recall} = \frac{\sum_{n=1}^N \frac{TP_n}{TP_n+FN_n}}{N} - 'samples' - for multilabel input, at first, recall is computed on a - per sample basis and then average across samples is returned. + where :math:`N` is the number of samples. :math:`n` in :math:`TP_n` and :math:`FN_n` + means that the measures are computed for sample :math:`n`, across labels. - .. math:: - \text{Sample-averaged Recall} = \frac{\sum_{n=1}^N \frac{TP_n}{TP_n+FN_n}}{N} + Incompatible with binary and multiclass inputs. - where :math:`N` is the number of samples. :math:`n` in :math:`TP_n` and :math:`FN_n` - means that the measures are computed for sample :math:`n`, across labels. + 'weighted' + like macro recall but considers class/label imbalance. For binary and multiclass + input, it computes metric for each class then returns average of them weighted by + support of classes (number of actual samples in each class). For multilabel input, + it computes recall for each label then returns average of them weighted by support + of labels (number of actual positive samples in each label). - Incompatible with binary and multiclass inputs. + .. math:: + Recall_k = \frac{TP_k}{TP_k+FN_k} - 'weighted' - like macro recall but considers class/label imbalance. For binary and multiclass - input, it computes metric for each class then returns average of them weighted by - support of classes (number of actual samples in each class). For multilabel input, - it computes recall for each label then returns average of them weighted by support - of labels (number of actual positive samples in each label). + .. math:: + \text{Weighted Recall} = \frac{\sum_{k=1}^C P_k * Recall_k}{N} - .. math:: - Recall_k = \frac{TP_k}{TP_k+FN_k} + where :math:`C` is the number of classes (2 in binary case). :math:`P_k` is the number + of samples belonged to class :math:`k` in binary and multiclass case, and the number of + positive samples belonged to label :math:`k` in multilabel case. - .. math:: - \text{Weighted Recall} = \frac{\sum_{k=1}^C P_k * Recall_k}{N} - - where :math:`C` is the number of classes (2 in binary case). :math:`P_k` is the number - of samples belonged to class :math:`k` in binary and multiclass case, and the number of - positive samples belonged to label :math:`k` in multilabel case. - - Note that for binary and multiclass data, weighted recall is equivalent - with accuracy, so use :class:`~ignite.metrics.accuracy.Accuracy`. - - macro - computes macro recall which is unweighted average of metric computed across - classes or labels. - - .. math:: - \text{Macro Recall} = \frac{\sum_{k=1}^C Recall_k}{C} - - where :math:`C` is the number of classes (2 in binary case). - - True - like macro option. For backward compatibility. - is_multilabel: flag to use in multilabel case. By default, value is False. - device: specifies which device updates are accumulated on. Setting the metric's - device to be the same as your ``update`` arguments ensures the ``update`` method is non-blocking. By - default, CPU. - skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be - true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` - Alternatively, ``output_transform`` can be used to handle this. - - Examples: - - For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`. - - .. include:: defaults.rst - :start-after: :orphan: - - Binary case. In binary and multilabel cases, the elements of - `y` and `y_pred` should have 0 or 1 values. - - .. testcode:: 1 - - metric = Recall() - two_class_metric = Recall(average=None) # Returns recall for both classes - metric.attach(default_evaluator, "recall") - two_class_metric.attach(default_evaluator, "both classes recall") - y_true = torch.tensor([1, 0, 1, 1, 0, 1]) - y_pred = torch.tensor([1, 0, 1, 0, 1, 1]) - state = default_evaluator.run([[y_pred, y_true]]) - print(f"Recall: {state.metrics['recall']}") - print(f"Recall for class 0 and class 1: {state.metrics['both classes recall']}") - - .. testoutput:: 1 - - Recall: 0.75 - Recall for class 0 and class 1: tensor([0.5000, 0.7500], dtype=torch.float64) - - Multiclass case - - .. testcode:: 2 - - metric = Recall() - macro_metric = Recall(average=True) - - metric.attach(default_evaluator, "recall") - macro_metric.attach(default_evaluator, "macro recall") - - y_true = torch.tensor([2, 0, 2, 1, 0]) - y_pred = torch.tensor([ - [0.0266, 0.1719, 0.3055], - [0.6886, 0.3978, 0.8176], - [0.9230, 0.0197, 0.8395], - [0.1785, 0.2670, 0.6084], - [0.8448, 0.7177, 0.7288] - ]) - state = default_evaluator.run([[y_pred, y_true]]) - print(f"Recall: {state.metrics['recall']}") - print(f"Macro Recall: {state.metrics['macro recall']}") - - .. testoutput:: 2 - - Recall: tensor([0.5000, 0.0000, 0.5000], dtype=torch.float64) - Macro Recall: 0.3333333333333333 - - Multilabel case, the shapes must be (batch_size, num_categories, ...) - - .. testcode:: 3 - - metric = Recall(is_multilabel=True) - micro_metric = Recall(is_multilabel=True, average='micro') - macro_metric = Recall(is_multilabel=True, average=True) - samples_metric = Recall(is_multilabel=True, average='samples') - - metric.attach(default_evaluator, "recall") - micro_metric.attach(default_evaluator, "micro recall") - macro_metric.attach(default_evaluator, "macro recall") - samples_metric.attach(default_evaluator, "samples recall") - - y_true = torch.tensor([ - [0, 0, 1], - [0, 0, 0], - [0, 0, 0], - [1, 0, 0], - [0, 1, 1], - ]) - y_pred = torch.tensor([ - [1, 1, 0], - [1, 0, 1], - [1, 0, 0], - [1, 0, 1], - [1, 1, 0], - ]) - state = default_evaluator.run([[y_pred, y_true]]) - print(f"Recall: {state.metrics['recall']}") - print(f"Micro Recall: {state.metrics['micro recall']}") - print(f"Macro Recall: {state.metrics['macro recall']}") - print(f"Samples Recall: {state.metrics['samples recall']}") - - .. testoutput:: 3 - - Recall: tensor([1., 1., 0.], dtype=torch.float64) - Micro Recall: 0.5 - Macro Recall: 0.6666666666666666 - Samples Recall: 0.3 - - Thresholding of predictions can be done as below: - - .. testcode:: 4 - - def thresholded_output_transform(output): - y_pred, y = output - y_pred = torch.round(y_pred) - return y_pred, y - - metric = Recall(output_transform=thresholded_output_transform) - metric.attach(default_evaluator, "recall") - y_true = torch.tensor([1, 0, 1, 1, 0, 1]) - y_pred = torch.tensor([0.6, 0.2, 0.9, 0.4, 0.7, 0.65]) - state = default_evaluator.run([[y_pred, y_true]]) - print(state.metrics['recall']) - - .. testoutput:: 4 - - 0.75 - - .. versionchanged:: 0.4.10 - Some new options were added to `average` parameter. - - .. versionchanged:: 0.5.1 - ``skip_unrolling`` argument is added. + Note that for binary and multiclass data, weighted recall is equivalent + with accuracy, so use :class:`~ignite.metrics.accuracy.Accuracy`. + + macro + computes macro recall which is unweighted average of metric computed across + classes or labels. + + .. math:: + \text{Macro Recall} = \frac{\sum_{k=1}^C Recall_k}{C} + + where :math:`C` is the number of classes (2 in binary case). + + True + like macro option. For backward compatibility. + is_multilabel: flag to use in multilabel case. By default, value is False. + device: specifies which device updates are accumulated on. Setting the metric's + device to be the same as your ``update`` arguments ensures the ``update`` method is non-blocking. By + default, CPU. + skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be + true for multi-output model, for example, if ``y_pred`` contains multi-output as ``(y_pred_a, y_pred_b)`` + Alternatively, ``output_transform`` can be used to handle this. + class_names: list of class name strings used to label per-class output when ``average=False`` + <<<<<<< HEAD + or ``average=None``. If provided, ``compute()`` returns a ``dict`` mapping each class + ======= + or ``average=None``. If provided, :meth:`compute` returns a ``dict`` mapping each class + >>>>>>> 7ea2eb50590597780ffae1ab3dc4c7ede3c80be7 + name to its metric value instead of a tensor. Must match the number of classes inferred + from the data. Default: ``None``. + Examples: + + For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`. + + .. include:: defaults.rst + :start-after: :orphan: + + Binary case. In binary and multilabel cases, the elements of + `y` and `y_pred` should have 0 or 1 values. + + .. testcode:: 1 + + metric = Recall() + two_class_metric = Recall(average=None) # Returns recall for both classes + metric.attach(default_evaluator, "recall") + two_class_metric.attach(default_evaluator, "both classes recall") + y_true = torch.tensor([1, 0, 1, 1, 0, 1]) + y_pred = torch.tensor([1, 0, 1, 0, 1, 1]) + state = default_evaluator.run([[y_pred, y_true]]) + print(f"Recall: {state.metrics['recall']}") + print(f"Recall for class 0 and class 1: {state.metrics['both classes recall']}") + + .. testoutput:: 1 + + Recall: 0.75 + Recall for class 0 and class 1: tensor([0.5000, 0.7500], dtype=torch.float64) + + Multiclass case + + .. testcode:: 2 + + metric = Recall() + macro_metric = Recall(average=True) + + metric.attach(default_evaluator, "recall") + macro_metric.attach(default_evaluator, "macro recall") + + y_true = torch.tensor([2, 0, 2, 1, 0]) + y_pred = torch.tensor([ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288] + ]) + state = default_evaluator.run([[y_pred, y_true]]) + print(f"Recall: {state.metrics['recall']}") + print(f"Macro Recall: {state.metrics['macro recall']}") + + .. testoutput:: 2 + + Recall: tensor([0.5000, 0.0000, 0.5000], dtype=torch.float64) + Macro Recall: 0.3333333333333333 + + Multilabel case, the shapes must be (batch_size, num_categories, ...) + + .. testcode:: 3 + + metric = Recall(is_multilabel=True) + micro_metric = Recall(is_multilabel=True, average='micro') + macro_metric = Recall(is_multilabel=True, average=True) + samples_metric = Recall(is_multilabel=True, average='samples') + + metric.attach(default_evaluator, "recall") + micro_metric.attach(default_evaluator, "micro recall") + macro_metric.attach(default_evaluator, "macro recall") + samples_metric.attach(default_evaluator, "samples recall") + + y_true = torch.tensor([ + [0, 0, 1], + [0, 0, 0], + [0, 0, 0], + [1, 0, 0], + [0, 1, 1], + ]) + y_pred = torch.tensor([ + [1, 1, 0], + [1, 0, 1], + [1, 0, 0], + [1, 0, 1], + [1, 1, 0], + ]) + state = default_evaluator.run([[y_pred, y_true]]) + print(f"Recall: {state.metrics['recall']}") + print(f"Micro Recall: {state.metrics['micro recall']}") + print(f"Macro Recall: {state.metrics['macro recall']}") + print(f"Samples Recall: {state.metrics['samples recall']}") + + .. testoutput:: 3 + + Recall: tensor([1., 1., 0.], dtype=torch.float64) + Micro Recall: 0.5 + Macro Recall: 0.6666666666666666 + Samples Recall: 0.3 + + Thresholding of predictions can be done as below: + + .. testcode:: 4 + + def thresholded_output_transform(output): + y_pred, y = output + y_pred = torch.round(y_pred) + return y_pred, y + + metric = Recall(output_transform=thresholded_output_transform) + metric.attach(default_evaluator, "recall") + y_true = torch.tensor([1, 0, 1, 1, 0, 1]) + y_pred = torch.tensor([0.6, 0.2, 0.9, 0.4, 0.7, 0.65]) + state = default_evaluator.run([[y_pred, y_true]]) + print(state.metrics['recall']) + + .. testoutput:: 4 + + 0.75 + + .. versionchanged:: 0.4.10 + Some new options were added to `average` parameter. + + .. versionchanged:: 0.5.1 + ``skip_unrolling`` argument is added. """ @reinit__is_reduced @@ -240,5 +247,10 @@ def update(self, output: Sequence[torch.Tensor]) -> None: if self._average == "weighted": self._weight += y.sum(dim=0) + if self._class_names is not None and len(self._class_names) != self._numerator.shape[0]: + raise ValueError( + f"class_names has {len(self._class_names)} entries but the metric computed " + f"{self._numerator.shape[0]} classes." + ) self._updated = True diff --git a/tests/ignite/metrics/test_precision.py b/tests/ignite/metrics/test_precision.py index 7bd80540f5de..d2121858c6a3 100644 --- a/tests/ignite/metrics/test_precision.py +++ b/tests/ignite/metrics/test_precision.py @@ -325,6 +325,86 @@ def test_incorrect_y_classes(average): assert pr._updated is False +@pytest.mark.parametrize( + "invalid_class_names", + [ + [0, 1, 2], # list of ints + "cat", # string instead of list + [0.1, 0.2], # list of floats + ["cat", 1], # mixed + ], +) +def test_class_names_invalid_type(invalid_class_names): + with pytest.raises(ValueError, match="class_names must be a list of strings"): + Precision(average=False, class_names=invalid_class_names) + + +@pytest.mark.parametrize("average", ["macro", "micro", "weighted", "samples", True]) +def test_class_names_incompatible_average(average): + with pytest.raises(ValueError, match="class_names is only applicable when average=False or average=None"): + Precision(average=average, class_names=["cat", "dog", "horse"]) + + +@pytest.mark.parametrize("average", [False, None]) +def test_class_names_multiclass(average): + class_names = ["cat", "dog", "horse"] + pr = Precision(average=average, class_names=class_names) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288], + ] + ) + y = torch.tensor([2, 0, 2, 1, 0]) + + pr.update((y_pred, y)) + result = pr.compute() + + assert isinstance(result, dict) + assert list(result.keys()) == class_names + assert result == pytest.approx({"cat": 0.5, "dog": 0.0, "horse": 0.3333333333333333}) + + +def test_class_names_length_mismatch(): + pr = Precision(average=False, class_names=["cat", "dog"]) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + ] + ) + y = torch.tensor([2, 0, 1]) + + with pytest.raises(ValueError, match="class_names has 2 entries but the metric computed 3 classes"): + pr.update((y_pred, y)) + + +def test_class_names_none_returns_tensor(): + pr = Precision(average=False) + + y_pred = torch.tensor( + [ + [0.0266, 0.1719, 0.3055], + [0.6886, 0.3978, 0.8176], + [0.9230, 0.0197, 0.8395], + [0.1785, 0.2670, 0.6084], + [0.8448, 0.7177, 0.7288], + ] + ) + y = torch.tensor([2, 0, 2, 1, 0]) + + pr.update((y_pred, y)) + result = pr.compute() + + assert isinstance(result, torch.Tensor) + + @pytest.mark.usefixtures("distributed") class TestDistributed: @pytest.mark.parametrize("average", [False, "macro", "weighted", "micro"])