diff --git a/docs/source/metrics.rst b/docs/source/metrics.rst index bac06b3067..af571020c7 100644 --- a/docs/source/metrics.rst +++ b/docs/source/metrics.rst @@ -39,6 +39,13 @@ Metrics .. autoclass:: DiceMetric :members: + `IoU Score` +---------- +.. autofunction:: compute_iou + +.. autoclass:: IoUMetric + :members: + `Generalized Dice Score` ------------------------ .. autofunction:: compute_generalized_dice diff --git a/monai/metrics/__init__.py b/monai/metrics/__init__.py index 0cefe056e8..ee22a2f16f 100644 --- a/monai/metrics/__init__.py +++ b/monai/metrics/__init__.py @@ -15,6 +15,7 @@ from .generalized_dice import GeneralizedDiceScore, compute_generalized_dice from .hausdorff_distance import HausdorffDistanceMetric, compute_hausdorff_distance, compute_percent_hausdorff_distance from .meandice import DiceMetric, compute_meandice +from .iou_metric import IoUMetric, compute_iou from .metric import Cumulative, CumulativeIterationMetric, IterationMetric, Metric from .regression import MAEMetric, MSEMetric, PSNRMetric, RMSEMetric, SSIMMetric from .rocauc import ROCAUCMetric, compute_roc_auc diff --git a/monai/metrics/iou_metric.py b/monai/metrics/iou_metric.py new file mode 100644 index 0000000000..a322de288b --- /dev/null +++ b/monai/metrics/iou_metric.py @@ -0,0 +1,140 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Union + +import torch + +from monai.metrics.utils import do_metric_reduction, ignore_background, is_binary_tensor +from monai.utils import MetricReduction + +from .metric import CumulativeIterationMetric + + +class IoUMetric(CumulativeIterationMetric): + """ + Compute average IOU score between two tensors. It can support both multi-classes and multi-labels tasks. + Input `y_pred` is compared with ground truth `y`. + `y_preds` is expected to have binarized predictions and `y` should be in one-hot format. You can use suitable transforms + in ``monai.transforms.post`` first to achieve binarized values. + The `include_background` parameter can be set to ``False`` to exclude + the first category (channel index 0) which is by convention assumed to be background. If the non-background + segmentations are small compared to the total image size they can get overwhelmed by the signal from the + background. + `y_preds` and `y` can be a list of channel-first Tensor (CHW[D]) or a batch-first Tensor (BCHW[D]). + Args: + include_background: whether to skip IOU computation on the first channel of + the predicted output. Defaults to ``True``. + reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values, + available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``, + ``"mean_channel"``, ``"sum_channel"``}, default to ``"mean"``. if "none", will not do reduction. + get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns (metric, not_nans). + Here `not_nans` count the number of not nans for the metric, thus its shape equals to the shape of the metric. + ignore_empty: whether to ignore empty ground truth cases during calculation. + If `True`, NaN value will be set for empty ground truth cases. + If `False`, 1 will be set if the predictions of empty ground truth cases are also empty. + """ + + def __init__( + self, + include_background: bool = True, + reduction: Union[MetricReduction, str] = MetricReduction.MEAN, + get_not_nans: bool = False, + ignore_empty: bool = True, + ) -> None: + super().__init__() + self.include_background = include_background + self.reduction = reduction + self.get_not_nans = get_not_nans + self.ignore_empty = ignore_empty + + def _compute_tensor(self, y_pred: torch.Tensor, y: torch.Tensor): # type: ignore + """ + Args: + y_pred: input data to compute, typical segmentation model output. + It must be one-hot format and first dim is batch, example shape: [16, 3, 32, 32]. The values + should be binarized. + y: ground truth to compute IOU metric. It must be one-hot format and first dim is batch. + The values should be binarized. + Raises: + ValueError: when `y` is not a binarized tensor. + ValueError: when `y_pred` has less than three dimensions. + """ + is_binary_tensor(y_pred, "y_pred") + is_binary_tensor(y, "y") + + dims = y_pred.ndimension() + if dims < 3: + raise ValueError(f"y_pred should have at least 3 dimensions (batch, channel, spatial), got {dims}.") + # compute iou (BxC) for each channel for each batch + return compute_iou( + y_pred=y_pred, y=y, include_background=self.include_background, ignore_empty=self.ignore_empty + ) + + def aggregate(self, reduction: Union[MetricReduction, str, None] = None): # type: ignore + """ + Execute reduction logic for the output of `compute_iou`. + Args: + reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values, + available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``, + ``"mean_channel"``, ``"sum_channel"``}, default to `self.reduction`. if "none", will not do reduction. + """ + data = self.get_buffer() + if not isinstance(data, torch.Tensor): + raise ValueError("the data to aggregate must be PyTorch Tensor.") + + # do metric reduction + f, not_nans = do_metric_reduction(data, reduction or self.reduction) + return (f, not_nans) if self.get_not_nans else f + + +def compute_iou( + y_pred: torch.Tensor, y: torch.Tensor, include_background: bool = True, ignore_empty: bool = True +) -> torch.Tensor: + """Computes IOU score metric from full size Tensor and collects average. + Args: + y_pred: input data to compute, typical segmentation model output. + It must be one-hot format and first dim is batch, example shape: [16, 3, 32, 32]. The values + should be binarized. + y: ground truth to compute IOU metric. It must be one-hot format and first dim is batch. + The values should be binarized. + include_background: whether to skip IOU computation on the first channel of + the predicted output. Defaults to True. + ignore_empty: whether to ignore empty ground truth cases during calculation. + If `True`, NaN value will be set for empty ground truth cases. + If `False`, 1 will be set if the predictions of empty ground truth cases are also empty. + Returns: + IOU scores per batch and per class, (shape [batch_size, num_classes]). + Raises: + ValueError: when `y_pred` and `y` have different shapes. + """ + + if not include_background: + y_pred, y = ignore_background(y_pred=y_pred, y=y) + + y = y.float() + y_pred = y_pred.float() + + if y.shape != y_pred.shape: + raise ValueError(f"y_pred and y should have same shapes, got {y_pred.shape} and {y.shape}.") + + # reducing only spatial dimensions (not batch nor channels) + n_len = len(y_pred.shape) + reduce_axis = list(range(2, n_len)) + intersection = torch.sum(y * y_pred, dim=reduce_axis) + + y_o = torch.sum(y, reduce_axis) + y_pred_o = torch.sum(y_pred, dim=reduce_axis) + union = y_o + y_pred_o - intersection + + if ignore_empty is True: + return torch.where(y_o > 0, (intersection) / union, torch.tensor(float("nan"), device=y_o.device)) + return torch.where(denominator > 0, (intersection) / union, torch.tensor(1.0, device=y_o.device)) diff --git a/tests/test_compute_iou.py b/tests/test_compute_iou.py new file mode 100644 index 0000000000..3bc61e80a7 --- /dev/null +++ b/tests/test_compute_iou.py @@ -0,0 +1,220 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +from parameterized import parameterized + +from monai.metrics import IoUMetric, compute_iou + +# keep background +TEST_CASE_1 = [ # y (1, 1, 2, 2), y_pred (1, 1, 2, 2), expected out (1, 1) + { + "y_pred": torch.tensor([[[[1.0, 0.0], [0.0, 1.0]]]]), + "y": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]), + "include_background": True, + }, + [[0.6667]], +] + +# remove background and not One-Hot target +TEST_CASE_2 = [ # y (2, 3, 2, 2), y_pred (2, 3, 2, 2), expected out (2, 2) (no background) + { + "y_pred": torch.tensor( + [ + [[[0.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]]], + [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 0.0]]], + ] + ), + "y": torch.tensor( + [ + [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 0.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]]], + [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 0.0]]], + ] + ), + "include_background": False, + }, + [[0.4000, 0.0000],[0.4000, 0.5000]], +] + +# should return Nan for all labels=0 case and skip for MeanIoU +TEST_CASE_3 = [ + { + "y_pred": torch.tensor( + [ + [[[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]], [[1.0, 1.0], [1.0, 1.0]]], + [[[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]], [[1.0, 1.0], [1.0, 1.0]]], + ] + ), + "y": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]], + [[[0.0, 1.0], [1.0, 0.0]], [[1.0, 0.0], [0.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]]], + ] + ), + "include_background": True, + }, + [[False, True, True], [False, False, True]], +] + +TEST_CASE_4 = [ + {"include_background": True, "reduction": "mean_batch", "get_not_nans": True}, + { + "y_pred": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]], + [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]], + ] + ), + "y": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]], + [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 0.0]]], + ] + ), + }, + [0.5416, 0.2500, 0.5000], +] + +TEST_CASE_5 = [ + {"include_background": True, "reduction": "mean", "get_not_nans": True}, + { + "y_pred": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]], + [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]], + ] + ), + "y": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]], + [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 0.0]]], + ] + ), + }, + 0.5555, +] + +TEST_CASE_6 = [ + {"include_background": True, "reduction": "sum_batch", "get_not_nans": True}, + { + "y_pred": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]], + [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]], + ] + ), + "y": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]], + [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]], + ] + ), + }, + [1.5000, 0.0000, 0.0000], +] + +TEST_CASE_7 = [ + {"include_background": True, "reduction": "mean", "get_not_nans": True}, + { + "y_pred": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]], + [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]], + ] + ), + "y": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]], + [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]], + ] + ), + }, + 0.7500, +] + +TEST_CASE_8 = [ + {"include_background": False, "reduction": "sum_batch", "get_not_nans": True}, + { + "y_pred": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]], + [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]], + ] + ), + "y": torch.tensor( + [ + [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]], + [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]], + ] + ), + }, + [0.0000, 0.0000], +] + +TEST_CASE_9 = [ + {"y": torch.ones((2, 2, 3, 3)), "y_pred": torch.ones((2, 2, 3, 3))}, + [[1.0000, 1.0000], [1.0000, 1.0000]], +] + +TEST_CASE_10 = [ + {"y": [torch.ones((2, 3, 3)), torch.ones((2, 3, 3))], "y_pred": [torch.ones((2, 3, 3)), torch.ones((2, 3, 3))]}, + [[1.0000, 1.0000], [1.0000, 1.0000]], +] + +TEST_CASE_11 = [ + {"y": torch.zeros((2, 2, 3, 3)), "y_pred": torch.zeros((2, 2, 3, 3)), "ignore_empty": False}, + [[1.0000, 1.0000], [1.0000, 1.0000]], +] + +TEST_CASE_12 = [ + {"y": torch.zeros((2, 2, 3, 3)), "y_pred": torch.ones((2, 2, 3, 3)), "ignore_empty": False}, + [[0.0000, 0.0000], [0.0000, 0.0000]], +] + + +class TestComputeMeanIoU(unittest.TestCase): + @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_9, TEST_CASE_11, TEST_CASE_12]) + def test_value(self, input_data, expected_value): + result = compute_iou(**input_data) + np.testing.assert_allclose(result.cpu().numpy(), expected_value, atol=1e-4) + + @parameterized.expand([TEST_CASE_3]) + def test_nans(self, input_data, expected_value): + result = compute_iou(**input_data) + self.assertTrue(np.allclose(np.isnan(result.cpu().numpy()), expected_value)) + + # IoUMetric class tests + @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_10]) + def test_value_class(self, input_data, expected_value): + + # same test as for compute_iou + vals = {} + vals["y_pred"] = input_data.pop("y_pred") + vals["y"] = input_data.pop("y") + iou_metric = IoUMetric(**input_data) + iou_metric(**vals) + result = iou_metric.aggregate(reduction="none") + np.testing.assert_allclose(result.cpu().numpy(), expected_value, atol=1e-4) + + @parameterized.expand([TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]) + def test_nans_class(self, params, input_data, expected_value): + + iou_metric = IoUMetric(**params) + iou_metric(**input_data) + result, _ = iou_metric.aggregate() + np.testing.assert_allclose(result.cpu().numpy(), expected_value, atol=1e-4) + + +if __name__ == "__main__": + unittest.main()