Mean IoU Implementation

docs/source/metrics.rst

@@ -39,6 +39,13 @@ Metrics
 .. autoclass:: DiceMetric
     :members:
 
+`Mean IoU`
+----------
+.. autofunction:: compute_meaniou
+
+.. autoclass:: MeanIoU
+    :members:
+
 `Generalized Dice Score`
 ------------------------
 .. autofunction:: compute_generalized_dice

monai/metrics/__init__.py

@@ -15,6 +15,7 @@
 from .generalized_dice import GeneralizedDiceScore, compute_generalized_dice
 from .hausdorff_distance import HausdorffDistanceMetric, compute_hausdorff_distance, compute_percent_hausdorff_distance
 from .meandice import DiceMetric, compute_meandice
+from .meaniou import MeanIoU, compute_meaniou
 from .metric import Cumulative, CumulativeIterationMetric, IterationMetric, Metric
 from .regression import MAEMetric, MSEMetric, PSNRMetric, RMSEMetric, SSIMMetric
 from .rocauc import ROCAUCMetric, compute_roc_auc

monai/metrics/meaniou.py

@@ -0,0 +1,149 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Union
+
+import torch
+
+from monai.metrics.utils import do_metric_reduction, ignore_background, is_binary_tensor
+from monai.utils import MetricReduction
+
+from .metric import CumulativeIterationMetric
+
+
+class MeanIoU(CumulativeIterationMetric):
+    """
+    Compute average IoU score between two tensors. It can support both multi-classes and multi-labels tasks.
+    Input `y_pred` is compared with ground truth `y`.
+    `y_pred` is expected to have binarized predictions and `y` should be in one-hot format. You can use suitable transforms
+    in ``monai.transforms.post`` first to achieve binarized values.
+    The `include_background` parameter can be set to ``False`` to exclude
+    the first category (channel index 0) which is by convention assumed to be background. If the non-background
+    segmentations are small compared to the total image size they can get overwhelmed by the signal from the
+    background.
+    `y_pred` and `y` can be a list of channel-first Tensor (CHW[D]) or a batch-first Tensor (BCHW[D]).
+
+    Args:
+        include_background: whether to skip IoU computation on the first channel of
+            the predicted output. Defaults to ``True``.
+        reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values,
+            available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+            ``"mean_channel"``, ``"sum_channel"``}, default to ``"mean"``. if "none", will not do reduction.
+        get_not_nans: whether to return the `not_nans` count, if True, aggregate() returns (metric, not_nans).
+            Here `not_nans` count the number of not nans for the metric, thus its shape equals to the shape of the metric.
+        ignore_empty: whether to ignore empty ground truth cases during calculation.
+            If `True`, NaN value will be set for empty ground truth cases.
+            If `False`, 1 will be set if the predictions of empty ground truth cases are also empty.
+
+    """
+
+    def __init__(
+        self,
+        include_background: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
+        get_not_nans: bool = False,
+        ignore_empty: bool = True,
+    ) -> None:
+        super().__init__()
+        self.include_background = include_background
+        self.reduction = reduction
+        self.get_not_nans = get_not_nans
+        self.ignore_empty = ignore_empty
+
+    def _compute_tensor(self, y_pred: torch.Tensor, y: torch.Tensor):  # type: ignore
+        """
+        Args:
+            y_pred: input data to compute, typical segmentation model output.
+                It must be one-hot format and first dim is batch, example shape: [16, 3, 32, 32]. The values
+                should be binarized.
+            y: ground truth to compute mean IoU metric. It must be one-hot format and first dim is batch.
+                The values should be binarized.
+
+        Raises:
+            ValueError: when `y` is not a binarized tensor.
+            ValueError: when `y_pred` has less than three dimensions.
+        """
+        is_binary_tensor(y_pred, "y_pred")
+        is_binary_tensor(y, "y")
+
+        dims = y_pred.ndimension()
+        if dims < 3:
+            raise ValueError(f"y_pred should have at least 3 dimensions (batch, channel, spatial), got {dims}.")
+        # compute IoU (BxC) for each channel for each batch
+        return compute_meaniou(
+            y_pred=y_pred, y=y, include_background=self.include_background, ignore_empty=self.ignore_empty
+        )
+
+    def aggregate(self, reduction: Union[MetricReduction, str, None] = None):  # type: ignore
+        """
+        Execute reduction logic for the output of `compute_meaniou`.
+
+        Args:
+            reduction: define mode of reduction to the metrics, will only apply reduction on `not-nan` values,
+                available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}, default to `self.reduction`. if "none", will not do reduction.
+
+        """
+        data = self.get_buffer()
+        if not isinstance(data, torch.Tensor):
+            raise ValueError("the data to aggregate must be PyTorch Tensor.")
+
+        # do metric reduction
+        f, not_nans = do_metric_reduction(data, reduction or self.reduction)
+        return (f, not_nans) if self.get_not_nans else f
+
+
+def compute_meaniou(
+    y_pred: torch.Tensor, y: torch.Tensor, include_background: bool = True, ignore_empty: bool = True
+) -> torch.Tensor:
+    """Computes IoU score metric from full size Tensor and collects average.
+
+    Args:
+        y_pred: input data to compute, typical segmentation model output.
+            It must be one-hot format and first dim is batch, example shape: [16, 3, 32, 32]. The values
+            should be binarized.
+        y: ground truth to compute mean IoU metric. It must be one-hot format and first dim is batch.
+            The values should be binarized.
+        include_background: whether to skip IoU computation on the first channel of
+            the predicted output. Defaults to True.
+        ignore_empty: whether to ignore empty ground truth cases during calculation.
+            If `True`, NaN value will be set for empty ground truth cases.
+            If `False`, 1 will be set if the predictions of empty ground truth cases are also empty.
+
+    Returns:
+        IoU scores per batch and per class, (shape [batch_size, num_classes]).
+
+    Raises:
+        ValueError: when `y_pred` and `y` have different shapes.
+
+    """
+
+    if not include_background:
+        y_pred, y = ignore_background(y_pred=y_pred, y=y)
+
+    y = y.float()
+    y_pred = y_pred.float()
+
+    if y.shape != y_pred.shape:
+        raise ValueError(f"y_pred and y should have same shapes, got {y_pred.shape} and {y.shape}.")
+
+    # reducing only spatial dimensions (not batch nor channels)
+    n_len = len(y_pred.shape)
+    reduce_axis = list(range(2, n_len))
+    intersection = torch.sum(y * y_pred, dim=reduce_axis)
+
+    y_o = torch.sum(y, reduce_axis)
+    y_pred_o = torch.sum(y_pred, dim=reduce_axis)
+    union = y_o + y_pred_o - intersection
+
+    if ignore_empty is True:
+        return torch.where(y_o > 0, (intersection) / union, torch.tensor(float("nan"), device=y_o.device))
+    return torch.where(union > 0, (intersection) / union, torch.tensor(1.0, device=y_o.device))

tests/test_compute_meaniou.py

@@ -0,0 +1,220 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.metrics import MeanIoU, compute_meaniou
+
+# keep background
+TEST_CASE_1 = [  # y (1, 1, 2, 2), y_pred (1, 1, 2, 2), expected out (1, 1)
+    {
+        "y_pred": torch.tensor([[[[1.0, 0.0], [0.0, 1.0]]]]),
+        "y": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]]),
+        "include_background": True,
+    },
+    [[0.6667]],
+]
+
+# remove background and not One-Hot target
+TEST_CASE_2 = [  # y (2, 3, 2, 2), y_pred (2, 3, 2, 2), expected out (2, 2) (no background)
+    {
+        "y_pred": torch.tensor(
+            [
+                [[[0.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]]],
+                [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 0.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 0.0]]],
+            ]
+        ),
+        "y": torch.tensor(
+            [
+                [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 0.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]]],
+                [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 0.0]]],
+            ]
+        ),
+        "include_background": False,
+    },
+    [[0.3333, 0.0000], [0.5000, 0.5000]],
+]
+
+# should return Nan for all labels=0 case and skip for MeanIoU
+TEST_CASE_3 = [
+    {
+        "y_pred": torch.tensor(
+            [
+                [[[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]], [[1.0, 1.0], [1.0, 1.0]]],
+                [[[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]], [[1.0, 1.0], [1.0, 1.0]]],
+            ]
+        ),
+        "y": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]],
+                [[[0.0, 1.0], [1.0, 0.0]], [[1.0, 0.0], [0.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]]],
+            ]
+        ),
+        "include_background": True,
+    },
+    [[False, True, True], [False, False, True]],
+]
+
+TEST_CASE_4 = [
+    {"include_background": True, "reduction": "mean_batch", "get_not_nans": True},
+    {
+        "y_pred": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]],
+                [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]],
+            ]
+        ),
+        "y": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]],
+                [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 0.0]]],
+            ]
+        ),
+    },
+    [0.5416, 0.2500, 0.5000],
+]
+
+TEST_CASE_5 = [
+    {"include_background": True, "reduction": "mean", "get_not_nans": True},
+    {
+        "y_pred": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]],
+                [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]],
+            ]
+        ),
+        "y": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]],
+                [[[0.0, 0.0], [0.0, 1.0]], [[1.0, 1.0], [0.0, 0.0]], [[0.0, 0.0], [1.0, 0.0]]],
+            ]
+        ),
+    },
+    0.5555,
+]
+
+TEST_CASE_6 = [
+    {"include_background": True, "reduction": "sum_batch", "get_not_nans": True},
+    {
+        "y_pred": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]],
+                [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]],
+            ]
+        ),
+        "y": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]],
+                [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]],
+            ]
+        ),
+    },
+    [1.5000, 0.0000, 0.0000],
+]
+
+TEST_CASE_7 = [
+    {"include_background": True, "reduction": "mean", "get_not_nans": True},
+    {
+        "y_pred": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]],
+                [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]],
+            ]
+        ),
+        "y": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]],
+                [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]],
+            ]
+        ),
+    },
+    0.7500,
+]
+
+TEST_CASE_8 = [
+    {"include_background": False, "reduction": "sum_batch", "get_not_nans": True},
+    {
+        "y_pred": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 0.0]], [[0.0, 1.0], [0.0, 0.0]], [[0.0, 1.0], [1.0, 1.0]]],
+                [[[1.0, 0.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 1.0]], [[0.0, 1.0], [1.0, 0.0]]],
+            ]
+        ),
+        "y": torch.tensor(
+            [
+                [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]],
+                [[[1.0, 1.0], [1.0, 1.0]], [[0.0, 0.0], [0.0, 0.0]], [[0.0, 0.0], [0.0, 0.0]]],
+            ]
+        ),
+    },
+    [0.0000, 0.0000],
+]
+
+TEST_CASE_9 = [
+    {"y": torch.ones((2, 2, 3, 3)), "y_pred": torch.ones((2, 2, 3, 3))},
+    [[1.0000, 1.0000], [1.0000, 1.0000]],
+]
+
+TEST_CASE_10 = [
+    {"y": [torch.ones((2, 3, 3)), torch.ones((2, 3, 3))], "y_pred": [torch.ones((2, 3, 3)), torch.ones((2, 3, 3))]},
+    [[1.0000, 1.0000], [1.0000, 1.0000]],
+]
+
+TEST_CASE_11 = [
+    {"y": torch.zeros((2, 2, 3, 3)), "y_pred": torch.zeros((2, 2, 3, 3)), "ignore_empty": False},
+    [[1.0000, 1.0000], [1.0000, 1.0000]],
+]
+
+TEST_CASE_12 = [
+    {"y": torch.zeros((2, 2, 3, 3)), "y_pred": torch.ones((2, 2, 3, 3)), "ignore_empty": False},
+    [[0.0000, 0.0000], [0.0000, 0.0000]],
+]
+
+
+class TestComputeMeanIoU(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_9, TEST_CASE_11, TEST_CASE_12])
+    def test_value(self, input_data, expected_value):
+        result = compute_meaniou(**input_data)
+        np.testing.assert_allclose(result.cpu().numpy(), expected_value, atol=1e-4)
+
+    @parameterized.expand([TEST_CASE_3])
+    def test_nans(self, input_data, expected_value):
+        result = compute_meaniou(**input_data)
+        self.assertTrue(np.allclose(np.isnan(result.cpu().numpy()), expected_value))
+
+    # MeanIoU class tests
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_10])
+    def test_value_class(self, input_data, expected_value):
+
+        # same test as for compute_meaniou
+        vals = {}
+        vals["y_pred"] = input_data.pop("y_pred")
+        vals["y"] = input_data.pop("y")
+        iou_metric = MeanIoU(**input_data)
+        iou_metric(**vals)
+        result = iou_metric.aggregate(reduction="none")
+        np.testing.assert_allclose(result.cpu().numpy(), expected_value, atol=1e-4)
+
+    @parameterized.expand([TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8])
+    def test_nans_class(self, params, input_data, expected_value):
+
+        iou_metric = MeanIoU(**params)
+        iou_metric(**input_data)
+        result, _ = iou_metric.aggregate()
+        np.testing.assert_allclose(result.cpu().numpy(), expected_value, atol=1e-4)
+
+
+if __name__ == "__main__":
+    unittest.main()