561 497 Provide metrics base APIs

docs/source/metrics.rst

@@ -10,6 +10,11 @@ Metrics
 ------
 .. autofunction:: compute_froc_score
 
+`Metric`
+--------
+.. autoclass:: Metric
+    :members:
+
 `Mean Dice`
 -----------
 .. autofunction:: compute_meandice
@@ -21,6 +26,9 @@ Metrics
 --------------------------
 .. autofunction:: compute_roc_auc
 
+.. autoclass:: ROCAUCMetric
+    :members:
+
 `Confusion matrix`
 ------------------
 .. autofunction:: get_confusion_matrix

monai/config/__init__.py

@@ -18,4 +18,4 @@
     print_gpu_info,
     print_system_info,
 )
-from .type_definitions import DtypeLike, IndexSelection, KeysCollection, NdarrayTensor
+from .type_definitions import DtypeLike, IndexSelection, KeysCollection, NdarrayTensor, TensorOrList

monai/config/type_definitions.py

@@ -9,12 +9,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Collection, Hashable, Iterable, TypeVar, Union
+from typing import Collection, Hashable, Iterable, Sequence, TypeVar, Union
 
 import numpy as np
 import torch
 
-__all__ = ["KeysCollection", "IndexSelection", "DtypeLike", "NdarrayTensor"]
+__all__ = ["KeysCollection", "IndexSelection", "DtypeLike", "NdarrayTensor", "TensorOrList"]
 
 """Commonly used concepts
 This module provides naming and type specifications for commonly used concepts
@@ -55,6 +55,7 @@
 container must be iterable.
 """
 
+
 DtypeLike = Union[
     np.dtype,
     type,
@@ -67,3 +68,10 @@
 # Generic type which can represent either a numpy.ndarray or a torch.Tensor
 # Unlike Union can create a dependence between parameter(s) / return(s)
 NdarrayTensor = TypeVar("NdarrayTensor", np.ndarray, torch.Tensor)
+
+
+TensorOrList = Union[torch.Tensor, Sequence[torch.Tensor]]
+"""TensorOrList
+
+The TensorOrList type is used for defining `batch-first Tensor` or `list of channel-first Tensor`.
+"""

monai/handlers/confusion_matrix.py

@@ -9,13 +9,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Callable, Union
+from typing import Callable, Union
 
 import torch
 
 from monai.handlers.iteration_metric import IterationMetric
-from monai.metrics import ConfusionMatrixMetric, compute_confusion_matrix_metric
-from monai.metrics.utils import MetricReduction, do_metric_reduction
+from monai.metrics import ConfusionMatrixMetric
+from monai.metrics.utils import MetricReduction
 
 
 class ConfusionMatrix(IterationMetric):
@@ -30,6 +30,7 @@ def __init__(
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
     ) -> None:
         """
 
@@ -47,6 +48,9 @@ def __init__(
             device: device specification in case of distributed computation usage.
             save_details: whether to save metric computation details per image, for example: TP/TN/FP/FN of every image.
                 default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
 
         See also:
             :py:meth:`monai.metrics.confusion_matrix`
@@ -55,7 +59,7 @@ def __init__(
             include_background=include_background,
             metric_name=metric_name,
             compute_sample=False,
-            reduction=MetricReduction.NONE,
+            reduction=reduction,
         )
         self.metric_name = metric_name
         super().__init__(
@@ -64,7 +68,3 @@ def __init__(
             device=device,
             save_details=save_details,
         )
-
-    def _reduce(self, scores) -> Any:
-        confusion_matrix, _ = do_metric_reduction(scores, MetricReduction.MEAN)
-        return compute_confusion_matrix_metric(self.metric_name, confusion_matrix)

monai/handlers/hausdorff_distance.py

@@ -32,6 +32,7 @@ def __init__(
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
     ) -> None:
         """
 
@@ -48,15 +49,17 @@ def __init__(
             device: device specification in case of distributed computation usage.
             save_details: whether to save metric computation details per image, for example: hausdorff distance
                 of every image. default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
 
         """
-        super().__init__(output_transform, device=device)
         metric_fn = HausdorffDistanceMetric(
             include_background=include_background,
             distance_metric=distance_metric,
             percentile=percentile,
             directed=directed,
-            reduction=MetricReduction.NONE,
+            reduction=reduction,
         )
         super().__init__(
             metric_fn=metric_fn,

monai/handlers/iteration_metric.py

@@ -14,19 +14,19 @@
 import torch
 
 from monai.handlers.utils import evenly_divisible_all_gather
-from monai.metrics import do_metric_reduction
-from monai.utils import MetricReduction, exact_version, optional_import
+from monai.metrics import Metric
+from monai.utils import exact_version, optional_import
 
 idist, _ = optional_import("ignite", "0.4.4", exact_version, "distributed")
-Metric, _ = optional_import("ignite.metrics", "0.4.4", exact_version, "Metric")
+IgniteMetric, _ = optional_import("ignite.metrics", "0.4.4", exact_version, "Metric")
 reinit__is_reduced, _ = optional_import("ignite.metrics.metric", "0.4.4", exact_version, "reinit__is_reduced")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
     Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 
-class IterationMetric(Metric):  # type: ignore[valid-type, misc] # due to optional_import
+class IterationMetric(IgniteMetric):  # type: ignore[valid-type, misc] # due to optional_import
     """
     Class for metrics that should be computed on every iteration and compute final results when epoch completed.
     Similar to the `EpochMetric` in ignite:
@@ -46,7 +46,7 @@ class IterationMetric(Metric):  # type: ignore[valid-type, misc] # due to option
 
     def __init__(
         self,
-        metric_fn: Callable,
+        metric_fn: Metric,
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
@@ -78,19 +78,7 @@ def update(self, output: Sequence[torch.Tensor]) -> None:
 
         y_pred, y = output
 
-        def _compute(y_pred, y):
-            if isinstance(y_pred, torch.Tensor):
-                y_pred = y_pred.detach()
-            if isinstance(y, torch.Tensor):
-                y = y.detach()
-            score = self.metric_fn(y_pred, y)
-            return score[0] if isinstance(score, (tuple, list)) else score
-
-        if isinstance(y_pred, (list, tuple)) or isinstance(y, (list, tuple)):
-            # if y_pred or y is a list of channel-first data, add batch dim and compute metric, then concat the scores
-            score = torch.cat([_compute(p_.unsqueeze(0), y_.unsqueeze(0)) for p_, y_ in zip(y_pred, y)], dim=0)
-        else:
-            score = _compute(y_pred, y)
+        score = self.metric_fn(y_pred, y)
         self._scores.append(score.to(self._device))
 
     def compute(self) -> Any:
@@ -117,6 +105,7 @@ def compute(self) -> Any:
         if idist.get_rank() == 0:
             # run compute_fn on zero rank only
             result = self._reduce(_scores)
+            result = result[0] if isinstance(result, (list, tuple)) else result
 
         if ws > 1:
             # broadcast result to all processes
@@ -125,7 +114,7 @@ def compute(self) -> Any:
         return result.item() if isinstance(result, torch.Tensor) else result
 
     def _reduce(self, scores) -> Any:
-        return do_metric_reduction(scores, MetricReduction.MEAN)[0]
+        return self.metric_fn.aggregate(scores)
 
     def attach(self, engine: Engine, name: str) -> None:
         """

monai/handlers/mean_dice.py

@@ -29,6 +29,7 @@ def __init__(
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
     ) -> None:
         """
 
@@ -39,14 +40,14 @@ def __init__(
             device: device specification in case of distributed computation usage.
             save_details: whether to save metric computation details per image, for example: mean dice of every image.
                 default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
 
         See also:
             :py:meth:`monai.metrics.meandice.compute_meandice`
         """
-        metric_fn = DiceMetric(
-            include_background=include_background,
-            reduction=MetricReduction.NONE,
-        )
+        metric_fn = DiceMetric(include_background=include_background, reduction=reduction)
         super().__init__(
             metric_fn=metric_fn,
             output_transform=output_transform,

monai/handlers/regression_metrics.py

@@ -28,6 +28,7 @@ def __init__(
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
     ) -> None:
         """
 
@@ -36,13 +37,14 @@ def __init__(
             device: device specification in case of distributed computation usage.
             save_details: whether to save metric computation details per image, for example: mean squared error of every image.
                 default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
 
         See also:
             :py:class:`monai.metrics.MSEMetric`
         """
-        metric_fn = MSEMetric(
-            reduction=MetricReduction.NONE,
-        )
+        metric_fn = MSEMetric(reduction=reduction)
         super().__init__(
             metric_fn=metric_fn,
             output_transform=output_transform,
@@ -61,6 +63,7 @@ def __init__(
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
     ) -> None:
         """
 
@@ -69,13 +72,14 @@ def __init__(
             device: device specification in case of distributed computation usage.
             save_details: whether to save metric computation details per image, for example: mean absolute error of every image.
                 default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
 
         See also:
             :py:class:`monai.metrics.MAEMetric`
         """
-        metric_fn = MAEMetric(
-            reduction=MetricReduction.NONE,
-        )
+        metric_fn = MAEMetric(reduction=reduction)
         super().__init__(
             metric_fn=metric_fn,
             output_transform=output_transform,
@@ -94,6 +98,7 @@ def __init__(
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
     ) -> None:
         """
 
@@ -102,13 +107,14 @@ def __init__(
             device: device specification in case of distributed computation usage.
             save_details: whether to save metric computation details per image, for example: root mean squared error of every image.
                 default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
 
         See also:
             :py:class:`monai.metrics.RMSEMetric`
         """
-        metric_fn = RMSEMetric(
-            reduction=MetricReduction.NONE,
-        )
+        metric_fn = RMSEMetric(reduction=reduction)
         super().__init__(
             metric_fn=metric_fn,
             output_transform=output_transform,
@@ -128,6 +134,7 @@ def __init__(
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
     ) -> None:
         """
 
@@ -138,14 +145,14 @@ def __init__(
             device: device specification in case of distributed computation usage.
             save_details: whether to save metric computation details per image, for example: PSNR of every image.
                 default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}
+                Define the mode to reduce computation result. Defaults to ``"mean"``.
 
         See also:
             :py:class:`monai.metrics.PSNRMetric`
         """
-        metric_fn = PSNRMetric(
-            max_val=max_val,
-            reduction=MetricReduction.NONE,
-        )
+        metric_fn = PSNRMetric(max_val=max_val, reduction=reduction)
         super().__init__(
             metric_fn=metric_fn,
             output_transform=output_transform,

monai/handlers/roc_auc.py

@@ -9,16 +9,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Callable, Union
+from typing import Any, Callable, Tuple, Union
 
 import torch
 
 from monai.handlers.utils import evenly_divisible_all_gather
-from monai.metrics import compute_roc_auc
+from monai.metrics import ROCAUCMetric
 from monai.utils import Average, exact_version, optional_import
 
 idist, _ = optional_import("ignite", "0.4.4", exact_version, "distributed")
 EpochMetric, _ = optional_import("ignite.metrics", "0.4.4", exact_version, "EpochMetric")
+reinit__is_reduced, _ = optional_import("ignite.metrics.metric", "0.4.4", exact_version, "reinit__is_reduced")
 
 
 class ROCAUC(EpochMetric):  # type: ignore[valid-type, misc]  # due to optional_import
@@ -56,21 +57,19 @@ def __init__(
         output_transform: Callable = lambda x: x,
         device: Union[str, torch.device] = "cpu",
     ) -> None:
-        def _compute_fn(pred, label):
-            return compute_roc_auc(
-                y_pred=pred,
-                y=label,
-                average=Average(average),
-            )
-
+        self.metric = ROCAUCMetric(average=Average(average))
         self._is_reduced: bool = False
         super().__init__(
-            compute_fn=_compute_fn,
+            compute_fn=lambda p, y: self.metric.aggregate(data=(p, y)),
             output_transform=output_transform,
             check_compute_fn=False,
             device=device,
         )
 
+    @reinit__is_reduced
+    def update(self, output: Tuple[torch.Tensor, torch.Tensor]) -> None:
+        super().update(output=self.metric(output[0], output[1]))
+
     def compute(self) -> Any:
         _prediction_tensor = torch.cat(self._predictions, dim=0)
         _target_tensor = torch.cat(self._targets, dim=0)