enhance persistent dataset tempfile

monai/apps/datasets.py

@@ -128,10 +128,8 @@ def _generate_data_list(self, dataset_dir: str) -> List[Dict]:
             image_files_list.extend(image_files[i])
             image_class.extend([i] * num_each[i])
             class_name.extend([class_names[i]] * num_each[i])
-        num_total = len(image_class)
 
         data = []
-
         length = len(image_files_list)
         indices = np.arange(length)
         self.randomize(indices)

monai/apps/deepgrow/transforms.py

@@ -185,8 +185,8 @@ def _get_signal(self, image, guidance):
             signal = np.zeros((len(guidance), image.shape[-2], image.shape[-1]), dtype=np.float32)
 
         sshape = signal.shape
-        for i in range(len(guidance)):
-            for point in guidance[i]:
+        for i, g_i in enumerate(guidance):
+            for point in g_i:
                 if np.any(np.asarray(point) < 0):
                     continue
 
@@ -924,8 +924,8 @@ def __init__(
     def _apply(self, image, guidance):
         slice_idx = guidance[2]  # (pos, neg, slice_idx)
         idx = []
-        for i in range(len(image.shape)):
-            idx.append(slice_idx) if i == self.axis else idx.append(slice(0, image.shape[i]))
+        for i, size_i in enumerate(image.shape):
+            idx.append(slice_idx) if i == self.axis else idx.append(slice(0, size_i))
 
         idx = tuple(idx)
         return image[idx], idx

monai/apps/pathology/datasets.py

@@ -274,7 +274,7 @@ def _calculate_mask_level(self, image: np.ndarray, mask: np.ndarray) -> Tuple[in
                 f"ratio 0: {ratios[0]} ({image_shape[0]} / {mask_shape[0]}),"
                 f"ratio 1: {ratios[1]} ({image_shape[1]} / {mask_shape[1]}),"
             )
-        elif not level.is_integer():
+        if not level.is_integer():
             raise ValueError(f"Mask is not at a regular level (ratio not power of 2), image / mask ratio: {ratios[0]}")
 
         return int(level), ratios[0]

monai/config/deviceconfig.py

@@ -218,12 +218,6 @@ def get_gpu_info() -> OrderedDict:
         _dict_append(output, f"GPU {gpu} Is multi GPU board", lambda: bool(gpu_info.is_multi_gpu_board))
         _dict_append(output, f"GPU {gpu} Multi processor count", lambda: gpu_info.multi_processor_count)
         _dict_append(output, f"GPU {gpu} Total memory (GB)", lambda: round(gpu_info.total_memory / 1024 ** 3, 1))
-        _dict_append(
-            output, f"GPU {gpu} Cached memory (GB)", lambda: round(torch.cuda.memory_reserved(gpu) / 1024 ** 3, 1)
-        )
-        _dict_append(
-            output, f"GPU {gpu} Allocated memory (GB)", lambda: round(torch.cuda.memory_allocated(gpu) / 1024 ** 3, 1)
-        )
         _dict_append(output, f"GPU {gpu} CUDA capability (maj.min)", lambda: f"{gpu_info.major}.{gpu_info.minor}")
 
     return output

monai/data/dataset.py

@@ -14,6 +14,7 @@
 import math
 import pickle
 import sys
+import tempfile
 import threading
 import time
 import warnings
@@ -240,12 +241,18 @@ def _cachecheck(self, item_transformed):
 
         _item_transformed = self._pre_transform(deepcopy(item_transformed))  # keep the original hashed
         if hashfile is not None:
-            # NOTE: Writing to ".temp_write_cache" and then using a nearly atomic rename operation
+            # NOTE: Writing to a temporary directory and then using a nearly atomic rename operation
             #       to make the cache more robust to manual killing of parent process
             #       which may leave partially written cache files in an incomplete state
-            temp_hash_file = hashfile.with_suffix(".temp_write_cache")
-            torch.save(_item_transformed, temp_hash_file)
-            temp_hash_file.rename(hashfile)
+            with tempfile.TemporaryDirectory() as tmpdirname:
+                temp_hash_file = Path(tmpdirname) / hashfile.name
+                torch.save(_item_transformed, temp_hash_file)
+                if temp_hash_file.is_file() and not hashfile.is_file():
+                    # On Unix, if target exists and is a file, it will be replaced silently if the user has permission.
+                    try:
+                        temp_hash_file.rename(hashfile)
+                    except FileExistsError:
+                        pass
         return _item_transformed
 
     def _transform(self, index: int):
@@ -408,6 +415,10 @@ def _fill_cache_start_reader(self):
                     new_size = size * 2
                     warnings.warn(f"Resizing the cache database from {int(size) >> 20}MB to {int(new_size) >> 20}MB.")
                     env.set_mapsize(new_size)
+                except lmdb.MapResizedError:
+                    # the mapsize is increased by another process
+                    # set_mapsize with a size of 0 to adopt the new size,
+                    env.set_mapsize(0)
             if not done:  # still has the map full error
                 size = env.info()["map_size"]
                 env.close()

monai/data/inverse_batch_transform.py

@@ -77,7 +77,7 @@ def __init__(
         self.batch_size = loader.batch_size
         self.num_workers = loader.num_workers if num_workers is None else num_workers
         self.collate_fn = collate_fn
-        self.pad_collation_used = loader.collate_fn == pad_list_data_collate
+        self.pad_collation_used = loader.collate_fn.__doc__ == pad_list_data_collate.__doc__
 
     def __call__(self, data: Dict[str, Any]) -> Any:
 

monai/losses/dice.py

@@ -590,8 +590,8 @@ def _compute_alpha_generalized_true_positives(self, flat_target: torch.Tensor) -
         if self.alpha_mode == "GDL":  # GDL style
             # Define alpha like in the generalized dice loss
             # i.e. the inverse of the volume of each class.
-            one_hot = F.one_hot(flat_target, num_classes=self.num_classes).permute(0, 2, 1).float()
-            volumes = torch.sum(one_hot, dim=2)
+            one_hot_f = F.one_hot(flat_target, num_classes=self.num_classes).permute(0, 2, 1).float()
+            volumes = torch.sum(one_hot_f, dim=2)
             alpha = 1.0 / (volumes + 1.0)
         else:  # default, i.e. like in the original paper
             # alpha weights are 0 for the background and 1 the other classes

monai/networks/nets/ahnet.py

@@ -390,10 +390,8 @@ def __init__(
         self.bn0 = norm_type(64)
         self.relu = relu_type(inplace=True)
         if upsample_mode in ["transpose", "nearest"]:
-            """
-            To maintain the determinism, the value of kernel_size and stride should be the same.
-            (you can check this link for reference: https://github.com/Project-MONAI/MONAI/pull/815 )
-            """
+            # To maintain the determinism, the value of kernel_size and stride should be the same.
+            # (you can check this link for reference: https://github.com/Project-MONAI/MONAI/pull/815 )
             self.maxpool = pool_type(kernel_size=(2, 2, 2)[-spatial_dims:], stride=2)
         else:
             self.maxpool = pool_type(kernel_size=(3, 3, 3)[-spatial_dims:], stride=2, padding=1)

monai/networks/nets/dynunet.py

@@ -181,8 +181,8 @@ def check_kernel_stride(self):
         if not (len(kernels) == len(strides) and len(kernels) >= 3):
             raise AssertionError(error_msg)
 
-        for idx in range(len(kernels)):
-            kernel, stride = kernels[idx], strides[idx]
+        for idx, k_i in enumerate(kernels):
+            kernel, stride = k_i, strides[idx]
             if not isinstance(kernel, int):
                 error_msg = "length of kernel_size in block {} should be the same as spatial_dims.".format(idx)
                 if len(kernel) != self.spatial_dims:

monai/networks/nets/efficientnet.py

@@ -702,8 +702,7 @@ def _make_same_padder(conv_op: Union[nn.Conv1d, nn.Conv2d, nn.Conv3d], image_siz
     padder = Pad["constantpad", len(padding) // 2]
     if sum(padding) > 0:
         return padder(padding=padding, value=0.0)
-    else:
-        return nn.Identity()
+    return nn.Identity()
 
 
 def _round_filters(filters: int, width_coefficient: Optional[float], depth_divisor: float) -> int:
@@ -763,8 +762,6 @@ def _calculate_output_image_size(input_image_size: List[int], stride: Union[int,
     Returns:
         output_image_size: output image/feature spatial size.
     """
-    # get number of spatial dimensions, corresponds to image spatial size length
-    num_dims = len(input_image_size)
 
     # checks to extract integer stride in case tuple was received
     if isinstance(stride, tuple):

monai/transforms/croppad/array.py

@@ -78,11 +78,11 @@ def _determine_data_pad_width(self, data_shape: Sequence[int]) -> List[Tuple[int
         self.spatial_size = fall_back_tuple(self.spatial_size, data_shape)
         if self.method == Method.SYMMETRIC:
             pad_width = []
-            for i in range(len(self.spatial_size)):
-                width = max(self.spatial_size[i] - data_shape[i], 0)
+            for i, sp_i in enumerate(self.spatial_size):
+                width = max(sp_i - data_shape[i], 0)
                 pad_width.append((width // 2, width - (width // 2)))
             return pad_width
-        return [(0, max(self.spatial_size[i] - data_shape[i], 0)) for i in range(len(self.spatial_size))]
+        return [(0, max(sp_i - data_shape[i], 0)) for i, sp_i in enumerate(self.spatial_size)]
 
     def __call__(self, img: np.ndarray, mode: Optional[Union[NumpyPadMode, str]] = None) -> np.ndarray:
         """

monai/transforms/croppad/batch.py

@@ -99,10 +99,10 @@ def __call__(self, batch: Any):
             padder = SpatialPad(max_shape, self.method, self.mode)  # type: ignore
             transform = padder if not output_to_tensor else Compose([padder, ToTensor()])
 
-            for idx in range(len(batch)):
-                im = batch[idx][key_or_idx]
+            for idx, batch_i in enumerate(batch):
+                im = batch_i[key_or_idx]
                 orig_size = im.shape[1:]
-                padded = transform(batch[idx][key_or_idx])
+                padded = transform(batch_i[key_or_idx])
                 batch = replace_element(padded, batch, idx, key_or_idx)
 
                 # If we have a dictionary of data, append to list

monai/transforms/utility/array.py

@@ -777,7 +777,7 @@ def __init__(self, orig_labels: Sequence, target_labels: Sequence, dtype: DtypeL
         """
         if len(orig_labels) != len(target_labels):
             raise ValueError("orig_labels and target_labels must have the same length.")
-        if all([o == z for o, z in zip(orig_labels, target_labels)]):
+        if all(o == z for o, z in zip(orig_labels, target_labels)):
             raise ValueError("orig_labels and target_labels are exactly the same, should be different to map.")
 
         self.orig_labels = orig_labels

monai/transforms/utils.py

@@ -339,8 +339,10 @@ def generate_pos_neg_label_crop_centers(
     valid_end = np.subtract(label_spatial_shape + np.array(1), spatial_size / np.array(2)).astype(np.uint16)
     # int generation to have full range on upper side, but subtract unfloored size/2 to prevent rounded range
     # from being too high
-    for i in range(len(valid_start)):  # need this because np.random.randint does not work with same start and end
-        if valid_start[i] == valid_end[i]:
+    for i, valid_s in enumerate(
+        valid_start
+    ):  # need this because np.random.randint does not work with same start and end
+        if valid_s == valid_end[i]:
             valid_end[i] += 1
 
     def _correct_centers(

monai/utils/jupyter_utils.py

@@ -342,5 +342,5 @@ def plot_status(self, logger, plot_func: Callable = plot_engine_status):
         which holds the internal lock during the plot generation.
         """
         with self.lock:
-            self.fig, axes = plot_func(title=self.status(), engine=self.engine, logger=logger, fig=self.fig)
+            self.fig, _ = plot_func(title=self.status(), engine=self.engine, logger=logger, fig=self.fig)
             return self.fig

monai/visualize/occlusion_sensitivity.py

@@ -299,15 +299,14 @@ def __call__(  # type: ignore
             sensitivity_ims_list, output_im_shape = self._compute_occlusion_sensitivity(x, b_box)
 
             # Loop over image for each classification
-            for i in range(len(sensitivity_ims_list)):
-
+            for i, sens_i in enumerate(sensitivity_ims_list):
                 # upsample
                 if self.upsampler is not None:
-                    if len(sensitivity_ims_list[i].shape) != len(x.shape):
+                    if len(sens_i.shape) != len(x.shape):
                         raise AssertionError
-                    if np.any(sensitivity_ims_list[i].shape != x.shape):
+                    if np.any(sens_i.shape != x.shape):
                         img_spatial = tuple(output_im_shape[1:])
-                        sensitivity_ims_list[i] = self.upsampler(img_spatial)(sensitivity_ims_list[i])
+                        sensitivity_ims_list[i] = self.upsampler(img_spatial)(sens_i)
 
             # Convert list of tensors to tensor
             sensitivity_ims = torch.stack(sensitivity_ims_list, dim=-1)

tests/test_affine_transform.py

@@ -311,7 +311,7 @@ def test_ill_affine_transform(self):
         with self.assertRaises(RuntimeError):  # dtype doesn't match
             affine = torch.as_tensor([[2.0, 0.0, 0.0], [0.0, 2.0, 0.0]], dtype=torch.float64)
             image = torch.arange(1.0, 13.0).view(1, 1, 3, 4).to(device=torch.device("cpu:0"))
-            out = AffineTransform((1, 2))(image, affine)
+            AffineTransform((1, 2))(image, affine)
 
     def test_forward_2d(self):
         x = torch.rand(2, 1, 4, 4)

tests/test_autoencoder.py

@@ -98,7 +98,7 @@ def test_script(self):
 
     def test_channel_stride_difference(self):
         with self.assertRaises(ValueError):
-            net = AutoEncoder(**TEST_CASE_FAIL)
+            AutoEncoder(**TEST_CASE_FAIL)
 
 
 if __name__ == "__main__":

tests/test_dice_focal_loss.py

@@ -67,7 +67,7 @@ def test_ill_shape(self):
 
     def test_ill_lambda(self):
         with self.assertRaisesRegex(ValueError, ""):
-            loss = DiceFocalLoss(lambda_dice=-1.0)
+            DiceFocalLoss(lambda_dice=-1.0)
 
     @SkipIfBeforePyTorchVersion((1, 7, 0))
     def test_script(self):

tests/test_focal_loss.py

@@ -178,8 +178,6 @@ def test_ill_opts(self):
         chn_target = torch.ones((1, 2, 3))
         with self.assertRaisesRegex(ValueError, ""):
             FocalLoss(reduction="unknown")(chn_input, chn_target)
-        with self.assertRaisesRegex(TypeError, ""):
-            FocalLoss(other_act="tanh")(chn_input, chn_target)
 
     def test_ill_shape(self):
         chn_input = torch.ones((1, 2, 3))

tests/test_handler_prob_map_producer.py

@@ -30,14 +30,16 @@
 
 class TestDataset(Dataset):
     def __init__(self, name, size):
-        self.data = [
-            {
-                "name": name,
-                "mask_shape": (size, size),
-                "mask_locations": [[i, i] for i in range(size)],
-                "level": 0,
-            }
-        ]
+        super().__init__(
+            data=[
+                {
+                    "name": name,
+                    "mask_shape": (size, size),
+                    "mask_locations": [[i, i] for i in range(size)],
+                    "level": 0,
+                }
+            ]
+        )
         self.len = size
 
     def __len__(self):

tests/test_lmdbdataset.py

@@ -10,6 +10,7 @@
 # limitations under the License.
 
 import os
+import shutil
 import tempfile
 import unittest
 
@@ -19,7 +20,7 @@
 
 from monai.data import LMDBDataset, json_hashing
 from monai.transforms import Compose, LoadImaged, SimulateDelayd, Transform
-from tests.utils import skip_if_windows
+from tests.utils import DistCall, DistTestCase, skip_if_windows
 
 TEST_CASE_1 = [
     Compose(
@@ -78,20 +79,21 @@
 ]
 
 
+class _InplaceXform(Transform):
+    def __call__(self, data):
+        if data:
+            data[0] = data[0] + np.pi
+        else:
+            data.append(1)
+        return data
+
+
 @skip_if_windows
 class TestLMDBDataset(unittest.TestCase):
     def test_cache(self):
         """testing no inplace change to the hashed item"""
         items = [[list(range(i))] for i in range(5)]
 
-        class _InplaceXform(Transform):
-            def __call__(self, data):
-                if data:
-                    data[0] = data[0] + np.pi
-                else:
-                    data.append(1)
-                return data
-
         with tempfile.TemporaryDirectory() as tempdir:
             ds = LMDBDataset(items, transform=_InplaceXform(), cache_dir=tempdir, lmdb_kwargs={"map_size": 10 * 1024})
             self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
@@ -177,5 +179,44 @@ def test_shape(self, transform, expected_shape, kwargs=None):
             self.assertTupleEqual(data2_postcached["extra"].shape, expected_shape)
 
 
+@skip_if_windows
+class TestMPLMDBDataset(DistTestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
+    @DistCall(nnodes=1, nproc_per_node=2)
+    def test_mp_cache(self):
+        items = [[list(range(i))] for i in range(5)]
+
+        ds = LMDBDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir, lmdb_kwargs={"map_size": 10 * 1024})
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+        ds1 = LMDBDataset(items, transform=_InplaceXform(), cache_dir=self.tempdir, lmdb_kwargs={"map_size": 10 * 1024})
+        self.assertEqual(list(ds1), list(ds))
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+
+        ds = LMDBDataset(
+            items,
+            transform=_InplaceXform(),
+            cache_dir=self.tempdir,
+            lmdb_kwargs={"map_size": 10 * 1024},
+            hash_func=json_hashing,
+        )
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+        ds1 = LMDBDataset(
+            items,
+            transform=_InplaceXform(),
+            cache_dir=self.tempdir,
+            lmdb_kwargs={"map_size": 10 * 1024},
+            hash_func=json_hashing,
+        )
+        self.assertEqual(list(ds1), list(ds))
+        self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]])
+
+        self.assertTrue(isinstance(ds1.info(), dict))
+
+
 if __name__ == "__main__":
     unittest.main()