From 102a550056d3755e3b9c59d646fb58d322f07ddd Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 28 Apr 2021 13:49:07 +0800 Subject: [PATCH 1/4] [DLMED] Enhance unit test and doc-string Signed-off-by: Nic Ma --- monai/data/dataset.py | 6 ++++-- tests/test_handler_smartcache.py | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index bfb3d8b86d..c29e44c70e 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -614,8 +614,10 @@ class SmartCacheDataset(Randomizable, CacheDataset): 4. Call `shutdown()` when training ends. Note: - This replacement will not work if setting the `multiprocessing_context` of DataLoader to `spawn` - or on windows(the default multiprocessing method is `spawn`) and setting `num_workers` greater than 0. + This replacement will not work for below cases: + 1. Set the `multiprocessing_context` of DataLoader to `spawn`. + 2. Run on windows(the default multiprocessing method is `spawn`) with `num_workers` greater than 0. + 3. Set `persistent_workers=True` in the PyTorch DataLoader. If using MONAI workflows, please add `SmartCacheHandler` to the handler list of trainer, otherwise, please make sure to call `start()`, `update_cache()`, `shutdown()` during training. diff --git a/tests/test_handler_smartcache.py b/tests/test_handler_smartcache.py index cfe68e98e2..5ff3a3cfb2 100644 --- a/tests/test_handler_smartcache.py +++ b/tests/test_handler_smartcache.py @@ -10,7 +10,7 @@ # limitations under the License. import unittest - +import sys import torch from ignite.engine import Engine @@ -37,7 +37,8 @@ def _train_func(engine, batch): # set up testing handler dataset = SmartCacheDataset(data, transform=None, replace_rate=0.2, cache_num=5, shuffle=False) - data_loader = torch.utils.data.DataLoader(dataset, batch_size=5) + workers = 2 if sys.platform != "darwin" else 0 + data_loader = torch.utils.data.DataLoader(dataset, batch_size=5, num_workers=workers, persistent_workers=False) SmartCacheHandler(dataset).attach(engine) engine.run(data_loader, max_epochs=5) From 0b046acad3b03e04dfc5e6aea1c0483ceb34f1f1 Mon Sep 17 00:00:00 2001 From: monai-bot Date: Wed, 28 Apr 2021 05:55:44 +0000 Subject: [PATCH 2/4] [MONAI] python code formatting Signed-off-by: monai-bot --- tests/test_handler_smartcache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_handler_smartcache.py b/tests/test_handler_smartcache.py index 5ff3a3cfb2..8947e5810f 100644 --- a/tests/test_handler_smartcache.py +++ b/tests/test_handler_smartcache.py @@ -9,8 +9,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import sys +import unittest + import torch from ignite.engine import Engine From 2dd616802cf9b0dca3d6cab28a9ada1596844091 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 28 Apr 2021 14:35:38 +0800 Subject: [PATCH 3/4] [DLMED] enhance unit test Signed-off-by: Nic Ma --- monai/data/dataset.py | 2 +- tests/test_handler_smartcache.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index c29e44c70e..b8ecd2b35a 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -617,7 +617,7 @@ class SmartCacheDataset(Randomizable, CacheDataset): This replacement will not work for below cases: 1. Set the `multiprocessing_context` of DataLoader to `spawn`. 2. Run on windows(the default multiprocessing method is `spawn`) with `num_workers` greater than 0. - 3. Set `persistent_workers=True` in the PyTorch DataLoader. + 3. Set the `persistent_workers` of DataLoader to `True` with `num_workers` greater than 0. If using MONAI workflows, please add `SmartCacheHandler` to the handler list of trainer, otherwise, please make sure to call `start()`, `update_cache()`, `shutdown()` during training. diff --git a/tests/test_handler_smartcache.py b/tests/test_handler_smartcache.py index 8947e5810f..181f237283 100644 --- a/tests/test_handler_smartcache.py +++ b/tests/test_handler_smartcache.py @@ -38,7 +38,7 @@ def _train_func(engine, batch): # set up testing handler dataset = SmartCacheDataset(data, transform=None, replace_rate=0.2, cache_num=5, shuffle=False) - workers = 2 if sys.platform != "darwin" else 0 + workers = 2 if sys.platform == "linux" else 0 data_loader = torch.utils.data.DataLoader(dataset, batch_size=5, num_workers=workers, persistent_workers=False) SmartCacheHandler(dataset).attach(engine) From c1ff5a71e12545e983869053ec7cc3e56ea40026 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 28 Apr 2021 15:16:33 +0800 Subject: [PATCH 4/4] [DLMED] fix CI test issue Signed-off-by: Nic Ma --- tests/test_handler_smartcache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_handler_smartcache.py b/tests/test_handler_smartcache.py index 181f237283..b67f1226cd 100644 --- a/tests/test_handler_smartcache.py +++ b/tests/test_handler_smartcache.py @@ -17,8 +17,10 @@ from monai.data import SmartCacheDataset from monai.handlers import SmartCacheHandler +from tests.utils import SkipIfBeforePyTorchVersion +@SkipIfBeforePyTorchVersion((1, 7)) class TestHandlerSmartCache(unittest.TestCase): def test_content(self): data = [0, 1, 2, 3, 4, 5, 6, 7, 8]