From 69aa7cac89f89dd2faf3b6d4e45b2b7d2b2420e0 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 7 Jul 2021 13:23:22 +0100 Subject: [PATCH 1/7] add use_deterministic_algorithms flag Signed-off-by: Wenqi Li --- monai/utils/__init__.py | 2 +- monai/utils/misc.py | 26 ++++++++++++++++++++++++-- monai/utils/module.py | 16 +--------------- tests/test_set_determinism.py | 17 ++++++++++++++++- 4 files changed, 42 insertions(+), 19 deletions(-) diff --git a/monai/utils/__init__.py b/monai/utils/__init__.py index 8f3869b95f..b0f3300202 100644 --- a/monai/utils/__init__.py +++ b/monai/utils/__init__.py @@ -48,6 +48,7 @@ fall_back_tuple, first, get_seed, + has_option, is_scalar, is_scalar_tensor, issequenceiterable, @@ -66,7 +67,6 @@ get_full_type_name, get_package_version, get_torch_version_tuple, - has_option, load_submodules, min_version, optional_import, diff --git a/monai/utils/misc.py b/monai/utils/misc.py index 84420e2887..84c2848804 100644 --- a/monai/utils/misc.py +++ b/monai/utils/misc.py @@ -22,6 +22,8 @@ import numpy as np import torch +from monai.utils.module import get_torch_version_tuple + __all__ = [ "zip_with", "star_zip_with", @@ -214,6 +216,7 @@ def get_seed() -> Optional[int]: def set_determinism( seed: Optional[int] = np.iinfo(np.uint32).max, + use_deterministic_algorithms: [bool] = False, additional_settings: Optional[Union[Sequence[Callable[[int], Any]], Callable[[int], Any]]] = None, ) -> None: """ @@ -224,8 +227,8 @@ def set_determinism( It is recommended to set a large seed, i.e. a number that has a good balance of 0 and 1 bits. Avoid having many 0 bits in the seed. if set to None, will disable deterministic training. - additional_settings: additional settings - that need to set random seed. + use_deterministic_algorithms: Set whether PyTorch operations must use "deterministic" algorithms. + additional_settings: additional settings that need to set random seed. """ if seed is None: @@ -254,6 +257,15 @@ def set_determinism( torch.backends.cudnn.deterministic = _flag_deterministic torch.backends.cudnn.benchmark = _flag_cudnn_benchmark + if use_deterministic_algorithms: + torch_ver = get_torch_version_tuple() + if torch_ver >= (1, 9, 0): + torch.use_deterministic_algorithms(True) + elif torch_ver >= (1, 7, 0): + torch.set_deterministic(True) + else: + warnings.warn("use_deterministic_algorithms=True, but PyTorch version is too low to set the mode.") + def list_to_dict(items): """ @@ -359,3 +371,13 @@ class ImageMetaKey: FILENAME_OR_OBJ = "filename_or_obj" PATCH_INDEX = "patch_index" + + +def has_option(obj, keywords: Union[str, Sequence[str]]) -> bool: + """ + Return a boolean indicating whether the given callable `obj` has the `keywords` in its signature. + """ + if not callable(obj): + return False + sig = inspect.signature(obj) + return all(key in sig.parameters for key in ensure_tuple(keywords)) diff --git a/monai/utils/module.py b/monai/utils/module.py index f6d7687bb6..2ccea2f05f 100644 --- a/monai/utils/module.py +++ b/monai/utils/module.py @@ -9,18 +9,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import inspect import sys import warnings from importlib import import_module from pkgutil import walk_packages from re import match -from typing import Any, Callable, List, Sequence, Tuple, Union +from typing import Any, Callable, List, Tuple import torch -from .misc import ensure_tuple - OPTIONAL_IMPORT_MSG_FMT = "{}" __all__ = [ @@ -32,7 +29,6 @@ "optional_import", "load_submodules", "get_full_type_name", - "has_option", "get_package_version", "get_torch_version_tuple", "PT_BEFORE_1_7", @@ -243,16 +239,6 @@ def __call__(self, *_args, **_kwargs): return _LazyRaise(), False -def has_option(obj, keywords: Union[str, Sequence[str]]) -> bool: - """ - Return a boolean indicating whether the given callable `obj` has the `keywords` in its signature. - """ - if not callable(obj): - return False - sig = inspect.signature(obj) - return all(key in sig.parameters for key in ensure_tuple(keywords)) - - def get_package_version(dep_name, default="NOT INSTALLED or UNKNOWN VERSION."): """ Try to load package and get version. If not found, return `default`. diff --git a/tests/test_set_determinism.py b/tests/test_set_determinism.py index bc4927007b..d35daaf9ee 100644 --- a/tests/test_set_determinism.py +++ b/tests/test_set_determinism.py @@ -14,7 +14,8 @@ import numpy as np import torch -from monai.utils import get_seed, set_determinism +from monai.utils import get_seed, get_torch_version_tuple, set_determinism +from tests.utils import SkipIfBeforePyTorchVersion, skip_if_no_cuda class TestSetDeterminism(unittest.TestCase): @@ -48,6 +49,20 @@ def test_values(self): self.assertTrue(not torch.backends.cudnn.benchmark) set_determinism(seed=None) + @SkipIfBeforePyTorchVersion((1, 7)) + @skip_if_no_cuda + def test_algo_flag(self): + if get_torch_version_tuple() <= (1, 8, 0): + current_val = torch.is_deterministic() + else: + current_val = torch.are_deterministic_algorithms_enabled() + set_determinism(1, use_deterministic_algorithms=True) + with self.assertRaises(RuntimeError): + torch.randn(10, requires_grad=True, device="cuda").index_select( + 0, torch.tensor([0], device="cuda") + ).backward() + set_determinism(None, use_deterministic_algorithms=current_val) + if __name__ == "__main__": unittest.main() From b34fb7abaaad7caa80d6b76c0089f796cec35259 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 7 Jul 2021 13:26:56 +0100 Subject: [PATCH 2/7] fixes typing Signed-off-by: Wenqi Li --- monai/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/utils/misc.py b/monai/utils/misc.py index 84c2848804..b09cf2df93 100644 --- a/monai/utils/misc.py +++ b/monai/utils/misc.py @@ -216,7 +216,7 @@ def get_seed() -> Optional[int]: def set_determinism( seed: Optional[int] = np.iinfo(np.uint32).max, - use_deterministic_algorithms: [bool] = False, + use_deterministic_algorithms: bool = False, additional_settings: Optional[Union[Sequence[Callable[[int], Any]], Callable[[int], Any]]] = None, ) -> None: """ From 9531643090a55b3a40b495fab473e0cec6e1acc0 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 7 Jul 2021 20:16:58 +0100 Subject: [PATCH 3/7] update tests Signed-off-by: Wenqi Li --- tests/test_set_determinism.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/test_set_determinism.py b/tests/test_set_determinism.py index d35daaf9ee..d96f0a1f0e 100644 --- a/tests/test_set_determinism.py +++ b/tests/test_set_determinism.py @@ -52,16 +52,12 @@ def test_values(self): @SkipIfBeforePyTorchVersion((1, 7)) @skip_if_no_cuda def test_algo_flag(self): - if get_torch_version_tuple() <= (1, 8, 0): - current_val = torch.is_deterministic() - else: - current_val = torch.are_deterministic_algorithms_enabled() set_determinism(1, use_deterministic_algorithms=True) with self.assertRaises(RuntimeError): - torch.randn(10, requires_grad=True, device="cuda").index_select( - 0, torch.tensor([0], device="cuda") - ).backward() - set_determinism(None, use_deterministic_algorithms=current_val) + x = torch.randn(20, 16, 50, 44, 31, requires_grad=True, device="cuda:0") + y = torch.nn.AvgPool3d((3, 2, 2), stride=(2, 1, 2))(x) + y.sum().backward() + set_determinism(None, use_deterministic_algorithms=False) if __name__ == "__main__": From 9d3ab1489d81044bc382c1d2ca5cd51f619f3c01 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 7 Jul 2021 21:00:04 +0100 Subject: [PATCH 4/7] fixes tests Signed-off-by: Wenqi Li --- monai/utils/misc.py | 4 ++-- tests/test_set_determinism.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/monai/utils/misc.py b/monai/utils/misc.py index b09cf2df93..fdf4dfe663 100644 --- a/monai/utils/misc.py +++ b/monai/utils/misc.py @@ -259,9 +259,9 @@ def set_determinism( if use_deterministic_algorithms: torch_ver = get_torch_version_tuple() - if torch_ver >= (1, 9, 0): + if torch_ver >= (1, 9): torch.use_deterministic_algorithms(True) - elif torch_ver >= (1, 7, 0): + elif torch_ver >= (1, 7): torch.set_deterministic(True) else: warnings.warn("use_deterministic_algorithms=True, but PyTorch version is too low to set the mode.") diff --git a/tests/test_set_determinism.py b/tests/test_set_determinism.py index d96f0a1f0e..58fbc05fe5 100644 --- a/tests/test_set_determinism.py +++ b/tests/test_set_determinism.py @@ -14,7 +14,7 @@ import numpy as np import torch -from monai.utils import get_seed, get_torch_version_tuple, set_determinism +from monai.utils import get_seed, set_determinism from tests.utils import SkipIfBeforePyTorchVersion, skip_if_no_cuda @@ -49,7 +49,7 @@ def test_values(self): self.assertTrue(not torch.backends.cudnn.benchmark) set_determinism(seed=None) - @SkipIfBeforePyTorchVersion((1, 7)) + @SkipIfBeforePyTorchVersion((1, 6)) @skip_if_no_cuda def test_algo_flag(self): set_determinism(1, use_deterministic_algorithms=True) From 0e5cdd12af3d0a32cb806046582451d9e592a895 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 7 Jul 2021 21:05:17 +0100 Subject: [PATCH 5/7] fixes tests Signed-off-by: Wenqi Li --- monai/utils/misc.py | 4 ++-- tests/test_set_determinism.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monai/utils/misc.py b/monai/utils/misc.py index fdf4dfe663..f4d840e88d 100644 --- a/monai/utils/misc.py +++ b/monai/utils/misc.py @@ -262,9 +262,9 @@ def set_determinism( if torch_ver >= (1, 9): torch.use_deterministic_algorithms(True) elif torch_ver >= (1, 7): - torch.set_deterministic(True) + torch.set_deterministic(True) # beta feature else: - warnings.warn("use_deterministic_algorithms=True, but PyTorch version is too low to set the mode.") + warnings.warn("use_deterministic_algorithms=True, but PyTorch version is too old to set the mode.") def list_to_dict(items): diff --git a/tests/test_set_determinism.py b/tests/test_set_determinism.py index 58fbc05fe5..f7690f7e74 100644 --- a/tests/test_set_determinism.py +++ b/tests/test_set_determinism.py @@ -49,7 +49,7 @@ def test_values(self): self.assertTrue(not torch.backends.cudnn.benchmark) set_determinism(seed=None) - @SkipIfBeforePyTorchVersion((1, 6)) + @SkipIfBeforePyTorchVersion((1, 8)) @skip_if_no_cuda def test_algo_flag(self): set_determinism(1, use_deterministic_algorithms=True) From bdb71533798b6a0f7e2465f347330d452cba9327 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 7 Jul 2021 21:13:16 +0100 Subject: [PATCH 6/7] comment Signed-off-by: Wenqi Li --- tests/test_set_determinism.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_set_determinism.py b/tests/test_set_determinism.py index f7690f7e74..63e5d7005b 100644 --- a/tests/test_set_determinism.py +++ b/tests/test_set_determinism.py @@ -49,7 +49,7 @@ def test_values(self): self.assertTrue(not torch.backends.cudnn.benchmark) set_determinism(seed=None) - @SkipIfBeforePyTorchVersion((1, 8)) + @SkipIfBeforePyTorchVersion((1, 8)) # beta feature @skip_if_no_cuda def test_algo_flag(self): set_determinism(1, use_deterministic_algorithms=True) From 014d1768b93b831d9dfc5282619c068969c9a36a Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 7 Jul 2021 22:37:01 +0100 Subject: [PATCH 7/7] update tests Signed-off-by: Wenqi Li --- monai/utils/misc.py | 8 ++++---- tests/test_set_determinism.py | 10 ++++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/monai/utils/misc.py b/monai/utils/misc.py index f4d840e88d..e42b144a74 100644 --- a/monai/utils/misc.py +++ b/monai/utils/misc.py @@ -216,7 +216,7 @@ def get_seed() -> Optional[int]: def set_determinism( seed: Optional[int] = np.iinfo(np.uint32).max, - use_deterministic_algorithms: bool = False, + use_deterministic_algorithms: Optional[bool] = None, additional_settings: Optional[Union[Sequence[Callable[[int], Any]], Callable[[int], Any]]] = None, ) -> None: """ @@ -257,12 +257,12 @@ def set_determinism( torch.backends.cudnn.deterministic = _flag_deterministic torch.backends.cudnn.benchmark = _flag_cudnn_benchmark - if use_deterministic_algorithms: + if use_deterministic_algorithms is not None: torch_ver = get_torch_version_tuple() if torch_ver >= (1, 9): - torch.use_deterministic_algorithms(True) + torch.use_deterministic_algorithms(use_deterministic_algorithms) elif torch_ver >= (1, 7): - torch.set_deterministic(True) # beta feature + torch.set_deterministic(use_deterministic_algorithms) # beta feature else: warnings.warn("use_deterministic_algorithms=True, but PyTorch version is too old to set the mode.") diff --git a/tests/test_set_determinism.py b/tests/test_set_determinism.py index 63e5d7005b..537aa36676 100644 --- a/tests/test_set_determinism.py +++ b/tests/test_set_determinism.py @@ -49,14 +49,20 @@ def test_values(self): self.assertTrue(not torch.backends.cudnn.benchmark) set_determinism(seed=None) + +class TestSetFlag(unittest.TestCase): + def setUp(self): + set_determinism(1, use_deterministic_algorithms=True) + @SkipIfBeforePyTorchVersion((1, 8)) # beta feature @skip_if_no_cuda - def test_algo_flag(self): - set_determinism(1, use_deterministic_algorithms=True) + def test_algo(self): with self.assertRaises(RuntimeError): x = torch.randn(20, 16, 50, 44, 31, requires_grad=True, device="cuda:0") y = torch.nn.AvgPool3d((3, 2, 2), stride=(2, 1, 2))(x) y.sum().backward() + + def tearDown(self): set_determinism(None, use_deterministic_algorithms=False)