From e68dec04e906efeda29b86f762dc91ce9c340ff1 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 13 Oct 2022 14:23:36 -0400 Subject: [PATCH 01/18] adding a multigpu tests Signed-off-by: Wenqi Li --- tests/test_fl_monai_algo_dist.py | 76 ++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 tests/test_fl_monai_algo_dist.py diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py new file mode 100644 index 0000000000..ecb6ecfc47 --- /dev/null +++ b/tests/test_fl_monai_algo_dist.py @@ -0,0 +1,76 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import unittest +import torch.distributed as dist + +from parameterized import parameterized + +from monai.bundle import ConfigParser +from monai.fl.client.monai_algo import MonaiAlgo +from monai.fl.utils.constants import ExtraItems +from monai.fl.utils.exchange_object import ExchangeObject +from tests.utils import SkipIfNoModule, DistCall, DistTestCase + +TEST_TRAIN_1 = [ + { + "bundle_root": os.path.join(os.path.dirname(__file__)), + "config_train_filename": os.path.join("testing_data", "config_fl_train.json"), + "config_evaluate_filename": None, + "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"), + "multi_gpu": True, + } +] + + +@SkipIfNoModule("ignite") +class TestFLMonaiAlgo(DistTestCase): + def setUp(self): + for input_params in TEST_TRAIN_1: + # get testing data dir and update train config; using the first to define data dir + if isinstance(input_params["config_train_filename"], list): + config_train_filename = input_params["config_train_filename"][0] + else: + config_train_filename = input_params["config_train_filename"] + with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f: + config_train = json.load(f) + + config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data") + + with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f: + json.dump(config_train, f, indent=4) + + + @parameterized.expand([TEST_TRAIN_1]) + @DistCall(nnodes=1, nproc_per_node=2) + def test_train(self, input_params): + # initialize algo + print(f"rank {dist.get_rank()}") + algo = MonaiAlgo(**input_params) + algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"}) + + # initialize model + parser = ConfigParser() + parser.read_config(os.path.join(input_params["bundle_root"], config_train_filename)) + parser.parse() + network = parser.get_parsed_content("network") + + data = ExchangeObject(weights=network.state_dict()) + + # test train + algo.train(data=data, extra={}) + + + +if __name__ == "__main__": + unittest.main() From da308e87de7dd74f36378f4a21131732f553ad15 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 13 Oct 2022 22:11:47 +0100 Subject: [PATCH 02/18] update to remove addchannel Signed-off-by: Wenqi Li --- tests/testing_data/config_fl_evaluate.json | 2 +- tests/testing_data/config_fl_train.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testing_data/config_fl_evaluate.json b/tests/testing_data/config_fl_evaluate.json index 84ab7988f6..89cebc67c3 100644 --- a/tests/testing_data/config_fl_evaluate.json +++ b/tests/testing_data/config_fl_evaluate.json @@ -22,7 +22,7 @@ "image_only": true }, { - "_target_": "AddChanneld", + "_target_": "EnsureChannelFirstD", "keys": [ "image" ] diff --git a/tests/testing_data/config_fl_train.json b/tests/testing_data/config_fl_train.json index 5954d2cfbc..067b25a952 100644 --- a/tests/testing_data/config_fl_train.json +++ b/tests/testing_data/config_fl_train.json @@ -30,7 +30,7 @@ "image_only": true }, { - "_target_": "AddChanneld", + "_target_": "EnsureChannelFirstD", "keys": [ "image" ] From 656f5f32317cfd8cb269960d57b22d434c909bf4 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 13 Oct 2022 22:13:37 +0100 Subject: [PATCH 03/18] update tests Signed-off-by: Wenqi Li --- tests/test_fl_monai_algo_dist.py | 34 ++++++++++++++------------------ 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py index ecb6ecfc47..3bd37ebc6d 100644 --- a/tests/test_fl_monai_algo_dist.py +++ b/tests/test_fl_monai_algo_dist.py @@ -12,15 +12,15 @@ import json import os import unittest -import torch.distributed as dist +import torch.distributed as dist from parameterized import parameterized from monai.bundle import ConfigParser from monai.fl.client.monai_algo import MonaiAlgo from monai.fl.utils.constants import ExtraItems from monai.fl.utils.exchange_object import ExchangeObject -from tests.utils import SkipIfNoModule, DistCall, DistTestCase +from tests.utils import DistCall, DistTestCase, SkipIfNoModule TEST_TRAIN_1 = [ { @@ -36,41 +36,37 @@ @SkipIfNoModule("ignite") class TestFLMonaiAlgo(DistTestCase): def setUp(self): - for input_params in TEST_TRAIN_1: - # get testing data dir and update train config; using the first to define data dir - if isinstance(input_params["config_train_filename"], list): - config_train_filename = input_params["config_train_filename"][0] - else: - config_train_filename = input_params["config_train_filename"] - with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f: - config_train = json.load(f) - - config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data") + input_params = TEST_TRAIN_1[0] + # get testing data dir and update train config; using the first to define data dir + if isinstance(input_params["config_train_filename"], list): + config_train_filename = input_params["config_train_filename"][0] + else: + config_train_filename = input_params["config_train_filename"] + with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f: + config_train = json.load(f) - with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f: - json.dump(config_train, f, indent=4) + config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data") + with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f: + json.dump(config_train, f, indent=4) @parameterized.expand([TEST_TRAIN_1]) @DistCall(nnodes=1, nproc_per_node=2) def test_train(self, input_params): # initialize algo - print(f"rank {dist.get_rank()}") algo = MonaiAlgo(**input_params) algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"}) + self.assertTrue(dist.get_rank() in (0, 1)) # initialize model parser = ConfigParser() - parser.read_config(os.path.join(input_params["bundle_root"], config_train_filename)) + parser.read_config(os.path.join(input_params["bundle_root"], input_params["config_train_filename"])) parser.parse() network = parser.get_parsed_content("network") - data = ExchangeObject(weights=network.state_dict()) - # test train algo.train(data=data, extra={}) - if __name__ == "__main__": unittest.main() From dc101c0b0b2476e31b1fba6423faa1c70a869b27 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 13 Oct 2022 22:15:34 +0100 Subject: [PATCH 04/18] update tests Signed-off-by: Wenqi Li --- tests/test_fl_monai_algo_dist.py | 2 +- tests/utils.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py index 3bd37ebc6d..a03d962b78 100644 --- a/tests/test_fl_monai_algo_dist.py +++ b/tests/test_fl_monai_algo_dist.py @@ -51,7 +51,7 @@ def setUp(self): json.dump(config_train, f, indent=4) @parameterized.expand([TEST_TRAIN_1]) - @DistCall(nnodes=1, nproc_per_node=2) + @DistCall(nnodes=1, nproc_per_node=2, init_method="no_init") def test_train(self, input_params): # initialize algo algo = MonaiAlgo(**input_params) diff --git a/tests/utils.py b/tests/utils.py index 1f632261dd..8b0748379e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -404,6 +404,7 @@ def __init__( timeout: Timeout for operations executed against the process group. init_method: URL specifying how to initialize the process group. Default is "env://" or "file:///d:/a_temp" (windows) if unspecified. + If ``"no_init"``, the `dist.init_process_group` must be called within the code to be tested. backend: The backend to use. Depending on build-time configurations, valid values include ``mpi``, ``gloo``, and ``nccl``. daemon: the process’s daemon flag. @@ -451,13 +452,14 @@ def run_process(self, func, local_rank, args, kwargs, results): if torch.cuda.is_available(): torch.cuda.set_device(int(local_rank)) # using device ids from CUDA_VISIBILE_DEVICES - dist.init_process_group( - backend=self.backend, - init_method=self.init_method, - timeout=self.timeout, - world_size=int(os.environ["WORLD_SIZE"]), - rank=int(os.environ["RANK"]), - ) + if init_method is not "no_init": + dist.init_process_group( + backend=self.backend, + init_method=self.init_method, + timeout=self.timeout, + world_size=int(os.environ["WORLD_SIZE"]), + rank=int(os.environ["RANK"]), + ) func(*args, **kwargs) # the primary node lives longer to # avoid _store_based_barrier, RuntimeError: Broken pipe From 47018e4a1fbc0e277006808864ac4e81033f2749 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 13 Oct 2022 22:16:45 +0100 Subject: [PATCH 05/18] update tests Signed-off-by: Wenqi Li --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index 8b0748379e..8e1c2a8f4a 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -452,7 +452,7 @@ def run_process(self, func, local_rank, args, kwargs, results): if torch.cuda.is_available(): torch.cuda.set_device(int(local_rank)) # using device ids from CUDA_VISIBILE_DEVICES - if init_method is not "no_init": + if self.init_method != "no_init": dist.init_process_group( backend=self.backend, init_method=self.init_method, From b379519c008baad936f1bb139615ac6e143371a5 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 13 Oct 2022 22:31:33 +0100 Subject: [PATCH 06/18] update Signed-off-by: Wenqi Li --- tests/test_fl_monai_algo_dist.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py index a03d962b78..850f93727b 100644 --- a/tests/test_fl_monai_algo_dist.py +++ b/tests/test_fl_monai_algo_dist.py @@ -9,7 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os import unittest @@ -22,12 +21,13 @@ from monai.fl.utils.exchange_object import ExchangeObject from tests.utils import DistCall, DistTestCase, SkipIfNoModule +_root_dir = os.path.join(os.path.dirname(__file__)) TEST_TRAIN_1 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), - "config_train_filename": os.path.join("testing_data", "config_fl_train.json"), + "bundle_root": _root_dir, + "config_train_filename": os.path.join(_root_dir, "testing_data", "config_fl_train.json"), "config_evaluate_filename": None, - "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"), + "config_filters_filename": os.path.join(_root_dir, "testing_data", "config_fl_filters.json"), "multi_gpu": True, } ] @@ -35,21 +35,6 @@ @SkipIfNoModule("ignite") class TestFLMonaiAlgo(DistTestCase): - def setUp(self): - input_params = TEST_TRAIN_1[0] - # get testing data dir and update train config; using the first to define data dir - if isinstance(input_params["config_train_filename"], list): - config_train_filename = input_params["config_train_filename"][0] - else: - config_train_filename = input_params["config_train_filename"] - with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f: - config_train = json.load(f) - - config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data") - - with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f: - json.dump(config_train, f, indent=4) - @parameterized.expand([TEST_TRAIN_1]) @DistCall(nnodes=1, nproc_per_node=2, init_method="no_init") def test_train(self, input_params): From f5f0128e49d290949d5e13a8d8d19da8a37c6f8e Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 13 Oct 2022 17:32:50 -0400 Subject: [PATCH 07/18] add train config Signed-off-by: Wenqi Li --- tests/testing_data/multi_gpu_train.json | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tests/testing_data/multi_gpu_train.json diff --git a/tests/testing_data/multi_gpu_train.json b/tests/testing_data/multi_gpu_train.json new file mode 100644 index 0000000000..4161e5274c --- /dev/null +++ b/tests/testing_data/multi_gpu_train.json @@ -0,0 +1,39 @@ +{ + "device": "$torch.device(f'cuda:{dist.get_rank()}')", + "network": { + "_target_": "torch.nn.parallel.DistributedDataParallel", + "module": "$@network_def.to(@device)", + "device_ids": [ + "@device" + ] + }, + "train#sampler": { + "_target_": "DistributedSampler", + "dataset": "@train#dataset", + "even_divisible": true, + "shuffle": true + }, + "train#dataloader#sampler": "@train#sampler", + "train#dataloader#shuffle": false, + "train#trainer#train_handlers": "$@train#handlers[: -2 if dist.get_rank() > 0 else None]", + "validate#sampler": { + "_target_": "DistributedSampler", + "dataset": "@validate#dataset", + "even_divisible": false, + "shuffle": false + }, + "validate#dataloader#sampler": "@validate#sampler", + "validate#evaluator#val_handlers": "$None if dist.get_rank() > 0 else @validate#handlers", + "training": [ + "$import torch.distributed as dist", + "$dist.init_process_group(backend='nccl')", + "$torch.cuda.set_device(@device)", + "$monai.utils.set_determinism(seed=123)", + "$setattr(torch.backends.cudnn, 'benchmark', True)", + "$import logging", + "$@train#trainer.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)", + "$@validate#evaluator.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)", + "$@train#trainer.run()", + "$dist.destroy_process_group()" + ] +} From 7ca2d0980c9ee888ab08ef822663cf8ca8fca62c Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Thu, 13 Oct 2022 17:57:13 -0400 Subject: [PATCH 08/18] update tests Signed-off-by: Wenqi Li --- tests/test_fl_monai_algo_dist.py | 37 +++++++++++++++++++--- tests/testing_data/multi_gpu_evaluate.json | 27 ++++++++++++++++ tests/testing_data/multi_gpu_train.json | 9 ------ 3 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 tests/testing_data/multi_gpu_evaluate.json diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py index 850f93727b..0886eac370 100644 --- a/tests/test_fl_monai_algo_dist.py +++ b/tests/test_fl_monai_algo_dist.py @@ -10,6 +10,7 @@ # limitations under the License. import os +from os.path import join as pathjoin import unittest import torch.distributed as dist @@ -21,13 +22,24 @@ from monai.fl.utils.exchange_object import ExchangeObject from tests.utils import DistCall, DistTestCase, SkipIfNoModule -_root_dir = os.path.join(os.path.dirname(__file__)) +_root_dir = pathjoin(os.path.dirname(__file__)) +_data_dir = pathjoin(_root_dir, "testing_data") TEST_TRAIN_1 = [ { "bundle_root": _root_dir, - "config_train_filename": os.path.join(_root_dir, "testing_data", "config_fl_train.json"), + "config_train_filename": [pathjoin(_data_dir, "config_fl_train.json"), pathjoin(_data_dir, "multi_gpu_train.json")], "config_evaluate_filename": None, - "config_filters_filename": os.path.join(_root_dir, "testing_data", "config_fl_filters.json"), + "config_filters_filename": pathjoin(_root_dir, "testing_data", "config_fl_filters.json"), + "multi_gpu": True, + } +] + +TEST_EVALUATE_1 = [ + { + "bundle_root": _root_dir, + "config_train_filename": None, + "config_evaluate_filename": [pathjoin(_data_dir, "config_fl_evaluate.json"), pathjoin(_data_dir, "multi_gpu_evaluate.json")], + "config_filters_filename": pathjoin(_data_dir, "config_fl_filters.json"), "multi_gpu": True, } ] @@ -45,13 +57,30 @@ def test_train(self, input_params): # initialize model parser = ConfigParser() - parser.read_config(os.path.join(input_params["bundle_root"], input_params["config_train_filename"])) + parser.read_config([pathjoin(input_params["bundle_root"], x) for x in input_params["config_train_filename"]]) parser.parse() network = parser.get_parsed_content("network") data = ExchangeObject(weights=network.state_dict()) # test train algo.train(data=data, extra={}) + @parameterized.expand([TEST_EVALUATE_1]) + @DistCall(nnodes=1, nproc_per_node=2, init_method="no_init") + def test_evaluate(self, input_params): + # initialize algo + algo = MonaiAlgo(**input_params) + algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"}) + self.assertTrue(dist.get_rank() in (0, 1)) + + # initialize model + parser = ConfigParser() + parser.read_config([os.path.join(input_params["bundle_root"], x) for x in input_params["config_evaluate_filename"]]) + parser.parse() + network = parser.get_parsed_content("network") + data = ExchangeObject(weights=network.state_dict()) + # test evaluate + algo.evaluate(data=data, extra={}) + if __name__ == "__main__": unittest.main() diff --git a/tests/testing_data/multi_gpu_evaluate.json b/tests/testing_data/multi_gpu_evaluate.json new file mode 100644 index 0000000000..7af24a6b2e --- /dev/null +++ b/tests/testing_data/multi_gpu_evaluate.json @@ -0,0 +1,27 @@ +{ + "device": "$torch.device(f'cuda:{dist.get_rank()}')", + "network": { + "_target_": "torch.nn.parallel.DistributedDataParallel", + "module": "$@network_def.to(@device)", + "device_ids": [ + "@device" + ] + }, + "validate#sampler": { + "_target_": "DistributedSampler", + "dataset": "@validate#dataset", + "even_divisible": false, + "shuffle": false + }, + "validate#dataloader#sampler": "@validate#sampler", + "evaluating": [ + "$import torch.distributed as dist", + "$dist.init_process_group(backend='nccl')", + "$torch.cuda.set_device(@device)", + "$setattr(torch.backends.cudnn, 'benchmark', True)", + "$import logging", + "$@validate#evaluator.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)", + "$@validate#evaluator.run()", + "$dist.destroy_process_group()" + ] +} diff --git a/tests/testing_data/multi_gpu_train.json b/tests/testing_data/multi_gpu_train.json index 4161e5274c..41fd7698db 100644 --- a/tests/testing_data/multi_gpu_train.json +++ b/tests/testing_data/multi_gpu_train.json @@ -16,14 +16,6 @@ "train#dataloader#sampler": "@train#sampler", "train#dataloader#shuffle": false, "train#trainer#train_handlers": "$@train#handlers[: -2 if dist.get_rank() > 0 else None]", - "validate#sampler": { - "_target_": "DistributedSampler", - "dataset": "@validate#dataset", - "even_divisible": false, - "shuffle": false - }, - "validate#dataloader#sampler": "@validate#sampler", - "validate#evaluator#val_handlers": "$None if dist.get_rank() > 0 else @validate#handlers", "training": [ "$import torch.distributed as dist", "$dist.init_process_group(backend='nccl')", @@ -32,7 +24,6 @@ "$setattr(torch.backends.cudnn, 'benchmark', True)", "$import logging", "$@train#trainer.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)", - "$@validate#evaluator.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)", "$@train#trainer.run()", "$dist.destroy_process_group()" ] From 47ae7b215a1e0a45dacc68e70585ca96bb86a355 Mon Sep 17 00:00:00 2001 From: monai-bot Date: Thu, 13 Oct 2022 22:03:21 +0000 Subject: [PATCH 09/18] [MONAI] code formatting Signed-off-by: monai-bot --- tests/test_fl_monai_algo_dist.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py index 0886eac370..ee4c0d81a3 100644 --- a/tests/test_fl_monai_algo_dist.py +++ b/tests/test_fl_monai_algo_dist.py @@ -10,8 +10,8 @@ # limitations under the License. import os -from os.path import join as pathjoin import unittest +from os.path import join as pathjoin import torch.distributed as dist from parameterized import parameterized @@ -27,7 +27,10 @@ TEST_TRAIN_1 = [ { "bundle_root": _root_dir, - "config_train_filename": [pathjoin(_data_dir, "config_fl_train.json"), pathjoin(_data_dir, "multi_gpu_train.json")], + "config_train_filename": [ + pathjoin(_data_dir, "config_fl_train.json"), + pathjoin(_data_dir, "multi_gpu_train.json"), + ], "config_evaluate_filename": None, "config_filters_filename": pathjoin(_root_dir, "testing_data", "config_fl_filters.json"), "multi_gpu": True, @@ -38,7 +41,10 @@ { "bundle_root": _root_dir, "config_train_filename": None, - "config_evaluate_filename": [pathjoin(_data_dir, "config_fl_evaluate.json"), pathjoin(_data_dir, "multi_gpu_evaluate.json")], + "config_evaluate_filename": [ + pathjoin(_data_dir, "config_fl_evaluate.json"), + pathjoin(_data_dir, "multi_gpu_evaluate.json"), + ], "config_filters_filename": pathjoin(_data_dir, "config_fl_filters.json"), "multi_gpu": True, } @@ -74,7 +80,9 @@ def test_evaluate(self, input_params): # initialize model parser = ConfigParser() - parser.read_config([os.path.join(input_params["bundle_root"], x) for x in input_params["config_evaluate_filename"]]) + parser.read_config( + [os.path.join(input_params["bundle_root"], x) for x in input_params["config_evaluate_filename"]] + ) parser.parse() network = parser.get_parsed_content("network") data = ExchangeObject(weights=network.state_dict()) From 5ad1b561ee4a002acf39eb3eb34c59f336b17437 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 14 Oct 2022 09:16:04 +0100 Subject: [PATCH 10/18] test abort finalize, no in-place config writting Signed-off-by: Wenqi Li --- tests/test_fl_monai_algo.py | 118 +++++++++------------ tests/testing_data/config_fl_evaluate.json | 2 +- tests/testing_data/config_fl_train.json | 2 +- 3 files changed, 50 insertions(+), 72 deletions(-) diff --git a/tests/test_fl_monai_algo.py b/tests/test_fl_monai_algo.py index 66722d0086..46e8c5efa3 100644 --- a/tests/test_fl_monai_algo.py +++ b/tests/test_fl_monai_algo.py @@ -9,7 +9,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json import os import unittest @@ -21,107 +20,110 @@ from monai.fl.utils.exchange_object import ExchangeObject from tests.utils import SkipIfNoModule +_root_dir = os.path.join(os.path.dirname(__file__)) +_data_dir = os.path.join(_root_dir, "testing_data") + TEST_TRAIN_1 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), - "config_train_filename": os.path.join("testing_data", "config_fl_train.json"), + "bundle_root": _data_dir, + "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"), "config_evaluate_filename": None, - "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"), + "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"), } ] TEST_TRAIN_2 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), - "config_train_filename": os.path.join("testing_data", "config_fl_train.json"), + "bundle_root": _data_dir, + "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"), "config_evaluate_filename": None, "config_filters_filename": None, } ] TEST_TRAIN_3 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), + "bundle_root": _data_dir, "config_train_filename": [ - os.path.join("testing_data", "config_fl_train.json"), - os.path.join("testing_data", "config_fl_train.json"), + os.path.join(_data_dir, "config_fl_train.json"), + os.path.join(_data_dir, "config_fl_train.json"), ], "config_evaluate_filename": None, "config_filters_filename": [ - os.path.join("testing_data", "config_fl_filters.json"), - os.path.join("testing_data", "config_fl_filters.json"), + os.path.join(_data_dir, "config_fl_filters.json"), + os.path.join(_data_dir, "config_fl_filters.json"), ], } ] TEST_EVALUATE_1 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), + "bundle_root": _data_dir, "config_train_filename": None, - "config_evaluate_filename": os.path.join("testing_data", "config_fl_evaluate.json"), - "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"), + "config_evaluate_filename": os.path.join(_data_dir, "config_fl_evaluate.json"), + "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"), } ] TEST_EVALUATE_2 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), + "bundle_root": _data_dir, "config_train_filename": None, - "config_evaluate_filename": os.path.join("testing_data", "config_fl_evaluate.json"), + "config_evaluate_filename": os.path.join(_data_dir, "config_fl_evaluate.json"), "config_filters_filename": None, } ] TEST_EVALUATE_3 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), + "bundle_root": _data_dir, "config_train_filename": None, "config_evaluate_filename": [ - os.path.join("testing_data", "config_fl_evaluate.json"), - os.path.join("testing_data", "config_fl_evaluate.json"), + os.path.join(_data_dir, "config_fl_evaluate.json"), + os.path.join(_data_dir, "config_fl_evaluate.json"), ], "config_filters_filename": [ - os.path.join("testing_data", "config_fl_filters.json"), - os.path.join("testing_data", "config_fl_filters.json"), + os.path.join(_data_dir, "config_fl_filters.json"), + os.path.join(_data_dir, "config_fl_filters.json"), ], } ] TEST_GET_WEIGHTS_1 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), - "config_train_filename": os.path.join("testing_data", "config_fl_train.json"), + "bundle_root": _data_dir, + "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"), "config_evaluate_filename": None, "send_weight_diff": False, - "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"), + "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"), } ] TEST_GET_WEIGHTS_2 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), + "bundle_root": _data_dir, "config_train_filename": None, "config_evaluate_filename": None, "send_weight_diff": False, - "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"), + "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"), } ] TEST_GET_WEIGHTS_3 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), - "config_train_filename": os.path.join("testing_data", "config_fl_train.json"), + "bundle_root": _data_dir, + "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"), "config_evaluate_filename": None, "send_weight_diff": True, - "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"), + "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"), } ] TEST_GET_WEIGHTS_4 = [ { - "bundle_root": os.path.join(os.path.dirname(__file__)), + "bundle_root": _data_dir, "config_train_filename": [ - os.path.join("testing_data", "config_fl_train.json"), - os.path.join("testing_data", "config_fl_train.json"), + os.path.join(_data_dir, "config_fl_train.json"), + os.path.join(_data_dir, "config_fl_train.json"), ], "config_evaluate_filename": None, "send_weight_diff": True, "config_filters_filename": [ - os.path.join("testing_data", "config_fl_filters.json"), - os.path.join("testing_data", "config_fl_filters.json"), + os.path.join(_data_dir, "config_fl_filters.json"), + os.path.join(_data_dir, "config_fl_filters.json"), ], } ] @@ -133,24 +135,20 @@ class TestFLMonaiAlgo(unittest.TestCase): def test_train(self, input_params): # get testing data dir and update train config; using the first to define data dir if isinstance(input_params["config_train_filename"], list): - config_train_filename = input_params["config_train_filename"][0] + config_train_filename = [ + os.path.join(input_params["bundle_root"], x) for x in input_params["config_train_filename"] + ] else: - config_train_filename = input_params["config_train_filename"] - with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f: - config_train = json.load(f) - - config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data") - - with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f: - json.dump(config_train, f, indent=4) + config_train_filename = os.path.join(input_params["bundle_root"], input_params["config_train_filename"]) # initialize algo algo = MonaiAlgo(**input_params) algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"}) + algo.abort() # initialize model parser = ConfigParser() - parser.read_config(os.path.join(input_params["bundle_root"], config_train_filename)) + parser.read_config(config_train_filename) parser.parse() network = parser.get_parsed_content("network") @@ -158,21 +156,17 @@ def test_train(self, input_params): # test train algo.train(data=data, extra={}) + algo.finalize() @parameterized.expand([TEST_EVALUATE_1, TEST_EVALUATE_2, TEST_EVALUATE_3]) def test_evaluate(self, input_params): # get testing data dir and update train config; using the first to define data dir if isinstance(input_params["config_evaluate_filename"], list): - config_eval_filename = input_params["config_evaluate_filename"][0] + config_eval_filename = [ + os.path.join(input_params["bundle_root"], x) for x in input_params["config_evaluate_filename"] + ] else: - config_eval_filename = input_params["config_evaluate_filename"] - with open(os.path.join(input_params["bundle_root"], config_eval_filename)) as f: - config_evaluate = json.load(f) - - config_evaluate["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data") - - with open(os.path.join(input_params["bundle_root"], config_eval_filename), "w") as f: - json.dump(config_evaluate, f, indent=4) + config_eval_filename = os.path.join(input_params["bundle_root"], input_params["config_evaluate_filename"]) # initialize algo algo = MonaiAlgo(**input_params) @@ -180,7 +174,7 @@ def test_evaluate(self, input_params): # initialize model parser = ConfigParser() - parser.read_config(os.path.join(input_params["bundle_root"], config_eval_filename)) + parser.read_config(config_eval_filename) parser.parse() network = parser.get_parsed_content("network") @@ -191,20 +185,6 @@ def test_evaluate(self, input_params): @parameterized.expand([TEST_GET_WEIGHTS_1, TEST_GET_WEIGHTS_2, TEST_GET_WEIGHTS_3, TEST_GET_WEIGHTS_4]) def test_get_weights(self, input_params): - # get testing data dir and update train config; using the first to define data dir - if input_params["config_train_filename"]: - if isinstance(input_params["config_train_filename"], list): - config_train_filename = input_params["config_train_filename"][0] - else: - config_train_filename = input_params["config_train_filename"] - with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f: - config_train = json.load(f) - - config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data") - - with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f: - json.dump(config_train, f, indent=4) - # initialize algo algo = MonaiAlgo(**input_params) algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"}) @@ -217,8 +197,6 @@ def test_get_weights(self, input_params): weights = algo.get_weights(extra={}) self.assertIsInstance(weights, ExchangeObject) - # TODO: test abort and finalize - if __name__ == "__main__": unittest.main() diff --git a/tests/testing_data/config_fl_evaluate.json b/tests/testing_data/config_fl_evaluate.json index 89cebc67c3..686b09440f 100644 --- a/tests/testing_data/config_fl_evaluate.json +++ b/tests/testing_data/config_fl_evaluate.json @@ -1,6 +1,6 @@ { "bundle_root": "tests/testing_data", - "dataset_dir": "tests/testing_data", + "dataset_dir": "@bundle_root", "imports": [ "$import os" ], diff --git a/tests/testing_data/config_fl_train.json b/tests/testing_data/config_fl_train.json index 067b25a952..1194785c86 100644 --- a/tests/testing_data/config_fl_train.json +++ b/tests/testing_data/config_fl_train.json @@ -1,6 +1,6 @@ { "bundle_root": "tests/testing_data", - "dataset_dir": "tests/testing_data", + "dataset_dir": "@bundle_root", "imports": [ "$import os" ], From a14d96a48de2bc25282624e1bb3408d43b625060 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 14 Oct 2022 09:40:22 +0100 Subject: [PATCH 11/18] cancel-in-progress if merged Signed-off-by: Wenqi Li --- .github/workflows/pythonapp-gpu.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pythonapp-gpu.yml b/.github/workflows/pythonapp-gpu.yml index 83175172fc..22f2a9ce7e 100644 --- a/.github/workflows/pythonapp-gpu.yml +++ b/.github/workflows/pythonapp-gpu.yml @@ -7,6 +7,7 @@ on: - main - releasing/* pull_request: + types: [opened, synchronize, closed] concurrency: # automatically cancel the previously triggered workflows when there's a newer version @@ -63,6 +64,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: apt install + if: github.event.pull_request.merged != true run: | # FIXME: workaround for https://github.com/Project-MONAI/MONAI/issues/4200 apt-key del 7fa2af80 && rm -rf /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list @@ -112,6 +114,7 @@ jobs: name: Optional Cupy dependency (cuda114) run: echo "cupy-cuda114" >> requirements-dev.txt - name: Install dependencies + if: github.event.pull_request.merged != true run: | which python python -m pip install --upgrade pip wheel @@ -146,6 +149,7 @@ jobs: if pgrep python; then pkill python; fi shell: bash - name: Upload coverage + if: github.event.pull_request.merged != true uses: codecov/codecov-action@v3 with: files: ./coverage.xml From 6c964bfd8c8c8c5a3d38a8ea902b2e4ed9a67bb2 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 14 Oct 2022 10:22:36 +0100 Subject: [PATCH 12/18] fixes dir Signed-off-by: Wenqi Li --- tests/test_fl_monai_algo_dist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py index b1b790d0d8..46cb7fe1d7 100644 --- a/tests/test_fl_monai_algo_dist.py +++ b/tests/test_fl_monai_algo_dist.py @@ -26,7 +26,7 @@ _data_dir = pathjoin(_root_dir, "testing_data") TEST_TRAIN_1 = [ { - "bundle_root": _root_dir, + "bundle_root": _data_dir, "config_train_filename": [ pathjoin(_data_dir, "config_fl_train.json"), pathjoin(_data_dir, "multi_gpu_train.json"), @@ -39,7 +39,7 @@ TEST_EVALUATE_1 = [ { - "bundle_root": _root_dir, + "bundle_root": _data_dir, "config_train_filename": None, "config_evaluate_filename": [ pathjoin(_data_dir, "config_fl_evaluate.json"), From 0af161ab4712bc2e9d4dd0efa9a64e15ad78060b Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 14 Oct 2022 11:05:02 +0100 Subject: [PATCH 13/18] test dist first Signed-off-by: Wenqi Li --- .github/workflows/pythonapp-gpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp-gpu.yml b/.github/workflows/pythonapp-gpu.yml index 22f2a9ce7e..09f0b85009 100644 --- a/.github/workflows/pythonapp-gpu.yml +++ b/.github/workflows/pythonapp-gpu.yml @@ -140,7 +140,8 @@ jobs: python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))' python -c "import monai; monai.config.print_config()" # build for the current self-hosted CI Tesla V100 - BUILD_MONAI=1 TORCH_CUDA_ARCH_LIST="7.0" ./runtests.sh --build --quick --unittests --disttests + BUILD_MONAI=1 TORCH_CUDA_ARCH_LIST="7.0" ./runtests.sh --build --disttests + ./runtests.sh --quick --unittests if [ ${{ matrix.environment }} = "PT110+CUDA102" ]; then # test the clang-format tool downloading once coverage run -m tests.clang_format_utils From 7c4914347bf89cfd476c6df40e83a4b5248b8811 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 14 Oct 2022 11:17:56 +0100 Subject: [PATCH 14/18] update to use abspath Signed-off-by: Wenqi Li --- tests/test_fl_monai_algo.py | 2 +- tests/test_fl_monai_algo_dist.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_fl_monai_algo.py b/tests/test_fl_monai_algo.py index 46e8c5efa3..0627235a18 100644 --- a/tests/test_fl_monai_algo.py +++ b/tests/test_fl_monai_algo.py @@ -20,7 +20,7 @@ from monai.fl.utils.exchange_object import ExchangeObject from tests.utils import SkipIfNoModule -_root_dir = os.path.join(os.path.dirname(__file__)) +_root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__))) _data_dir = os.path.join(_root_dir, "testing_data") TEST_TRAIN_1 = [ diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py index 46cb7fe1d7..0f83bb3c71 100644 --- a/tests/test_fl_monai_algo_dist.py +++ b/tests/test_fl_monai_algo_dist.py @@ -22,7 +22,7 @@ from monai.fl.utils.exchange_object import ExchangeObject from tests.utils import DistCall, DistTestCase, SkipIfNoModule, skip_if_no_cuda -_root_dir = pathjoin(os.path.dirname(__file__)) +_root_dir = os.path.abspath(pathjoin(os.path.dirname(__file__))) _data_dir = pathjoin(_root_dir, "testing_data") TEST_TRAIN_1 = [ { From 1621a1e67b7b0ac0facdc0eac9377e26f5afb8c6 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 14 Oct 2022 12:35:25 +0100 Subject: [PATCH 15/18] smaller tests Signed-off-by: Wenqi Li --- tests/testing_data/config_fl_evaluate.json | 2 +- tests/testing_data/config_fl_train.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testing_data/config_fl_evaluate.json b/tests/testing_data/config_fl_evaluate.json index 686b09440f..113596070a 100644 --- a/tests/testing_data/config_fl_evaluate.json +++ b/tests/testing_data/config_fl_evaluate.json @@ -62,7 +62,7 @@ "dataloader": { "_target_": "DataLoader", "dataset": "@validate#dataset", - "batch_size": 128, + "batch_size": 3, "shuffle": false, "num_workers": 4 }, diff --git a/tests/testing_data/config_fl_train.json b/tests/testing_data/config_fl_train.json index 1194785c86..f53a95bc02 100644 --- a/tests/testing_data/config_fl_train.json +++ b/tests/testing_data/config_fl_train.json @@ -96,7 +96,7 @@ "dataloader": { "_target_": "DataLoader", "dataset": "@train#dataset", - "batch_size": 128, + "batch_size": 3, "shuffle": true, "num_workers": 2 }, From 4fcef5a35a2da9b26e0928d2b803bfd329f7370d Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 14 Oct 2022 14:48:31 +0100 Subject: [PATCH 16/18] skip <1.11 Signed-off-by: Wenqi Li --- tests/test_fl_monai_algo_dist.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py index 0f83bb3c71..11f64ea318 100644 --- a/tests/test_fl_monai_algo_dist.py +++ b/tests/test_fl_monai_algo_dist.py @@ -20,7 +20,7 @@ from monai.fl.client.monai_algo import MonaiAlgo from monai.fl.utils.constants import ExtraItems from monai.fl.utils.exchange_object import ExchangeObject -from tests.utils import DistCall, DistTestCase, SkipIfNoModule, skip_if_no_cuda +from tests.utils import DistCall, DistTestCase, SkipIfBeforePyTorchVersion, SkipIfNoModule, skip_if_no_cuda _root_dir = os.path.abspath(pathjoin(os.path.dirname(__file__))) _data_dir = pathjoin(_root_dir, "testing_data") @@ -52,6 +52,7 @@ @SkipIfNoModule("ignite") +@SkipIfBeforePyTorchVersion((1, 11, 1)) class TestFLMonaiAlgo(DistTestCase): @parameterized.expand([TEST_TRAIN_1]) @DistCall(nnodes=1, nproc_per_node=2, init_method="no_init") From 215378387ff1befde803fda1c942cd17cb9efba6 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 14 Oct 2022 16:01:51 +0100 Subject: [PATCH 17/18] skip v1.7 tests Signed-off-by: Wenqi Li --- tests/test_resample_backends.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_resample_backends.py b/tests/test_resample_backends.py index 912a97378c..6d231183a9 100644 --- a/tests/test_resample_backends.py +++ b/tests/test_resample_backends.py @@ -20,7 +20,7 @@ from monai.transforms import Resample from monai.transforms.utils import create_grid from monai.utils import GridSampleMode, GridSamplePadMode, NdimageMode, SplineMode, convert_to_numpy -from tests.utils import assert_allclose, is_tf32_env +from tests.utils import SkipIfBeforePyTorchVersion, assert_allclose, is_tf32_env _rtol = 1e-3 if is_tf32_env() else 1e-4 @@ -40,6 +40,7 @@ TEST_IDENTITY.append([dict(device=device), p_s, interp_s, pad_s, (1, 21, 23, 24)]) +@SkipIfBeforePyTorchVersion((1, 9, 1)) class TestResampleBackends(unittest.TestCase): @parameterized.expand(TEST_IDENTITY) def test_resample_identity(self, input_param, im_type, interp, padding, input_shape): From aabedbe006ef4af5a092a0b62ca30b45995c110f Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Fri, 14 Oct 2022 18:56:59 +0100 Subject: [PATCH 18/18] fixes downloading tests Signed-off-by: Wenqi Li --- tests/test_auto3dseg_ensemble.py | 9 +++++---- tests/test_auto3dseg_hpo.py | 10 +++++----- tests/utils.py | 2 ++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/test_auto3dseg_ensemble.py b/tests/test_auto3dseg_ensemble.py index de56247257..7b9656f1ac 100644 --- a/tests/test_auto3dseg_ensemble.py +++ b/tests/test_auto3dseg_ensemble.py @@ -22,7 +22,7 @@ from monai.data import create_test_image_3d from monai.utils import optional_import from monai.utils.enums import AlgoEnsembleKeys -from tests.utils import SkipIfBeforePyTorchVersion, skip_if_no_cuda, skip_if_quick +from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda, skip_if_quick _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter") @@ -110,9 +110,10 @@ def test_ensemble(self) -> None: ConfigParser.export_config_file(data_src, data_src_cfg) - bundle_generator = BundleGen( - algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg - ) + with skip_if_downloading_fails(): + bundle_generator = BundleGen( + algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg + ) bundle_generator.generate(work_dir, num_fold=2) history = bundle_generator.get_history() diff --git a/tests/test_auto3dseg_hpo.py b/tests/test_auto3dseg_hpo.py index 7cd53d99dc..708828eed4 100644 --- a/tests/test_auto3dseg_hpo.py +++ b/tests/test_auto3dseg_hpo.py @@ -23,7 +23,7 @@ from monai.bundle.config_parser import ConfigParser from monai.data import create_test_image_3d from monai.utils import optional_import -from tests.utils import SkipIfBeforePyTorchVersion, skip_if_no_cuda +from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter") optuna, has_optuna = optional_import("optuna") @@ -104,10 +104,10 @@ def setUp(self) -> None: } ConfigParser.export_config_file(data_src, data_src_cfg) - - bundle_generator = BundleGen( - algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg - ) + with skip_if_downloading_fails(): + bundle_generator = BundleGen( + algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg + ) bundle_generator.generate(work_dir, num_fold=2) self.history = bundle_generator.get_history() diff --git a/tests/utils.py b/tests/utils.py index 8e1c2a8f4a..b16b4b13fb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -137,6 +137,8 @@ def skip_if_downloading_fails(): raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e if "md5 check" in str(rt_e): raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e + if "account limit" in str(rt_e): # HTTP Error 503: Egress is over the account limit + raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e raise rt_e