adding a multigpu test of TestFLMonaiAlgo

.github/workflows/pythonapp-gpu.yml

@@ -7,6 +7,7 @@ on:
       - main
       - releasing/*
   pull_request:
+    types: [opened, synchronize, closed]
 
 concurrency:
   # automatically cancel the previously triggered workflows when there's a newer version
@@ -63,6 +64,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: apt install
+      if: github.event.pull_request.merged != true
       run: |
         # FIXME: workaround for https://github.com/Project-MONAI/MONAI/issues/4200
         apt-key del 7fa2af80 && rm -rf /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list
@@ -112,6 +114,7 @@ jobs:
       name: Optional Cupy dependency (cuda114)
       run: echo "cupy-cuda114" >> requirements-dev.txt
     - name: Install dependencies
+      if: github.event.pull_request.merged != true
       run: |
         which python
         python -m pip install --upgrade pip wheel
@@ -137,7 +140,8 @@ jobs:
         python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
         python -c "import monai; monai.config.print_config()"
         # build for the current self-hosted CI Tesla V100
-        BUILD_MONAI=1 TORCH_CUDA_ARCH_LIST="7.0" ./runtests.sh --build --quick --unittests --disttests
+        BUILD_MONAI=1 TORCH_CUDA_ARCH_LIST="7.0" ./runtests.sh --build --disttests
+        ./runtests.sh --quick --unittests
         if [ ${{ matrix.environment }} = "PT110+CUDA102" ]; then
           # test the clang-format tool downloading once
           coverage run -m tests.clang_format_utils
@@ -146,6 +150,7 @@ jobs:
         if pgrep python; then pkill python; fi
       shell: bash
     - name: Upload coverage
+      if: github.event.pull_request.merged != true
       uses: codecov/codecov-action@v3
       with:
         files: ./coverage.xml

tests/test_auto3dseg_ensemble.py

@@ -22,7 +22,7 @@
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
 from monai.utils.enums import AlgoEnsembleKeys
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_no_cuda, skip_if_quick
+from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda, skip_if_quick
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 
@@ -110,9 +110,10 @@ def test_ensemble(self) -> None:
 
         ConfigParser.export_config_file(data_src, data_src_cfg)
 
-        bundle_generator = BundleGen(
-            algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
-        )
+        with skip_if_downloading_fails():
+            bundle_generator = BundleGen(
+                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+            )
         bundle_generator.generate(work_dir, num_fold=2)
         history = bundle_generator.get_history()
 

tests/test_auto3dseg_hpo.py

@@ -23,7 +23,7 @@
 from monai.bundle.config_parser import ConfigParser
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_no_cuda
+from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 optuna, has_optuna = optional_import("optuna")
@@ -104,10 +104,10 @@ def setUp(self) -> None:
         }
 
         ConfigParser.export_config_file(data_src, data_src_cfg)
-
-        bundle_generator = BundleGen(
-            algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
-        )
+        with skip_if_downloading_fails():
+            bundle_generator = BundleGen(
+                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+            )
         bundle_generator.generate(work_dir, num_fold=2)
 
         self.history = bundle_generator.get_history()

tests/test_fl_monai_algo.py

@@ -9,7 +9,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import os
 import unittest
 
@@ -21,107 +20,110 @@
 from monai.fl.utils.exchange_object import ExchangeObject
 from tests.utils import SkipIfNoModule
 
+_root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__)))
+_data_dir = os.path.join(_root_dir, "testing_data")
+
 TEST_TRAIN_1 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
-        "config_train_filename": os.path.join("testing_data", "config_fl_train.json"),
+        "bundle_root": _data_dir,
+        "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"),
         "config_evaluate_filename": None,
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_TRAIN_2 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
-        "config_train_filename": os.path.join("testing_data", "config_fl_train.json"),
+        "bundle_root": _data_dir,
+        "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"),
         "config_evaluate_filename": None,
         "config_filters_filename": None,
     }
 ]
 TEST_TRAIN_3 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": [
-            os.path.join("testing_data", "config_fl_train.json"),
-            os.path.join("testing_data", "config_fl_train.json"),
+            os.path.join(_data_dir, "config_fl_train.json"),
+            os.path.join(_data_dir, "config_fl_train.json"),
         ],
         "config_evaluate_filename": None,
         "config_filters_filename": [
-            os.path.join("testing_data", "config_fl_filters.json"),
-            os.path.join("testing_data", "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
         ],
     }
 ]
 
 TEST_EVALUATE_1 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": None,
-        "config_evaluate_filename": os.path.join("testing_data", "config_fl_evaluate.json"),
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_evaluate_filename": os.path.join(_data_dir, "config_fl_evaluate.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_EVALUATE_2 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": None,
-        "config_evaluate_filename": os.path.join("testing_data", "config_fl_evaluate.json"),
+        "config_evaluate_filename": os.path.join(_data_dir, "config_fl_evaluate.json"),
         "config_filters_filename": None,
     }
 ]
 TEST_EVALUATE_3 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": None,
         "config_evaluate_filename": [
-            os.path.join("testing_data", "config_fl_evaluate.json"),
-            os.path.join("testing_data", "config_fl_evaluate.json"),
+            os.path.join(_data_dir, "config_fl_evaluate.json"),
+            os.path.join(_data_dir, "config_fl_evaluate.json"),
         ],
         "config_filters_filename": [
-            os.path.join("testing_data", "config_fl_filters.json"),
-            os.path.join("testing_data", "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
         ],
     }
 ]
 
 TEST_GET_WEIGHTS_1 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
-        "config_train_filename": os.path.join("testing_data", "config_fl_train.json"),
+        "bundle_root": _data_dir,
+        "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"),
         "config_evaluate_filename": None,
         "send_weight_diff": False,
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_GET_WEIGHTS_2 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": None,
         "config_evaluate_filename": None,
         "send_weight_diff": False,
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_GET_WEIGHTS_3 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
-        "config_train_filename": os.path.join("testing_data", "config_fl_train.json"),
+        "bundle_root": _data_dir,
+        "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"),
         "config_evaluate_filename": None,
         "send_weight_diff": True,
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_GET_WEIGHTS_4 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": [
-            os.path.join("testing_data", "config_fl_train.json"),
-            os.path.join("testing_data", "config_fl_train.json"),
+            os.path.join(_data_dir, "config_fl_train.json"),
+            os.path.join(_data_dir, "config_fl_train.json"),
         ],
         "config_evaluate_filename": None,
         "send_weight_diff": True,
         "config_filters_filename": [
-            os.path.join("testing_data", "config_fl_filters.json"),
-            os.path.join("testing_data", "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
         ],
     }
 ]
@@ -133,54 +135,46 @@ class TestFLMonaiAlgo(unittest.TestCase):
     def test_train(self, input_params):
         # get testing data dir and update train config; using the first to define data dir
         if isinstance(input_params["config_train_filename"], list):
-            config_train_filename = input_params["config_train_filename"][0]
+            config_train_filename = [
+                os.path.join(input_params["bundle_root"], x) for x in input_params["config_train_filename"]
+            ]
         else:
-            config_train_filename = input_params["config_train_filename"]
-        with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f:
-            config_train = json.load(f)
-
-        config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data")
-
-        with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f:
-            json.dump(config_train, f, indent=4)
+            config_train_filename = os.path.join(input_params["bundle_root"], input_params["config_train_filename"])
 
         # initialize algo
         algo = MonaiAlgo(**input_params)
         algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"})
+        algo.abort()
 
         # initialize model
         parser = ConfigParser()
-        parser.read_config(os.path.join(input_params["bundle_root"], config_train_filename))
+        parser.read_config(config_train_filename)
         parser.parse()
         network = parser.get_parsed_content("network")
 
         data = ExchangeObject(weights=network.state_dict())
 
         # test train
         algo.train(data=data, extra={})
+        algo.finalize()
 
     @parameterized.expand([TEST_EVALUATE_1, TEST_EVALUATE_2, TEST_EVALUATE_3])
     def test_evaluate(self, input_params):
         # get testing data dir and update train config; using the first to define data dir
         if isinstance(input_params["config_evaluate_filename"], list):
-            config_eval_filename = input_params["config_evaluate_filename"][0]
+            config_eval_filename = [
+                os.path.join(input_params["bundle_root"], x) for x in input_params["config_evaluate_filename"]
+            ]
         else:
-            config_eval_filename = input_params["config_evaluate_filename"]
-        with open(os.path.join(input_params["bundle_root"], config_eval_filename)) as f:
-            config_evaluate = json.load(f)
-
-        config_evaluate["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data")
-
-        with open(os.path.join(input_params["bundle_root"], config_eval_filename), "w") as f:
-            json.dump(config_evaluate, f, indent=4)
+            config_eval_filename = os.path.join(input_params["bundle_root"], input_params["config_evaluate_filename"])
 
         # initialize algo
         algo = MonaiAlgo(**input_params)
         algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"})
 
         # initialize model
         parser = ConfigParser()
-        parser.read_config(os.path.join(input_params["bundle_root"], config_eval_filename))
+        parser.read_config(config_eval_filename)
         parser.parse()
         network = parser.get_parsed_content("network")
 
@@ -191,20 +185,6 @@ def test_evaluate(self, input_params):
 
     @parameterized.expand([TEST_GET_WEIGHTS_1, TEST_GET_WEIGHTS_2, TEST_GET_WEIGHTS_3, TEST_GET_WEIGHTS_4])
     def test_get_weights(self, input_params):
-        # get testing data dir and update train config; using the first to define data dir
-        if input_params["config_train_filename"]:
-            if isinstance(input_params["config_train_filename"], list):
-                config_train_filename = input_params["config_train_filename"][0]
-            else:
-                config_train_filename = input_params["config_train_filename"]
-            with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f:
-                config_train = json.load(f)
-
-            config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data")
-
-            with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f:
-                json.dump(config_train, f, indent=4)
-
         # initialize algo
         algo = MonaiAlgo(**input_params)
         algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"})
@@ -217,8 +197,6 @@ def test_get_weights(self, input_params):
             weights = algo.get_weights(extra={})
             self.assertIsInstance(weights, ExchangeObject)
 
-    # TODO: test abort and finalize
-
 
 if __name__ == "__main__":
     unittest.main()