diff --git a/.github/workflows/pythonapp-gpu.yml b/.github/workflows/pythonapp-gpu.yml
index 83175172fc..09f0b85009 100644
--- a/.github/workflows/pythonapp-gpu.yml
+++ b/.github/workflows/pythonapp-gpu.yml
@@ -7,6 +7,7 @@ on:
       - main
       - releasing/*
   pull_request:
+    types: [opened, synchronize, closed]
 
 concurrency:
   # automatically cancel the previously triggered workflows when there's a newer version
@@ -63,6 +64,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: apt install
+      if: github.event.pull_request.merged != true
       run: |
         # FIXME: workaround for https://github.com/Project-MONAI/MONAI/issues/4200
         apt-key del 7fa2af80 && rm -rf /etc/apt/sources.list.d/nvidia-ml.list /etc/apt/sources.list.d/cuda.list
@@ -112,6 +114,7 @@ jobs:
       name: Optional Cupy dependency (cuda114)
       run: echo "cupy-cuda114" >> requirements-dev.txt
     - name: Install dependencies
+      if: github.event.pull_request.merged != true
       run: |
         which python
         python -m pip install --upgrade pip wheel
@@ -137,7 +140,8 @@ jobs:
         python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
         python -c "import monai; monai.config.print_config()"
         # build for the current self-hosted CI Tesla V100
-        BUILD_MONAI=1 TORCH_CUDA_ARCH_LIST="7.0" ./runtests.sh --build --quick --unittests --disttests
+        BUILD_MONAI=1 TORCH_CUDA_ARCH_LIST="7.0" ./runtests.sh --build --disttests
+        ./runtests.sh --quick --unittests
         if [ ${{ matrix.environment }} = "PT110+CUDA102" ]; then
           # test the clang-format tool downloading once
           coverage run -m tests.clang_format_utils
@@ -146,6 +150,7 @@ jobs:
         if pgrep python; then pkill python; fi
       shell: bash
     - name: Upload coverage
+      if: github.event.pull_request.merged != true
       uses: codecov/codecov-action@v3
       with:
         files: ./coverage.xml
diff --git a/tests/test_auto3dseg_ensemble.py b/tests/test_auto3dseg_ensemble.py
index de56247257..7b9656f1ac 100644
--- a/tests/test_auto3dseg_ensemble.py
+++ b/tests/test_auto3dseg_ensemble.py
@@ -22,7 +22,7 @@
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
 from monai.utils.enums import AlgoEnsembleKeys
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_no_cuda, skip_if_quick
+from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda, skip_if_quick
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 
@@ -110,9 +110,10 @@ def test_ensemble(self) -> None:
 
         ConfigParser.export_config_file(data_src, data_src_cfg)
 
-        bundle_generator = BundleGen(
-            algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
-        )
+        with skip_if_downloading_fails():
+            bundle_generator = BundleGen(
+                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+            )
         bundle_generator.generate(work_dir, num_fold=2)
         history = bundle_generator.get_history()
 
diff --git a/tests/test_auto3dseg_hpo.py b/tests/test_auto3dseg_hpo.py
index 7cd53d99dc..708828eed4 100644
--- a/tests/test_auto3dseg_hpo.py
+++ b/tests/test_auto3dseg_hpo.py
@@ -23,7 +23,7 @@
 from monai.bundle.config_parser import ConfigParser
 from monai.data import create_test_image_3d
 from monai.utils import optional_import
-from tests.utils import SkipIfBeforePyTorchVersion, skip_if_no_cuda
+from tests.utils import SkipIfBeforePyTorchVersion, skip_if_downloading_fails, skip_if_no_cuda
 
 _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 optuna, has_optuna = optional_import("optuna")
@@ -104,10 +104,10 @@ def setUp(self) -> None:
         }
 
         ConfigParser.export_config_file(data_src, data_src_cfg)
-
-        bundle_generator = BundleGen(
-            algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
-        )
+        with skip_if_downloading_fails():
+            bundle_generator = BundleGen(
+                algo_path=work_dir, data_stats_filename=da_output_yaml, data_src_cfg_name=data_src_cfg
+            )
         bundle_generator.generate(work_dir, num_fold=2)
 
         self.history = bundle_generator.get_history()
diff --git a/tests/test_fl_monai_algo.py b/tests/test_fl_monai_algo.py
index 66722d0086..0627235a18 100644
--- a/tests/test_fl_monai_algo.py
+++ b/tests/test_fl_monai_algo.py
@@ -9,7 +9,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import os
 import unittest
 
@@ -21,107 +20,110 @@
 from monai.fl.utils.exchange_object import ExchangeObject
 from tests.utils import SkipIfNoModule
 
+_root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__)))
+_data_dir = os.path.join(_root_dir, "testing_data")
+
 TEST_TRAIN_1 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
-        "config_train_filename": os.path.join("testing_data", "config_fl_train.json"),
+        "bundle_root": _data_dir,
+        "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"),
         "config_evaluate_filename": None,
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_TRAIN_2 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
-        "config_train_filename": os.path.join("testing_data", "config_fl_train.json"),
+        "bundle_root": _data_dir,
+        "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"),
         "config_evaluate_filename": None,
         "config_filters_filename": None,
     }
 ]
 TEST_TRAIN_3 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": [
-            os.path.join("testing_data", "config_fl_train.json"),
-            os.path.join("testing_data", "config_fl_train.json"),
+            os.path.join(_data_dir, "config_fl_train.json"),
+            os.path.join(_data_dir, "config_fl_train.json"),
         ],
         "config_evaluate_filename": None,
         "config_filters_filename": [
-            os.path.join("testing_data", "config_fl_filters.json"),
-            os.path.join("testing_data", "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
         ],
     }
 ]
 
 TEST_EVALUATE_1 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": None,
-        "config_evaluate_filename": os.path.join("testing_data", "config_fl_evaluate.json"),
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_evaluate_filename": os.path.join(_data_dir, "config_fl_evaluate.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_EVALUATE_2 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": None,
-        "config_evaluate_filename": os.path.join("testing_data", "config_fl_evaluate.json"),
+        "config_evaluate_filename": os.path.join(_data_dir, "config_fl_evaluate.json"),
         "config_filters_filename": None,
     }
 ]
 TEST_EVALUATE_3 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": None,
         "config_evaluate_filename": [
-            os.path.join("testing_data", "config_fl_evaluate.json"),
-            os.path.join("testing_data", "config_fl_evaluate.json"),
+            os.path.join(_data_dir, "config_fl_evaluate.json"),
+            os.path.join(_data_dir, "config_fl_evaluate.json"),
         ],
         "config_filters_filename": [
-            os.path.join("testing_data", "config_fl_filters.json"),
-            os.path.join("testing_data", "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
         ],
     }
 ]
 
 TEST_GET_WEIGHTS_1 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
-        "config_train_filename": os.path.join("testing_data", "config_fl_train.json"),
+        "bundle_root": _data_dir,
+        "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"),
         "config_evaluate_filename": None,
         "send_weight_diff": False,
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_GET_WEIGHTS_2 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": None,
         "config_evaluate_filename": None,
         "send_weight_diff": False,
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_GET_WEIGHTS_3 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
-        "config_train_filename": os.path.join("testing_data", "config_fl_train.json"),
+        "bundle_root": _data_dir,
+        "config_train_filename": os.path.join(_data_dir, "config_fl_train.json"),
         "config_evaluate_filename": None,
         "send_weight_diff": True,
-        "config_filters_filename": os.path.join("testing_data", "config_fl_filters.json"),
+        "config_filters_filename": os.path.join(_data_dir, "config_fl_filters.json"),
     }
 ]
 TEST_GET_WEIGHTS_4 = [
     {
-        "bundle_root": os.path.join(os.path.dirname(__file__)),
+        "bundle_root": _data_dir,
         "config_train_filename": [
-            os.path.join("testing_data", "config_fl_train.json"),
-            os.path.join("testing_data", "config_fl_train.json"),
+            os.path.join(_data_dir, "config_fl_train.json"),
+            os.path.join(_data_dir, "config_fl_train.json"),
         ],
         "config_evaluate_filename": None,
         "send_weight_diff": True,
         "config_filters_filename": [
-            os.path.join("testing_data", "config_fl_filters.json"),
-            os.path.join("testing_data", "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
+            os.path.join(_data_dir, "config_fl_filters.json"),
         ],
     }
 ]
@@ -133,24 +135,20 @@ class TestFLMonaiAlgo(unittest.TestCase):
     def test_train(self, input_params):
         # get testing data dir and update train config; using the first to define data dir
         if isinstance(input_params["config_train_filename"], list):
-            config_train_filename = input_params["config_train_filename"][0]
+            config_train_filename = [
+                os.path.join(input_params["bundle_root"], x) for x in input_params["config_train_filename"]
+            ]
         else:
-            config_train_filename = input_params["config_train_filename"]
-        with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f:
-            config_train = json.load(f)
-
-        config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data")
-
-        with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f:
-            json.dump(config_train, f, indent=4)
+            config_train_filename = os.path.join(input_params["bundle_root"], input_params["config_train_filename"])
 
         # initialize algo
         algo = MonaiAlgo(**input_params)
         algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"})
+        algo.abort()
 
         # initialize model
         parser = ConfigParser()
-        parser.read_config(os.path.join(input_params["bundle_root"], config_train_filename))
+        parser.read_config(config_train_filename)
         parser.parse()
         network = parser.get_parsed_content("network")
 
@@ -158,21 +156,17 @@ def test_train(self, input_params):
 
         # test train
         algo.train(data=data, extra={})
+        algo.finalize()
 
     @parameterized.expand([TEST_EVALUATE_1, TEST_EVALUATE_2, TEST_EVALUATE_3])
     def test_evaluate(self, input_params):
         # get testing data dir and update train config; using the first to define data dir
         if isinstance(input_params["config_evaluate_filename"], list):
-            config_eval_filename = input_params["config_evaluate_filename"][0]
+            config_eval_filename = [
+                os.path.join(input_params["bundle_root"], x) for x in input_params["config_evaluate_filename"]
+            ]
         else:
-            config_eval_filename = input_params["config_evaluate_filename"]
-        with open(os.path.join(input_params["bundle_root"], config_eval_filename)) as f:
-            config_evaluate = json.load(f)
-
-        config_evaluate["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data")
-
-        with open(os.path.join(input_params["bundle_root"], config_eval_filename), "w") as f:
-            json.dump(config_evaluate, f, indent=4)
+            config_eval_filename = os.path.join(input_params["bundle_root"], input_params["config_evaluate_filename"])
 
         # initialize algo
         algo = MonaiAlgo(**input_params)
@@ -180,7 +174,7 @@ def test_evaluate(self, input_params):
 
         # initialize model
         parser = ConfigParser()
-        parser.read_config(os.path.join(input_params["bundle_root"], config_eval_filename))
+        parser.read_config(config_eval_filename)
         parser.parse()
         network = parser.get_parsed_content("network")
 
@@ -191,20 +185,6 @@ def test_evaluate(self, input_params):
 
     @parameterized.expand([TEST_GET_WEIGHTS_1, TEST_GET_WEIGHTS_2, TEST_GET_WEIGHTS_3, TEST_GET_WEIGHTS_4])
     def test_get_weights(self, input_params):
-        # get testing data dir and update train config; using the first to define data dir
-        if input_params["config_train_filename"]:
-            if isinstance(input_params["config_train_filename"], list):
-                config_train_filename = input_params["config_train_filename"][0]
-            else:
-                config_train_filename = input_params["config_train_filename"]
-            with open(os.path.join(input_params["bundle_root"], config_train_filename)) as f:
-                config_train = json.load(f)
-
-            config_train["dataset_dir"] = os.path.join(os.path.dirname(__file__), "testing_data")
-
-            with open(os.path.join(input_params["bundle_root"], config_train_filename), "w") as f:
-                json.dump(config_train, f, indent=4)
-
         # initialize algo
         algo = MonaiAlgo(**input_params)
         algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"})
@@ -217,8 +197,6 @@ def test_get_weights(self, input_params):
             weights = algo.get_weights(extra={})
             self.assertIsInstance(weights, ExchangeObject)
 
-    # TODO: test abort and finalize
-
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_fl_monai_algo_dist.py b/tests/test_fl_monai_algo_dist.py
new file mode 100644
index 0000000000..11f64ea318
--- /dev/null
+++ b/tests/test_fl_monai_algo_dist.py
@@ -0,0 +1,97 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+from os.path import join as pathjoin
+
+import torch.distributed as dist
+from parameterized import parameterized
+
+from monai.bundle import ConfigParser
+from monai.fl.client.monai_algo import MonaiAlgo
+from monai.fl.utils.constants import ExtraItems
+from monai.fl.utils.exchange_object import ExchangeObject
+from tests.utils import DistCall, DistTestCase, SkipIfBeforePyTorchVersion, SkipIfNoModule, skip_if_no_cuda
+
+_root_dir = os.path.abspath(pathjoin(os.path.dirname(__file__)))
+_data_dir = pathjoin(_root_dir, "testing_data")
+TEST_TRAIN_1 = [
+    {
+        "bundle_root": _data_dir,
+        "config_train_filename": [
+            pathjoin(_data_dir, "config_fl_train.json"),
+            pathjoin(_data_dir, "multi_gpu_train.json"),
+        ],
+        "config_evaluate_filename": None,
+        "config_filters_filename": pathjoin(_root_dir, "testing_data", "config_fl_filters.json"),
+        "multi_gpu": True,
+    }
+]
+
+TEST_EVALUATE_1 = [
+    {
+        "bundle_root": _data_dir,
+        "config_train_filename": None,
+        "config_evaluate_filename": [
+            pathjoin(_data_dir, "config_fl_evaluate.json"),
+            pathjoin(_data_dir, "multi_gpu_evaluate.json"),
+        ],
+        "config_filters_filename": pathjoin(_data_dir, "config_fl_filters.json"),
+        "multi_gpu": True,
+    }
+]
+
+
+@SkipIfNoModule("ignite")
+@SkipIfBeforePyTorchVersion((1, 11, 1))
+class TestFLMonaiAlgo(DistTestCase):
+    @parameterized.expand([TEST_TRAIN_1])
+    @DistCall(nnodes=1, nproc_per_node=2, init_method="no_init")
+    @skip_if_no_cuda
+    def test_train(self, input_params):
+        # initialize algo
+        algo = MonaiAlgo(**input_params)
+        algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"})
+        self.assertTrue(dist.get_rank() in (0, 1))
+
+        # initialize model
+        parser = ConfigParser()
+        parser.read_config([pathjoin(input_params["bundle_root"], x) for x in input_params["config_train_filename"]])
+        parser.parse()
+        network = parser.get_parsed_content("network")
+        data = ExchangeObject(weights=network.state_dict())
+        # test train
+        algo.train(data=data, extra={})
+
+    @parameterized.expand([TEST_EVALUATE_1])
+    @DistCall(nnodes=1, nproc_per_node=2, init_method="no_init")
+    @skip_if_no_cuda
+    def test_evaluate(self, input_params):
+        # initialize algo
+        algo = MonaiAlgo(**input_params)
+        algo.initialize(extra={ExtraItems.CLIENT_NAME: "test_fl"})
+        self.assertTrue(dist.get_rank() in (0, 1))
+
+        # initialize model
+        parser = ConfigParser()
+        parser.read_config(
+            [os.path.join(input_params["bundle_root"], x) for x in input_params["config_evaluate_filename"]]
+        )
+        parser.parse()
+        network = parser.get_parsed_content("network")
+        data = ExchangeObject(weights=network.state_dict())
+        # test evaluate
+        algo.evaluate(data=data, extra={})
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_resample_backends.py b/tests/test_resample_backends.py
index 912a97378c..6d231183a9 100644
--- a/tests/test_resample_backends.py
+++ b/tests/test_resample_backends.py
@@ -20,7 +20,7 @@
 from monai.transforms import Resample
 from monai.transforms.utils import create_grid
 from monai.utils import GridSampleMode, GridSamplePadMode, NdimageMode, SplineMode, convert_to_numpy
-from tests.utils import assert_allclose, is_tf32_env
+from tests.utils import SkipIfBeforePyTorchVersion, assert_allclose, is_tf32_env
 
 _rtol = 1e-3 if is_tf32_env() else 1e-4
 
@@ -40,6 +40,7 @@
                 TEST_IDENTITY.append([dict(device=device), p_s, interp_s, pad_s, (1, 21, 23, 24)])
 
 
+@SkipIfBeforePyTorchVersion((1, 9, 1))
 class TestResampleBackends(unittest.TestCase):
     @parameterized.expand(TEST_IDENTITY)
     def test_resample_identity(self, input_param, im_type, interp, padding, input_shape):
diff --git a/tests/testing_data/config_fl_evaluate.json b/tests/testing_data/config_fl_evaluate.json
index 84ab7988f6..113596070a 100644
--- a/tests/testing_data/config_fl_evaluate.json
+++ b/tests/testing_data/config_fl_evaluate.json
@@ -1,6 +1,6 @@
 {
     "bundle_root": "tests/testing_data",
-    "dataset_dir": "tests/testing_data",
+    "dataset_dir": "@bundle_root",
     "imports": [
         "$import os"
     ],
@@ -22,7 +22,7 @@
                 "image_only": true
             },
             {
-                "_target_": "AddChanneld",
+                "_target_": "EnsureChannelFirstD",
                 "keys": [
                     "image"
                 ]
@@ -62,7 +62,7 @@
         "dataloader": {
             "_target_": "DataLoader",
             "dataset": "@validate#dataset",
-            "batch_size": 128,
+            "batch_size": 3,
             "shuffle": false,
             "num_workers": 4
         },
diff --git a/tests/testing_data/config_fl_train.json b/tests/testing_data/config_fl_train.json
index 5954d2cfbc..f53a95bc02 100644
--- a/tests/testing_data/config_fl_train.json
+++ b/tests/testing_data/config_fl_train.json
@@ -1,6 +1,6 @@
 {
     "bundle_root": "tests/testing_data",
-    "dataset_dir": "tests/testing_data",
+    "dataset_dir": "@bundle_root",
     "imports": [
         "$import os"
     ],
@@ -30,7 +30,7 @@
                 "image_only": true
             },
             {
-                "_target_": "AddChanneld",
+                "_target_": "EnsureChannelFirstD",
                 "keys": [
                     "image"
                 ]
@@ -96,7 +96,7 @@
         "dataloader": {
             "_target_": "DataLoader",
             "dataset": "@train#dataset",
-            "batch_size": 128,
+            "batch_size": 3,
             "shuffle": true,
             "num_workers": 2
         },
diff --git a/tests/testing_data/multi_gpu_evaluate.json b/tests/testing_data/multi_gpu_evaluate.json
new file mode 100644
index 0000000000..7af24a6b2e
--- /dev/null
+++ b/tests/testing_data/multi_gpu_evaluate.json
@@ -0,0 +1,27 @@
+{
+    "device": "$torch.device(f'cuda:{dist.get_rank()}')",
+    "network": {
+        "_target_": "torch.nn.parallel.DistributedDataParallel",
+        "module": "$@network_def.to(@device)",
+        "device_ids": [
+            "@device"
+        ]
+    },
+    "validate#sampler": {
+        "_target_": "DistributedSampler",
+        "dataset": "@validate#dataset",
+        "even_divisible": false,
+        "shuffle": false
+    },
+    "validate#dataloader#sampler": "@validate#sampler",
+    "evaluating": [
+        "$import torch.distributed as dist",
+        "$dist.init_process_group(backend='nccl')",
+        "$torch.cuda.set_device(@device)",
+        "$setattr(torch.backends.cudnn, 'benchmark', True)",
+        "$import logging",
+        "$@validate#evaluator.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)",
+        "$@validate#evaluator.run()",
+        "$dist.destroy_process_group()"
+    ]
+}
diff --git a/tests/testing_data/multi_gpu_train.json b/tests/testing_data/multi_gpu_train.json
new file mode 100644
index 0000000000..41fd7698db
--- /dev/null
+++ b/tests/testing_data/multi_gpu_train.json
@@ -0,0 +1,30 @@
+{
+    "device": "$torch.device(f'cuda:{dist.get_rank()}')",
+    "network": {
+        "_target_": "torch.nn.parallel.DistributedDataParallel",
+        "module": "$@network_def.to(@device)",
+        "device_ids": [
+            "@device"
+        ]
+    },
+    "train#sampler": {
+        "_target_": "DistributedSampler",
+        "dataset": "@train#dataset",
+        "even_divisible": true,
+        "shuffle": true
+    },
+    "train#dataloader#sampler": "@train#sampler",
+    "train#dataloader#shuffle": false,
+    "train#trainer#train_handlers": "$@train#handlers[: -2 if dist.get_rank() > 0 else None]",
+    "training": [
+        "$import torch.distributed as dist",
+        "$dist.init_process_group(backend='nccl')",
+        "$torch.cuda.set_device(@device)",
+        "$monai.utils.set_determinism(seed=123)",
+        "$setattr(torch.backends.cudnn, 'benchmark', True)",
+        "$import logging",
+        "$@train#trainer.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)",
+        "$@train#trainer.run()",
+        "$dist.destroy_process_group()"
+    ]
+}
diff --git a/tests/utils.py b/tests/utils.py
index 1f632261dd..b16b4b13fb 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -137,6 +137,8 @@ def skip_if_downloading_fails():
             raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e
         if "md5 check" in str(rt_e):
             raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e
+        if "account limit" in str(rt_e):  # HTTP Error 503: Egress is over the account limit
+            raise unittest.SkipTest(f"error while downloading: {rt_e}") from rt_e
         raise rt_e
 
 
@@ -404,6 +406,7 @@ def __init__(
             timeout: Timeout for operations executed against the process group.
             init_method: URL specifying how to initialize the process group.
                 Default is "env://" or "file:///d:/a_temp" (windows) if unspecified.
+                If ``"no_init"``, the `dist.init_process_group` must be called within the code to be tested.
             backend: The backend to use. Depending on build-time configurations,
                 valid values include ``mpi``, ``gloo``, and ``nccl``.
             daemon: the process’s daemon flag.
@@ -451,13 +454,14 @@ def run_process(self, func, local_rank, args, kwargs, results):
             if torch.cuda.is_available():
                 torch.cuda.set_device(int(local_rank))  # using device ids from CUDA_VISIBILE_DEVICES
 
-            dist.init_process_group(
-                backend=self.backend,
-                init_method=self.init_method,
-                timeout=self.timeout,
-                world_size=int(os.environ["WORLD_SIZE"]),
-                rank=int(os.environ["RANK"]),
-            )
+            if self.init_method != "no_init":
+                dist.init_process_group(
+                    backend=self.backend,
+                    init_method=self.init_method,
+                    timeout=self.timeout,
+                    world_size=int(os.environ["WORLD_SIZE"]),
+                    rank=int(os.environ["RANK"]),
+                )
             func(*args, **kwargs)
             # the primary node lives longer to
             # avoid _store_based_barrier, RuntimeError: Broken pipe