diff --git a/.dockerignore b/.dockerignore
index 549e63bad5..4e1161bfb2 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -4,8 +4,10 @@ __pycache__/
 docs/
 
 .coverage
+.coverage.*
+.coverage/
+coverage.xml
 .readthedocs.yml
-*.md
 *.toml
 
 !README.md
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 61b8814857..f7024f1a08 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -11,7 +11,7 @@ A few sentences describing the changes proposed in this pull request.
 - [x] Non-breaking change (fix or new feature that would not break existing functionality).
 - [ ] Breaking change (fix or new feature that would cause existing functionality to change).
 - [ ] New tests added to cover the changes.
-- [ ] Integration tests passed locally by running `./runtests.sh --codeformat --coverage`.
-- [ ] Quick tests passed locally by running `./runtests.sh --quick`.
+- [ ] Integration tests passed locally by running `./runtests.sh -f -u --net --coverage`.
+- [ ] Quick tests passed locally by running `./runtests.sh --quick --unittests`.
 - [ ] In-line docstrings updated.
 - [ ] Documentation updated, tested `make html` command in the `docs/` folder.
diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
index e568ba9e15..e215ec98e4 100644
--- a/.github/workflows/cron.yml
+++ b/.github/workflows/cron.yml
@@ -3,6 +3,8 @@ name: crons
 on:
   schedule:
     - cron: "0 2 * * *"  # at 02:00 UTC
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
 
 jobs:
   cron-gpu:
@@ -13,7 +15,7 @@ jobs:
     runs-on: [self-hosted, linux, x64, common]
     strategy:
       matrix:
-        pytorch-version: [1.5.0, 1.5.1, 1.6.0, latest]
+        pytorch-version: [1.5.1, 1.6.0, 1.7.1, latest]
     steps:
     - uses: actions/checkout@v2
     - name: Install the dependencies
@@ -23,15 +25,12 @@ jobs:
         python -m pip uninstall -y torch torchvision
         if [ ${{ matrix.pytorch-version }} == "latest" ]; then
           python -m pip install torch torchvision
-        elif [ ${{ matrix.pytorch-version }} == "1.5.0" ]; then
-          python -m pip install torch==1.5.0
-          python -m pip install torchvision==0.6.0
         elif [ ${{ matrix.pytorch-version }} == "1.5.1" ]; then
-          python -m pip install torch==1.5.1
-          python -m pip install torchvision==0.6.1
+          python -m pip install torch==1.5.1 torchvision==0.6.1
         elif [ ${{ matrix.pytorch-version }} == "1.6.0" ]; then
-          python -m pip install torch==1.6.0
-          python -m pip install torchvision==0.7.0
+          python -m pip install torch==1.6.0 torchvision==0.7.0
+        elif [ ${{ matrix.pytorch-version }} == "1.7.1" ]; then
+          python -m pip install torch==1.7.1 torchvision==0.8.2
         fi
         python -m pip install -r requirements-dev.txt
         python -m pip list
@@ -43,10 +42,14 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
-        python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --coverage
+        python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
+        BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
+        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
+        if pgrep python; then pkill python; fi
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with:
@@ -56,12 +59,16 @@ jobs:
   cron-pt-image:
     if: github.repository == 'Project-MONAI/MONAI'
     container:
-      image: nvcr.io/nvidia/pytorch:20.12-py3  # testing with the latest pytorch base image
+      image: nvcr.io/nvidia/pytorch:21.02-py3  # testing with the latest pytorch base image
       options: "--gpus all"
     runs-on: [self-hosted, linux, x64, common]
     steps:
     - uses: actions/checkout@v2
-    - name: Install the dependencies
+    - name: Install APT dependencies
+      run: |
+        apt-get update
+        DEBIAN_FRONTEND="noninteractive" apt-get install -y libopenslide0
+    - name: Install Python dependencies
       run: |
         which python
         python -m pip install --upgrade pip wheel
@@ -75,10 +82,14 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
-        python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --coverage
+        python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
+        BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
+        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
+        if pgrep python; then pkill python; fi
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with:
@@ -100,13 +111,54 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
         ngc --version
-        BUILD_MONAI=1 ./runtests.sh --coverage --pytype
+        BUILD_MONAI=1 ./runtests.sh --coverage --pytype --unittests  # unit tests with pytype checks, coverage report
+        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
+        if pgrep python; then pkill python; fi
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with:
         fail_ci_if_error: false
         file: ./coverage.xml
+
+  cron-tutorial-notebooks:
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: cron-gpu  # so that monai itself is verified first
+    container:
+      image: nvcr.io/nvidia/pytorch:21.02-py3  # testing with the latest pytorch base image
+      options: "--gpus all --ipc=host"
+    runs-on: [self-hosted, linux, x64, common]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Install MONAI
+      id: monai-install
+      run: |
+        which python
+        python -m pip install --upgrade pip wheel
+        python -m pip install -r requirements-dev.txt
+        BUILD_MONAI=0 python setup.py develop  # install monai
+        nvidia-smi
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        echo "::set-output name=devices::$CUDA_VISIBLE_DEVICES"
+    - name: Checkout tutorials and install their requirements
+      run: |
+        cd /opt
+        git clone --depth 1 --branch master --single-branch https://github.com/Project-MONAI/tutorials.git  # latest commit of master branch
+        cd tutorials
+        python -m pip install -r requirements.txt
+    - name: Run tutorial notebooks
+      timeout-minutes: 150
+      run: |
+        export CUDA_VISIBLE_DEVICES=${{ steps.monai-install.outputs.devices }}
+        echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
+        cd /opt/tutorials
+        $(pwd)/runner.sh
+        if pgrep python; then pkill python; fi
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
new file mode 100644
index 0000000000..32f1fd2056
--- /dev/null
+++ b/.github/workflows/docker.yml
@@ -0,0 +1,123 @@
+name: docker
+# versioning: compute a static version file
+# local_docker: use the version file to build docker images
+# docker_test_latest: test the latest internal docker image (has flake)
+# docker_test_dockerhub: test the latest dockerhub release (no flake)
+on:
+  # master only docker deployment and quick tests
+  push:
+    branches:
+      - master
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  versioning:
+    # compute versioning file from python setup.py
+    # upload as artifact
+    # (also used in release.yml)
+    if: github.repository == 'Project-MONAI/MONAI'
+    container:
+      image: localhost:5000/local_monai:latest
+    runs-on: [self-hosted, linux, x64, build_only]
+    steps:
+      - uses: actions/checkout@v2
+        # full history so that we can git describe
+        with:
+          ref: master
+          fetch-depth: 0
+      - shell: bash
+        run: |
+          git describe
+          python setup.py build
+          cat build/lib/monai/_version.py
+      - name: Upload version
+        uses: actions/upload-artifact@v2
+        with:
+          name: _version.py
+          path: build/lib/monai/_version.py
+      - name: Clean up directory
+        shell: bash
+        run: |
+          ls -al
+          rm -rf {*,.[^.]*}
+
+  local_docker:
+    # builds two versions: local_monai:latest and local_monai:dockerhub
+    # latest: used for local tests
+    # dockerhub: release, no flake package
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: versioning
+    runs-on: [self-hosted, linux, x64, build_only]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: master
+    - name: Download version
+      uses: actions/download-artifact@v2
+      with:
+        name: _version.py
+    - name: docker_build
+      shell: bash
+      run: |
+        # get tag info for versioning
+        cat _version.py
+        mv _version.py monai/
+        # build and run original docker image for local registry
+        docker build -t localhost:5000/local_monai:latest -f Dockerfile .
+        docker push localhost:5000/local_monai:latest
+        # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com
+        sed -i '/flake/d' requirements-dev.txt
+        docker build -t projectmonai/monai:latest -f Dockerfile .
+        # also push as tag "dockerhub" to local registry
+        docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub
+        docker push localhost:5000/local_monai:dockerhub
+        # distribute as always w/ tag "latest" to hub.docker.com
+        echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin
+        docker push projectmonai/monai:latest
+        docker logout
+        docker image prune -f
+
+  docker_test_latest:
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: local_docker
+    container:
+      image: localhost:5000/local_monai:latest
+    runs-on: [self-hosted, linux, x64, common]
+    steps:
+    - name: Import
+      run: |
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
+        python -c 'import monai; monai.config.print_config()'
+        cd /opt/monai
+        ls -al
+        ngc --version
+        python -m tests.min_tests
+        if pgrep python; then pkill python; fi
+      env:
+        QUICKTEST: True
+
+  docker_test_dockerhub:
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: local_docker
+    container:
+      image: localhost:5000/local_monai:dockerhub
+    runs-on: [self-hosted, linux, x64, common]
+    steps:
+    - name: Import
+      run: |
+        export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
+        echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
+        python -c 'import monai; monai.config.print_config()'
+        cd /opt/monai
+        ls -al
+        ngc --version
+        python -m tests.min_tests
+        if pgrep python; then pkill python; fi
+      env:
+        QUICKTEST: True
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index 003a746de4..e78393f357 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -7,7 +7,7 @@ on:
 jobs:
   integration-py3:
     container:
-      image: nvcr.io/nvidia/pytorch:20.03-py3  # CUDA 10.2
+      image: nvcr.io/nvidia/pytorch:20.12-py3  # CUDA 11.1
       options: --gpus all
     runs-on: [self-hosted, linux, x64, common]
     steps:
@@ -28,13 +28,13 @@ jobs:
         path: |
           ~/.cache/pip
           ~/.cache/torch
-        key: docker-20-03-py3-pip-${{ steps.pip-cache.outputs.datew }}
+        key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }}
     - name: Install the dependencies
       run: |
         which python
         python -m pip install --upgrade pip wheel
         python -m pip uninstall -y torch torchvision
-        python -m pip install torch==1.7.1 torchvision==0.8.2
+        python -m pip install torch==1.8.1+cu111 torchvision==0.9.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html
         python -m pip install -r requirements-dev.txt
     - name: Run integration tests
       run: |
@@ -42,9 +42,14 @@ jobs:
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
         python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
         BUILD_MONAI=1 ./runtests.sh --net
+        BUILD_MONAI=1 ./runtests.sh --unittests
+        if pgrep python; then pkill python; fi
+      shell: bash
     - name: Add reaction
       uses: peter-evans/create-or-update-comment@v1
       with:
diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml
index 8e92ea0ed7..229c9cf784 100644
--- a/.github/workflows/pythonapp.yml
+++ b/.github/workflows/pythonapp.yml
@@ -9,7 +9,7 @@ on:
 
 jobs:
   # caching of these jobs:
-  #   - docker-20-03-py3-pip- (shared)
+  #   - docker-py3-pip- (shared)
   #   - ubuntu py37 pip-
   #   - os-latest-pip- (shared)
   flake8-py3:
@@ -39,9 +39,9 @@ jobs:
         # clean up temporary files
         $(pwd)/runtests.sh --clean
         # Git hub actions have 2 cores, so parallize pytype
-        $(pwd)/runtests.sh --nounittests --codeformat -j 2
+        $(pwd)/runtests.sh --codeformat -j 2
 
-  quick-py3:  # full dependencies installed
+  quick-py3:  # full dependencies installed tests for different OS
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -80,13 +80,16 @@ jobs:
     - if: runner.os == 'windows'
       name: Install torch cpu from pytorch.org (Windows only)
       run: |
-        python -m pip install torch==1.7.1+cpu torchvision==0.8.2+cpu -f https://download.pytorch.org/whl/torch_stable.html
+        python -m pip install torch==1.8.1+cpu torchvision==0.9.1+cpu -f https://download.pytorch.org/whl/torch_stable.html
         # min. requirements for windows instances
         python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines(txt[1:12]); f.close()"
+    - if: runner.os == 'macos'
+      name: Remove cucim installation (Mac only)
+      run: |
+        python -c "f=open('requirements-dev.txt', 'r'); txt=f.readlines(); f.close(); print(txt); f=open('requirements-dev.txt', 'w'); f.writelines([t for t in txt if not t.startswith('cucim')]); f.close()"
     - name: Install the dependencies
       run: |
-        python -m pip install torch==1.7.1
-        python -m pip install torchvision==0.8.2
+        python -m pip install torch==1.8.1 torchvision==0.9.1
         cat "requirements-dev.txt"
         python -m pip install -r requirements-dev.txt
         python -m pip list
@@ -102,7 +105,7 @@ jobs:
       env:
         QUICKTEST: True
 
-  min-dep-py3:  # min dependencies installed
+  min-dep-os:  # min dependencies installed tests for different OS
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -134,11 +137,56 @@ jobs:
     - if: runner.os == 'windows'
       name: Install torch cpu from pytorch.org (Windows only)
       run: |
-        python -m pip install torch==1.7.1+cpu -f https://download.pytorch.org/whl/torch_stable.html
+        python -m pip install torch==1.8.1+cpu -f https://download.pytorch.org/whl/torch_stable.html
+    - name: Install the dependencies
+      run: |
+        # min. requirements
+        python -m pip install torch==1.8.1
+        python -m pip install -r requirements-min.txt
+        python -m pip list
+        BUILD_MONAI=0 python setup.py develop  # no compile of extensions
+      shell: bash
+    - name: Run quick tests (CPU ${{ runner.os }})
+      run: |
+        python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
+        python -c "import monai; monai.config.print_config()"
+        python -m tests.min_tests
+      env:
+        QUICKTEST: True
+
+  min-dep-py3:  # min dependencies installed tests for different python
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.6, 3.7]
+    timeout-minutes: 40
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Prepare pip wheel
+      run: |
+        which python
+        python -m pip install --user --upgrade pip setuptools wheel
+    - name: cache weekly timestamp
+      id: pip-cache
+      run: |
+        echo "::set-output name=datew::$(date '+%Y-%V')"
+        echo "::set-output name=dir::$(pip cache dir)"
+      shell: bash
+    - name: cache for pip
+      uses: actions/cache@v2
+      id: cache
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: ubuntu-latest-latest-pip-${{ steps.pip-cache.outputs.datew }}
     - name: Install the dependencies
       run: |
         # min. requirements
-        python -m pip install torch==1.7.1
+        python -m pip install torch==1.8.1
         python -m pip install -r requirements-min.txt
         python -m pip list
         BUILD_MONAI=0 python setup.py develop  # no compile of extensions
@@ -156,15 +204,13 @@ jobs:
     strategy:
       matrix:
         environment:
-          - "PT15+CUDA101"
           - "PT16+CUDA102"
           - "PT16+CUDA110"
           - "PT17+CUDA102"
           - "PT17+CUDA110"
+          - "PT18+CUDA102"
+          - "PT18+CUDA112"
         include:
-          - environment: PT15+CUDA101
-            pytorch: "torch==1.5.0+cu101 torchvision==0.6.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html"
-            base: "nvcr.io/nvidia/cuda:10.1-devel-ubuntu18.04"
           - environment: PT16+CUDA102
             pytorch: "torch==1.6.0 torchvision==0.7.0"
             base: "nvcr.io/nvidia/cuda:10.2-devel-ubuntu18.04"
@@ -179,6 +225,13 @@ jobs:
             # we explicitly set pytorch to -h to avoid pip install error
             pytorch: "-h"
             base: "nvcr.io/nvidia/pytorch:20.09-py3"
+          - environment: PT18+CUDA102
+            pytorch: "torch==1.8.1 torchvision==0.9.1"
+            base: "nvcr.io/nvidia/cuda:10.2-devel-ubuntu18.04"
+          - environment: PT18+CUDA112
+            # we explicitly set pytorch to -h to avoid pip install error
+            pytorch: "-h"
+            base: "nvcr.io/nvidia/pytorch:21.02-py3"
     container:
       image: ${{ matrix.base }}
       options: --gpus all
@@ -187,7 +240,10 @@ jobs:
     - uses: actions/checkout@v2
     - name: apt install
       run: |
-        if [ ${{ matrix.environment }} != "PT16+CUDA110" ]; then \
+        if [ ${{ matrix.environment }} = "PT16+CUDA102" ] || \
+           [ ${{ matrix.environment }} = "PT17+CUDA102" ] || \
+           [ ${{ matrix.environment }} = "PT18+CUDA102" ]
+        then
         PYVER=3.6 PYSFX=3 DISTUTILS=python3-distutils && \
         apt-get update && apt-get install -y --no-install-recommends \
           curl \
@@ -217,28 +273,36 @@ jobs:
         ln -s /usr/bin/python$PYVER /usr/bin/python`echo $PYVER | cut -c1-1` &&
         curl -O https://bootstrap.pypa.io/get-pip.py && \
         python get-pip.py && \
-        rm get-pip.py ; fi
+        rm get-pip.py;
+        fi
     - name: Install dependencies
       run: |
         which python
         python -m pip install --upgrade pip wheel
         python -m pip install ${{ matrix.pytorch }}
         python -m pip install -r requirements-dev.txt
+        python -m pip list
     - name: Run quick tests (GPU)
       run: |
-        python -m pip list
         nvidia-smi
+        export LAUNCH_DELAY=$(python -c "import numpy; print(numpy.random.randint(30) * 10)")
+        echo "Sleep $LAUNCH_DELAY"
+        sleep $LAUNCH_DELAY
         export CUDA_VISIBLE_DEVICES=$(coverage run -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
-        python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
+        python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
         python -c "import monai; monai.config.print_config()"
-        BUILD_MONAI=1 ./runtests.sh --quick
-        if [ ${{ matrix.environment }} == "PT16+CUDA110" ]; then
+        BUILD_MONAI=1 ./runtests.sh --quick --unittests
+        if [ ${{ matrix.environment }} = "PT18+CUDA112" ]; then
           # test the clang-format tool downloading once
           coverage run -m tests.clang_format_utils
         fi
         coverage xml
+        if pgrep python; then pkill python; fi
+      shell: bash
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with:
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 840194b1da..00e28ecd52 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -83,3 +83,70 @@ jobs:
         password: ${{ secrets.TEST_PYPI }}
         repository_url: https://test.pypi.org/legacy/
 
+  versioning:
+    # compute versioning file from python setup.py
+    # upload as artifact
+    # (also used in docker.yml)
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: packaging
+    container:
+      image: localhost:5000/local_monai:latest
+    runs-on: [self-hosted, linux, x64, build_only]
+    steps:
+      - uses: actions/checkout@v2
+        # full history so that we can git describe
+        with:
+          ref: master
+          fetch-depth: 0
+      - shell: bash
+        run: |
+          git describe
+          python setup.py build
+          cat build/lib/monai/_version.py
+      - name: Upload version
+        uses: actions/upload-artifact@v2
+        with:
+          name: _version.py
+          path: build/lib/monai/_version.py
+      - name: Clean up directory
+        shell: bash
+        run: |
+          ls -al
+          rm -rf {*,.[^.]*}
+
+  release_tag_docker:
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: versioning
+    runs-on: [self-hosted, linux, x64, build_only]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: master
+      - name: Download version
+        uses: actions/download-artifact@v2
+        with:
+          name: _version.py
+      - name: Set tag
+        id: versioning
+        run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
+      - name: Check tag
+        env:
+          RELEASE_VERSION: ${{ steps.versioning.outputs.tag }}
+        run: |
+          echo "$RELEASE_VERSION"
+          cat _version.py
+      - if: startsWith(github.ref, 'refs/tags/')
+        name: build with the tag
+        env:
+          RELEASE_VERSION: ${{ steps.versioning.outputs.tag }}
+        shell: bash
+        run: |
+          # get tag info for versioning
+          mv _version.py monai/
+          # remove flake package as it is not needed on hub.docker.com
+          sed -i '/flake/d' requirements-dev.txt
+          docker build -t projectmonai/monai:"$RELEASE_VERSION" -f Dockerfile .
+          # distribute with a tag to hub.docker.com
+          echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin
+          docker push projectmonai/monai:"$RELEASE_VERSION"
+          docker logout
diff --git a/.github/workflows/setupapp.yml b/.github/workflows/setupapp.yml
index 7656eb4828..1b4c37b6e8 100644
--- a/.github/workflows/setupapp.yml
+++ b/.github/workflows/setupapp.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   # caching of these jobs:
-  #   - docker-20-03-py3-pip- (shared)
+  #   - docker-py3-pip- (shared)
   #   - ubuntu py36 37 38-pip-
   #   - os-latest-pip (shared)
   coverage-py3:
@@ -30,7 +30,7 @@ jobs:
         path: |
           ~/.cache/pip
           ~/.cache/torch
-        key: docker-20-03-py3-pip-${{ steps.pip-cache.outputs.datew }}
+        key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }}
     - name: Install the dependencies
       run: |
         which python
@@ -41,17 +41,25 @@ jobs:
     - name: Run unit tests report coverage
       run: |
         python -m pip list
+        export LAUNCH_DELAY=$[ $RANDOM % 16 * 60 ]
+        echo "Sleep $LAUNCH_DELAY"
+        sleep $LAUNCH_DELAY
         nvidia-smi
         export CUDA_VISIBLE_DEVICES=$(python -m tests.utils)
         echo $CUDA_VISIBLE_DEVICES
+        trap 'if pgrep python; then pkill python; fi;' ERR
+        python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null &
         python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))"
-        python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))'
-        BUILD_MONAI=1 ./runtests.sh --coverage
+        python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))'
+        BUILD_MONAI=1 ./runtests.sh --coverage --unittests  # unit tests with coverage report
+        BUILD_MONAI=1 ./runtests.sh --coverage --net  # integration tests with coverage report
         coverage xml
+        if pgrep python; then pkill python; fi
+      shell: bash
     - name: Upload coverage
       uses: codecov/codecov-action@v1
       with:
-        fail_ci_if_error: true
+        fail_ci_if_error: false
         file: ./coverage.xml
 
   test-py3x:
@@ -88,7 +96,7 @@ jobs:
       run: |
         python -m pip list
         python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))'
-        BUILD_MONAI=1 ./runtests.sh --quick
+        BUILD_MONAI=1 ./runtests.sh --quick --unittests
         coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1
@@ -138,44 +146,3 @@ jobs:
         python -m tests.min_tests
       env:
         QUICKTEST: True
-
-  local_docker:
-    if: github.repository == 'Project-MONAI/MONAI'
-    runs-on: [self-hosted, linux, x64, build_only]
-    # we only push built container if it is built from master branch
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        ref: master
-    - name: docker_build
-      run: |
-        # build and run original docker image for local registry
-        docker build -t localhost:5000/local_monai:latest -f Dockerfile .
-        docker push localhost:5000/local_monai:latest
-        # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com
-        sed -i '/flake/d' requirements-dev.txt
-        docker build -t projectmonai/monai:latest -f Dockerfile .
-        # also push as tag "dockerhub" to local registry
-        docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub
-        docker push localhost:5000/local_monai:dockerhub
-        # distribute as always w/ tag "latest" to hub.docker.com
-        echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin
-        docker push projectmonai/monai:latest
-        docker logout
-
-  docker:
-    if: github.repository == 'Project-MONAI/MONAI'
-    needs: local_docker
-    container:
-      image: localhost:5000/local_monai:latest
-    runs-on: [self-hosted, linux, x64, common]
-    steps:
-    - name: Import
-      run: |
-        python -c 'import monai; monai.config.print_config()'
-        cd /opt/monai
-        ls -al
-        ngc --version
-        python -m tests.min_tests
-      env:
-        QUICKTEST: True
diff --git a/.github/workflows/weekly-preview.yml b/.github/workflows/weekly-preview.yml
index bb68a0801d..df5f52f57a 100644
--- a/.github/workflows/weekly-preview.yml
+++ b/.github/workflows/weekly-preview.yml
@@ -32,7 +32,7 @@ jobs:
         export YEAR_WEEK=$(date +'%y%U')
         echo "Year week for tag is ${YEAR_WEEK}"
         if ! [[ $YEAR_WEEK =~ ^[0-9]{4}$ ]] ; then echo "Wrong 'year week' format.  Should be 4 digits."; exit 1 ; fi
-        git tag "0.5.dev${YEAR_WEEK}"
+        git tag "0.6.dev${YEAR_WEEK}"
         git log -1
         git tag --list
         python setup.py sdist bdist_wheel
diff --git a/.gitignore b/.gitignore
index 0d1455d70d..7444d7f2f9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,6 +40,7 @@ htmlcov/
 .tox/
 .coverage
 .coverage.*
+.coverage/
 .cache
 nosetests.xml
 coverage.xml
@@ -47,6 +48,9 @@ coverage.xml
 .hypothesis/
 .pytest_cache/
 
+# temporary unittest artifacts
+tests/testing_data/temp_*
+
 # Translations
 *.mo
 *.pot
@@ -124,6 +128,7 @@ temp/
 # temporary testing data MedNIST
 tests/testing_data/MedNIST*
 tests/testing_data/*Hippocampus*
+tests/testing_data/*.tiff
 
 # clang format tool
 .clang-format-bin/
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 56e65a7d92..2a6882d08b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,87 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.5.0] - 2020-04-09
+### Added
+* Overview document for [feature highlights in v0.5.0](https://github.com/Project-MONAI/MONAI/blob/master/docs/source/highlights.md)
+* Invertible spatial transforms
+  * `InvertibleTransform` base APIs
+  * Batch inverse and decollating APIs
+  * Inverse of `Compose`
+  * Batch inverse event handling
+  * Test-time augmentation as an application
+* Initial support of learning-based image registration:
+  * Bending energy, LNCC, and global mutual information loss
+  * Fully convolutional architectures
+  * Dense displacement field, dense velocity field computation
+  * Warping with high-order interpolation with C++/CUDA implementations
+* Deepgrow modules for interactive segmentation:
+  * Workflows with simulations of clicks
+  * Distance-based transforms for guidance signals
+* Digital pathology support:
+  * Efficient whole slide imaging IO and sampling with Nvidia cuCIM and SmartCache
+  * FROC measurements for lesion
+  * Probabilistic post-processing for lesion detection
+  * TorchVision classification model adaptor for fully convolutional analysis
+* 12 new transforms, grid patch dataset, `ThreadDataLoader`, EfficientNets B0-B7
+* 4 iteration events for the engine for finer control of workflows
+* New C++/CUDA extensions:
+  * Conditional random field
+  * Fast bilateral filtering using the permutohedral lattice
+* Metrics summary reporting and saving APIs
+* DiceCELoss, DiceFocalLoss, a multi-scale wrapper for segmentation loss computation
+* Data loading utilities：
+  * `decollate_batch`
+  * `PadListDataCollate` with inverse support
+* Support of slicing syntax for `Dataset`
+* Initial Torchscript support for the loss modules
+* Learning rate finder
+* Allow for missing keys in the dictionary-based transforms
+* Support of checkpoint loading for transfer learning
+* Various summary and plotting utilities for Jupyter notebooks
+* Contributor Covenant Code of Conduct
+* Major CI/CD enhancements covering the tutorial repository
+* Fully compatible with PyTorch 1.8
+* Initial nightly CI/CD pipelines using Nvidia Blossom Infrastructure
+
+### Changed
+* Enhanced `list_data_collate` error handling
+* Unified iteration metric APIs
+* `densenet*` extensions are renamed to `DenseNet*`
+* `se_res*` network extensions are renamed to `SERes*`
+* Transform base APIs are rearranged into `compose`, `inverse`, and `transform`
+* `_do_transform` flag for the random augmentations is unified via `RandomizableTransform`
+* Decoupled post-processing steps, e.g. `softmax`, `to_onehot_y`, from the metrics computations
+* Moved the distributed samplers to `monai.data.samplers` from `monai.data.utils`
+* Engine's data loaders now accept generic iterables as input
+* Workflows now accept additional custom events and state properties
+* Various type hints according to Numpy 1.20
+* Refactored testing utility `runtests.sh` to have `--unittest` and `--net` (integration tests) options
+* Base Docker image upgraded to `nvcr.io/nvidia/pytorch:21.02-py3` from `nvcr.io/nvidia/pytorch:20.10-py3`
+* Docker images are now built with self-hosted environments
+* Primary contact email updated to `monai.contact@gmail.com`
+* Now using GitHub Discussions as the primary communication forum
+
+### Removed
+* Compatibility tests for PyTorch 1.5.x
+* Format specific loaders, e.g. `LoadNifti`, `NiftiDataset`
+* Assert statements from non-test files
+* `from module import *` statements, addressed flake8 F403
+
+### Fixed
+* Uses American English spelling for code, as per PyTorch
+* Code coverage now takes multiprocessing runs into account
+* SmartCache with initial shuffling
+* `ConvertToMultiChannelBasedOnBratsClasses` now supports channel-first inputs
+* Checkpoint handler to save with non-root permissions
+* Fixed an issue for exiting the distributed unit tests
+* Unified `DynUNet` to have single tensor output w/o deep supervision
+* `SegmentationSaver` now supports user-specified data types and a `squeeze_end_dims` flag
+* Fixed `*Saver` event handlers output filenames with a `data_root_dir` option
+* Load image functions now ensure little-endian
+* Fixed the test runner to support regex-based test case matching
+* Usability issues in the event handlers
+
 ## [0.4.0] - 2020-12-15
 ### Added
 * Overview document for [feature highlights in v0.4.0](https://github.com/Project-MONAI/MONAI/blob/master/docs/source/highlights.md)
@@ -173,7 +254,8 @@ the postprocessing steps should be used before calling the metrics methods
 
 [highlights]: https://github.com/Project-MONAI/MONAI/blob/master/docs/source/highlights.md
 
-[Unreleased]: https://github.com/Project-MONAI/MONAI/compare/0.4.0...HEAD
+[Unreleased]: https://github.com/Project-MONAI/MONAI/compare/0.5.0...HEAD
+[0.5.0]: https://github.com/Project-MONAI/MONAI/compare/0.4.0...0.5.0
 [0.4.0]: https://github.com/Project-MONAI/MONAI/compare/0.3.0...0.4.0
 [0.3.0]: https://github.com/Project-MONAI/MONAI/compare/0.2.0...0.3.0
 [0.2.0]: https://github.com/Project-MONAI/MONAI/compare/0.1.0...0.2.0
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 01a4773b5a..79cbbfabac 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -24,7 +24,7 @@ We are happy to talk with you about your needs for MONAI and your ideas for cont
 
 ### Does it belong in PyTorch instead of MONAI?
 
-MONAI is based on the PyTorch and Numpy libraries. These libraries implement what we consider to be best practice for general scientific computing and deep learning functionality. MONAI builds on these with a strong focus on medical applications. As such, it is a good idea to consider whether your functionality is medical-application specific or not. General deep learning functionality may be better off in PyTorch; you can find their contribution guidelines [here](https://pytorch.org/docs/stable/community/contribution_guide.html).
+MONAI is part of [PyTorch Ecosystem](https://pytorch.org/ecosystem/), and mainly based on the PyTorch and Numpy libraries. These libraries implement what we consider to be best practice for general scientific computing and deep learning functionality. MONAI builds on these with a strong focus on medical applications. As such, it is a good idea to consider whether your functionality is medical-application specific or not. General deep learning functionality may be better off in PyTorch; you can find their contribution guidelines [here](https://pytorch.org/docs/stable/community/contribution_guide.html).
 
 ## The contribution process
 
@@ -51,11 +51,15 @@ Coding style is checked and enforced by flake8, black, and isort, using [a flake
 Before submitting a pull request, we recommend that all linting should pass, by running the following command locally:
 
 ```bash
-pip install -U -r requirements-dev.txt  # install the latest tools
-./runtests.sh --codeformat --nounittests  # runs the linting tools only
+# optionally update the dependencies and dev tools
+python -m pip install -U pip
+python -m pip install -U -r requirements-dev.txt
+
+# run the linting and type checking tools
+./runtests.sh --codeformat
 
 # try to fix the coding style errors automatically
-./runtests.sh --autofix --nounittests
+./runtests.sh --autofix
 ```
 
 License information: all source code files should start with this paragraph:
@@ -86,7 +90,7 @@ MONAI tests are located under `tests/`.
 - The unit test's file name follows `test_[module_name].py`.
 - The integration test's file name follows `test_integration_[workflow_name].py`.
 
-A bash script (`runtests.sh`) is provided to run all tests locally
+A bash script (`runtests.sh`) is provided to run all tests locally.
 Please run ``./runtests.sh -h`` to see all options.
 
 To run a particular test, for example `tests/test_dice_loss.py`:
@@ -98,16 +102,16 @@ Before submitting a pull request, we recommend that all linting and unit tests
 should pass, by running the following command locally:
 
 ```bash
-./runtests.sh --codeformat --coverage
+./runtests.sh -f -u --net --coverage
 ```
 or (for new features that would not break existing functionality):
 
 ```bash
-./runtests.sh --quick
+./runtests.sh --quick --unittests
 ```
 
 It is recommended that the new test `test_[module_name].py` is constructed by using only
-python 3.6+ build-in functions, `torch`, `numpy`, and `parameterized` packages.
+python 3.6+ build-in functions, `torch`, `numpy`, `coverage` (for reporting code coverages) and `parameterized` (for organising test cases) packages.
 If it requires any other external packages, please make sure:
 - the packages are listed in [`requirements-dev.txt`](requirements-dev.txt)
 - the new test `test_[module_name].py` is added to the `exclude_cases` in [`./tests/min_tests.py`](./tests/min_tests.py) so that
@@ -141,7 +145,7 @@ Before submitting a pull request, it is recommended to:
 - check the auto-generated documentation (by browsing `./docs/build/html/index.html` with a web browser)
 - type `make clean` in `docs/` folder to remove the current build files.
 
-Please type `make help` for all supported format options.
+Please type `make help` in `docs/` folder for all supported format options.
 
 #### Automatic code formatting
 MONAI provides support of automatic Python code formatting via [a customised GitHub action](https://github.com/Project-MONAI/monai-code-formatter).
@@ -251,6 +255,7 @@ All code review comments should be specific, constructive, and actionable.
 1. Read carefully the descriptions of the pull request and the files changed, write comments if needed.
 1. Make in-line comments to specific code segments, [request for changes](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-request-reviews) if needed.
 1. Review any further code changes until all comments addressed by the contributors.
+1. Comment to trigger `/black` and/or `/integration-test` for optional auto code formatting and [integration tests](.github/workflows/integration.yml).
 1. Merge the pull request to the master branch.
 1. Close the corresponding task ticket on [the issue list][monai issue list].
 
diff --git a/Dockerfile b/Dockerfile
index 47976b97b1..23be9ae1c3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,8 +9,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:20.12-py3
-
+# To build with a different base image
+# please run `docker build` using the `--build-arg PYTORCH_IMAGE=...` flag.
+ARG PYTORCH_IMAGE=nvcr.io/nvidia/pytorch:21.02-py3
 FROM ${PYTORCH_IMAGE}
 
 LABEL maintainer="monai.contact@gmail.com"
@@ -29,10 +30,9 @@ RUN cp /tmp/requirements.txt /tmp/req.bak \
 # please specify exact files and folders to be copied -- else, basically always, the Docker build process cannot cache
 # this or anything below it and always will build from at most here; one file change leads to no caching from here on...
 
-COPY LICENSE setup.py setup.cfg versioneer.py runtests.sh .gitignore .gitattributes README.md MANIFEST.in ./
+COPY LICENSE CHANGELOG.md CODE_OF_CONDUCT.md CONTRIBUTING.md README.md versioneer.py setup.py setup.cfg runtests.sh MANIFEST.in ./
 COPY tests ./tests
 COPY monai ./monai
-COPY .git ./.git
 RUN BUILD_MONAI=1 FORCE_CUDA=1 python setup.py develop \
   && rm -rf build __pycache__
 
@@ -43,6 +43,9 @@ RUN wget -q ${NGC_CLI_URI} && \
     unzip ngccli_cat_linux.zip && chmod u+x ngc && \
     md5sum -c ngc.md5 && \
     rm -rf ngccli_cat_linux.zip ngc.md5
+RUN apt-get update \
+  && DEBIAN_FRONTEND="noninteractive" apt-get install -y libopenslide0  \
+  && rm -rf /var/lib/apt/lists/*
 # append /opt/tools to runtime path for NGC CLI to be accessible from all file system locations
 ENV PATH=${PATH}:/opt/tools
 WORKDIR /opt/monai
diff --git a/README.md b/README.md
index f06a2d146f..a36bf0ef2f 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ Its ambitions are:
 
 ## Features
 > _The codebase is currently under active development._
-> _Please see [the technical highlights](https://docs.monai.io/en/latest/highlights.html) of the current milestone release._
+> _Please see [the technical highlights](https://docs.monai.io/en/latest/highlights.html) and [What's New](https://docs.monai.io/en/latest/whatsnew.html) of the current milestone release._
 
 - flexible pre-processing for multi-dimensional medical imaging data;
 - compositional & portable APIs for ease of integration in existing workflows;
@@ -62,3 +62,6 @@ Ask and answer questions over on [MONAI's GitHub Discussions tab](https://github
 - Issue tracker: https://github.com/Project-MONAI/MONAI/issues
 - Wiki: https://github.com/Project-MONAI/MONAI/wiki
 - Test status: https://github.com/Project-MONAI/MONAI/actions
+- PyPI package: https://pypi.org/project/monai/
+- Weekly previews: https://pypi.org/project/monai-weekly/
+- Docker Hub: https://hub.docker.com/r/projectmonai/monai
diff --git a/docs/images/3d_paired.png b/docs/images/3d_paired.png
new file mode 100644
index 0000000000..dd751c8e16
Binary files /dev/null and b/docs/images/3d_paired.png differ
diff --git a/docs/images/deepgrow.png b/docs/images/deepgrow.png
new file mode 100644
index 0000000000..d006bd0d09
Binary files /dev/null and b/docs/images/deepgrow.png differ
diff --git a/docs/images/deepgrow_scheme.png b/docs/images/deepgrow_scheme.png
new file mode 100644
index 0000000000..9b4e400839
Binary files /dev/null and b/docs/images/deepgrow_scheme.png differ
diff --git a/docs/images/invert_transforms.png b/docs/images/invert_transforms.png
new file mode 100644
index 0000000000..fa3863f373
Binary files /dev/null and b/docs/images/invert_transforms.png differ
diff --git a/docs/images/lr_finder.png b/docs/images/lr_finder.png
new file mode 100644
index 0000000000..ed9ba69770
Binary files /dev/null and b/docs/images/lr_finder.png differ
diff --git a/docs/images/metrics_report.png b/docs/images/metrics_report.png
new file mode 100644
index 0000000000..a317fcdc21
Binary files /dev/null and b/docs/images/metrics_report.png differ
diff --git a/docs/images/pathology.png b/docs/images/pathology.png
new file mode 100644
index 0000000000..da12ad23e7
Binary files /dev/null and b/docs/images/pathology.png differ
diff --git a/docs/images/tta.png b/docs/images/tta.png
new file mode 100644
index 0000000000..6c4e18ffa0
Binary files /dev/null and b/docs/images/tta.png differ
diff --git a/docs/requirements.txt b/docs/requirements.txt
index d046bc53cf..acc983129f 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,16 +1,16 @@
 -f https://download.pytorch.org/whl/cpu/torch-1.6.0%2Bcpu-cp37-cp37m-linux_x86_64.whl
 torch>=1.5
-pytorch-ignite==0.4.2
+pytorch-ignite==0.4.4
 numpy>=1.17
-itk>=5.0
+itk>=5.0, <=5.1.2
 nibabel
 parameterized
 scikit-image>=0.14.2
 tensorboard
 commonmark==0.9.1
 recommonmark==0.6.0
-Sphinx==3.3.0
-sphinx-rtd-theme==0.5.0
+Sphinx==3.5.3
+sphinx-rtd-theme==0.5.2
 sphinxcontrib-applehelp
 sphinxcontrib-devhelp
 sphinxcontrib-htmlhelp
diff --git a/docs/source/apps.rst b/docs/source/apps.rst
index b8c8b4d341..29d835514f 100644
--- a/docs/source/apps.rst
+++ b/docs/source/apps.rst
@@ -46,9 +46,44 @@ Applications
     :members:
 .. autoclass:: AddRandomGuidanced
     :members:
+.. autoclass:: AddGuidanceFromPointsd
+    :members:
 .. autoclass:: SpatialCropForegroundd
     :members:
+.. autoclass:: SpatialCropGuidanced
+    :members:
+.. autoclass:: RestoreLabeld
+    :members:
+.. autoclass:: ResizeGuidanced
+    :members:
 .. autoclass:: FindDiscrepancyRegionsd
     :members:
 .. autoclass:: FindAllValidSlicesd
     :members:
+.. autoclass:: Fetch2DSliced
+    :members:
+
+`Pathology`
+-----------
+
+.. automodule:: monai.apps.pathology.datasets
+.. autoclass:: PatchWSIDataset
+    :members:
+.. autoclass:: SmartCachePatchWSIDataset
+    :members:
+.. autoclass:: MaskedInferenceWSIDataset
+    :members:
+
+.. automodule:: monai.apps.pathology.handlers
+.. autoclass:: ProbMapProducer
+    :members:
+
+.. automodule:: monai.apps.pathology.metrics
+.. autoclass:: LesionFROC
+    :members:
+
+.. automodule:: monai.apps.pathology.utils
+.. autofunction:: compute_multi_instance_mask
+.. autofunction:: compute_isolated_tumor_cells
+.. autoclass:: PathologyProbNMS
+    :members:
diff --git a/docs/source/data.rst b/docs/source/data.rst
index 11609964c3..66fadd549b 100644
--- a/docs/source/data.rst
+++ b/docs/source/data.rst
@@ -68,6 +68,12 @@ Generic Interfaces
 .. autoclass:: ImageDataset
   :members:
   :special-members: __getitem__
+  
+`NPZDictItemDataset`
+~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: NPZDictItemDataset
+  :members:
+  :special-members: __getitem__
 
 Patch-based dataset
 -------------------
@@ -77,6 +83,11 @@ Patch-based dataset
 .. autoclass:: GridPatchDataset
   :members:
 
+`PatchIter`
+~~~~~~~~~~~
+.. autoclass:: PatchIter
+  :members:
+
 `PatchDataset`
 ~~~~~~~~~~~~~~
 .. autoclass:: PatchDataset
@@ -105,6 +116,10 @@ PILReader
 .. autoclass:: PILReader
   :members:
 
+WSIReader
+~~~~~~~~~
+.. autoclass:: WSIReader
+  :members:
 
 Nifti format handling
 ---------------------
@@ -151,6 +166,9 @@ DistributedSampler
 ~~~~~~~~~~~~~~~~~~
 .. autoclass:: monai.data.DistributedSampler
 
+DistributedWeightedRandomSampler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: monai.data.DistributedWeightedRandomSampler
 
 Decathlon Datalist
 ~~~~~~~~~~~~~~~~~~
@@ -165,3 +183,12 @@ DataLoader
 ThreadBuffer
 ~~~~~~~~~~~~
 .. autoclass:: monai.data.ThreadBuffer
+
+
+BatchInverseTransform
+~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: monai.data.BatchInverseTransform
+
+TestTimeAugmentation
+~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: monai.data.TestTimeAugmentation
diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst
index a629b28b27..7c8498e37a 100644
--- a/docs/source/handlers.rst
+++ b/docs/source/handlers.rst
@@ -110,3 +110,23 @@ SmartCache handler
 ------------------
 .. autoclass:: SmartCacheHandler
     :members:
+
+Parameter Scheduler handler
+---------------------------
+.. autoclass:: ParamSchedulerHandler
+    :members:
+
+EarlyStop handler
+-----------------
+.. autoclass:: EarlyStopHandler
+    :members:
+
+GarbageCollector handler
+------------------------
+.. autoclass:: GarbageCollector
+    :members:
+
+Transform inverter
+------------------
+.. autoclass:: TransformInverter
+    :members:
diff --git a/docs/source/highlights.md b/docs/source/highlights.md
index 29302bda77..d6fcc473c6 100644
--- a/docs/source/highlights.md
+++ b/docs/source/highlights.md
@@ -1,8 +1,8 @@
-# Modules in v0.4.0
+# Modules overview
 
 MONAI aims at supporting deep learning in medical image analysis at multiple granularities.
 This figure shows a typical example of the end-to-end workflow in medical deep learning area:
-![image](../images/end_to_end.png)
+![an end to end workflow](../images/end_to_end.png)
 
 ## MONAI architecture
 The design principle of MONAI is to provide flexible and light APIs for users with varying expertise.
@@ -12,7 +12,7 @@ The design principle of MONAI is to provide flexible and light APIs for users wi
 4. Researchers contribute implementations based on the state-of-the-art for the latest research challenges, including COVID-19 image analysis, Model Parallel, etc.
 
 The overall architecture and modules are shown in the following figure:
-![image](../images/arch_modules_v0.4.png)
+![architecture overview](../images/arch_modules_v0.4.png)
 The rest of this page provides more details for each module.
 
 * [Data I/O, processing and augmentation](#medical-image-data-i-o-processing-and-augmentation)
@@ -26,6 +26,7 @@ The rest of this page provides more details for each module.
 * [Workflows](#workflows)
 * [Research](#research)
 * [GPU acceleration](#gpu-acceleration)
+* [Applications](#applications)
 
 ## Medical image data I/O, processing and augmentation
 Medical images require highly specialized methods for I/O, preprocessing, and augmentation. Medical images are often in specialized formats with rich meta-information, and the data volumes are often high-dimensional. These require carefully designed manipulation procedures. The medical imaging focus of MONAI is enabled by powerful and flexible image transformations that facilitate user-friendly, reproducible, optimized medical data pre-processing pipelines.
@@ -49,7 +50,7 @@ transformations. These currently include, for example:
 - `Rand3DElastic`: Random elastic deformation and affine in 3D
 
 [2D transforms tutorial](https://github.com/Project-MONAI/tutorials/blob/master/modules/transforms_demo_2d.ipynb) shows the detailed usage of several MONAI medical image specific transforms.
-![image](../images/medical_transforms.png)
+![2d transform examples](../images/medical_transforms.png)
 
 ### 3. Fused spatial transforms and GPU acceleration
 As medical image volumes are usually large (in multi-dimensional arrays), pre-processing performance affects the overall pipeline speed. MONAI provides affine transforms to execute fused spatial operations, supports GPU acceleration via native PyTorch for high performance.
@@ -70,7 +71,7 @@ new_img = affine(image, spatial_size=(300, 400), mode='bilinear')
 Experiments and test results are available at [Fused transforms test](https://github.com/Project-MONAI/tutorials/blob/master/acceleration/transform_speed.ipynb).
 
 Currently all the geometric image transforms (Spacing, Zoom, Rotate, Resize, etc.) are designed based on the PyTorch native interfaces. [Geometric transforms tutorial](https://github.com/Project-MONAI/tutorials/blob/master/modules/3d_image_transforms.ipynb) indicates the usage of affine transforms with 3D medical images.
-![image](../images/affine.png)
+![3d transform examples](../images/affine.png)
 
 ### 4. Randomly crop out batch images based on positive/negative ratio
 Medical image data volume may be too large to fit into GPU memory. A widely-used approach is to randomly draw small size data samples during training and run a “sliding window” routine for inference.  MONAI currently provides general random sampling strategies including class-balanced fixed ratio sampling which may help stabilize the patch-based training process. A typical example is in [Spleen 3D segmentation tutorial](https://github.com/Project-MONAI/tutorials/blob/master/3d_segmentation/spleen_segmentation_3d.ipynb), which achieves the class-balanced sampling with `RandCropByPosNegLabel` transform.
@@ -98,7 +99,7 @@ monai.utils.set_determinism(seed=0, additional_settings=None)
 To apply different transforms on the same data and concatenate the results, MONAI provides `CopyItems` transform to make copies of specified items in the data dictionary and `ConcatItems` transform to combine specified items on the expected dimension, and also provides `DeleteItems` transform to delete unnecessary items to save memory.
 
 Typical usage is to scale the intensity of the same image into different ranges and concatenate the results together.
-![image](../images/multi_transform_chains.png)
+![multiple transform chains](../images/multi_transform_chains.png)
 
 ### 7. Debug transforms with DataStats
 When transforms are combined with the "compose" function, it's not easy to track the output of specific transform. To help debug errors in the composed transforms, MONAI provides utility transforms such as `DataStats` to print out intermediate data properties such as `data shape`, `value range`, `data value`, `Additional information`, etc. It's a self-contained transform and can be integrated into any transform chain.
@@ -112,7 +113,7 @@ MONAI also provides post-processing transforms for handling the model outputs. C
 - Extracting contour of segmentation result, which can be used to map to original image and evaluate the model, as below figure (d) and (e).
 
 After applying the post-processing transforms, it's easier to compute metrics, save model output into files or visualize data in the TensorBoard. [Post transforms tutorial](https://github.com/Project-MONAI/tutorials/blob/master/modules/post_transforms.ipynb) shows an example with several main post transforms.
-![image](../images/post_transforms.png)
+![post-processing transforms](../images/post_transforms.png)
 
 ### 9. Integrate third-party transforms
 The design of MONAI transforms emphasis code readability and usability. It works for array data or dictionary-based data. MONAI also provides `Adaptor` tools to accommodate different data format for 3rd party transforms. To convert the data shapes or types, utility transforms such as `ToTensor`, `ToNumpy`, `SqueezeDim` are also provided. So it's easy to enhance the transform chain by seamlessly integrating transforms from external packages, including: `ITK`, `BatchGenerator`, `TorchIO` and `Rising`.
@@ -121,24 +122,43 @@ For more details, please check out the tutorial: [integrate 3rd party transforms
 
 ### 10. IO factory for medical image formats
 Many popular image formats exist in the medical domain, and they are quite different with rich metadata information. To easily handle different medical image formats in the same pipeline, [MONAI provides `LoadImage` transform](https://github.com/Project-MONAI/tutorials/blob/master/modules/load_medical_images.ipynb), which can automatically choose image readers based on the supported suffixes and in below priority order:
-- User-specified reader at runtime when call this loader.
-- Registered readers from the latest to the first in list.
+- User-specified reader at runtime when calling this loader.
+- Registered readers from the latest to the first in the list.
 - Default readers: (nii, nii.gz -> NibabelReader), (png, jpg, bmp -> PILReader), (npz, npy -> NumpyReader), (others -> ITKReader).
 
-The `ImageReader` API is quite straight-forward, users can easily extend for their own customized image readers.
+The `ImageReader` API is quite straightforward, users can easily extend it for their own customized image readers.
 
 With these pre-defined image readers, MONAI can load images in formats: `NIfTI`, `DICOM`, `PNG`, `JPG`, `BMP`, `NPY/NPZ`, etc.
 
+### 11. Save transform data into NIfTI or PNG files
+To convert images into files or debug the transform chain, MONAI provides `SaveImage` transform. Users can inject this transform into the transform chain to save the results.
+
+### 12. Automatically ensure `channel-first` data shape
+Medical images have different shape formats. They can be `channel-last`, `channel-first` or even `no-channel`. We may, for example, want to load several `no-channel` images and stack them as `channel-first` data. To improve the user experience, MONAI provided an `EnsureChannelFirst` transform to automatically detect data shape according to the meta information and convert it to the `channel-first` format consistently.
+
+### 13. Invert spatial transforms and test-time augmentations
+It is often desirable to invert the previously applied spatial transforms (resize, flip, rotate, zoom, crop, pad, etc.) with the deep learning workflows, for example, to resume to the original imaging space after processing the image data in a normalized data space.  We enhance almost all the spatial transforms with an `inverse` operation and release this experimental feature in v0.5.0. Users can easily invert all the spatial transforms for one transformed data item or a batch of data items. It also can be achieved within the workflows by using the `TransformInverter` handler.
+
+If the pipeline includes random transformations, users may want to observe the effect that these transformations have on the output. The typical approach is that we pass the same input through the transforms multiple times with different random realizations. Then use the inverse transforms to move all the results to a common space, and calculate the metrics. MONAI provided `TestTimeAugmentation` for this feature, which by default will calculate the `mode`, `mean`, `standard deviation` and `volume variation coefficient`.
+
+[Invert transforms and TTA tutorials](https://github.com/Project-MONAI/tutorials/blob/master/modules/inverse_transforms_and_test_time_augmentations.ipynb) introduce details about the API with usage examples.
+
+(1) The last column is the inverted data of model output:
+![invert transform](../images/invert_transforms.png)
+
+(2) The TTA results of `mode`, `mean` and `standard deviation`:
+![test time augmentation](../images/tta.png)
+
 ## Datasets
 ### 1. Cache IO and transforms data to accelerate training
 Users often need to train the model with many (potentially thousands of) epochs over the data to achieve the desired model quality. A native PyTorch implementation may repeatedly load data and run the same preprocessing steps for every epoch during training, which can be time-consuming and unnecessary, especially when the medical image volumes are large.
 
 MONAI provides a multi-threads `CacheDataset` and `LMDBDataset` to accelerate these transformation steps during training by storing the intermediate outcomes before the first randomized transform in the transform chain. Enabling this feature could potentially give 10x training speedups in the [Datasets experiment](https://github.com/Project-MONAI/tutorials/blob/master/acceleration/dataset_type_performance.ipynb).
-![image](../images/cache_dataset.png)
+![digital pathology](../images/cache_dataset.png)
 
 ### 2. Cache intermediate outcomes into persistent storage
 The `PersistentDataset` is similar to the CacheDataset, where the intermediate cache values are persisted to disk storage or LMDB for rapid retrieval between experimental runs (as is the case when tuning hyperparameters), or when the entire data set size exceeds available memory. The `PersistentDataset` could achieve similar performance when comparing to `CacheDataset` in [Datasets experiment](https://github.com/Project-MONAI/tutorials/blob/master/acceleration/dataset_type_performance.ipynb).
-![image](../images/datasets_speed.png)
+![cachedataset speed](../images/datasets_speed.png)
 
 ### 3. SmartCache mechanism for big datasets
 During training with very big volume dataset, an efficient approach is to only train with a subset of the dataset in an epoch and dynamically replace part of the subset in every epoch. It's the `SmartCache` mechanism in [NVIDIA Clara-train SDK](https://docs.nvidia.com/clara/tlt-mi/clara-train-sdk-v3.0/nvmidl/additional_features/smart_cache.html#smart-cache).
@@ -188,20 +208,24 @@ To quickly get started with popular training data in the medical domain, MONAI p
 MONAI always welcome new contributions of public datasets, please refer to existing Datasets and leverage the download and extracting APIs, etc. [Public datasets tutorial](https://github.com/Project-MONAI/tutorials/blob/master/modules/public_datasets.ipynb) indicates how to quickly set up training workflows with `MedNISTDataset` and `DecathlonDataset` and how to create a new `Dataset` for public data.
 
 The common workflow of predefined datasets:
-![image](../images/dataset_progress.png)
+![pre-defined dataset](../images/dataset_progress.png)
 
 ### 7. Partition dataset for cross validation
 The `partition_dataset` utility in MONAI can perform several kinds of mechanism to partition dataset for training and validation or cross-validation. It supports shuffling based on a specified random seed, and will return a set of datasets, each dataset contains one partition. And it can split the dataset based on specified ratios or evenly split into `num_partitions`. For given class labels, it can also make sure the same ratio of classes in every partition.
 
 ## Losses
-There are domain-specific loss functions in the medical imaging research which are not typically used in the generic computer vision tasks. As an important module of MONAI, these loss functions are implemented in PyTorch, such as `DiceLoss`, `GeneralizedDiceLoss`, `MaskedDiceLoss`, `TverskyLoss` and `FocalLoss`, etc.
+There are domain-specific loss functions in the medical imaging research which are not typically used in the generic computer vision tasks. As an important module of MONAI, these loss functions are implemented in PyTorch, such as `DiceLoss`, `GeneralizedDiceLoss`, `MaskedDiceLoss`, `TverskyLoss`, `FocalLoss`, `DiceCELoss`, and `DiceFocalLoss`, etc.
 
 ## Optimizers
 MONAI provides several advanced features in optimizers to help accelerate the training or fine-tuning progress. For example, `Novograd` optimizer can be used to converge obviously faster than traditional optimizers. And users can easily define different learning rates for the model layers based [on the `generate_param_groups` utility API](https://github.com/Project-MONAI/tutorials/blob/master/modules/layer_wise_learning_rate.ipynb).
 
+Another important feature is `LearningRateFinder`. The learning rate range test increases the learning rate in a pre-training run between two boundaries in a linear or exponential manner. It provides valuable information on how well the network can be trained over a range of learning rates and what the optimal learning rates are. [LearningRateFinder tutorial](https://github.com/Project-MONAI/tutorials/blob/master/modules/learning_rate.ipynb) indicates the API usage examples.
+![learning rate finder plot](../images/lr_finder.png)
+
 ## Network architectures
 Some deep neural network architectures have shown to be particularly effective for medical imaging analysis tasks. MONAI implements reference networks with the aims of both flexibility and code readability.
 
+### 1. Predefined layers and blocks
 To leverage the common network layers and blocks, MONAI provides several predefined layers and blocks which are compatible with 1D, 2D and 3D networks. Users can easily integrate the layer factories in their own networks.
 
 For example:
@@ -215,6 +239,8 @@ name, dimension = Conv.CONVTRANS, 3
 conv_type = Conv[name, dimension]
 add_module('conv1', conv_type(in_channels, out_channels, kernel_size=1, bias=False))
 ```
+
+### 2. Implementation of generic 2D/3D networks
 And there are several 1D/2D/3D-compatible implementations of intermediate blocks and generic networks, such as UNet, DynUNet, DenseNet, GAN, AHNet, VNet, SENet(and SEResNet, SEResNeXt), SegResNet, etc. All the networks can support PyTorch serialization pipeline based on `torch.jit.script`.
 
 ## Evaluation
@@ -228,7 +254,7 @@ A typical process is:
 2. Iteratively run batched window inferences until all windows are analyzed.
 3. Aggregate the inference outputs to a single segmentation map.
 4. Save the results to file or compute some evaluation metrics.
-![image](../images/sliding_window.png)
+![sliding window scheme](../images/sliding_window.png)
 
 The [Spleen 3D segmentation tutorial](https://github.com/Project-MONAI/tutorials/blob/master/3d_segmentation/spleen_segmentation_3d.ipynb) leverages `SlidingWindow` inference for validation.
 
@@ -237,17 +263,21 @@ Various useful evaluation metrics have been used to measure the quality of medic
 
 For example, `Mean Dice` score can be used for segmentation tasks, and the area under the ROC curve(`ROCAUC`) for classification tasks. We continue to integrate more options.
 
+### 3. Metrics report generation
+During evaluation, users usually save the metrics of every input image, then analyze the bad cases to improve the deep learning pipeline. To save detailed information of metrics, MONAI provided a handler `MetricsSaver`, which can save the final metric values, raw metric of every model output channel of every input image, metrics summary report of operations: `mean`, `median`, `max`, `min`, `90percent`, `std`, etc. The `MeanDice` reports of validation with prostate dataset are as below:
+![metrics report example](../images/metrics_report.png)
+
 ## Visualization
 Beyond the simple point and curve plotting, MONAI provides intuitive interfaces to visualize multidimensional data as GIF animations in TensorBoard. This could provide a quick qualitative assessment of the model by visualizing, for example, the volumetric inputs, segmentation maps, and intermediate feature maps. A runnable example with visualization is available at [UNet training example](https://github.com/Project-MONAI/tutorials/blob/master/3d_segmentation/torch/unet_training_dict.py).
 
 And to visualize the class activation mapping for a trained classification model, MONAI provides CAM, GradCAM, GradCAM++ APIs for both 2D and 3D models:
 
-![image](../images/cam.png)
+![CAM visualization example](../images/cam.png)
 
 The above example is generated by computing [GradCAM/GradCAM++ from a lung CT lesion classification model](https://github.com/Project-MONAI/tutorials/tree/master/modules/interpretability).
 
 ## Result writing
-Currently MONAI supports writing the model outputs as NIfTI files or PNG files for segmentation tasks, and as CSV files for classification tasks. And the writers can restore the data spacing, orientation or shape according to the `original_shape` or `original_affine` information from the input image.
+Currently, MONAI supports writing the model outputs as NIfTI files or PNG files for segmentation tasks, and as CSV files for classification tasks. And the writers can restore the data spacing, orientation or shape according to the `original_shape` or `original_affine` information from the input image.
 
 A rich set of formats will be supported soon, along with relevant statistics and evaluation metrics automatically computed from the outputs.
 
@@ -255,11 +285,11 @@ A rich set of formats will be supported soon, along with relevant statistics and
 To quickly set up training and evaluation experiments, MONAI provides a set of workflows to significantly simplify the modules and allow for fast prototyping.
 
 These features decouple the domain-specific components and the generic machine learning processes. They also provide a set of unify APIs for higher level applications (such as AutoML, Federated Learning).
-The trainers and evaluators of the workflows are compatible with pytorch-ignite `Engine` and `Event-Handler` mechanism. There are rich event handlers in MONAI to independently attach to the trainer or evaluator.
+The trainers and evaluators of the workflows are compatible with pytorch-ignite `Engine` and `Event-Handler` mechanism. There are rich event handlers in MONAI to independently attach to the trainer or evaluator, and users can register additional `custom events` to workflows.
 
 ### 1. General workflows pipeline
-The workflow and event handlers are shown as below:
-![image](../images/workflows.png)
+The workflow and some of MONAI event handlers are shown as below:
+![workflow pipeline](../images/workflows.png)
 
 The end-to-end training and evaluation examples are available at [Workflow examples](https://github.com/Project-MONAI/tutorials/tree/master/modules/engines).
 
@@ -270,9 +300,14 @@ Models ensemble is a popular strategy in machine learning and deep learning area
 3. Execute inference on the test data with all the K models.
 4. Compute the average values with weights or vote the most common value as the final result.
 
-![image](../images/models_ensemble.png)
+![model ensemble](../images/models_ensemble.png)
 More details of practice is at [Model ensemble tutorial](https://github.com/Project-MONAI/tutorials/blob/master/modules/models_ensemble.ipynb).
 
+### 3. Transfer learning for different input / output classes
+`Transfer-learning` is a very common and efficient training approach, especially in the medical-specific domain where obtaining large datasets for training can be difficult. So transfer-learning from a pre-trained checkpoint can significantly improve the model metrics and shorten training time.
+
+MONAI provided `CheckpointLoader` to load a checkpoint for the workflow before training, and it allows some `layer names` of the current network don't match the checkpoint, or some `layer shapes` don't match the checkpoint, which can be useful if the current task has different input image classes or output classes.
+
 ## Research
 There are several research prototypes in MONAI corresponding to the recently published papers that address advanced research problems.
 We always welcome contributions in forms of comments, suggestions, and code implementations.
@@ -283,13 +318,13 @@ The generic patterns/modules identified from the research prototypes will be int
 [A reimplementation](https://monai.io/research/coplenet-pneumonia-lesion-segmentation) of the COPLE-Net originally proposed by:
 
 G. Wang, X. Liu, C. Li, Z. Xu, J. Ruan, H. Zhu, T. Meng, K. Li, N. Huang, S. Zhang. (2020) "A Noise-robust Framework for Automatic Segmentation of COVID-19 Pneumonia Lesions from CT Images." IEEE Transactions on Medical Imaging. 2020. [DOI: 10.1109/TMI.2020.3000314](https://doi.org/10.1109/TMI.2020.3000314)
-![image](../images/coplenet.png)
+![coplenet](../images/coplenet.png)
 
 ### 2. LAMP: Large Deep Nets with Automated Model Parallelism for Image Segmentation
 [A reimplementation](https://monai.io/research/lamp-automated-model-parallelism) of the LAMP system originally proposed by:
 
 Wentao Zhu, Can Zhao, Wenqi Li, Holger Roth, Ziyue Xu, and Daguang Xu (2020) "LAMP: Large Deep Nets with Automated Model Parallelism for Image Segmentation." MICCAI 2020 (Early Accept, paper link: https://arxiv.org/abs/2006.12575)
-![image](../images/unet-pipe.png)
+![LAMP UNet](../images/unet-pipe.png)
 
 ## GPU acceleration
 NVIDIA GPUs have been widely applied in many areas of deep learning training and evaluation, and the CUDA parallel computation shows obvious acceleration when comparing to traditional computation methods. To fully leverage GPU features, many popular mechanisms raised, like automatic mixed precision (AMP), distributed data parallel, etc. MONAI can support these features and provides rich examples.
@@ -299,19 +334,42 @@ In 2017, NVIDIA researchers developed a methodology for mixed-precision training
 
 For the PyTorch 1.6 release, developers at NVIDIA and Facebook moved mixed precision functionality into PyTorch core as the AMP package, `torch.cuda.amp`.
 
-MONAI workflows can easily set `amp=True/False` in `SupervisedTrainer` or `SupervisedEvaluator` during training or evaluation to enable/disable AMP. And we tried to compare the training speed if AMP ON/OFF on Tesla V100 GPU with CUDA 11 and PyTorch 1.6, got some benchmark for reference:
-![image](../images/amp_training_v100.png)
-We also executed the same test program on Testa A100 GPU with the same software environment, got much faster benchmark for reference:
-![image](../images/amp_training_a100.png)
+MONAI workflows can easily set `amp=True/False` in `SupervisedTrainer` or `SupervisedEvaluator` during training or evaluation to enable/disable AMP. And we tried to compare the training speed if AMP ON/OFF on NVIDIA V100 GPU with CUDA 11 and PyTorch 1.6, got some benchmark for reference:
+![amp v100 results](../images/amp_training_v100.png)
+We also executed the same test program on NVIDIA A100 GPU with the same software environment, got much faster benchmark for reference:
+![amp a100 results](../images/amp_training_a100.png)
 More details is available at [AMP training tutorial](https://github.com/Project-MONAI/tutorials/blob/master/acceleration/automatic_mixed_precision.ipynb).
 We also tried to combine AMP with `CacheDataset` and `Novograd` optimizer to achieve the fast training in MONAI, able to obtain approximately 12x speedup compared with a Pytorch native implementation when the training converges at a validation mean dice of 0.93. Benchmark for reference:
-![image](../images/fast_training.png)
+![fast training results](../images/fast_training.png)
 More details is available at [Fast training tutorial](https://github.com/Project-MONAI/tutorials/blob/master/acceleration/fast_training_tutorial.ipynb).
 
 ### 2. Distributed data parallel
 Distributed data parallel is an important feature of PyTorch to connect multiple GPU devices on single or multiple nodes to train or evaluate models. MONAI provides demos for reference: train/evaluate with PyTorch DDP, train/evaluate with Horovod, train/evaluate with Ignite DDP, partition dataset and train with SmartCacheDataset, as well as a real world training example based on Decathlon challenge Task01 - Brain Tumor segmentation.
-The demo contains distributed caching, training, and validation. We tried to train this example on NVIDIA NGC server, got some performance benchmarks for reference(PyTorch 1.6, CUDA 11, Tesla V100 GPUs):
-![image](../images/distributed_training.png)
+The demo contains distributed caching, training, and validation. We tried to train this example on NVIDIA NGC server, got some performance benchmarks for reference(PyTorch 1.6, CUDA 11, NVIDIA V100 GPUs):
+
+![distributed training results](../images/distributed_training.png)
 
 ### 3. C++/CUDA optimized modules
-To accelerate some heavy computation progress, C++/CUDA implementation can be an impressive method, which usually brings even hundreds of times faster performance. MONAI contains some C++/CUDA optimized modules, like `Resampler`,and fully support C++/CUDA programs in CI/CD and building package.
+To accelerate some heavy computation progress, C++/CUDA implementation can be an impressive method, which usually brings even hundreds of times faster performance. MONAI contains some C++/CUDA optimized modules, like `Resampler`, `Conditional random field (CRF)`, `Fast bilateral filtering using the permutohedral lattice`, and fully support C++/CUDA programs in CI/CD and building package.
+
+## Applications
+The research area of medical image deep learning is expanding fast. To apply the latest achievements into applications, MONAI contains many application components to build end-to-end solutions or prototypes for other similar use cases.
+
+### 1. DeepGrow modules for interactive segmentation
+[A reimplementation](https://github.com/Project-MONAI/MONAI/tree/master/monai/apps/deepgrow) of the DeepGrow components, which is deep learning based semi-automated segmentation approach that aims to be a "smart" interactive tool for region of interest delineation in medical images, originally proposed by:
+
+Sakinis, Tomas, et al. "Interactive segmentation of medical images through fully convolutional neural networks." arXiv preprint arXiv:1903.08205 (2019).
+
+![deepgrow scheme](../images/deepgrow.png)
+
+### 2. Lesion detection in digital pathology
+[Implementation](https://github.com/Project-MONAI/MONAI/tree/master/monai/apps/pathology) of the pathology detection components, which includes efficient whole slide imaging IO and sampling with NVIDIA cuCIM library and SmartCache mechanism, FROC measurements for lesion and probabilistic post-processing for lesion detection.
+
+![digital pathology](../images/pathology.png)
+
+### 3. Learning-based image registration
+Starting from v0.5.0, MONAI provides experimental features for building learning-based 2D/3D registration workflows.  These include image similarity measures as loss functions, bending energy as model regularization, network architectures, warping modules. The components can be used to build the major unsupervised and weakly-supervised algorithms.
+
+The following figure shows the registration of CT images acquired at different time points for a single patient using MONAI:
+
+![3d registration](../images/3d_paired.png)
diff --git a/docs/source/index.rst b/docs/source/index.rst
index ea21428e6e..143c407a72 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -45,6 +45,7 @@ Technical documentation is available at `docs.monai.io <https://docs.monai.io>`_
    :maxdepth: 1
    :caption: Feature highlights
 
+   whatsnew.md
    highlights.md
 
 .. toctree::
@@ -91,9 +92,8 @@ Links
 - FAQ: https://github.com/Project-MONAI/MONAI/wiki/Frequently-asked-questions-and-answers
 - Test status: https://github.com/Project-MONAI/MONAI/actions
 - PyPI package: https://pypi.org/project/monai/
+- Weekly previews: https://pypi.org/project/monai-weekly/
 - Docker Hub: https://hub.docker.com/r/projectmonai/monai
-- Google Group: https://groups.google.com/forum/#!forum/project-monai
-- Reddit: https://www.reddit.com/r/projectmonai/
 
 
 Indices and tables
diff --git a/docs/source/inferers.rst b/docs/source/inferers.rst
index 544e695d2e..e358e603bd 100644
--- a/docs/source/inferers.rst
+++ b/docs/source/inferers.rst
@@ -30,3 +30,9 @@ Inferers
 .. autoclass:: SlidingWindowInferer
     :members:
     :special-members: __call__
+
+`SaliencyInferer`
+~~~~~~~~~~~~~~~~~
+.. autoclass:: SaliencyInferer
+    :members:
+    :special-members: __call__
diff --git a/docs/source/installation.md b/docs/source/installation.md
index cb540b1559..b3dbcc90f8 100644
--- a/docs/source/installation.md
+++ b/docs/source/installation.md
@@ -59,11 +59,11 @@ for the latest features:
 
 ### Option 1 (as a part of your system-wide module):
 ```bash
-pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
+pip install git+https://github.com/Project-MONAI/MONAI#egg=monai
 ```
 or, to build with MONAI Cpp/CUDA extensions:
 ```bash
-BUILD_MONAI=1 pip install git+https://github.com/Project-MONAI/MONAI#egg=MONAI
+BUILD_MONAI=1 pip install git+https://github.com/Project-MONAI/MONAI#egg=monai
 ```
 this command will download and install the current master branch of [MONAI from
 GitHub](https://github.com/Project-MONAI/MONAI).
diff --git a/docs/source/losses.rst b/docs/source/losses.rst
index 5e19219fee..eea6656a24 100644
--- a/docs/source/losses.rst
+++ b/docs/source/losses.rst
@@ -48,6 +48,11 @@ Segmentation Losses
 .. autoclass:: DiceCELoss
     :members:
 
+`DiceFocalLoss`
+~~~~~~~~~~~~~~~
+.. autoclass:: DiceFocalLoss
+    :members:
+
 `FocalLoss`
 ~~~~~~~~~~~
 .. autoclass:: FocalLoss
diff --git a/docs/source/metrics.rst b/docs/source/metrics.rst
index 32a3faf380..7cfd10f196 100644
--- a/docs/source/metrics.rst
+++ b/docs/source/metrics.rst
@@ -6,6 +6,10 @@ Metrics
 =======
 .. currentmodule:: monai.metrics
 
+`FROC`
+------
+.. autofunction:: compute_froc_score
+
 `Mean Dice`
 -----------
 .. autofunction:: compute_meandice
diff --git a/docs/source/networks.rst b/docs/source/networks.rst
index e0ac0f2d75..baee107620 100644
--- a/docs/source/networks.rst
+++ b/docs/source/networks.rst
@@ -20,6 +20,11 @@ Blocks
 .. autoclass:: Convolution
     :members:
 
+`CRF`
+~~~~~~~~~~~~~
+.. autoclass:: CRF
+    :members:
+
 `ResidualUnit`
 ~~~~~~~~~~~~~~
 .. autoclass:: ResidualUnit
@@ -30,6 +35,11 @@ Blocks
 .. autoclass:: Swish
     :members:
 
+`MemoryEfficientSwish`
+~~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: MemoryEfficientSwish
+    :members:
+
 `Mish`
 ~~~~~~
 .. autoclass:: Mish
@@ -94,7 +104,7 @@ Blocks
 .. autoclass:: SEResNetBottleneck
     :members:
 
-`Squeeze-and-Excitation ResneXt Bottleneck`
+`Squeeze-and-Excitation ResNeXt Bottleneck`
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autoclass:: SEResNeXtBottleneck
     :members:
@@ -119,6 +129,21 @@ Blocks
 .. autoclass:: Subpixelupsample
 .. autoclass:: SubpixelUpSample
 
+`Registration Residual Conv Block`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: RegistrationResidualConvBlock
+    :members:
+
+`Registration Down Sample Block`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: RegistrationDownSampleBlock
+    :members:
+
+`Registration Extraction Block`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: RegistrationExtractionBlock
+    :members:
+
 `LocalNet DownSample Block`
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. autoclass:: LocalNetDownSampleBlock
@@ -271,10 +296,11 @@ Nets
 ~~~~~~~~~~
 .. autoclass:: DenseNet
   :members:
-.. autofunction:: densenet121
-.. autofunction:: densenet169
-.. autofunction:: densenet201
-.. autofunction:: densenet264
+
+`EfficientNet`
+~~~~~~~~~~~~~~
+.. autoclass:: EfficientNet
+  :members:
 
 `SegResNet`
 ~~~~~~~~~~~
@@ -290,12 +316,6 @@ Nets
 ~~~~~~~
 .. autoclass:: SENet
   :members:
-.. autofunction:: senet154
-.. autofunction:: se_resnet50
-.. autofunction:: se_resnet101
-.. autofunction:: se_resnet152
-.. autofunction:: se_resnext50_32x4d
-.. autofunction:: se_resnext101_32x4d
 
 `HighResNet`
 ~~~~~~~~~~~~
@@ -330,6 +350,16 @@ Nets
 .. autoclass:: VNet
   :members:
 
+`RegUNet`
+~~~~~~~~~~
+.. autoclass:: RegUNet
+  :members:
+
+`GlobalNet`
+~~~~~~~~~~~~
+.. autoclass:: GlobalNet
+  :members:
+
 `LocalNet`
 ~~~~~~~~~~~
 .. autoclass:: LocalNet
@@ -375,6 +405,11 @@ Nets
 .. autoclass:: Critic
   :members:
 
+`TorchVisionFullyConvModel`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. autoclass:: TorchVisionFullyConvModel
+  :members:
+
 Utilities
 ---------
 .. automodule:: monai.networks.utils
diff --git a/docs/source/transforms.rst b/docs/source/transforms.rst
index 5b550f7885..4f039b9c35 100644
--- a/docs/source/transforms.rst
+++ b/docs/source/transforms.rst
@@ -27,12 +27,23 @@ Generic Interfaces
 .. autoclass:: Randomizable
     :members:
 
+`RandomizableTransform`
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: RandomizableTransform
+    :members:
+
 `Compose`
 ^^^^^^^^^
 .. autoclass:: Compose
     :members:
     :special-members: __call__
 
+`InvertibleTransform`
+^^^^^^^^^^^^^^^^^^^^^
+.. autoclass:: InvertibleTransform
+    :members:
+
+
 Vanilla Transforms
 ------------------
 
@@ -132,6 +143,24 @@ Intensity
     :members:
     :special-members: __call__
 
+`StdShiftIntensity`
+"""""""""""""""""""
+.. autoclass:: StdShiftIntensity
+    :members:
+    :special-members: __call__
+
+`RandStdShiftIntensity`
+"""""""""""""""""""""""
+.. autoclass:: RandStdShiftIntensity
+    :members:
+    :special-members: __call__
+
+`RandBiasField`
+"""""""""""""""
+.. autoclass:: RandBiasField
+    :members:
+    :special-members: __call__
+
 `ScaleIntensity`
 """"""""""""""""
 .. autoclass:: ScaleIntensity
@@ -276,6 +305,11 @@ Post-processing
     :members:
     :special-members: __call__
 
+`Prob NMS`
+""""""""""
+.. autoclass:: ProbNMS
+  :members:
+
 `VoteEnsemble`
 """"""""""""""
 .. autoclass:: VoteEnsemble
@@ -309,6 +343,12 @@ Spatial
     :members:
     :special-members: __call__
 
+`RandAxisFlip`
+""""""""""""""
+.. autoclass:: RandAxisFlip
+    :members:
+    :special-members: __call__
+
 `RandZoom`
 """"""""""
 .. autoclass:: RandZoom
@@ -426,6 +466,12 @@ Utility
     :members:
     :special-members: __call__
 
+`EnsureChannelFirst`
+""""""""""""""""""""
+.. autoclass:: EnsureChannelFirst
+    :members:
+    :special-members: __call__
+
 `RepeatChannel`
 """""""""""""""
 .. autoclass:: RepeatChannel
@@ -516,6 +562,12 @@ Utility
     :members:
     :special-members: __call__
 
+`MapLabelValue`
+"""""""""""""""
+.. autoclass:: MapLabelValue
+    :members:
+    :special-members: __call__
+
 Dictionary Transforms
 ---------------------
 
@@ -615,6 +667,24 @@ Instensity (Dict)
     :members:
     :special-members: __call__
 
+`StdShiftIntensityd`
+""""""""""""""""""""
+.. autoclass:: StdShiftIntensityd
+    :members:
+    :special-members: __call__
+
+`RandStdShiftIntensityd`
+""""""""""""""""""""""""
+.. autoclass:: RandStdShiftIntensityd
+    :members:
+    :special-members: __call__
+
+`RandBiasFieldd`
+""""""""""""""""
+.. autoclass:: RandBiasFieldd
+    :members:
+    :special-members: __call__
+
 `ScaleIntensityd`
 """""""""""""""""
 .. autoclass:: ScaleIntensityd
@@ -786,6 +856,12 @@ Spatial (Dict)
     :members:
     :special-members: __call__
 
+`RandAxisFlipd`
+"""""""""""""""
+.. autoclass:: RandAxisFlipd
+    :members:
+    :special-members: __call__
+
 `Rotated`
 """""""""
 .. autoclass:: Rotated
@@ -828,6 +904,12 @@ Spatial (Dict)
     :members:
     :special-members: __call__
 
+`Affined`
+"""""""""
+.. autoclass:: Affined
+    :members:
+    :special-members: __call__
+
 `RandAffined`
 """""""""""""
 .. autoclass:: RandAffined
@@ -873,6 +955,12 @@ Utility (Dict)
     :members:
     :special-members: __call__
 
+`EnsureChannelFirstd`
+"""""""""""""""""""""
+.. autoclass:: EnsureChannelFirstd
+    :members:
+    :special-members: __call__
+
 `RepeatChanneld`
 """"""""""""""""
 .. autoclass:: RepeatChanneld
@@ -987,6 +1075,12 @@ Utility (Dict)
     :members:
     :special-members: __call__
 
+`MapLabelValued`
+""""""""""""""""
+.. autoclass:: MapLabelValued
+    :members:
+    :special-members: __call__
+
 Transform Adaptors
 ------------------
 .. automodule:: monai.transforms.adaptors
diff --git a/docs/source/utils.rst b/docs/source/utils.rst
index e0b993da60..855954fd29 100644
--- a/docs/source/utils.rst
+++ b/docs/source/utils.rst
@@ -27,6 +27,7 @@ Misc
 .. automodule:: monai.utils.misc
   :members:
 
+
 Profiling
 ---------
 .. automodule:: monai.utils.profiling
diff --git a/docs/source/whatsnew.md b/docs/source/whatsnew.md
new file mode 100644
index 0000000000..e7cbcd5b39
--- /dev/null
+++ b/docs/source/whatsnew.md
@@ -0,0 +1,73 @@
+# What's new in 0.5.0 🎉
+
+## Invert spatial transforms and test-time augmentations
+It is often desirable to invert the previously applied spatial transforms (resize, flip, rotate, zoom, crop, pad, etc.) with the deep learning workflows, for example, to resume to the original imaging space after processing the image data in a normalized data space.  We enhance almost all the spatial transforms with an `inverse` operation and release this experimental feature in v0.5.0. Users can easily invert all the spatial transforms for one transformed data item or a batch of data items. It also can be achieved within the workflows by using the `TransformInverter` handler.
+
+If the pipeline includes random transformations, users may want to observe the effect that these transformations have on the output. The typical approach is that we pass the same input through the transforms multiple times with different random realizations. Then use the inverse transforms to move all the results to a common space, and calculate the metrics. MONAI provided `TestTimeAugmentation` for this feature, which by default will calculate the `mode`, `mean`, `standard deviation` and `volume variation coefficient`.
+
+[Invert transforms and TTA tutorials](https://github.com/Project-MONAI/tutorials/blob/master/modules/inverse_transforms_and_test_time_augmentations.ipynb) introduce details about the API with examples.
+
+(1) The last column is the inverted data of model output:
+![invert transform](../images/invert_transforms.png)
+
+(2) The TTA results of `mode`, `mean` and `standard deviation`:
+![test time augmentation](../images/tta.png)
+
+## Lesion detection in digital pathology
+MONAI starts to support digital pathology deep learning tasks. The initial [implementation](https://github.com/Project-MONAI/MONAI/tree/master/monai/apps/pathology) of the pathology detection components includes:
+- Efficient whole slide imaging IO with NVIDIA cuCIM library
+- Patch-based sampling and training strategies with the SmartCache mechanism
+- FROC measurements for lesion detection
+- Probabilistic post-processing for lesion ROIs.
+
+![digital pathology](../images/pathology.png)
+
+## DeepGrow modules for interactive segmentation
+Towards an interactive workflow with manual input during training and inference,
+[a reimplementation](https://github.com/Project-MONAI/MONAI/tree/master/monai/apps/deepgrow) of the DeepGrow components is included in this release.
+DeepGrow is a deep learning based semi-automated segmentation approach that aims to be a "smart" interactive tool for regions of interest delineation in medical images.
+
+![deepgrow scheme](../images/deepgrow_scheme.png)
+
+An end-to-end example is presented at [`project-monai/tutorials`](https://github.com/Project-MONAI/tutorials/tree/master/deepgrow/ignite).
+![deepgrow end-to-end](../images/deepgrow.png)
+
+## Learning-based image registration
+Starting from v0.5.0, MONAI provides experimental features for building learning-based 2D/3D registration workflows. These include image similarity measures as loss functions, bending energy as model regularization, network architectures, warping modules. The components can be used to build the major unsupervised and weakly-supervised algorithms.
+
+The following figure shows the registration of CT images acquired at different time points for a single patient using MONAI:
+
+![3d registration](../images/3d_paired.png)
+
+## Various usability improvements
+### IO factory for medical image formats
+Many popular image formats exist in the medical domain, and they are quite different with rich metadata information. To easily handle different medical image formats in the same pipeline, [MONAI provides `LoadImage` transform](https://github.com/Project-MONAI/tutorials/blob/master/modules/load_medical_images.ipynb), which can automatically choose image readers based on the supported suffixes and in the below priority order:
+- User-specified reader at runtime when call this loader.
+- Registered readers from the latest to the first in list.
+- Default readers: (nii, nii.gz -> NibabelReader), (png, jpg, bmp -> PILReader), (npz, npy -> NumpyReader), (others -> ITKReader).
+
+The `ImageReader` API is quite straight-forward, users can easily extend for their own customized image readers.
+
+With these pre-defined image readers, MONAI can load images in formats: `NIfTI`, `DICOM`, `PNG`, `JPG`, `BMP`, `NPY/NPZ`, etc.
+
+### Save transform data into NIfTI or PNG files
+To convert images into files or debug the transform chain, MONAI provides `SaveImage` transform. Users can inject this transform into the transform chain to save the results.
+
+### Automatically ensure `channel-first` data shape
+Medical images have different shape formats. They can be `channel-last`, `channel-first` or even `no-channel`. We may, for example, want to load several `no-channel` images and stack them as `channel-first` data. To improve the user experience, MONAI provided an `EnsureChannelFirst` transform to automatically detect data shape according to the meta information and convert it to the `channel-first` format consistently.
+
+### Network architectures
+Various ready-to-use architectures with pretrained model weights from `torch.hub`.
+
+### Result writing
+Currently MONAI supports writing the model outputs as NIfTI files or PNG files for segmentation tasks, and as CSV files for classification tasks. And the writers can restore the data spacing, orientation or shape according to the `original_shape` or `original_affine` information from the input image.
+
+A rich set of formats will be supported soon, along with relevant statistics and evaluation metrics automatically computed from the outputs.
+
+### Transfer learning for different input / output classes
+`Transfer-learning` is a very common and efficient training approach, especially in the medical-specific domain where obtaining large datasets for training can be difficult. So transfer-learning from a pre-trained checkpoint can significantly improve the model metrics and shorten training time.
+
+MONAI provided `CheckpointLoader` to load a checkpoint for the workflow before training, and it allows some `layer names` of current network don't match the checkpoint, or some `layer shapes` don't match the checkpoint, which can be useful if the current task has different input image classes or output classes.
+
+### C++/CUDA optimized modules
+To accelerate some heavy computation progress, C++/CUDA implementation can be an impressive method, which usually brings even hundreds of times faster performance. MONAI contains some C++/CUDA optimized modules, like `Resampler`, `Conditional random field (CRF)`, `Fast bilateral filtering using the permutohedral lattice`, and fully support C++/CUDA programs in CI/CD and building package.
diff --git a/monai/__init__.py b/monai/__init__.py
index 910698ee14..3bb89cc348 100644
--- a/monai/__init__.py
+++ b/monai/__init__.py
@@ -19,7 +19,7 @@
 
 version_dict = get_versions()
 __version__ = version_dict.get("version", "0+unknown")
-__revision_id__ = version_dict.get("full-revisionid", None)
+__revision_id__ = version_dict.get("full-revisionid")
 del get_versions, version_dict
 
 __copyright__ = "(c) 2020 - 2021 MONAI Consortium"
@@ -44,3 +44,19 @@
 
 # load all modules, this will trigger all export decorations
 load_submodules(sys.modules[__name__], True, exclude_pattern=excludes)
+
+__all__ = [
+    "apps",
+    "config",
+    "data",
+    "engines",
+    "handlers",
+    "inferers",
+    "losses",
+    "metrics",
+    "networks",
+    "optimizers",
+    "transforms",
+    "utils",
+    "visualize",
+]
diff --git a/monai/_version.py b/monai/_version.py
index 1b31d5fd1a..79f569dd79 100644
--- a/monai/_version.py
+++ b/monai/_version.py
@@ -1,3 +1,4 @@
+
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
 # feature). Distribution tarballs (built by setup.py sdist) and build
@@ -5,7 +6,7 @@
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.18 (https://github.com/warner/python-versioneer)
+# versioneer-0.19 (https://github.com/python-versioneer/python-versioneer)
 
 """Git implementation of _version.py."""
 
@@ -56,7 +57,7 @@ class NotThisMethod(Exception):
 
 
 def register_vcs_handler(vcs, method):  # decorator
-    """Decorator to mark a method as the handler for a particular VCS."""
+    """Create decorator to mark a method as the handler of a VCS."""
     def decorate(f):
         """Store f in HANDLERS[vcs][method]."""
         if vcs not in HANDLERS:
@@ -92,9 +93,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
         if verbose:
             print("unable to find command, tried %s" % (commands,))
         return None, None
-    stdout = p.communicate()[0].strip()
-    if sys.version_info[0] >= 3:
-        stdout = stdout.decode()
+    stdout = p.communicate()[0].strip().decode()
     if p.returncode != 0:
         if verbose:
             print("unable to run %s (error)" % dispcmd)
@@ -164,6 +163,10 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         raise NotThisMethod("no keywords at all, weird")
     date = keywords.get("date")
     if date is not None:
+        # Use only the last line.  Previous lines may contain GPG signature
+        # information.
+        date = date.splitlines()[-1]
+
         # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
         # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
         # -like" string, which we must then edit to make compliant), because
@@ -299,6 +302,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     # commit date: see ISO-8601 comment in git_versions_from_keywords()
     date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
                        cwd=root)[0].strip()
+    # Use only the last line.  Previous lines may contain GPG signature
+    # information.
+    date = date.splitlines()[-1]
     pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
 
     return pieces
@@ -337,18 +343,18 @@ def render_pep440(pieces):
 
 
 def render_pep440_pre(pieces):
-    """TAG[.post.devDISTANCE] -- No -dirty.
+    """TAG[.post0.devDISTANCE] -- No -dirty.
 
     Exceptions:
-    1: no tags. 0.post.devDISTANCE
+    1: no tags. 0.post0.devDISTANCE
     """
     if pieces["closest-tag"]:
         rendered = pieces["closest-tag"]
         if pieces["distance"]:
-            rendered += ".post.dev%d" % pieces["distance"]
+            rendered += ".post0.dev%d" % pieces["distance"]
     else:
         # exception #1
-        rendered = "0.post.dev%d" % pieces["distance"]
+        rendered = "0.post0.dev%d" % pieces["distance"]
     return rendered
 
 
@@ -494,7 +500,7 @@ def get_versions():
         # versionfile_source is the relative path from the top of the source
         # tree (where the .git directory might live) to this file. Invert
         # this to find the root from __file__.
-        for i in cfg.versionfile_source.split('/'):  # lgtm[py/unused-loop-variable]
+        for i in cfg.versionfile_source.split('/'):
             root = os.path.dirname(root)
     except NameError:
         return {"version": "0+unknown", "full-revisionid": None,
diff --git a/monai/apps/datasets.py b/monai/apps/datasets.py
index f0416b8c4f..a193b7391e 100644
--- a/monai/apps/datasets.py
+++ b/monai/apps/datasets.py
@@ -11,7 +11,7 @@
 
 import os
 import sys
-from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+from typing import Callable, Dict, List, Optional, Sequence, Union
 
 import numpy as np
 
@@ -98,8 +98,8 @@ def __init__(
             self, data, transform, cache_num=cache_num, cache_rate=cache_rate, num_workers=num_workers
         )
 
-    def randomize(self, data: Optional[Any] = None) -> None:
-        self.rann = self.R.random()
+    def randomize(self, data: List[int]) -> None:
+        self.R.shuffle(data)
 
     def get_num_classes(self) -> int:
         """Get number of classes."""
@@ -132,22 +132,26 @@ def _generate_data_list(self, dataset_dir: str) -> List[Dict]:
 
         data = []
 
-        for i in range(num_total):
-            self.randomize()
-            if self.section == "training":
-                if self.rann < self.val_frac + self.test_frac:
-                    continue
-            elif self.section == "validation":
-                if self.rann >= self.val_frac:
-                    continue
-            elif self.section == "test":
-                if self.rann < self.val_frac or self.rann >= self.val_frac + self.test_frac:
-                    continue
-            else:
-                raise ValueError(
-                    f'Unsupported section: {self.section}, available options are ["training", "validation", "test"].'
-                )
+        length = len(image_files_list)
+        indices = np.arange(length)
+        self.randomize(indices)
+
+        test_length = int(length * self.test_frac)
+        val_length = int(length * self.val_frac)
+        if self.section == "test":
+            section_indices = indices[:test_length]
+        elif self.section == "validation":
+            section_indices = indices[test_length : test_length + val_length]
+        elif self.section == "training":
+            section_indices = indices[test_length + val_length :]
+        else:
+            raise ValueError(
+                f'Unsupported section: {self.section}, available options are ["training", "validation", "test"].'
+            )
+
+        for i in section_indices:
             data.append({"image": image_files_list[i], "label": image_class[i], "class_name": class_name[i]})
+
         return data
 
 
diff --git a/monai/apps/deepgrow/dataset.py b/monai/apps/deepgrow/dataset.py
index 45cfbde6ea..acaeba0bc3 100644
--- a/monai/apps/deepgrow/dataset.py
+++ b/monai/apps/deepgrow/dataset.py
@@ -22,7 +22,7 @@
 def create_dataset(
     datalist,
     output_dir: str,
-    dimension,
+    dimension: int,
     pixdim,
     image_key: str = "image",
     label_key: str = "label",
@@ -138,7 +138,7 @@ def _save_data_2d(vol_idx, vol_image, vol_label, dataset_dir, relative_path):
     if len(vol_image.shape) == 4:
         logging.info(
             "4D-Image, pick only first series; Image: {}; Label: {}".format(
-                vol_image.shape, vol_label.shape if vol_label else None
+                vol_image.shape, vol_label.shape if vol_label is not None else None
             )
         )
         vol_image = vol_image[0]
@@ -216,7 +216,7 @@ def _save_data_3d(vol_idx, vol_image, vol_label, dataset_dir, relative_path):
     if len(vol_image.shape) == 4:
         logging.info(
             "4D-Image, pick only first series; Image: {}; Label: {}".format(
-                vol_image.shape, vol_label.shape if vol_label else None
+                vol_image.shape, vol_label.shape if vol_label is not None else None
             )
         )
         vol_image = vol_image[0]
diff --git a/monai/apps/deepgrow/interaction.py b/monai/apps/deepgrow/interaction.py
index 77e271a9eb..8a64ad7cf9 100644
--- a/monai/apps/deepgrow/interaction.py
+++ b/monai/apps/deepgrow/interaction.py
@@ -13,9 +13,9 @@
 import torch
 
 from monai.engines import SupervisedEvaluator, SupervisedTrainer
-from monai.engines.utils import CommonKeys
 from monai.engines.workflow import Events
 from monai.transforms import Compose
+from monai.utils.enums import CommonKeys
 
 
 class Interaction:
diff --git a/monai/apps/deepgrow/transforms.py b/monai/apps/deepgrow/transforms.py
index f178360031..1a56357db6 100644
--- a/monai/apps/deepgrow/transforms.py
+++ b/monai/apps/deepgrow/transforms.py
@@ -8,18 +8,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from typing import Callable, Optional, Sequence, Union
+from typing import Callable, Dict, Optional, Sequence, Union
 
 import numpy as np
 import torch
 
 from monai.config import IndexSelection, KeysCollection
 from monai.networks.layers import GaussianFilter
-from monai.transforms import SpatialCrop
-from monai.transforms.compose import MapTransform, Randomizable, Transform
+from monai.transforms import Resize, SpatialCrop
+from monai.transforms.transform import MapTransform, Randomizable, Transform
 from monai.transforms.utils import generate_spatial_bounding_box
-from monai.utils import min_version, optional_import
+from monai.utils import InterpolateMode, ensure_tuple_rep, min_version, optional_import
 
 measure, _ = optional_import("skimage.measure", "0.14.2", min_version)
 distance_transform_cdt, _ = optional_import("scipy.ndimage.morphology", name="distance_transform_cdt")
@@ -48,7 +47,7 @@ def _apply(self, label):
         return np.asarray(sids)
 
     def __call__(self, data):
-        d = dict(data)
+        d: Dict = dict(data)
         label = d[self.label]
         if label.shape[0] != 1:
             raise ValueError("Only supports single channel labels!")
@@ -62,12 +61,13 @@ def __call__(self, data):
         return d
 
 
-class AddInitialSeedPointd(Randomizable, Transform):
+class AddInitialSeedPointd(Randomizable):
     """
     Add random guidance as initial seed point for a given label.
 
     Note that the label is of size (C, D, H, W) or (C, H, W)
-    The guidance is of size (2, N, # of dims) where N is number of guidance added
+
+    The guidance is of size (2, N, # of dims) where N is number of guidance added.
     # of dims = 4 when C, D, H, W; # of dims = 3 when (C, H, W)
 
     Args:
@@ -87,13 +87,21 @@ def __init__(
         connected_regions: int = 5,
     ):
         self.label = label
-        self.sids = sids
-        self.sid = sid
+        self.sids_key = sids
+        self.sid_key = sid
+        self.sid = None
         self.guidance = guidance
         self.connected_regions = connected_regions
 
-    def randomize(self, data=None):
-        pass
+    def randomize(self, data):
+        sid = data.get(self.sid_key, None)
+        sids = data.get(self.sids_key, None)
+        if sids is not None:
+            if sid is None or sid not in sids:
+                sid = self.R.choice(sids, replace=False)
+        else:
+            sid = None
+        self.sid = sid
 
     def _apply(self, label, sid):
         dimensions = 3 if len(label.shape) > 3 else 2
@@ -106,7 +114,8 @@ def _apply(self, label, sid):
 
         label = (label > 0.5).astype(np.float32)
         blobs_labels = measure.label(label.astype(int), background=0) if dims == 2 else label
-        assert np.max(blobs_labels) > 0, "Not a valid Label"
+        if np.max(blobs_labels) <= 0:
+            raise AssertionError("Not a valid Label")
 
         pos_guidance = []
         for ridx in range(1, 2 if dims == 3 else self.connected_regions + 1):
@@ -134,14 +143,8 @@ def _apply(self, label, sid):
 
     def __call__(self, data):
         d = dict(data)
-        sid = d.get(self.sid, None)
-        sids = d.get(self.sids, None)
-        if sids is not None:
-            if sid is None or sid not in sids:
-                sid = self.R.choice(sids, replace=False)
-        else:
-            sid = None
-        d[self.guidance] = self._apply(d[self.label], sid)
+        self.randomize(data)
+        d[self.guidance] = self._apply(d[self.label], self.sid)
         return d
 
 
@@ -232,6 +235,7 @@ class FindDiscrepancyRegionsd(Transform):
     Find discrepancy between prediction and actual during click interactions during training.
 
     If batched is true:
+
         label is in shape (B, C, D, H, W) or (B, C, H, W)
         pred has same shape as label
         discrepancy will have shape (B, 2, C, D, H, W) or (B, 2, C, H, W)
@@ -279,11 +283,11 @@ def __call__(self, data):
         return d
 
 
-class AddRandomGuidanced(Randomizable, Transform):
+class AddRandomGuidanced(Randomizable):
     """
     Add random guidance based on discrepancies that were found between label and prediction.
 
-    If batched is True:
+    If batched is True, input shape is as below:
 
         Guidance is of shape (B, 2, N, # of dim) where B is batch size, 2 means positive and negative,
         N means how many guidance points, # of dim is the total number of dimensions of the image
@@ -291,7 +295,15 @@ class AddRandomGuidanced(Randomizable, Transform):
 
         Discrepancy is of shape (B, 2, C, D, H, W) or (B, 2, C, H, W)
 
-        Probability is of shape (B,)
+        Probability is of shape (B, 1)
+
+    else:
+
+        Guidance is of shape (2, N, # of dim)
+
+        Discrepancy is of shape (2, C, D, H, W) or (2, C, H, W)
+
+        Probability is of shape (1)
 
     Args:
         guidance: key to guidance source.
@@ -389,7 +401,7 @@ class SpatialCropForegroundd(MapTransform):
     """
     Crop only the foreground object of the expected images.
 
-    Difference VS CropForegroundd:
+    Difference VS :py:class:`monai.transforms.CropForegroundd`:
 
       1. If the bounding box is smaller than spatial size in all dimensions then this transform will crop the
          object using box's center and spatial_size.
@@ -399,9 +411,11 @@ class SpatialCropForegroundd(MapTransform):
 
     The typical usage is to help training and evaluation if the valid part is small in the whole medical image.
     The valid part can be determined by any field in the data with `source_key`, for example:
+
     - Select values > 0 in image field as the foreground and crop on all fields specified by `keys`.
     - Select label = 3 in label field as the foreground to crop on all fields specified by `keys`.
     - Select label > 0 in the third channel of a One-Hot label field as the foreground to crop all `keys` fields.
+
     Users can define arbitrary function to select expected foreground from the whole source image or specified
     channels. And it can also add margin to every dim of the bounding box of foreground object.
 
@@ -422,6 +436,7 @@ class SpatialCropForegroundd(MapTransform):
         end_coord_key: key to record the end coordinate of spatial bounding box for foreground.
         original_shape_key: key to record original shape for foreground.
         cropped_shape_key: key to record cropped shape for foreground.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -437,8 +452,9 @@ def __init__(
         end_coord_key: str = "foreground_end_coord",
         original_shape_key: str = "foreground_original_shape",
         cropped_shape_key: str = "foreground_cropped_shape",
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
 
         self.source_key = source_key
         self.spatial_size = list(spatial_size)
@@ -457,8 +473,8 @@ def __call__(self, data):
             d[self.source_key], self.select_fn, self.channel_indices, self.margin
         )
 
-        center = np.mean([box_start, box_end], axis=0).astype(int).tolist()
-        current_size = np.subtract(box_end, box_start).astype(int).tolist()
+        center = list(np.mean([box_start, box_end], axis=0).astype(int))
+        current_size = list(np.subtract(box_end, box_start).astype(int))
 
         if np.all(np.less(current_size, self.spatial_size)):
             cropper = SpatialCrop(roi_center=center, roi_size=self.spatial_size)
@@ -467,7 +483,228 @@ def __call__(self, data):
         else:
             cropper = SpatialCrop(roi_start=box_start, roi_end=box_end)
 
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            meta_key = f"{key}_{self.meta_key_postfix}"
+            d[meta_key][self.start_coord_key] = box_start
+            d[meta_key][self.end_coord_key] = box_end
+            d[meta_key][self.original_shape_key] = d[key].shape
+
+            image = cropper(d[key])
+            d[meta_key][self.cropped_shape_key] = image.shape
+            d[key] = image
+        return d
+
+
+# Transforms to support Inference for Deepgrow models
+class AddGuidanceFromPointsd(Transform):
+    """
+    Add guidance based on user clicks.
+
+    We assume the input is loaded by LoadImaged and has the shape of (H, W, D) originally.
+    Clicks always specify the coordinates in (H, W, D)
+
+    If depth_first is True:
+
+        Input is now of shape (D, H, W), will return guidance that specifies the coordinates in (D, H, W)
+
+    else:
+
+        Input is now of shape (H, W, D), will return guidance that specifies the coordinates in (H, W, D)
+
+    Args:
+        ref_image: key to reference image to fetch current and original image details.
+        guidance: output key to store guidance.
+        foreground: key that represents user foreground (+ve) clicks.
+        background: key that represents user background (-ve) clicks.
+        axis: axis that represents slices in 3D volume. (axis to Depth)
+        depth_first: if depth (slices) is positioned at first dimension.
+        dimensions: dimensions based on model used for deepgrow (2D vs 3D).
+        slice_key: key that represents applicable slice to add guidance.
+        meta_key_postfix: use `{ref_image}_{postfix}` to to fetch the meta data according to the key data,
+            default is `meta_dict`, the meta data is a dictionary object.
+            For example, to handle key `image`,  read/write affine matrices from the
+            metadata `image_meta_dict` dictionary's `affine` field.
+    """
+
+    def __init__(
+        self,
+        ref_image,
+        guidance: str = "guidance",
+        foreground: str = "foreground",
+        background: str = "background",
+        axis: int = 0,
+        depth_first: bool = True,
+        dimensions: int = 2,
+        slice_key: str = "slice",
+        meta_key_postfix: str = "meta_dict",
+    ):
+        self.ref_image = ref_image
+        self.guidance = guidance
+        self.foreground = foreground
+        self.background = background
+        self.axis = axis
+        self.depth_first = depth_first
+        self.dimensions = dimensions
+        self.slice = slice_key
+        self.meta_key_postfix = meta_key_postfix
+
+    def _apply(self, pos_clicks, neg_clicks, factor, slice_num):
+        pos = neg = []
+
+        if self.dimensions == 2:
+            points = list(pos_clicks)
+            points.extend(neg_clicks)
+            points = np.array(points)
+
+            slices = list(np.unique(points[:, self.axis]))
+            slice_idx = slices[0] if slice_num is None else next(x for x in slices if x == slice_num)
+
+            if len(pos_clicks):
+                pos_clicks = np.array(pos_clicks)
+                pos = (pos_clicks[np.where(pos_clicks[:, self.axis] == slice_idx)] * factor)[:, 1:].astype(int).tolist()
+            if len(neg_clicks):
+                neg_clicks = np.array(neg_clicks)
+                neg = (neg_clicks[np.where(neg_clicks[:, self.axis] == slice_idx)] * factor)[:, 1:].astype(int).tolist()
+
+            guidance = [pos, neg, slice_idx]
+        else:
+            if len(pos_clicks):
+                pos = np.multiply(pos_clicks, factor).astype(int).tolist()
+            if len(neg_clicks):
+                neg = np.multiply(neg_clicks, factor).astype(int).tolist()
+            guidance = [pos, neg]
+        return guidance
+
+    def __call__(self, data):
+        d = dict(data)
+        meta_dict_key = f"{self.ref_image}_{self.meta_key_postfix}"
+        if meta_dict_key not in d:
+            raise RuntimeError(f"Missing meta_dict {meta_dict_key} in data!")
+        if "spatial_shape" not in d[meta_dict_key]:
+            raise RuntimeError('Missing "spatial_shape" in meta_dict!')
+        original_shape = d[meta_dict_key]["spatial_shape"]
+        current_shape = list(d[self.ref_image].shape)
+
+        if self.depth_first:
+            if self.axis != 0:
+                raise RuntimeError("Depth first means the depth axis should be 0.")
+            # in here we assume the depth dimension was in the last dimension of "original_shape"
+            original_shape = np.roll(original_shape, 1)
+
+        factor = np.array(current_shape) / original_shape
+
+        fg_bg_clicks = []
+        for key in [self.foreground, self.background]:
+            clicks = d[key]
+            clicks = list(np.array(clicks).astype(int))
+            if self.depth_first:
+                for i in range(len(clicks)):
+                    clicks[i] = list(np.roll(clicks[i], 1))
+            fg_bg_clicks.append(clicks)
+        d[self.guidance] = self._apply(fg_bg_clicks[0], fg_bg_clicks[1], factor, d.get(self.slice))
+        return d
+
+
+class SpatialCropGuidanced(MapTransform):
+    """
+    Crop image based on guidance with minimal spatial size.
+
+    - If the bounding box is smaller than spatial size in all dimensions then this transform will crop the
+      object using box's center and spatial_size.
+
+    - This transform will set "start_coord_key", "end_coord_key", "original_shape_key" and "cropped_shape_key"
+      in data[{key}_{meta_key_postfix}]
+
+    Input data is of shape (C, spatial_1, [spatial_2, ...])
+
+    Args:
+        keys: keys of the corresponding items to be transformed.
+        guidance: key to the guidance. It is used to generate the bounding box of foreground
+        spatial_size: minimal spatial size of the image patch e.g. [128, 128, 128] to fit in.
+        margin: add margin value to spatial dims of the bounding box, if only 1 value provided, use it for all dims.
+        meta_key_postfix: use `key_{postfix}` to to fetch the meta data according to the key data,
+            default is `meta_dict`, the meta data is a dictionary object.
+            For example, to handle key `image`,  read/write affine matrices from the
+            metadata `image_meta_dict` dictionary's `affine` field.
+        start_coord_key: key to record the start coordinate of spatial bounding box for foreground.
+        end_coord_key: key to record the end coordinate of spatial bounding box for foreground.
+        original_shape_key: key to record original shape for foreground.
+        cropped_shape_key: key to record cropped shape for foreground.
+        allow_missing_keys: don't raise exception if key is missing.
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        guidance: str,
+        spatial_size,
+        margin=20,
+        meta_key_postfix="meta_dict",
+        start_coord_key: str = "foreground_start_coord",
+        end_coord_key: str = "foreground_end_coord",
+        original_shape_key: str = "foreground_original_shape",
+        cropped_shape_key: str = "foreground_cropped_shape",
+        allow_missing_keys: bool = False,
+    ) -> None:
+        super().__init__(keys, allow_missing_keys)
+
+        self.guidance = guidance
+        self.spatial_size = list(spatial_size)
+        self.margin = margin
+        self.meta_key_postfix = meta_key_postfix
+        self.start_coord_key = start_coord_key
+        self.end_coord_key = end_coord_key
+        self.original_shape_key = original_shape_key
+        self.cropped_shape_key = cropped_shape_key
+
+    def bounding_box(self, points, img_shape):
+        ndim = len(img_shape)
+        margin = ensure_tuple_rep(self.margin, ndim)
+        for m in margin:
+            if m < 0:
+                raise ValueError("margin value should not be negative number.")
+
+        box_start = [0] * ndim
+        box_end = [0] * ndim
+
+        for di in range(ndim):
+            dt = points[..., di]
+            min_d = max(min(dt - margin[di]), 0)
+            max_d = min(img_shape[di], max(dt + margin[di] + 1))
+            box_start[di], box_end[di] = min_d, max_d
+        return box_start, box_end
+
+    def __call__(self, data):
+        d: Dict = dict(data)
+        guidance = d[self.guidance]
+        original_spatial_shape = d[self.keys[0]].shape[1:]
+        box_start, box_end = self.bounding_box(np.array(guidance[0] + guidance[1]), original_spatial_shape)
+        center = list(np.mean([box_start, box_end], axis=0).astype(int))
+        spatial_size = self.spatial_size
+
+        box_size = list(np.subtract(box_end, box_start).astype(int))
+        spatial_size = spatial_size[-len(box_size) :]
+
+        if len(spatial_size) < len(box_size):
+            # If the data is in 3D and spatial_size is specified as 2D [256,256]
+            # Then we will get all slices in such case
+            diff = len(box_size) - len(spatial_size)
+            spatial_size = list(original_spatial_shape[1 : (1 + diff)]) + spatial_size
+
+        if np.all(np.less(box_size, spatial_size)):
+            if len(center) == 3:
+                # 3D Deepgrow: set center to be middle of the depth dimension (D)
+                center[0] = spatial_size[0] // 2
+            cropper = SpatialCrop(roi_center=center, roi_size=spatial_size)
+        else:
+            cropper = SpatialCrop(roi_start=box_start, roi_end=box_end)
+
+        # update bounding box in case it was corrected by the SpatialCrop constructor
+        box_start = np.array([s.start for s in cropper.slices])
+        box_end = np.array([s.stop for s in cropper.slices])
+        for key in self.key_iterator(d):
+            if not np.array_equal(d[key].shape[1:], original_spatial_shape):
+                raise RuntimeError("All the image specified in keys should have same spatial shape")
             meta_key = f"{key}_{self.meta_key_postfix}"
             d[meta_key][self.start_coord_key] = box_start
             d[meta_key][self.end_coord_key] = box_end
@@ -476,4 +713,230 @@ def __call__(self, data):
             image = cropper(d[key])
             d[meta_key][self.cropped_shape_key] = image.shape
             d[key] = image
+
+        pos_clicks, neg_clicks = guidance[0], guidance[1]
+        pos = np.subtract(pos_clicks, box_start).tolist() if len(pos_clicks) else []
+        neg = np.subtract(neg_clicks, box_start).tolist() if len(neg_clicks) else []
+
+        d[self.guidance] = [pos, neg]
+        return d
+
+
+class ResizeGuidanced(Transform):
+    """
+    Resize the guidance based on cropped vs resized image.
+
+    This transform assumes that the images have been cropped and resized. And the shape after cropped is store inside
+    the meta dict of ref image.
+
+    Args:
+        guidance: key to guidance
+        ref_image: key to reference image to fetch current and original image details
+        meta_key_postfix: use `{ref_image}_{postfix}` to to fetch the meta data according to the key data,
+            default is `meta_dict`, the meta data is a dictionary object.
+            For example, to handle key `image`,  read/write affine matrices from the
+            metadata `image_meta_dict` dictionary's `affine` field.
+        cropped_shape_key: key that records cropped shape for foreground.
+    """
+
+    def __init__(
+        self,
+        guidance: str,
+        ref_image: str,
+        meta_key_postfix="meta_dict",
+        cropped_shape_key: str = "foreground_cropped_shape",
+    ) -> None:
+        self.guidance = guidance
+        self.ref_image = ref_image
+        self.meta_key_postfix = meta_key_postfix
+        self.cropped_shape_key = cropped_shape_key
+
+    def __call__(self, data):
+        d = dict(data)
+        guidance = d[self.guidance]
+        meta_dict: Dict = d[f"{self.ref_image}_{self.meta_key_postfix}"]
+        current_shape = d[self.ref_image].shape[1:]
+        cropped_shape = meta_dict[self.cropped_shape_key][1:]
+        factor = np.divide(current_shape, cropped_shape)
+
+        pos_clicks, neg_clicks = guidance[0], guidance[1]
+        pos = np.multiply(pos_clicks, factor).astype(int).tolist() if len(pos_clicks) else []
+        neg = np.multiply(neg_clicks, factor).astype(int).tolist() if len(neg_clicks) else []
+
+        d[self.guidance] = [pos, neg]
+        return d
+
+
+class RestoreLabeld(MapTransform):
+    """
+    Restores label based on the ref image.
+
+    The ref_image is assumed that it went through the following transforms:
+
+        1. Fetch2DSliced (If 2D)
+        2. Spacingd
+        3. SpatialCropGuidanced
+        4. Resized
+
+    And its shape is assumed to be (C, D, H, W)
+
+    This transform tries to undo these operation so that the result label can be overlapped with original volume.
+    It does the following operation:
+
+        1. Undo Resized
+        2. Undo SpatialCropGuidanced
+        3. Undo Spacingd
+        4. Undo Fetch2DSliced
+
+    The resulting label is of shape (D, H, W)
+
+    Args:
+        keys: keys of the corresponding items to be transformed.
+        ref_image: reference image to fetch current and original image details
+        slice_only: apply only to an applicable slice, in case of 2D model/prediction
+        mode: {``"constant"``, ``"edge"``, ``"linear_ramp"``, ``"maximum"``, ``"mean"``,
+            ``"median"``, ``"minimum"``, ``"reflect"``, ``"symmetric"``, ``"wrap"``, ``"empty"``}
+            One of the listed string values or a user supplied function for padding. Defaults to ``"constant"``.
+            See also: https://numpy.org/doc/1.18/reference/generated/numpy.pad.html
+        align_corners: Geometrically, we consider the pixels of the input as squares rather than points.
+            See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+            It also can be a sequence of bool, each element corresponds to a key in ``keys``.
+        meta_key_postfix: use `{ref_image}_{meta_key_postfix}` to to fetch the meta data according to the key data,
+            default is `meta_dict`, the meta data is a dictionary object.
+            For example, to handle key `image`,  read/write affine matrices from the
+            metadata `image_meta_dict` dictionary's `affine` field.
+        start_coord_key: key that records the start coordinate of spatial bounding box for foreground.
+        end_coord_key: key that records the end coordinate of spatial bounding box for foreground.
+        original_shape_key: key that records original shape for foreground.
+        cropped_shape_key: key that records cropped shape for foreground.
+        allow_missing_keys: don't raise exception if key is missing.
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        ref_image: str,
+        slice_only: bool = False,
+        mode: Union[Sequence[Union[InterpolateMode, str]], InterpolateMode, str] = InterpolateMode.NEAREST,
+        align_corners: Union[Sequence[Optional[bool]], Optional[bool]] = None,
+        meta_key_postfix: str = "meta_dict",
+        start_coord_key: str = "foreground_start_coord",
+        end_coord_key: str = "foreground_end_coord",
+        original_shape_key: str = "foreground_original_shape",
+        cropped_shape_key: str = "foreground_cropped_shape",
+        allow_missing_keys: bool = False,
+    ) -> None:
+        super().__init__(keys, allow_missing_keys)
+        self.ref_image = ref_image
+        self.slice_only = slice_only
+        self.mode = ensure_tuple_rep(mode, len(self.keys))
+        self.align_corners = ensure_tuple_rep(align_corners, len(self.keys))
+        self.meta_key_postfix = meta_key_postfix
+        self.start_coord_key = start_coord_key
+        self.end_coord_key = end_coord_key
+        self.original_shape_key = original_shape_key
+        self.cropped_shape_key = cropped_shape_key
+
+    def __call__(self, data):
+        d = dict(data)
+        meta_dict: Dict = d[f"{self.ref_image}_{self.meta_key_postfix}"]
+
+        for key, mode, align_corners in self.key_iterator(d, self.mode, self.align_corners):
+            image = d[key]
+
+            # Undo Resize
+            current_shape = image.shape
+            cropped_shape = meta_dict[self.cropped_shape_key]
+            if np.any(np.not_equal(current_shape, cropped_shape)):
+                resizer = Resize(spatial_size=cropped_shape[1:], mode=mode)
+                image = resizer(image, mode=mode, align_corners=align_corners)
+
+            # Undo Crop
+            original_shape = meta_dict[self.original_shape_key]
+            result = np.zeros(original_shape, dtype=np.float32)
+            box_start = meta_dict[self.start_coord_key]
+            box_end = meta_dict[self.end_coord_key]
+
+            spatial_dims = min(len(box_start), len(image.shape[1:]))
+            slices = [slice(None)] + [slice(s, e) for s, e in zip(box_start[:spatial_dims], box_end[:spatial_dims])]
+            slices = tuple(slices)
+            result[slices] = image
+
+            # Undo Spacing
+            current_size = result.shape[1:]
+            # change spatial_shape from HWD to DHW
+            spatial_shape = list(np.roll(meta_dict["spatial_shape"], 1))
+            spatial_size = spatial_shape[-len(current_size) :]
+
+            if np.any(np.not_equal(current_size, spatial_size)):
+                resizer = Resize(spatial_size=spatial_size, mode=mode)
+                result = resizer(result, mode=mode, align_corners=align_corners)
+
+            # Undo Slicing
+            slice_idx = meta_dict.get("slice_idx")
+            if slice_idx is None or self.slice_only:
+                final_result = result if len(result.shape) <= 3 else result[0]
+            else:
+                slice_idx = meta_dict["slice_idx"][0]
+                final_result = np.zeros(tuple(spatial_shape))
+                final_result[slice_idx] = result
+            d[key] = final_result
+
+            meta = d.get(f"{key}_{self.meta_key_postfix}")
+            if meta is None:
+                meta = dict()
+                d[f"{key}_{self.meta_key_postfix}"] = meta
+            meta["slice_idx"] = slice_idx
+            meta["affine"] = meta_dict["original_affine"]
+        return d
+
+
+class Fetch2DSliced(MapTransform):
+    """
+    Fetch one slice in case of a 3D volume.
+
+    The volume only contains spatial coordinates.
+
+    Args:
+        keys: keys of the corresponding items to be transformed.
+        guidance: key that represents guidance.
+        axis: axis that represents slice in 3D volume.
+        meta_key_postfix: use `key_{meta_key_postfix}` to to fetch the meta data according to the key data,
+            default is `meta_dict`, the meta data is a dictionary object.
+            For example, to handle key `image`,  read/write affine matrices from the
+            metadata `image_meta_dict` dictionary's `affine` field.
+        allow_missing_keys: don't raise exception if key is missing.
+    """
+
+    def __init__(
+        self,
+        keys,
+        guidance="guidance",
+        axis: int = 0,
+        meta_key_postfix: str = "meta_dict",
+        allow_missing_keys: bool = False,
+    ):
+        super().__init__(keys, allow_missing_keys)
+        self.guidance = guidance
+        self.axis = axis
+        self.meta_key_postfix = meta_key_postfix
+
+    def _apply(self, image, guidance):
+        slice_idx = guidance[2]  # (pos, neg, slice_idx)
+        idx = []
+        for i in range(len(image.shape)):
+            idx.append(slice_idx) if i == self.axis else idx.append(slice(0, image.shape[i]))
+
+        idx = tuple(idx)
+        return image[idx], idx
+
+    def __call__(self, data):
+        d = dict(data)
+        guidance = d[self.guidance]
+        if len(guidance) < 3:
+            raise RuntimeError("Guidance does not container slice_idx!")
+        for key in self.key_iterator(d):
+            img_slice, idx = self._apply(d[key], guidance)
+            d[key] = img_slice
+            d[f"{key}_{self.meta_key_postfix}"]["slice_idx"] = idx
         return d
diff --git a/monai/apps/pathology/__init__.py b/monai/apps/pathology/__init__.py
new file mode 100644
index 0000000000..203e1a80d7
--- /dev/null
+++ b/monai/apps/pathology/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .datasets import MaskedInferenceWSIDataset, PatchWSIDataset, SmartCacheDataset
+from .handlers import ProbMapProducer
+from .metrics import LesionFROC
+from .utils import PathologyProbNMS, compute_isolated_tumor_cells, compute_multi_instance_mask
diff --git a/monai/apps/pathology/datasets.py b/monai/apps/pathology/datasets.py
new file mode 100644
index 0000000000..cba8cd2da9
--- /dev/null
+++ b/monai/apps/pathology/datasets.py
@@ -0,0 +1,311 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
+
+import numpy as np
+
+from monai.data import Dataset, SmartCacheDataset
+from monai.data.image_reader import WSIReader
+from monai.utils import ensure_tuple_rep
+
+__all__ = ["PatchWSIDataset", "SmartCachePatchWSIDataset", "MaskedInferenceWSIDataset"]
+
+
+class PatchWSIDataset(Dataset):
+    """
+    This dataset reads whole slide images, extracts regions, and creates patches.
+    It also reads labels for each patch and provides each patch with its associated class labels.
+
+    Args:
+        data: the list of input samples including image, location, and label (see the note below for more details).
+        region_size: the size of regions to be extracted from the whole slide image.
+        grid_shape: the grid shape on which the patches should be extracted.
+        patch_size: the size of patches extracted from the region on the grid.
+        transform: transforms to be executed on input data.
+        image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
+            Defaults to CuCIM.
+
+    Note:
+        The input data has the following form as an example:
+        `[{"image": "path/to/image1.tiff", "location": [200, 500], "label": [0,0,0,1]}]`.
+
+        This means from "image1.tiff" extract a region centered at the given location `location`
+        with the size of `region_size`, and then extract patches with the size of `patch_size`
+        from a grid with the shape of `grid_shape`.
+        Be aware the the `grid_shape` should construct a grid with the same number of element as `labels`,
+        so for this example the `grid_shape` should be (2, 2).
+
+    """
+
+    def __init__(
+        self,
+        data: List,
+        region_size: Union[int, Tuple[int, int]],
+        grid_shape: Union[int, Tuple[int, int]],
+        patch_size: Union[int, Tuple[int, int]],
+        transform: Optional[Callable] = None,
+        image_reader_name: str = "cuCIM",
+    ):
+        super().__init__(data, transform)
+
+        self.region_size = ensure_tuple_rep(region_size, 2)
+        self.grid_shape = ensure_tuple_rep(grid_shape, 2)
+        self.patch_size = ensure_tuple_rep(patch_size, 2)
+
+        self.image_path_list = list({x["image"] for x in self.data})
+        self.image_reader_name = image_reader_name
+        self.image_reader = WSIReader(image_reader_name)
+        self.wsi_object_dict = None
+        if self.image_reader_name != "openslide":
+            # OpenSlide causes memory issue if we prefetch image objects
+            self._fetch_wsi_objects()
+
+    def _fetch_wsi_objects(self):
+        """Load all the image objects and reuse them when asked for an item."""
+        self.wsi_object_dict = {}
+        for image_path in self.image_path_list:
+            self.wsi_object_dict[image_path] = self.image_reader.read(image_path)
+
+    def __getitem__(self, index):
+        sample = self.data[index]
+        if self.image_reader_name == "openslide":
+            img_obj = self.image_reader.read(sample["image"])
+        else:
+            img_obj = self.wsi_object_dict[sample["image"]]
+        location = [sample["location"][i] - self.region_size[i] // 2 for i in range(len(self.region_size))]
+        images, _ = self.image_reader.get_data(
+            img=img_obj,
+            location=location,
+            size=self.region_size,
+            grid_shape=self.grid_shape,
+            patch_size=self.patch_size,
+        )
+        labels = np.array(sample["label"], dtype=np.float32)
+        # expand dimensions to have 4 dimension as batch, class, height, and width.
+        for _ in range(4 - labels.ndim):
+            labels = np.expand_dims(labels, 1)
+        patches = [{"image": images[i], "label": labels[i]} for i in range(len(sample["label"]))]
+        if self.transform:
+            patches = self.transform(patches)
+        return patches
+
+
+class SmartCachePatchWSIDataset(SmartCacheDataset):
+    """Add SmartCache functionality to `PatchWSIDataset`.
+
+    Args:
+        data: the list of input samples including image, location, and label (see `PatchWSIDataset` for more details)
+        region_size: the region to be extracted from the whole slide image.
+        grid_shape: the grid shape on which the patches should be extracted.
+        patch_size: the size of patches extracted from the region on the grid.
+        image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
+            Defaults to CuCIM.
+        transform: transforms to be executed on input data.
+        replace_rate: percentage of the cached items to be replaced in every epoch.
+        cache_num: number of items to be cached. Default is `sys.maxsize`.
+            will take the minimum of (cache_num, data_length x cache_rate, data_length).
+        cache_rate: percentage of cached data in total, default is 1.0 (cache all).
+            will take the minimum of (cache_num, data_length x cache_rate, data_length).
+        num_init_workers: the number of worker threads to initialize the cache for first epoch.
+            If num_init_workers is None then the number returned by os.cpu_count() is used.
+        num_replace_workers: the number of worker threads to prepare the replacement cache for every epoch.
+            If num_replace_workers is None then the number returned by os.cpu_count() is used.
+        progress: whether to display a progress bar when caching for the first epoch.
+
+    """
+
+    def __init__(
+        self,
+        data: List,
+        region_size: Union[int, Tuple[int, int]],
+        grid_shape: Union[int, Tuple[int, int]],
+        patch_size: Union[int, Tuple[int, int]],
+        transform: Union[Sequence[Callable], Callable],
+        image_reader_name: str = "cuCIM",
+        replace_rate: float = 0.5,
+        cache_num: int = sys.maxsize,
+        cache_rate: float = 1.0,
+        num_init_workers: Optional[int] = None,
+        num_replace_workers: Optional[int] = None,
+        progress: bool = True,
+    ):
+        patch_wsi_dataset = PatchWSIDataset(
+            data=data,
+            region_size=region_size,
+            grid_shape=grid_shape,
+            patch_size=patch_size,
+            image_reader_name=image_reader_name,
+        )
+        super().__init__(
+            data=patch_wsi_dataset,  # type: ignore
+            transform=transform,
+            replace_rate=replace_rate,
+            cache_num=cache_num,
+            cache_rate=cache_rate,
+            num_init_workers=num_init_workers,
+            num_replace_workers=num_replace_workers,
+            progress=progress,
+            shuffle=False,
+        )
+
+
+class MaskedInferenceWSIDataset(Dataset):
+    """
+    This dataset load the provided foreground masks at an arbitrary resolution level,
+    and extract patches based on that mask from the associated whole slide image.
+
+    Args:
+        data: a list of sample including the path to the whole slide image and the path to the mask.
+            Like this: `[{"image": "path/to/image1.tiff", "mask": "path/to/mask1.npy}, ...]"`.
+        patch_size: the size of patches to be extracted from the whole slide image for inference.
+        transform: transforms to be executed on extracted patches.
+        image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
+        Defaults to CuCIM.
+
+    Note:
+        The resulting output (probability maps) after performing inference using this dataset is
+            supposed to be the same size as the foreground mask and not the original wsi image size.
+    """
+
+    def __init__(
+        self,
+        data: List[Dict["str", "str"]],
+        patch_size: Union[int, Tuple[int, int]],
+        transform: Optional[Callable] = None,
+        image_reader_name: str = "cuCIM",
+    ) -> None:
+        super().__init__(data, transform)
+
+        self.patch_size = ensure_tuple_rep(patch_size, 2)
+
+        # set up whole slide image reader
+        self.image_reader_name = image_reader_name
+        self.image_reader = WSIReader(image_reader_name)
+
+        # process data and create a list of dictionaries containing all required data and metadata
+        self.data = self._prepare_data(data)
+
+        # calculate cumulative number of patches for all the samples
+        self.num_patches_per_sample = [len(d["image_locations"]) for d in self.data]
+        self.num_patches = sum(self.num_patches_per_sample)
+        self.cum_num_patches = np.cumsum([0] + self.num_patches_per_sample[:-1])
+
+    def _prepare_data(self, input_data: List[Dict["str", "str"]]) -> List[Dict]:
+        prepared_data = []
+        for sample in input_data:
+            prepared_sample = self._prepare_a_sample(sample)
+            prepared_data.append(prepared_sample)
+        return prepared_data
+
+    def _prepare_a_sample(self, sample: Dict["str", "str"]) -> Dict:
+        """
+        Preprocess input data to load WSIReader object and the foreground mask,
+        and define the locations where patches need to be extracted.
+
+        Args:
+            sample: one sample, a dictionary containing path to the whole slide image and the foreground mask.
+                For example: `{"image": "path/to/image1.tiff", "mask": "path/to/mask1.npy}`
+
+        Return:
+            A dictionary containing:
+                "name": the base name of the whole slide image,
+                "image": the WSIReader image object,
+                "mask_shape": the size of the foreground mask,
+                "mask_locations": the list of non-zero pixel locations (x, y) on the foreground mask,
+                "image_locations": the list of pixel locations (x, y) on the whole slide image where patches are extracted, and
+                "level": the resolution level of the mask with respect to the whole slide image.
+        }
+        """
+        image = self.image_reader.read(sample["image"])
+        mask = np.load(sample["mask"]).T
+        try:
+            level, ratio = self._calculate_mask_level(image, mask)
+        except ValueError as err:
+            err.args = (sample["mask"],) + err.args
+            raise
+
+        # get all indices for non-zero pixels of the foreground mask
+        mask_locations = np.vstack(mask.nonzero()).T
+
+        # convert mask locations to image locations to extract patches
+        image_locations = (mask_locations + 0.5) * ratio - np.array(self.patch_size) // 2
+
+        return {
+            "name": os.path.splitext(os.path.basename(sample["image"]))[0],
+            "image": image,
+            "mask_shape": mask.shape,
+            "mask_locations": mask_locations.astype(int).tolist(),
+            "image_locations": image_locations.astype(int).tolist(),
+            "level": level,
+        }
+
+    def _calculate_mask_level(self, image: np.ndarray, mask: np.ndarray) -> Tuple[int, float]:
+        """
+        Calculate level of the mask and its ratio with respect to the whole slide image
+
+        Args:
+            image: the original whole slide image
+            mask: a mask, that can be down-sampled at an arbitrary level.
+                Note that down-sampling ratio should be 2^N and equal in all dimension.
+
+        Return:
+            tuple: (level, ratio) where ratio is 2^level
+
+        """
+        image_shape = image.shape
+        mask_shape = mask.shape
+        ratios = [image_shape[i] / mask_shape[i] for i in range(2)]
+        level = np.log2(ratios[0])
+
+        if ratios[0] != ratios[1]:
+            raise ValueError(
+                "Image/Mask ratio across dimensions does not match!"
+                f"ratio 0: {ratios[0]} ({image_shape[0]} / {mask_shape[0]}),"
+                f"ratio 1: {ratios[1]} ({image_shape[1]} / {mask_shape[1]}),"
+            )
+        elif not level.is_integer():
+            raise ValueError(f"Mask is not at a regular level (ratio not power of 2), image / mask ratio: {ratios[0]}")
+
+        return int(level), ratios[0]
+
+    def _load_a_patch(self, index):
+        """
+        Load sample given the index
+
+        Since index is sequential and the patches are coming in an stream from different images,
+        this method, first, finds the whole slide image and the patch that should be extracted,
+        then it loads the patch and provide it with its image name and the corresponding mask location.
+        """
+        sample_num = np.argmax(self.cum_num_patches > index) - 1
+        sample = self.data[sample_num]
+        patch_num = index - self.cum_num_patches[sample_num]
+        location_on_image = sample["image_locations"][patch_num]
+        location_on_mask = sample["mask_locations"][patch_num]
+
+        image, _ = self.image_reader.get_data(
+            img=sample["image"],
+            location=location_on_image,
+            size=self.patch_size,
+        )
+        processed_sample = {"image": image, "name": sample["name"], "mask_location": location_on_mask}
+        return processed_sample
+
+    def __len__(self):
+        return self.num_patches
+
+    def __getitem__(self, index):
+        patch = [self._load_a_patch(index)]
+        if self.transform:
+            patch = self.transform(patch)
+        return patch
diff --git a/monai/apps/pathology/handlers.py b/monai/apps/pathology/handlers.py
new file mode 100644
index 0000000000..f0790c20b1
--- /dev/null
+++ b/monai/apps/pathology/handlers.py
@@ -0,0 +1,114 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+from typing import TYPE_CHECKING, Dict, Optional
+
+import numpy as np
+
+from monai.config import DtypeLike
+from monai.utils import exact_version, optional_import
+
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+if TYPE_CHECKING:
+    from ignite.engine import Engine
+else:
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+
+
+class ProbMapProducer:
+    """
+    Event handler triggered on completing every iteration to save the probability map
+    """
+
+    def __init__(
+        self,
+        output_dir: str = "./",
+        output_postfix: str = "",
+        dtype: DtypeLike = np.float64,
+        name: Optional[str] = None,
+    ) -> None:
+        """
+        Args:
+            output_dir: output directory to save probability maps.
+            output_postfix: a string appended to all output file names.
+            dtype: the data type in which the probability map is stored. Default np.float64.
+            name: identifier of logging.logger to use, defaulting to `engine.logger`.
+
+        """
+        self.logger = logging.getLogger(name)
+        self._name = name
+        self.output_dir = output_dir
+        self.output_postfix = output_postfix
+        self.dtype = dtype
+        self.prob_map: Dict[str, np.ndarray] = {}
+        self.level: Dict[str, int] = {}
+        self.counter: Dict[str, int] = {}
+        self.num_done_images: int = 0
+        self.num_images: int = 0
+
+    def attach(self, engine: Engine) -> None:
+        """
+        Args:
+            engine: Ignite Engine, it can be a trainer, validator or evaluator.
+        """
+
+        self.num_images = len(engine.data_loader.dataset.data)
+
+        for sample in engine.data_loader.dataset.data:
+            name = sample["name"]
+            self.prob_map[name] = np.zeros(sample["mask_shape"], dtype=self.dtype)
+            self.counter[name] = len(sample["mask_locations"])
+            self.level[name] = sample["level"]
+
+        if self._name is None:
+            self.logger = engine.logger
+        if not engine.has_event_handler(self, Events.ITERATION_COMPLETED):
+            engine.add_event_handler(Events.ITERATION_COMPLETED, self)
+        if not engine.has_event_handler(self.finalize, Events.COMPLETED):
+            engine.add_event_handler(Events.COMPLETED, self.finalize)
+
+    def __call__(self, engine: Engine) -> None:
+        """
+        This method assumes self.batch_transform will extract metadata from the input batch.
+
+        Args:
+            engine: Ignite Engine, it can be a trainer, validator or evaluator.
+        """
+        names = engine.state.batch["name"]
+        locs = engine.state.batch["mask_location"]
+        pred = engine.state.output["pred"]
+        for i, name in enumerate(names):
+            self.prob_map[name][locs[0][i], locs[1][i]] = pred[i]
+            self.counter[name] -= 1
+            if self.counter[name] == 0:
+                self.save_prob_map(name)
+
+    def save_prob_map(self, name: str) -> None:
+        """
+        This method save the probability map for an image, when its inference is finished,
+        and delete that probability map from memory.
+
+        Args:
+            name: the name of image to be saved.
+        """
+        file_path = os.path.join(self.output_dir, name)
+        np.save(file_path + self.output_postfix + ".npy", self.prob_map[name])
+
+        self.num_done_images += 1
+        self.logger.info(f"Inference of '{name}' is done [{self.num_done_images}/{self.num_images}]!")
+        del self.prob_map[name]
+        del self.counter[name]
+        del self.level[name]
+
+    def finalize(self, engine: Engine):
+        self.logger.info(f"Probability map is created for {self.num_done_images}/{self.num_images} images!")
diff --git a/monai/apps/pathology/metrics.py b/monai/apps/pathology/metrics.py
new file mode 100644
index 0000000000..2140de0080
--- /dev/null
+++ b/monai/apps/pathology/metrics.py
@@ -0,0 +1,184 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Dict, List, Tuple
+
+import numpy as np
+
+from monai.apps.pathology.utils import PathologyProbNMS, compute_isolated_tumor_cells, compute_multi_instance_mask
+from monai.data.image_reader import WSIReader
+from monai.metrics import compute_fp_tp_probs, compute_froc_curve_data, compute_froc_score
+from monai.utils import min_version, optional_import
+
+if TYPE_CHECKING:
+    from tqdm import tqdm
+
+    has_tqdm = True
+else:
+    tqdm, has_tqdm = optional_import("tqdm", "4.47.0", min_version, "tqdm")
+
+if not has_tqdm:
+
+    def tqdm(x):
+        return x
+
+
+class LesionFROC:
+    """
+    Evaluate with Free Response Operating Characteristic (FROC) score.
+
+    Args:
+        data: either the list of dictionaries containing probability maps (inference result) and
+            tumor mask (ground truth), as below, or the path to a json file containing such list.
+            `{
+            "prob_map": "path/to/prob_map_1.npy",
+            "tumor_mask": "path/to/ground_truth_1.tiff",
+            "level": 6,
+            "pixel_spacing": 0.243
+            }`
+        grow_distance: Euclidean distance (in micrometer) by which to grow the label the ground truth's tumors.
+            Defaults to 75, which is the equivalent size of 5 tumor cells.
+        itc_diameter: the maximum diameter of a region (in micrometer) to be considered as an isolated tumor cell.
+            Defaults to 200.
+        eval_thresholds: the false positive rates for calculating the average sensitivity.
+            Defaults to (0.25, 0.5, 1, 2, 4, 8) which is the same as the CAMELYON 16 Challenge.
+        nms_sigma: the standard deviation for gaussian filter of non-maximal suppression. Defaults to 0.0.
+        nms_prob_threshold: the probability threshold of non-maximal suppression. Defaults to 0.5.
+        nms_box_size: the box size (in pixel) to be removed around the the pixel for non-maximal suppression.
+        image_reader_name: the name of library to be used for loading whole slide imaging, either CuCIM or OpenSlide.
+            Defaults to CuCIM.
+
+    Note:
+        For more info on `nms_*` parameters look at monai.utils.prob_nms.ProbNMS`.
+
+    """
+
+    def __init__(
+        self,
+        data: List[Dict],
+        grow_distance: int = 75,
+        itc_diameter: int = 200,
+        eval_thresholds: Tuple = (0.25, 0.5, 1, 2, 4, 8),
+        nms_sigma: float = 0.0,
+        nms_prob_threshold: float = 0.5,
+        nms_box_size: int = 48,
+        image_reader_name: str = "cuCIM",
+    ) -> None:
+
+        self.data = data
+        self.grow_distance = grow_distance
+        self.itc_diameter = itc_diameter
+        self.eval_thresholds = eval_thresholds
+        self.image_reader = WSIReader(image_reader_name)
+        self.nms = PathologyProbNMS(
+            sigma=nms_sigma,
+            prob_threshold=nms_prob_threshold,
+            box_size=nms_box_size,
+        )
+
+    def prepare_inference_result(self, sample: Dict):
+        """
+        Prepare the probability map for detection evaluation.
+
+        """
+        # load the probability map (the result of model inference)
+        prob_map = np.load(sample["prob_map"])
+
+        # apply non-maximal suppression
+        nms_outputs = self.nms(probs_map=prob_map, resolution_level=sample["level"])
+
+        # separate nms outputs
+        if nms_outputs:
+            probs, x_coord, y_coord = zip(*nms_outputs)
+        else:
+            probs, x_coord, y_coord = [], [], []
+
+        return np.array(probs), np.array(x_coord), np.array(y_coord)
+
+    def prepare_ground_truth(self, sample):
+        """
+        Prepare the ground truth for evaluation based on the binary tumor mask
+
+        """
+        # load binary tumor masks
+        img_obj = self.image_reader.read(sample["tumor_mask"])
+        tumor_mask = self.image_reader.get_data(img_obj, level=sample["level"])[0][0]
+
+        # calculate pixel spacing at the mask level
+        mask_pixel_spacing = sample["pixel_spacing"] * pow(2, sample["level"])
+
+        # compute multi-instance mask from a binary mask
+        grow_pixel_threshold = self.grow_distance / (mask_pixel_spacing * 2)
+        tumor_mask = compute_multi_instance_mask(mask=tumor_mask, threshold=grow_pixel_threshold)
+
+        # identify isolated tumor cells
+        itc_threshold = (self.itc_diameter + self.grow_distance) / mask_pixel_spacing
+        itc_labels = compute_isolated_tumor_cells(tumor_mask=tumor_mask, threshold=itc_threshold)
+
+        return tumor_mask, itc_labels
+
+    def compute_fp_tp(self):
+        """
+        Compute false positive and true positive probabilities for tumor detection,
+        by comparing the model outputs with the prepared ground truths for all samples
+
+        """
+        total_fp_probs, total_tp_probs = [], []
+        total_num_targets = 0
+        num_images = len(self.data)
+
+        for sample in tqdm(self.data):
+            probs, y_coord, x_coord = self.prepare_inference_result(sample)
+            ground_truth, itc_labels = self.prepare_ground_truth(sample)
+            # compute FP and TP probabilities for a pair of an image and an ground truth mask
+            fp_probs, tp_probs, num_targets = compute_fp_tp_probs(
+                probs=probs,
+                y_coord=y_coord,
+                x_coord=x_coord,
+                evaluation_mask=ground_truth,
+                labels_to_exclude=itc_labels,
+                resolution_level=sample["level"],
+            )
+            total_fp_probs.extend(fp_probs)
+            total_tp_probs.extend(tp_probs)
+            total_num_targets += num_targets
+
+        return (
+            np.array(total_fp_probs),
+            np.array(total_tp_probs),
+            total_num_targets,
+            num_images,
+        )
+
+    def evaluate(self):
+        """
+        Evaluate the detection performance of a model based on the model probability map output,
+        the ground truth tumor mask, and their associated metadata (e.g., pixel_spacing, level)
+        """
+        # compute false positive (FP) and true positive (TP) probabilities for all images
+        fp_probs, tp_probs, num_targets, num_images = self.compute_fp_tp()
+
+        # compute FROC curve given the evaluation of all images
+        fps_per_image, total_sensitivity = compute_froc_curve_data(
+            fp_probs=fp_probs,
+            tp_probs=tp_probs,
+            num_targets=num_targets,
+            num_images=num_images,
+        )
+
+        # compute FROC score give specific evaluation threshold
+        froc_score = compute_froc_score(
+            fps_per_image=fps_per_image,
+            total_sensitivity=total_sensitivity,
+            eval_thresholds=self.eval_thresholds,
+        )
+
+        return froc_score
diff --git a/monai/apps/pathology/utils.py b/monai/apps/pathology/utils.py
new file mode 100644
index 0000000000..0d1f530bff
--- /dev/null
+++ b/monai/apps/pathology/utils.py
@@ -0,0 +1,85 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Union
+
+import numpy as np
+import torch
+
+from monai.transforms.post.array import ProbNMS
+from monai.utils import optional_import
+
+measure, _ = optional_import("skimage.measure")
+ndimage, _ = optional_import("scipy.ndimage")
+
+
+def compute_multi_instance_mask(mask: np.ndarray, threshold: float):
+    """
+    This method computes the segmentation mask according to the binary tumor mask.
+
+    Args:
+        mask: the binary mask array
+        threshold: the threshold to fill holes
+    """
+
+    neg = 255 - mask * 255
+    distance = ndimage.morphology.distance_transform_edt(neg)
+    binary = distance < threshold
+
+    filled_image = ndimage.morphology.binary_fill_holes(binary)
+    multi_instance_mask = measure.label(filled_image, connectivity=2)
+
+    return multi_instance_mask
+
+
+def compute_isolated_tumor_cells(tumor_mask: np.ndarray, threshold: float) -> List[int]:
+    """
+    This method computes identifies Isolated Tumor Cells (ITC) and return their labels.
+
+    Args:
+        tumor_mask: the tumor mask.
+        threshold: the threshold (at the mask level) to define an isolated tumor cell (ITC).
+            A region with the longest diameter less than this threshold is considered as an ITC.
+    """
+    max_label = np.amax(tumor_mask)
+    properties = measure.regionprops(tumor_mask, coordinates="rc")
+    itc_list = []
+    for i in range(max_label):  # type: ignore
+        if properties[i].major_axis_length < threshold:
+            itc_list.append(i + 1)
+
+    return itc_list
+
+
+class PathologyProbNMS(ProbNMS):
+    """
+    This class extends monai.utils.ProbNMS and add the `resolution` option for
+    Pathology.
+    """
+
+    def __call__(
+        self,
+        probs_map: Union[np.ndarray, torch.Tensor],
+        resolution_level: int = 0,
+    ):
+        """
+        probs_map: the input probabilities map, it must have shape (H[, W, ...]).
+        resolution_level: the level at which the probabilities map is made.
+        """
+        resolution = pow(2, resolution_level)
+        org_outputs = ProbNMS.__call__(self, probs_map)
+        outputs = []
+        for org_output in org_outputs:
+            prob = org_output[0]
+            coord = np.asarray(org_output[1:])
+            coord_wsi = ((coord + 0.5) * resolution).astype(int)
+            outputs.append([prob] + list(coord_wsi))
+        return outputs
diff --git a/monai/config/deviceconfig.py b/monai/config/deviceconfig.py
index be77a1d975..213be56b5c 100644
--- a/monai/config/deviceconfig.py
+++ b/monai/config/deviceconfig.py
@@ -131,7 +131,8 @@ def get_system_info() -> OrderedDict:
     elif output["System"] == "Darwin":
         _dict_append(output, "Mac version", lambda: platform.mac_ver()[0])
     else:
-        linux_ver = re.search(r'PRETTY_NAME="(.*)"', open("/etc/os-release", "r").read())
+        with open("/etc/os-release", "r") as rel_f:
+            linux_ver = re.search(r'PRETTY_NAME="(.*)"', rel_f.read())
         if linux_ver:
             _dict_append(output, "Linux version", lambda: linux_ver.group(1))
 
diff --git a/monai/csrc/ext.cpp b/monai/csrc/ext.cpp
index 2e0644bc78..b4bb0f2c04 100644
--- a/monai/csrc/ext.cpp
+++ b/monai/csrc/ext.cpp
@@ -29,14 +29,20 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
 
   // resample bound mode
   py::enum_<monai::BoundType>(m, "BoundType")
-      .value("replicate", monai::BoundType::Replicate)
-      .value("dct1", monai::BoundType::DCT1)
-      .value("dct2", monai::BoundType::DCT2)
-      .value("dst1", monai::BoundType::DST1)
-      .value("dst2", monai::BoundType::DST2)
-      .value("dft", monai::BoundType::DFT)
-      .value("sliding", monai::BoundType::Sliding)
-      .value("zero", monai::BoundType::Zero)
+      .value("replicate", monai::BoundType::Replicate, "a a a | a b c d | d d d")
+      .value("nearest", monai::BoundType::Replicate, "a a a | a b c d | d d d")
+      .value("dct1", monai::BoundType::DCT1, "d c b | a b c d | c b a")
+      .value("mirror", monai::BoundType::DCT1, "d c b | a b c d | c b a")
+      .value("dct2", monai::BoundType::DCT2, "c b a | a b c d | d c b")
+      .value("reflect", monai::BoundType::DCT2, "c b a | a b c d | d c b")
+      .value("dst1", monai::BoundType::DST1, "-b -a 0 | a b c d | 0 -d -c")
+      .value("antimirror", monai::BoundType::DST1, "-b -a 0 | a b c d | 0 -d -c")
+      .value("dst2", monai::BoundType::DST2, "-c -b -a | a b c d | -d -c -b")
+      .value("antireflect", monai::BoundType::DST2, "-c -b -a | a b c d | -d -c -b")
+      .value("dft", monai::BoundType::DFT, "b c d | a b c d | a b c")
+      .value("wrap", monai::BoundType::DFT, "b c d | a b c d | a b c")
+      //   .value("sliding", monai::BoundType::Sliding)
+      .value("zero", monai::BoundType::Zero, "0 0 0 | a b c d | 0 0 0")
       .export_values();
 
   // resample interpolation mode
diff --git a/monai/csrc/filtering/bilateral/bilateralfilter_cuda_phl.cu b/monai/csrc/filtering/bilateral/bilateralfilter_cuda_phl.cu
index 603ab689cf..17dc9e7ebd 100644
--- a/monai/csrc/filtering/bilateral/bilateralfilter_cuda_phl.cu
+++ b/monai/csrc/filtering/bilateral/bilateralfilter_cuda_phl.cu
@@ -95,7 +95,7 @@ void BilateralFilterPHLCuda(
   cudaMalloc(&data, desc.batchCount * desc.channelStride * desc.channelCount * sizeof(scalar_t));
   cudaMalloc(&features, desc.batchCount * desc.channelStride * featureChannelCount * sizeof(scalar_t));
 
-  // Prparing constant memory
+  // Preparing constant memory
   cudaMemcpyToSymbol(cBatchStride, &desc.batchStride, sizeof(int));
   cudaMemcpyToSymbol(cChannelStride, &desc.channelStride, sizeof(int));
   cudaMemcpyToSymbol(cSpatialStrides, desc.strides, sizeof(int) * desc.dimensions);
diff --git a/monai/csrc/filtering/permutohedral/hash_table.cuh b/monai/csrc/filtering/permutohedral/hash_table.cuh
index 7d9d7eb163..f9893dffe2 100644
--- a/monai/csrc/filtering/permutohedral/hash_table.cuh
+++ b/monai/csrc/filtering/permutohedral/hash_table.cuh
@@ -15,7 +15,7 @@ limitations under the License.
 
 //#define USE_ADDITIVE_HASH
 
-// turn this on if you want to get slighly less memory consumption and slightly longer run times.
+// turn this on if you want to get slightly less memory consumption and slightly longer run times.
 //#define LINEAR_D_MEMORY
 
 #define USE_CUSTOM_MODULO
diff --git a/monai/csrc/filtering/permutohedral/permutohedral.cpp b/monai/csrc/filtering/permutohedral/permutohedral.cpp
index 5d6916b8f4..04ef6fa4da 100644
--- a/monai/csrc/filtering/permutohedral/permutohedral.cpp
+++ b/monai/csrc/filtering/permutohedral/permutohedral.cpp
@@ -1,3 +1,16 @@
+/*
+Copyright 2020 - 2021 MONAI Consortium
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 #include "utils/common_utils.h"
 #include "utils/meta_macros.h"
 
diff --git a/monai/csrc/resample/pushpull.h b/monai/csrc/resample/pushpull.h
index 45fd5ce564..1c20cc0114 100644
--- a/monai/csrc/resample/pushpull.h
+++ b/monai/csrc/resample/pushpull.h
@@ -69,8 +69,8 @@ at::Tensor grid_pull(
   CHECK_STRIDED(grid_opt)
   CHECK_SAME_DEVICE(input_opt, grid_opt)
   CHECK_SAME_DTYPE(input_opt, grid_opt)
-  CHECK_SPATIAL_2D_OR_3D(input)
-  CHECK_SPATIAL_2D_OR_3D(grid)
+  CHECK_SPATIAL_1D_2D_OR_3D(input)
+  CHECK_SPATIAL_1D_2D_OR_3D(grid)
   CHECK_GRID_COMPONENT(grid, grid.dim())
   CHECK_SPATIAL_NOT_EMPTY(input)
   CHECK_SPATIAL_NOT_EMPTY(grid)
@@ -165,8 +165,8 @@ at::Tensor grid_push(
   CHECK_STRIDED(grid_opt)
   CHECK_SAME_DEVICE(input_opt, grid_opt)
   CHECK_SAME_DTYPE(input_opt, grid_opt)
-  CHECK_SPATIAL_2D_OR_3D(input)
-  CHECK_SPATIAL_2D_OR_3D(grid)
+  CHECK_SPATIAL_1D_2D_OR_3D(input)
+  CHECK_SPATIAL_1D_2D_OR_3D(grid)
   CHECK_GRID_COMPONENT(grid, grid.dim())
   CHECK_SPATIAL_NOT_EMPTY(input)
   CHECK_SPATIAL_NOT_EMPTY(grid)
@@ -175,7 +175,10 @@ at::Tensor grid_push(
   CHECK_VEC_NOT_EMPTY(interpolation_mode);
 
   if (source_size.empty()) {
-    auto size = c10::IntArrayRef({input.size(2), input.size(3), input.dim() == 5 ? input.size(4) : 1});
+    auto size = c10::IntArrayRef(
+        {input.dim() >= 3 ? input.size(2) : 1,
+         input.dim() >= 4 ? input.size(3) : 1,
+         input.dim() >= 5 ? input.size(4) : 1});
     if (input.is_cuda())
 #ifdef WITH_CUDA
       return cuda::pushpull(
@@ -295,14 +298,15 @@ at::Tensor grid_count(
   CHECK_DEFINED(grid)
   auto grid_opt = grid.options();
   CHECK_STRIDED(grid_opt)
-  CHECK_SPATIAL_2D_OR_3D(grid)
+  CHECK_SPATIAL_1D_2D_OR_3D(grid)
   CHECK_GRID_COMPONENT(grid, grid.dim())
   CHECK_SPATIAL_NOT_EMPTY(grid)
   CHECK_VEC_NOT_EMPTY(bound_mode);
   CHECK_VEC_NOT_EMPTY(interpolation_mode);
 
   if (source_size.empty()) {
-    auto size = c10::IntArrayRef({grid.size(1), grid.size(2), grid.dim() == 5 ? grid.size(3) : 1});
+    auto size = c10::IntArrayRef(
+        {grid.dim() >= 3 ? grid.size(2) : 1, grid.dim() >= 4 ? grid.size(3) : 1, grid.dim() >= 5 ? grid.size(4) : 1});
     if (grid.is_cuda())
 #ifdef WITH_CUDA
       return cuda::pushpull(
@@ -422,8 +426,8 @@ at::Tensor grid_grad(
   CHECK_STRIDED(grid_opt)
   CHECK_SAME_DEVICE(input_opt, grid_opt)
   CHECK_SAME_DTYPE(input_opt, grid_opt)
-  CHECK_SPATIAL_2D_OR_3D(input)
-  CHECK_SPATIAL_2D_OR_3D(grid)
+  CHECK_SPATIAL_1D_2D_OR_3D(input)
+  CHECK_SPATIAL_1D_2D_OR_3D(grid)
   CHECK_GRID_COMPONENT(grid, grid.dim())
   CHECK_SPATIAL_NOT_EMPTY(input)
   CHECK_SPATIAL_NOT_EMPTY(grid)
diff --git a/monai/csrc/resample/pushpull_cpu.cpp b/monai/csrc/resample/pushpull_cpu.cpp
index 40743a6cf1..dd10dd76ee 100644
--- a/monai/csrc/resample/pushpull_cpu.cpp
+++ b/monai/csrc/resample/pushpull_cpu.cpp
@@ -18,13 +18,14 @@ limitations under the License.
 // It handles boundary conditions and interpolation orders defined in
 // `utils/resample_utils.h` and `utils/resample_utils.h`.
 // These parameters can be specified per dimension.
-// Isotorpic 0-th and 1-st order interpolation have their own (faster)
+// Isotropic 0-th and 1-st order interpolation have their own (faster)
 // implementations. Sliding boundary conditions are also implemented
 // separately.
 
 // TODO:
 // . [DONE] generic 3d
 // . [DONE] generic 2d
+// . [DONE] generic 1d
 // . sliding nearest 3d
 // . sliding nearest 2d
 // . sliding linear 3d
@@ -37,6 +38,7 @@ limitations under the License.
 // . input bound/inter are always vectors -> clean unused constructors
 
 #include <ATen/ATen.h>
+#include <limits>
 #include <tuple>
 #include "bounds_common.h"
 #include "interpolation_common.h"
@@ -44,7 +46,7 @@ limitations under the License.
 //#include <cstdio>
 
 // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-// CPU/GPU -specific parameters
+// CPU-specific parameters
 #include <ATen/Parallel.h>
 namespace {
 // This parameter specifies the minimum number of voxels that should be
@@ -74,18 +76,27 @@ MONAI_NAMESPACE_DEVICE { // cpu
   namespace { // anonymous namespace > everything inside has internal linkage
 
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-  //                        GENERIC PUSHPULL CLASS
+  //                        INDEXING UTILS
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-  // This class implements the bulk of the code.
-  // /!\ No type and shape checking is performed here.
 
-  template <typename scalar_t, typename offset_t>
-  class PushPullImpl {
+  // This class reads and sets all the parameters that will later be used
+  // by the algorithm in PushPullImpl. All of this is done outside of the
+  // implementation class so that we do not depend on generic types. The
+  // point is to pre-allocate all necessary tensors so that we can check
+  // if they're all compatible with 32 bit math. If it's the case, we can
+  // dispatch to a 32b cuda implementation, which might increase
+  // performance. Else, we use 64 bit math to compute offsets.
+  // (On CPU, we always use 64 bit offsets because it doesn't make a huge
+  // difference. It would be different if we had a vectorized
+  // implementation as in PyTorch).
+  class PushPullAllocator {
    public:
+    static constexpr int64_t max_int32 = std::numeric_limits<int32_t>::max();
+
     // ~~~ CONSTRUCTORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     MONAI_HOST
-    PushPullImpl(
+    PushPullAllocator(
         int dim,
         BoundVectorRef bound,
         InterpolationVectorRef interpolation,
@@ -125,101 +136,418 @@ MONAI_NAMESPACE_DEVICE { // cpu
       iso = interpolation0 == interpolation1 && interpolation0 == interpolation2;
     }
 
-    MONAI_HOST
-    PushPullImpl(
-        int dim,
-        BoundType bound,
-        InterpolationVectorRef interpolation,
-        bool extrapolate,
-        bool do_pull,
-        bool do_push,
-        bool do_count,
-        bool do_grad,
-        bool do_sgrad)
-        : dim(dim),
-          bound0(bound),
-          bound1(bound),
-          bound2(bound),
-          interpolation0(interpolation.size() > 0 ? interpolation[0] : InterpolationType::Linear),
-          interpolation1(
-              interpolation.size() > 1       ? interpolation[1]
-                  : interpolation.size() > 0 ? interpolation[0]
-                                             : InterpolationType::Linear),
-          interpolation2(
-              interpolation.size() > 2       ? interpolation[2]
-                  : interpolation.size() > 1 ? interpolation[1]
-                  : interpolation.size() > 0 ? interpolation[0]
-                                             : InterpolationType::Linear),
-          extrapolate(extrapolate),
-          do_pull(do_pull),
-          do_push(do_push),
-          do_count(do_count),
-          do_grad(do_grad),
-          do_sgrad(do_sgrad) {
-      iso = interpolation0 == interpolation1 && interpolation0 == interpolation2;
+    // ~~~ FUNCTORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    // Usually used for pull:
+    // - do_pull  -> return source[grid]
+    // - do_push  -> fails
+    // - do_grad  -> return J(source)[grid]
+    // - do_sgrad -> return H(source)[grid]
+    MONAI_HOST void ioset(const Tensor& source, const Tensor& grid) {
+      init_all();
+      init_source(source);
+      init_grid(grid);
+      init_output();
     }
 
-    MONAI_HOST
-    PushPullImpl(
-        int dim,
-        BoundVectorRef bound,
-        InterpolationType interpolation,
-        bool extrapolate,
-        bool do_pull,
-        bool do_push,
-        bool do_count,
-        bool do_grad,
-        bool do_sgrad)
-        : dim(dim),
-          bound0(bound.size() > 0 ? bound[0] : BoundType::Replicate),
-          bound1(
-              bound.size() > 1       ? bound[1]
-                  : bound.size() > 0 ? bound[0]
-                                     : BoundType::Replicate),
-          bound2(
-              bound.size() > 2       ? bound[2]
-                  : bound.size() > 1 ? bound[1]
-                  : bound.size() > 0 ? bound[0]
-                                     : BoundType::Replicate),
-          interpolation0(interpolation),
-          interpolation1(interpolation),
-          interpolation2(interpolation),
-          extrapolate(extrapolate),
-          do_pull(do_pull),
-          do_push(do_push),
-          do_count(do_count),
-          do_grad(do_grad),
-          do_sgrad(do_sgrad) {
-      iso = interpolation0 == interpolation1 && interpolation0 == interpolation2;
+    // Usually used for pull_backward:
+    // - do_pull  -> return source[grid]
+    // - do_push  -> return push(target, grid, source.shape)
+    // - do_grad  -> return J(source)[grid]
+    // - do_sgrad -> return H(source)[grid]
+    MONAI_HOST void ioset(const Tensor& source, const Tensor& grid, const Tensor& target) {
+      init_all();
+      init_source(source);
+      init_grid(grid);
+      init_target(target);
+      init_output();
     }
 
-    MONAI_HOST
-    PushPullImpl(
-        int dim,
-        BoundType bound,
-        InterpolationType interpolation,
-        bool extrapolate,
-        bool do_pull,
-        bool do_push,
-        bool do_count,
-        bool do_grad,
-        bool do_sgrad)
-        : dim(dim),
-          bound0(bound),
-          bound1(bound),
-          bound2(bound),
-          interpolation0(interpolation),
-          interpolation1(interpolation),
-          interpolation2(interpolation),
-          extrapolate(extrapolate),
-          do_pull(do_pull),
-          do_push(do_push),
-          do_count(do_count),
-          do_grad(do_grad),
-          do_sgrad(do_sgrad) {
-      iso = interpolation0 == interpolation1 && interpolation0 == interpolation2;
+    // Usually used for push:
+    // - do_pull  -> fails
+    // - do_push  -> return push(target, grid, source_size)
+    // - do_grad  -> fails
+    // - do_sgrad -> fails
+    MONAI_HOST void ioset(IntArrayRef source_size, const Tensor& grid, const Tensor& target) {
+      init_all();
+      init_source(source_size);
+      init_grid(grid);
+      init_target(target);
+      init_output();
     }
 
+    // Usually used for count:
+    // - do_pull  -> fails
+    // - do_push  -> return push(ones, grid, source_size)
+    // - do_grad  -> fails
+    // - do_sgrad -> fails
+    MONAI_HOST void ioset(IntArrayRef source_size, const Tensor& grid) {
+      init_all();
+      init_source(source_size);
+      init_grid(grid);
+      init_output();
+    }
+
+    // We just check that all tensors that we own are compatible with 32b math
+    bool canUse32BitIndexMath(int64_t max_elem = max_int32) const {
+      return src_32b_ok && trgt_32b_ok && grid_32b_ok && grad_32b_ok && out_32b_ok;
+    }
+
+   private:
+    // Copied from aten/src/ATen/native/IndexingUtils.cpp in PyTorch 1.6.
+    // It is used to decide to which pointer type we should dispatch to.
+    // Basically, we need to make sure that the "furthest" element we need
+    // to reach is less than max_elem away.
+    static bool tensorCanUse32BitIndexMath(const Tensor& t, int64_t max_elem = max_int32) {
+      int64_t elements = t.numel();
+      if (elements >= max_elem) {
+        return false;
+      }
+      if (elements == 0) {
+        return max_elem > 0;
+      }
+
+      int64_t offset = 0;
+      int64_t linearId = elements - 1;
+
+      // NOTE: Assumes all strides are positive, which is true for now
+      for (int i = t.dim() - 1; i >= 0; --i) {
+        int64_t curDimIndex = linearId % t.size(i);
+        int64_t curDimOffset = curDimIndex * t.stride(i);
+        offset += curDimOffset;
+        linearId /= t.size(i);
+      }
+
+      if (offset >= max_elem) {
+        return false;
+      }
+
+      return true;
+    }
+
+    // ~~~ COMPONENTS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    MONAI_HOST void init_all();
+    MONAI_HOST void init_source(const Tensor& source);
+    MONAI_HOST void init_source(IntArrayRef source_size);
+    MONAI_HOST void init_grid(const Tensor& grid);
+    MONAI_HOST void init_target(const Tensor& target);
+    MONAI_HOST void init_output();
+
+    // ~~~ OPTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    int dim; // dimensionality (2 or 3)
+    BoundType bound0; // boundary condition  // x|W
+    BoundType bound1; // boundary condition  // y|H
+    BoundType bound2; // boundary condition  // z|D
+    InterpolationType interpolation0; // interpolation order // x|W
+    InterpolationType interpolation1; // interpolation order // y|H
+    InterpolationType interpolation2; // interpolation order // z|D
+    bool iso; // isotropic interpolation?
+    bool extrapolate; // compute out-of-bound values
+    bool do_pull; // sample a volume
+    bool do_push; // splat a volume
+    bool do_count; // splatting weights (= jacobian determinant)
+    bool do_grad; // backprop: gradient of grid // pull
+    bool do_sgrad; // sample spatial gradients
+
+    // ~~~ NAVIGATORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    std::deque<Tensor> output;
+    TensorOptions src_opt;
+    TensorOptions grid_opt;
+    TensorOptions trgt_opt;
+    int64_t N;
+    int64_t C;
+    int64_t src_X;
+    int64_t src_Y;
+    int64_t src_Z;
+    int64_t trgt_X;
+    int64_t trgt_Y;
+    int64_t trgt_Z;
+    int64_t trgt_K;
+    int64_t src_sN;
+    int64_t src_sC;
+    int64_t src_sX;
+    int64_t src_sY;
+    int64_t src_sZ;
+    bool src_32b_ok;
+    void* src_ptr;
+    int64_t trgt_sN;
+    int64_t trgt_sC;
+    int64_t trgt_sX;
+    int64_t trgt_sY;
+    int64_t trgt_sZ;
+    int64_t trgt_sK;
+    bool trgt_32b_ok;
+    void* trgt_ptr;
+    int64_t grid_sN;
+    int64_t grid_sC;
+    int64_t grid_sX;
+    int64_t grid_sY;
+    int64_t grid_sZ;
+    bool grid_32b_ok;
+    void* grid_ptr;
+    int64_t out_sN;
+    int64_t out_sC;
+    int64_t out_sX;
+    int64_t out_sY;
+    int64_t out_sZ;
+    int64_t out_sK; // gradient dimension
+    bool out_32b_ok;
+    void* out_ptr;
+    int64_t grad_sN;
+    int64_t grad_sC;
+    int64_t grad_sX;
+    int64_t grad_sY;
+    int64_t grad_sZ;
+    bool grad_32b_ok;
+    void* grad_ptr;
+
+    // Allow PushPullImpl's constructor to access PushPullAllocator's
+    // private members.
+    template <typename scalar_t, typename offset_t>
+    friend class PushPullImpl;
+  };
+
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                          INITIALISATION
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  MONAI_HOST
+  void PushPullAllocator::init_all() {
+    src_opt = grid_opt = trgt_opt = TensorOptions();
+    N = C = 1L;
+    src_X = src_Y = src_Z = 1L;
+    trgt_X = trgt_Y = trgt_Z = 1L;
+    trgt_K = 0L;
+    src_sN = src_sC = src_sX = src_sY = src_sZ = 0L;
+    grid_sN = grid_sC = grid_sX = grid_sY = grid_sZ = 0L;
+    grad_sN = grad_sC = grad_sX = grad_sY = grad_sZ = 0L;
+    trgt_sN = trgt_sC = trgt_sX = trgt_sY = trgt_sZ = trgt_sK = 0L;
+    out_sN = out_sC = out_sX = out_sY = out_sZ = out_sK = 0L;
+    src_ptr = trgt_ptr = grid_ptr = out_ptr = grad_ptr = static_cast<float*>(0);
+    src_32b_ok = trgt_32b_ok = grid_32b_ok = out_32b_ok = grad_32b_ok = true;
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_source(const Tensor& source) {
+    N = source.size(0);
+    C = source.size(1);
+    src_X = source.size(2);
+    src_Y = dim < 2 ? 1L : source.size(3);
+    src_Z = dim < 3 ? 1L : source.size(4);
+    src_sN = source.stride(0);
+    src_sC = source.stride(1);
+    src_sX = source.stride(2);
+    src_sY = dim < 2 ? 0L : source.stride(3);
+    src_sZ = dim < 3 ? 0L : source.stride(4);
+    src_ptr = source.data_ptr();
+    src_opt = source.options();
+    src_32b_ok = tensorCanUse32BitIndexMath(source);
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_source(IntArrayRef source_size) {
+    src_X = source_size[0];
+    src_Y = dim < 2 ? 1L : source_size[1];
+    src_Z = dim < 3 ? 1L : source_size[2];
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_grid(const Tensor& grid) {
+    N = grid.size(0);
+    trgt_X = grid.size(1);
+    trgt_Y = dim < 2 ? 1L : grid.size(2);
+    trgt_Z = dim < 3 ? 1L : grid.size(3);
+    grid_sN = grid.stride(0);
+    grid_sX = grid.stride(1);
+    grid_sY = dim < 2 ? 0L : grid.stride(2);
+    grid_sZ = dim < 3 ? 0L : grid.stride(3);
+    grid_sC = grid.stride(dim == 1 ? 2 : dim == 2 ? 3 : 4);
+    grid_ptr = grid.data_ptr();
+    grid_opt = grid.options();
+    grid_32b_ok = tensorCanUse32BitIndexMath(grid);
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_target(const Tensor& target) {
+    N = target.size(0);
+    C = target.size(1);
+    trgt_X = target.size(2);
+    trgt_Y = dim < 2 ? 1L : target.size(3);
+    trgt_Z = dim < 3 ? 1L : target.size(4);
+    trgt_K = target.dim() == dim + 3 ? target.size(dim == 1 ? 3 : dim == 2 ? 4 : 5) : 0L;
+    trgt_sN = target.stride(0);
+    trgt_sC = target.stride(1);
+    trgt_sX = target.stride(2);
+    trgt_sY = dim < 2 ? 0L : target.stride(3);
+    trgt_sZ = dim < 3 ? 0L : target.stride(4);
+    trgt_sK = target.dim() == dim + 3 ? target.stride(dim == 1 ? 3 : dim == 2 ? 4 : 5) : 0L;
+    trgt_ptr = target.data_ptr();
+    trgt_opt = target.options();
+    trgt_32b_ok = tensorCanUse32BitIndexMath(target);
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_output() {
+    output.clear();
+    if (do_pull) {
+      if (dim == 1)
+        output.push_back(at::empty({N, C, trgt_X}, src_opt));
+      else if (dim == 2)
+        output.push_back(at::empty({N, C, trgt_X, trgt_Y}, src_opt));
+      else
+        output.push_back(at::empty({N, C, trgt_X, trgt_Y, trgt_Z}, src_opt));
+      auto pull = output.back();
+      out_sN = pull.stride(0);
+      out_sC = pull.stride(1);
+      out_sX = pull.stride(2);
+      out_sY = dim < 2 ? 0L : pull.stride(3);
+      out_sZ = dim < 3 ? 0L : pull.stride(4);
+      out_sK = 0L;
+      out_ptr = pull.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(pull);
+    } else if (do_sgrad) {
+      if (dim == 1)
+        output.push_back(at::empty({N, C, trgt_X, 1}, src_opt));
+      else if (dim == 2)
+        output.push_back(at::empty({N, C, trgt_X, trgt_Y, 2}, src_opt));
+      else
+        output.push_back(at::empty({N, C, trgt_X, trgt_Y, trgt_Z, 3}, src_opt));
+      auto sgrad = output.back();
+      out_sN = sgrad.stride(0);
+      out_sC = sgrad.stride(1);
+      out_sX = sgrad.stride(2);
+      out_sY = dim < 2 ? 0L : sgrad.stride(3);
+      out_sZ = dim < 3 ? 0L : sgrad.stride(4);
+      out_sK = sgrad.stride(dim == 1 ? 3 : dim == 2 ? 4 : 5);
+      out_ptr = sgrad.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(sgrad);
+
+      if (iso && interpolation0 == InterpolationType::Nearest)
+        sgrad.zero_();
+      if (iso && interpolation0 == InterpolationType::Linear && dim == 1)
+        sgrad.zero_();
+    } else if (do_push) {
+      if (dim == 1)
+        output.push_back(at::zeros({N, C, src_X}, trgt_opt));
+      else if (dim == 2)
+        output.push_back(at::zeros({N, C, src_X, src_Y}, trgt_opt));
+      else
+        output.push_back(at::zeros({N, C, src_X, src_Y, src_Z}, trgt_opt));
+      auto push = output.back();
+      out_sN = push.stride(0);
+      out_sC = push.stride(1);
+      out_sX = push.stride(2);
+      out_sY = dim < 2 ? 0L : push.stride(3);
+      out_sZ = dim < 3 ? 0L : push.stride(4);
+      out_sK = 0L;
+      out_ptr = push.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(push);
+    } else if (do_count) {
+      if (dim == 1)
+        output.push_back(at::zeros({N, 1, src_X}, grid_opt));
+      else if (dim == 2)
+        output.push_back(at::zeros({N, 1, src_X, src_Y}, grid_opt));
+      else
+        output.push_back(at::zeros({N, 1, src_X, src_Y, src_Z}, grid_opt));
+      auto count = output.back();
+      out_sN = count.stride(0);
+      out_sC = count.stride(1);
+      out_sX = count.stride(2);
+      out_sY = dim < 2 ? 0L : count.stride(3);
+      out_sZ = dim < 3 ? 0L : count.stride(4);
+      out_sK = 0L;
+      out_ptr = count.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(count);
+    }
+    if (do_grad) {
+      if (dim == 1)
+        output.push_back(at::zeros({N, trgt_X, 1}, grid_opt));
+      else if (dim == 2)
+        output.push_back(at::zeros({N, trgt_X, trgt_Y, 2}, grid_opt));
+      else
+        output.push_back(at::zeros({N, trgt_X, trgt_Y, trgt_Z, 3}, grid_opt));
+      auto grad = output.back();
+      grad_sN = grad.stride(0);
+      grad_sX = grad.stride(1);
+      grad_sY = dim < 2 ? 0L : grad.stride(2);
+      grad_sZ = dim < 3 ? 0L : grad.stride(3);
+      grad_sC = grad.stride(dim == 1 ? 2 : dim == 2 ? 3 : 4);
+      grad_ptr = grad.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(grad);
+
+      if (iso && interpolation0 == InterpolationType::Nearest)
+        grad.zero_();
+    }
+  }
+
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                        GENERIC PUSHPULL CLASS
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  // This class implements the bulk of the code.
+  // /!\ No type and shape checking is performed here.
+
+  template <typename scalar_t, typename offset_t>
+  class PushPullImpl {
+   public:
+    // ~~~ CONSTRUCTOR ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    PushPullImpl(const PushPullAllocator& info)
+        : output(info.output),
+          dim(info.dim),
+          bound0(info.bound0),
+          bound1(info.bound1),
+          bound2(info.bound2),
+          interpolation0(info.interpolation0),
+          interpolation1(info.interpolation1),
+          interpolation2(info.interpolation1),
+          iso(info.iso),
+          extrapolate(info.extrapolate),
+          do_pull(info.do_pull),
+          do_push(info.do_push),
+          do_count(info.do_count),
+          do_grad(info.do_grad),
+          do_sgrad(info.do_sgrad),
+          N(static_cast<offset_t>(info.N)),
+          C(static_cast<offset_t>(info.C)),
+          src_X(static_cast<offset_t>(info.src_X)),
+          src_Y(static_cast<offset_t>(info.src_Y)),
+          src_Z(static_cast<offset_t>(info.src_Z)),
+          trgt_X(static_cast<offset_t>(info.trgt_X)),
+          trgt_Y(static_cast<offset_t>(info.trgt_Y)),
+          trgt_Z(static_cast<offset_t>(info.trgt_Z)),
+          trgt_K(static_cast<offset_t>(info.trgt_K)),
+          src_sN(static_cast<offset_t>(info.src_sN)),
+          src_sC(static_cast<offset_t>(info.src_sC)),
+          src_sX(static_cast<offset_t>(info.src_sX)),
+          src_sY(static_cast<offset_t>(info.src_sY)),
+          src_sZ(static_cast<offset_t>(info.src_sZ)),
+          src_ptr(static_cast<scalar_t*>(info.src_ptr)),
+          trgt_sN(static_cast<offset_t>(info.trgt_sN)),
+          trgt_sC(static_cast<offset_t>(info.trgt_sC)),
+          trgt_sX(static_cast<offset_t>(info.trgt_sX)),
+          trgt_sY(static_cast<offset_t>(info.trgt_sY)),
+          trgt_sZ(static_cast<offset_t>(info.trgt_sZ)),
+          trgt_sK(static_cast<offset_t>(info.trgt_sK)),
+          trgt_ptr(static_cast<scalar_t*>(info.trgt_ptr)),
+          grid_sN(static_cast<offset_t>(info.grid_sN)),
+          grid_sC(static_cast<offset_t>(info.grid_sC)),
+          grid_sX(static_cast<offset_t>(info.grid_sX)),
+          grid_sY(static_cast<offset_t>(info.grid_sY)),
+          grid_sZ(static_cast<offset_t>(info.grid_sZ)),
+          grid_ptr(static_cast<scalar_t*>(info.grid_ptr)),
+          out_sN(static_cast<offset_t>(info.out_sN)),
+          out_sC(static_cast<offset_t>(info.out_sC)),
+          out_sX(static_cast<offset_t>(info.out_sX)),
+          out_sY(static_cast<offset_t>(info.out_sY)),
+          out_sZ(static_cast<offset_t>(info.out_sZ)),
+          out_sK(static_cast<offset_t>(info.out_sK)),
+          out_ptr(static_cast<scalar_t*>(info.out_ptr)),
+          grad_sN(static_cast<offset_t>(info.grad_sN)),
+          grad_sC(static_cast<offset_t>(info.grad_sC)),
+          grad_sX(static_cast<offset_t>(info.grad_sX)),
+          grad_sY(static_cast<offset_t>(info.grad_sY)),
+          grad_sZ(static_cast<offset_t>(info.grad_sZ)),
+          grad_ptr(static_cast<scalar_t*>(info.grad_ptr)) {}
+
     // ~~~ PUBLIC VALUE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     std::deque<Tensor> output;
@@ -247,39 +575,8 @@ MONAI_NAMESPACE_DEVICE { // cpu
     // }
 
     // ~~~ FUNCTORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    MONAI_HOST void ioset // Pull
-        (const Tensor& source, const Tensor& grid) {
-      init_all();
-      init_source(source);
-      init_grid(grid);
-      init_output();
-    }
-
-    MONAI_HOST void ioset(const Tensor& source, const Tensor& grid, const Tensor& target) {
-      init_all();
-      init_source(source);
-      init_grid(grid);
-      init_target(target);
-      init_output();
-    }
-
-    MONAI_HOST void ioset // Push
-        (IntArrayRef source_size, const Tensor& grid, const Tensor& target) {
-      init_all();
-      init_source(source_size);
-      init_grid(grid);
-      init_target(target);
-      init_output();
-    }
-
-    MONAI_HOST void ioset // Count
-        (IntArrayRef source_size, const Tensor& grid) {
-      init_all();
-      init_source(source_size);
-      init_grid(grid);
-      init_output();
-    }
 
+    // Loop over all voxels
     void loop() const;
 
     MONAI_HOST MONAI_DEVICE int64_t voxcount() const {
@@ -288,14 +585,18 @@ MONAI_NAMESPACE_DEVICE { // cpu
 
    private:
     // ~~~ COMPONENTS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    MONAI_HOST void init_all();
-    MONAI_HOST void init_source(const Tensor& source);
-    MONAI_HOST void init_source(IntArrayRef source_size);
-    MONAI_HOST void init_grid(const Tensor& grid);
-    MONAI_HOST void init_target(const Tensor& target);
-    MONAI_HOST void init_output();
+    MONAI_DEVICE void check1d(offset_t w, offset_t n) const;
     MONAI_DEVICE void check2d(offset_t w, offset_t h, offset_t n) const;
     MONAI_DEVICE void check3d(offset_t w, offset_t h, offset_t d, offset_t n) const;
+    MONAI_DEVICE void interpolate1d(scalar_t x, offset_t w, offset_t n) const;
+    MONAI_DEVICE void interpolate1d_nearest(scalar_t x, offset_t w, offset_t n) const;
+    MONAI_DEVICE void interpolate1d_linear(scalar_t x, offset_t w, offset_t n) const;
+    MONAI_DEVICE void interpolate1d_sliding(scalar_t x, offset_t w, offset_t n) const { /*TODO*/
+    }
+    MONAI_DEVICE void interpolate1d_sliding_nearest(scalar_t x, offset_t w, offset_t n) const { /*TODO*/
+    }
+    MONAI_DEVICE void interpolate1d_sliding_linear(scalar_t x, offset_t w, offset_t n) const { /*TODO*/
+    }
     MONAI_DEVICE void interpolate2d(scalar_t x, scalar_t y, offset_t w, offset_t h, offset_t n) const;
     MONAI_DEVICE void interpolate2d_nearest(scalar_t x, scalar_t y, offset_t w, offset_t h, offset_t n) const;
     MONAI_DEVICE void interpolate2d_bilinear(scalar_t x, scalar_t y, offset_t w, offset_t h, offset_t n) const;
@@ -370,9 +671,6 @@ MONAI_NAMESPACE_DEVICE { // cpu
     bool do_sgrad; // sample spatial gradients
 
     // ~~~ NAVIGATORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    TensorOptions src_opt;
-    TensorOptions grid_opt;
-    TensorOptions trgt_opt;
     offset_t N;
     offset_t C;
     offset_t src_X;
@@ -402,174 +700,24 @@ MONAI_NAMESPACE_DEVICE { // cpu
     offset_t grid_sZ;
     scalar_t* grid_ptr;
     offset_t out_sN;
-    offset_t out_sC;
-    offset_t out_sX;
-    offset_t out_sY;
-    offset_t out_sZ;
-    offset_t out_sK; // gradient dimension
-    scalar_t* out_ptr;
-    offset_t grad_sN;
-    offset_t grad_sC;
-    offset_t grad_sX;
-    offset_t grad_sY;
-    offset_t grad_sZ;
-    scalar_t* grad_ptr;
-  };
-
-  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-  //                          INITIALISATION
-  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-  template <typename scalar_t, typename offset_t>
-  void PushPullImpl<scalar_t, offset_t>::init_all() {
-    src_opt = grid_opt = trgt_opt = TensorOptions();
-    N = C = static_cast<offset_t>(1);
-    src_X = src_Y = src_Z = static_cast<offset_t>(1);
-    trgt_X = trgt_Y = trgt_Z = trgt_K = static_cast<offset_t>(1);
-    src_sN = src_sC = src_sX = src_sY = src_sZ = static_cast<offset_t>(0);
-    grid_sN = grid_sC = grid_sX = grid_sY = grid_sZ = static_cast<offset_t>(0);
-    grad_sN = grad_sC = grad_sX = grad_sY = grad_sZ = static_cast<offset_t>(0);
-    trgt_sN = trgt_sC = trgt_sX = trgt_sY = trgt_sZ = trgt_sK = static_cast<offset_t>(0);
-    out_sN = out_sC = out_sX = out_sY = out_sZ = out_sK = static_cast<offset_t>(0);
-    src_ptr = trgt_ptr = grid_ptr = out_ptr = grad_ptr = static_cast<scalar_t*>(0);
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_source(const Tensor& source) {
-    N = source.size(0);
-    C = source.size(1);
-    src_X = source.size(2);
-    src_Y = source.size(3);
-    src_Z = dim == 2 ? static_cast<offset_t>(1) : source.size(4);
-    src_sN = source.stride(0);
-    src_sC = source.stride(1);
-    src_sX = source.stride(2);
-    src_sY = source.stride(3);
-    src_sZ = dim == 2 ? static_cast<offset_t>(0) : source.stride(4);
-    src_ptr = source.data_ptr<scalar_t>();
-    src_opt = source.options();
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_source(IntArrayRef source_size) {
-    src_X = source_size[0];
-    src_Y = source_size[1];
-    src_Z = dim == 2 ? static_cast<offset_t>(1) : source_size[2];
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_grid(const Tensor& grid) {
-    N = grid.size(0);
-    trgt_X = grid.size(1);
-    trgt_Y = grid.size(2);
-    trgt_Z = dim == 2 ? static_cast<offset_t>(1) : grid.size(3);
-    grid_sN = grid.stride(0);
-    grid_sX = grid.stride(1);
-    grid_sY = grid.stride(2);
-    grid_sZ = dim == 2 ? static_cast<offset_t>(0) : grid.stride(3);
-    grid_sC = grid.stride(dim == 2 ? 3 : 4);
-    grid_ptr = grid.data_ptr<scalar_t>();
-    grid_opt = grid.options();
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_target(const Tensor& target) {
-    N = target.size(0);
-    C = target.size(1);
-    trgt_X = target.size(2);
-    trgt_Y = target.size(3);
-    trgt_Z = dim == 2 ? static_cast<offset_t>(1) : target.size(4);
-    trgt_K = target.dim() == dim + 3 ? target.size(dim == 2 ? 4 : 5) : static_cast<offset_t>(1);
-    trgt_sN = target.stride(0);
-    trgt_sC = target.stride(1);
-    trgt_sX = target.stride(2);
-    trgt_sY = target.stride(3);
-    trgt_sZ = dim == 2 ? static_cast<offset_t>(0) : target.stride(4);
-    trgt_sK = target.dim() == dim + 3 ? target.stride(dim == 2 ? 4 : 5) : static_cast<offset_t>(0);
-    trgt_ptr = target.data_ptr<scalar_t>();
-    trgt_opt = target.options();
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_output() {
-    output.clear();
-    if (do_pull) {
-      if (dim == 2)
-        output.push_back(at::empty({N, C, trgt_X, trgt_Y}, src_opt));
-      else
-        output.push_back(at::empty({N, C, trgt_X, trgt_Y, trgt_Z}, src_opt));
-      auto pull = output.back();
-      out_sN = pull.stride(0);
-      out_sC = pull.stride(1);
-      out_sX = pull.stride(2);
-      out_sY = pull.stride(3);
-      out_sZ = dim == 2 ? static_cast<offset_t>(0) : pull.stride(4);
-      out_sK = static_cast<offset_t>(0);
-      out_ptr = pull.template data_ptr<scalar_t>();
-    } else if (do_sgrad) {
-      if (dim == 2)
-        output.push_back(at::empty({N, C, trgt_X, trgt_Y, 2}, src_opt));
-      else
-        output.push_back(at::empty({N, C, trgt_X, trgt_Y, trgt_Z, 3}, src_opt));
-      auto sgrad = output.back();
-      out_sN = sgrad.stride(0);
-      out_sC = sgrad.stride(1);
-      out_sX = sgrad.stride(2);
-      out_sY = sgrad.stride(3);
-      out_sZ = dim == 2 ? static_cast<offset_t>(0) : sgrad.stride(4);
-      out_sK = sgrad.stride(dim == 2 ? 4 : 5);
-      out_ptr = sgrad.template data_ptr<scalar_t>();
-
-      if (iso && interpolation0 == InterpolationType::Nearest)
-        sgrad.zero_();
-    } else if (do_push) {
-      if (dim == 2)
-        output.push_back(at::zeros({N, C, src_X, src_Y}, trgt_opt));
-      else
-        output.push_back(at::zeros({N, C, src_X, src_Y, src_Z}, trgt_opt));
-      auto push = output.back();
-      out_sN = push.stride(0);
-      out_sC = push.stride(1);
-      out_sX = push.stride(2);
-      out_sY = push.stride(3);
-      out_sZ = dim == 2 ? static_cast<offset_t>(0) : push.stride(4);
-      out_sK = static_cast<offset_t>(0);
-      out_ptr = push.template data_ptr<scalar_t>();
-    } else if (do_count) {
-      if (dim == 2)
-        output.push_back(at::zeros({N, 1, src_X, src_Y}, grid_opt));
-      else
-        output.push_back(at::zeros({N, 1, src_X, src_Y, src_Z}, grid_opt));
-      auto count = output.back();
-      out_sN = count.stride(0);
-      out_sC = count.stride(1);
-      out_sX = count.stride(2);
-      out_sY = count.stride(3);
-      out_sZ = dim == 2 ? static_cast<offset_t>(0) : count.stride(4);
-      out_sK = static_cast<offset_t>(0);
-      out_ptr = count.template data_ptr<scalar_t>();
-    }
-    if (do_grad) {
-      if (dim == 2)
-        output.push_back(at::zeros({N, src_X, src_Y, 2}, grid_opt));
-      else
-        output.push_back(at::zeros({N, src_X, src_Y, src_Z, 3}, grid_opt));
-      auto grad = output.back();
-      grad_sN = grad.stride(0);
-      grad_sX = grad.stride(1);
-      grad_sY = grad.stride(2);
-      grad_sZ = dim == 2 ? static_cast<offset_t>(0) : grad.stride(3);
-      grad_sC = grad.stride(dim == 2 ? 3 : 4);
-      grad_ptr = grad.template data_ptr<scalar_t>();
-
-      if (iso && interpolation0 == InterpolationType::Nearest)
-        grad.zero_();
-    }
-  }
+    offset_t out_sC;
+    offset_t out_sX;
+    offset_t out_sY;
+    offset_t out_sZ;
+    offset_t out_sK; // gradient dimension
+    scalar_t* out_ptr;
+    offset_t grad_sN;
+    offset_t grad_sC;
+    offset_t grad_sX;
+    offset_t grad_sY;
+    offset_t grad_sZ;
+    scalar_t* grad_ptr;
+  };
 
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   //                             LOOP
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
   // This bit loops over all target voxels. We therefore need to
   // convert linear indices to multivariate indices. The way I do it
   // might not be optimal.
@@ -586,7 +734,10 @@ MONAI_NAMESPACE_DEVICE { // cpu
       // parallelize across voxels.
       at::parallel_for(0, N, 0, [&](offset_t start, offset_t end) {
         for (offset_t n = start; n < end; ++n) {
-          if (dim == 2) {
+          if (dim == 1) {
+            for (offset_t w = 0; w < trgt_X; ++w)
+              check1d(w, n);
+          } else if (dim == 2) {
             for (offset_t h = 0; h < trgt_Y; ++h)
               for (offset_t w = 0; w < trgt_X; ++w)
                 check2d(w, h, n);
@@ -600,8 +751,8 @@ MONAI_NAMESPACE_DEVICE { // cpu
       });
       return;
     }
-#endif
 
+#endif
     // Parallelize across voxels
     offset_t trgt_NXYZ = trgt_Z * trgt_Y * trgt_X * N;
     offset_t trgt_XYZ = trgt_Z * trgt_Y * trgt_X;
@@ -615,7 +766,9 @@ MONAI_NAMESPACE_DEVICE { // cpu
         h = (i / trgt_Z) % trgt_Y;
         d = i % trgt_Z;
 
-        if (dim == 2)
+        if (dim == 1)
+          check1d(w, n);
+        else if (dim == 2)
           check2d(w, h, n);
         else
           check3d(w, h, d, n);
@@ -631,6 +784,59 @@ MONAI_NAMESPACE_DEVICE { // cpu
   // 1) read the [x,y,z] source coordinate for the current target voxel
   // 3) check if the source coordinate is in bounds
 
+  template <typename scalar_t, typename offset_t>
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::check3d(offset_t w, offset_t h, offset_t d, offset_t n) const {
+    // get the corresponding input x, y, z co-ordinates from grid
+    scalar_t* grid_ptr_NXYZ = grid_ptr + n * grid_sN + w * grid_sX + h * grid_sY + d * grid_sZ;
+    scalar_t x = *grid_ptr_NXYZ;
+    scalar_t y = grid_ptr_NXYZ[grid_sC];
+    scalar_t z = grid_ptr_NXYZ[grid_sC * 2];
+
+    // Check if out-of-bound
+    if (!(extrapolate ||
+          (inbounds(x, src_X, static_cast<scalar_t>(TINY)) && inbounds(y, src_Y, static_cast<scalar_t>(TINY)) &&
+           inbounds(z, src_Z, static_cast<scalar_t>(TINY))))) {
+      if (do_pull || do_sgrad) {
+        scalar_t* out_ptr_NCXYZ = out_ptr + n * out_sN + w * out_sX + h * out_sY + d * out_sZ;
+        for (offset_t c = 0; c < C; ++c, out_ptr_NCXYZ += out_sC) {
+          *out_ptr_NCXYZ = static_cast<scalar_t>(0);
+          if (do_sgrad) {
+            out_ptr_NCXYZ[out_sK] = static_cast<scalar_t>(0);
+            out_ptr_NCXYZ[out_sK * 2] = static_cast<scalar_t>(0);
+          }
+        }
+      }
+      if (do_grad) {
+        scalar_t* grad_ptr_NXYZ = grad_ptr + n * grad_sN + w * grad_sX + h * grad_sY + d * grad_sZ;
+        (*grad_ptr_NXYZ) = static_cast<scalar_t>(0);
+        grad_ptr_NXYZ[grad_sC] = static_cast<scalar_t>(0);
+        grad_ptr_NXYZ[grad_sC * 2] = static_cast<scalar_t>(0);
+      }
+      return;
+    }
+
+    // Next step
+    if (bound0 == BoundType::Sliding) {
+      if (iso)
+        switch (static_cast<int>(interpolation0)) {
+          case 0:
+            return interpolate3d_sliding_nearest(x, y, z, w, h, d, n);
+          case 1:
+            return interpolate3d_sliding_trilinear(x, y, z, w, h, d, n);
+        }
+      return interpolate3d_sliding(x, y, z, w, h, d, n);
+    } else {
+      if (iso)
+        switch (static_cast<int>(interpolation0)) {
+          case 0:
+            return interpolate3d_nearest(x, y, z, w, h, d, n);
+          case 1:
+            return interpolate3d_trilinear(x, y, z, w, h, d, n);
+        }
+      return interpolate3d(x, y, z, w, h, d, n);
+    }
+  }
+
   template <typename scalar_t, typename offset_t>
   MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::check2d(offset_t w, offset_t h, offset_t n) const {
     // get the corresponding input x, y, z co-ordinates from grid
@@ -642,7 +848,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
     if (!(extrapolate ||
           (inbounds(x, src_X, static_cast<scalar_t>(TINY)) && inbounds(y, src_Y, static_cast<scalar_t>(TINY))))) {
       if (do_pull || do_sgrad) {
-        scalar_t* out_ptr_NCXY = out_ptr + n * out_sN + w * out_sZ + h * out_sY;
+        scalar_t* out_ptr_NCXY = out_ptr + n * out_sN + w * out_sX + h * out_sY;
         for (offset_t c = 0; c < C; ++c, out_ptr_NCXY += out_sC) {
           *out_ptr_NCXY = static_cast<scalar_t>(0);
           if (do_sgrad)
@@ -680,32 +886,25 @@ MONAI_NAMESPACE_DEVICE { // cpu
   }
 
   template <typename scalar_t, typename offset_t>
-  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::check3d(offset_t w, offset_t h, offset_t d, offset_t n) const {
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::check1d(offset_t w, offset_t n) const {
     // get the corresponding input x, y, z co-ordinates from grid
-    scalar_t* grid_ptr_NXYZ = grid_ptr + n * grid_sN + w * grid_sX + h * grid_sY + d * grid_sZ;
-    scalar_t x = *grid_ptr_NXYZ;
-    scalar_t y = grid_ptr_NXYZ[grid_sC];
-    scalar_t z = grid_ptr_NXYZ[grid_sC * 2];
+    scalar_t* grid_ptr_NX = grid_ptr + n * grid_sN + w * grid_sX;
+    scalar_t x = *grid_ptr_NX;
 
     // Check if out-of-bound
-    if (!(extrapolate ||
-          (inbounds(x, src_X, static_cast<scalar_t>(TINY)) && inbounds(y, src_Y, static_cast<scalar_t>(TINY)) &&
-           inbounds(z, src_Z, static_cast<scalar_t>(TINY))))) {
+    if (!(extrapolate || inbounds(x, src_X, static_cast<scalar_t>(TINY)))) {
       if (do_pull || do_sgrad) {
-        scalar_t* out_ptr_NCXYZ = out_ptr + n * out_sN + w * out_sX + h * out_sY + d * out_sZ;
-        for (offset_t c = 0; c < C; ++c, out_ptr_NCXYZ += out_sC) {
-          *out_ptr_NCXYZ = static_cast<scalar_t>(0);
-          if (do_sgrad) {
-            out_ptr_NCXYZ[out_sK] = static_cast<scalar_t>(0);
-            out_ptr_NCXYZ[out_sK * 2] = static_cast<scalar_t>(0);
-          }
+        scalar_t* out_ptr_NCX = out_ptr + n * out_sN + w * out_sX;
+        for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC) {
+          *out_ptr_NCX = static_cast<scalar_t>(0);
+          if (do_sgrad)
+            out_ptr_NCX[out_sK] = static_cast<scalar_t>(0);
         }
       }
       if (do_grad) {
-        scalar_t* grad_ptr_NXYZ = grad_ptr + n * grad_sN + w * grad_sX + h * grad_sY + d * grad_sZ;
-        (*grad_ptr_NXYZ) = static_cast<scalar_t>(0);
-        grad_ptr_NXYZ[grad_sC] = static_cast<scalar_t>(0);
-        grad_ptr_NXYZ[grad_sC * 2] = static_cast<scalar_t>(0);
+        scalar_t* grad_ptr_NX = grad_ptr + n * grad_sN + w * grad_sX;
+        (*grad_ptr_NX) = static_cast<scalar_t>(0);
+        grad_ptr_NX[grad_sC] = static_cast<scalar_t>(0);
       }
       return;
     }
@@ -715,20 +914,20 @@ MONAI_NAMESPACE_DEVICE { // cpu
       if (iso)
         switch (static_cast<int>(interpolation0)) {
           case 0:
-            return interpolate3d_sliding_nearest(x, y, z, w, h, d, n);
+            return interpolate1d_sliding_nearest(x, w, n);
           case 1:
-            return interpolate3d_sliding_trilinear(x, y, z, w, h, d, n);
+            return interpolate1d_sliding_linear(x, w, n);
         }
-      return interpolate3d_sliding(x, y, z, w, h, d, n);
+      return interpolate1d_sliding(x, w, n);
     } else {
       if (iso)
         switch (static_cast<int>(interpolation0)) {
           case 0:
-            return interpolate3d_nearest(x, y, z, w, h, d, n);
+            return interpolate1d_nearest(x, w, n);
           case 1:
-            return interpolate3d_trilinear(x, y, z, w, h, d, n);
+            return interpolate1d_linear(x, w, n);
         }
-      return interpolate3d(x, y, z, w, h, d, n);
+      return interpolate1d(x, w, n);
     }
   }
 
@@ -763,7 +962,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
     if (trgt_ptr && (do_push || do_grad))
       for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXYZ += trgt_sC) {
         target[c] = *trgt_ptr_NCXYZ;
-        if (trgt_K > 1) {
+        if (trgt_K > 0) {
           target[c + C] = trgt_ptr_NCXYZ[trgt_sK];
           target[c + C * 2] = trgt_ptr_NCXYZ[trgt_sK * 2];
         }
@@ -881,7 +1080,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
 
           // ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           else if (do_push) {
-            if (trgt_K == 1) {
+            if (trgt_K == 0) {
               // Diff w.r.t. push/pull
               scalar_t* out_ptr_NC = out_ptr_NC0;
               for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC)
@@ -904,7 +1103,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
 
           // ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Grad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           if (do_grad) {
-            if (trgt_K == 1) {
+            if (trgt_K == 0) {
               // Diff w.r.t. pull/push
               scalar_t* src_ptr_NC = src_ptr_NC0;
               scalar_t dot = static_cast<scalar_t>(0);
@@ -973,7 +1172,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
     if (trgt_ptr && (do_push || do_grad))
       for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXY += trgt_sC) {
         target[c] = *trgt_ptr_NCXY;
-        if (trgt_K > 1) {
+        if (trgt_K > 0) {
           target[c + C] = trgt_ptr_NCXY[trgt_sK];
         }
       }
@@ -1066,7 +1265,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
 
         // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         else if (do_push) {
-          if (trgt_K == 1) {
+          if (trgt_K == 0) {
             // Diff w.r.t. push/pull
             scalar_t* out_ptr_NC = out_ptr_NC0;
             for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC)
@@ -1088,7 +1287,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
 
         // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Grad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         if (do_grad) {
-          if (trgt_K == 1) {
+          if (trgt_K == 0) {
             // Diff w.r.t. pull/push
             scalar_t* src_ptr_NC = src_ptr_NC0;
             scalar_t dot = static_cast<scalar_t>(0);
@@ -1125,6 +1324,150 @@ MONAI_NAMESPACE_DEVICE { // cpu
     }
   }
 
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                     GENERIC INTERPOLATION 1D
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  template <typename scalar_t, typename offset_t>
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::interpolate1d(scalar_t x, offset_t w, offset_t n) const {
+    // Get corner pixel values from (x, y)
+    offset_t bx0, bx1;
+    interpolation::bounds(interpolation0, x, bx0, bx1);
+    offset_t dbx = bx1 - bx0;
+
+    // Pre-compute offsets and target value
+    scalar_t* src_ptr_NC0 = src_ptr + n * src_sN;
+    scalar_t* out_ptr_NC0 = out_ptr + n * out_sN;
+    scalar_t* out_ptr_NCX0 = out_ptr + n * out_sN + w * out_sX;
+    scalar_t* trgt_ptr_NCX = trgt_ptr + n * trgt_sN + w * trgt_sX;
+    scalar_t target[2 * MONAI_MAX_NUM_CHANNELS];
+    if (trgt_ptr && (do_push || do_grad))
+      for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC) {
+        target[c] = *trgt_ptr_NCX;
+        if (trgt_K > 0) {
+          target[c + C] = trgt_ptr_NCX[trgt_sK];
+        }
+      }
+
+    // Initialize output
+    scalar_t* out_ptr_NCX = out_ptr_NCX0;
+    if (do_pull || do_sgrad) {
+      for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC) {
+        *out_ptr_NCX = static_cast<scalar_t>(0);
+        if (do_sgrad) {
+          out_ptr_NCX[out_sK] = static_cast<scalar_t>(0);
+        }
+      }
+    }
+
+    // Pre-compute indices/weights/grad
+    scalar_t wx[8]; // B-spline weights
+    scalar_t gx[8]; // B-spline derivatives
+    scalar_t hx[8]; // B-spline 2nd derivatives
+    offset_t ix[8]; // Warped indices
+    uint8_t sx[8]; // Warped indices
+
+    {
+      scalar_t *owx = static_cast<scalar_t*>(wx), *ogx = static_cast<scalar_t*>(gx), *ohx = static_cast<scalar_t*>(hx);
+      offset_t* oix = static_cast<offset_t*>(ix);
+      uint8_t* osx = static_cast<uint8_t*>(sx);
+      for (offset_t bx = bx0; bx <= bx1; ++bx) {
+        scalar_t dx = x - bx;
+        *(owx++) = interpolation::fastweight(interpolation0, dx);
+        if (do_grad || do_sgrad)
+          *(ogx++) = interpolation::fastgrad(interpolation0, dx);
+        if (do_grad && trgt_sK > 1)
+          *(ohx++) = interpolation::fasthess(interpolation0, dx);
+        *(osx++) = bound::sign(bound0, bx, src_X);
+        *(oix++) = bound::index(bound0, bx, src_X);
+      }
+    }
+
+    // Convolve coefficients with basis functions
+    scalar_t ogx;
+    ogx = static_cast<scalar_t>(0);
+    for (offset_t i = 0; i <= dbx; ++i) {
+      offset_t oox = ix[i] * out_sX;
+      offset_t osx = ix[i] * src_sX;
+      uint8_t sxx = sx[i];
+      scalar_t wxx = wx[i];
+      scalar_t gxx = gx[i];
+      scalar_t hxx = hx[i];
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pull ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      if (do_pull) {
+        scalar_t* src_ptr_NC = src_ptr_NC0;
+        scalar_t* out_ptr_NCX = out_ptr_NCX0;
+        for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC)
+          *out_ptr_NCX += bound::get(src_ptr_NC, osx, sxx) * wxx;
+      }
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SGrad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      else if (do_sgrad) {
+        scalar_t* src_ptr_NC = src_ptr_NC0;
+        scalar_t* out_ptr_NCX = out_ptr_NCX0;
+        for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC) {
+          scalar_t src = bound::get(src_ptr_NC, osx, sxx);
+          *out_ptr_NCX += src * gxx;
+        }
+      }
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      else if (do_push) {
+        if (trgt_K == 0) {
+          // Diff w.r.t. push/pull
+          scalar_t* out_ptr_NC = out_ptr_NC0;
+          for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC)
+            bound::add(out_ptr_NC, oox, wxx * target[c], sxx);
+        } else {
+          // Diff w.r.t. sgrad
+          scalar_t* out_ptr_NC = out_ptr_NC0;
+          for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC) {
+            scalar_t val = gxx * target[c];
+            bound::add(out_ptr_NC, oox, val, sxx);
+          }
+        }
+      }
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Count ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      else if (do_count) {
+        bound::add(out_ptr_NC0, oox, wxx, sxx);
+      }
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Grad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      if (do_grad) {
+        if (trgt_K == 0) {
+          // Diff w.r.t. pull/push
+          scalar_t* src_ptr_NC = src_ptr_NC0;
+          scalar_t dot = static_cast<scalar_t>(0);
+          for (offset_t c = 0; c < C; ++c, src_ptr_NC += src_sC) {
+            scalar_t src = bound::get(src_ptr_NC, osx, sxx);
+            dot += (trgt_ptr ? src * target[c] : src);
+            // trgt_ptr == 0 in the backward pass of 'count'
+          }
+          ogx += gxx * dot;
+        } else {
+          // Diff w.r.t. sgrad
+          scalar_t* src_ptr_NC = src_ptr_NC0;
+          scalar_t dot;
+          dot = static_cast<scalar_t>(0);
+          for (offset_t c = 0; c < C; ++c, src_ptr_NC += src_sC) {
+            scalar_t src = bound::get(src_ptr_NC, osx, sxx);
+            dot += src * target[c];
+          }
+          ogx += hxx * dot;
+        }
+      }
+
+    } // x
+
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Grad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    if (do_grad) {
+      scalar_t* grad_ptr_NX = grad_ptr + n * grad_sN + w * grad_sX;
+      (*grad_ptr_NX) = ogx;
+    }
+  }
+
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   //                     LINEAR INTERPOLATION 3D
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1214,7 +1557,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
       scalar_t* trgt_ptr_NCXYZ = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY + d * trgt_sZ;
       scalar_t* src_ptr_NC = src_ptr + n * src_sN;
 
-      if (trgt_K == 1) {
+      if (trgt_K == 0) {
         // backward w.r.t. push/pull
         for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXYZ += trgt_sC, src_ptr_NC += src_sC) {
           scalar_t src;
@@ -1376,7 +1719,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
       o111 = ix1 * out_sX + iy1 * out_sY + iz1 * out_sZ;
       scalar_t* trgt_ptr_NCXYZ = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY + d * trgt_sZ;
       scalar_t* out_ptr_NC = out_ptr + n * out_sN;
-      if (trgt_K == 1) {
+      if (trgt_K == 0) {
         // Diff w.r.t. push/pull
         for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXYZ += trgt_sC, out_ptr_NC += out_sC) {
           scalar_t trgt = *trgt_ptr_NCXYZ;
@@ -1461,7 +1804,6 @@ MONAI_NAMESPACE_DEVICE { // cpu
     scalar_t w10 = dx1 * dy0;
     scalar_t w01 = dx0 * dy1;
     scalar_t w11 = dx1 * dy1;
-    ;
 
     // Sign (/!\ compute sign before warping indices)
     int8_t sx1 = bound::sign(bound0, ix0 + 1, src_X);
@@ -1500,7 +1842,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
       scalar_t* trgt_ptr_NCXY = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY;
       scalar_t* src_ptr_NC = src_ptr + n * src_sN;
 
-      if (trgt_K == 1) {
+      if (trgt_K == 0) {
         // backward w.r.t. push/pull
         for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXY += trgt_sC, src_ptr_NC += src_sC) {
           scalar_t src;
@@ -1547,9 +1889,9 @@ MONAI_NAMESPACE_DEVICE { // cpu
         }
       }
 
-      scalar_t* grad_ptr_NXYZ = grad_ptr + n * grad_sN + w * grad_sX + h * grad_sY;
-      (*grad_ptr_NXYZ) = gx;
-      grad_ptr_NXYZ[grad_sC] = gy;
+      scalar_t* grad_ptr_NXY = grad_ptr + n * grad_sN + w * grad_sX + h * grad_sY;
+      (*grad_ptr_NXY) = gx;
+      grad_ptr_NXY[grad_sC] = gy;
     }
     // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pull ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     if (do_pull) {
@@ -1591,7 +1933,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
       o11 = ix1 * out_sX + iy1 * out_sY;
       scalar_t* trgt_ptr_NCXY = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY;
       scalar_t* out_ptr_NC = out_ptr + n * out_sN;
-      if (trgt_K == 1) {
+      if (trgt_K == 0) {
         // Diff w.r.t. push/pull
         for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXY += trgt_sC, out_ptr_NC += out_sC) {
           scalar_t trgt = *trgt_ptr_NCXY;
@@ -1632,6 +1974,123 @@ MONAI_NAMESPACE_DEVICE { // cpu
     }
   }
 
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                     LINEAR INTERPOLATION 1D
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  template <typename scalar_t, typename offset_t>
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::interpolate1d_linear(scalar_t x, offset_t w, offset_t n) const {
+    // Get corner pixel values from (x)
+    offset_t ix0 = static_cast<offset_t>(std::floor(x));
+
+    // Interpolation weights (inversely proportional to distance)
+    scalar_t w1 = x - ix0;
+    scalar_t w0 = 1. - w1;
+
+    // Sign (/!\ compute sign before warping indices)
+    int8_t s1 = bound::sign(bound0, ix0 + 1, src_X);
+    int8_t s0 = bound::sign(bound0, ix0, src_X);
+
+    // Warp indices
+    offset_t ix1;
+    ix1 = bound::index(bound0, ix0 + 1, src_X);
+    ix0 = bound::index(bound0, ix0, src_X);
+
+    // Offsets into source volume
+    offset_t o0, o1;
+    if (do_pull || do_grad || do_sgrad) {
+      o0 = ix0 * src_sX;
+      o1 = ix1 * src_sX;
+    }
+
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~ Grid gradient ~~~~~~~~~~~~~~~~~~~~~~~~~~
+    if (do_grad) {
+      if (trgt_K == 0) {
+        // backward w.r.t. push/pull
+
+        o0 = ix0 * src_sX;
+        o1 = ix1 * src_sX;
+        scalar_t gx = static_cast<scalar_t>(0);
+        scalar_t* trgt_ptr_NCX = trgt_ptr + n * trgt_sN + w * trgt_sX;
+        scalar_t* src_ptr_NC = src_ptr + n * src_sN;
+
+        for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC, src_ptr_NC += src_sC) {
+          scalar_t src;
+          scalar_t trgt = trgt_ptr ? *trgt_ptr_NCX : static_cast<scalar_t>(1);
+          // ^ trgt_ptr == 0 during the backward pass of count
+          src = bound::get(src_ptr_NC, o0, s0);
+          if (trgt_ptr)
+            src *= trgt;
+          gx -= src;
+          src = bound::get(src_ptr_NC, o1, s1);
+          if (trgt_ptr)
+            src *= trgt;
+          gx += src;
+        }
+
+        scalar_t* grad_ptr_NX = grad_ptr + n * grad_sN + w * grad_sX;
+        (*grad_ptr_NX) = gx;
+      } else {
+        // backward w.r.t. sgrad
+        // -> zero (make sure this is done at initialization)
+      }
+    }
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pull ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    if (do_pull) {
+      o0 = ix0 * src_sX;
+      o1 = ix1 * src_sX;
+      scalar_t* out_ptr_NCX = out_ptr + n * out_sN + w * out_sX;
+      scalar_t* src_ptr_NC = src_ptr + n * src_sN;
+      for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC) {
+        *out_ptr_NCX = bound::get(src_ptr_NC, o0, s0) * w0 + bound::get(src_ptr_NC, o1, s1) * w1;
+      }
+    }
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SGrad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    else if (do_sgrad) {
+      o0 = ix0 * src_sX;
+      o1 = ix1 * src_sX;
+      scalar_t* out_ptr_NCX = out_ptr + n * out_sN + w * out_sX;
+      scalar_t* src_ptr_NC = src_ptr + n * src_sN;
+
+      for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC) {
+        *out_ptr_NCX = bound::get(src_ptr_NC, o1, s1) - bound::get(src_ptr_NC, o0, s0);
+      }
+    }
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    else if (do_push) {
+      // Offsets into 'push' volume
+      o0 = ix0 * out_sX;
+      o1 = ix1 * out_sX;
+      scalar_t* trgt_ptr_NCX = trgt_ptr + n * trgt_sN + w * trgt_sX;
+      scalar_t* out_ptr_NC = out_ptr + n * out_sN;
+      if (trgt_K == 0) {
+        // Diff w.r.t. push/pull
+        for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC, out_ptr_NC += out_sC) {
+          scalar_t trgt = *trgt_ptr_NCX;
+          bound::add(out_ptr_NC, o0, w0 * trgt, s0);
+          bound::add(out_ptr_NC, o1, w1 * trgt, s1);
+        }
+      } else {
+        // Diff w.r.t. sgrad
+        for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC, out_ptr_NC += out_sC) {
+          scalar_t trgt0 = *trgt_ptr_NCX;
+          bound::add(out_ptr_NC, o0, -trgt0, s0);
+          bound::add(out_ptr_NC, o1, trgt0, s1);
+        }
+      }
+    }
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    else if (do_count) {
+      // Offsets into 'push' volume
+      o0 = ix0 * out_sX;
+      o1 = ix1 * out_sX;
+
+      scalar_t* out_ptr_N = out_ptr + n * out_sN;
+      bound::add(out_ptr_N, o0, w0, s0);
+      bound::add(out_ptr_N, o1, w1, s1);
+    }
+  }
+
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   //                  NEAREST NEIGHBOR INTERPOLATION 3D
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1666,7 +2125,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
       scalar_t* src_ptr_NC = src_ptr + n * src_sN;
       for (offset_t c = 0; c < C; ++c, out_ptr_NCXYZ += out_sC, src_ptr_NC += src_sC)
         *out_ptr_NCXYZ = bound::get(src_ptr_NC, o, s);
-    } else if (do_push && trgt_K == 1) {
+    } else if (do_push && trgt_K == 0) {
       offset_t o = iz * out_sZ + iy * out_sY + ix * out_sX;
       scalar_t* trgt_ptr_NCXYZ = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY + d * trgt_sZ;
       scalar_t* out_ptr_NC = out_ptr + n * out_sN;
@@ -1709,7 +2168,7 @@ MONAI_NAMESPACE_DEVICE { // cpu
       scalar_t* src_ptr_NC = src_ptr + n * src_sN;
       for (offset_t c = 0; c < C; ++c, out_ptr_NCXY += out_sC, src_ptr_NC += src_sC)
         *out_ptr_NCXY = bound::get(src_ptr_NC, o, s);
-    } else if (do_push && trgt_K == 1) {
+    } else if (do_push && trgt_K == 0) {
       offset_t o = iy * out_sY + ix * out_sX;
       scalar_t* trgt_ptr_NCXY = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY;
       scalar_t* out_ptr_NC = out_ptr + n * out_sN;
@@ -1722,10 +2181,48 @@ MONAI_NAMESPACE_DEVICE { // cpu
         bound::add(out_ptr_NC, o, static_cast<scalar_t>(1), s);
     }
   }
+
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                  NEAREST NEIGHBOR INTERPOLATION 1D
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  template <typename scalar_t, typename offset_t>
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::interpolate1d_nearest(scalar_t x, offset_t w, offset_t n) const {
+    offset_t i = static_cast<offset_t>(std::round(x));
+
+    // Boundary condition (/!\ compute sign before warping indices)
+    int8_t s = bound::sign(bound0, i, src_X);
+    i = bound::index(bound0, i, src_X);
+
+    if (do_pull) {
+      offset_t o = i * src_sX;
+      scalar_t* out_ptr_NCX = out_ptr + n * out_sN + w * out_sX;
+      scalar_t* src_ptr_NC = src_ptr + n * src_sN;
+      for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC)
+        *out_ptr_NCX = bound::get(src_ptr_NC, o, s);
+    } else if (do_push && trgt_K == 0) {
+      offset_t o = i * out_sX;
+      scalar_t* trgt_ptr_NCX = trgt_ptr + n * trgt_sN + w * trgt_sX;
+      scalar_t* out_ptr_NC = out_ptr + n * out_sN;
+      for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC, out_ptr_NC += out_sC)
+        bound::add(out_ptr_NC, o, *trgt_ptr_NCX, s);
+    } else if (do_count) {
+      offset_t o = i * out_sX;
+      scalar_t* out_ptr_NC = out_ptr + n * out_sN;
+      for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC)
+        bound::add(out_ptr_NC, o, static_cast<scalar_t>(1), s);
+    }
+  }
+
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   //            LINEAR INTERPOLATION 3D + SLIDING BOUNDARY
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   // TODO
+
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                  CUDA KERNEL (MUST BE OUT OF CLASS)
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
   } // namespace
 
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1757,8 +2254,6 @@ MONAI_NAMESPACE_DEVICE { // cpu
   PUSHPULL_INSTANTIATE1(BoundType); \
   PUSHPULL_INSTANTIATE1(BoundVectorRef)
 
-  // ~~~ CPU ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
   // Two arguments (source, grid)
   // > `bound` and `interpolation` can be single arguments or vectors.
   template <typename BoundType, typename InterpolationType, typename SourceType>
@@ -1773,12 +2268,14 @@ MONAI_NAMESPACE_DEVICE { // cpu
       bool do_count,
       bool do_grad,
       bool do_sgrad) {
+    PushPullAllocator info(
+        grid.dim() - 2, bound, interpolation, extrapolate, do_pull, do_push, do_count, do_grad, do_sgrad);
+    info.ioset(source, grid);
+
     return AT_DISPATCH_FLOATING_TYPES(grid.scalar_type(), "pushpull", [&] {
-      PushPullImpl<scalar_t, int32_t> f(
-          grid.dim() - 2, bound, interpolation, extrapolate, do_pull, do_push, do_count, do_grad, do_sgrad);
-      f.ioset(source, grid);
-      f.loop();
-      return f.output;
+      PushPullImpl<scalar_t, int64_t> algo(info);
+      algo.loop();
+      return algo.output;
     });
   }
 
@@ -1798,17 +2295,18 @@ MONAI_NAMESPACE_DEVICE { // cpu
       bool do_count,
       bool do_grad,
       bool do_sgrad) {
+    PushPullAllocator info(
+        grid.dim() - 2, bound, interpolation, extrapolate, do_pull, do_push, do_count, do_grad, do_sgrad);
+    info.ioset(source, grid, target);
+
     return AT_DISPATCH_FLOATING_TYPES(grid.scalar_type(), "pushpull", [&] {
-      PushPullImpl<scalar_t, int32_t> f(
-          grid.dim() - 2, bound, interpolation, extrapolate, do_pull, do_push, do_count, do_grad, do_sgrad);
-      f.ioset(source, grid, target);
-      f.loop();
-      return f.output;
+      PushPullImpl<scalar_t, int64_t> algo(info);
+      algo.loop();
+      return algo.output;
     });
   }
 
   PUSHPULL_INSTANTIATE;
 
-} // namespace <device>
-
+} // namespace cpu
 } // namespace monai
diff --git a/monai/csrc/resample/pushpull_cuda.cu b/monai/csrc/resample/pushpull_cuda.cu
index ecfeb562ab..38d34ffe98 100644
--- a/monai/csrc/resample/pushpull_cuda.cu
+++ b/monai/csrc/resample/pushpull_cuda.cu
@@ -25,6 +25,7 @@ limitations under the License.
 // TODO:
 // . [DONE] generic 3d
 // . [DONE] generic 2d
+// . [DONE] generic 1d
 // . sliding nearest 3d
 // . sliding nearest 2d
 // . sliding linear 3d
@@ -37,6 +38,7 @@ limitations under the License.
 // . input bound/inter are always vectors -> clean unused constructors
 
 #include <ATen/ATen.h>
+#include <limits>
 #include <tuple>
 #include "bounds_common.h"
 #include "interpolation_common.h"
@@ -71,18 +73,27 @@ MONAI_NAMESPACE_DEVICE { // cuda
   namespace { // anonymous namespace > everything inside has internal linkage
 
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-  //                        GENERIC PUSHPULL CLASS
+  //                        INDEXING UTILS
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-  // This class implements the bulk of the code.
-  // /!\ No type and shape checking is performed here.
 
-  template <typename scalar_t, typename offset_t>
-  class PushPullImpl {
+  // This class reads and sets all the parameters that will later be used
+  // by the algorithm in PushPullImpl. All of this is done outside of the
+  // implementation class so that we do not depend on generic types. The
+  // point is to pre-allocate all necessary tensors so that we can check
+  // if they're all compatible with 32 bit math. If it's the case, we can
+  // dispatch to a 32b cuda implementation, which might increase
+  // performance. Else, we use 64 bit math to compute offsets.
+  // (On CPU, we always use 64 bit offsets because it doesn't make a huge
+  // difference. It would be different if we had a vectorized
+  // implementation as in PyTorch).
+  class PushPullAllocator {
    public:
+    static constexpr int64_t max_int32 = std::numeric_limits<int32_t>::max();
+
     // ~~~ CONSTRUCTORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     MONAI_HOST
-    PushPullImpl(
+    PushPullAllocator(
         int dim,
         BoundVectorRef bound,
         InterpolationVectorRef interpolation,
@@ -122,100 +133,417 @@ MONAI_NAMESPACE_DEVICE { // cuda
       iso = interpolation0 == interpolation1 && interpolation0 == interpolation2;
     }
 
-    MONAI_HOST
-    PushPullImpl(
-        int dim,
-        BoundType bound,
-        InterpolationVectorRef interpolation,
-        bool extrapolate,
-        bool do_pull,
-        bool do_push,
-        bool do_count,
-        bool do_grad,
-        bool do_sgrad)
-        : dim(dim),
-          bound0(bound),
-          bound1(bound),
-          bound2(bound),
-          interpolation0(interpolation.size() > 0 ? interpolation[0] : InterpolationType::Linear),
-          interpolation1(
-              interpolation.size() > 1       ? interpolation[1]
-                  : interpolation.size() > 0 ? interpolation[0]
-                                             : InterpolationType::Linear),
-          interpolation2(
-              interpolation.size() > 2       ? interpolation[2]
-                  : interpolation.size() > 1 ? interpolation[1]
-                  : interpolation.size() > 0 ? interpolation[0]
-                                             : InterpolationType::Linear),
-          extrapolate(extrapolate),
-          do_pull(do_pull),
-          do_push(do_push),
-          do_count(do_count),
-          do_grad(do_grad),
-          do_sgrad(do_sgrad) {
-      iso = interpolation0 == interpolation1 && interpolation0 == interpolation2;
+    // ~~~ FUNCTORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    // Usually used for pull:
+    // - do_pull  -> return source[grid]
+    // - do_push  -> fails
+    // - do_grad  -> return J(source)[grid]
+    // - do_sgrad -> return H(source)[grid]
+    MONAI_HOST void ioset(const Tensor& source, const Tensor& grid) {
+      init_all();
+      init_source(source);
+      init_grid(grid);
+      init_output();
     }
 
-    MONAI_HOST
-    PushPullImpl(
-        int dim,
-        BoundVectorRef bound,
-        InterpolationType interpolation,
-        bool extrapolate,
-        bool do_pull,
-        bool do_push,
-        bool do_count,
-        bool do_grad,
-        bool do_sgrad)
-        : dim(dim),
-          bound0(bound.size() > 0 ? bound[0] : BoundType::Replicate),
-          bound1(
-              bound.size() > 1       ? bound[1]
-                  : bound.size() > 0 ? bound[0]
-                                     : BoundType::Replicate),
-          bound2(
-              bound.size() > 2       ? bound[2]
-                  : bound.size() > 1 ? bound[1]
-                  : bound.size() > 0 ? bound[0]
-                                     : BoundType::Replicate),
-          interpolation0(interpolation),
-          interpolation1(interpolation),
-          interpolation2(interpolation),
-          extrapolate(extrapolate),
-          do_pull(do_pull),
-          do_push(do_push),
-          do_count(do_count),
-          do_grad(do_grad),
-          do_sgrad(do_sgrad) {
-      iso = interpolation0 == interpolation1 && interpolation0 == interpolation2;
+    // Usually used for pull_backward:
+    // - do_pull  -> return source[grid]
+    // - do_push  -> return push(target, grid, source.shape)
+    // - do_grad  -> return J(source)[grid]
+    // - do_sgrad -> return H(source)[grid]
+    MONAI_HOST void ioset(const Tensor& source, const Tensor& grid, const Tensor& target) {
+      init_all();
+      init_source(source);
+      init_grid(grid);
+      init_target(target);
+      init_output();
     }
 
-    MONAI_HOST
-    PushPullImpl(
-        int dim,
-        BoundType bound,
-        InterpolationType interpolation,
-        bool extrapolate,
-        bool do_pull,
-        bool do_push,
-        bool do_count,
-        bool do_grad,
-        bool do_sgrad)
-        : dim(dim),
-          bound0(bound),
-          bound1(bound),
-          bound2(bound),
-          interpolation0(interpolation),
-          interpolation1(interpolation),
-          interpolation2(interpolation),
-          extrapolate(extrapolate),
-          do_pull(do_pull),
-          do_push(do_push),
-          do_count(do_count),
-          do_grad(do_grad),
-          do_sgrad(do_sgrad) {
-      iso = interpolation0 == interpolation1 && interpolation0 == interpolation2;
+    // Usually used for push:
+    // - do_pull  -> fails
+    // - do_push  -> return push(target, grid, source_size)
+    // - do_grad  -> fails
+    // - do_sgrad -> fails
+    MONAI_HOST void ioset(IntArrayRef source_size, const Tensor& grid, const Tensor& target) {
+      init_all();
+      init_source(source_size);
+      init_grid(grid);
+      init_target(target);
+      init_output();
+    }
+
+    // Usually used for count:
+    // - do_pull  -> fails
+    // - do_push  -> return push(ones, grid, source_size)
+    // - do_grad  -> fails
+    // - do_sgrad -> fails
+    MONAI_HOST void ioset(IntArrayRef source_size, const Tensor& grid) {
+      init_all();
+      init_source(source_size);
+      init_grid(grid);
+      init_output();
+    }
+
+    // We just check that all tensors that we own are compatible with 32b math
+    bool canUse32BitIndexMath(int64_t max_elem = max_int32) const {
+      return src_32b_ok && trgt_32b_ok && grid_32b_ok && grad_32b_ok && out_32b_ok;
+    }
+
+   private:
+    // Copied from aten/src/ATen/native/IndexingUtils.cpp in PyTorch 1.6.
+    // It is used to decide to which pointer type we should dispatch to.
+    // Basically, we need to make sure that the "furthest" element we need
+    // to reach is less than max_elem away.
+    static bool tensorCanUse32BitIndexMath(const Tensor& t, int64_t max_elem = max_int32) {
+      int64_t elements = t.numel();
+      if (elements >= max_elem) {
+        return false;
+      }
+      if (elements == 0) {
+        return max_elem > 0;
+      }
+
+      int64_t offset = 0;
+      int64_t linearId = elements - 1;
+
+      // NOTE: Assumes all strides are positive, which is true for now
+      for (int i = t.dim() - 1; i >= 0; --i) {
+        int64_t curDimIndex = linearId % t.size(i);
+        int64_t curDimOffset = curDimIndex * t.stride(i);
+        offset += curDimOffset;
+        linearId /= t.size(i);
+      }
+
+      if (offset >= max_elem) {
+        return false;
+      }
+
+      return true;
+    }
+
+    // ~~~ COMPONENTS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    MONAI_HOST void init_all();
+    MONAI_HOST void init_source(const Tensor& source);
+    MONAI_HOST void init_source(IntArrayRef source_size);
+    MONAI_HOST void init_grid(const Tensor& grid);
+    MONAI_HOST void init_target(const Tensor& target);
+    MONAI_HOST void init_output();
+
+    // ~~~ OPTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    int dim; // dimensionality (2 or 3)
+    BoundType bound0; // boundary condition  // x|W
+    BoundType bound1; // boundary condition  // y|H
+    BoundType bound2; // boundary condition  // z|D
+    InterpolationType interpolation0; // interpolation order // x|W
+    InterpolationType interpolation1; // interpolation order // y|H
+    InterpolationType interpolation2; // interpolation order // z|D
+    bool iso; // isotropic interpolation?
+    bool extrapolate; // compute out-of-bound values
+    bool do_pull; // sample a volume
+    bool do_push; // splat a volume
+    bool do_count; // splatting weights (= jacobian determinant)
+    bool do_grad; // backprop: gradient of grid // pull
+    bool do_sgrad; // sample spatial gradients
+
+    // ~~~ NAVIGATORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    std::deque<Tensor> output;
+    TensorOptions src_opt;
+    TensorOptions grid_opt;
+    TensorOptions trgt_opt;
+    int64_t N;
+    int64_t C;
+    int64_t src_X;
+    int64_t src_Y;
+    int64_t src_Z;
+    int64_t trgt_X;
+    int64_t trgt_Y;
+    int64_t trgt_Z;
+    int64_t trgt_K;
+    int64_t src_sN;
+    int64_t src_sC;
+    int64_t src_sX;
+    int64_t src_sY;
+    int64_t src_sZ;
+    bool src_32b_ok;
+    void* src_ptr;
+    int64_t trgt_sN;
+    int64_t trgt_sC;
+    int64_t trgt_sX;
+    int64_t trgt_sY;
+    int64_t trgt_sZ;
+    int64_t trgt_sK;
+    bool trgt_32b_ok;
+    void* trgt_ptr;
+    int64_t grid_sN;
+    int64_t grid_sC;
+    int64_t grid_sX;
+    int64_t grid_sY;
+    int64_t grid_sZ;
+    bool grid_32b_ok;
+    void* grid_ptr;
+    int64_t out_sN;
+    int64_t out_sC;
+    int64_t out_sX;
+    int64_t out_sY;
+    int64_t out_sZ;
+    int64_t out_sK; // gradient dimension
+    bool out_32b_ok;
+    void* out_ptr;
+    int64_t grad_sN;
+    int64_t grad_sC;
+    int64_t grad_sX;
+    int64_t grad_sY;
+    int64_t grad_sZ;
+    bool grad_32b_ok;
+    void* grad_ptr;
+
+    // Allow PushPullImpl's constructor to access PushPullAllocator's
+    // private members.
+    template <typename scalar_t, typename offset_t>
+    friend class PushPullImpl;
+  };
+
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                          INITIALISATION
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  MONAI_HOST
+  void PushPullAllocator::init_all() {
+    src_opt = grid_opt = trgt_opt = TensorOptions();
+    N = C = 1L;
+    src_X = src_Y = src_Z = 1L;
+    trgt_X = trgt_Y = trgt_Z = 1L;
+    trgt_K = 0L;
+    src_sN = src_sC = src_sX = src_sY = src_sZ = 0L;
+    grid_sN = grid_sC = grid_sX = grid_sY = grid_sZ = 0L;
+    grad_sN = grad_sC = grad_sX = grad_sY = grad_sZ = 0L;
+    trgt_sN = trgt_sC = trgt_sX = trgt_sY = trgt_sZ = trgt_sK = 0L;
+    out_sN = out_sC = out_sX = out_sY = out_sZ = out_sK = 0L;
+    src_ptr = trgt_ptr = grid_ptr = out_ptr = grad_ptr = static_cast<float*>(0);
+    src_32b_ok = trgt_32b_ok = grid_32b_ok = out_32b_ok = grad_32b_ok = true;
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_source(const Tensor& source) {
+    N = source.size(0);
+    C = source.size(1);
+    src_X = source.size(2);
+    src_Y = dim < 2 ? 1L : source.size(3);
+    src_Z = dim < 3 ? 1L : source.size(4);
+    src_sN = source.stride(0);
+    src_sC = source.stride(1);
+    src_sX = source.stride(2);
+    src_sY = dim < 2 ? 0L : source.stride(3);
+    src_sZ = dim < 3 ? 0L : source.stride(4);
+    src_ptr = source.data_ptr();
+    src_opt = source.options();
+    src_32b_ok = tensorCanUse32BitIndexMath(source);
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_source(IntArrayRef source_size) {
+    src_X = source_size[0];
+    src_Y = dim < 2 ? 1L : source_size[1];
+    src_Z = dim < 3 ? 1L : source_size[2];
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_grid(const Tensor& grid) {
+    N = grid.size(0);
+    trgt_X = grid.size(1);
+    trgt_Y = dim < 2 ? 1L : grid.size(2);
+    trgt_Z = dim < 3 ? 1L : grid.size(3);
+    grid_sN = grid.stride(0);
+    grid_sX = grid.stride(1);
+    grid_sY = dim < 2 ? 0L : grid.stride(2);
+    grid_sZ = dim < 3 ? 0L : grid.stride(3);
+    grid_sC = grid.stride(dim == 1 ? 2 : dim == 2 ? 3 : 4);
+    grid_ptr = grid.data_ptr();
+    grid_opt = grid.options();
+    grid_32b_ok = tensorCanUse32BitIndexMath(grid);
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_target(const Tensor& target) {
+    N = target.size(0);
+    C = target.size(1);
+    trgt_X = target.size(2);
+    trgt_Y = dim < 2 ? 1L : target.size(3);
+    trgt_Z = dim < 3 ? 1L : target.size(4);
+    trgt_K = target.dim() == dim + 3 ? target.size(dim == 1 ? 3 : dim == 2 ? 4 : 5) : 0L;
+    trgt_sN = target.stride(0);
+    trgt_sC = target.stride(1);
+    trgt_sX = target.stride(2);
+    trgt_sY = dim < 2 ? 0L : target.stride(3);
+    trgt_sZ = dim < 3 ? 0L : target.stride(4);
+    trgt_sK = target.dim() == dim + 3 ? target.stride(dim == 1 ? 3 : dim == 2 ? 4 : 5) : 0L;
+    trgt_ptr = target.data_ptr();
+    trgt_opt = target.options();
+    trgt_32b_ok = tensorCanUse32BitIndexMath(target);
+  }
+
+  MONAI_HOST
+  void PushPullAllocator::init_output() {
+    output.clear();
+    if (do_pull) {
+      if (dim == 1)
+        output.push_back(at::empty({N, C, trgt_X}, src_opt));
+      else if (dim == 2)
+        output.push_back(at::empty({N, C, trgt_X, trgt_Y}, src_opt));
+      else
+        output.push_back(at::empty({N, C, trgt_X, trgt_Y, trgt_Z}, src_opt));
+      auto pull = output.back();
+      out_sN = pull.stride(0);
+      out_sC = pull.stride(1);
+      out_sX = pull.stride(2);
+      out_sY = dim < 2 ? 0L : pull.stride(3);
+      out_sZ = dim < 3 ? 0L : pull.stride(4);
+      out_sK = 0L;
+      out_ptr = pull.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(pull);
+    } else if (do_sgrad) {
+      if (dim == 1)
+        output.push_back(at::empty({N, C, trgt_X, 1}, src_opt));
+      else if (dim == 2)
+        output.push_back(at::empty({N, C, trgt_X, trgt_Y, 2}, src_opt));
+      else
+        output.push_back(at::empty({N, C, trgt_X, trgt_Y, trgt_Z, 3}, src_opt));
+      auto sgrad = output.back();
+      out_sN = sgrad.stride(0);
+      out_sC = sgrad.stride(1);
+      out_sX = sgrad.stride(2);
+      out_sY = dim < 2 ? 0L : sgrad.stride(3);
+      out_sZ = dim < 3 ? 0L : sgrad.stride(4);
+      out_sK = sgrad.stride(dim == 1 ? 3 : dim == 2 ? 4 : 5);
+      out_ptr = sgrad.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(sgrad);
+
+      if (iso && interpolation0 == InterpolationType::Nearest)
+        sgrad.zero_();
+      if (iso && interpolation0 == InterpolationType::Linear && dim == 1)
+        sgrad.zero_();
+    } else if (do_push) {
+      if (dim == 1)
+        output.push_back(at::zeros({N, C, src_X}, trgt_opt));
+      else if (dim == 2)
+        output.push_back(at::zeros({N, C, src_X, src_Y}, trgt_opt));
+      else
+        output.push_back(at::zeros({N, C, src_X, src_Y, src_Z}, trgt_opt));
+      auto push = output.back();
+      out_sN = push.stride(0);
+      out_sC = push.stride(1);
+      out_sX = push.stride(2);
+      out_sY = dim < 2 ? 0L : push.stride(3);
+      out_sZ = dim < 3 ? 0L : push.stride(4);
+      out_sK = 0L;
+      out_ptr = push.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(push);
+    } else if (do_count) {
+      if (dim == 1)
+        output.push_back(at::zeros({N, 1, src_X}, grid_opt));
+      else if (dim == 2)
+        output.push_back(at::zeros({N, 1, src_X, src_Y}, grid_opt));
+      else
+        output.push_back(at::zeros({N, 1, src_X, src_Y, src_Z}, grid_opt));
+      auto count = output.back();
+      out_sN = count.stride(0);
+      out_sC = count.stride(1);
+      out_sX = count.stride(2);
+      out_sY = dim < 2 ? 0L : count.stride(3);
+      out_sZ = dim < 3 ? 0L : count.stride(4);
+      out_sK = 0L;
+      out_ptr = count.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(count);
+    }
+    if (do_grad) {
+      if (dim == 1)
+        output.push_back(at::zeros({N, trgt_X, 1}, grid_opt));
+      else if (dim == 2)
+        output.push_back(at::zeros({N, trgt_X, trgt_Y, 2}, grid_opt));
+      else
+        output.push_back(at::zeros({N, trgt_X, trgt_Y, trgt_Z, 3}, grid_opt));
+      auto grad = output.back();
+      grad_sN = grad.stride(0);
+      grad_sX = grad.stride(1);
+      grad_sY = dim < 2 ? 0L : grad.stride(2);
+      grad_sZ = dim < 3 ? 0L : grad.stride(3);
+      grad_sC = grad.stride(dim == 1 ? 2 : dim == 2 ? 3 : 4);
+      grad_ptr = grad.data_ptr();
+      out_32b_ok = tensorCanUse32BitIndexMath(grad);
+
+      if (iso && interpolation0 == InterpolationType::Nearest)
+        grad.zero_();
     }
+  }
+
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                        GENERIC PUSHPULL CLASS
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  // This class implements the bulk of the code.
+  // /!\ No type and shape checking is performed here.
+
+  template <typename scalar_t, typename offset_t>
+  class PushPullImpl {
+   public:
+    // ~~~ CONSTRUCTOR ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    PushPullImpl(const PushPullAllocator& info)
+        : output(info.output),
+          dim(info.dim),
+          bound0(info.bound0),
+          bound1(info.bound1),
+          bound2(info.bound2),
+          interpolation0(info.interpolation0),
+          interpolation1(info.interpolation1),
+          interpolation2(info.interpolation1),
+          iso(info.iso),
+          extrapolate(info.extrapolate),
+          do_pull(info.do_pull),
+          do_push(info.do_push),
+          do_count(info.do_count),
+          do_grad(info.do_grad),
+          do_sgrad(info.do_sgrad),
+          N(static_cast<offset_t>(info.N)),
+          C(static_cast<offset_t>(info.C)),
+          src_X(static_cast<offset_t>(info.src_X)),
+          src_Y(static_cast<offset_t>(info.src_Y)),
+          src_Z(static_cast<offset_t>(info.src_Z)),
+          trgt_X(static_cast<offset_t>(info.trgt_X)),
+          trgt_Y(static_cast<offset_t>(info.trgt_Y)),
+          trgt_Z(static_cast<offset_t>(info.trgt_Z)),
+          trgt_K(static_cast<offset_t>(info.trgt_K)),
+          src_sN(static_cast<offset_t>(info.src_sN)),
+          src_sC(static_cast<offset_t>(info.src_sC)),
+          src_sX(static_cast<offset_t>(info.src_sX)),
+          src_sY(static_cast<offset_t>(info.src_sY)),
+          src_sZ(static_cast<offset_t>(info.src_sZ)),
+          src_ptr(static_cast<scalar_t*>(info.src_ptr)),
+          trgt_sN(static_cast<offset_t>(info.trgt_sN)),
+          trgt_sC(static_cast<offset_t>(info.trgt_sC)),
+          trgt_sX(static_cast<offset_t>(info.trgt_sX)),
+          trgt_sY(static_cast<offset_t>(info.trgt_sY)),
+          trgt_sZ(static_cast<offset_t>(info.trgt_sZ)),
+          trgt_sK(static_cast<offset_t>(info.trgt_sK)),
+          trgt_ptr(static_cast<scalar_t*>(info.trgt_ptr)),
+          grid_sN(static_cast<offset_t>(info.grid_sN)),
+          grid_sC(static_cast<offset_t>(info.grid_sC)),
+          grid_sX(static_cast<offset_t>(info.grid_sX)),
+          grid_sY(static_cast<offset_t>(info.grid_sY)),
+          grid_sZ(static_cast<offset_t>(info.grid_sZ)),
+          grid_ptr(static_cast<scalar_t*>(info.grid_ptr)),
+          out_sN(static_cast<offset_t>(info.out_sN)),
+          out_sC(static_cast<offset_t>(info.out_sC)),
+          out_sX(static_cast<offset_t>(info.out_sX)),
+          out_sY(static_cast<offset_t>(info.out_sY)),
+          out_sZ(static_cast<offset_t>(info.out_sZ)),
+          out_sK(static_cast<offset_t>(info.out_sK)),
+          out_ptr(static_cast<scalar_t*>(info.out_ptr)),
+          grad_sN(static_cast<offset_t>(info.grad_sN)),
+          grad_sC(static_cast<offset_t>(info.grad_sC)),
+          grad_sX(static_cast<offset_t>(info.grad_sX)),
+          grad_sY(static_cast<offset_t>(info.grad_sY)),
+          grad_sZ(static_cast<offset_t>(info.grad_sZ)),
+          grad_ptr(static_cast<scalar_t*>(info.grad_ptr)) {}
 
     // ~~~ PUBLIC VALUE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -244,39 +572,9 @@ MONAI_NAMESPACE_DEVICE { // cuda
     // }
 
     // ~~~ FUNCTORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    MONAI_HOST void ioset // Pull
-        (const Tensor& source, const Tensor& grid) {
-      init_all();
-      init_source(source);
-      init_grid(grid);
-      init_output();
-    }
-
-    MONAI_HOST void ioset(const Tensor& source, const Tensor& grid, const Tensor& target) {
-      init_all();
-      init_source(source);
-      init_grid(grid);
-      init_target(target);
-      init_output();
-    }
-
-    MONAI_HOST void ioset // Push
-        (IntArrayRef source_size, const Tensor& grid, const Tensor& target) {
-      init_all();
-      init_source(source_size);
-      init_grid(grid);
-      init_target(target);
-      init_output();
-    }
-
-    MONAI_HOST void ioset // Count
-        (IntArrayRef source_size, const Tensor& grid) {
-      init_all();
-      init_source(source_size);
-      init_grid(grid);
-      init_output();
-    }
 
+    // Loop over voxels that belong to one CUDA block
+    // This function is called by the CUDA kernel
     MONAI_DEVICE void loop(int threadIdx, int blockIdx, int blockDim, int gridDim) const;
 
     MONAI_HOST MONAI_DEVICE int64_t voxcount() const {
@@ -285,14 +583,18 @@ MONAI_NAMESPACE_DEVICE { // cuda
 
    private:
     // ~~~ COMPONENTS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    MONAI_HOST void init_all();
-    MONAI_HOST void init_source(const Tensor& source);
-    MONAI_HOST void init_source(IntArrayRef source_size);
-    MONAI_HOST void init_grid(const Tensor& grid);
-    MONAI_HOST void init_target(const Tensor& target);
-    MONAI_HOST void init_output();
+    MONAI_DEVICE void check1d(offset_t w, offset_t n) const;
     MONAI_DEVICE void check2d(offset_t w, offset_t h, offset_t n) const;
     MONAI_DEVICE void check3d(offset_t w, offset_t h, offset_t d, offset_t n) const;
+    MONAI_DEVICE void interpolate1d(scalar_t x, offset_t w, offset_t n) const;
+    MONAI_DEVICE void interpolate1d_nearest(scalar_t x, offset_t w, offset_t n) const;
+    MONAI_DEVICE void interpolate1d_linear(scalar_t x, offset_t w, offset_t n) const;
+    MONAI_DEVICE void interpolate1d_sliding(scalar_t x, offset_t w, offset_t n) const { /*TODO*/
+    }
+    MONAI_DEVICE void interpolate1d_sliding_nearest(scalar_t x, offset_t w, offset_t n) const { /*TODO*/
+    }
+    MONAI_DEVICE void interpolate1d_sliding_linear(scalar_t x, offset_t w, offset_t n) const { /*TODO*/
+    }
     MONAI_DEVICE void interpolate2d(scalar_t x, scalar_t y, offset_t w, offset_t h, offset_t n) const;
     MONAI_DEVICE void interpolate2d_nearest(scalar_t x, scalar_t y, offset_t w, offset_t h, offset_t n) const;
     MONAI_DEVICE void interpolate2d_bilinear(scalar_t x, scalar_t y, offset_t w, offset_t h, offset_t n) const;
@@ -367,9 +669,6 @@ MONAI_NAMESPACE_DEVICE { // cuda
     bool do_sgrad; // sample spatial gradients
 
     // ~~~ NAVIGATORS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    TensorOptions src_opt;
-    TensorOptions grid_opt;
-    TensorOptions trgt_opt;
     offset_t N;
     offset_t C;
     offset_t src_X;
@@ -396,173 +695,22 @@ MONAI_NAMESPACE_DEVICE { // cuda
     offset_t grid_sC;
     offset_t grid_sX;
     offset_t grid_sY;
-    offset_t grid_sZ;
-    scalar_t* grid_ptr;
-    offset_t out_sN;
-    offset_t out_sC;
-    offset_t out_sX;
-    offset_t out_sY;
-    offset_t out_sZ;
-    offset_t out_sK; // gradient dimension
-    scalar_t* out_ptr;
-    offset_t grad_sN;
-    offset_t grad_sC;
-    offset_t grad_sX;
-    offset_t grad_sY;
-    offset_t grad_sZ;
-    scalar_t* grad_ptr;
-  };
-
-  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-  //                          INITIALISATION
-  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-  template <typename scalar_t, typename offset_t>
-  void PushPullImpl<scalar_t, offset_t>::init_all() {
-    src_opt = grid_opt = trgt_opt = TensorOptions();
-    N = C = static_cast<offset_t>(1);
-    src_X = src_Y = src_Z = static_cast<offset_t>(1);
-    trgt_X = trgt_Y = trgt_Z = trgt_K = static_cast<offset_t>(1);
-    src_sN = src_sC = src_sX = src_sY = src_sZ = static_cast<offset_t>(0);
-    grid_sN = grid_sC = grid_sX = grid_sY = grid_sZ = static_cast<offset_t>(0);
-    grad_sN = grad_sC = grad_sX = grad_sY = grad_sZ = static_cast<offset_t>(0);
-    trgt_sN = trgt_sC = trgt_sX = trgt_sY = trgt_sZ = trgt_sK = static_cast<offset_t>(0);
-    out_sN = out_sC = out_sX = out_sY = out_sZ = out_sK = static_cast<offset_t>(0);
-    src_ptr = trgt_ptr = grid_ptr = out_ptr = grad_ptr = static_cast<scalar_t*>(0);
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_source(const Tensor& source) {
-    N = source.size(0);
-    C = source.size(1);
-    src_X = source.size(2);
-    src_Y = source.size(3);
-    src_Z = dim == 2 ? static_cast<offset_t>(1) : source.size(4);
-    src_sN = source.stride(0);
-    src_sC = source.stride(1);
-    src_sX = source.stride(2);
-    src_sY = source.stride(3);
-    src_sZ = dim == 2 ? static_cast<offset_t>(0) : source.stride(4);
-    src_ptr = source.data_ptr<scalar_t>();
-    src_opt = source.options();
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_source(IntArrayRef source_size) {
-    src_X = source_size[0];
-    src_Y = source_size[1];
-    src_Z = dim == 2 ? static_cast<offset_t>(1) : source_size[2];
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_grid(const Tensor& grid) {
-    N = grid.size(0);
-    trgt_X = grid.size(1);
-    trgt_Y = grid.size(2);
-    trgt_Z = dim == 2 ? static_cast<offset_t>(1) : grid.size(3);
-    grid_sN = grid.stride(0);
-    grid_sX = grid.stride(1);
-    grid_sY = grid.stride(2);
-    grid_sZ = dim == 2 ? static_cast<offset_t>(0) : grid.stride(3);
-    grid_sC = grid.stride(dim == 2 ? 3 : 4);
-    grid_ptr = grid.data_ptr<scalar_t>();
-    grid_opt = grid.options();
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_target(const Tensor& target) {
-    N = target.size(0);
-    C = target.size(1);
-    trgt_X = target.size(2);
-    trgt_Y = target.size(3);
-    trgt_Z = dim == 2 ? static_cast<offset_t>(1) : target.size(4);
-    trgt_K = target.dim() == dim + 3 ? target.size(dim == 2 ? 4 : 5) : static_cast<offset_t>(1);
-    trgt_sN = target.stride(0);
-    trgt_sC = target.stride(1);
-    trgt_sX = target.stride(2);
-    trgt_sY = target.stride(3);
-    trgt_sZ = dim == 2 ? static_cast<offset_t>(0) : target.stride(4);
-    trgt_sK = target.dim() == dim + 3 ? target.stride(dim == 2 ? 4 : 5) : static_cast<offset_t>(0);
-    trgt_ptr = target.data_ptr<scalar_t>();
-    trgt_opt = target.options();
-  }
-
-  template <typename scalar_t, typename offset_t>
-  MONAI_HOST void PushPullImpl<scalar_t, offset_t>::init_output() {
-    output.clear();
-    if (do_pull) {
-      if (dim == 2)
-        output.push_back(at::empty({N, C, trgt_X, trgt_Y}, src_opt));
-      else
-        output.push_back(at::empty({N, C, trgt_X, trgt_Y, trgt_Z}, src_opt));
-      auto pull = output.back();
-      out_sN = pull.stride(0);
-      out_sC = pull.stride(1);
-      out_sX = pull.stride(2);
-      out_sY = pull.stride(3);
-      out_sZ = dim == 2 ? static_cast<offset_t>(0) : pull.stride(4);
-      out_sK = static_cast<offset_t>(0);
-      out_ptr = pull.template data_ptr<scalar_t>();
-    } else if (do_sgrad) {
-      if (dim == 2)
-        output.push_back(at::empty({N, C, trgt_X, trgt_Y, 2}, src_opt));
-      else
-        output.push_back(at::empty({N, C, trgt_X, trgt_Y, trgt_Z, 3}, src_opt));
-      auto sgrad = output.back();
-      out_sN = sgrad.stride(0);
-      out_sC = sgrad.stride(1);
-      out_sX = sgrad.stride(2);
-      out_sY = sgrad.stride(3);
-      out_sZ = dim == 2 ? static_cast<offset_t>(0) : sgrad.stride(4);
-      out_sK = sgrad.stride(dim == 2 ? 4 : 5);
-      out_ptr = sgrad.template data_ptr<scalar_t>();
-
-      if (iso && interpolation0 == InterpolationType::Nearest)
-        sgrad.zero_();
-    } else if (do_push) {
-      if (dim == 2)
-        output.push_back(at::zeros({N, C, src_X, src_Y}, trgt_opt));
-      else
-        output.push_back(at::zeros({N, C, src_X, src_Y, src_Z}, trgt_opt));
-      auto push = output.back();
-      out_sN = push.stride(0);
-      out_sC = push.stride(1);
-      out_sX = push.stride(2);
-      out_sY = push.stride(3);
-      out_sZ = dim == 2 ? static_cast<offset_t>(0) : push.stride(4);
-      out_sK = static_cast<offset_t>(0);
-      out_ptr = push.template data_ptr<scalar_t>();
-    } else if (do_count) {
-      if (dim == 2)
-        output.push_back(at::zeros({N, 1, src_X, src_Y}, grid_opt));
-      else
-        output.push_back(at::zeros({N, 1, src_X, src_Y, src_Z}, grid_opt));
-      auto count = output.back();
-      out_sN = count.stride(0);
-      out_sC = count.stride(1);
-      out_sX = count.stride(2);
-      out_sY = count.stride(3);
-      out_sZ = dim == 2 ? static_cast<offset_t>(0) : count.stride(4);
-      out_sK = static_cast<offset_t>(0);
-      out_ptr = count.template data_ptr<scalar_t>();
-    }
-    if (do_grad) {
-      if (dim == 2)
-        output.push_back(at::zeros({N, src_X, src_Y, 2}, grid_opt));
-      else
-        output.push_back(at::zeros({N, src_X, src_Y, src_Z, 3}, grid_opt));
-      auto grad = output.back();
-      grad_sN = grad.stride(0);
-      grad_sX = grad.stride(1);
-      grad_sY = grad.stride(2);
-      grad_sZ = dim == 2 ? static_cast<offset_t>(0) : grad.stride(3);
-      grad_sC = grad.stride(dim == 2 ? 3 : 4);
-      grad_ptr = grad.template data_ptr<scalar_t>();
-
-      if (iso && interpolation0 == InterpolationType::Nearest)
-        grad.zero_();
-    }
-  }
+    offset_t grid_sZ;
+    scalar_t* grid_ptr;
+    offset_t out_sN;
+    offset_t out_sC;
+    offset_t out_sX;
+    offset_t out_sY;
+    offset_t out_sZ;
+    offset_t out_sK; // gradient dimension
+    scalar_t* out_ptr;
+    offset_t grad_sN;
+    offset_t grad_sC;
+    offset_t grad_sX;
+    offset_t grad_sY;
+    offset_t grad_sZ;
+    scalar_t* grad_ptr;
+  };
 
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   //                             LOOP
@@ -583,7 +731,9 @@ MONAI_NAMESPACE_DEVICE { // cuda
       h = (i / trgt_Z) % trgt_Y;
       d = i % trgt_Z;
 
-      if (dim == 2)
+      if (dim == 1)
+        check1d(w, n);
+      else if (dim == 2)
         check2d(w, h, n);
       else
         check3d(w, h, d, n);
@@ -598,6 +748,59 @@ MONAI_NAMESPACE_DEVICE { // cuda
   // 1) read the [x,y,z] source coordinate for the current target voxel
   // 3) check if the source coordinate is in bounds
 
+  template <typename scalar_t, typename offset_t>
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::check3d(offset_t w, offset_t h, offset_t d, offset_t n) const {
+    // get the corresponding input x, y, z co-ordinates from grid
+    scalar_t* grid_ptr_NXYZ = grid_ptr + n * grid_sN + w * grid_sX + h * grid_sY + d * grid_sZ;
+    scalar_t x = *grid_ptr_NXYZ;
+    scalar_t y = grid_ptr_NXYZ[grid_sC];
+    scalar_t z = grid_ptr_NXYZ[grid_sC * 2];
+
+    // Check if out-of-bound
+    if (!(extrapolate ||
+          (inbounds(x, src_X, static_cast<scalar_t>(TINY)) && inbounds(y, src_Y, static_cast<scalar_t>(TINY)) &&
+           inbounds(z, src_Z, static_cast<scalar_t>(TINY))))) {
+      if (do_pull || do_sgrad) {
+        scalar_t* out_ptr_NCXYZ = out_ptr + n * out_sN + w * out_sX + h * out_sY + d * out_sZ;
+        for (offset_t c = 0; c < C; ++c, out_ptr_NCXYZ += out_sC) {
+          *out_ptr_NCXYZ = static_cast<scalar_t>(0);
+          if (do_sgrad) {
+            out_ptr_NCXYZ[out_sK] = static_cast<scalar_t>(0);
+            out_ptr_NCXYZ[out_sK * 2] = static_cast<scalar_t>(0);
+          }
+        }
+      }
+      if (do_grad) {
+        scalar_t* grad_ptr_NXYZ = grad_ptr + n * grad_sN + w * grad_sX + h * grad_sY + d * grad_sZ;
+        (*grad_ptr_NXYZ) = static_cast<scalar_t>(0);
+        grad_ptr_NXYZ[grad_sC] = static_cast<scalar_t>(0);
+        grad_ptr_NXYZ[grad_sC * 2] = static_cast<scalar_t>(0);
+      }
+      return;
+    }
+
+    // Next step
+    if (bound0 == BoundType::Sliding) {
+      if (iso)
+        switch (static_cast<int>(interpolation0)) {
+          case 0:
+            return interpolate3d_sliding_nearest(x, y, z, w, h, d, n);
+          case 1:
+            return interpolate3d_sliding_trilinear(x, y, z, w, h, d, n);
+        }
+      return interpolate3d_sliding(x, y, z, w, h, d, n);
+    } else {
+      if (iso)
+        switch (static_cast<int>(interpolation0)) {
+          case 0:
+            return interpolate3d_nearest(x, y, z, w, h, d, n);
+          case 1:
+            return interpolate3d_trilinear(x, y, z, w, h, d, n);
+        }
+      return interpolate3d(x, y, z, w, h, d, n);
+    }
+  }
+
   template <typename scalar_t, typename offset_t>
   MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::check2d(offset_t w, offset_t h, offset_t n) const {
     // get the corresponding input x, y, z co-ordinates from grid
@@ -609,7 +812,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
     if (!(extrapolate ||
           (inbounds(x, src_X, static_cast<scalar_t>(TINY)) && inbounds(y, src_Y, static_cast<scalar_t>(TINY))))) {
       if (do_pull || do_sgrad) {
-        scalar_t* out_ptr_NCXY = out_ptr + n * out_sN + w * out_sZ + h * out_sY;
+        scalar_t* out_ptr_NCXY = out_ptr + n * out_sN + w * out_sX + h * out_sY;
         for (offset_t c = 0; c < C; ++c, out_ptr_NCXY += out_sC) {
           *out_ptr_NCXY = static_cast<scalar_t>(0);
           if (do_sgrad)
@@ -647,32 +850,25 @@ MONAI_NAMESPACE_DEVICE { // cuda
   }
 
   template <typename scalar_t, typename offset_t>
-  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::check3d(offset_t w, offset_t h, offset_t d, offset_t n) const {
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::check1d(offset_t w, offset_t n) const {
     // get the corresponding input x, y, z co-ordinates from grid
-    scalar_t* grid_ptr_NXYZ = grid_ptr + n * grid_sN + w * grid_sX + h * grid_sY + d * grid_sZ;
-    scalar_t x = *grid_ptr_NXYZ;
-    scalar_t y = grid_ptr_NXYZ[grid_sC];
-    scalar_t z = grid_ptr_NXYZ[grid_sC * 2];
+    scalar_t* grid_ptr_NX = grid_ptr + n * grid_sN + w * grid_sX;
+    scalar_t x = *grid_ptr_NX;
 
     // Check if out-of-bound
-    if (!(extrapolate ||
-          (inbounds(x, src_X, static_cast<scalar_t>(TINY)) && inbounds(y, src_Y, static_cast<scalar_t>(TINY)) &&
-           inbounds(z, src_Z, static_cast<scalar_t>(TINY))))) {
+    if (!(extrapolate || inbounds(x, src_X, static_cast<scalar_t>(TINY)))) {
       if (do_pull || do_sgrad) {
-        scalar_t* out_ptr_NCXYZ = out_ptr + n * out_sN + w * out_sX + h * out_sY + d * out_sZ;
-        for (offset_t c = 0; c < C; ++c, out_ptr_NCXYZ += out_sC) {
-          *out_ptr_NCXYZ = static_cast<scalar_t>(0);
-          if (do_sgrad) {
-            out_ptr_NCXYZ[out_sK] = static_cast<scalar_t>(0);
-            out_ptr_NCXYZ[out_sK * 2] = static_cast<scalar_t>(0);
-          }
+        scalar_t* out_ptr_NCX = out_ptr + n * out_sN + w * out_sX;
+        for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC) {
+          *out_ptr_NCX = static_cast<scalar_t>(0);
+          if (do_sgrad)
+            out_ptr_NCX[out_sK] = static_cast<scalar_t>(0);
         }
       }
       if (do_grad) {
-        scalar_t* grad_ptr_NXYZ = grad_ptr + n * grad_sN + w * grad_sX + h * grad_sY + d * grad_sZ;
-        (*grad_ptr_NXYZ) = static_cast<scalar_t>(0);
-        grad_ptr_NXYZ[grad_sC] = static_cast<scalar_t>(0);
-        grad_ptr_NXYZ[grad_sC * 2] = static_cast<scalar_t>(0);
+        scalar_t* grad_ptr_NX = grad_ptr + n * grad_sN + w * grad_sX;
+        (*grad_ptr_NX) = static_cast<scalar_t>(0);
+        grad_ptr_NX[grad_sC] = static_cast<scalar_t>(0);
       }
       return;
     }
@@ -682,20 +878,20 @@ MONAI_NAMESPACE_DEVICE { // cuda
       if (iso)
         switch (static_cast<int>(interpolation0)) {
           case 0:
-            return interpolate3d_sliding_nearest(x, y, z, w, h, d, n);
+            return interpolate1d_sliding_nearest(x, w, n);
           case 1:
-            return interpolate3d_sliding_trilinear(x, y, z, w, h, d, n);
+            return interpolate1d_sliding_linear(x, w, n);
         }
-      return interpolate3d_sliding(x, y, z, w, h, d, n);
+      return interpolate1d_sliding(x, w, n);
     } else {
       if (iso)
         switch (static_cast<int>(interpolation0)) {
           case 0:
-            return interpolate3d_nearest(x, y, z, w, h, d, n);
+            return interpolate1d_nearest(x, w, n);
           case 1:
-            return interpolate3d_trilinear(x, y, z, w, h, d, n);
+            return interpolate1d_linear(x, w, n);
         }
-      return interpolate3d(x, y, z, w, h, d, n);
+      return interpolate1d(x, w, n);
     }
   }
 
@@ -730,7 +926,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
     if (trgt_ptr && (do_push || do_grad))
       for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXYZ += trgt_sC) {
         target[c] = *trgt_ptr_NCXYZ;
-        if (trgt_K > 1) {
+        if (trgt_K > 0) {
           target[c + C] = trgt_ptr_NCXYZ[trgt_sK];
           target[c + C * 2] = trgt_ptr_NCXYZ[trgt_sK * 2];
         }
@@ -848,7 +1044,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
 
           // ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           else if (do_push) {
-            if (trgt_K == 1) {
+            if (trgt_K == 0) {
               // Diff w.r.t. push/pull
               scalar_t* out_ptr_NC = out_ptr_NC0;
               for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC)
@@ -871,7 +1067,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
 
           // ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Grad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
           if (do_grad) {
-            if (trgt_K == 1) {
+            if (trgt_K == 0) {
               // Diff w.r.t. pull/push
               scalar_t* src_ptr_NC = src_ptr_NC0;
               scalar_t dot = static_cast<scalar_t>(0);
@@ -940,7 +1136,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
     if (trgt_ptr && (do_push || do_grad))
       for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXY += trgt_sC) {
         target[c] = *trgt_ptr_NCXY;
-        if (trgt_K > 1) {
+        if (trgt_K > 0) {
           target[c + C] = trgt_ptr_NCXY[trgt_sK];
         }
       }
@@ -1033,7 +1229,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
 
         // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         else if (do_push) {
-          if (trgt_K == 1) {
+          if (trgt_K == 0) {
             // Diff w.r.t. push/pull
             scalar_t* out_ptr_NC = out_ptr_NC0;
             for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC)
@@ -1055,7 +1251,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
 
         // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Grad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         if (do_grad) {
-          if (trgt_K == 1) {
+          if (trgt_K == 0) {
             // Diff w.r.t. pull/push
             scalar_t* src_ptr_NC = src_ptr_NC0;
             scalar_t dot = static_cast<scalar_t>(0);
@@ -1092,6 +1288,150 @@ MONAI_NAMESPACE_DEVICE { // cuda
     }
   }
 
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                     GENERIC INTERPOLATION 1D
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  template <typename scalar_t, typename offset_t>
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::interpolate1d(scalar_t x, offset_t w, offset_t n) const {
+    // Get corner pixel values from (x, y)
+    offset_t bx0, bx1;
+    interpolation::bounds(interpolation0, x, bx0, bx1);
+    offset_t dbx = bx1 - bx0;
+
+    // Pre-compute offsets and target value
+    scalar_t* src_ptr_NC0 = src_ptr + n * src_sN;
+    scalar_t* out_ptr_NC0 = out_ptr + n * out_sN;
+    scalar_t* out_ptr_NCX0 = out_ptr + n * out_sN + w * out_sX;
+    scalar_t* trgt_ptr_NCX = trgt_ptr + n * trgt_sN + w * trgt_sX;
+    scalar_t target[2 * MONAI_MAX_NUM_CHANNELS];
+    if (trgt_ptr && (do_push || do_grad))
+      for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC) {
+        target[c] = *trgt_ptr_NCX;
+        if (trgt_K > 0) {
+          target[c + C] = trgt_ptr_NCX[trgt_sK];
+        }
+      }
+
+    // Initialize output
+    scalar_t* out_ptr_NCX = out_ptr_NCX0;
+    if (do_pull || do_sgrad) {
+      for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC) {
+        *out_ptr_NCX = static_cast<scalar_t>(0);
+        if (do_sgrad) {
+          out_ptr_NCX[out_sK] = static_cast<scalar_t>(0);
+        }
+      }
+    }
+
+    // Pre-compute indices/weights/grad
+    scalar_t wx[8]; // B-spline weights
+    scalar_t gx[8]; // B-spline derivatives
+    scalar_t hx[8]; // B-spline 2nd derivatives
+    offset_t ix[8]; // Warped indices
+    uint8_t sx[8]; // Warped indices
+
+    {
+      scalar_t *owx = static_cast<scalar_t*>(wx), *ogx = static_cast<scalar_t*>(gx), *ohx = static_cast<scalar_t*>(hx);
+      offset_t* oix = static_cast<offset_t*>(ix);
+      uint8_t* osx = static_cast<uint8_t*>(sx);
+      for (offset_t bx = bx0; bx <= bx1; ++bx) {
+        scalar_t dx = x - bx;
+        *(owx++) = interpolation::fastweight(interpolation0, dx);
+        if (do_grad || do_sgrad)
+          *(ogx++) = interpolation::fastgrad(interpolation0, dx);
+        if (do_grad && trgt_sK > 1)
+          *(ohx++) = interpolation::fasthess(interpolation0, dx);
+        *(osx++) = bound::sign(bound0, bx, src_X);
+        *(oix++) = bound::index(bound0, bx, src_X);
+      }
+    }
+
+    // Convolve coefficients with basis functions
+    scalar_t ogx;
+    ogx = static_cast<scalar_t>(0);
+    for (offset_t i = 0; i <= dbx; ++i) {
+      offset_t oox = ix[i] * out_sX;
+      offset_t osx = ix[i] * src_sX;
+      uint8_t sxx = sx[i];
+      scalar_t wxx = wx[i];
+      scalar_t gxx = gx[i];
+      scalar_t hxx = hx[i];
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pull ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      if (do_pull) {
+        scalar_t* src_ptr_NC = src_ptr_NC0;
+        scalar_t* out_ptr_NCX = out_ptr_NCX0;
+        for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC)
+          *out_ptr_NCX += bound::get(src_ptr_NC, osx, sxx) * wxx;
+      }
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SGrad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      else if (do_sgrad) {
+        scalar_t* src_ptr_NC = src_ptr_NC0;
+        scalar_t* out_ptr_NCX = out_ptr_NCX0;
+        for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC) {
+          scalar_t src = bound::get(src_ptr_NC, osx, sxx);
+          *out_ptr_NCX += src * gxx;
+        }
+      }
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      else if (do_push) {
+        if (trgt_K == 0) {
+          // Diff w.r.t. push/pull
+          scalar_t* out_ptr_NC = out_ptr_NC0;
+          for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC)
+            bound::add(out_ptr_NC, oox, wxx * target[c], sxx);
+        } else {
+          // Diff w.r.t. sgrad
+          scalar_t* out_ptr_NC = out_ptr_NC0;
+          for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC) {
+            scalar_t val = gxx * target[c];
+            bound::add(out_ptr_NC, oox, val, sxx);
+          }
+        }
+      }
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Count ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      else if (do_count) {
+        bound::add(out_ptr_NC0, oox, wxx, sxx);
+      }
+
+      // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Grad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+      if (do_grad) {
+        if (trgt_K == 0) {
+          // Diff w.r.t. pull/push
+          scalar_t* src_ptr_NC = src_ptr_NC0;
+          scalar_t dot = static_cast<scalar_t>(0);
+          for (offset_t c = 0; c < C; ++c, src_ptr_NC += src_sC) {
+            scalar_t src = bound::get(src_ptr_NC, osx, sxx);
+            dot += (trgt_ptr ? src * target[c] : src);
+            // trgt_ptr == 0 in the backward pass of 'count'
+          }
+          ogx += gxx * dot;
+        } else {
+          // Diff w.r.t. sgrad
+          scalar_t* src_ptr_NC = src_ptr_NC0;
+          scalar_t dot;
+          dot = static_cast<scalar_t>(0);
+          for (offset_t c = 0; c < C; ++c, src_ptr_NC += src_sC) {
+            scalar_t src = bound::get(src_ptr_NC, osx, sxx);
+            dot += src * target[c];
+          }
+          ogx += hxx * dot;
+        }
+      }
+
+    } // x
+
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Grad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    if (do_grad) {
+      scalar_t* grad_ptr_NX = grad_ptr + n * grad_sN + w * grad_sX;
+      (*grad_ptr_NX) = ogx;
+    }
+  }
+
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   //                     LINEAR INTERPOLATION 3D
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1181,7 +1521,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
       scalar_t* trgt_ptr_NCXYZ = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY + d * trgt_sZ;
       scalar_t* src_ptr_NC = src_ptr + n * src_sN;
 
-      if (trgt_K == 1) {
+      if (trgt_K == 0) {
         // backward w.r.t. push/pull
         for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXYZ += trgt_sC, src_ptr_NC += src_sC) {
           scalar_t src;
@@ -1343,7 +1683,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
       o111 = ix1 * out_sX + iy1 * out_sY + iz1 * out_sZ;
       scalar_t* trgt_ptr_NCXYZ = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY + d * trgt_sZ;
       scalar_t* out_ptr_NC = out_ptr + n * out_sN;
-      if (trgt_K == 1) {
+      if (trgt_K == 0) {
         // Diff w.r.t. push/pull
         for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXYZ += trgt_sC, out_ptr_NC += out_sC) {
           scalar_t trgt = *trgt_ptr_NCXYZ;
@@ -1428,7 +1768,6 @@ MONAI_NAMESPACE_DEVICE { // cuda
     scalar_t w10 = dx1 * dy0;
     scalar_t w01 = dx0 * dy1;
     scalar_t w11 = dx1 * dy1;
-    ;
 
     // Sign (/!\ compute sign before warping indices)
     int8_t sx1 = bound::sign(bound0, ix0 + 1, src_X);
@@ -1467,7 +1806,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
       scalar_t* trgt_ptr_NCXY = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY;
       scalar_t* src_ptr_NC = src_ptr + n * src_sN;
 
-      if (trgt_K == 1) {
+      if (trgt_K == 0) {
         // backward w.r.t. push/pull
         for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXY += trgt_sC, src_ptr_NC += src_sC) {
           scalar_t src;
@@ -1514,9 +1853,9 @@ MONAI_NAMESPACE_DEVICE { // cuda
         }
       }
 
-      scalar_t* grad_ptr_NXYZ = grad_ptr + n * grad_sN + w * grad_sX + h * grad_sY;
-      (*grad_ptr_NXYZ) = gx;
-      grad_ptr_NXYZ[grad_sC] = gy;
+      scalar_t* grad_ptr_NXY = grad_ptr + n * grad_sN + w * grad_sX + h * grad_sY;
+      (*grad_ptr_NXY) = gx;
+      grad_ptr_NXY[grad_sC] = gy;
     }
     // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pull ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     if (do_pull) {
@@ -1558,7 +1897,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
       o11 = ix1 * out_sX + iy1 * out_sY;
       scalar_t* trgt_ptr_NCXY = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY;
       scalar_t* out_ptr_NC = out_ptr + n * out_sN;
-      if (trgt_K == 1) {
+      if (trgt_K == 0) {
         // Diff w.r.t. push/pull
         for (offset_t c = 0; c < C; ++c, trgt_ptr_NCXY += trgt_sC, out_ptr_NC += out_sC) {
           scalar_t trgt = *trgt_ptr_NCXY;
@@ -1599,6 +1938,123 @@ MONAI_NAMESPACE_DEVICE { // cuda
     }
   }
 
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                     LINEAR INTERPOLATION 1D
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  template <typename scalar_t, typename offset_t>
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::interpolate1d_linear(scalar_t x, offset_t w, offset_t n) const {
+    // Get corner pixel values from (x)
+    offset_t ix0 = static_cast<offset_t>(std::floor(x));
+
+    // Interpolation weights (inversely proportional to distance)
+    scalar_t w1 = x - ix0;
+    scalar_t w0 = 1. - w1;
+
+    // Sign (/!\ compute sign before warping indices)
+    int8_t s1 = bound::sign(bound0, ix0 + 1, src_X);
+    int8_t s0 = bound::sign(bound0, ix0, src_X);
+
+    // Warp indices
+    offset_t ix1;
+    ix1 = bound::index(bound0, ix0 + 1, src_X);
+    ix0 = bound::index(bound0, ix0, src_X);
+
+    // Offsets into source volume
+    offset_t o0, o1;
+    if (do_pull || do_grad || do_sgrad) {
+      o0 = ix0 * src_sX;
+      o1 = ix1 * src_sX;
+    }
+
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~ Grid gradient ~~~~~~~~~~~~~~~~~~~~~~~~~~
+    if (do_grad) {
+      if (trgt_K == 0) {
+        // backward w.r.t. push/pull
+
+        o0 = ix0 * src_sX;
+        o1 = ix1 * src_sX;
+        scalar_t gx = static_cast<scalar_t>(0);
+        scalar_t* trgt_ptr_NCX = trgt_ptr + n * trgt_sN + w * trgt_sX;
+        scalar_t* src_ptr_NC = src_ptr + n * src_sN;
+
+        for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC, src_ptr_NC += src_sC) {
+          scalar_t src;
+          scalar_t trgt = trgt_ptr ? *trgt_ptr_NCX : static_cast<scalar_t>(1);
+          // ^ trgt_ptr == 0 during the backward pass of count
+          src = bound::get(src_ptr_NC, o0, s0);
+          if (trgt_ptr)
+            src *= trgt;
+          gx -= src;
+          src = bound::get(src_ptr_NC, o1, s1);
+          if (trgt_ptr)
+            src *= trgt;
+          gx += src;
+        }
+
+        scalar_t* grad_ptr_NX = grad_ptr + n * grad_sN + w * grad_sX;
+        (*grad_ptr_NX) = gx;
+      } else {
+        // backward w.r.t. sgrad
+        // -> zero (make sure this is done at initialization)
+      }
+    }
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Pull ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    if (do_pull) {
+      o0 = ix0 * src_sX;
+      o1 = ix1 * src_sX;
+      scalar_t* out_ptr_NCX = out_ptr + n * out_sN + w * out_sX;
+      scalar_t* src_ptr_NC = src_ptr + n * src_sN;
+      for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC) {
+        *out_ptr_NCX = bound::get(src_ptr_NC, o0, s0) * w0 + bound::get(src_ptr_NC, o1, s1) * w1;
+      }
+    }
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SGrad ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    else if (do_sgrad) {
+      o0 = ix0 * src_sX;
+      o1 = ix1 * src_sX;
+      scalar_t* out_ptr_NCX = out_ptr + n * out_sN + w * out_sX;
+      scalar_t* src_ptr_NC = src_ptr + n * src_sN;
+
+      for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC) {
+        *out_ptr_NCX = bound::get(src_ptr_NC, o1, s1) - bound::get(src_ptr_NC, o0, s0);
+      }
+    }
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    else if (do_push) {
+      // Offsets into 'push' volume
+      o0 = ix0 * out_sX;
+      o1 = ix1 * out_sX;
+      scalar_t* trgt_ptr_NCX = trgt_ptr + n * trgt_sN + w * trgt_sX;
+      scalar_t* out_ptr_NC = out_ptr + n * out_sN;
+      if (trgt_K == 0) {
+        // Diff w.r.t. push/pull
+        for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC, out_ptr_NC += out_sC) {
+          scalar_t trgt = *trgt_ptr_NCX;
+          bound::add(out_ptr_NC, o0, w0 * trgt, s0);
+          bound::add(out_ptr_NC, o1, w1 * trgt, s1);
+        }
+      } else {
+        // Diff w.r.t. sgrad
+        for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC, out_ptr_NC += out_sC) {
+          scalar_t trgt0 = *trgt_ptr_NCX;
+          bound::add(out_ptr_NC, o0, -trgt0, s0);
+          bound::add(out_ptr_NC, o1, trgt0, s1);
+        }
+      }
+    }
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Push ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    else if (do_count) {
+      // Offsets into 'push' volume
+      o0 = ix0 * out_sX;
+      o1 = ix1 * out_sX;
+
+      scalar_t* out_ptr_N = out_ptr + n * out_sN;
+      bound::add(out_ptr_N, o0, w0, s0);
+      bound::add(out_ptr_N, o1, w1, s1);
+    }
+  }
+
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   //                  NEAREST NEIGHBOR INTERPOLATION 3D
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1633,7 +2089,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
       scalar_t* src_ptr_NC = src_ptr + n * src_sN;
       for (offset_t c = 0; c < C; ++c, out_ptr_NCXYZ += out_sC, src_ptr_NC += src_sC)
         *out_ptr_NCXYZ = bound::get(src_ptr_NC, o, s);
-    } else if (do_push && trgt_K == 1) {
+    } else if (do_push && trgt_K == 0) {
       offset_t o = iz * out_sZ + iy * out_sY + ix * out_sX;
       scalar_t* trgt_ptr_NCXYZ = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY + d * trgt_sZ;
       scalar_t* out_ptr_NC = out_ptr + n * out_sN;
@@ -1676,7 +2132,7 @@ MONAI_NAMESPACE_DEVICE { // cuda
       scalar_t* src_ptr_NC = src_ptr + n * src_sN;
       for (offset_t c = 0; c < C; ++c, out_ptr_NCXY += out_sC, src_ptr_NC += src_sC)
         *out_ptr_NCXY = bound::get(src_ptr_NC, o, s);
-    } else if (do_push && trgt_K == 1) {
+    } else if (do_push && trgt_K == 0) {
       offset_t o = iy * out_sY + ix * out_sX;
       scalar_t* trgt_ptr_NCXY = trgt_ptr + n * trgt_sN + w * trgt_sX + h * trgt_sY;
       scalar_t* out_ptr_NC = out_ptr + n * out_sN;
@@ -1689,6 +2145,39 @@ MONAI_NAMESPACE_DEVICE { // cuda
         bound::add(out_ptr_NC, o, static_cast<scalar_t>(1), s);
     }
   }
+
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+  //                  NEAREST NEIGHBOR INTERPOLATION 1D
+  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  template <typename scalar_t, typename offset_t>
+  MONAI_DEVICE void PushPullImpl<scalar_t, offset_t>::interpolate1d_nearest(scalar_t x, offset_t w, offset_t n) const {
+    offset_t i = static_cast<offset_t>(std::round(x));
+
+    // Boundary condition (/!\ compute sign before warping indices)
+    int8_t s = bound::sign(bound0, i, src_X);
+    i = bound::index(bound0, i, src_X);
+
+    if (do_pull) {
+      offset_t o = i * src_sX;
+      scalar_t* out_ptr_NCX = out_ptr + n * out_sN + w * out_sX;
+      scalar_t* src_ptr_NC = src_ptr + n * src_sN;
+      for (offset_t c = 0; c < C; ++c, out_ptr_NCX += out_sC, src_ptr_NC += src_sC)
+        *out_ptr_NCX = bound::get(src_ptr_NC, o, s);
+    } else if (do_push && trgt_K == 0) {
+      offset_t o = i * out_sX;
+      scalar_t* trgt_ptr_NCX = trgt_ptr + n * trgt_sN + w * trgt_sX;
+      scalar_t* out_ptr_NC = out_ptr + n * out_sN;
+      for (offset_t c = 0; c < C; ++c, trgt_ptr_NCX += trgt_sC, out_ptr_NC += out_sC)
+        bound::add(out_ptr_NC, o, *trgt_ptr_NCX, s);
+    } else if (do_count) {
+      offset_t o = i * out_sX;
+      scalar_t* out_ptr_NC = out_ptr + n * out_sN;
+      for (offset_t c = 0; c < C; ++c, out_ptr_NC += out_sC)
+        bound::add(out_ptr_NC, o, static_cast<scalar_t>(1), s);
+    }
+  }
+
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   //            LINEAR INTERPOLATION 3D + SLIDING BOUNDARY
   // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1736,8 +2225,6 @@ MONAI_NAMESPACE_DEVICE { // cuda
   PUSHPULL_INSTANTIATE1(BoundType); \
   PUSHPULL_INSTANTIATE1(BoundVectorRef)
 
-  // ~~~ CUDA ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
   // Two arguments (source, grid)
   // > `bound` and `interpolation` can be single arguments or vectors.
   template <typename BoundType, typename InterpolationType, typename SourceType>
@@ -1752,12 +2239,20 @@ MONAI_NAMESPACE_DEVICE { // cuda
       bool do_count,
       bool do_grad,
       bool do_sgrad) {
+    PushPullAllocator info(
+        grid.dim() - 2, bound, interpolation, extrapolate, do_pull, do_push, do_count, do_grad, do_sgrad);
+    info.ioset(source, grid);
+
     return AT_DISPATCH_FLOATING_TYPES_AND_HALF(grid.scalar_type(), "pushpull", [&] {
-      PushPullImpl<scalar_t, int64_t> f(
-          grid.dim() - 2, bound, interpolation, extrapolate, do_pull, do_push, do_count, do_grad, do_sgrad);
-      f.ioset(source, grid);
-      pushpull_kernel<<<GET_BLOCKS(f.voxcount()), CUDA_NUM_THREADS, 0, at::cuda::getCurrentCUDAStream()>>>(f);
-      return f.output;
+      if (info.canUse32BitIndexMath()) {
+        PushPullImpl<scalar_t, int32_t> algo(info);
+        pushpull_kernel<<<GET_BLOCKS(algo.voxcount()), CUDA_NUM_THREADS, 0, at::cuda::getCurrentCUDAStream()>>>(algo);
+        return algo.output;
+      } else {
+        PushPullImpl<scalar_t, int64_t> algo(info);
+        pushpull_kernel<<<GET_BLOCKS(algo.voxcount()), CUDA_NUM_THREADS, 0, at::cuda::getCurrentCUDAStream()>>>(algo);
+        return algo.output;
+      }
     });
   }
 
@@ -1777,17 +2272,24 @@ MONAI_NAMESPACE_DEVICE { // cuda
       bool do_count,
       bool do_grad,
       bool do_sgrad) {
+    PushPullAllocator info(
+        grid.dim() - 2, bound, interpolation, extrapolate, do_pull, do_push, do_count, do_grad, do_sgrad);
+    info.ioset(source, grid, target);
+
     return AT_DISPATCH_FLOATING_TYPES_AND_HALF(grid.scalar_type(), "pushpull", [&] {
-      PushPullImpl<scalar_t, int64_t> f(
-          grid.dim() - 2, bound, interpolation, extrapolate, do_pull, do_push, do_count, do_grad, do_sgrad);
-      f.ioset(source, grid, target);
-      pushpull_kernel<<<GET_BLOCKS(f.voxcount()), CUDA_NUM_THREADS, 0, at::cuda::getCurrentCUDAStream()>>>(f);
-      return f.output;
+      if (info.canUse32BitIndexMath()) {
+        PushPullImpl<scalar_t, int32_t> algo(info);
+        pushpull_kernel<<<GET_BLOCKS(algo.voxcount()), CUDA_NUM_THREADS, 0, at::cuda::getCurrentCUDAStream()>>>(algo);
+        return algo.output;
+      } else {
+        PushPullImpl<scalar_t, int64_t> algo(info);
+        pushpull_kernel<<<GET_BLOCKS(algo.voxcount()), CUDA_NUM_THREADS, 0, at::cuda::getCurrentCUDAStream()>>>(algo);
+        return algo.output;
+      }
     });
   }
 
   PUSHPULL_INSTANTIATE;
 
-} // namespace <device>
-
+} // namespace gpu
 } // namespace monai
diff --git a/monai/csrc/utils/common_utils.h b/monai/csrc/utils/common_utils.h
index 882312acb3..4d09377e65 100644
--- a/monai/csrc/utils/common_utils.h
+++ b/monai/csrc/utils/common_utils.h
@@ -26,10 +26,10 @@ limitations under the License.
       value.layout() == at::kStrided,                                     \
       "(): expected " #value "to have torch.strided layout, but it has ", \
       value.layout());
-#define CHECK_SPATIAL_2D_OR_3D(value)                               \
-  TORCH_CHECK(                                                      \
-      (value.dim() == 4 || value.dim() == 5),                       \
-      "(): expected 4D or 5D " #value " but got input with sizes ", \
+#define CHECK_SPATIAL_1D_2D_OR_3D(value)                                \
+  TORCH_CHECK(                                                          \
+      (value.dim() == 3 || value.dim() == 4 || value.dim() == 5),       \
+      "(): expected 3D, 4D or 5D " #value " but got input with sizes ", \
       value.sizes());
 #define CHECK_GRID_COMPONENT(value, dim)           \
   TORCH_CHECK(                                     \
@@ -42,18 +42,18 @@ limitations under the License.
 #define CHECK_SAME_DEVICE(value1, value2)     \
   TORCH_CHECK(                                \
       value1.device() == value2.device(),     \
-      "(): expected " #value2 " and " #value2 \
+      "(): expected " #value1 " and " #value2 \
       " to be on same device, "               \
-      "but " #value2 " is on ",               \
+      "but " #value1 " is on ",               \
       value1.device(),                        \
       " and " #value2 " is on ",              \
       value2.device());
 #define CHECK_SAME_DTYPE(value1, value2)      \
   TORCH_CHECK(                                \
       value1.dtype() == value2.dtype(),       \
-      "(): expected " #value2 " and " #value2 \
+      "(): expected " #value1 " and " #value2 \
       " to have the same dtype, "             \
-      "but " #value2 " has ",                 \
+      "but " #value1 " has ",                 \
       value1.dtype(),                         \
       " and " #value2 " has ",                \
       value2.dtype());
@@ -67,14 +67,15 @@ limitations under the License.
         i,                                                                                    \
         " being empty");                                                                      \
   }
-#define CHECK_GRID_TARGET_COMPAT(value1, value2)                                                                  \
-  TORCH_CHECK(                                                                                                    \
-      value2.size(0) == value1.size(0) && value2.size(2) == value1.size(1) && value2.size(3) == value1.size(2) && \
-          (value2.dim() == 4 || value2.size(4) == value1.size(3)),                                                \
-      "(): expected " #value2 " and " #value1                                                                     \
-      " to have same batch, width, height and (optionally) depth sizes, but got " #value2 " with sizes ",         \
-      value2.sizes(),                                                                                             \
-      " and " #value1 " with sizes ",                                                                             \
+#define CHECK_GRID_TARGET_COMPAT(value1, value2)                                                          \
+  TORCH_CHECK(                                                                                            \
+      value2.size(0) == value1.size(0) && (value2.dim() <= 2 || value2.size(2) == value1.size(1)) &&      \
+          (value2.dim() <= 3 || value2.size(3) == value1.size(2)) &&                                      \
+          (value2.dim() <= 4 || value2.size(4) == value1.size(3)),                                        \
+      "(): expected " #value2 " and " #value1                                                             \
+      " to have same batch, width, height and (optionally) depth sizes, but got " #value2 " with sizes ", \
+      value2.sizes(),                                                                                     \
+      " and " #value1 " with sizes ",                                                                     \
       value1.sizes());
 #define CHECK_SPATIAL_LENGTH(value, dim) \
   TORCH_CHECK(((int64_t)(value.size()) == dim - 2), "(): expected ", dim, #value " elements but got ", value.size());
diff --git a/monai/csrc/utils/resample_utils.h b/monai/csrc/utils/resample_utils.h
index 4735d13ca1..bbdf258b4c 100644
--- a/monai/csrc/utils/resample_utils.h
+++ b/monai/csrc/utils/resample_utils.h
@@ -62,7 +62,9 @@ namespace monai {
 template <typename scalar_t, typename offset_t>
 static inline void cpuAtomicAdd(scalar_t* ptr, offset_t offset, scalar_t value) {
 #if AT_PARALLEL_OPENMP
+#if _OPENMP
 #pragma omp atomic
+#endif
 #endif
   ptr[offset] += value;
 }
diff --git a/monai/data/__init__.py b/monai/data/__init__.py
index e0db1e17ae..adb27a608e 100644
--- a/monai/data/__init__.py
+++ b/monai/data/__init__.py
@@ -17,27 +17,31 @@
     CacheNTransDataset,
     Dataset,
     LMDBDataset,
+    NPZDictItemDataset,
     PersistentDataset,
     SmartCacheDataset,
     ZipDataset,
 )
 from .decathlon_datalist import load_decathlon_datalist, load_decathlon_properties
-from .grid_dataset import GridPatchDataset, PatchDataset
+from .grid_dataset import GridPatchDataset, PatchDataset, PatchIter
 from .image_dataset import ImageDataset
-from .image_reader import ImageReader, ITKReader, NibabelReader, NumpyReader, PILReader
+from .image_reader import ImageReader, ITKReader, NibabelReader, NumpyReader, PILReader, WSIReader
+from .inverse_batch_transform import BatchInverseTransform
 from .iterable_dataset import IterableDataset
 from .nifti_saver import NiftiSaver
 from .nifti_writer import write_nifti
 from .png_saver import PNGSaver
 from .png_writer import write_png
+from .samplers import DistributedSampler, DistributedWeightedRandomSampler
 from .synthetic import create_test_image_2d, create_test_image_3d
-from .thread_buffer import ThreadBuffer
+from .test_time_augmentation import TestTimeAugmentation
+from .thread_buffer import ThreadBuffer, ThreadDataLoader
 from .utils import (
-    DistributedSampler,
     compute_importance_map,
     compute_shape_offset,
     correct_nifti_header_if_necessary,
     create_file_basename,
+    decollate_batch,
     dense_patch_slices,
     get_random_patch,
     get_valid_patch_size,
@@ -46,6 +50,7 @@
     iter_patch_slices,
     json_hashing,
     list_data_collate,
+    pad_list_data_collate,
     partition_dataset,
     partition_dataset_classes,
     pickle_hashing,
diff --git a/monai/data/dataset.py b/monai/data/dataset.py
index b93f03151f..a09050e5bc 100644
--- a/monai/data/dataset.py
+++ b/monai/data/dataset.py
@@ -10,6 +10,7 @@
 # limitations under the License.
 
 
+import collections.abc
 import math
 import pickle
 import sys
@@ -19,12 +20,14 @@
 from copy import deepcopy
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, List, Optional, Sequence, Union
+from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Union
 
+import numpy as np
 import torch
 from torch.utils.data import Dataset as _TorchDataset
+from torch.utils.data import Subset
 
-from monai.data.utils import pickle_hashing
+from monai.data.utils import first, pickle_hashing
 from monai.transforms import Compose, Randomizable, Transform, apply_transform
 from monai.utils import MAX_SEED, get_seed, min_version, optional_import
 
@@ -42,6 +45,9 @@ class Dataset(_TorchDataset):
     """
     A generic dataset with a length property and an optional callable data transform
     when fetching a data sample.
+    If passing slicing indices, will return a PyTorch Subset, for example: `data: Subset = dataset[1:4]`,
+    for more details, please check: https://pytorch.org/docs/stable/data.html#torch.utils.data.Subset
+
     For example, typical input data can be a list of dictionaries::
 
         [{                            {                            {
@@ -51,26 +57,39 @@ class Dataset(_TorchDataset):
          },                           },                           }]
     """
 
-    def __init__(self, data: Sequence, transform: Optional[Callable] = None, progress: bool = True) -> None:
+    def __init__(self, data: Sequence, transform: Optional[Callable] = None) -> None:
         """
         Args:
             data: input data to load and transform to generate dataset for model.
             transform: a callable data transform on input data.
-            progress: whether to display a progress bar.
+
         """
         self.data = data
         self.transform = transform
-        self.progress = progress
 
     def __len__(self) -> int:
         return len(self.data)
 
-    def __getitem__(self, index: int):
-        data = self.data[index]
-        if self.transform is not None:
-            data = apply_transform(self.transform, data)
+    def _transform(self, index: int):
+        """
+        Fetch single data item from `self.data`.
+        """
+        data_i = self.data[index]
+        return apply_transform(self.transform, data_i) if self.transform is not None else data_i
 
-        return data
+    def __getitem__(self, index: Union[int, slice, Sequence[int]]):
+        """
+        Returns a `Subset` if `index` is a slice or Sequence, a data item otherwise.
+        """
+        if isinstance(index, slice):
+            # dataset[:42]
+            start, stop, step = index.indices(len(self))
+            indices = range(start, stop, step)
+            return Subset(dataset=self, indices=indices)
+        if isinstance(index, collections.abc.Sequence):
+            # dataset[[1, 3, 4]]
+            return Subset(dataset=self, indices=index)
+        return self._transform(index)
 
 
 class PersistentDataset(Dataset):
@@ -78,6 +97,8 @@ class PersistentDataset(Dataset):
     Persistent storage of pre-computed values to efficiently manage larger than memory dictionary format data,
     it can operate transforms for specific fields.  Results from the non-random transform components are computed
     when first used, and stored in the `cache_dir` for rapid retrieval on subsequent uses.
+    If passing slicing indices, will return a PyTorch Subset, for example: `data: Subset = dataset[1:4]`,
+    for more details, please check: https://pytorch.org/docs/stable/data.html#torch.utils.data.Subset
 
     For example, typical input data can be a list of dictionaries::
 
@@ -117,7 +138,6 @@ def __init__(
         transform: Union[Sequence[Callable], Callable],
         cache_dir: Optional[Union[Path, str]] = None,
         hash_func: Callable[..., bytes] = pickle_hashing,
-        progress: bool = True,
     ) -> None:
         """
         Args:
@@ -132,11 +152,11 @@ def __init__(
                 If the cache_dir doesn't exist, will automatically create it.
             hash_func: a callable to compute hash from data items to be cached.
                 defaults to `monai.data.utils.pickle_hashing`.
-            progress: whether to display a progress bar.
+
         """
         if not isinstance(transform, Compose):
             transform = Compose(transform)
-        super().__init__(data=data, transform=transform, progress=progress)
+        super().__init__(data=data, transform=transform)
         self.cache_dir = Path(cache_dir) if cache_dir is not None else None
         self.hash_func = hash_func
         if self.cache_dir is not None:
@@ -228,7 +248,7 @@ def _cachecheck(self, item_transformed):
             temp_hash_file.rename(hashfile)
         return _item_transformed
 
-    def __getitem__(self, index: int):
+    def _transform(self, index: int):
         pre_random_item = self._cachecheck(self.data[index])
         return self._post_transform(pre_random_item)
 
@@ -349,7 +369,8 @@ def __init__(
             lmdb_kwargs: additional keyword arguments to the lmdb environment.
                 for more details please visit: https://lmdb.readthedocs.io/en/release/#environment-class
         """
-        super().__init__(data=data, transform=transform, cache_dir=cache_dir, hash_func=hash_func, progress=progress)
+        super().__init__(data=data, transform=transform, cache_dir=cache_dir, hash_func=hash_func)
+        self.progress = progress
         if not self.cache_dir:
             raise ValueError("cache_dir must be specified.")
         self.db_file = self.cache_dir / f"{db_name}.lmdb"
@@ -445,6 +466,8 @@ class CacheDataset(Dataset):
 
     To improve the caching efficiency, please always put as many as possible non-random transforms
     before the randomized ones when composing the chain of transforms.
+    If passing slicing indices, will return a PyTorch Subset, for example: `data: Subset = dataset[1:4]`,
+    for more details, please check: https://pytorch.org/docs/stable/data.html#torch.utils.data.Subset
 
     For example, if the transform is a `Compose` of::
 
@@ -489,7 +512,8 @@ def __init__(
         """
         if not isinstance(transform, Compose):
             transform = Compose(transform)
-        super().__init__(data=data, transform=transform, progress=progress)
+        super().__init__(data=data, transform=transform)
+        self.progress = progress
         self.cache_num = min(int(cache_num), int(len(data) * cache_rate), len(data))
         self.num_workers = num_workers
         if self.num_workers is not None:
@@ -527,10 +551,10 @@ def _load_cache_item(self, idx: int):
             item = apply_transform(_transform, item)
         return item
 
-    def __getitem__(self, index):
-        if index >= self.cache_num:
+    def _transform(self, index: int):
+        if index % len(self) >= self.cache_num:  # support negative index
             # no cache for this index, execute all the transforms directly
-            return super(CacheDataset, self).__getitem__(index)
+            return super()._transform(index)
         # load data from cache and execute from the first random transform
         start_run = False
         if self._cache is None:
@@ -545,7 +569,7 @@ def __getitem__(self, index):
         return data
 
 
-class SmartCacheDataset(CacheDataset):
+class SmartCacheDataset(Randomizable, CacheDataset):
     """
     Re-implementation of the SmartCache mechanism in NVIDIA Clara-train SDK.
     At any time, the cache pool only keeps a subset of the whole dataset. In each epoch, only the items
@@ -559,6 +583,8 @@ class SmartCacheDataset(CacheDataset):
     where r is the configured replace rate).
     For more details, please refer to:
     https://docs.nvidia.com/clara/tlt-mi/clara-train-sdk-v3.0/nvmidl/additional_features/smart_cache.html#smart-cache
+    If passing slicing indices, will return a PyTorch Subset, for example: `data: Subset = dataset[1:4]`,
+    for more details, please check: https://pytorch.org/docs/stable/data.html#torch.utils.data.Subset
 
     For example, if we have 5 images: `[image1, image2, image3, image4, image5]`, and `cache_num=4`, `replace_rate=0.25`.
     so the actual training images cached and replaced for every epoch are as below::
@@ -580,6 +606,24 @@ class SmartCacheDataset(CacheDataset):
         This replacement will not work if setting the `multiprocessing_context` of DataLoader to `spawn`
         or on windows(the default multiprocessing method is `spawn`) and setting `num_workers` greater than 0.
 
+        If using MONAI workflows, please add `SmartCacheHandler` to the handler list of trainer,
+        otherwise, please make sure to call `start()`, `update_cache()`, `shutdown()` during training.
+
+    Args:
+        data: input data to load and transform to generate dataset for model.
+        transform: transforms to execute operations on input data.
+        replace_rate: percentage of the cached items to be replaced in every epoch.
+        cache_num: number of items to be cached. Default is `sys.maxsize`.
+            will take the minimum of (cache_num, data_length x cache_rate, data_length).
+        cache_rate: percentage of cached data in total, default is 1.0 (cache all).
+            will take the minimum of (cache_num, data_length x cache_rate, data_length).
+        num_init_workers: the number of worker threads to initialize the cache for first epoch.
+            If num_init_workers is None then the number returned by os.cpu_count() is used.
+        num_replace_workers: the number of worker threads to prepare the replacement cache for every epoch.
+            If num_replace_workers is None then the number returned by os.cpu_count() is used.
+        progress: whether to display a progress bar when caching for the first epoch.
+        shuffle: whether to shuffle the whole data list before preparing the cache content for first epoch.
+        seed: random seed if shuffle is `True`, default to `0`.
     """
 
     def __init__(
@@ -590,30 +634,26 @@ def __init__(
         cache_num: int = sys.maxsize,
         cache_rate: float = 1.0,
         num_init_workers: Optional[int] = None,
-        num_replace_workers: int = 0,
+        num_replace_workers: Optional[int] = None,
+        progress: bool = True,
+        shuffle: bool = True,
+        seed: int = 0,
     ) -> None:
-        """
-        Args:
-            data: input data to load and transform to generate dataset for model.
-            transform: transforms to execute operations on input data.
-            replace_rate: percentage of the cached items to be replaced in every epoch.
-            cache_num: number of items to be cached. Default is `sys.maxsize`.
-                will take the minimum of (cache_num, data_length x cache_rate, data_length).
-            cache_rate: percentage of cached data in total, default is 1.0 (cache all).
-                will take the minimum of (cache_num, data_length x cache_rate, data_length).
-            num_init_workers: the number of worker threads to initialize the cache for first epoch.
-                If num_init_workers is None then the number returned by os.cpu_count() is used.
-            num_replace_workers: the number of worker threads to prepare the replacement cache for every epoch.
-                if 0, run in main thread, no separate thread will open.
-        """
-        super().__init__(data, transform, cache_num, cache_rate, num_init_workers)
+        if shuffle:
+            self.set_random_state(seed=seed)
+            self.randomize(data)
+
+        super().__init__(data, transform, cache_num, cache_rate, num_init_workers, progress)
         if self._cache is None:
             self._cache = self._fill_cache()
         if self.cache_num >= len(data):
             warnings.warn("cache_num is greater or equal than dataset length, fall back to regular CacheDataset.")
         if replace_rate <= 0:
             raise ValueError("replace_rate must be greater than 0, otherwise, please use CacheDataset.")
-        self.num_replace_workers: int = num_replace_workers
+
+        self.num_replace_workers: Optional[int] = num_replace_workers
+        if self.num_replace_workers is not None:
+            self.num_replace_workers = max(int(self.num_replace_workers), 1)
 
         self._total_num: int = len(data)
         self._replace_num: int = min(math.ceil(self.cache_num * replace_rate), len(data) - self.cache_num)
@@ -628,6 +668,12 @@ def __init__(
 
         self._compute_data_idx()
 
+    def randomize(self, data: Sequence) -> None:
+        try:
+            self.R.shuffle(data)
+        except TypeError as e:
+            warnings.warn(f"input data can't be shuffled in SmartCacheDataset with numpy.random.shuffle(): {e}.")
+
     def _compute_data_idx(self):
         """
         Update the replacement data position in the total data.
@@ -743,12 +789,9 @@ def _compute_replacements(self):
         It can support multi-threads to accelerate the computation progress.
 
         """
-        if self.num_replace_workers > 0:
-            with ThreadPool(self.num_replace_workers) as p:
-                p.map(self._replace_cache_thread, list(range(self._replace_num)))
-        else:
-            for i in range(self._replace_num):
-                self._replace_cache_thread(i)
+        with ThreadPool(self.num_replace_workers) as p:
+            p.map(self._replace_cache_thread, list(range(self._replace_num)))
+
         self._replace_done = True
 
     def _try_manage_replacement(self, check_round):
@@ -793,6 +836,8 @@ class ZipDataset(Dataset):
     finally return (img, imgmeta, seg, segmeta).
     And if the datasets don't have same length, use the minimum length of them as the length
     of ZipDataset.
+    If passing slicing indices, will return a PyTorch Subset, for example: `data: Subset = dataset[1:4]`,
+    for more details, please check: https://pytorch.org/docs/stable/data.html#torch.utils.data.Subset
 
     Examples::
 
@@ -817,7 +862,7 @@ def __init__(self, datasets: Sequence, transform: Optional[Callable] = None) ->
     def __len__(self) -> int:
         return min((len(dataset) for dataset in self.data))
 
-    def __getitem__(self, index: int):
+    def _transform(self, index: int):
         def to_list(x):
             return list(x) if isinstance(x, (tuple, list)) else [x]
 
@@ -927,3 +972,55 @@ def __getitem__(self, index: int):
         if isinstance(transform, Randomizable):
             transform.set_random_state(seed=self._seed)
         return self.dataset[index]
+
+
+class NPZDictItemDataset(Dataset):
+    """
+    Represents a dataset from a loaded NPZ file. The members of the file to load are named in the keys of `keys` and
+    stored under the keyed name. All loaded arrays must have the same 0-dimension (batch) size. Items are always dicts
+    mapping names to an item extracted from the loaded arrays.
+    If passing slicing indices, will return a PyTorch Subset, for example: `data: Subset = dataset[1:4]`,
+    for more details, please check: https://pytorch.org/docs/stable/data.html#torch.utils.data.Subset
+
+    Args:
+        npzfile: Path to .npz file or stream containing .npz file data
+        keys: Maps keys to load from file to name to store in dataset
+        transform: Transform to apply to batch dict
+        other_keys: secondary data to load from file and store in dict `other_keys`, not returned by __getitem__
+    """
+
+    def __init__(
+        self,
+        npzfile: Union[str, IO],
+        keys: Dict[str, str],
+        transform: Optional[Callable] = None,
+        other_keys: Optional[Sequence[str]] = (),
+    ):
+        self.npzfile: Union[str, IO] = npzfile if isinstance(npzfile, str) else "STREAM"
+        self.keys: Dict[str, str] = dict(keys)
+        dat = np.load(npzfile)
+
+        self.arrays = {storedk: dat[datak] for datak, storedk in self.keys.items()}
+        self.length = self.arrays[first(self.keys.values())].shape[0]
+
+        self.other_keys = {} if other_keys is None else {k: dat[k] for k in other_keys}
+
+        for k, v in self.arrays.items():
+            if v.shape[0] != self.length:
+                raise ValueError(
+                    "All loaded arrays must have the same first dimension "
+                    f"size {self.length}, array `{k}` has size {v.shape[0]}"
+                )
+
+        super().__init__([], transform)
+
+    def __len__(self):
+        return self.length
+
+    def _transform(self, index: int):
+        data = {k: v[index] for k, v in self.arrays.items()}
+
+        if self.transform is not None:
+            data = apply_transform(self.transform, data)
+
+        return data
diff --git a/monai/data/decathlon_datalist.py b/monai/data/decathlon_datalist.py
index 6167e83e47..11fb5edd28 100644
--- a/monai/data/decathlon_datalist.py
+++ b/monai/data/decathlon_datalist.py
@@ -17,34 +17,43 @@
 
 
 @overload
-def _compute_path(base_dir: str, element: str) -> str:
+def _compute_path(base_dir: str, element: str, check_path: bool = False) -> str:
     ...
 
 
 @overload
-def _compute_path(base_dir: str, element: List[str]) -> List[str]:
+def _compute_path(base_dir: str, element: List[str], check_path: bool = False) -> List[str]:
     ...
 
 
-def _compute_path(base_dir, element):
+def _compute_path(base_dir, element, check_path=False):
     """
     Args:
         base_dir: the base directory of the dataset.
         element: file path(s) to append to directory.
+        check_path: if `True`, only compute when the result is an existing path.
 
     Raises:
         TypeError: When ``element`` contains a non ``str``.
         TypeError: When ``element`` type is not in ``Union[list, str]``.
 
     """
+
+    def _join_path(base_dir: str, item: str):
+        result = os.path.normpath(os.path.join(base_dir, item))
+        if check_path and not os.path.exists(result):
+            # if not an existing path, don't join with base dir
+            return item
+        return result
+
     if isinstance(element, str):
-        return os.path.normpath(os.path.join(base_dir, element))
+        return _join_path(base_dir, element)
     if isinstance(element, list):
         for e in element:
             if not isinstance(e, str):
-                raise TypeError(f"Every file path in element must be a str but got {type(element).__name__}.")
-        return [os.path.normpath(os.path.join(base_dir, e)) for e in element]
-    raise TypeError(f"element must be one of (str, list) but is {type(element).__name__}.")
+                return element
+        return [_join_path(base_dir, e) for e in element]
+    return element
 
 
 def _append_paths(base_dir: str, is_segmentation: bool, items: List[Dict]) -> List[Dict]:
@@ -63,9 +72,12 @@ def _append_paths(base_dir: str, is_segmentation: bool, items: List[Dict]) -> Li
             raise TypeError(f"Every item in items must be a dict but got {type(item).__name__}.")
         for k, v in item.items():
             if k == "image":
-                item[k] = _compute_path(base_dir, v)
+                item[k] = _compute_path(base_dir, v, check_path=False)
             elif is_segmentation and k == "label":
-                item[k] = _compute_path(base_dir, v)
+                item[k] = _compute_path(base_dir, v, check_path=False)
+            else:
+                # for other items, auto detect whether it's a valid path
+                item[k] = _compute_path(base_dir, v, check_path=True)
     return items
 
 
diff --git a/monai/data/grid_dataset.py b/monai/data/grid_dataset.py
index f85569d88a..b789b9c032 100644
--- a/monai/data/grid_dataset.py
+++ b/monai/data/grid_dataset.py
@@ -9,9 +9,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import math
 from typing import Callable, Dict, Optional, Sequence, Union
 
+import numpy as np
 import torch
 from torch.utils.data import IterableDataset
 
@@ -20,31 +20,25 @@
 from monai.transforms import apply_transform
 from monai.utils import NumpyPadMode, ensure_tuple
 
-__all__ = ["PatchDataset", "GridPatchDataset"]
+__all__ = ["PatchDataset", "GridPatchDataset", "PatchIter"]
 
 
-class GridPatchDataset(IterableDataset):
+class PatchIter:
     """
-    Yields patches from arrays read from an input dataset. The patches are chosen in a contiguous grid sampling scheme.
+    A class to return a patch generator with predefined properties such as `patch_size`.
+    Typically used with :py:class:`monai.data.GridPatchDataset`.
     """
 
     def __init__(
         self,
-        dataset: Sequence,
         patch_size: Sequence[int],
         start_pos: Sequence[int] = (),
         mode: Union[NumpyPadMode, str] = NumpyPadMode.WRAP,
         **pad_opts: Dict,
-    ) -> None:
+    ):
         """
-        Initializes this dataset in terms of the input dataset and patch size. The `patch_size` is the size of the
-        patch to sample from the input arrays. It is assumed the arrays first dimension is the channel dimension which
-        will be yielded in its entirety so this should not be specified in `patch_size`. For example, for an input 3D
-        array with 1 channel of size (1, 20, 20, 20) a regular grid sampling of eight patches (1, 10, 10, 10) would be
-        specified by a `patch_size` of (10, 10, 10).
 
         Args:
-            dataset: the dataset to read array data from
             patch_size: size of patches to generate slices for, 0/None selects whole dimension
             start_pos: starting position in the array, default is 0 for each dimension
             mode: {``"constant"``, ``"edge"``, ``"linear_ramp"``, ``"maximum"``, ``"mean"``,
@@ -52,32 +46,123 @@ def __init__(
                 One of the listed string values or a user supplied function. Defaults to ``"wrap"``.
                 See also: https://numpy.org/doc/1.18/reference/generated/numpy.pad.html
             pad_opts: padding options, see numpy.pad
-        """
 
-        self.dataset = dataset
+        Note:
+            The `patch_size` is the size of the
+            patch to sample from the input arrays. It is assumed the arrays first dimension is the channel dimension which
+            will be yielded in its entirety so this should not be specified in `patch_size`. For example, for an input 3D
+            array with 1 channel of size (1, 20, 20, 20) a regular grid sampling of eight patches (1, 10, 10, 10) would be
+            specified by a `patch_size` of (10, 10, 10).
+
+        """
         self.patch_size = (None,) + tuple(patch_size)
         self.start_pos = ensure_tuple(start_pos)
         self.mode: NumpyPadMode = NumpyPadMode(mode)
         self.pad_opts = pad_opts
 
+    def __call__(self, array):
+        """
+        Args:
+            array: the image to generate patches from.
+        """
+        yield from iter_patch(
+            array,
+            patch_size=self.patch_size,  # expand to have the channel dim
+            start_pos=self.start_pos,
+            copy_back=False,
+            mode=self.mode,
+            **self.pad_opts,
+        )
+
+
+class GridPatchDataset(IterableDataset):
+    """
+    Yields patches from images read from an image dataset.
+    Typically used with `PatchIter` so that the patches are chosen in a contiguous grid sampling scheme.
+
+     .. code-block:: python
+
+        import numpy as np
+
+        from monai.data import GridPatchDataset, DataLoader, PatchIter
+        from monai.transforms import RandShiftIntensity
+
+        # image-level dataset
+        images = [np.arange(16, dtype=float).reshape(1, 4, 4),
+                  np.arange(16, dtype=float).reshape(1, 4, 4)]
+        # image-level patch generator, "grid sampling"
+        patch_iter = PatchIter(patch_size=(2, 2), start_pos=(0, 0))
+        # patch-level intensity shifts
+        patch_intensity = RandShiftIntensity(offsets=1.0, prob=1.0)
+
+        # construct the dataset
+        ds = GridPatchDataset(dataset=images,
+                              patch_iter=patch_iter,
+                              transform=patch_intensity)
+        # use the grid patch dataset
+        for item in DataLoader(ds, batch_size=2, num_workers=2):
+            print("patch size:", item[0].shape)
+            print("coordinates:", item[1])
+
+        # >>> patch size: torch.Size([2, 1, 2, 2])
+        #     coordinates: tensor([[[0, 1], [0, 2], [0, 2]],
+        #                          [[0, 1], [2, 4], [0, 2]]])
+
+    """
+
+    def __init__(
+        self,
+        dataset: Sequence,
+        patch_iter: Callable,
+        transform: Optional[Callable] = None,
+        with_coordinates: bool = True,
+    ) -> None:
+        """
+        Initializes this dataset in terms of the image dataset, patch generator, and an optional transform.
+
+        Args:
+            dataset: the dataset to read image data from.
+            patch_iter: converts an input image (item from dataset) into a iterable of image patches.
+                `patch_iter(dataset[idx])` must yield a tuple: (patches, coordinates).
+                see also: :py:class:`monai.data.PatchIter`.
+            transform: a callable data transform operates on the patches.
+            with_coordinates: whether to yield the coordinates of each patch, default to `True`.
+
+        """
+
+        self.dataset = dataset
+        self.patch_iter = patch_iter
+        self.transform = transform
+        self.with_coordinates = with_coordinates
+
     def __iter__(self):
         worker_info = torch.utils.data.get_worker_info()
-        iter_start = 0
-        iter_end = len(self.dataset)
+        iter_start, iter_end = 0, 1
+        try:
+            iter_end = len(self.dataset)  # TODO: support iterable self.dataset
+        except TypeError:
+            raise NotImplementedError("image dataset must implement `len()`.")
 
         if worker_info is not None:
             # split workload
-            per_worker = int(math.ceil((iter_end - iter_start) / float(worker_info.num_workers)))
-            worker_id = worker_info.id
-            iter_start = iter_start + worker_id * per_worker
+            per_worker = int(np.ceil((iter_end - iter_start) / float(worker_info.num_workers)))
+            iter_start = iter_start + worker_info.id * per_worker
             iter_end = min(iter_start + per_worker, iter_end)
 
         for index in range(iter_start, iter_end):
-            arrays = self.dataset[index]
-
-            iters = [iter_patch(a, self.patch_size, self.start_pos, False, self.mode, **self.pad_opts) for a in arrays]
-
-            yield from zip(*iters)
+            image = self.dataset[index]
+            if not self.with_coordinates:
+                for patch, *_ in self.patch_iter(image):  # patch_iter to yield at least 1 item: patch
+                    out_patch = (
+                        patch if self.transform is None else apply_transform(self.transform, patch, map_items=False)
+                    )
+                    yield out_patch
+            else:
+                for patch, slices, *_ in self.patch_iter(image):  # patch_iter to yield at least 2 items: patch, coords
+                    out_patch = (
+                        patch if self.transform is None else apply_transform(self.transform, patch, map_items=False)
+                    )
+                    yield out_patch, slices
 
 
 class PatchDataset(Dataset):
@@ -95,8 +180,8 @@ class PatchDataset(Dataset):
         from monai.transforms import RandSpatialCropSamples, RandShiftIntensity
 
         # image dataset
-        images = [np.arange(16, dtype=np.float).reshape(1, 4, 4),
-                  np.arange(16, dtype=np.float).reshape(1, 4, 4)]
+        images = [np.arange(16, dtype=float).reshape(1, 4, 4),
+                  np.arange(16, dtype=float).reshape(1, 4, 4)]
         # image patch sampler
         n_samples = 5
         sampler = RandSpatialCropSamples(roi_size=(3, 3), num_samples=n_samples,
@@ -142,7 +227,7 @@ def __init__(
     def __len__(self) -> int:
         return len(self.data) * self.samples_per_image
 
-    def __getitem__(self, index: int):
+    def _transform(self, index: int):
         image_id = int(index / self.samples_per_image)
         image = self.data[image_id]
         patches = self.patch_func(image)
diff --git a/monai/data/image_reader.py b/monai/data/image_reader.py
index e458833979..08aa6c6bbf 100644
--- a/monai/data/image_reader.py
+++ b/monai/data/image_reader.py
@@ -19,26 +19,31 @@
 
 from monai.config import DtypeLike, KeysCollection
 from monai.data.utils import correct_nifti_header_if_necessary
-from monai.utils import ensure_tuple, optional_import
+from monai.transforms.utility.array import EnsureChannelFirst
+from monai.utils import ensure_tuple, ensure_tuple_rep, optional_import
 
 from .utils import is_supported_format
 
 if TYPE_CHECKING:
+    import cucim
     import itk  # type: ignore
     import nibabel as nib
+    import openslide
     from itk import Image  # type: ignore
     from nibabel.nifti1 import Nifti1Image
     from PIL import Image as PILImage
 
-    has_itk = has_nib = has_pil = True
+    has_itk = has_nib = has_pil = has_cim = has_osl = True
 else:
     itk, has_itk = optional_import("itk", allow_namespace_pkg=True)
     Image, _ = optional_import("itk", allow_namespace_pkg=True, name="Image")
     nib, has_nib = optional_import("nibabel")
     Nifti1Image, _ = optional_import("nibabel.nifti1", name="Nifti1Image")
     PILImage, has_pil = optional_import("PIL.Image")
+    cucim, has_cim = optional_import("cucim")
+    openslide, has_osl = optional_import("openslide")
 
-__all__ = ["ImageReader", "ITKReader", "NibabelReader", "NumpyReader", "PILReader"]
+__all__ = ["ImageReader", "ITKReader", "NibabelReader", "NumpyReader", "PILReader", "WSIReader"]
 
 
 class ImageReader(ABC):
@@ -109,6 +114,17 @@ def _copy_compatible_dict(from_dict: Dict, to_dict: Dict):
             )
 
 
+def _stack_images(image_list: List, meta_dict: Dict):
+    if len(image_list) > 1:
+        if meta_dict.get("original_channel_dim", None) not in ("no_channel", None):
+            raise RuntimeError("can not read a list of images which already have channel dimension.")
+        meta_dict["original_channel_dim"] = 0
+        img_array = np.stack(image_list, axis=0)
+    else:
+        img_array = image_list[0]
+    return img_array
+
+
 class ITKReader(ImageReader):
     """
     Load medical images based on ITK library.
@@ -200,11 +216,12 @@ def get_data(self, img):
             header["original_affine"] = self._get_affine(i)
             header["affine"] = header["original_affine"].copy()
             header["spatial_shape"] = self._get_spatial_shape(i)
-            img_array.append(self._get_array_data(i))
+            data = self._get_array_data(i)
+            img_array.append(data)
+            header["original_channel_dim"] = "no_channel" if len(data.shape) == len(header["spatial_shape"]) else -1
             _copy_compatible_dict(header, compatible_meta)
 
-        img_array_ = np.stack(img_array, axis=0) if len(img_array) > 1 else img_array[0]
-        return img_array_, compatible_meta
+        return _stack_images(img_array, compatible_meta), compatible_meta
 
     def _get_meta_dict(self, img) -> Dict:
         """
@@ -236,7 +253,7 @@ def _get_meta_dict(self, img) -> Dict:
         meta_dict["direction"] = itk.array_from_matrix(img.GetDirection())
         return meta_dict
 
-    def _get_affine(self, img) -> np.ndarray:
+    def _get_affine(self, img):
         """
         Get or construct the affine matrix of the image, it can be used to correct
         spacing, orientation or execute spatial transforms.
@@ -252,12 +269,12 @@ def _get_affine(self, img) -> np.ndarray:
         origin = np.asarray(img.GetOrigin())
 
         direction = np.asarray(direction)
-        affine = np.eye(direction.shape[0] + 1)
+        affine: np.ndarray = np.eye(direction.shape[0] + 1)
         affine[(slice(-1), slice(-1))] = direction @ np.diag(spacing)
         affine[(slice(-1), -1)] = origin
-        return np.asarray(affine)
+        return affine
 
-    def _get_spatial_shape(self, img) -> np.ndarray:
+    def _get_spatial_shape(self, img):
         """
         Get the spatial shape of image data, it doesn't contain the channel dim.
 
@@ -265,6 +282,7 @@ def _get_spatial_shape(self, img) -> np.ndarray:
             img: a ITK image object loaded from a image file.
 
         """
+        # the img data should have no channel dim or the last dim is channel
         shape = list(itk.size(img))
         shape.reverse()
         return np.asarray(shape)
@@ -371,11 +389,12 @@ def get_data(self, img):
                 i = nib.as_closest_canonical(i)
                 header["affine"] = self._get_affine(i)
             header["spatial_shape"] = self._get_spatial_shape(i)
-            img_array.append(self._get_array_data(i))
+            data = self._get_array_data(i)
+            img_array.append(data)
+            header["original_channel_dim"] = "no_channel" if len(data.shape) == len(header["spatial_shape"]) else -1
             _copy_compatible_dict(header, compatible_meta)
 
-        img_array_ = np.stack(img_array, axis=0) if len(img_array) > 1 else img_array[0]
-        return img_array_, compatible_meta
+        return _stack_images(img_array, compatible_meta), compatible_meta
 
     def _get_meta_dict(self, img) -> Dict:
         """
@@ -387,7 +406,7 @@ def _get_meta_dict(self, img) -> Dict:
         """
         return dict(img.header)
 
-    def _get_affine(self, img) -> np.ndarray:
+    def _get_affine(self, img):
         """
         Get the affine matrix of the image, it can be used to correct
         spacing, orientation or execute spatial transforms.
@@ -398,7 +417,7 @@ def _get_affine(self, img) -> np.ndarray:
         """
         return np.array(img.affine, copy=True)
 
-    def _get_spatial_shape(self, img) -> np.ndarray:
+    def _get_spatial_shape(self, img):
         """
         Get the spatial shape of image data, it doesn't contain the channel dim.
 
@@ -408,9 +427,10 @@ def _get_spatial_shape(self, img) -> np.ndarray:
         """
         ndim = img.header["dim"][0]
         spatial_rank = min(ndim, 3)
+        # the img data should have no channel dim or the last dim is channel
         return np.asarray(img.header["dim"][1 : spatial_rank + 1])
 
-    def _get_array_data(self, img) -> np.ndarray:
+    def _get_array_data(self, img):
         """
         Get the raw array data of the image, converted to Numpy array.
 
@@ -504,12 +524,12 @@ def get_data(self, img):
         for i in ensure_tuple(img):
             header = {}
             if isinstance(i, np.ndarray):
+                # can not detect the channel dim of numpy array, use all the dims as spatial_shape
                 header["spatial_shape"] = i.shape
             img_array.append(i)
             _copy_compatible_dict(header, compatible_meta)
 
-        img_array_ = np.stack(img_array, axis=0) if len(img_array) > 1 else img_array[0]
-        return img_array_, compatible_meta
+        return _stack_images(img_array, compatible_meta), compatible_meta
 
 
 class PILReader(ImageReader):
@@ -582,11 +602,12 @@ def get_data(self, img):
         for i in ensure_tuple(img):
             header = self._get_meta_dict(i)
             header["spatial_shape"] = self._get_spatial_shape(i)
-            img_array.append(np.asarray(i))
+            data = np.asarray(i)
+            img_array.append(data)
+            header["original_channel_dim"] = "no_channel" if len(data.shape) == len(header["spatial_shape"]) else -1
             _copy_compatible_dict(header, compatible_meta)
 
-        img_array_ = np.stack(img_array, axis=0) if len(img_array) > 1 else img_array[0]
-        return img_array_, compatible_meta
+        return _stack_images(img_array, compatible_meta), compatible_meta
 
     def _get_meta_dict(self, img) -> Dict:
         """
@@ -602,10 +623,191 @@ def _get_meta_dict(self, img) -> Dict:
             "height": img.height,
         }
 
-    def _get_spatial_shape(self, img) -> np.ndarray:
+    def _get_spatial_shape(self, img):
         """
         Get the spatial shape of image data, it doesn't contain the channel dim.
         Args:
             img: a PIL Image object loaded from a image file.
         """
+        # the img data should have no channel dim or the last dim is channel
         return np.asarray((img.width, img.height))
+
+
+class WSIReader(ImageReader):
+    """
+    Read whole slide imaging and extract patches.
+
+    Args:
+        reader_lib: backend library to load the images, available options: "OpenSlide" or "cuCIM".
+
+    """
+
+    def __init__(self, reader_lib: str = "OpenSlide"):
+        super().__init__()
+        self.reader_lib = reader_lib.lower()
+        if self.reader_lib == "openslide":
+            if has_osl:
+                self.wsi_reader = openslide.OpenSlide
+        elif self.reader_lib == "cucim":
+            if has_cim:
+                self.wsi_reader = cucim.CuImage
+        else:
+            raise ValueError('`reader_lib` should be either "cuCIM" or "OpenSlide"')
+
+    def verify_suffix(self, filename: Union[Sequence[str], str]) -> bool:
+        """
+        Verify whether the specified file or files format is supported by WSI reader.
+
+        Args:
+            filename: file name or a list of file names to read.
+                if a list of files, verify all the suffixes.
+        """
+        return is_supported_format(filename, ["tif", "tiff"])
+
+    def read(self, data: Union[Sequence[str], str, np.ndarray], **kwargs):
+        """
+        Read image data from specified file or files.
+        Note that the returned object is CuImage or list of CuImage objects.
+
+        Args:
+            data: file name or a list of file names to read.
+
+        """
+        if (self.reader_lib == "openslide") and (not has_osl):
+            raise ImportError("No module named 'openslide'")
+        if (self.reader_lib == "cucim") and (not has_cim):
+            raise ImportError("No module named 'cucim'")
+
+        img_: List = []
+
+        filenames: Sequence[str] = ensure_tuple(data)
+        for name in filenames:
+            img = self.wsi_reader(name)
+            if self.reader_lib == "openslide":
+                img.shape = (img.dimensions[1], img.dimensions[0], 3)
+            img_.append(img)
+
+        return img_ if len(filenames) > 1 else img_[0]
+
+    def get_data(
+        self,
+        img,
+        location: Tuple[int, int] = (0, 0),
+        size: Optional[Tuple[int, int]] = None,
+        level: int = 0,
+        dtype: DtypeLike = np.uint8,
+        grid_shape: Tuple[int, int] = (1, 1),
+        patch_size: Optional[Union[int, Tuple[int, int]]] = None,
+    ):
+        """
+        Extract regions as numpy array from WSI image and return them.
+
+        Args:
+            img: a WSIReader image object loaded from a file, or list of CuImage objects
+            location: (x_min, y_min) tuple giving the top left pixel in the level 0 reference frame,
+            or list of tuples (default=(0, 0))
+            size: (height, width) tuple giving the region size, or list of tuples (default to full image size)
+            This is the size of image at the given level (`level`)
+            level: the level number, or list of level numbers (default=0)
+            dtype: the data type of output image
+            grid_shape: (row, columns) tuple define a grid to extract patches on that
+            patch_size: (height, width) the size of extracted patches at the given level
+        """
+
+        if self.reader_lib == "openslide" and size is None:
+            # the maximum size is set to WxH
+            size = (
+                img.shape[0] // (2 ** level) - location[0],
+                img.shape[1] // (2 ** level) - location[1],
+            )
+
+        region = self._extract_region(img, location=location, size=size, level=level, dtype=dtype)
+
+        metadata: Dict = {}
+        metadata["spatial_shape"] = size
+        metadata["original_channel_dim"] = -1
+        region = EnsureChannelFirst()(region, metadata)
+        if patch_size is None:
+            patches = region
+        else:
+            tuple_patch_size = ensure_tuple_rep(patch_size, 2)
+            patches = self._extract_patches(
+                region,
+                patch_size=tuple_patch_size,  # type: ignore
+                grid_shape=grid_shape,
+                dtype=dtype,
+            )
+
+        return patches, metadata
+
+    def _extract_region(
+        self,
+        img_obj,
+        size: Optional[Tuple[int, int]],
+        location: Tuple[int, int] = (0, 0),
+        level: int = 0,
+        dtype: DtypeLike = np.uint8,
+    ):
+        # reverse the order of dimensions for size and location to be compatible with image shape
+        location = location[::-1]
+        if size is None:
+            region = img_obj.read_region(location=location, level=level)
+        else:
+            size = size[::-1]
+            region = img_obj.read_region(location=location, size=size, level=level)
+
+        region = self.convert_to_rgb_array(region, dtype)
+        return region
+
+    def convert_to_rgb_array(
+        self,
+        raw_region,
+        dtype: DtypeLike = np.uint8,
+    ):
+        """Convert to RGB mode and numpy array"""
+        if self.reader_lib == "openslide":
+            # convert to RGB
+            raw_region = raw_region.convert("RGB")
+            # convert to numpy
+            raw_region = np.asarray(raw_region, dtype=dtype)
+        else:
+            num_channels = len(raw_region.channel_names)
+            # convert to numpy
+            raw_region = np.asarray(raw_region, dtype=dtype)
+            # remove alpha channel if exist (RGBA)
+            if num_channels > 3:
+                raw_region = raw_region[:, :, :3]
+
+        return raw_region
+
+    def _extract_patches(
+        self,
+        region: np.ndarray,
+        grid_shape: Tuple[int, int] = (1, 1),
+        patch_size: Optional[Tuple[int, int]] = None,
+        dtype: DtypeLike = np.uint8,
+    ):
+        if patch_size is None and grid_shape == (1, 1):
+            return region
+
+        n_patches = grid_shape[0] * grid_shape[1]
+        region_size = region.shape[1:]
+
+        if patch_size is None:
+            patch_size = (region_size[0] // grid_shape[0], region_size[1] // grid_shape[1])
+
+        # split the region into patches on the grid and center crop them to patch size
+        flat_patch_grid = np.zeros((n_patches, 3, patch_size[0], patch_size[1]), dtype=dtype)
+        start_points = [
+            np.round(region_size[i] * (0.5 + np.arange(grid_shape[i])) / grid_shape[i] - patch_size[i] / 2).astype(int)
+            for i in range(2)
+        ]
+        idx = 0
+        for y_start in start_points[1]:
+            for x_start in start_points[0]:
+                x_end = x_start + patch_size[0]
+                y_end = y_start + patch_size[1]
+                flat_patch_grid[idx] = region[:, x_start:x_end, y_start:y_end]
+                idx += 1
+
+        return flat_patch_grid
diff --git a/monai/data/inverse_batch_transform.py b/monai/data/inverse_batch_transform.py
new file mode 100644
index 0000000000..3035a1910d
--- /dev/null
+++ b/monai/data/inverse_batch_transform.py
@@ -0,0 +1,95 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from typing import Any, Callable, Dict, Hashable, Optional, Sequence
+
+import numpy as np
+from torch.utils.data.dataloader import DataLoader as TorchDataLoader
+
+from monai.data.dataloader import DataLoader
+from monai.data.dataset import Dataset
+from monai.data.utils import decollate_batch, no_collation, pad_list_data_collate
+from monai.transforms.croppad.batch import PadListDataCollate
+from monai.transforms.inverse import InvertibleTransform
+from monai.transforms.transform import Transform
+from monai.utils import first
+
+__all__ = ["BatchInverseTransform"]
+
+
+class _BatchInverseDataset(Dataset):
+    def __init__(
+        self,
+        data: Sequence[Any],
+        transform: InvertibleTransform,
+        pad_collation_used: bool,
+    ) -> None:
+        super().__init__(data, transform)
+        self.invertible_transform = transform
+        self.pad_collation_used = pad_collation_used
+
+    def _transform(self, index: int) -> Dict[Hashable, np.ndarray]:
+        data = dict(self.data[index])
+        # If pad collation was used, then we need to undo this first
+        if self.pad_collation_used:
+            data = PadListDataCollate.inverse(data)
+
+        if not isinstance(self.invertible_transform, InvertibleTransform):
+            warnings.warn("transform is not invertible, can't invert transform for the input data.")
+            return data
+        return self.invertible_transform.inverse(data)
+
+
+class BatchInverseTransform(Transform):
+    """
+    Perform inverse on a batch of data. This is useful if you have inferred a batch of images and want to invert
+    them all.
+    """
+
+    def __init__(
+        self,
+        transform: InvertibleTransform,
+        loader: TorchDataLoader,
+        collate_fn: Optional[Callable] = no_collation,
+        num_workers: Optional[int] = 0,
+    ) -> None:
+        """
+        Args:
+            transform: a callable data transform on input data.
+            loader: data loader used to run `transforms` and generate the batch of data.
+            collate_fn: how to collate data after inverse transformations.
+                default won't do any collation, so the output will be a list of size batch size.
+            num_workers: number of workers when run data loader for inverse transforms,
+                default to 0 as only run 1 iteration and multi-processing may be even slower.
+                if the transforms are really slow, set num_workers for multi-processing.
+                if set to `None`, use the `num_workers` of the transform data loader.
+        """
+        self.transform = transform
+        self.batch_size = loader.batch_size
+        self.num_workers = loader.num_workers if num_workers is None else num_workers
+        self.collate_fn = collate_fn
+        self.pad_collation_used = loader.collate_fn == pad_list_data_collate
+
+    def __call__(self, data: Dict[str, Any]) -> Any:
+
+        decollated_data = decollate_batch(data)
+        inv_ds = _BatchInverseDataset(decollated_data, self.transform, self.pad_collation_used)
+        inv_loader = DataLoader(
+            inv_ds, batch_size=self.batch_size, num_workers=self.num_workers, collate_fn=self.collate_fn
+        )
+        try:
+            return first(inv_loader)
+        except RuntimeError as re:
+            re_str = str(re)
+            if "equal size" in re_str:
+                re_str += "\nMONAI hint: try creating `BatchInverseTransform` with `collate_fn=lambda x: x`."
+            raise RuntimeError(re_str)
diff --git a/monai/data/nifti_saver.py b/monai/data/nifti_saver.py
index 01e701b1a6..0ff719023c 100644
--- a/monai/data/nifti_saver.py
+++ b/monai/data/nifti_saver.py
@@ -27,6 +27,8 @@ class NiftiSaver:
     Typically, the data can be segmentation predictions, call `save` for single data
     or call `save_batch` to save a batch of data together. If no meta data provided,
     use index from 0 as the filename prefix.
+
+    NB: image should include channel dimension: [B],C,H,W,[D].
     """
 
     def __init__(
@@ -40,6 +42,8 @@ def __init__(
         align_corners: bool = False,
         dtype: DtypeLike = np.float64,
         output_dtype: DtypeLike = np.float32,
+        squeeze_end_dims: bool = True,
+        data_root_dir: str = "",
     ) -> None:
         """
         Args:
@@ -60,6 +64,21 @@ def __init__(
             dtype: data type for resampling computation. Defaults to ``np.float64`` for best precision.
                 If None, use the data type of input data.
             output_dtype: data type for saving data. Defaults to ``np.float32``.
+            squeeze_end_dims: if True, any trailing singleton dimensions will be removed (after the channel
+                has been moved to the end). So if input is (C,H,W,D), this will be altered to (H,W,D,C), and
+                then if C==1, it will be saved as (H,W,D). If D also ==1, it will be saved as (H,W). If false,
+                image will always be saved as (H,W,D,C).
+            data_root_dir: if not empty, it specifies the beginning parts of the input file's
+                absolute path. it's used to compute `input_file_rel_path`, the relative path to the file from
+                `data_root_dir` to preserve folder structure when saving in case there are files in different
+                folders with the same file names. for example:
+                input_file_name: /foo/bar/test1/image.nii,
+                postfix: seg
+                output_ext: nii.gz
+                output_dir: /output,
+                data_root_dir: /foo/bar,
+                output will be: /output/test1/image/image_seg.nii.gz
+
         """
         self.output_dir = output_dir
         self.output_postfix = output_postfix
@@ -71,6 +90,8 @@ def __init__(
         self.dtype = dtype
         self.output_dtype = output_dtype
         self._data_index = 0
+        self.squeeze_end_dims = squeeze_end_dims
+        self.data_root_dir = data_root_dir
 
     def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict] = None) -> None:
         """
@@ -81,6 +102,7 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
             - ``'original_affine'`` -- for data orientation handling, defaulting to an identity matrix.
             - ``'affine'`` -- for data output affine, defaulting to an identity matrix.
             - ``'spatial_shape'`` -- for data output shape.
+            - ``'patch_index'`` -- if the data is a patch of big image, append the patch index to filename.
 
         When meta_data is specified, the saver will try to resample batch data from the space
         defined by "affine" to the space defined by "original_affine".
@@ -100,20 +122,27 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
         original_affine = meta_data.get("original_affine", None) if meta_data else None
         affine = meta_data.get("affine", None) if meta_data else None
         spatial_shape = meta_data.get("spatial_shape", None) if meta_data else None
+        patch_index = meta_data.get(Key.PATCH_INDEX, None) if meta_data else None
 
         if isinstance(data, torch.Tensor):
             data = data.detach().cpu().numpy()
 
-        filename = create_file_basename(self.output_postfix, filename, self.output_dir)
-        filename = f"{filename}{self.output_ext}"
+        path = create_file_basename(self.output_postfix, filename, self.output_dir, self.data_root_dir, patch_index)
+        path = f"{path}{self.output_ext}"
         # change data shape to be (channel, h, w, d)
         while len(data.shape) < 4:
             data = np.expand_dims(data, -1)
         # change data to "channel last" format and write to nifti format file
         data = np.moveaxis(np.asarray(data), 0, -1)
+
+        # if desired, remove trailing singleton dimensions
+        if self.squeeze_end_dims:
+            while data.shape[-1] == 1:
+                data = np.squeeze(data, -1)
+
         write_nifti(
             data,
-            file_name=filename,
+            file_name=path,
             affine=affine,
             target_affine=original_affine,
             resample=self.resample,
@@ -142,6 +171,7 @@ def save_batch(self, batch_data: Union[torch.Tensor, np.ndarray], meta_data: Opt
         Args:
             batch_data: target batch data content that save into NIfTI format.
             meta_data: every key-value in the meta_data is corresponding to a batch of data.
+
         """
         for i, data in enumerate(batch_data):  # save a batch of files
-            self.save(data, {k: meta_data[k][i] for k in meta_data} if meta_data else None)
+            self.save(data=data, meta_data={k: meta_data[k][i] for k in meta_data} if meta_data is not None else None)
diff --git a/monai/data/png_saver.py b/monai/data/png_saver.py
index 4c4c847824..880f6b204f 100644
--- a/monai/data/png_saver.py
+++ b/monai/data/png_saver.py
@@ -36,6 +36,7 @@ def __init__(
         resample: bool = True,
         mode: Union[InterpolateMode, str] = InterpolateMode.NEAREST,
         scale: Optional[int] = None,
+        data_root_dir: str = "",
     ) -> None:
         """
         Args:
@@ -48,6 +49,16 @@ def __init__(
                 See also: https://pytorch.org/docs/stable/nn.functional.html#interpolate
             scale: {``255``, ``65535``} postprocess data by clipping to [0, 1] and scaling
                 [0, 255] (uint8) or [0, 65535] (uint16). Default is None to disable scaling.
+            data_root_dir: if not empty, it specifies the beginning parts of the input file's
+                absolute path. it's used to compute `input_file_rel_path`, the relative path to the file from
+                `data_root_dir` to preserve folder structure when saving in case there are files in different
+                folders with the same file names. for example:
+                input_file_name: /foo/bar/test1/image.png,
+                postfix: seg
+                output_ext: png
+                output_dir: /output,
+                data_root_dir: /foo/bar,
+                output will be: /output/test1/image/image_seg.png
 
         """
         self.output_dir = output_dir
@@ -56,6 +67,7 @@ def __init__(
         self.resample = resample
         self.mode: InterpolateMode = InterpolateMode(mode)
         self.scale = scale
+        self.data_root_dir = data_root_dir
 
         self._data_index = 0
 
@@ -66,6 +78,7 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
 
             - ``'filename_or_obj'`` -- for output file name creation, corresponding to filename or object.
             - ``'spatial_shape'`` -- for data output shape.
+            - ``'patch_index'`` -- if the data is a patch of big image, append the patch index to filename.
 
         If meta_data is None, use the default index (starting from 0) as the filename.
 
@@ -86,12 +99,13 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
         filename = meta_data[Key.FILENAME_OR_OBJ] if meta_data else str(self._data_index)
         self._data_index += 1
         spatial_shape = meta_data.get("spatial_shape", None) if meta_data and self.resample else None
+        patch_index = meta_data.get(Key.PATCH_INDEX, None) if meta_data else None
 
         if isinstance(data, torch.Tensor):
             data = data.detach().cpu().numpy()
 
-        filename = create_file_basename(self.output_postfix, filename, self.output_dir)
-        filename = f"{filename}{self.output_ext}"
+        path = create_file_basename(self.output_postfix, filename, self.output_dir, self.data_root_dir, patch_index)
+        path = f"{path}{self.output_ext}"
 
         if data.shape[0] == 1:
             data = data.squeeze(0)
@@ -102,7 +116,7 @@ def save(self, data: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict]
 
         write_png(
             np.asarray(data),
-            file_name=filename,
+            file_name=path,
             output_spatial_shape=spatial_shape,
             mode=self.mode,
             scale=self.scale,
@@ -114,6 +128,7 @@ def save_batch(self, batch_data: Union[torch.Tensor, np.ndarray], meta_data: Opt
         Args:
             batch_data: target batch data content that save into png format.
             meta_data: every key-value in the meta_data is corresponding to a batch of data.
+
         """
         for i, data in enumerate(batch_data):  # save a batch of files
-            self.save(data, {k: meta_data[k][i] for k in meta_data} if meta_data else None)
+            self.save(data=data, meta_data={k: meta_data[k][i] for k in meta_data} if meta_data is not None else None)
diff --git a/monai/data/samplers.py b/monai/data/samplers.py
new file mode 100644
index 0000000000..8bba79c9b0
--- /dev/null
+++ b/monai/data/samplers.py
@@ -0,0 +1,119 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional, Sequence
+
+import torch
+from torch.utils.data import Dataset
+from torch.utils.data import DistributedSampler as _TorchDistributedSampler
+
+__all__ = ["DistributedSampler", "DistributedWeightedRandomSampler"]
+
+
+class DistributedSampler(_TorchDistributedSampler):
+    """
+    Enhance PyTorch DistributedSampler to support non-evenly divisible sampling.
+
+    Args:
+        dataset: Dataset used for sampling.
+        even_divisible: if False, different ranks can have different data length.
+            for example, input data: [1, 2, 3, 4, 5], rank 0: [1, 3, 5], rank 1: [2, 4].
+        num_replicas: number of processes participating in distributed training.
+            by default, `world_size` is retrieved from the current distributed group.
+        rank: rank of the current process within `num_replicas`. by default,
+            `rank` is retrieved from the current distributed group.
+        shuffle: if `True`, sampler will shuffle the indices, default to True.
+        kwargs: additional arguments for `DistributedSampler` super class, can be `seed` and `drop_last`.
+
+    More information about DistributedSampler, please check:
+    https://github.com/pytorch/pytorch/blob/master/torch/utils/data/distributed.py
+
+    """
+
+    def __init__(
+        self,
+        dataset: Dataset,
+        even_divisible: bool = True,
+        num_replicas: Optional[int] = None,
+        rank: Optional[int] = None,
+        shuffle: bool = True,
+        **kwargs,
+    ):
+        super().__init__(dataset=dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle, **kwargs)
+
+        if not even_divisible:
+            data_len = len(dataset)  # type: ignore
+            extra_size = self.total_size - data_len
+            if self.rank + extra_size >= self.num_replicas:
+                self.num_samples -= 1
+            self.total_size = data_len
+
+
+class DistributedWeightedRandomSampler(DistributedSampler):
+    """
+    Extend the `DistributedSampler` to support weighted sampling.
+    Refer to `torch.utils.data.WeightedRandomSampler`, for more details please check:
+    https://github.com/pytorch/pytorch/blob/master/torch/utils/data/sampler.py#L150
+
+    Args:
+        dataset: Dataset used for sampling.
+        weights: a sequence of weights, not necessary summing up to one, length should exactly
+            match the full dataset.
+        num_samples_per_rank: number of samples to draw for every rank, sample from
+            the distributed subset of dataset.
+            if None, default to the length of dataset split by DistributedSampler.
+        generator: PyTorch Generator used in sampling.
+        even_divisible: if False, different ranks can have different data length.
+            for example, input data: [1, 2, 3, 4, 5], rank 0: [1, 3, 5], rank 1: [2, 4].'
+        num_replicas: number of processes participating in distributed training.
+            by default, `world_size` is retrieved from the current distributed group.
+        rank: rank of the current process within `num_replicas`. by default,
+            `rank` is retrieved from the current distributed group.
+        shuffle: if `True`, sampler will shuffle the indices, default to True.
+        kwargs: additional arguments for `DistributedSampler` super class, can be `seed` and `drop_last`.
+
+    """
+
+    def __init__(
+        self,
+        dataset: Dataset,
+        weights: Sequence[float],
+        num_samples_per_rank: Optional[int] = None,
+        generator: Optional[torch.Generator] = None,
+        even_divisible: bool = True,
+        num_replicas: Optional[int] = None,
+        rank: Optional[int] = None,
+        shuffle: bool = True,
+        **kwargs,
+    ):
+        super().__init__(
+            dataset=dataset,
+            even_divisible=even_divisible,
+            num_replicas=num_replicas,
+            rank=rank,
+            shuffle=shuffle,
+            **kwargs,
+        )
+        self.weights = weights
+        self.num_samples_per_rank = num_samples_per_rank if num_samples_per_rank is not None else self.num_samples
+        self.generator = generator
+
+    def __iter__(self):
+        indices = list(super().__iter__())
+        weights = torch.as_tensor([self.weights[i] for i in indices], dtype=torch.double)
+        # sample based on the provided weights
+        rand_tensor = torch.multinomial(weights, self.num_samples_per_rank, True, generator=self.generator)
+
+        for i in rand_tensor:
+            yield indices[i]
+
+    def __len__(self):
+        return self.num_samples_per_rank
diff --git a/monai/data/test_time_augmentation.py b/monai/data/test_time_augmentation.py
new file mode 100644
index 0000000000..9e4497d5e4
--- /dev/null
+++ b/monai/data/test_time_augmentation.py
@@ -0,0 +1,199 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+from monai.data.dataloader import DataLoader
+from monai.data.dataset import Dataset
+from monai.data.inverse_batch_transform import BatchInverseTransform
+from monai.data.utils import list_data_collate, pad_list_data_collate
+from monai.transforms.compose import Compose
+from monai.transforms.inverse import InvertibleTransform
+from monai.transforms.transform import Randomizable
+from monai.transforms.utils import allow_missing_keys_mode
+from monai.utils.enums import CommonKeys, InverseKeys
+from monai.utils.module import optional_import
+
+if TYPE_CHECKING:
+    from tqdm import tqdm
+
+    has_tqdm = True
+else:
+    tqdm, has_tqdm = optional_import("tqdm", name="tqdm")
+
+__all__ = ["TestTimeAugmentation"]
+
+
+class TestTimeAugmentation:
+    """
+    Class for performing test time augmentations. This will pass the same image through the network multiple times.
+
+    The user passes transform(s) to be applied to each realisation, and provided that at least one of those transforms
+    is random, the network's output will vary. Provided that inverse transformations exist for all supplied spatial
+    transforms, the inverse can be applied to each realisation of the network's output. Once in the same spatial
+    reference, the results can then be combined and metrics computed.
+
+    Test time augmentations are a useful feature for computing network uncertainty, as well as observing the network's
+    dependency on the applied random transforms.
+
+    Reference:
+        Wang et al.,
+        Aleatoric uncertainty estimation with test-time augmentation for medical image segmentation with convolutional
+        neural networks,
+        https://doi.org/10.1016/j.neucom.2019.01.103
+
+    Args:
+        transform: transform (or composed) to be applied to each realisation. At least one transform must be of type
+            `Randomizable`. All random transforms must be of type `InvertibleTransform`.
+        batch_size: number of realisations to infer at once.
+        num_workers: how many subprocesses to use for data.
+        inferrer_fn: function to use to perform inference.
+        device: device on which to perform inference.
+        image_key: key used to extract image from input dictionary.
+        label_key: key used to extract label from input dictionary.
+        meta_key_postfix: use `key_{postfix}` to to fetch the meta data according to the key data,
+            default is `meta_dict`, the meta data is a dictionary object.
+            For example, to handle key `image`,  read/write affine matrices from the
+            metadata `image_meta_dict` dictionary's `affine` field.
+        return_full_data: normally, metrics are returned (mode, mean, std, vvc). Setting this flag to `True` will return the
+            full data. Dimensions will be same size as when passing a single image through `inferrer_fn`, with a dimension appended
+            equal in size to `num_examples` (N), i.e., `[N,C,H,W,[D]]`.
+        progress: whether to display a progress bar.
+
+    Example:
+        .. code-block:: python
+
+            transform = RandAffined(keys, ...)
+            post_trans = Compose([Activations(sigmoid=True), AsDiscrete(threshold_values=True)])
+
+            tt_aug = TestTimeAugmentation(
+                transform, batch_size=5, num_workers=0, inferrer_fn=lambda x: post_trans(model(x)), device=device
+            )
+            mode, mean, std, vvc = tt_aug(test_data)
+    """
+
+    def __init__(
+        self,
+        transform: InvertibleTransform,
+        batch_size: int,
+        num_workers: int,
+        inferrer_fn: Callable,
+        device: Optional[Union[str, torch.device]] = "cuda" if torch.cuda.is_available() else "cpu",
+        image_key=CommonKeys.IMAGE,
+        label_key=CommonKeys.LABEL,
+        meta_key_postfix="meta_dict",
+        return_full_data: bool = False,
+        progress: bool = True,
+    ) -> None:
+        self.transform = transform
+        self.batch_size = batch_size
+        self.num_workers = num_workers
+        self.inferrer_fn = inferrer_fn
+        self.device = device
+        self.image_key = image_key
+        self.label_key = label_key
+        self.meta_key_postfix = meta_key_postfix
+        self.return_full_data = return_full_data
+        self.progress = progress
+
+        # check that the transform has at least one random component, and that all random transforms are invertible
+        self._check_transforms()
+
+    def _check_transforms(self):
+        """Should be at least 1 random transform, and all random transforms should be invertible."""
+        ts = [self.transform] if not isinstance(self.transform, Compose) else self.transform.transforms
+        randoms = np.array([isinstance(t, Randomizable) for t in ts])
+        invertibles = np.array([isinstance(t, InvertibleTransform) for t in ts])
+        # check at least 1 random
+        if sum(randoms) == 0:
+            raise RuntimeError(
+                "Requires a `Randomizable` transform or a `Compose` containing at least one `Randomizable` transform."
+            )
+        # check that whenever randoms is True, invertibles is also true
+        for r, i in zip(randoms, invertibles):
+            if r and not i:
+                raise RuntimeError(
+                    f"All applied random transform(s) must be invertible. Problematic transform: {type(r).__name__}"
+                )
+
+    def __call__(
+        self, data: Dict[str, Any], num_examples: int = 10
+    ) -> Union[Tuple[np.ndarray, np.ndarray, np.ndarray, float], np.ndarray]:
+        """
+        Args:
+            data: dictionary data to be processed.
+            num_examples: number of realisations to be processed and results combined.
+
+        Returns:
+            - if `return_full_data==False`: mode, mean, std, vvc. The mode, mean and standard deviation are calculated across
+                `num_examples` outputs at each voxel. The volume variation coefficient (VVC) is `std/mean` across the whole output,
+                including `num_examples`. See original paper for clarification.
+            - if `return_full_data==False`: data is returned as-is after applying the `inferrer_fn` and then concatenating across
+                the first dimension containing `num_examples`. This allows the user to perform their own analysis if desired.
+        """
+        d = dict(data)
+
+        # check num examples is multiple of batch size
+        if num_examples % self.batch_size != 0:
+            raise ValueError("num_examples should be multiple of batch size.")
+
+        # generate batch of data of size == batch_size, dataset and dataloader
+        data_in = [d] * num_examples
+        ds = Dataset(data_in, self.transform)
+        dl = DataLoader(ds, self.num_workers, batch_size=self.batch_size, collate_fn=pad_list_data_collate)
+
+        label_transform_key = self.label_key + InverseKeys.KEY_SUFFIX
+
+        # create inverter
+        inverter = BatchInverseTransform(self.transform, dl, collate_fn=list_data_collate)
+
+        outputs: List[np.ndarray] = []
+
+        for batch_data in tqdm(dl) if has_tqdm and self.progress else dl:
+
+            batch_images = batch_data[self.image_key].to(self.device)
+
+            # do model forward pass
+            batch_output = self.inferrer_fn(batch_images)
+            if isinstance(batch_output, torch.Tensor):
+                batch_output = batch_output.detach().cpu()
+            if isinstance(batch_output, np.ndarray):
+                batch_output = torch.Tensor(batch_output)
+
+            # create a dictionary containing the inferred batch and their transforms
+            inferred_dict = {self.label_key: batch_output, label_transform_key: batch_data[label_transform_key]}
+            # if meta dict is present, add that too (required for some inverse transforms)
+            label_meta_dict_key = f"{self.label_key}_{self.meta_key_postfix}"
+            if label_meta_dict_key in batch_data:
+                inferred_dict[label_meta_dict_key] = batch_data[label_meta_dict_key]
+
+            # do inverse transformation (allow missing keys as only inverting label)
+            with allow_missing_keys_mode(self.transform):  # type: ignore
+                inv_batch = inverter(inferred_dict)
+
+            # append
+            outputs.append(inv_batch[self.label_key])
+
+        # output
+        output: np.ndarray = np.concatenate(outputs)
+
+        if self.return_full_data:
+            return output
+
+        # calculate metrics
+        mode = np.array(torch.mode(torch.Tensor(output.astype(np.int64)), dim=0).values)
+        mean: np.ndarray = np.mean(output, axis=0)  # type: ignore
+        std: np.ndarray = np.std(output, axis=0)  # type: ignore
+        vvc: float = (np.std(output) / np.mean(output)).item()
+        return mode, mean, std, vvc
diff --git a/monai/data/thread_buffer.py b/monai/data/thread_buffer.py
index 252fdd6a21..8ea71e3555 100644
--- a/monai/data/thread_buffer.py
+++ b/monai/data/thread_buffer.py
@@ -13,6 +13,8 @@
 from queue import Empty, Full, Queue
 from threading import Thread
 
+from monai.data import DataLoader, Dataset
+
 
 class ThreadBuffer:
     """
@@ -73,3 +75,20 @@ def __iter__(self):
                     pass  # queue was empty this time, try again
         finally:
             self.stop()  # ensure thread completion
+
+
+class ThreadDataLoader(DataLoader):
+    """
+    Subclass of `DataLoader` using a `ThreadBuffer` object to implement `__iter__` method asynchronously. This will
+    iterate over data from the loader as expected however the data is generated on a separate thread. Use this class
+    where a `DataLoader` instance is required and not just an iterable object.
+    """
+
+    def __init__(self, dataset: Dataset, num_workers: int = 0, **kwargs):
+        super().__init__(dataset, num_workers, **kwargs)
+
+        # ThreadBuffer will use the inherited __iter__ instead of the one defined below
+        self.buffer = ThreadBuffer(super().__iter__())
+
+    def __iter__(self):
+        yield from self.buffer
diff --git a/monai/data/utils.py b/monai/data/utils.py
index acc6d2e97a..d39f2702ff 100644
--- a/monai/data/utils.py
+++ b/monai/data/utils.py
@@ -15,14 +15,13 @@
 import os
 import pickle
 import warnings
-from collections import defaultdict
+from collections import abc, defaultdict
 from itertools import product, starmap
 from pathlib import PurePath
-from typing import Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Generator, Iterable, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 import torch
-from torch.utils.data import DistributedSampler as _TorchDistributedSampler
 from torch.utils.data._utils.collate import default_collate
 
 from monai.networks.layers.simplelayers import GaussianFilter
@@ -36,6 +35,8 @@
     first,
     optional_import,
 )
+from monai.utils.enums import Method
+from monai.utils.misc import issequenceiterable
 
 nib, _ = optional_import("nibabel")
 
@@ -59,10 +60,12 @@
     "partition_dataset",
     "partition_dataset_classes",
     "select_cross_validation_folds",
-    "DistributedSampler",
     "json_hashing",
     "pickle_hashing",
     "sorted_dict",
+    "decollate_batch",
+    "pad_list_data_collate",
+    "no_collation",
 ]
 
 
@@ -170,7 +173,7 @@ def iter_patch(
     copy_back: bool = True,
     mode: Union[NumpyPadMode, str] = NumpyPadMode.WRAP,
     **pad_opts: Dict,
-) -> Generator[np.ndarray, None, None]:
+):
     """
     Yield successive patches from `arr` of size `patch_size`. The iteration can start from position `start_pos` in `arr`
     but drawing from a padded array extended by the `patch_size` in each dimension (so these coordinates can be negative
@@ -190,6 +193,15 @@ def iter_patch(
     Yields:
         Patches of array data from `arr` which are views into a padded array which can be modified, if `copy_back` is
         True these changes will be reflected in `arr` once the iteration completes.
+
+    Note:
+        coordinate format is:
+
+            [1st_dim_start, 1st_dim_end,
+             2nd_dim_start, 2nd_dim_end,
+             ...,
+             Nth_dim_start, Nth_dim_end]]
+
     """
     # ensure patchSize and startPos are the right length
     patch_size_ = get_valid_patch_size(arr.shape, patch_size)
@@ -206,7 +218,9 @@ def iter_patch(
     iter_size = tuple(s + p for s, p in zip(arr.shape, patch_size_))
 
     for slices in iter_patch_slices(iter_size, patch_size_, start_pos_padded):
-        yield arrpad[slices]
+        # compensate original image padding
+        coords_no_pad = tuple((coord.start - p, coord.stop - p) for coord, p in zip(slices, patch_size_))
+        yield arrpad[slices], np.asarray(coords_no_pad)  # data and coords (in numpy; works with torch loader)
 
     # copy back data from the padded image if required
     if copy_back:
@@ -240,7 +254,137 @@ def list_data_collate(batch: Sequence):
     """
     elem = batch[0]
     data = [i for k in batch for i in k] if isinstance(elem, list) else batch
-    return default_collate(data)
+    try:
+        elem = batch[0]
+        key = None
+        if isinstance(elem, abc.Mapping):
+            ret = {}
+            for k in elem:
+                key = k
+                ret[k] = default_collate([d[k] for d in data])
+            return ret
+        return default_collate(data)
+    except RuntimeError as re:
+        re_str = str(re)
+        if "equal size" in re_str:
+            if key is not None:
+                re_str += f"\nCollate error on the key '{key}' of dictionary data."
+            re_str += (
+                "\n\nMONAI hint: if your transforms intentionally create images of different shapes, creating your "
+                + "`DataLoader` with `collate_fn=pad_list_data_collate` might solve this problem (check its "
+                + "documentation)."
+            )
+        raise RuntimeError(re_str)
+    except TypeError as re:
+        re_str = str(re)
+        if "numpy" in re_str and "Tensor" in re_str:
+            if key is not None:
+                re_str += f"\nCollate error on the key '{key}' of dictionary data."
+            re_str += (
+                "\n\nMONAI hint: if your transforms intentionally create mixtures of torch Tensor and numpy ndarray, "
+                + "creating your `DataLoader` with `collate_fn=pad_list_data_collate` might solve this problem "
+                + "(check its documentation)."
+            )
+        raise TypeError(re_str)
+
+
+def decollate_batch(data: dict, batch_size: Optional[int] = None) -> List[dict]:
+    """De-collate a batch of data (for example, as produced by a `DataLoader`).
+
+    Returns a list of dictionaries. Each dictionary will only contain the data for a given batch.
+
+    Images originally stored as (B,C,H,W,[D]) will be returned as (C,H,W,[D]). Other information,
+    such as metadata, may have been stored in a list (or a list inside nested dictionaries). In
+    this case we return the element of the list corresponding to the batch idx.
+
+    Return types aren't guaranteed to be the same as the original, since numpy arrays will have been
+    converted to torch.Tensor, and tuples/lists may have been converted to lists of tensors
+
+    For example:
+
+    .. code-block:: python
+
+        batch_data = {
+            "image": torch.rand((2,1,10,10)),
+            "image_meta_dict": {"scl_slope": torch.Tensor([0.0, 0.0])}
+        }
+        out = decollate_batch(batch_data)
+        print(len(out))
+        >>> 2
+
+        print(out[0])
+        >>> {'image': tensor([[[4.3549e-01...43e-01]]]), 'image_meta_dict': {'scl_slope': 0.0}}
+
+    Args:
+        data: data to be de-collated.
+        batch_size: number of batches in data. If `None` is passed, try to figure out batch size.
+    """
+    if not isinstance(data, dict):
+        raise RuntimeError("Only currently implemented for dictionary data (might be trivial to adapt).")
+    if batch_size is None:
+        for v in data.values():
+            if isinstance(v, torch.Tensor):
+                batch_size = v.shape[0]
+                break
+    if batch_size is None:
+        raise RuntimeError("Couldn't determine batch size, please specify as argument.")
+
+    def torch_to_single(d: torch.Tensor):
+        """If input is a torch.Tensor with only 1 element, return just the element."""
+        return d if d.numel() > 1 else d.item()
+
+    def decollate(data: Any, idx: int):
+        """Recursively de-collate."""
+        if isinstance(data, dict):
+            return {k: decollate(v, idx) for k, v in data.items()}
+        if isinstance(data, torch.Tensor):
+            out = data[idx]
+            return torch_to_single(out)
+        if isinstance(data, list):
+            if len(data) == 0:
+                return data
+            if isinstance(data[0], torch.Tensor):
+                return [torch_to_single(d[idx]) for d in data]
+            if issequenceiterable(data[0]):
+                return [decollate(d, idx) for d in data]
+            return data[idx]
+        raise TypeError(f"Not sure how to de-collate type: {type(data)}")
+
+    return [{key: decollate(data[key], idx) for key in data.keys()} for idx in range(batch_size)]
+
+
+def pad_list_data_collate(
+    batch: Sequence,
+    method: Union[Method, str] = Method.SYMMETRIC,
+    mode: Union[NumpyPadMode, str] = NumpyPadMode.CONSTANT,
+):
+    """
+    Function version of :py:class:`monai.transforms.croppad.batch.PadListDataCollate`.
+
+    Same as MONAI's ``list_data_collate``, except any tensors are centrally padded to match the shape of the biggest
+    tensor in each dimension. This transform is useful if some of the applied transforms generate batch data of
+    different sizes.
+
+    This can be used on both list and dictionary data. In the case of the dictionary data, this transform will be added
+    to the list of invertible transforms.
+
+    The inverse can be called using the static method: `monai.transforms.croppad.batch.PadListDataCollate.inverse`.
+
+    Args:
+        batch: batch of data to pad-collate
+        method: padding method (see :py:class:`monai.transforms.SpatialPad`)
+        mode: padding mode (see :py:class:`monai.transforms.SpatialPad`)
+    """
+    from monai.transforms.croppad.batch import PadListDataCollate  # needs to be here to avoid circular import
+
+    return PadListDataCollate(method, mode)(batch)
+
+
+def no_collation(x):
+    """
+    No any collation operation.
+    """
+    return x
 
 
 def worker_init_fn(worker_id: int) -> None:
@@ -266,6 +410,8 @@ def set_rnd(obj, seed: int) -> int:
         obj.set_random_state(seed=seed % MAX_SEED)
         return seed + 1  # a different seed for the next component
     for key in obj.__dict__:
+        if key.startswith("__"):  # skip the private methods
+            continue
         seed = set_rnd(obj.__dict__[key], seed=seed)
     return seed
 
@@ -462,6 +608,7 @@ def create_file_basename(
     input_file_name: str,
     folder_path: str,
     data_root_dir: str = "",
+    patch_index: Optional[int] = None,
 ) -> str:
     """
     Utility function to create the path to the output file based on the input
@@ -470,7 +617,12 @@ def create_file_basename(
 
         `folder_path/input_file_name (no ext.) /input_file_name (no ext.)[_postfix]`
 
-    otherwise the relative path with respect to `data_root_dir` will be inserted.
+    otherwise the relative path with respect to `data_root_dir` will be inserted, for example:
+    input_file_name: /foo/bar/test1/image.png,
+    postfix: seg
+    folder_path: /output,
+    data_root_dir: /foo/bar,
+    output will be: /output/test1/image/image_seg
 
     Args:
         postfix: output name's postfix
@@ -480,6 +632,7 @@ def create_file_basename(
             absolute path. This is used to compute `input_file_rel_path`, the relative path to the file from
             `data_root_dir` to preserve folder structure when saving in case there are files in different
             folders with the same file names.
+        patch_index: if not None, append the patch index to filename.
     """
 
     # get the filename and directory
@@ -498,11 +651,15 @@ def create_file_basename(
     if not os.path.exists(subfolder_path):
         os.makedirs(subfolder_path)
 
-    if postfix:
+    if len(postfix) > 0:
         # add the sub-folder plus the postfix name to become the file basename in the output path
         output = os.path.join(subfolder_path, filename + "_" + postfix)
     else:
         output = os.path.join(subfolder_path, filename)
+
+    if patch_index is not None:
+        output += f"_{patch_index}"
+
     return os.path.abspath(output)
 
 
@@ -763,34 +920,6 @@ def select_cross_validation_folds(partitions: Sequence[Iterable], folds: Union[S
     return [data_item for fold_id in ensure_tuple(folds) for data_item in partitions[fold_id]]
 
 
-class DistributedSampler(_TorchDistributedSampler):
-    """
-    Enhance PyTorch DistributedSampler to support non-evenly divisible sampling.
-
-    Args:
-        even_divisible: if False, different ranks can have different data length.
-        for example, input data: [1, 2, 3, 4, 5], rank 0: [1, 3, 5], rank 1: [2, 4].
-
-    More information about DistributedSampler, please check:
-    https://github.com/pytorch/pytorch/blob/master/torch/utils/data/distributed.py
-
-    """
-
-    def __init__(self, even_divisible: bool = True, *args, **kwargs):
-        self.total_size: int = 0
-        self.rank: int = 0
-        self.num_samples: int = 0
-        self.num_replicas: int = 0
-        super().__init__(*args, **kwargs)
-
-        if not even_divisible:
-            data_len = len(kwargs["dataset"])
-            extra_size = self.total_size - data_len
-            if self.rank + extra_size >= self.num_replicas:
-                self.num_samples -= 1
-            self.total_size = data_len
-
-
 def json_hashing(item) -> bytes:
     """
 
diff --git a/monai/engines/__init__.py b/monai/engines/__init__.py
index 8256680735..d3a14f6104 100644
--- a/monai/engines/__init__.py
+++ b/monai/engines/__init__.py
@@ -12,4 +12,4 @@
 from .evaluator import EnsembleEvaluator, Evaluator, SupervisedEvaluator
 from .multi_gpu_supervised_trainer import create_multigpu_supervised_evaluator, create_multigpu_supervised_trainer
 from .trainer import GanTrainer, SupervisedTrainer, Trainer
-from .utils import CommonKeys, GanKeys, IterationEvents, default_make_latent, default_prepare_batch, get_devices_spec
+from .utils import GanKeys, IterationEvents, default_make_latent, default_prepare_batch, get_devices_spec
diff --git a/monai/engines/evaluator.py b/monai/engines/evaluator.py
index 0b7167fb3a..e1fecb745d 100644
--- a/monai/engines/evaluator.py
+++ b/monai/engines/evaluator.py
@@ -9,25 +9,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Sequence, Tuple
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union
 
 import torch
 from torch.utils.data import DataLoader
 
-from monai.engines.utils import CommonKeys as Keys
 from monai.engines.utils import IterationEvents, default_prepare_batch
 from monai.engines.workflow import Workflow
 from monai.inferers import Inferer, SimpleInferer
-from monai.networks.utils import eval_mode
+from monai.networks.utils import eval_mode, train_mode
 from monai.transforms import Transform
-from monai.utils import ensure_tuple, exact_version, optional_import
+from monai.utils import ForwardMode, ensure_tuple, exact_version, optional_import
+from monai.utils.enums import CommonKeys as Keys
 
 if TYPE_CHECKING:
-    from ignite.engine import Engine
+    from ignite.engine import Engine, EventEnum
     from ignite.metrics import Metric
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
-    Metric, _ = optional_import("ignite.metrics", "0.4.2", exact_version, "Metric")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+    Metric, _ = optional_import("ignite.metrics", "0.4.4", exact_version, "Metric")
+    EventEnum, _ = optional_import("ignite.engine", "0.4.4", exact_version, "EventEnum")
 
 __all__ = ["Evaluator", "SupervisedEvaluator", "EnsembleEvaluator"]
 
@@ -38,7 +39,7 @@ class Evaluator(Workflow):
 
     Args:
         device: an object representing the device on which to run.
-        val_data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        val_data_loader: Ignite engine use data_loader to run, must be Iterable or torch.DataLoader.
         epoch_length: number of iterations for one epoch, default to `len(val_data_loader)`.
         non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously
             with respect to the host. For other cases, this argument has no effect.
@@ -54,13 +55,19 @@ class Evaluator(Workflow):
         val_handlers: every handler is a set of Ignite Event-Handlers, must have `attach` function, like:
             CheckpointHandler, StatsHandler, SegmentationSaver, etc.
         amp: whether to enable auto-mixed-precision evaluation, default is False.
+        mode: model forward mode during evaluation, should be 'eval' or 'train',
+            which maps to `model.eval()` or `model.train()`, default to 'eval'.
+        event_names: additional custom ignite events that will register to the engine.
+            new events can be a list of str or `ignite.engine.events.EventEnum`.
+        event_to_attr: a dictionary to map an event to a state attribute, then add to `engine.state`.
+            for more details, check: https://github.com/pytorch/ignite/blob/v0.4.4.post1/ignite/engine/engine.py#L160
 
     """
 
     def __init__(
         self,
         device: torch.device,
-        val_data_loader: DataLoader,
+        val_data_loader: Union[Iterable, DataLoader],
         epoch_length: Optional[int] = None,
         non_blocking: bool = False,
         prepare_batch: Callable = default_prepare_batch,
@@ -70,6 +77,9 @@ def __init__(
         additional_metrics: Optional[Dict[str, Metric]] = None,
         val_handlers: Optional[Sequence] = None,
         amp: bool = False,
+        mode: Union[ForwardMode, str] = ForwardMode.EVAL,
+        event_names: Optional[List[Union[str, EventEnum]]] = None,
+        event_to_attr: Optional[dict] = None,
     ) -> None:
         super().__init__(
             device=device,
@@ -84,7 +94,16 @@ def __init__(
             additional_metrics=additional_metrics,
             handlers=val_handlers,
             amp=amp,
+            event_names=event_names,
+            event_to_attr=event_to_attr,
         )
+        mode = ForwardMode(mode)
+        if mode == ForwardMode.EVAL:
+            self.mode = eval_mode
+        elif mode == ForwardMode.TRAIN:
+            self.mode = train_mode
+        else:
+            raise ValueError(f"unsupported mode: {mode}, should be 'eval' or 'train'.")
 
     def run(self, global_epoch: int = 1) -> None:
         """
@@ -110,7 +129,7 @@ class SupervisedEvaluator(Evaluator):
 
     Args:
         device: an object representing the device on which to run.
-        val_data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        val_data_loader: Ignite engine use data_loader to run, must be Iterable, typically be torch.DataLoader.
         network: use the network to run model forward.
         epoch_length: number of iterations for one epoch, default to `len(val_data_loader)`.
         non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously
@@ -128,13 +147,19 @@ class SupervisedEvaluator(Evaluator):
         val_handlers: every handler is a set of Ignite Event-Handlers, must have `attach` function, like:
             CheckpointHandler, StatsHandler, SegmentationSaver, etc.
         amp: whether to enable auto-mixed-precision evaluation, default is False.
+        mode: model forward mode during evaluation, should be 'eval' or 'train',
+            which maps to `model.eval()` or `model.train()`, default to 'eval'.
+        event_names: additional custom ignite events that will register to the engine.
+            new events can be a list of str or `ignite.engine.events.EventEnum`.
+        event_to_attr: a dictionary to map an event to a state attribute, then add to `engine.state`.
+            for more details, check: https://github.com/pytorch/ignite/blob/v0.4.4.post1/ignite/engine/engine.py#L160
 
     """
 
     def __init__(
         self,
         device: torch.device,
-        val_data_loader: DataLoader,
+        val_data_loader: Union[Iterable, DataLoader],
         network: torch.nn.Module,
         epoch_length: Optional[int] = None,
         non_blocking: bool = False,
@@ -146,6 +171,9 @@ def __init__(
         additional_metrics: Optional[Dict[str, Metric]] = None,
         val_handlers: Optional[Sequence] = None,
         amp: bool = False,
+        mode: Union[ForwardMode, str] = ForwardMode.EVAL,
+        event_names: Optional[List[Union[str, EventEnum]]] = None,
+        event_to_attr: Optional[dict] = None,
     ) -> None:
         super().__init__(
             device=device,
@@ -159,15 +187,15 @@ def __init__(
             additional_metrics=additional_metrics,
             val_handlers=val_handlers,
             amp=amp,
+            mode=mode,
+            # add the iteration events
+            event_names=[IterationEvents] if event_names is None else event_names + [IterationEvents],
+            event_to_attr=event_to_attr,
         )
 
         self.network = network
         self.inferer = SimpleInferer() if inferer is None else inferer
 
-    def _register_additional_events(self):
-        super()._register_additional_events()
-        self.register_events(*IterationEvents)
-
     def _iteration(self, engine: Engine, batchdata: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
         """
         callback function for the Supervised Evaluation processing logic of 1 iteration in Ignite Engine.
@@ -195,17 +223,18 @@ def _iteration(self, engine: Engine, batchdata: Dict[str, torch.Tensor]) -> Dict
             inputs, targets, args, kwargs = batch
 
         # put iteration outputs into engine.state
-        engine.state.output = output = {Keys.IMAGE: inputs, Keys.LABEL: targets}
+        engine.state.output = {Keys.IMAGE: inputs, Keys.LABEL: targets}
         # execute forward computation
-        with eval_mode(self.network):
+        with self.mode(self.network):
             if self.amp:
                 with torch.cuda.amp.autocast():
-                    output[Keys.PRED] = self.inferer(inputs, self.network, *args, **kwargs)
+                    engine.state.output[Keys.PRED] = self.inferer(inputs, self.network, *args, **kwargs)
             else:
-                output[Keys.PRED] = self.inferer(inputs, self.network, *args, **kwargs)
+                engine.state.output[Keys.PRED] = self.inferer(inputs, self.network, *args, **kwargs)
         engine.fire_event(IterationEvents.FORWARD_COMPLETED)
+        engine.fire_event(IterationEvents.MODEL_COMPLETED)
 
-        return output
+        return engine.state.output
 
 
 class EnsembleEvaluator(Evaluator):
@@ -215,7 +244,7 @@ class EnsembleEvaluator(Evaluator):
 
     Args:
         device: an object representing the device on which to run.
-        val_data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        val_data_loader: Ignite engine use data_loader to run, must be Iterable, typically be torch.DataLoader.
         epoch_length: number of iterations for one epoch, default to `len(val_data_loader)`.
         networks: use the networks to run model forward in order.
         pred_keys: the keys to store every prediction data.
@@ -235,13 +264,19 @@ class EnsembleEvaluator(Evaluator):
         val_handlers: every handler is a set of Ignite Event-Handlers, must have `attach` function, like:
             CheckpointHandler, StatsHandler, SegmentationSaver, etc.
         amp: whether to enable auto-mixed-precision evaluation, default is False.
+        mode: model forward mode during evaluation, should be 'eval' or 'train',
+            which maps to `model.eval()` or `model.train()`, default to 'eval'.
+        event_names: additional custom ignite events that will register to the engine.
+            new events can be a list of str or `ignite.engine.events.EventEnum`.
+        event_to_attr: a dictionary to map an event to a state attribute, then add to `engine.state`.
+            for more details, check: https://github.com/pytorch/ignite/blob/v0.4.4.post1/ignite/engine/engine.py#L160
 
     """
 
     def __init__(
         self,
         device: torch.device,
-        val_data_loader: DataLoader,
+        val_data_loader: Union[Iterable, DataLoader],
         networks: Sequence[torch.nn.Module],
         pred_keys: Sequence[str],
         epoch_length: Optional[int] = None,
@@ -254,6 +289,9 @@ def __init__(
         additional_metrics: Optional[Dict[str, Metric]] = None,
         val_handlers: Optional[Sequence] = None,
         amp: bool = False,
+        mode: Union[ForwardMode, str] = ForwardMode.EVAL,
+        event_names: Optional[List[Union[str, EventEnum]]] = None,
+        event_to_attr: Optional[dict] = None,
     ) -> None:
         super().__init__(
             device=device,
@@ -267,16 +305,16 @@ def __init__(
             additional_metrics=additional_metrics,
             val_handlers=val_handlers,
             amp=amp,
+            mode=mode,
+            # add the iteration events
+            event_names=[IterationEvents] if event_names is None else event_names + [IterationEvents],
+            event_to_attr=event_to_attr,
         )
 
         self.networks = ensure_tuple(networks)
         self.pred_keys = ensure_tuple(pred_keys)
         self.inferer = SimpleInferer() if inferer is None else inferer
 
-    def _register_additional_events(self):
-        super()._register_additional_events()
-        self.register_events(*IterationEvents)
-
     def _iteration(self, engine: Engine, batchdata: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
         """
         callback function for the Supervised Evaluation processing logic of 1 iteration in Ignite Engine.
@@ -307,14 +345,17 @@ def _iteration(self, engine: Engine, batchdata: Dict[str, torch.Tensor]) -> Dict
             inputs, targets, args, kwargs = batch
 
         # put iteration outputs into engine.state
-        engine.state.output = output = {Keys.IMAGE: inputs, Keys.LABEL: targets}
+        engine.state.output = {Keys.IMAGE: inputs, Keys.LABEL: targets}
         for idx, network in enumerate(self.networks):
-            with eval_mode(network):
+            with self.mode(network):
                 if self.amp:
                     with torch.cuda.amp.autocast():
-                        output.update({self.pred_keys[idx]: self.inferer(inputs, network, *args, **kwargs)})
+                        engine.state.output.update(
+                            {self.pred_keys[idx]: self.inferer(inputs, network, *args, **kwargs)}
+                        )
                 else:
-                    output.update({self.pred_keys[idx]: self.inferer(inputs, network, *args, **kwargs)})
+                    engine.state.output.update({self.pred_keys[idx]: self.inferer(inputs, network, *args, **kwargs)})
         engine.fire_event(IterationEvents.FORWARD_COMPLETED)
+        engine.fire_event(IterationEvents.MODEL_COMPLETED)
 
-        return output
+        return engine.state.output
diff --git a/monai/engines/multi_gpu_supervised_trainer.py b/monai/engines/multi_gpu_supervised_trainer.py
index d12e012a56..d0e09443fa 100644
--- a/monai/engines/multi_gpu_supervised_trainer.py
+++ b/monai/engines/multi_gpu_supervised_trainer.py
@@ -19,15 +19,15 @@
 from monai.engines.utils import get_devices_spec
 from monai.utils import exact_version, optional_import
 
-create_supervised_trainer, _ = optional_import("ignite.engine", "0.4.2", exact_version, "create_supervised_trainer")
-create_supervised_evaluator, _ = optional_import("ignite.engine", "0.4.2", exact_version, "create_supervised_evaluator")
-_prepare_batch, _ = optional_import("ignite.engine", "0.4.2", exact_version, "_prepare_batch")
+create_supervised_trainer, _ = optional_import("ignite.engine", "0.4.4", exact_version, "create_supervised_trainer")
+create_supervised_evaluator, _ = optional_import("ignite.engine", "0.4.4", exact_version, "create_supervised_evaluator")
+_prepare_batch, _ = optional_import("ignite.engine", "0.4.4", exact_version, "_prepare_batch")
 if TYPE_CHECKING:
     from ignite.engine import Engine
     from ignite.metrics import Metric
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
-    Metric, _ = optional_import("ignite.metrics", "0.4.2", exact_version, "Metric")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+    Metric, _ = optional_import("ignite.metrics", "0.4.4", exact_version, "Metric")
 
 __all__ = [
     "create_multigpu_supervised_trainer",
diff --git a/monai/engines/trainer.py b/monai/engines/trainer.py
index efb2ab12fa..e9e31a1b16 100644
--- a/monai/engines/trainer.py
+++ b/monai/engines/trainer.py
@@ -9,25 +9,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Sequence, Tuple, Union
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union
 
 import torch
 from torch.optim.optimizer import Optimizer
 from torch.utils.data import DataLoader
 
-from monai.engines.utils import CommonKeys as Keys
 from monai.engines.utils import GanKeys, IterationEvents, default_make_latent, default_prepare_batch
 from monai.engines.workflow import Workflow
 from monai.inferers import Inferer, SimpleInferer
 from monai.transforms import Transform
 from monai.utils import exact_version, optional_import
+from monai.utils.enums import CommonKeys as Keys
 
 if TYPE_CHECKING:
-    from ignite.engine import Engine
+    from ignite.engine import Engine, EventEnum
     from ignite.metrics import Metric
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
-    Metric, _ = optional_import("ignite.metrics", "0.4.2", exact_version, "Metric")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+    Metric, _ = optional_import("ignite.metrics", "0.4.4", exact_version, "Metric")
+    EventEnum, _ = optional_import("ignite.engine", "0.4.4", exact_version, "EventEnum")
 
 __all__ = ["Trainer", "SupervisedTrainer", "GanTrainer"]
 
@@ -58,7 +59,7 @@ class SupervisedTrainer(Trainer):
     Args:
         device: an object representing the device on which to run.
         max_epochs: the total epoch number for trainer to run.
-        train_data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        train_data_loader: Ignite engine use data_loader to run, must be Iterable or torch.DataLoader.
         network: to train with this network.
         optimizer: the optimizer associated to the network.
         loss_function: the loss function associated to the optimizer.
@@ -78,6 +79,10 @@ class SupervisedTrainer(Trainer):
         train_handlers: every handler is a set of Ignite Event-Handlers, must have `attach` function, like:
             CheckpointHandler, StatsHandler, SegmentationSaver, etc.
         amp: whether to enable auto-mixed-precision training, default is False.
+        event_names: additional custom ignite events that will register to the engine.
+            new events can be a list of str or `ignite.engine.events.EventEnum`.
+        event_to_attr: a dictionary to map an event to a state attribute, then add to `engine.state`.
+            for more details, check: https://github.com/pytorch/ignite/blob/v0.4.4.post1/ignite/engine/engine.py#L160
 
     """
 
@@ -85,7 +90,7 @@ def __init__(
         self,
         device: torch.device,
         max_epochs: int,
-        train_data_loader: DataLoader,
+        train_data_loader: Union[Iterable, DataLoader],
         network: torch.nn.Module,
         optimizer: Optimizer,
         loss_function: Callable,
@@ -99,8 +104,9 @@ def __init__(
         additional_metrics: Optional[Dict[str, Metric]] = None,
         train_handlers: Optional[Sequence] = None,
         amp: bool = False,
+        event_names: Optional[List[Union[str, EventEnum]]] = None,
+        event_to_attr: Optional[dict] = None,
     ) -> None:
-        # set up Ignite engine and environments
         super().__init__(
             device=device,
             max_epochs=max_epochs,
@@ -114,6 +120,9 @@ def __init__(
             additional_metrics=additional_metrics,
             handlers=train_handlers,
             amp=amp,
+            # add the iteration events
+            event_names=[IterationEvents] if event_names is None else event_names + [IterationEvents],
+            event_to_attr=event_to_attr,
         )
 
         self.network = network
@@ -121,10 +130,6 @@ def __init__(
         self.loss_function = loss_function
         self.inferer = SimpleInferer() if inferer is None else inferer
 
-    def _register_additional_events(self):
-        super()._register_additional_events()
-        self.register_events(*IterationEvents)
-
     def _iteration(self, engine: Engine, batchdata: Dict[str, torch.Tensor]):
         """
         Callback function for the Supervised Training processing logic of 1 iteration in Ignite Engine.
@@ -152,12 +157,12 @@ def _iteration(self, engine: Engine, batchdata: Dict[str, torch.Tensor]):
         else:
             inputs, targets, args, kwargs = batch
         # put iteration outputs into engine.state
-        engine.state.output = output = {Keys.IMAGE: inputs, Keys.LABEL: targets}
+        engine.state.output = {Keys.IMAGE: inputs, Keys.LABEL: targets}
 
         def _compute_pred_loss():
-            output[Keys.PRED] = self.inferer(inputs, self.network, *args, **kwargs)
+            engine.state.output[Keys.PRED] = self.inferer(inputs, self.network, *args, **kwargs)
             engine.fire_event(IterationEvents.FORWARD_COMPLETED)
-            output[Keys.LOSS] = self.loss_function(output[Keys.PRED], targets).mean()
+            engine.state.output[Keys.LOSS] = self.loss_function(engine.state.output[Keys.PRED], targets).mean()
             engine.fire_event(IterationEvents.LOSS_COMPLETED)
 
         self.network.train()
@@ -165,18 +170,18 @@ def _compute_pred_loss():
         if self.amp and self.scaler is not None:
             with torch.cuda.amp.autocast():
                 _compute_pred_loss()
-            self.scaler.scale(output[Keys.LOSS]).backward()
+            self.scaler.scale(engine.state.output[Keys.LOSS]).backward()
             engine.fire_event(IterationEvents.BACKWARD_COMPLETED)
             self.scaler.step(self.optimizer)
             self.scaler.update()
         else:
             _compute_pred_loss()
-            output[Keys.LOSS].backward()
+            engine.state.output[Keys.LOSS].backward()
             engine.fire_event(IterationEvents.BACKWARD_COMPLETED)
             self.optimizer.step()
-        engine.fire_event(IterationEvents.OPTIMIZER_COMPLETED)
+        engine.fire_event(IterationEvents.MODEL_COMPLETED)
 
-        return output
+        return engine.state.output
 
 
 class GanTrainer(Trainer):
@@ -251,6 +256,9 @@ def __init__(
         additional_metrics: Optional[Dict[str, Metric]] = None,
         train_handlers: Optional[Sequence] = None,
     ):
+        if not isinstance(train_data_loader, DataLoader):
+            raise ValueError("train_data_loader must be PyTorch DataLoader.")
+
         # set up Ignite engine and environments
         super().__init__(
             device=device,
@@ -296,7 +304,7 @@ def _iteration(
             raise ValueError("must provide batch data for current iteration.")
 
         d_input = self.prepare_batch(batchdata, engine.state.device, engine.non_blocking)
-        batch_size = self.data_loader.batch_size
+        batch_size = self.data_loader.batch_size  # type: ignore
         g_input = self.g_prepare_batch(batch_size, self.latent_shape, engine.state.device, engine.non_blocking)
         g_output = self.g_inferer(g_input, self.g_network)
 
diff --git a/monai/engines/utils.py b/monai/engines/utils.py
index 8f5899f2a5..d16ab3cfbb 100644
--- a/monai/engines/utils.py
+++ b/monai/engines/utils.py
@@ -14,15 +14,15 @@
 import torch
 
 from monai.utils import exact_version, optional_import
+from monai.utils.enums import CommonKeys
 
 if TYPE_CHECKING:
     from ignite.engine import EventEnum
 else:
-    EventEnum, _ = optional_import("ignite.engine", "0.4.2", exact_version, "EventEnum")
+    EventEnum, _ = optional_import("ignite.engine", "0.4.4", exact_version, "EventEnum")
 
 __all__ = [
     "IterationEvents",
-    "CommonKeys",
     "GanKeys",
     "get_devices_spec",
     "default_prepare_batch",
@@ -38,30 +38,14 @@ class IterationEvents(EventEnum):
     `FORWARD_COMPLETED` is the Event when `network(image, label)` completed.
     `LOSS_COMPLETED` is the Event when `loss(pred, label)` completed.
     `BACKWARD_COMPLETED` is the Event when `loss.backward()` completed.
+    `MODEL_COMPLETED` is the Event when all the model related operations completed.
 
     """
 
     FORWARD_COMPLETED = "forward_completed"
     LOSS_COMPLETED = "loss_completed"
     BACKWARD_COMPLETED = "backward_completed"
-    OPTIMIZER_COMPLETED = "optimizer_completed"
-
-
-class CommonKeys:
-    """
-    A set of common keys for dictionary based supervised training process.
-    `IMAGE` is the input image data.
-    `LABEL` is the training or evaluation label of segmentation or classification task.
-    `PRED` is the prediction data of model output.
-    `LOSS` is the loss value of current iteration.
-    `INFO` is some useful information during training or evaluation, like loss value, etc.
-
-    """
-
-    IMAGE = "image"
-    LABEL = "label"
-    PRED = "pred"
-    LOSS = "loss"
+    MODEL_COMPLETED = "model_completed"
 
 
 class GanKeys:
diff --git a/monai/engines/workflow.py b/monai/engines/workflow.py
index d6415c1966..4018dabc40 100644
--- a/monai/engines/workflow.py
+++ b/monai/engines/workflow.py
@@ -9,26 +9,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Callable, Dict, Optional, Sequence
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Sequence, Union
 
 import torch
 import torch.distributed as dist
 from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
 
-from monai.engines.utils import default_prepare_batch
+from monai.engines.utils import IterationEvents, default_prepare_batch
 from monai.transforms import apply_transform
 from monai.utils import ensure_tuple, exact_version, optional_import
 
-IgniteEngine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
-State, _ = optional_import("ignite.engine", "0.4.2", exact_version, "State")
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
+IgniteEngine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+State, _ = optional_import("ignite.engine", "0.4.4", exact_version, "State")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+
 if TYPE_CHECKING:
-    from ignite.engine import Engine
+    from ignite.engine import Engine, EventEnum
     from ignite.metrics import Metric
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
-    Metric, _ = optional_import("ignite.metrics", "0.4.2", exact_version, "Metric")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+    Metric, _ = optional_import("ignite.metrics", "0.4.4", exact_version, "Metric")
+    EventEnum, _ = optional_import("ignite.engine", "0.4.4", exact_version, "EventEnum")
 
 
 class Workflow(IgniteEngine):  # type: ignore[valid-type, misc] # due to optional_import
@@ -44,7 +46,7 @@ class Workflow(IgniteEngine):  # type: ignore[valid-type, misc] # due to optiona
     Args:
         device: an object representing the device on which to run.
         max_epochs: the total epoch number for engine to run, validator and evaluator have only 1 epoch.
-        data_loader: Ignite engine use data_loader to run, must be torch.DataLoader.
+        data_loader: Ignite engine use data_loader to run, must be Iterable or torch.DataLoader.
         epoch_length: number of iterations for one epoch, default to `len(data_loader)`.
         non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously
             with respect to the host. For other cases, this argument has no effect.
@@ -60,6 +62,10 @@ class Workflow(IgniteEngine):  # type: ignore[valid-type, misc] # due to optiona
         handlers: every handler is a set of Ignite Event-Handlers, must have `attach` function, like:
             CheckpointHandler, StatsHandler, SegmentationSaver, etc.
         amp: whether to enable auto-mixed-precision training or inference, default is False.
+        event_names: additional custom ignite events that will register to the engine.
+            new events can be a list of str or `ignite.engine.events.EventEnum`.
+        event_to_attr: a dictionary to map an event to a state attribute, then add to `engine.state`.
+            for more details, check: https://github.com/pytorch/ignite/blob/v0.4.4.post1/ignite/engine/engine.py#L160
 
     Raises:
         TypeError: When ``device`` is not a ``torch.Device``.
@@ -73,7 +79,7 @@ def __init__(
         self,
         device: torch.device,
         max_epochs: int,
-        data_loader: DataLoader,
+        data_loader: Union[Iterable, DataLoader],
         epoch_length: Optional[int] = None,
         non_blocking: bool = False,
         prepare_batch: Callable = default_prepare_batch,
@@ -83,6 +89,8 @@ def __init__(
         additional_metrics: Optional[Dict[str, Metric]] = None,
         handlers: Optional[Sequence] = None,
         amp: bool = False,
+        event_names: Optional[List[Union[str, EventEnum]]] = None,
+        event_to_attr: Optional[dict] = None,
     ) -> None:
         if iteration_update is not None:
             super().__init__(iteration_update)
@@ -90,14 +98,20 @@ def __init__(
             super().__init__(self._iteration)
         if not isinstance(device, torch.device):
             raise TypeError(f"device must be a torch.device but is {type(device).__name__}.")
-        if not isinstance(data_loader, DataLoader):
-            raise TypeError(f"data_loader must be a torch.utils.data.DataLoader but is {type(data_loader).__name__}.")
-        sampler = data_loader.__dict__["sampler"]
-        if isinstance(sampler, DistributedSampler):
 
-            @self.on(Events.EPOCH_STARTED)
-            def set_sampler_epoch(engine: Engine):
-                sampler.set_epoch(engine.state.epoch)
+        if isinstance(data_loader, DataLoader):
+            sampler = data_loader.__dict__["sampler"]
+            if isinstance(sampler, DistributedSampler):
+
+                @self.on(Events.EPOCH_STARTED)
+                def set_sampler_epoch(engine: Engine):
+                    sampler.set_epoch(engine.state.epoch)
+
+            if epoch_length is None:
+                epoch_length = len(data_loader)
+        else:
+            if epoch_length is None:
+                raise ValueError("if data_loader is not PyTorch DataLoader, must specify the epoch_length.")
 
         # set all sharable data for the workflow based on Ignite engine.state
         self.state = State(
@@ -106,7 +120,7 @@ def set_sampler_epoch(engine: Engine):
             iteration=0,
             epoch=0,
             max_epochs=max_epochs,
-            epoch_length=len(data_loader) if epoch_length is None else epoch_length,
+            epoch_length=epoch_length,
             output=None,
             batch=None,
             metrics={},
@@ -122,7 +136,17 @@ def set_sampler_epoch(engine: Engine):
         self.prepare_batch = prepare_batch
         self.amp = amp
 
-        self._register_additional_events()
+        if event_names is not None:
+            if not isinstance(event_names, list):
+                raise ValueError("event_names must be a list or string or EventEnum.")
+            for name in event_names:
+                if isinstance(name, str):
+                    self.register_events(name, event_to_attr=event_to_attr)
+                elif issubclass(name, EventEnum):
+                    self.register_events(*name, event_to_attr=event_to_attr)
+                else:
+                    raise ValueError("event_names must be a list or string or EventEnum.")
+
         if post_transform is not None:
             self._register_post_transforms(post_transform)
         if key_metric is not None:
@@ -130,26 +154,17 @@ def set_sampler_epoch(engine: Engine):
         if handlers is not None:
             self._register_handlers(handlers)
 
-    def _register_additional_events(self):
-        """
-        Register more ignite Events to the engine.
-
-        """
-        pass
-
-    def _register_post_transforms(self, posttrans):
+    def _register_post_transforms(self, posttrans: Callable):
         """
         Register the post transforms to the engine, will execute them as a chain when iteration completed.
 
         """
 
-        @self.on(Events.ITERATION_COMPLETED)
+        @self.on(IterationEvents.MODEL_COMPLETED)
         def run_post_transform(engine: Engine) -> None:
-            if posttrans is None:
-                raise AssertionError
             engine.state.output = apply_transform(posttrans, engine.state.output)
 
-    def _register_metrics(self, k_metric, add_metrics):
+    def _register_metrics(self, k_metric: Dict, add_metrics: Optional[Dict] = None):
         """
         Register the key metric and additional metrics to the engine, supports ignite Metrics.
 
@@ -174,7 +189,7 @@ def _compare_metrics(engine: Engine) -> None:
                     engine.state.best_metric = current_val_metric
                     engine.state.best_metric_epoch = engine.state.epoch
 
-    def _register_handlers(self, handlers):
+    def _register_handlers(self, handlers: Sequence):
         """
         Register the handlers to the engine, supports ignite Handlers with `attach` API.
 
diff --git a/monai/handlers/__init__.py b/monai/handlers/__init__.py
index 8f73f7f2fd..b0dbb82127 100644
--- a/monai/handlers/__init__.py
+++ b/monai/handlers/__init__.py
@@ -13,18 +13,22 @@
 from .checkpoint_saver import CheckpointSaver
 from .classification_saver import ClassificationSaver
 from .confusion_matrix import ConfusionMatrix
+from .earlystop_handler import EarlyStopHandler
+from .garbage_collector import GarbageCollector
 from .hausdorff_distance import HausdorffDistance
 from .iteration_metric import IterationMetric
 from .lr_schedule_handler import LrScheduleHandler
 from .mean_dice import MeanDice
-from .metric_logger import MetricLogger
+from .metric_logger import MetricLogger, MetricLoggerKeys
 from .metrics_saver import MetricsSaver
+from .parameter_scheduler import ParamSchedulerHandler
 from .roc_auc import ROCAUC
 from .segmentation_saver import SegmentationSaver
 from .smartcache_handler import SmartCacheHandler
 from .stats_handler import StatsHandler
 from .surface_distance import SurfaceDistance
 from .tensorboard_handlers import TensorBoardHandler, TensorBoardImageHandler, TensorBoardStatsHandler
+from .transform_inverter import TransformInverter
 from .utils import (
     evenly_divisible_all_gather,
     stopping_fn_from_loss,
diff --git a/monai/handlers/checkpoint_loader.py b/monai/handlers/checkpoint_loader.py
index 648cc8360a..6d8f065f1e 100644
--- a/monai/handlers/checkpoint_loader.py
+++ b/monai/handlers/checkpoint_loader.py
@@ -13,15 +13,16 @@
 from typing import TYPE_CHECKING, Dict, Optional
 
 import torch
+import torch.nn as nn
 
 from monai.utils import exact_version, optional_import
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
-Checkpoint, _ = optional_import("ignite.handlers", "0.4.2", exact_version, "Checkpoint")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+Checkpoint, _ = optional_import("ignite.handlers", "0.4.4", exact_version, "Checkpoint")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 
 class CheckpointLoader:
@@ -44,6 +45,12 @@ class CheckpointLoader:
             first load the module to CPU and then copy each parameter to where it was
             saved, which would result in all processes on the same machine using the
             same set of devices.
+        strict: whether to strictly enforce that the keys in `state_dict` match the keys
+            returned by `torch.nn.Module.state_dict` function. default to `True`.
+        strict_shape: whether to enforce the data shape of the matched layers in the checkpoint,
+            `if `False`, it will skip the layers that have different data shape with checkpoint content.
+            This can be useful advanced feature for transfer learning. users should totally
+            understand which layers will have different shape. default to `True`.
 
     """
 
@@ -53,6 +60,8 @@ def __init__(
         load_dict: Dict,
         name: Optional[str] = None,
         map_location: Optional[Dict] = None,
+        strict: bool = True,
+        strict_shape: bool = True,
     ) -> None:
         if load_path is None:
             raise AssertionError("must provide clear path to load checkpoint.")
@@ -63,6 +72,8 @@ def __init__(
         self.load_dict = load_dict
         self._name = name
         self.map_location = map_location
+        self.strict = strict
+        self.strict_shape = strict_shape
 
     def attach(self, engine: Engine) -> None:
         """
@@ -80,5 +91,30 @@ def __call__(self, engine: Engine) -> None:
         """
         checkpoint = torch.load(self.load_path, map_location=self.map_location)
 
-        Checkpoint.load_objects(to_load=self.load_dict, checkpoint=checkpoint)
+        if not self.strict_shape:
+            k, _ = list(self.load_dict.items())[0]
+            # single object and checkpoint is directly a state_dict
+            if len(self.load_dict) == 1 and k not in checkpoint:
+                checkpoint = {k: checkpoint}
+
+            # skip items that don't match data shape
+            for k, obj in self.load_dict.items():
+                if isinstance(obj, (nn.DataParallel, nn.parallel.DistributedDataParallel)):
+                    obj = obj.module
+                if isinstance(obj, torch.nn.Module):
+                    d = obj.state_dict()
+                    checkpoint[k] = {k: v for k, v in checkpoint[k].items() if k in d and v.shape == d[k].shape}
+
+        # save current max epochs setting in the engine, don't overwrite it if larger than max_epochs in checkpoint
+        prior_max_epochs = engine.state.max_epochs
+        Checkpoint.load_objects(to_load=self.load_dict, checkpoint=checkpoint, strict=self.strict)
+        if engine.state.epoch > prior_max_epochs:
+            raise ValueError(
+                f"Epoch count ({engine.state.epoch}) in checkpoint is larger than "
+                f"the `engine.state.max_epochs` ({prior_max_epochs}) of engine. To further train from checkpoint, "
+                "construct trainer with `max_epochs` larger than checkpoint's epoch count. "
+                "To use checkpoint for inference, no need to load state_dict for the engine."
+            )
+        engine.state.max_epochs = prior_max_epochs
+
         self.logger.info(f"Restored all variables from {self.load_path}")
diff --git a/monai/handlers/checkpoint_saver.py b/monai/handlers/checkpoint_saver.py
index 1808e6b251..68857e17ff 100644
--- a/monai/handlers/checkpoint_saver.py
+++ b/monai/handlers/checkpoint_saver.py
@@ -15,16 +15,15 @@
 
 from monai.utils import exact_version, optional_import
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
-Checkpoint, _ = optional_import("ignite.handlers", "0.4.2", exact_version, "Checkpoint")
-BaseSaveHandler, _ = optional_import("ignite.handlers.checkpoint", "0.4.2", exact_version, "BaseSaveHandler")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+Checkpoint, _ = optional_import("ignite.handlers", "0.4.4", exact_version, "Checkpoint")
 
 if TYPE_CHECKING:
     from ignite.engine import Engine
     from ignite.handlers import DiskSaver
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
-    DiskSaver, _ = optional_import("ignite.handlers", "0.4.2", exact_version, "DiskSaver")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+    DiskSaver, _ = optional_import("ignite.handlers", "0.4.4", exact_version, "DiskSaver")
 
 
 class CheckpointSaver:
@@ -60,6 +59,8 @@ class CheckpointSaver:
             if `True`, then will save an object in the checkpoint file with key `checkpointer` to be consistent
             with ignite: https://github.com/pytorch/ignite/blob/master/ignite/handlers/checkpoint.py#L99.
             typically, it's used to resume training and compare current metric with previous N values.
+        key_metric_greater_or_equal: if `True`, the latest equally scored model is stored. Otherwise,
+            save the the first equally scored model. default to `False`.
         epoch_level: save checkpoint during training for every N epochs or every N iterations.
             `True` is epoch level, `False` is iteration level.
         save_interval: save checkpoint every N epochs, default is 0 to save no checkpoint.
@@ -90,6 +91,7 @@ def __init__(
         key_metric_n_saved: int = 1,
         key_metric_filename: Optional[str] = None,
         key_metric_save_state: bool = False,
+        key_metric_greater_or_equal: bool = False,
         epoch_level: bool = True,
         save_interval: int = 0,
         n_saved: Optional[int] = None,
@@ -113,7 +115,9 @@ class _DiskSaver(DiskSaver):
             """
 
             def __init__(self, dirname: str, filename: Optional[str] = None):
-                super().__init__(dirname=dirname, require_empty=False)
+                # set `atomic=False` as `atomic=True` only gives read/write permission to the user who saved the file,
+                # without group/others read permission
+                super().__init__(dirname=dirname, require_empty=False, atomic=False)
                 self.filename = filename
 
             def __call__(self, checkpoint: Dict, filename: str, metadata: Optional[Dict] = None) -> None:
@@ -163,6 +167,7 @@ def _score_func(engine: Engine):
                 score_name="key_metric",
                 n_saved=key_metric_n_saved,
                 include_self=key_metric_save_state,
+                greater_or_equal=key_metric_greater_or_equal,
             )
 
         if save_interval > 0:
diff --git a/monai/handlers/classification_saver.py b/monai/handlers/classification_saver.py
index 33ce7c7ec8..98f917330f 100644
--- a/monai/handlers/classification_saver.py
+++ b/monai/handlers/classification_saver.py
@@ -17,12 +17,12 @@
 from monai.utils import ImageMetaKey as Key
 from monai.utils import exact_version, optional_import
 
-idist, _ = optional_import("ignite", "0.4.2", exact_version, "distributed")
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
+idist, _ = optional_import("ignite", "0.4.4", exact_version, "distributed")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 
 class ClassificationSaver:
diff --git a/monai/handlers/confusion_matrix.py b/monai/handlers/confusion_matrix.py
index 1741aa305a..551fd29199 100644
--- a/monai/handlers/confusion_matrix.py
+++ b/monai/handlers/confusion_matrix.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Callable, Optional
+from typing import Any, Callable, Union
 
 import torch
 
@@ -28,7 +28,7 @@ def __init__(
         include_background: bool = True,
         metric_name: str = "hit_rate",
         output_transform: Callable = lambda x: x,
-        device: Optional[torch.device] = None,
+        device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
     ) -> None:
         """
diff --git a/monai/handlers/earlystop_handler.py b/monai/handlers/earlystop_handler.py
new file mode 100644
index 0000000000..0d140a9994
--- /dev/null
+++ b/monai/handlers/earlystop_handler.py
@@ -0,0 +1,95 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING, Callable, Optional
+
+from monai.utils import exact_version, optional_import
+
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+EarlyStopping, _ = optional_import("ignite.handlers", "0.4.4", exact_version, "EarlyStopping")
+
+if TYPE_CHECKING:
+    from ignite.engine import Engine
+else:
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+
+
+class EarlyStopHandler:
+    """
+    EarlyStopHandler acts as an Ignite handler to stop training if no improvement after a given number of events.
+    It‘s based on the `EarlyStopping` handler in ignite.
+
+    Args:
+        patience: number of events to wait if no improvement and then stop the training.
+        score_function: It should be a function taking a single argument, an :class:`~ignite.engine.engine.Engine`
+            object that the handler attached, can be a trainer or validator, and return a score `float`.
+            an improvement is considered if the score is higher.
+        trainer: trainer engine to stop the run if no improvement, if None, must call `set_trainer()` before training.
+        min_delta: a minimum increase in the score to qualify as an improvement,
+            i.e. an increase of less than or equal to `min_delta`, will count as no improvement.
+        cumulative_delta: if True, `min_delta` defines an increase since the last `patience` reset, otherwise,
+            it defines an increase after the last event, default to False.
+        epoch_level: check early stopping for every epoch or every iteration of the attached engine,
+            `True` is epoch level, `False` is iteration level, default to epoch level.
+
+    Note:
+        If in distributed training and uses loss value of every iteration to detect early stopping,
+        the values may be different in different ranks.
+        User may attach this handler to validator engine to detect validation metrics and stop the training,
+        in this case, the `score_function` is executed on validator engine and `trainer` is the trainer engine.
+
+    """
+
+    def __init__(
+        self,
+        patience: int,
+        score_function: Callable,
+        trainer: Optional[Engine] = None,
+        min_delta: float = 0.0,
+        cumulative_delta: bool = False,
+        epoch_level: bool = True,
+    ) -> None:
+        self.patience = patience
+        self.score_function = score_function
+        self.min_delta = min_delta
+        self.cumulative_delta = cumulative_delta
+        self.epoch_level = epoch_level
+        self._handler = None
+
+        if trainer is not None:
+            self.set_trainer(trainer=trainer)
+
+    def attach(self, engine: Engine) -> None:
+        """
+        Args:
+            engine: Ignite Engine, it can be a trainer, validator or evaluator.
+        """
+        if self.epoch_level:
+            engine.add_event_handler(Events.EPOCH_COMPLETED, self)
+        else:
+            engine.add_event_handler(Events.ITERATION_COMPLETED, self)
+
+    def set_trainer(self, trainer: Engine):
+        """
+        Set trainer to execute early stop if not setting properly in `__init__()`.
+        """
+        self._handler = EarlyStopping(
+            patience=self.patience,
+            score_function=self.score_function,
+            trainer=trainer,
+            min_delta=self.min_delta,
+            cumulative_delta=self.cumulative_delta,
+        )
+
+    def __call__(self, engine: Engine) -> None:
+        if self._handler is None:
+            raise RuntimeError("please set trainer in __init__() or call set_trainer() before training.")
+        self._handler(engine)
diff --git a/monai/handlers/garbage_collector.py b/monai/handlers/garbage_collector.py
new file mode 100644
index 0000000000..7bb59c9049
--- /dev/null
+++ b/monai/handlers/garbage_collector.py
@@ -0,0 +1,80 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gc
+from typing import TYPE_CHECKING
+
+from monai.utils import exact_version, optional_import
+
+if TYPE_CHECKING:
+    from ignite.engine import Engine, Events
+else:
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+    Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+
+
+class GarbageCollector:
+    """
+    Run garbage collector after each epoch
+
+    Args:
+        trigger_event: the event that trigger a call to this handler.
+            - "epoch", after completion of each epoch (equivalent of ignite.engine.Events.EPOCH_COMPLETED)
+            - "iteration", after completion of each iteration (equivalent of ignite.engine.Events.ITERATION_COMPLETED)
+            - any ignite built-in event from ignite.engine.Events.
+            Defaults to "epoch".
+        log_level: log level (integer) for some garbage collection information as below. Defaults to 10 (DEBUG).
+            - 50 (CRITICAL)
+            - 40 (ERROR)
+            - 30 (WARNING)
+            - 20 (INFO)
+            - 10 (DEBUG)
+            - 0 (NOTSET)
+    """
+
+    def __init__(self, trigger_event: str = "epoch", log_level: int = 10):
+        if isinstance(trigger_event, Events):
+            self.trigger_event = trigger_event
+        elif trigger_event.lower() == "epoch":
+            self.trigger_event = Events.EPOCH_COMPLETED
+        elif trigger_event.lower() == "iteration":
+            self.trigger_event = Events.ITERATION_COMPLETED
+        else:
+            raise ValueError(
+                f"'trigger_event' should be either epoch, iteration, or an ignite built-in event from"
+                f" ignite.engine.Events, '{trigger_event}' was given."
+            )
+
+        self.log_level = log_level
+
+    def attach(self, engine: Engine) -> None:
+        if not engine.has_event_handler(self, self.trigger_event):
+            engine.add_event_handler(self.trigger_event, self)
+
+    def __call__(self, engine: Engine) -> None:
+        """
+        This method calls python garbage collector.
+
+        Args:
+            engine: Ignite Engine, it should be either a trainer or validator.
+        """
+        # get count before garbage collection
+        pre_count = gc.get_count()
+        # fits call to garbage collector
+        gc.collect()
+        # second call to garbage collector
+        unreachable = gc.collect()
+        # get count after garbage collection
+        after_count = gc.get_count()
+        engine.logger.log(
+            self.log_level,
+            f"Garbage Count: [before: {pre_count}] -> [after: {after_count}] (unreachable : {unreachable})",
+        )
diff --git a/monai/handlers/hausdorff_distance.py b/monai/handlers/hausdorff_distance.py
index 7ac52d642a..042a587852 100644
--- a/monai/handlers/hausdorff_distance.py
+++ b/monai/handlers/hausdorff_distance.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional
+from typing import Callable, Optional, Union
 
 import torch
 
@@ -30,7 +30,7 @@ def __init__(
         percentile: Optional[float] = None,
         directed: bool = False,
         output_transform: Callable = lambda x: x,
-        device: Optional[torch.device] = None,
+        device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
     ) -> None:
         """
diff --git a/monai/handlers/iteration_metric.py b/monai/handlers/iteration_metric.py
index 641efad243..42f9828afd 100644
--- a/monai/handlers/iteration_metric.py
+++ b/monai/handlers/iteration_metric.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Any, Callable, List, Optional, Sequence
+from typing import TYPE_CHECKING, Any, Callable, List, Optional, Sequence, Union
 
 import torch
 
@@ -17,13 +17,13 @@
 from monai.metrics import do_metric_reduction
 from monai.utils import MetricReduction, exact_version, optional_import
 
-idist, _ = optional_import("ignite", "0.4.2", exact_version, "distributed")
-Metric, _ = optional_import("ignite.metrics", "0.4.2", exact_version, "Metric")
-reinit__is_reduced, _ = optional_import("ignite.metrics.metric", "0.4.2", exact_version, "reinit__is_reduced")
+idist, _ = optional_import("ignite", "0.4.4", exact_version, "distributed")
+Metric, _ = optional_import("ignite.metrics", "0.4.4", exact_version, "Metric")
+reinit__is_reduced, _ = optional_import("ignite.metrics.metric", "0.4.4", exact_version, "reinit__is_reduced")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 
 class IterationMetric(Metric):  # type: ignore[valid-type, misc] # due to optional_import
@@ -46,7 +46,7 @@ def __init__(
         self,
         metric_fn: Callable,
         output_transform: Callable = lambda x: x,
-        device: Optional[torch.device] = None,
+        device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
     ) -> None:
         self._is_reduced: bool = False
@@ -73,11 +73,19 @@ def update(self, output: Sequence[torch.Tensor]) -> None:
         """
         if len(output) != 2:
             raise ValueError(f"output must have length 2, got {len(output)}.")
+
         y_pred, y = output
-        score = self.metric_fn(y_pred, y)
-        if isinstance(score, (tuple, list)):
-            score = score[0]
-        self._scores.append(score)
+
+        def _compute(y_pred, y):
+            score = self.metric_fn(y_pred, y)
+            return score[0] if isinstance(score, (tuple, list)) else score
+
+        if isinstance(y_pred, (list, tuple)) and isinstance(y, (list, tuple)):
+            # if a list of channel-first data, add batch dim and compute metric, then concat the scores
+            score = torch.cat([_compute(p_.unsqueeze(0), y_.unsqueeze(0)) for p_, y_ in zip(y_pred, y)], dim=0)
+        else:
+            score = _compute(y_pred, y)
+        self._scores.append(score.to(self._device))
 
     def compute(self) -> Any:
         """
diff --git a/monai/handlers/lr_schedule_handler.py b/monai/handlers/lr_schedule_handler.py
index e5593f07ff..3b300537b2 100644
--- a/monai/handlers/lr_schedule_handler.py
+++ b/monai/handlers/lr_schedule_handler.py
@@ -16,11 +16,11 @@
 
 from monai.utils import ensure_tuple, exact_version, optional_import
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 
 class LrScheduleHandler:
diff --git a/monai/handlers/mean_dice.py b/monai/handlers/mean_dice.py
index 7decc3ab9b..6d51c534cf 100644
--- a/monai/handlers/mean_dice.py
+++ b/monai/handlers/mean_dice.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional
+from typing import Callable, Union
 
 import torch
 
@@ -27,7 +27,7 @@ def __init__(
         self,
         include_background: bool = True,
         output_transform: Callable = lambda x: x,
-        device: Optional[torch.device] = None,
+        device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
     ) -> None:
         """
diff --git a/monai/handlers/metric_logger.py b/monai/handlers/metric_logger.py
index fdd60da57c..f9a3913c56 100644
--- a/monai/handlers/metric_logger.py
+++ b/monai/handlers/metric_logger.py
@@ -10,23 +10,71 @@
 # limitations under the License.
 
 from collections import defaultdict
-from typing import TYPE_CHECKING, Callable, DefaultDict, List
+from enum import Enum
+from threading import RLock
+from typing import TYPE_CHECKING, Callable, DefaultDict, List, Optional
 
 from monai.utils import exact_version, optional_import
+from monai.utils.enums import CommonKeys
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+
+
+def _get_loss_from_output(output, loss_key: str = CommonKeys.LOSS):
+    return output[loss_key].item()
+
+
+class MetricLoggerKeys(Enum):
+    METRICS = "Metrics"
+    LOSS = "Loss"
 
 
 class MetricLogger:
-    def __init__(self, loss_transform: Callable = lambda x: x, metric_transform: Callable = lambda x: x) -> None:
+    """
+    Collect per-iteration metrics and loss value from the attached trainer. This will also collect metric values from
+    a given evaluator object which is expected to perform evaluation at the end of training epochs. This class is
+    useful for collecting loss and metric values in one place for storage with checkpoint savers (`state_dict` and
+    `load_state_dict` methods provided as expected by Pytorch and Ignite) and for graphing during training.
+
+    Example::
+        # construct an evaluator saving mean dice metric values in the key "val_mean_dice"
+        evaluator = SupervisedEvaluator(..., key_val_metric={"val_mean_dice": MeanDice(...)})
+
+        # construct the logger and associate with evaluator to extract metric values from
+        logger = MetricLogger(evaluator=evaluator)
+
+        # construct the trainer with the logger passed in as a handler so that it logs loss values
+        trainer = SupervisedTrainer(..., train_handlers=[logger, ValidationHandler(1, evaluator)])
+
+        # run training, logger.loss will be a list of (iteration, loss) values, logger.metrics a dict with key
+        # "val_mean_dice" storing a list of (iteration, metric) values
+        trainer.run()
+
+    Args:
+        loss_transform: Converts the `output` value from the trainer's state into a loss value
+        metric_transform: Converts the metric value coming from the trainer/evaluator's state into a storable value
+        evaluator: Optional evaluator to consume metric results from at the end of its evaluation run
+    """
+
+    def __init__(
+        self,
+        loss_transform: Callable = _get_loss_from_output,
+        metric_transform: Callable = lambda x: x,
+        evaluator: Optional[Engine] = None,
+    ) -> None:
         self.loss_transform = loss_transform
         self.metric_transform = metric_transform
         self.loss: List = []
         self.metrics: DefaultDict = defaultdict(list)
+        self.iteration = 0
+        self.lock = RLock()
+
+        if evaluator is not None:
+            self.attach_evaluator(evaluator)
 
     def attach(self, engine: Engine) -> None:
         """
@@ -35,21 +83,46 @@ def attach(self, engine: Engine) -> None:
         """
         engine.add_event_handler(Events.ITERATION_COMPLETED, self)
 
+    def attach_evaluator(self, evaluator: Engine) -> None:
+        """
+        Attach event  handlers to the given evaluator to log metric values from it.
+
+        Args:
+            evaluator: Ignite Engine implementing network evaluation
+        """
+        evaluator.add_event_handler(Events.COMPLETED, self.log_metrics)
+
     def __call__(self, engine: Engine) -> None:
         """
         Args:
             engine: Ignite Engine, it can be a trainer, validator or evaluator.
         """
-        self.loss.append(self.loss_transform(engine.state.output))
+        with self.lock:
+            self.iteration = engine.state.iteration
+            lossval = self.loss_transform(engine.state.output)
+
+            self.loss.append((self.iteration, lossval))
+            self.log_metrics(engine)
+
+    def log_metrics(self, engine: Engine) -> None:
+        """
+        Log metrics from the given Engine's state member.
+
+        Args:
+            engine: Ignite Engine to log from
+        """
+        with self.lock:
+            for m, v in engine.state.metrics.items():
+                v = self.metric_transform(v)
+                self.metrics[m].append((self.iteration, v))
 
-        for m, v in engine.state.metrics.items():
-            v = self.metric_transform(v)
-            #             # metrics may not be added on the first timestep, pad the list if this is the case
-            #             # so that each metric list is the same length as self.loss
-            #             if len(self.metrics[m])==0:
-            #                 self.metrics[m].append([v[0]]*len(self.loss))
+    def state_dict(self):
+        return {MetricLoggerKeys.LOSS: self.loss, MetricLoggerKeys.METRICS: self.metrics}
 
-            self.metrics[m].append(v)
+    def load_state_dict(self, state_dict):
+        self.loss[:] = state_dict[MetricLoggerKeys.LOSS]
+        self.metrics.clear()
+        self.metrics.update(state_dict[MetricLoggerKeys.METRICS])
 
 
 metriclogger = MetricLogger
diff --git a/monai/handlers/metrics_saver.py b/monai/handlers/metrics_saver.py
index 87d7223c96..082c370e48 100644
--- a/monai/handlers/metrics_saver.py
+++ b/monai/handlers/metrics_saver.py
@@ -15,12 +15,12 @@
 from monai.utils import ImageMetaKey as Key
 from monai.utils import ensure_tuple, exact_version, optional_import
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
-idist, _ = optional_import("ignite", "0.4.2", exact_version, "distributed")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+idist, _ = optional_import("ignite", "0.4.4", exact_version, "distributed")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 
 class MetricsSaver:
diff --git a/monai/handlers/parameter_scheduler.py b/monai/handlers/parameter_scheduler.py
new file mode 100644
index 0000000000..35ba044586
--- /dev/null
+++ b/monai/handlers/parameter_scheduler.py
@@ -0,0 +1,174 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from bisect import bisect_right
+from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union
+
+from monai.utils import exact_version, optional_import
+
+if TYPE_CHECKING:
+    from ignite.engine import Engine, Events
+else:
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+    Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+
+
+class ParamSchedulerHandler:
+    """
+    General purpose scheduler for parameters values. By default it can schedule in a linear, exponential, step or
+    multistep function. One can also pass Callables to have customized scheduling logic.
+
+    Args:
+        parameter_setter (Callable): Function that sets the required parameter
+        value_calculator (Union[str,Callable]): Either a string ('linear', 'exponential', 'step' or 'multistep')
+         or Callable for custom logic.
+        vc_kwargs (Dict): Dictionary that stores the required parameters for the value_calculator.
+        epoch_level (bool): Whether the the step is based on epoch or iteration. Defaults to False.
+        name (Optional[str]): Identifier of logging.logger to use, if None, defaulting to ``engine.logger``.
+        event (Optional[str]): Event to which the handler attaches. Defaults to Events.ITERATION_COMPLETED.
+    """
+
+    def __init__(
+        self,
+        parameter_setter: Callable,
+        value_calculator: Union[str, Callable],
+        vc_kwargs: Dict,
+        epoch_level: bool = False,
+        name: Optional[str] = None,
+        event=Events.ITERATION_COMPLETED,
+    ):
+        self.epoch_level = epoch_level
+        self.event = event
+
+        self._calculators = {
+            "linear": self._linear,
+            "exponential": self._exponential,
+            "step": self._step,
+            "multistep": self._multistep,
+        }
+
+        self._parameter_setter = parameter_setter
+        self._vc_kwargs = vc_kwargs
+        self._value_calculator = self._get_value_calculator(value_calculator=value_calculator)
+
+        self.logger = logging.getLogger(name)
+        self._name = name
+
+    def _get_value_calculator(self, value_calculator):
+        if isinstance(value_calculator, str):
+            return self._calculators[value_calculator]
+        if callable(value_calculator):
+            return value_calculator
+        raise ValueError(
+            f"value_calculator must be either a string from {list(self._calculators.keys())} or a Callable."
+        )
+
+    def __call__(self, engine: Engine):
+        if self.epoch_level:
+            self._vc_kwargs["current_step"] = engine.state.epoch
+        else:
+            self._vc_kwargs["current_step"] = engine.state.iteration
+
+        new_value = self._value_calculator(**self._vc_kwargs)
+        self._parameter_setter(new_value)
+
+    def attach(self, engine: Engine) -> None:
+        """
+        Args:
+            engine: Ignite Engine that is used for training.
+        """
+        if self._name is None:
+            self.logger = engine.logger
+        engine.add_event_handler(self.event, self)
+
+    @staticmethod
+    def _linear(
+        initial_value: float, step_constant: int, step_max_value: int, max_value: float, current_step: int
+    ) -> float:
+        """
+        Keeps the parameter value to zero until step_zero steps passed and then linearly increases it to 1 until an
+        additional step_one steps passed. Continues the trend until it reaches max_value.
+
+        Args:
+            initial_value (float): Starting value of the parameter.
+            step_constant (int): Step index until parameter's value is kept constant.
+            step_max_value (int): Step index at which parameter's value becomes max_value.
+            max_value (float): Max parameter value.
+            current_step (int): Current step index.
+
+        Returns:
+            float: new parameter value
+        """
+        if current_step <= step_constant:
+            delta = 0.0
+        elif current_step > step_max_value:
+            delta = max_value - initial_value
+        else:
+            delta = (max_value - initial_value) / (step_max_value - step_constant) * (current_step - step_constant)
+
+        return initial_value + delta
+
+    @staticmethod
+    def _exponential(initial_value: float, gamma: float, current_step: int) -> float:
+        """
+        Decays the parameter value by gamma every step.
+
+        Based on the closed form of ExponentialLR from Pytorch
+        https://github.com/pytorch/pytorch/blob/master/torch/optim/lr_scheduler.py#L457
+
+        Args:
+            initial_value (float): Starting value of the parameter.
+            gamma (float): Multiplicative factor of parameter value decay.
+            current_step (int): Current step index.
+
+        Returns:
+            float: new parameter value
+        """
+        return initial_value * gamma ** current_step
+
+    @staticmethod
+    def _step(initial_value: float, gamma: float, step_size: int, current_step: int) -> float:
+        """
+        Decays the parameter value by gamma every step_size.
+
+        Based on StepLR from Pytorch.
+        https://github.com/pytorch/pytorch/blob/master/torch/optim/lr_scheduler.py#L377
+
+        Args:
+            initial_value (float): Starting value of the parameter.
+            gamma (float): Multiplicative factor of parameter value decay.
+            step_size (int): Period of parameter value decay.
+            current_step (int): Current step index.
+
+        Returns
+            float: new parameter value
+        """
+        return initial_value * gamma ** (current_step // step_size)
+
+    @staticmethod
+    def _multistep(initial_value: float, gamma: float, milestones: List[int], current_step: int) -> float:
+        """
+        Decays the parameter value by gamma once the number of steps reaches one of the milestones.
+
+        Based on MultiStepLR from Pytorch.
+        https://github.com/pytorch/pytorch/blob/master/torch/optim/lr_scheduler.py#L424
+
+        Args:
+            initial_value (float): Starting value of the parameter.
+            gamma (float): Multiplicative factor of parameter value decay.
+            milestones (List[int]): List of step indices. Must be increasing.
+            current_step (int): Current step index.
+
+        Returns:
+            float: new parameter value
+        """
+        return initial_value * gamma ** bisect_right(milestones, current_step)
diff --git a/monai/handlers/roc_auc.py b/monai/handlers/roc_auc.py
index 2273b9ee89..8011dab8db 100644
--- a/monai/handlers/roc_auc.py
+++ b/monai/handlers/roc_auc.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Callable, Optional, Union
+from typing import Any, Callable, Union
 
 import torch
 
@@ -17,8 +17,8 @@
 from monai.metrics import compute_roc_auc
 from monai.utils import Average, exact_version, optional_import
 
-idist, _ = optional_import("ignite", "0.4.2", exact_version, "distributed")
-EpochMetric, _ = optional_import("ignite.metrics", "0.4.2", exact_version, "EpochMetric")
+idist, _ = optional_import("ignite", "0.4.4", exact_version, "distributed")
+EpochMetric, _ = optional_import("ignite.metrics", "0.4.4", exact_version, "EpochMetric")
 
 
 class ROCAUC(EpochMetric):  # type: ignore[valid-type, misc]  # due to optional_import
@@ -27,10 +27,6 @@ class ROCAUC(EpochMetric):  # type: ignore[valid-type, misc]  # due to optional_
     accumulating predictions and the ground-truth during an epoch and applying `compute_roc_auc`.
 
     Args:
-        to_onehot_y: whether to convert `y` into the one-hot format. Defaults to False.
-        softmax: whether to add softmax function to `y_pred` before computation. Defaults to False.
-        other_act: callable function to replace `softmax` as activation layer if needed, Defaults to ``None``.
-            for example: `other_act = lambda x: torch.log_softmax(x)`.
         average: {``"macro"``, ``"weighted"``, ``"micro"``, ``"none"``}
             Type of averaging performed if not binary classification. Defaults to ``"macro"``.
 
@@ -56,20 +52,14 @@ class ROCAUC(EpochMetric):  # type: ignore[valid-type, misc]  # due to optional_
 
     def __init__(
         self,
-        to_onehot_y: bool = False,
-        softmax: bool = False,
-        other_act: Optional[Callable] = None,
         average: Union[Average, str] = Average.MACRO,
         output_transform: Callable = lambda x: x,
-        device: Optional[torch.device] = None,
+        device: Union[str, torch.device] = "cpu",
     ) -> None:
         def _compute_fn(pred, label):
             return compute_roc_auc(
                 y_pred=pred,
                 y=label,
-                to_onehot_y=to_onehot_y,
-                softmax=softmax,
-                other_act=other_act,
                 average=Average(average),
             )
 
diff --git a/monai/handlers/segmentation_saver.py b/monai/handlers/segmentation_saver.py
index a46918b893..c3eca3b9c8 100644
--- a/monai/handlers/segmentation_saver.py
+++ b/monai/handlers/segmentation_saver.py
@@ -10,6 +10,7 @@
 # limitations under the License.
 
 import logging
+import warnings
 from typing import TYPE_CHECKING, Callable, Optional, Union
 
 import numpy as np
@@ -18,16 +19,19 @@
 from monai.transforms import SaveImage
 from monai.utils import GridSampleMode, GridSamplePadMode, InterpolateMode, exact_version, optional_import
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 
 class SegmentationSaver:
     """
     Event handler triggered on completing every iteration to save the segmentation predictions into files.
+    It can extract the input image meta data(filename, affine, original_shape, etc.) and resample the predictions
+    based on the meta data.
+
     """
 
     def __init__(
@@ -41,6 +45,8 @@ def __init__(
         scale: Optional[int] = None,
         dtype: DtypeLike = np.float64,
         output_dtype: DtypeLike = np.float32,
+        squeeze_end_dims: bool = True,
+        data_root_dir: str = "",
         batch_transform: Callable = lambda x: x,
         output_transform: Callable = lambda x: x,
         name: Optional[str] = None,
@@ -77,8 +83,24 @@ def __init__(
                 If None, use the data type of input data.
                 It's used for Nifti format only.
             output_dtype: data type for saving data. Defaults to ``np.float32``, it's used for Nifti format only.
+            squeeze_end_dims: if True, any trailing singleton dimensions will be removed (after the channel
+                has been moved to the end). So if input is (C,H,W,D), this will be altered to (H,W,D,C), and
+                then if C==1, it will be saved as (H,W,D). If D also ==1, it will be saved as (H,W). If false,
+                image will always be saved as (H,W,D,C).
+                it's used for NIfTI format only.
+            data_root_dir: if not empty, it specifies the beginning parts of the input file's
+                absolute path. it's used to compute `input_file_rel_path`, the relative path to the file from
+                `data_root_dir` to preserve folder structure when saving in case there are files in different
+                folders with the same file names. for example:
+                input_file_name: /foo/bar/test1/image.nii,
+                output_postfix: seg
+                output_ext: nii.gz
+                output_dir: /output,
+                data_root_dir: /foo/bar,
+                output will be: /output/test1/image/image_seg.nii.gz
             batch_transform: a callable that is used to transform the
                 ignite.engine.batch into expected format to extract the meta_data dictionary.
+                it can be used to extract the input image meta data: filename, affine, original_shape, etc.
             output_transform: a callable that is used to transform the
                 ignite.engine.output into the form expected image data.
                 The first dimension of this transform's output will be treated as the
@@ -96,8 +118,10 @@ def __init__(
             scale=scale,
             dtype=dtype,
             output_dtype=output_dtype,
-            save_batch=True,
+            squeeze_end_dims=squeeze_end_dims,
+            data_root_dir=data_root_dir,
         )
+        self.resample = resample
         self.batch_transform = batch_transform
         self.output_transform = output_transform
 
@@ -124,5 +148,16 @@ def __call__(self, engine: Engine) -> None:
         """
         meta_data = self.batch_transform(engine.state.batch)
         engine_output = self.output_transform(engine.state.output)
-        self._saver(engine_output, meta_data)
+        if isinstance(engine_output, (tuple, list)):
+            # if a list of data in shape: [channel, H, W, [D]], save every item separately
+            if self.resample:
+                warnings.warn("if saving inverted data, please set `resample=False` as it's already resampled.")
+
+            self._saver.save_batch = False
+            for i, d in enumerate(engine_output):
+                self._saver(d, {k: meta_data[k][i] for k in meta_data} if meta_data is not None else None)
+        else:
+            # if the data is in shape: [batch, channel, H, W, [D]]
+            self._saver.save_batch = True
+            self._saver(engine_output, meta_data)
         self.logger.info("saved all the model outputs into files.")
diff --git a/monai/handlers/smartcache_handler.py b/monai/handlers/smartcache_handler.py
index 423d87c22a..821f883d91 100644
--- a/monai/handlers/smartcache_handler.py
+++ b/monai/handlers/smartcache_handler.py
@@ -14,11 +14,11 @@
 from monai.data import SmartCacheDataset
 from monai.utils import exact_version, optional_import
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 
 class SmartCacheHandler:
diff --git a/monai/handlers/stats_handler.py b/monai/handlers/stats_handler.py
index 24d844569f..6d4a4e958b 100644
--- a/monai/handlers/stats_handler.py
+++ b/monai/handlers/stats_handler.py
@@ -17,11 +17,11 @@
 
 from monai.utils import exact_version, is_scalar, optional_import
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 DEFAULT_KEY_VAL_FORMAT = "{}: {:.4f} "
 DEFAULT_TAG = "Loss"
diff --git a/monai/handlers/surface_distance.py b/monai/handlers/surface_distance.py
index d3fa69bfce..7c2322354a 100644
--- a/monai/handlers/surface_distance.py
+++ b/monai/handlers/surface_distance.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Optional
+from typing import Callable, Union
 
 import torch
 
@@ -29,7 +29,7 @@ def __init__(
         symmetric: bool = False,
         distance_metric: str = "euclidean",
         output_transform: Callable = lambda x: x,
-        device: Optional[torch.device] = None,
+        device: Union[str, torch.device] = "cpu",
         save_details: bool = True,
     ) -> None:
         """
diff --git a/monai/handlers/tensorboard_handlers.py b/monai/handlers/tensorboard_handlers.py
index 4ee88bcfc9..57dad81fec 100644
--- a/monai/handlers/tensorboard_handlers.py
+++ b/monai/handlers/tensorboard_handlers.py
@@ -18,12 +18,12 @@
 from monai.utils import exact_version, is_scalar, optional_import
 from monai.visualize import plot_2d_or_3d_image
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
 if TYPE_CHECKING:
     from ignite.engine import Engine
     from torch.utils.tensorboard import SummaryWriter
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
     SummaryWriter, _ = optional_import("torch.utils.tensorboard", name="SummaryWriter")
 
 DEFAULT_TAG = "Loss"
@@ -79,7 +79,9 @@ def __init__(
         summary_writer: Optional[SummaryWriter] = None,
         log_dir: str = "./runs",
         epoch_event_writer: Optional[Callable[[Engine, SummaryWriter], Any]] = None,
+        epoch_interval: int = 1,
         iteration_event_writer: Optional[Callable[[Engine, SummaryWriter], Any]] = None,
+        iteration_interval: int = 1,
         output_transform: Callable = lambda x: x,
         global_epoch_transform: Callable = lambda x: x,
         tag_name: str = DEFAULT_TAG,
@@ -91,8 +93,10 @@ def __init__(
             log_dir: if using default SummaryWriter, write logs to this directory, default is `./runs`.
             epoch_event_writer: customized callable TensorBoard writer for epoch level.
                 Must accept parameter "engine" and "summary_writer", use default event writer if None.
+            epoch_interval: the epoch interval at which the epoch_event_writer is called. Defaults to 1.
             iteration_event_writer: customized callable TensorBoard writer for iteration level.
                 Must accept parameter "engine" and "summary_writer", use default event writer if None.
+            iteration_interval: the iteration interval at which the iteration_event_writer is called. Defaults to 1.
             output_transform: a callable that is used to transform the
                 ``ignite.engine.output`` into a scalar to plot, or a dictionary of {key: scalar}.
                 In the latter case, the output string will be formatted as key: value.
@@ -104,7 +108,9 @@ def __init__(
         """
         super().__init__(summary_writer=summary_writer, log_dir=log_dir)
         self.epoch_event_writer = epoch_event_writer
+        self.epoch_interval = epoch_interval
         self.iteration_event_writer = iteration_event_writer
+        self.iteration_interval = iteration_interval
         self.output_transform = output_transform
         self.global_epoch_transform = global_epoch_transform
         self.tag_name = tag_name
@@ -118,9 +124,11 @@ def attach(self, engine: Engine) -> None:
 
         """
         if not engine.has_event_handler(self.iteration_completed, Events.ITERATION_COMPLETED):
-            engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
+            engine.add_event_handler(
+                Events.ITERATION_COMPLETED(every=self.iteration_interval), self.iteration_completed
+            )
         if not engine.has_event_handler(self.epoch_completed, Events.EPOCH_COMPLETED):
-            engine.add_event_handler(Events.EPOCH_COMPLETED, self.epoch_completed)
+            engine.add_event_handler(Events.EPOCH_COMPLETED(every=self.epoch_interval), self.epoch_completed)
 
     def epoch_completed(self, engine: Engine) -> None:
         """
diff --git a/monai/handlers/transform_inverter.py b/monai/handlers/transform_inverter.py
new file mode 100644
index 0000000000..a8d5a509df
--- /dev/null
+++ b/monai/handlers/transform_inverter.py
@@ -0,0 +1,124 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from copy import deepcopy
+from typing import TYPE_CHECKING, Callable, Optional, Sequence, Union
+
+from torch.utils.data import DataLoader as TorchDataLoader
+
+from monai.data import BatchInverseTransform
+from monai.data.utils import no_collation
+from monai.engines.utils import CommonKeys, IterationEvents
+from monai.transforms import InvertibleTransform, ToTensor, allow_missing_keys_mode, convert_inverse_interp_mode
+from monai.utils import InverseKeys, ensure_tuple, ensure_tuple_rep, exact_version, optional_import
+
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+if TYPE_CHECKING:
+    from ignite.engine import Engine
+else:
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
+
+
+class TransformInverter:
+    """
+    Ignite handler to automatically invert `transforms`.
+    It takes `engine.state.output` as the input data and uses the transforms information from `engine.state.batch`.
+    The outputs are stored in `engine.state.output` with key: "{output_key}_{postfix}".
+    """
+
+    def __init__(
+        self,
+        transform: InvertibleTransform,
+        loader: TorchDataLoader,
+        output_keys: Union[str, Sequence[str]] = CommonKeys.PRED,
+        batch_keys: Union[str, Sequence[str]] = CommonKeys.IMAGE,
+        meta_key_postfix: str = "meta_dict",
+        collate_fn: Optional[Callable] = no_collation,
+        postfix: str = "inverted",
+        nearest_interp: Union[bool, Sequence[bool]] = True,
+        num_workers: Optional[int] = 0,
+    ) -> None:
+        """
+        Args:
+            transform: a callable data transform on input data.
+            loader: data loader used to run transforms and generate the batch of data.
+            output_keys: the key of expected data in `ignite.engine.output`, invert transforms on it.
+                it also can be a list of keys, will invert transform for each of them. Default to "pred".
+            batch_keys: the key of input data in `ignite.engine.batch`. will get the applied transforms
+                for this input data, then invert them for the expected data with `output_keys`.
+                It can also be a list of keys, each matches to the `output_keys` data. default to "image".
+            meta_key_postfix: use `{batch_key}_{postfix}` to to fetch the meta data according to the key data,
+                default is `meta_dict`, the meta data is a dictionary object.
+                For example, to handle key `image`,  read/write affine matrices from the
+                metadata `image_meta_dict` dictionary's `affine` field.
+            collate_fn: how to collate data after inverse transformations.
+                default won't do any collation, so the output will be a list of size batch size.
+            postfix: will save the inverted result into `ignite.engine.output` with key `{output_key}_{postfix}`.
+            nearest_interp: whether to use `nearest` interpolation mode when inverting the spatial transforms,
+                default to `True`. If `False`, use the same interpolation mode as the original transform.
+                it also can be a list of bool, each matches to the `output_keys` data.
+            num_workers: number of workers when run data loader for inverse transforms,
+                default to 0 as only run one iteration and multi-processing may be even slower.
+                Set to `None`, to use the `num_workers` of the input transform data loader.
+
+        """
+        self.transform = transform
+        self.inverter = BatchInverseTransform(
+            transform=transform,
+            loader=loader,
+            collate_fn=collate_fn,
+            num_workers=num_workers,
+        )
+        self.output_keys = ensure_tuple(output_keys)
+        self.batch_keys = ensure_tuple_rep(batch_keys, len(self.output_keys))
+        self.meta_key_postfix = meta_key_postfix
+        self.postfix = postfix
+        self.nearest_interp = ensure_tuple_rep(nearest_interp, len(self.output_keys))
+        self._totensor = ToTensor()
+
+    def attach(self, engine: Engine) -> None:
+        """
+        Args:
+            engine: Ignite Engine, it can be a trainer, validator or evaluator.
+        """
+        engine.add_event_handler(IterationEvents.MODEL_COMPLETED, self)
+
+    def __call__(self, engine: Engine) -> None:
+        """
+        Args:
+            engine: Ignite Engine, it can be a trainer, validator or evaluator.
+        """
+        for output_key, batch_key, nearest_interp in zip(self.output_keys, self.batch_keys, self.nearest_interp):
+            transform_key = batch_key + InverseKeys.KEY_SUFFIX
+            if transform_key not in engine.state.batch:
+                warnings.warn(f"all the transforms on `{batch_key}` are not InvertibleTransform.")
+                continue
+
+            transform_info = engine.state.batch[transform_key]
+            if nearest_interp:
+                transform_info = convert_inverse_interp_mode(
+                    trans_info=deepcopy(transform_info),
+                    mode="nearest",
+                    align_corners=None,
+                )
+
+            segs_dict = {
+                batch_key: engine.state.output[output_key].detach().cpu(),
+                transform_key: transform_info,
+            }
+            meta_dict_key = f"{batch_key}_{self.meta_key_postfix}"
+            if meta_dict_key in engine.state.batch:
+                segs_dict[meta_dict_key] = engine.state.batch[meta_dict_key]
+
+            with allow_missing_keys_mode(self.transform):  # type: ignore
+                inverted_key = f"{output_key}_{self.postfix}"
+                engine.state.output[inverted_key] = [self._totensor(i[batch_key]) for i in self.inverter(segs_dict)]
diff --git a/monai/handlers/utils.py b/monai/handlers/utils.py
index 3e36af0652..4ae38b908a 100644
--- a/monai/handlers/utils.py
+++ b/monai/handlers/utils.py
@@ -11,18 +11,18 @@
 
 import os
 from collections import OrderedDict
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Union
 
 import numpy as np
 import torch
 
 from monai.utils import ensure_tuple, exact_version, get_torch_version_tuple, optional_import
 
-idist, _ = optional_import("ignite", "0.4.2", exact_version, "distributed")
+idist, _ = optional_import("ignite", "0.4.4", exact_version, "distributed")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 __all__ = [
     "stopping_fn_from_metric",
@@ -33,7 +33,7 @@
 ]
 
 
-def stopping_fn_from_metric(metric_name: str) -> Callable[[Engine], Any]:
+def stopping_fn_from_metric(metric_name: str):
     """
     Returns a stopping function for ignite.handlers.EarlyStopping using the given metric name.
     """
@@ -44,7 +44,7 @@ def stopping_fn(engine: Engine):
     return stopping_fn
 
 
-def stopping_fn_from_loss() -> Callable[[Engine], Any]:
+def stopping_fn_from_loss():
     """
     Returns a stopping function for ignite.handlers.EarlyStopping using the loss value.
     """
@@ -75,7 +75,7 @@ def evenly_divisible_all_gather(data: torch.Tensor) -> torch.Tensor:
     # make sure the data is evenly-divisible on multi-GPUs
     length = data.shape[0]
     all_lens = idist.all_gather(length)
-    max_len = max(all_lens).item()
+    max_len = max(all_lens)
     if length < max_len:
         size = [max_len - length] + list(data.shape[1:])
         data = torch.cat([data, data.new_full(size, 0)], dim=0)
@@ -85,27 +85,39 @@ def evenly_divisible_all_gather(data: torch.Tensor) -> torch.Tensor:
     return torch.cat([data[i * max_len : i * max_len + l, ...] for i, l in enumerate(all_lens)], dim=0)
 
 
-def string_list_all_gather(strings: List[str], delimiter: str = "\t") -> List[str]:
+def string_list_all_gather(strings: List[str]) -> List[str]:
     """
     Utility function for distributed data parallel to all gather a list of strings.
+    Note that if the item in `strings` is longer than 1024 chars, it will be truncated to 1024:
+    https://github.com/pytorch/ignite/blob/master/ignite/distributed/comp_models/base.py#L92
 
     Args:
         strings: a list of strings to all gather.
-        delimiter: use the delimiter to join the string list to be a long string,
-            then all gather across ranks and split to a list. default to "\t".
 
     """
-    if idist.get_world_size() <= 1:
+    world_size = idist.get_world_size()
+    if world_size <= 1:
         return strings
 
-    _joined = delimiter.join(strings)
+    result: List[List[str]] = [[] for _ in range(world_size)]
+    # get length of strings
+    length = len(strings)
+    all_lens = idist.all_gather(length)
+    max_len = max(all_lens)
+    # pad the item to make sure the same length
+    if length < max_len:
+        strings = strings + ["" for _ in range(max_len - length)]
+
     if get_torch_version_tuple() > (1, 6, 0):
-        # all gather across all ranks
-        _joined = delimiter.join(idist.all_gather(_joined))
+        for s in strings:
+            gathered = idist.all_gather(s)
+            for i, g in enumerate(gathered):
+                if len(g) > 0:
+                    result[i].append(g)
     else:
         raise RuntimeError("string all_gather can not be supported in PyTorch < 1.7.0.")
 
-    return _joined.split(delimiter)
+    return [i for k in result for i in k]
 
 
 def write_metrics_reports(
@@ -151,7 +163,6 @@ def write_metrics_reports(
         with open(os.path.join(save_dir, "metrics.csv"), "w") as f:
             for k, v in metrics.items():
                 f.write(f"{k}{deli}{str(v)}\n")
-
     if metric_details is not None and len(metric_details) > 0:
         for k, v in metric_details.items():
             if isinstance(v, torch.Tensor):
diff --git a/monai/handlers/validation_handler.py b/monai/handlers/validation_handler.py
index 9cc2e926f4..fbd4b7862e 100644
--- a/monai/handlers/validation_handler.py
+++ b/monai/handlers/validation_handler.py
@@ -9,16 +9,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from monai.engines.evaluator import Evaluator
 from monai.utils import exact_version, optional_import
 
-Events, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Events")
+Events, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
 if TYPE_CHECKING:
     from ignite.engine import Engine
 else:
-    Engine, _ = optional_import("ignite.engine", "0.4.2", exact_version, "Engine")
+    Engine, _ = optional_import("ignite.engine", "0.4.4", exact_version, "Engine")
 
 
 class ValidationHandler:
@@ -28,11 +28,12 @@ class ValidationHandler:
 
     """
 
-    def __init__(self, validator: Evaluator, interval: int, epoch_level: bool = True) -> None:
+    def __init__(self, interval: int, validator: Optional[Evaluator] = None, epoch_level: bool = True) -> None:
         """
         Args:
-            validator: run the validator when trigger validation, suppose to be Evaluator.
             interval: do validation every N epochs or every N iterations during training.
+            validator: run the validator when trigger validation, suppose to be Evaluator.
+                if None, should call `set_validator()` before training.
             epoch_level: execute validation every N epochs or N iterations.
                 `True` is epoch level, `False` is iteration level.
 
@@ -40,12 +41,20 @@ def __init__(self, validator: Evaluator, interval: int, epoch_level: bool = True
             TypeError: When ``validator`` is not a ``monai.engines.evaluator.Evaluator``.
 
         """
-        if not isinstance(validator, Evaluator):
+        if validator is not None and not isinstance(validator, Evaluator):
             raise TypeError(f"validator must be a monai.engines.evaluator.Evaluator but is {type(validator).__name__}.")
         self.validator = validator
         self.interval = interval
         self.epoch_level = epoch_level
 
+    def set_validator(self, validator: Evaluator):
+        """
+        Set validator if not setting in the __init__().
+        """
+        if not isinstance(validator, Evaluator):
+            raise TypeError(f"validator must be a monai.engines.evaluator.Evaluator but is {type(validator).__name__}.")
+        self.validator = validator
+
     def attach(self, engine: Engine) -> None:
         """
         Args:
@@ -61,4 +70,6 @@ def __call__(self, engine: Engine) -> None:
         Args:
             engine: Ignite Engine, it can be a trainer, validator or evaluator.
         """
+        if self.validator is None:
+            raise RuntimeError("please set validator in __init__() or call `set_validator()` before training.")
         self.validator.run(engine.state.epoch)
diff --git a/monai/inferers/__init__.py b/monai/inferers/__init__.py
index 1cdea77b0f..030344728d 100644
--- a/monai/inferers/__init__.py
+++ b/monai/inferers/__init__.py
@@ -9,5 +9,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .inferer import Inferer, SimpleInferer, SlidingWindowInferer
+from .inferer import Inferer, SaliencyInferer, SimpleInferer, SlidingWindowInferer
 from .utils import sliding_window_inference
diff --git a/monai/inferers/inferer.py b/monai/inferers/inferer.py
index b17afb4e1d..ecb2c2c178 100644
--- a/monai/inferers/inferer.py
+++ b/monai/inferers/inferer.py
@@ -10,14 +10,16 @@
 # limitations under the License.
 
 from abc import ABC, abstractmethod
-from typing import Any, Callable, Sequence, Union
+from typing import Any, Callable, Optional, Sequence, Union
 
 import torch
+import torch.nn as nn
 
 from monai.inferers.utils import sliding_window_inference
 from monai.utils import BlendMode, PytorchPadMode
+from monai.visualize import CAM, GradCAM, GradCAMpp
 
-__all__ = ["Inferer", "SimpleInferer", "SlidingWindowInferer"]
+__all__ = ["Inferer", "SimpleInferer", "SlidingWindowInferer", "SaliencyInferer"]
 
 
 class Inferer(ABC):
@@ -190,3 +192,54 @@ def __call__(
             *args,
             **kwargs,
         )
+
+
+class SaliencyInferer(Inferer):
+    """
+    SaliencyInferer is inference with activation maps.
+
+    Args:
+        cam_name: expected CAM method name, should be: "CAM", "GradCAM" or "GradCAMpp".
+        target_layers: name of the model layer to generate the feature map.
+        class_idx: index of the class to be visualized. if None, default to argmax(logits).
+        args: other optional args to be passed to the `__init__` of cam.
+        kwargs: other optional keyword args to be passed to `__init__` of cam.
+
+    """
+
+    def __init__(self, cam_name: str, target_layers: str, class_idx: Optional[int] = None, *args, **kwargs) -> None:
+        Inferer.__init__(self)
+        if cam_name.lower() not in ("cam", "gradcam", "gradcampp"):
+            raise ValueError("cam_name should be: 'CAM', 'GradCAM' or 'GradCAMpp'.")
+        self.cam_name = cam_name.lower()
+        self.target_layers = target_layers
+        self.class_idx = class_idx
+        self.args = args
+        self.kwargs = kwargs
+
+    def __call__(  # type: ignore
+        self,
+        inputs: torch.Tensor,
+        network: nn.Module,
+        *args: Any,
+        **kwargs: Any,
+    ):
+        """Unified callable function API of Inferers.
+
+        Args:
+            inputs: model input data for inference.
+            network: target model to execute inference.
+                supports callables such as ``lambda x: my_torch_model(x, additional_config)``
+            args: other optional args to be passed to the `__call__` of cam.
+            kwargs: other optional keyword args to be passed to `__call__` of cam.
+
+        """
+        cam: Union[CAM, GradCAM, GradCAMpp]
+        if self.cam_name == "cam":
+            cam = CAM(network, self.target_layers, *self.args, **self.kwargs)
+        elif self.cam_name == "gradcam":
+            cam = GradCAM(network, self.target_layers, *self.args, **self.kwargs)
+        else:
+            cam = GradCAMpp(network, self.target_layers, *self.args, **self.kwargs)
+
+        return cam(inputs, self.class_idx, *args, **kwargs)
diff --git a/monai/losses/__init__.py b/monai/losses/__init__.py
index b9146a6962..78a0fbc191 100644
--- a/monai/losses/__init__.py
+++ b/monai/losses/__init__.py
@@ -13,11 +13,14 @@
 from .dice import (
     Dice,
     DiceCELoss,
+    DiceFocalLoss,
     DiceLoss,
     GeneralizedDiceLoss,
     GeneralizedWassersteinDiceLoss,
     MaskedDiceLoss,
     dice,
+    dice_ce,
+    dice_focal,
     generalized_dice,
     generalized_wasserstein_dice,
 )
diff --git a/monai/losses/dice.py b/monai/losses/dice.py
index c284660cc6..47af8ea171 100644
--- a/monai/losses/dice.py
+++ b/monai/losses/dice.py
@@ -10,7 +10,7 @@
 # limitations under the License.
 
 import warnings
-from typing import Callable, Optional, Union
+from typing import Callable, List, Optional, Sequence, Union
 
 import numpy as np
 import torch
@@ -18,6 +18,7 @@
 import torch.nn.functional as F
 from torch.nn.modules.loss import _Loss
 
+from monai.losses.focal_loss import FocalLoss
 from monai.networks import one_hot
 from monai.utils import LossReduction, Weight
 
@@ -54,7 +55,7 @@ def __init__(
     ) -> None:
         """
         Args:
-            include_background: if False channel index 0 (background category) is excluded from the calculation.
+            include_background: if False, channel index 0 (background category) is excluded from the calculation.
             to_onehot_y: whether to convert `y` into the one-hot format. Defaults to False.
             sigmoid: if True, apply a sigmoid function to the prediction.
             softmax: if True, apply a softmax function to the prediction.
@@ -101,10 +102,12 @@ def __init__(
     def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
         """
         Args:
-            input: the shape should be BNH[WD].
-            target: the shape should be BNH[WD].
+            input: the shape should be BNH[WD], where N is the number of classes.
+            target: the shape should be BNH[WD] or B1H[WD], where N is the number of classes.
 
         Raises:
+            AssertionError: When input and target (after one hot transform if setted)
+                have different shapes.
             ValueError: When ``self.reduction`` is not one of ["mean", "sum", "none"].
 
         """
@@ -136,10 +139,10 @@ def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
                 input = input[:, 1:]
 
         if target.shape != input.shape:
-            raise AssertionError(f"ground truth has differing shape ({target.shape}) from input ({input.shape})")
+            raise AssertionError(f"ground truth has different shape ({target.shape}) from input ({input.shape})")
 
         # reducing only spatial dimensions (not batch nor channels)
-        reduce_axis = list(range(2, len(input.shape)))
+        reduce_axis: List[int] = torch.arange(2, len(input.shape)).tolist()
         if self.batch:
             # reducing spatial dimensions and batch
             reduce_axis = [0] + reduce_axis
@@ -268,23 +271,26 @@ def __init__(
             raise TypeError(f"other_act must be None or callable but is {type(other_act).__name__}.")
         if int(sigmoid) + int(softmax) + int(other_act is not None) > 1:
             raise ValueError("Incompatible values: more than 1 of [sigmoid=True, softmax=True, other_act is not None].")
+
         self.include_background = include_background
         self.to_onehot_y = to_onehot_y
         self.sigmoid = sigmoid
         self.softmax = softmax
         self.other_act = other_act
 
-        w_type = Weight(w_type)
-        self.w_func: Callable = torch.ones_like
-        if w_type == Weight.SIMPLE:
-            self.w_func = torch.reciprocal
-        elif w_type == Weight.SQUARE:
-            self.w_func = lambda x: torch.reciprocal(x * x)
+        self.w_type = Weight(w_type)
 
         self.smooth_nr = float(smooth_nr)
         self.smooth_dr = float(smooth_dr)
         self.batch = batch
 
+    def w_func(self, grnd):
+        if self.w_type == Weight.SIMPLE:
+            return torch.reciprocal(grnd)
+        if self.w_type == Weight.SQUARE:
+            return torch.reciprocal(grnd * grnd)
+        return torch.ones_like(grnd)
+
     def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
         """
         Args:
@@ -325,7 +331,7 @@ def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
             raise AssertionError(f"ground truth has differing shape ({target.shape}) from input ({input.shape})")
 
         # reducing only spatial dimensions (not batch nor channels)
-        reduce_axis = list(range(2, len(input.shape)))
+        reduce_axis: List[int] = torch.arange(2, len(input.shape)).tolist()
         if self.batch:
             reduce_axis = [0] + reduce_axis
         intersection = torch.sum(target * input, reduce_axis)
@@ -595,15 +601,12 @@ def _compute_alpha_generalized_true_positives(self, flat_target: torch.Tensor) -
 
 class DiceCELoss(_Loss):
     """
-    Compute both Dice loss and Cross Entropy Loss, and return the sum of these two losses.
-    Input logits `input` (BNHW[D] where N is number of classes) is compared with ground truth `target` (BNHW[D]).
-    Axis N of `input` is expected to have logit predictions for each class rather than being image channels,
-    while the same axis of `target` can be 1 or N (one-hot format). The `smooth_nr` and `smooth_dr` parameters are
-    values added for dice loss part to the intersection and union components of the inter-over-union calculation
-    to smooth results respectively, these values should be small. The `include_background` class attribute can be
-    set to False for an instance of the loss to exclude the first category (channel index 0) which is by convention
-    assumed to be background. If the non-background segmentations are small compared to the total image size they can get
-    overwhelmed by the signal from the background so excluding it in such cases helps convergence.
+    Compute both Dice loss and Cross Entropy Loss, and return the weighted sum of these two losses.
+    The details of Dice loss is shown in ``monai.losses.DiceLoss``.
+    The details of Cross Entropy Loss is shown in ``torch.nn.CrossEntropyLoss``. In this implementation,
+    two deprecated parameters ``size_average`` and ``reduce``, and the parameter ``ignore_index`` are
+    not supported.
+
     """
 
     def __init__(
@@ -620,11 +623,13 @@ def __init__(
         smooth_dr: float = 1e-5,
         batch: bool = False,
         ce_weight: Optional[torch.Tensor] = None,
+        lambda_dice: float = 1.0,
+        lambda_ce: float = 1.0,
     ) -> None:
         """
         Args:
-            ``ce_weight`` is only used for cross entropy loss, ``reduction`` is used for both losses and other
-            parameters are only used for dice loss.
+            ``ce_weight`` and ``lambda_ce`` are only used for cross entropy loss.
+            ``reduction`` is used for both losses and other parameters are only used for dice loss.
 
             include_background: if False channel index 0 (background category) is excluded from the calculation.
             to_onehot_y: whether to convert `y` into the one-hot format. Defaults to False.
@@ -650,6 +655,10 @@ def __init__(
                 before any `reduction`.
             ce_weight: a rescaling weight given to each class for cross entropy loss.
                 See ``torch.nn.CrossEntropyLoss()`` for more information.
+            lambda_dice: the trade-off weight value for dice loss. The value should be no less than 0.0.
+                Defaults to 1.0.
+            lambda_ce: the trade-off weight value for cross entropy loss. The value should be no less than 0.0.
+                Defaults to 1.0.
 
         """
         super().__init__()
@@ -670,6 +679,12 @@ def __init__(
             weight=ce_weight,
             reduction=reduction,
         )
+        if lambda_dice < 0.0:
+            raise ValueError("lambda_dice should be no less than 0.0.")
+        if lambda_ce < 0.0:
+            raise ValueError("lambda_ce should be no less than 0.0.")
+        self.lambda_dice = lambda_dice
+        self.lambda_ce = lambda_ce
 
     def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
         """
@@ -679,7 +694,7 @@ def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
 
         Raises:
             ValueError: When number of dimensions for input and target are different.
-            ValueError: When number of channels for target is nither 1 or the same as input.
+            ValueError: When number of channels for target is neither 1 nor the same as input.
 
         """
         if len(input.shape) != len(target.shape):
@@ -695,11 +710,123 @@ def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
             target = torch.squeeze(target, dim=1)
         target = target.long()
         ce_loss = self.cross_entropy(input, target)
-        total_loss: torch.Tensor = dice_loss + ce_loss
+        total_loss: torch.Tensor = self.lambda_dice * dice_loss + self.lambda_ce * ce_loss
+        return total_loss
+
+
+class DiceFocalLoss(_Loss):
+    """
+    Compute both Dice loss and Focal Loss, and return the weighted sum of these two losses.
+    The details of Dice loss is shown in ``monai.losses.DiceLoss``.
+    The details of Focal Loss is shown in ``monai.losses.FocalLoss``.
+
+    """
+
+    def __init__(
+        self,
+        include_background: bool = True,
+        to_onehot_y: bool = False,
+        sigmoid: bool = False,
+        softmax: bool = False,
+        other_act: Optional[Callable] = None,
+        squared_pred: bool = False,
+        jaccard: bool = False,
+        reduction: str = "mean",
+        smooth_nr: float = 1e-5,
+        smooth_dr: float = 1e-5,
+        batch: bool = False,
+        gamma: float = 2.0,
+        focal_weight: Optional[Union[Sequence[float], float, int, torch.Tensor]] = None,
+        lambda_dice: float = 1.0,
+        lambda_focal: float = 1.0,
+    ) -> None:
+        """
+        Args:
+            ``gamma``, ``focal_weight`` and ``lambda_focal`` are only used for focal loss.
+            ``include_background``, ``to_onehot_y``and ``reduction`` are used for both losses
+            and other parameters are only used for dice loss.
+            include_background: if False channel index 0 (background category) is excluded from the calculation.
+            to_onehot_y: whether to convert `y` into the one-hot format. Defaults to False.
+            sigmoid: if True, apply a sigmoid function to the prediction.
+            softmax: if True, apply a softmax function to the prediction.
+            other_act: if don't want to use `sigmoid` or `softmax`, use other callable function to execute
+                other activation layers, Defaults to ``None``. for example:
+                `other_act = torch.tanh`.
+            squared_pred: use squared versions of targets and predictions in the denominator or not.
+            jaccard: compute Jaccard Index (soft IoU) instead of dice or not.
+            reduction: {``"none"``, ``"mean"``, ``"sum"``}
+                Specifies the reduction to apply to the output. Defaults to ``"mean"``.
+
+                - ``"none"``: no reduction will be applied.
+                - ``"mean"``: the sum of the output will be divided by the number of elements in the output.
+                - ``"sum"``: the output will be summed.
+
+            smooth_nr: a small constant added to the numerator to avoid zero.
+            smooth_dr: a small constant added to the denominator to avoid nan.
+            batch: whether to sum the intersection and union areas over the batch dimension before the dividing.
+                Defaults to False, a Dice loss value is computed independently from each item in the batch
+                before any `reduction`.
+            gamma: value of the exponent gamma in the definition of the Focal loss.
+            focal_weight: weights to apply to the voxels of each class. If None no weights are applied.
+                The input can be a single value (same weight for all classes), a sequence of values (the length
+                of the sequence should be the same as the number of classes).
+            lambda_dice: the trade-off weight value for dice loss. The value should be no less than 0.0.
+                Defaults to 1.0.
+            lambda_focal: the trade-off weight value for focal loss. The value should be no less than 0.0.
+                Defaults to 1.0.
+
+        """
+        super().__init__()
+        self.dice = DiceLoss(
+            include_background=include_background,
+            to_onehot_y=to_onehot_y,
+            sigmoid=sigmoid,
+            softmax=softmax,
+            other_act=other_act,
+            squared_pred=squared_pred,
+            jaccard=jaccard,
+            reduction=reduction,
+            smooth_nr=smooth_nr,
+            smooth_dr=smooth_dr,
+            batch=batch,
+        )
+        self.focal = FocalLoss(
+            include_background=include_background,
+            to_onehot_y=to_onehot_y,
+            gamma=gamma,
+            weight=focal_weight,
+            reduction=reduction,
+        )
+        if lambda_dice < 0.0:
+            raise ValueError("lambda_dice should be no less than 0.0.")
+        if lambda_focal < 0.0:
+            raise ValueError("lambda_focal should be no less than 0.0.")
+        self.lambda_dice = lambda_dice
+        self.lambda_focal = lambda_focal
+
+    def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            input: the shape should be BNH[WD]. The input should be the original logits
+                due to the restriction of ``monai.losses.FocalLoss``.
+            target: the shape should be BNH[WD] or B1H[WD].
+
+        Raises:
+            ValueError: When number of dimensions for input and target are different.
+            ValueError: When number of channels for target is neither 1 nor the same as input.
+
+        """
+        if len(input.shape) != len(target.shape):
+            raise ValueError("the number of dimensions for input and target should be the same.")
+
+        dice_loss = self.dice(input, target)
+        focal_loss = self.focal(input, target)
+        total_loss: torch.Tensor = self.lambda_dice * dice_loss + self.lambda_focal * focal_loss
         return total_loss
 
 
 dice = Dice = DiceLoss
 dice_ce = DiceCELoss
+dice_focal = DiceFocalLoss
 generalized_dice = GeneralizedDiceLoss
 generalized_wasserstein_dice = GeneralizedWassersteinDiceLoss
diff --git a/monai/losses/focal_loss.py b/monai/losses/focal_loss.py
index da7c63e571..5e0ccd3179 100644
--- a/monai/losses/focal_loss.py
+++ b/monai/losses/focal_loss.py
@@ -9,16 +9,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Optional, Union
+import warnings
+from typing import Optional, Sequence, Union
 
 import torch
 import torch.nn.functional as F
-from torch.nn.modules.loss import _WeightedLoss
+from torch.nn.modules.loss import _Loss
 
+from monai.networks import one_hot
 from monai.utils import LossReduction
 
 
-class FocalLoss(_WeightedLoss):
+class FocalLoss(_Loss):
     """
     Reimplementation of the Focal Loss described in:
 
@@ -29,15 +31,23 @@ class FocalLoss(_WeightedLoss):
 
     def __init__(
         self,
+        include_background: bool = True,
+        to_onehot_y: bool = False,
         gamma: float = 2.0,
-        weight: Optional[torch.Tensor] = None,
+        weight: Optional[Union[Sequence[float], float, int, torch.Tensor]] = None,
         reduction: Union[LossReduction, str] = LossReduction.MEAN,
     ) -> None:
         """
         Args:
+            include_background: if False, channel index 0 (background category) is excluded from the calculation.
+            to_onehot_y: whether to convert `y` into the one-hot format. Defaults to False.
             gamma: value of the exponent gamma in the definition of the Focal loss.
             weight: weights to apply to the voxels of each class. If None no weights are applied.
                 This corresponds to the weights `\alpha` in [1].
+                The input can be a single value (same weight for all classes), a sequence of values (the length
+                of the sequence should be the same as the number of classes, if not ``include_background``, the
+                number should not include class 0).
+                The value/values should be no less than 0. Defaults to None.
             reduction: {``"none"``, ``"mean"``, ``"sum"``}
                 Specifies the reduction to apply to the output. Defaults to ``"mean"``.
 
@@ -57,80 +67,85 @@ def __init__(
                 fl(pred, grnd)
 
         """
-        super(FocalLoss, self).__init__(weight=weight, reduction=LossReduction(reduction).value)
+        super(FocalLoss, self).__init__(reduction=LossReduction(reduction).value)
+        self.include_background = include_background
+        self.to_onehot_y = to_onehot_y
         self.gamma = gamma
-        self.weight: Optional[torch.Tensor] = None
+        self.weight: Optional[Union[Sequence[float], float, int, torch.Tensor]] = weight
 
-    def forward(self, logits: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+    def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
         """
         Args:
-            logits: the shape should be BCH[WD].
-                where C (greater than 1) is the number of classes.
-                Softmax over the logits is integrated in this module for improved numerical stability.
-            target: the shape should be B1H[WD] or BCH[WD].
-                If the target's shape is B1H[WD], the target that this loss expects should be a class index
-                in the range [0, C-1] where C is the number of classes.
+            input: the shape should be BNH[WD], where N is the number of classes.
+                The input should be the original logits since it will be transferred by
+                `F.log_softmax` in the forward function.
+            target: the shape should be BNH[WD] or B1H[WD], where N is the number of classes.
 
         Raises:
-            ValueError: When ``target`` ndim differs from ``logits``.
-            ValueError: When ``target`` channel is not 1 and ``target`` shape differs from ``logits``.
+            AssertionError: When input and target (after one hot transform if setted)
+                have different shapes.
             ValueError: When ``self.reduction`` is not one of ["mean", "sum", "none"].
+            ValueError: When ``self.weight`` is a sequence and the length is not equal to the
+                number of classes.
+            ValueError: When ``self.weight`` is/contains a value that is less than 0.
 
         """
-        i = logits
+        n_pred_ch = input.shape[1]
+
+        if self.to_onehot_y:
+            if n_pred_ch == 1:
+                warnings.warn("single channel prediction, `to_onehot_y=True` ignored.")
+            else:
+                target = one_hot(target, num_classes=n_pred_ch)
+
+        if not self.include_background:
+            if n_pred_ch == 1:
+                warnings.warn("single channel prediction, `include_background=False` ignored.")
+            else:
+                # if skipping background, removing first channel
+                target = target[:, 1:]
+                input = input[:, 1:]
+
+        if target.shape != input.shape:
+            raise AssertionError(f"ground truth has different shape ({target.shape}) from input ({input.shape})")
+
+        i = input
         t = target
 
-        if i.ndimension() != t.ndimension():
-            raise ValueError(f"logits and target ndim must match, got logits={i.ndimension()} target={t.ndimension()}.")
-
-        if t.shape[1] != 1 and t.shape[1] != i.shape[1]:
-            raise ValueError(
-                "target must have one channel or have the same shape as the logits. "
-                "If it has one channel, it should be a class index in the range [0, C-1] "
-                f"where C is the number of classes inferred from 'logits': C={i.shape[1]}. "
-            )
-        if i.shape[1] == 1:
-            raise NotImplementedError("Single-channel predictions not supported.")
-
-        # Change the shape of logits and target to
-        # num_batch x num_class x num_voxels.
-        if i.dim() > 2:
-            i = i.view(i.size(0), i.size(1), -1)  # N,C,H,W => N,C,H*W
-            t = t.view(t.size(0), t.size(1), -1)  # N,1,H,W => N,1,H*W or N,C,H*W
-        else:  # Compatibility with classification.
-            i = i.unsqueeze(2)  # N,C => N,C,1
-            t = t.unsqueeze(2)  # N,1 => N,1,1 or N,C,1
-
-        # Compute the log proba (more stable numerically than softmax).
-        logpt = F.log_softmax(i, dim=1)  # N,C,H*W
-        # Keep only log proba values of the ground truth class for each voxel.
-        if target.shape[1] == 1:
-            logpt = logpt.gather(1, t.long())  # N,C,H*W => N,1,H*W
-            logpt = torch.squeeze(logpt, dim=1)  # N,1,H*W => N,H*W
+        # Change the shape of input and target to B x N x num_voxels.
+        i = i.view(i.size(0), i.size(1), -1)
+        t = t.view(t.size(0), t.size(1), -1)
 
+        # Compute the log proba.
+        logpt = F.log_softmax(i, dim=1)
         # Get the proba
-        pt = torch.exp(logpt)  # N,H*W or N,C,H*W
+        pt = torch.exp(logpt)  # B,H*W or B,N,H*W
 
         if self.weight is not None:
-            self.weight = self.weight.to(i)
+            class_weight: Optional[torch.Tensor] = None
+            if isinstance(self.weight, (float, int)):
+                class_weight = torch.as_tensor([self.weight] * i.size(1))
+            else:
+                class_weight = torch.as_tensor(self.weight)
+                if class_weight.size(0) != i.size(1):
+                    raise ValueError(
+                        "the length of the weight sequence should be the same as the number of classes. "
+                        + "If `include_background=False`, the number should not include class 0."
+                    )
+            if class_weight.min() < 0:
+                raise ValueError("the value/values of weights should be no less than 0.")
+            class_weight = class_weight.to(i)
             # Convert the weight to a map in which each voxel
             # has the weight associated with the ground-truth label
             # associated with this voxel in target.
-            at = self.weight[None, :, None]  # C => 1,C,1
-            at = at.expand((t.size(0), -1, t.size(2)))  # 1,C,1 => N,C,H*W
-            if target.shape[1] == 1:
-                at = at.gather(1, t.long())  # selection of the weights  => N,1,H*W
-                at = torch.squeeze(at, dim=1)  # N,1,H*W => N,H*W
+            at = class_weight[None, :, None]  # N => 1,N,1
+            at = at.expand((t.size(0), -1, t.size(2)))  # 1,N,1 => B,N,H*W
             # Multiply the log proba by their weights.
             logpt = logpt * at
 
         # Compute the loss mini-batch.
         weight = torch.pow(-pt + 1.0, self.gamma)
-        if target.shape[1] == 1:
-            loss = torch.mean(-weight * logpt, dim=1)  # N
-        else:
-            loss = torch.mean(-weight * t * logpt, dim=-1)  # N,C
-
+        loss = torch.mean(-weight * t * logpt, dim=-1)
         if self.reduction == LossReduction.SUM.value:
             return loss.sum()
         if self.reduction == LossReduction.NONE.value:
diff --git a/monai/losses/image_dissimilarity.py b/monai/losses/image_dissimilarity.py
index b229a0c08f..eed5808aa3 100644
--- a/monai/losses/image_dissimilarity.py
+++ b/monai/losses/image_dissimilarity.py
@@ -61,17 +61,15 @@ class LocalNormalizedCrossCorrelationLoss(_Loss):
 
     def __init__(
         self,
-        in_channels: int,
         ndim: int = 3,
         kernel_size: int = 3,
         kernel_type: str = "rectangular",
         reduction: Union[LossReduction, str] = LossReduction.MEAN,
-        smooth_nr: float = 1e-7,
-        smooth_dr: float = 1e-7,
+        smooth_nr: float = 1e-5,
+        smooth_dr: float = 1e-5,
     ) -> None:
         """
         Args:
-            in_channels: number of input channels
             ndim: number of spatial ndimensions, {``1``, ``2``, ``3``}. Defaults to 3.
             kernel_size: kernel spatial size, must be odd.
             kernel_type: {``"rectangular"``, ``"triangular"``, ``"gaussian"``}. Defaults to ``"rectangular"``.
@@ -85,7 +83,6 @@ def __init__(
             smooth_dr: a small constant added to the denominator to avoid nan.
         """
         super(LocalNormalizedCrossCorrelationLoss, self).__init__(reduction=LossReduction(reduction).value)
-        self.in_channels = in_channels
 
         self.ndim = ndim
         if self.ndim not in [1, 2, 3]:
@@ -119,8 +116,6 @@ def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
         Raises:
             ValueError: When ``self.reduction`` is not one of ["mean", "sum", "none"].
         """
-        if pred.shape[1] != self.in_channels:
-            raise ValueError(f"expecting pred with {self.in_channels} channels, got pred of shape {pred.shape}")
         if pred.ndim - 2 != self.ndim:
             raise ValueError(f"expecting pred with {self.ndim} spatial dimensions, got pred of shape {pred.shape}")
         if target.shape != pred.shape:
@@ -129,11 +124,11 @@ def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
         t2, p2, tp = target ** 2, pred ** 2, target * pred
         kernel, kernel_vol = self.kernel.to(pred), self.kernel_vol.to(pred)
         # sum over kernel
-        t_sum = separable_filtering(target, kernels=[kernel] * self.ndim)
-        p_sum = separable_filtering(pred, kernels=[kernel] * self.ndim)
-        t2_sum = separable_filtering(t2, kernels=[kernel] * self.ndim)
-        p2_sum = separable_filtering(p2, kernels=[kernel] * self.ndim)
-        tp_sum = separable_filtering(tp, kernels=[kernel] * self.ndim)
+        t_sum = separable_filtering(target, kernels=[kernel.to(pred)] * self.ndim)
+        p_sum = separable_filtering(pred, kernels=[kernel.to(pred)] * self.ndim)
+        t2_sum = separable_filtering(t2, kernels=[kernel.to(pred)] * self.ndim)
+        p2_sum = separable_filtering(p2, kernels=[kernel.to(pred)] * self.ndim)
+        tp_sum = separable_filtering(tp, kernels=[kernel.to(pred)] * self.ndim)
 
         # average over kernel
         t_avg = t_sum / kernel_vol
@@ -151,6 +146,8 @@ def forward(self, pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
         cross = tp_sum - p_avg * t_sum
         t_var = t2_sum - t_avg * t_sum  # std[t] ** 2
         p_var = p2_sum - p_avg * p_sum  # std[p] ** 2
+        t_var = torch.max(t_var, torch.zeros_like(t_var))
+        p_var = torch.max(p_var, torch.zeros_like(p_var))
         ncc: torch.Tensor = (cross * cross + self.smooth_nr) / (t_var * p_var + self.smooth_dr)
         # shape = (batch, 1, D, H, W)
 
diff --git a/monai/losses/multi_scale.py b/monai/losses/multi_scale.py
index 5a17bc2d07..af23e03440 100644
--- a/monai/losses/multi_scale.py
+++ b/monai/losses/multi_scale.py
@@ -82,8 +82,8 @@ def forward(self, y_true: torch.Tensor, y_pred: torch.Tensor) -> torch.Tensor:
                 else:
                     loss_list.append(
                         self.loss(
-                            separable_filtering(y_pred, [self.kernel_fn(s)] * (y_true.ndim - 2)),
-                            separable_filtering(y_true, [self.kernel_fn(s)] * (y_true.ndim - 2)),
+                            separable_filtering(y_pred, [self.kernel_fn(s).to(y_pred)] * (y_true.ndim - 2)),
+                            separable_filtering(y_true, [self.kernel_fn(s).to(y_pred)] * (y_true.ndim - 2)),
                         )
                     )
             loss = torch.stack(loss_list, dim=0)
diff --git a/monai/losses/tversky.py b/monai/losses/tversky.py
index b1c45a74a2..1d75b9e8cc 100644
--- a/monai/losses/tversky.py
+++ b/monai/losses/tversky.py
@@ -10,7 +10,7 @@
 # limitations under the License.
 
 import warnings
-from typing import Callable, Optional, Union
+from typing import Callable, List, Optional, Union
 
 import torch
 from torch.nn.modules.loss import _Loss
@@ -139,7 +139,7 @@ def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
         g1 = 1 - g0
 
         # reducing only spatial dimensions (not batch nor channels)
-        reduce_axis = list(range(2, len(input.shape)))
+        reduce_axis: List[int] = torch.arange(2, len(input.shape)).tolist()
         if self.batch:
             # reducing spatial dimensions and batch
             reduce_axis = [0] + reduce_axis
diff --git a/monai/metrics/__init__.py b/monai/metrics/__init__.py
index 818413c30d..35dea5f387 100644
--- a/monai/metrics/__init__.py
+++ b/monai/metrics/__init__.py
@@ -10,6 +10,7 @@
 # limitations under the License.
 
 from .confusion_matrix import ConfusionMatrixMetric, compute_confusion_matrix_metric, get_confusion_matrix
+from .froc import compute_fp_tp_probs, compute_froc_curve_data, compute_froc_score
 from .hausdorff_distance import HausdorffDistanceMetric, compute_hausdorff_distance, compute_percent_hausdorff_distance
 from .meandice import DiceMetric, compute_meandice
 from .rocauc import compute_roc_auc
diff --git a/monai/metrics/confusion_matrix.py b/monai/metrics/confusion_matrix.py
index a0c840d45a..9c15b320eb 100644
--- a/monai/metrics/confusion_matrix.py
+++ b/monai/metrics/confusion_matrix.py
@@ -112,8 +112,7 @@ def __call__(self, y_pred: torch.Tensor, y: torch.Tensor):
                 results.append(f)
                 results.append(not_nans)
             return results
-        else:
-            return confusion_matrix
+        return confusion_matrix
 
 
 def get_confusion_matrix(
diff --git a/monai/metrics/froc.py b/monai/metrics/froc.py
new file mode 100644
index 0000000000..faebbbf7a6
--- /dev/null
+++ b/monai/metrics/froc.py
@@ -0,0 +1,137 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+
+def compute_fp_tp_probs(
+    probs: Union[np.ndarray, torch.Tensor],
+    y_coord: Union[np.ndarray, torch.Tensor],
+    x_coord: Union[np.ndarray, torch.Tensor],
+    evaluation_mask: Union[np.ndarray, torch.Tensor],
+    labels_to_exclude: Optional[List] = None,
+    resolution_level: int = 0,
+):
+    """
+    This function is modified from the official evaluation code of
+    `CAMELYON 16 Challenge <https://camelyon16.grand-challenge.org/>`_, and used to distinguish
+    true positive and false positive predictions. A true positive prediction is defined when
+    the detection point is within the annotated ground truth region.
+
+    Args:
+        probs: an array with shape (n,) that represents the probabilities of the detections.
+            Where, n is the number of predicted detections.
+        y_coord: an array with shape (n,) that represents the Y-coordinates of the detections.
+        x_coord: an array with shape (n,) that represents the X-coordinates of the detections.
+        evaluation_mask: the ground truth mask for evaluation.
+        labels_to_exclude: labels in this list will not be counted for metric calculation.
+        resolution_level: the level at which the evaluation mask is made.
+
+    Returns:
+        fp_probs: an array that contains the probabilities of the false positive detections.
+        tp_probs: an array that contains the probabilities of the True positive detections.
+        num_targets: the total number of targets (excluding `labels_to_exclude`) for all images under evaluation.
+
+    """
+    if not (probs.shape == y_coord.shape == x_coord.shape):
+        raise AssertionError("the shapes for coordinates and probabilities should be the same.")
+
+    if isinstance(probs, torch.Tensor):
+        probs = probs.detach().cpu().numpy()
+    if isinstance(y_coord, torch.Tensor):
+        y_coord = y_coord.detach().cpu().numpy()
+    if isinstance(x_coord, torch.Tensor):
+        x_coord = x_coord.detach().cpu().numpy()
+    if isinstance(evaluation_mask, torch.Tensor):
+        evaluation_mask = evaluation_mask.detach().cpu().numpy()
+
+    if labels_to_exclude is None:
+        labels_to_exclude = []
+
+    max_label = np.max(evaluation_mask)
+    tp_probs = np.zeros((max_label,), dtype=np.float32)
+
+    y_coord = (y_coord / pow(2, resolution_level)).astype(int)
+    x_coord = (x_coord / pow(2, resolution_level)).astype(int)
+
+    hittedlabel = evaluation_mask[y_coord, x_coord]
+    fp_probs = probs[np.where(hittedlabel == 0)]
+    for i in range(1, max_label + 1):
+        if i not in labels_to_exclude and i in hittedlabel:
+            tp_probs[i - 1] = probs[np.where(hittedlabel == i)].max()
+
+    num_targets = max_label - len(labels_to_exclude)
+    return fp_probs, tp_probs, num_targets
+
+
+def compute_froc_curve_data(
+    fp_probs: Union[np.ndarray, torch.Tensor],
+    tp_probs: Union[np.ndarray, torch.Tensor],
+    num_targets: int,
+    num_images: int,
+):
+    """
+    This function is modified from the official evaluation code of
+    `CAMELYON 16 Challenge <https://camelyon16.grand-challenge.org/>`_, and used to compute
+    the required data for plotting the Free Response Operating Characteristic (FROC) curve.
+
+    Args:
+        fp_probs: an array that contains the probabilities of the false positive detections for all
+            images under evaluation.
+        tp_probs: an array that contains the probabilities of the True positive detections for all
+            images under evaluation.
+        num_targets: the total number of targets (excluding `labels_to_exclude`) for all images under evaluation.
+        num_images: the number of images under evaluation.
+
+    """
+    if type(fp_probs) is not type(tp_probs):
+        raise AssertionError("fp and tp probs should have same type.")
+    if isinstance(fp_probs, torch.Tensor):
+        fp_probs = fp_probs.detach().cpu().numpy()
+    if isinstance(tp_probs, torch.Tensor):
+        tp_probs = tp_probs.detach().cpu().numpy()
+
+    total_fps, total_tps = [], []
+    all_probs = sorted(set(list(fp_probs) + list(tp_probs)))
+    for thresh in all_probs[1:]:
+        total_fps.append((fp_probs >= thresh).sum())
+        total_tps.append((tp_probs >= thresh).sum())
+    total_fps.append(0)
+    total_tps.append(0)
+    fps_per_image = np.asarray(total_fps) / float(num_images)
+    total_sensitivity = np.asarray(total_tps) / float(num_targets)
+    return fps_per_image, total_sensitivity
+
+
+def compute_froc_score(
+    fps_per_image: np.ndarray,
+    total_sensitivity: np.ndarray,
+    eval_thresholds: Tuple = (0.25, 0.5, 1, 2, 4, 8),
+):
+    """
+    This function is modified from the official evaluation code of
+    `CAMELYON 16 Challenge <https://camelyon16.grand-challenge.org/>`_, and used to compute
+    the challenge's second evaluation metric, which is defined as the average sensitivity at
+    the predefined false positive rates per whole slide image.
+
+    Args:
+        fps_per_image: the average number of false positives per image for different thresholds.
+        total_sensitivity: sensitivities (true positive rates) for different thresholds.
+        eval_thresholds: the false positive rates for calculating the average sensitivity. Defaults
+            to (0.25, 0.5, 1, 2, 4, 8) which is the same as the CAMELYON 16 Challenge.
+
+    """
+    interp_sens = np.interp(eval_thresholds, fps_per_image[::-1], total_sensitivity[::-1])
+    return np.mean(interp_sens)
diff --git a/monai/metrics/rocauc.py b/monai/metrics/rocauc.py
index 80a6671dfa..a6d70b6dd8 100644
--- a/monai/metrics/rocauc.py
+++ b/monai/metrics/rocauc.py
@@ -9,13 +9,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import warnings
-from typing import Callable, Optional, Union, cast
+from typing import Union, cast
 
 import numpy as np
 import torch
 
-from monai.networks import one_hot
 from monai.utils import Average
 
 
@@ -53,9 +51,6 @@ def _calculate(y: torch.Tensor, y_pred: torch.Tensor) -> float:
 def compute_roc_auc(
     y_pred: torch.Tensor,
     y: torch.Tensor,
-    to_onehot_y: bool = False,
-    softmax: bool = False,
-    other_act: Optional[Callable] = None,
     average: Union[Average, str] = Average.MACRO,
 ):
     """Computes Area Under the Receiver Operating Characteristic Curve (ROC AUC). Referring to:
@@ -67,10 +62,6 @@ def compute_roc_auc(
             it must be One-Hot format and first dim is batch, example shape: [16] or [16, 2].
         y: ground truth to compute ROC AUC metric, the first dim is batch.
             example shape: [16, 1] will be converted into [16, 2] (where `2` is inferred from `y_pred`).
-        to_onehot_y: whether to convert `y` into the one-hot format. Defaults to False.
-        softmax: whether to add softmax function to `y_pred` before computation. Defaults to False.
-        other_act: callable function to replace `softmax` as activation layer if needed, Defaults to ``None``.
-            for example: `other_act = lambda x: torch.log_softmax(x)`.
         average: {``"macro"``, ``"weighted"``, ``"micro"``, ``"none"``}
             Type of averaging performed if not binary classification.
             Defaults to ``"macro"``.
@@ -86,8 +77,6 @@ def compute_roc_auc(
     Raises:
         ValueError: When ``y_pred`` dimension is not one of [1, 2].
         ValueError: When ``y`` dimension is not one of [1, 2].
-        ValueError: When ``softmax=True`` and ``other_act is not None``. Incompatible values.
-        TypeError: When ``other_act`` is not an ``Optional[Callable]``.
         ValueError: When ``average`` is not one of ["macro", "weighted", "micro", "none"].
 
     Note:
@@ -107,22 +96,7 @@ def compute_roc_auc(
         y = y.squeeze(dim=-1)
 
     if y_pred_ndim == 1:
-        if to_onehot_y:
-            warnings.warn("y_pred has only one channel, to_onehot_y=True ignored.")
-        if softmax:
-            warnings.warn("y_pred has only one channel, softmax=True ignored.")
         return _calculate(y, y_pred)
-    n_classes = y_pred.shape[1]
-    if to_onehot_y:
-        y = one_hot(y, n_classes)
-    if softmax and other_act is not None:
-        raise ValueError("Incompatible values: softmax=True and other_act is not None.")
-    if softmax:
-        y_pred = y_pred.float().softmax(dim=1)
-    if other_act is not None:
-        if not callable(other_act):
-            raise TypeError(f"other_act must be None or callable but is {type(other_act).__name__}.")
-        y_pred = other_act(y_pred)
 
     if y.shape != y_pred.shape:
         raise AssertionError("data shapes of y_pred and y do not match.")
diff --git a/monai/networks/blocks/__init__.py b/monai/networks/blocks/__init__.py
index 4a2e31928e..ed6ac12430 100644
--- a/monai/networks/blocks/__init__.py
+++ b/monai/networks/blocks/__init__.py
@@ -10,13 +10,15 @@
 # limitations under the License.
 
 from .acti_norm import ADN
-from .activation import Mish, Swish
+from .activation import MemoryEfficientSwish, Mish, Swish
 from .aspp import SimpleASPP
 from .convolutions import Convolution, ResidualUnit
+from .crf import CRF
 from .downsample import MaxAvgPool
 from .dynunet_block import UnetBasicBlock, UnetOutBlock, UnetResBlock, UnetUpBlock, get_output_padding, get_padding
 from .fcn import FCN, GCN, MCFCN, Refine
 from .localnet_block import LocalNetDownSampleBlock, LocalNetFeatureExtractorBlock, LocalNetUpSampleBlock
+from .regunet_block import RegistrationDownSampleBlock, RegistrationExtractionBlock, RegistrationResidualConvBlock
 from .segresnet_block import ResBlock
 from .squeeze_and_excitation import (
     ChannelSELayer,
diff --git a/monai/networks/blocks/activation.py b/monai/networks/blocks/activation.py
index ef6c74f282..f6a04e830e 100644
--- a/monai/networks/blocks/activation.py
+++ b/monai/networks/blocks/activation.py
@@ -17,7 +17,7 @@ class Swish(nn.Module):
     r"""Applies the element-wise function:
 
     .. math::
-        \text{Swish}(x) = x * \text{Sigmoid}(\alpha * x) for constant value alpha.
+        \text{Swish}(x) = x * \text{Sigmoid}(\alpha * x) ~~~~\text{for constant value}~ \alpha.
 
     Citation: Searching for Activation Functions, Ramachandran et al., 2017, https://arxiv.org/abs/1710.05941.
 
@@ -43,6 +43,57 @@ def forward(self, input: torch.Tensor) -> torch.Tensor:
         return input * torch.sigmoid(self.alpha * input)
 
 
+class SwishImplementation(torch.autograd.Function):
+    r"""Memory efficient implementation for training
+    Follows recommendation from:
+    https://github.com/lukemelas/EfficientNet-PyTorch/issues/18#issuecomment-511677853
+
+    Results in ~ 30% memory saving during training as compared to Swish()
+    """
+
+    @staticmethod
+    def forward(ctx, input):
+        result = input * torch.sigmoid(input)
+        ctx.save_for_backward(input)
+        return result
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        input = ctx.saved_tensors[0]
+        sigmoid_input = torch.sigmoid(input)
+        return grad_output * (sigmoid_input * (1 + input * (1 - sigmoid_input)))
+
+
+class MemoryEfficientSwish(nn.Module):
+    r"""Applies the element-wise function:
+
+    .. math::
+        \text{Swish}(x) = x * \text{Sigmoid}(\alpha * x) ~~~~\text{for constant value}~ \alpha=1.
+
+    Memory efficient implementation for training following recommendation from:
+    https://github.com/lukemelas/EfficientNet-PyTorch/issues/18#issuecomment-511677853
+
+    Results in ~ 30% memory saving during training as compared to Swish()
+
+    Citation: Searching for Activation Functions, Ramachandran et al., 2017, https://arxiv.org/abs/1710.05941.
+
+    Shape:
+        - Input: :math:`(N, *)` where `*` means, any number of additional
+          dimensions
+        - Output: :math:`(N, *)`, same shape as the input
+
+
+    Examples::
+
+        >>> m = Act['memswish']()
+        >>> input = torch.randn(2)
+        >>> output = m(input)
+    """
+
+    def forward(self, input: torch.Tensor):
+        return SwishImplementation.apply(input)
+
+
 class Mish(nn.Module):
     r"""Applies the element-wise function:
 
diff --git a/monai/networks/blocks/convolutions.py b/monai/networks/blocks/convolutions.py
index 7bfb3b47e4..39ce60e3f8 100644
--- a/monai/networks/blocks/convolutions.py
+++ b/monai/networks/blocks/convolutions.py
@@ -30,6 +30,34 @@ class Convolution(nn.Sequential):
 
         -- (Conv|ConvTrans) --
 
+    For example:
+
+    .. code-block:: python
+
+        from monai.networks.blocks import Convolution
+
+        conv = Convolution(
+            dimensions=3,
+            in_channels=1,
+            out_channels=1,
+            adn_ordering="ADN",
+            act=("prelu", {"init": 0.2}),
+            dropout=0.1,
+            norm=("layer", {"normalized_shape": (10, 10, 10)}),
+        )
+        print(conv)
+
+    output::
+
+        Convolution(
+          (conv): Conv3d(1, 1, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+          (adn): ADN(
+            (A): PReLU(num_parameters=1)
+            (D): Dropout(p=0.1, inplace=False)
+            (N): LayerNorm((10, 10, 10), eps=1e-05, elementwise_affine=True)
+          )
+        )
+
     Args:
         dimensions: number of spatial dimensions.
         in_channels: number of input channels.
@@ -142,6 +170,44 @@ class ResidualUnit(nn.Module):
     """
     Residual module with multiple convolutions and a residual connection.
 
+    For example:
+
+    .. code-block:: python
+
+        from monai.networks.blocks import ResidualUnit
+
+        convs = ResidualUnit(
+            dimensions=3,
+            in_channels=1,
+            out_channels=1,
+            adn_ordering="AN",
+            act=("prelu", {"init": 0.2}),
+            norm=("layer", {"normalized_shape": (10, 10, 10)}),
+        )
+        print(convs)
+
+    output::
+
+        ResidualUnit(
+          (conv): Sequential(
+            (unit0): Convolution(
+              (conv): Conv3d(1, 1, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+              (adn): ADN(
+                (A): PReLU(num_parameters=1)
+                (N): LayerNorm((10, 10, 10), eps=1e-05, elementwise_affine=True)
+              )
+            )
+            (unit1): Convolution(
+              (conv): Conv3d(1, 1, kernel_size=(3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1))
+              (adn): ADN(
+                (A): PReLU(num_parameters=1)
+                (N): LayerNorm((10, 10, 10), eps=1e-05, elementwise_affine=True)
+              )
+            )
+          )
+          (residual): Identity()
+        )
+
     Args:
         dimensions: number of spatial dimensions.
         in_channels: number of input channels.
diff --git a/monai/networks/blocks/crf.py b/monai/networks/blocks/crf.py
new file mode 100644
index 0000000000..29d4ef4216
--- /dev/null
+++ b/monai/networks/blocks/crf.py
@@ -0,0 +1,140 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from torch.nn.functional import conv1d, conv2d, conv3d, pad, softmax
+
+from monai.networks.layers.filtering import PHLFilter
+
+__all__ = ["CRF"]
+
+
+class CRF(torch.nn.Module):
+    """
+    Conditional Random Field: Combines message passing with a class
+    compatibility convolution into an iterative process designed
+    to successively minimise the energy of the class labeling.
+
+    In this implementation, the message passing step is a weighted
+    combination of a gaussian filter and a bilateral filter.
+    The bilateral term is included to respect existing structure
+    within the reference tensor.
+
+    See:
+        https://arxiv.org/abs/1502.03240
+    """
+
+    def __init__(
+        self,
+        bilateral_weight: float = 1.0,
+        gaussian_weight: float = 1.0,
+        bilateral_spatial_sigma: float = 5.0,
+        bilateral_color_sigma: float = 0.5,
+        gaussian_spatial_sigma: float = 5.0,
+        update_factor: float = 3.0,
+        compatibility_kernel_range: int = 1,
+        iterations: int = 5,
+    ):
+        """
+        Args:
+            bilateral_weight: the weighting of the bilateral term in the message passing step.
+            gaussian_weight: the weighting of the gaussian term in the message passing step.
+            bilateral_spatial_sigma: standard deviation in spatial coordinates for the bilateral term.
+            bilateral_color_sigma: standard deviation in color space for the bilateral term.
+            gaussian_spatial_sigma: standard deviation in spatial coordinates for the gaussian term.
+            update_factor: determines the magnitude of each update.
+            compatibility_kernel_range: the range of the kernel used in the compatibility convolution.
+            iterations: the number of iterations.
+        """
+        super(CRF, self).__init__()
+        self.bilateral_weight = bilateral_weight
+        self.gaussian_weight = gaussian_weight
+        self.bilateral_spatial_sigma = bilateral_spatial_sigma
+        self.bilateral_color_sigma = bilateral_color_sigma
+        self.gaussian_spatial_sigma = gaussian_spatial_sigma
+        self.update_factor = update_factor
+        self.compatibility_kernel_range = compatibility_kernel_range
+        self.iterations = iterations
+
+    def forward(self, input_tensor: torch.Tensor, reference_tensor: torch.Tensor):
+        """
+        Args:
+            input_tensor: tensor containing initial class logits.
+            reference_tensor: the reference tensor used to guide the message passing.
+
+        Returns:
+            output (torch.Tensor): output tensor.
+        """
+
+        # useful values
+        spatial_dim = input_tensor.dim() - 2
+        class_count = input_tensor.size(1)
+        padding = self.compatibility_kernel_range
+
+        # constructing spatial feature tensor
+        spatial_features = _create_coordinate_tensor(reference_tensor)
+
+        # constructing final feature tensors for bilateral and gaussian kernel
+        bilateral_features = torch.cat(
+            [spatial_features / self.bilateral_spatial_sigma, reference_tensor / self.bilateral_color_sigma], dim=1
+        )
+        gaussian_features = spatial_features / self.gaussian_spatial_sigma
+
+        # compatibility matrix (potts model (1 - diag) for now)
+        compatibility_matrix = _potts_model_weights(class_count).to(device=input_tensor.device)
+
+        # expanding matrix to kernel
+        compatibility_kernel = _expand_matrix_to_kernel(
+            compatibility_matrix, spatial_dim, self.compatibility_kernel_range
+        )
+
+        # choosing convolution function
+        conv = [conv1d, conv2d, conv3d][spatial_dim - 1]
+
+        # setting up output tensor
+        output_tensor = softmax(input_tensor, dim=1)
+
+        # mean field loop
+        for _ in range(self.iterations):
+
+            # message passing step for both kernels
+            bliateral_output = PHLFilter.apply(output_tensor, bilateral_features)
+            gaussian_output = PHLFilter.apply(output_tensor, gaussian_features)
+
+            # combining filter outputs
+            combined_output = self.bilateral_weight * bliateral_output + self.gaussian_weight * gaussian_output
+
+            # compatibility convolution
+            combined_output = pad(combined_output, 2 * spatial_dim * [padding], mode="replicate")
+            compatibility_update = conv(combined_output, compatibility_kernel)
+
+            # update and normalize
+            output_tensor = softmax(input_tensor - self.update_factor * compatibility_update, dim=1)
+
+        return output_tensor
+
+
+# helper methods
+def _create_coordinate_tensor(tensor):
+    axes = [torch.arange(tensor.size(i)) for i in range(2, tensor.dim())]
+    grids = torch.meshgrid(axes)
+    coords = torch.stack(grids).to(device=tensor.device, dtype=tensor.dtype)
+    return torch.stack(tensor.size(0) * [coords], dim=0)
+
+
+def _potts_model_weights(class_count):
+    return (1 - torch.diag(torch.ones(class_count))).unsqueeze(-1)
+
+
+def _expand_matrix_to_kernel(matrix, spatial_dim, kernel_range):
+    reshape_arg = (matrix.size(0), matrix.size(1)) + spatial_dim * (1,)
+    expand_arg = (-1, -1) + spatial_dim * (1 + 2 * kernel_range,)
+    return matrix.reshape(reshape_arg).expand(expand_arg)
diff --git a/monai/networks/blocks/localnet_block.py b/monai/networks/blocks/localnet_block.py
index 4166c08774..3997d42436 100644
--- a/monai/networks/blocks/localnet_block.py
+++ b/monai/networks/blocks/localnet_block.py
@@ -1,3 +1,14 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from typing import Optional, Sequence, Tuple, Type, Union
 
 import torch
@@ -249,7 +260,7 @@ def forward(self, x, mid) -> torch.Tensor:
         Args:
             x: feature to be up-sampled, in shape (batch, ``in_channels``, insize_1, insize_2, [insize_3])
             mid: mid-level feature saved during down-sampling,
-                in shape (batch, ``out_channels``, midsize_1, midsize_2, [midnsize_3])
+                in shape (batch, ``out_channels``, midsize_1, midsize_2, [midsize_3])
 
         Raises:
             ValueError: when ``midsize != insize * 2``
diff --git a/monai/networks/blocks/regunet_block.py b/monai/networks/blocks/regunet_block.py
new file mode 100644
index 0000000000..d2cd3518b9
--- /dev/null
+++ b/monai/networks/blocks/regunet_block.py
@@ -0,0 +1,271 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional, Sequence, Tuple, Type, Union
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from monai.networks.blocks import Convolution
+from monai.networks.layers import Conv, Norm, Pool, same_padding
+
+
+def get_conv_block(
+    spatial_dims: int,
+    in_channels: int,
+    out_channels: int,
+    kernel_size: Union[Sequence[int], int] = 3,
+    strides: int = 1,
+    padding: Optional[Union[Tuple[int, ...], int]] = None,
+    act: Optional[Union[Tuple, str]] = "RELU",
+    norm: Optional[Union[Tuple, str]] = "BATCH",
+    initializer: Optional[str] = "kaiming_uniform",
+) -> nn.Module:
+    if padding is None:
+        padding = same_padding(kernel_size)
+    conv_block = Convolution(
+        spatial_dims,
+        in_channels,
+        out_channels,
+        kernel_size=kernel_size,
+        strides=strides,
+        act=act,
+        norm=norm,
+        bias=False,
+        conv_only=False,
+        padding=padding,
+    )
+    conv_type: Type[Union[nn.Conv1d, nn.Conv2d, nn.Conv3d]] = Conv[Conv.CONV, spatial_dims]
+    for m in conv_block.modules():
+        if isinstance(m, conv_type):
+            if initializer == "kaiming_uniform":
+                nn.init.kaiming_normal_(torch.as_tensor(m.weight))
+            elif initializer == "zeros":
+                nn.init.zeros_(torch.as_tensor(m.weight))
+            else:
+                raise ValueError(
+                    f"initializer {initializer} is not supported, " "currently supporting kaiming_uniform and zeros"
+                )
+    return conv_block
+
+
+def get_conv_layer(
+    spatial_dims: int,
+    in_channels: int,
+    out_channels: int,
+    kernel_size: Union[Sequence[int], int] = 3,
+) -> nn.Module:
+    padding = same_padding(kernel_size)
+    return Convolution(
+        spatial_dims,
+        in_channels,
+        out_channels,
+        kernel_size=kernel_size,
+        bias=False,
+        conv_only=True,
+        padding=padding,
+    )
+
+
+class RegistrationResidualConvBlock(nn.Module):
+    """
+    A block with skip links and layer - norm - activation.
+    Only changes the number of channels, the spatial size is kept same.
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        out_channels: int,
+        num_layers: int = 2,
+        kernel_size: int = 3,
+    ):
+        """
+
+        Args:
+            spatial_dims: number of spatial dimensions
+            in_channels: number of input channels
+            out_channels: number of output channels
+            num_layers: number of layers inside the block
+            kernel_size: kernel_size
+        """
+        super(RegistrationResidualConvBlock, self).__init__()
+        self.num_layers = num_layers
+        self.layers = nn.ModuleList(
+            [
+                get_conv_layer(
+                    spatial_dims=spatial_dims,
+                    in_channels=in_channels if i == 0 else out_channels,
+                    out_channels=out_channels,
+                    kernel_size=kernel_size,
+                )
+                for i in range(num_layers)
+            ]
+        )
+        self.norms = nn.ModuleList([Norm[Norm.BATCH, spatial_dims](out_channels) for _ in range(num_layers)])
+        self.acts = nn.ModuleList([nn.ReLU() for _ in range(num_layers)])
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+
+        Args:
+            x: Tensor in shape (batch, ``in_channels``, insize_1, insize_2, [insize_3])
+
+        Returns:
+            Tensor in shape (batch, ``out_channels``, insize_1, insize_2, [insize_3]),
+            with the same spatial size as ``x``
+        """
+        skip = x
+        for i, (conv, norm, act) in enumerate(zip(self.layers, self.norms, self.acts)):
+            x = conv(x)
+            x = norm(x)
+            if i == self.num_layers - 1:
+                # last block
+                x = x + skip
+            x = act(x)
+        return x
+
+
+class RegistrationDownSampleBlock(nn.Module):
+    """
+    A down-sample module used in RegUNet to half the spatial size.
+    The number of channels is kept same.
+
+    Adapted from:
+        DeepReg (https://github.com/DeepRegNet/DeepReg)
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        channels: int,
+        pooling: bool,
+    ) -> None:
+        """
+        Args:
+            spatial_dims: number of spatial dimensions.
+            channels: channels
+            pooling: use MaxPool if True, strided conv if False
+        """
+        super(RegistrationDownSampleBlock, self).__init__()
+        if pooling:
+            self.layer = Pool[Pool.MAX, spatial_dims](kernel_size=2)
+        else:
+            self.layer = get_conv_block(
+                spatial_dims=spatial_dims,
+                in_channels=channels,
+                out_channels=channels,
+                kernel_size=2,
+                strides=2,
+                padding=0,
+            )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        """
+        Halves the spatial dimensions and keeps the same channel.
+        output in shape (batch, ``channels``, insize_1 / 2, insize_2 / 2, [insize_3 / 2]),
+
+        Args:
+            x: Tensor in shape (batch, ``channels``, insize_1, insize_2, [insize_3])
+
+        Raises:
+            ValueError: when input spatial dimensions are not even.
+        """
+        for i in x.shape[2:]:
+            if i % 2 != 0:
+                raise ValueError("expecting x spatial dimensions be even, " f"got x of shape {x.shape}")
+        out: torch.Tensor = self.layer(x)
+        return out
+
+
+def get_deconv_block(
+    spatial_dims: int,
+    in_channels: int,
+    out_channels: int,
+) -> nn.Module:
+    return Convolution(
+        dimensions=spatial_dims,
+        in_channels=in_channels,
+        out_channels=out_channels,
+        strides=2,
+        act="RELU",
+        norm="BATCH",
+        bias=False,
+        is_transposed=True,
+        padding=1,
+        output_padding=1,
+    )
+
+
+class RegistrationExtractionBlock(nn.Module):
+    """
+    The Extraction Block used in RegUNet.
+    Extracts feature from each ``extract_levels`` and takes the average.
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        extract_levels: Tuple[int],
+        num_channels: Union[Tuple[int], List[int]],
+        out_channels: int,
+        kernel_initializer: Optional[str] = "kaiming_uniform",
+        activation: Optional[str] = None,
+    ):
+        """
+
+        Args:
+            spatial_dims: number of spatial dimensions
+            extract_levels: spatial levels to extract feature from, 0 refers to the input scale
+            num_channels: number of channels at each scale level,
+                List or Tuple of length equals to `depth` of the RegNet
+            out_channels: number of output channels
+            kernel_initializer: kernel initializer
+            activation: kernel activation function
+        """
+        super(RegistrationExtractionBlock, self).__init__()
+        self.extract_levels = extract_levels
+        self.max_level = max(extract_levels)
+        self.layers = nn.ModuleList(
+            [
+                get_conv_block(
+                    spatial_dims=spatial_dims,
+                    in_channels=num_channels[d],
+                    out_channels=out_channels,
+                    norm=None,
+                    act=activation,
+                    initializer=kernel_initializer,
+                )
+                for d in extract_levels
+            ]
+        )
+
+    def forward(self, x: List[torch.Tensor], image_size: List[int]) -> torch.Tensor:
+        """
+
+        Args:
+            x: Decoded feature at different spatial levels, sorted from deep to shallow
+            image_size: output image size
+
+        Returns:
+            Tensor of shape (batch, `out_channels`, size1, size2, size3), where (size1, size2, size3) = ``image_size``
+        """
+        feature_list = [
+            F.interpolate(
+                layer(x[self.max_level - level]),
+                size=image_size,
+            )
+            for layer, level in zip(self.layers, self.extract_levels)
+        ]
+        out: torch.Tensor = torch.mean(torch.stack(feature_list, dim=0), dim=0)
+        return out
diff --git a/monai/networks/blocks/warp.py b/monai/networks/blocks/warp.py
index eb4c09fa72..d916c026ff 100644
--- a/monai/networks/blocks/warp.py
+++ b/monai/networks/blocks/warp.py
@@ -1,47 +1,89 @@
-from typing import List, Optional, Union
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import warnings
+from typing import List
 
 import torch
 from torch import nn
 from torch.nn import functional as F
 
-from monai.utils import GridSamplePadMode
+from monai.config.deviceconfig import USE_COMPILED
+from monai.networks.layers.spatial_transforms import grid_pull
+from monai.utils import GridSampleMode, GridSamplePadMode, optional_import
+
+_C, _ = optional_import("monai._C")
+
+__all__ = ["Warp", "DVF2DDF"]
 
 
 class Warp(nn.Module):
     """
-    Warp an image with given DDF.
+    Warp an image with given dense displacement field (DDF).
     """
 
     def __init__(
         self,
-        spatial_dims: int,
-        mode: int = 1,
-        padding_mode: Optional[Union[GridSamplePadMode, str]] = GridSamplePadMode.ZEROS,
+        mode=GridSampleMode.BILINEAR.value,
+        padding_mode=GridSamplePadMode.BORDER.value,
     ):
         """
-        Args:
-            spatial_dims: {2, 3}. number of spatial dimensions
-            mode: interpolation mode to calculate output values, defaults to 1.
-                Possible values are::
-
-                    - 0 or 'nearest'    or InterpolationType.nearest
-                    - 1 or 'linear'     or InterpolationType.linear
-                    - 2 or 'quadratic'  or InterpolationType.quadratic
-                    - 3 or 'cubic'      or InterpolationType.cubic
-                    - 4 or 'fourth'     or InterpolationType.fourth
-                    - etc.
-            padding_mode: {``"zeros"``, ``"border"``, ``"reflection"``}
-                Padding mode for outside grid values. Defaults to ``"border"``.
-                See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+        For pytorch native APIs, the possible values are:
+
+            - mode: ``"nearest"``, ``"bilinear"``, ``"bicubic"``.
+            - padding_mode: ``"zeros"``, ``"border"``, ``"reflection"``
+
+        See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+
+        For MONAI C++/CUDA extensions, the possible values are:
+
+            - mode: ``"nearest"``, ``"bilinear"``, ``"bicubic"``, 0, 1, ...
+            - padding_mode: ``"zeros"``, ``"border"``, ``"reflection"``, 0, 1, ...
+
+        See also: :py:class:`monai.networks.layers.grid_pull`
         """
         super(Warp, self).__init__()
-        if spatial_dims not in [2, 3]:
-            raise ValueError(f"got unsupported spatial_dims={spatial_dims}, only support 2-d and 3-d input")
-        self.spatial_dims = spatial_dims
-        if mode < 0:
-            raise ValueError(f"do not support negative mode, got mode={mode}")
-        self.mode = mode
-        self.padding_mode: GridSamplePadMode = GridSamplePadMode(padding_mode)
+        # resolves _interp_mode for different methods
+
+        if USE_COMPILED:
+            if mode in (inter.value for inter in GridSampleMode):
+                mode = GridSampleMode(mode)
+                if mode == GridSampleMode.BILINEAR:
+                    mode = 1
+                elif mode == GridSampleMode.NEAREST:
+                    mode = 0
+                elif mode == GridSampleMode.BICUBIC:
+                    mode = 3
+                else:
+                    mode = 1  # default to linear
+            self._interp_mode = mode
+        else:
+            warnings.warn("monai.networks.blocks.Warp: Using PyTorch native grid_sample.")
+            self._interp_mode = GridSampleMode(mode).value
+
+        # resolves _padding_mode for different methods
+        if USE_COMPILED:
+            if padding_mode in (pad.value for pad in GridSamplePadMode):
+                padding_mode = GridSamplePadMode(padding_mode)
+                if padding_mode == GridSamplePadMode.ZEROS:
+                    padding_mode = 7
+                elif padding_mode == GridSamplePadMode.BORDER:
+                    padding_mode = 0
+                elif padding_mode == GridSamplePadMode.REFLECTION:
+                    padding_mode = 1
+                else:
+                    padding_mode = 0  # default to nearest
+            self._padding_mode = padding_mode
+        else:
+            self._padding_mode = GridSamplePadMode(padding_mode).value
 
     @staticmethod
     def get_reference_grid(ddf: torch.Tensor) -> torch.Tensor:
@@ -51,14 +93,7 @@ def get_reference_grid(ddf: torch.Tensor) -> torch.Tensor:
         grid = grid.to(ddf)
         return grid
 
-    @staticmethod
-    def normalize_grid(grid: torch.Tensor) -> torch.Tensor:
-        # (batch, ..., self.spatial_dims)
-        for i, dim in enumerate(grid.shape[1:-1]):
-            grid[..., i] = grid[..., i] * 2 / (dim - 1) - 1
-        return grid
-
-    def forward(self, image: torch.Tensor, ddf: torch.Tensor) -> torch.Tensor:
+    def forward(self, image: torch.Tensor, ddf: torch.Tensor):
         """
         Args:
             image: Tensor in shape (batch, num_channels, H, W[, D])
@@ -67,55 +102,39 @@ def forward(self, image: torch.Tensor, ddf: torch.Tensor) -> torch.Tensor:
         Returns:
             warped_image in the same shape as image (batch, num_channels, H, W[, D])
         """
-        if len(image.shape) != 2 + self.spatial_dims:
-            raise ValueError(f"expecting {self.spatial_dims + 2}-d input, " f"got input in shape {image.shape}")
-        if len(ddf.shape) != 2 + self.spatial_dims or ddf.shape[1] != self.spatial_dims:
-            raise ValueError(
-                f"expecting {self.spatial_dims + 2}-d ddf with {self.spatial_dims} channels, "
-                f"got ddf in shape {ddf.shape}"
-            )
-        if image.shape[0] != ddf.shape[0] or image.shape[2:] != ddf.shape[2:]:
+        spatial_dims = len(image.shape) - 2
+        if spatial_dims not in (2, 3):
+            raise NotImplementedError(f"got unsupported spatial_dims={spatial_dims}, currently support 2 or 3.")
+        ddf_shape = (image.shape[0], spatial_dims) + tuple(image.shape[2:])
+        if ddf.shape != ddf_shape:
             raise ValueError(
-                "expecting image and ddf of same batch size and spatial size, "
-                f"got image of shape {image.shape}, ddf of shape {ddf.shape}"
+                f"Given input {spatial_dims}-d image shape {image.shape}, " f"the input DDF shape must be {ddf_shape}."
             )
-
         grid = self.get_reference_grid(ddf) + ddf
-        grid = grid.permute([0] + list(range(2, 2 + self.spatial_dims)) + [1])  # (batch, ..., self.spatial_dims)
-
-        if self.mode > 1:
-            raise ValueError(f"{self.mode}-order interpolation not yet implemented.")
-            # if not USE_COMPILED:
-            #     raise ValueError(f"cannot perform {self.mode}-order interpolation without C compile.")
-            # _padding_mode = self.padding_mode.value
-            # if _padding_mode == "zeros":
-            #     bound = 7
-            # elif _padding_mode == "border":
-            #     bound = 0
-            # else:
-            #     bound = 1
-            # warped_image: torch.Tensor = grid_pull(
-            #     image,
-            #     grid,
-            #     bound=bound,
-            #     extrapolate=True,
-            #     interpolation=self.mode,
-            # )
-        else:
-            grid = self.normalize_grid(grid)
-            index_ordering: List[int] = list(range(self.spatial_dims - 1, -1, -1))
+        grid = grid.permute([0] + list(range(2, 2 + spatial_dims)) + [1])  # (batch, ..., spatial_dims)
+
+        if not USE_COMPILED:  # pytorch native grid_sample
+            for i, dim in enumerate(grid.shape[1:-1]):
+                grid[..., i] = grid[..., i] * 2 / (dim - 1) - 1
+            index_ordering: List[int] = list(range(spatial_dims - 1, -1, -1))
             grid = grid[..., index_ordering]  # z, y, x -> x, y, z
-            _interp_mode = "bilinear" if self.mode == 1 else "nearest"
-            warped_image = F.grid_sample(
-                image, grid, mode=_interp_mode, padding_mode=self.padding_mode.value, align_corners=True
+            return F.grid_sample(
+                image, grid, mode=self._interp_mode, padding_mode=f"{self._padding_mode}", align_corners=True
             )
 
-        return warped_image
+        # using csrc resampling
+        return grid_pull(
+            image,
+            grid,
+            bound=self._padding_mode,
+            extrapolate=True,
+            interpolation=self._interp_mode,
+        )
 
 
 class DVF2DDF(nn.Module):
     """
-    Layer calculates a dense velocity field (DVF) from a dense displacement field (DDF)
+    Layer calculates a dense displacement field (DDF) from a dense velocity field (DVF)
     with scaling and squaring.
 
     Adapted from:
@@ -125,16 +144,15 @@ class DVF2DDF(nn.Module):
 
     def __init__(
         self,
-        spatial_dims: int,
         num_steps: int = 7,
-        mode: int = 1,
-        padding_mode: Optional[Union[GridSamplePadMode, str]] = GridSamplePadMode.ZEROS,
+        mode=GridSampleMode.BILINEAR.value,
+        padding_mode=GridSamplePadMode.ZEROS.value,
     ):
         super(DVF2DDF, self).__init__()
         if num_steps <= 0:
             raise ValueError(f"expecting positive num_steps, got {num_steps}")
         self.num_steps = num_steps
-        self.warp_layer = Warp(spatial_dims=spatial_dims, mode=mode, padding_mode=padding_mode)
+        self.warp_layer = Warp(mode=mode, padding_mode=padding_mode)
 
     def forward(self, dvf):
         """
@@ -142,7 +160,7 @@ def forward(self, dvf):
             dvf: dvf to be transformed, in shape (batch, ``spatial_dims``, H, W[,D])
 
         Returns:
-
+            a dense displacement field
         """
         ddf: torch.Tensor = dvf / (2 ** self.num_steps)
         for _ in range(self.num_steps):
diff --git a/monai/networks/layers/factories.py b/monai/networks/layers/factories.py
index ec36b2ed95..9165a8ebe4 100644
--- a/monai/networks/layers/factories.py
+++ b/monai/networks/layers/factories.py
@@ -256,6 +256,13 @@ def swish_factory():
     return Swish
 
 
+@Act.factory_function("memswish")
+def memswish_factory():
+    from monai.networks.blocks.activation import MemoryEfficientSwish
+
+    return MemoryEfficientSwish
+
+
 @Act.factory_function("mish")
 def mish_factory():
     from monai.networks.blocks.activation import Mish
diff --git a/monai/networks/layers/filtering.py b/monai/networks/layers/filtering.py
index 1bec725c7e..3b2214d59a 100644
--- a/monai/networks/layers/filtering.py
+++ b/monai/networks/layers/filtering.py
@@ -32,7 +32,7 @@ class BilateralFilter(torch.autograd.Function):
         input: input tensor.
 
         spatial sigma: the standard deviation of the spatial blur. Higher values can
-            hurt performace when not using the approximate method (see fast approx).
+            hurt performance when not using the approximate method (see fast approx).
 
         color sigma: the standard deviation of the color blur. Lower values preserve
             edges better whilst higher values tend to a simple gaussian spatial blur.
@@ -47,15 +47,17 @@ class BilateralFilter(torch.autograd.Function):
 
     @staticmethod
     def forward(ctx, input, spatial_sigma=5, color_sigma=0.5, fast_approx=True):
-        ctx.save_for_backward(spatial_sigma, color_sigma, fast_approx)
+        ctx.ss = spatial_sigma
+        ctx.cs = color_sigma
+        ctx.fa = fast_approx
         output_data = _C.bilateral_filter(input, spatial_sigma, color_sigma, fast_approx)
         return output_data
 
     @staticmethod
     def backward(ctx, grad_output):
-        spatial_sigma, color_sigma, fast_approx = ctx.saved_variables
+        spatial_sigma, color_sigma, fast_approx = ctx.ss, ctx.cs, ctx.fa
         grad_input = _C.bilateral_filter(grad_output, spatial_sigma, color_sigma, fast_approx)
-        return grad_input
+        return grad_input, None, None, None
 
 
 class PHLFilter(torch.autograd.Function):
@@ -93,6 +95,7 @@ def forward(ctx, input, features, sigmas=None):
 
     @staticmethod
     def backward(ctx, grad_output):
-        scaled_features = ctx.saved_variables
-        grad_input = PHLFilter.scale(grad_output, scaled_features)
-        return grad_input
+        raise NotImplementedError("PHLFilter does not currently support Backpropagation")
+        # scaled_features, = ctx.saved_variables
+        # grad_input = _C.phl_filter(grad_output, scaled_features)
+        # return grad_input
diff --git a/monai/networks/layers/simplelayers.py b/monai/networks/layers/simplelayers.py
index f560526db8..6737e54da7 100644
--- a/monai/networks/layers/simplelayers.py
+++ b/monai/networks/layers/simplelayers.py
@@ -10,14 +10,14 @@
 # limitations under the License.
 
 import math
-from typing import Sequence, Union, cast
+from typing import List, Sequence, Union
 
 import torch
 import torch.nn.functional as F
 from torch import nn
 from torch.autograd import Function
 
-from monai.networks.layers.convutils import gaussian_1d, same_padding
+from monai.networks.layers.convutils import gaussian_1d
 from monai.networks.layers.factories import Conv
 from monai.utils import (
     PT_BEFORE_1_7,
@@ -164,9 +164,45 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
         return x.reshape(shape)
 
 
-def separable_filtering(
-    x: torch.Tensor, kernels: Union[Sequence[torch.Tensor], torch.Tensor], mode: str = "zeros"
+def _separable_filtering_conv(
+    input_: torch.Tensor,
+    kernels: List[torch.Tensor],
+    pad_mode: str,
+    d: int,
+    spatial_dims: int,
+    paddings: List[int],
+    num_channels: int,
 ) -> torch.Tensor:
+
+    if d < 0:
+        return input_
+
+    s = [1] * len(input_.shape)
+    s[d + 2] = -1
+    _kernel = kernels[d].reshape(s)
+
+    # if filter kernel is unity, don't convolve
+    if _kernel.numel() == 1 and _kernel[0] == 1:
+        return _separable_filtering_conv(input_, kernels, pad_mode, d - 1, spatial_dims, paddings, num_channels)
+
+    _kernel = _kernel.repeat([num_channels, 1] + [1] * spatial_dims)
+    _padding = [0] * spatial_dims
+    _padding[d] = paddings[d]
+    conv_type = [F.conv1d, F.conv2d, F.conv3d][spatial_dims - 1]
+
+    # translate padding for input to torch.nn.functional.pad
+    _reversed_padding_repeated_twice: List[List[int]] = [[p, p] for p in reversed(_padding)]
+    _sum_reversed_padding_repeated_twice: List[int] = sum(_reversed_padding_repeated_twice, [])
+    padded_input = F.pad(input_, _sum_reversed_padding_repeated_twice, mode=pad_mode)
+
+    return conv_type(
+        input=_separable_filtering_conv(padded_input, kernels, pad_mode, d - 1, spatial_dims, paddings, num_channels),
+        weight=_kernel,
+        groups=num_channels,
+    )
+
+
+def separable_filtering(x: torch.Tensor, kernels: List[torch.Tensor], mode: str = "zeros") -> torch.Tensor:
     """
     Apply 1-D convolutions along each spatial dimension of `x`.
 
@@ -186,36 +222,12 @@ def separable_filtering(
         raise TypeError(f"x must be a torch.Tensor but is {type(x).__name__}.")
 
     spatial_dims = len(x.shape) - 2
-    _kernels = [
-        torch.as_tensor(s, dtype=torch.float, device=s.device if isinstance(s, torch.Tensor) else None)
-        for s in ensure_tuple_rep(kernels, spatial_dims)
-    ]
-    _paddings = [cast(int, (same_padding(k.shape[0]))) for k in _kernels]
+    _kernels = [s.float() for s in kernels]
+    _paddings = [(k.shape[0] - 1) // 2 for k in _kernels]
     n_chs = x.shape[1]
+    pad_mode = "constant" if mode == "zeros" else mode
 
-    def _conv(input_: torch.Tensor, d: int) -> torch.Tensor:
-        if d < 0:
-            return input_
-        s = [1] * len(input_.shape)
-        s[d + 2] = -1
-        _kernel = kernels[d].reshape(s)
-        # if filter kernel is unity, don't convolve
-        if _kernel.numel() == 1 and _kernel[0] == 1:
-            return _conv(input_, d - 1)
-        _kernel = _kernel.repeat([n_chs, 1] + [1] * spatial_dims)
-        _padding = [0] * spatial_dims
-        _padding[d] = _paddings[d]
-        conv_type = [F.conv1d, F.conv2d, F.conv3d][spatial_dims - 1]
-        # translate padding for input to torch.nn.functional.pad
-        _reversed_padding_repeated_twice = [p for p in reversed(_padding) for _ in range(2)]
-        pad_mode = "constant" if mode == "zeros" else mode
-        return conv_type(
-            input=_conv(F.pad(input_, _reversed_padding_repeated_twice, mode=pad_mode), d - 1),
-            weight=_kernel,
-            groups=n_chs,
-        )
-
-    return _conv(x, spatial_dims - 1)
+    return _separable_filtering_conv(x, kernels, pad_mode, spatial_dims - 1, spatial_dims, _paddings, n_chs)
 
 
 class SavitzkyGolayFilter(nn.Module):
@@ -254,8 +266,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
         x = torch.as_tensor(x, device=x.device if isinstance(x, torch.Tensor) else None)
         if torch.is_complex(x):
             raise ValueError("x must be real.")
-        else:
-            x = x.to(dtype=torch.float)
+        x = x.to(dtype=torch.float)
 
         if (self.axis < 0) or (self.axis > len(x.shape) - 1):
             raise ValueError("Invalid axis for shape of x.")
diff --git a/monai/networks/layers/spatial_transforms.py b/monai/networks/layers/spatial_transforms.py
index 175fd05694..03031f3340 100644
--- a/monai/networks/layers/spatial_transforms.py
+++ b/monai/networks/layers/spatial_transforms.py
@@ -35,17 +35,15 @@ def forward(ctx, input, grid, interpolation, bound, extrapolate):
 
     @staticmethod
     def backward(ctx, grad):
-        var = ctx.saved_variables
+        if not (ctx.needs_input_grad[0] or ctx.needs_input_grad[1]):
+            return None, None, None, None, None
+        var = ctx.saved_tensors
         opt = ctx.opt
-        grad_input = grad_grid = None
         grads = _C.grid_pull_backward(grad, *var, *opt)
         if ctx.needs_input_grad[0]:
-            grad_input = grads[0]
-            if ctx.needs_input_grad[1]:
-                grad_grid = grads[1]
-        elif ctx.needs_input_grad[1]:
-            grad_grid = grads[0]
-        return grad_input, grad_grid, None, None, None
+            return grads[0], grads[1] if ctx.needs_input_grad[1] else None, None, None, None
+        if ctx.needs_input_grad[1]:
+            return None, grads[0], None, None, None
 
 
 def grid_pull(input: torch.Tensor, grid: torch.Tensor, interpolation="linear", bound="zero", extrapolate: bool = True):
@@ -60,7 +58,9 @@ def grid_pull(input: torch.Tensor, grid: torch.Tensor, interpolation="linear", b
         - 2 or 'quadratic'  or InterpolationType.quadratic
         - 3 or 'cubic'      or InterpolationType.cubic
         - 4 or 'fourth'     or InterpolationType.fourth
-        - etc.
+        - 5 or 'fifth'      or InterpolationType.fifth
+        - 6 or 'sixth'      or InterpolationType.sixth
+        - 7 or 'seventh'    or InterpolationType.seventh
 
     A list of values can be provided, in the order [W, H, D],
     to specify dimension-specific interpolation orders.
@@ -68,14 +68,13 @@ def grid_pull(input: torch.Tensor, grid: torch.Tensor, interpolation="linear", b
     `bound` can be an int, a string or a BoundType.
     Possible values are::
 
-        - 0 or 'replicate'  or BoundType.replicate
-        - 1 or 'dct1'       or BoundType.dct1
-        - 2 or 'dct2'       or BoundType.dct2
-        - 3 or 'dst1'       or BoundType.dst1
-        - 4 or 'dst2'       or BoundType.dst2
-        - 5 or 'dft'        or BoundType.dft
-        - 6 or 'sliding'    or BoundType.sliding [not implemented]
-        - 7 or 'zero'       or BoundType.zero
+        - 0 or 'replicate' or 'nearest'      or BoundType.replicate
+        - 1 or 'dct1'      or 'mirror'       or BoundType.dct1
+        - 2 or 'dct2'      or 'reflect'      or BoundType.dct2
+        - 3 or 'dst1'      or 'antimirror'   or BoundType.dst1
+        - 4 or 'dst2'      or 'antireflect'  or BoundType.dst2
+        - 5 or 'dft'       or 'wrap'         or BoundType.dft
+        - 7 or 'zero'                        or BoundType.zero
 
     A list of values can be provided, in the order [W, H, D],
     to specify dimension-specific boundary conditions.
@@ -87,15 +86,17 @@ def grid_pull(input: torch.Tensor, grid: torch.Tensor, interpolation="linear", b
         - `dct2` corresponds to Neumann boundary conditions (symmetric)
         - `dst2` corresponds to Dirichlet boundary conditions (antisymmetric)
 
-    See:
-        https://en.wikipedia.org/wiki/Discrete_cosine_transform
-        https://en.wikipedia.org/wiki/Discrete_sine_transform
+    See Also:
+        - https://en.wikipedia.org/wiki/Discrete_cosine_transform
+        - https://en.wikipedia.org/wiki/Discrete_sine_transform
+        - ``help(monai._C.BoundType)``
+        - ``help(monai._C.InterpolationType)``
 
     Args:
         input: Input image. `(B, C, Wi, Hi, Di)`.
-        grid: Deformation field. `(B, Wo, Ho, Do, 2|3)`.
+        grid: Deformation field. `(B, Wo, Ho, Do, 1|2|3)`.
         interpolation (int or list[int] , optional): Interpolation order.
-            Defaults to `1`.
+            Defaults to `'linear'`.
         bound (BoundType, or list[BoundType], optional): Boundary conditions.
             Defaults to `'zero'`.
         extrapolate: Extrapolate out-of-bound data.
@@ -106,11 +107,10 @@ def grid_pull(input: torch.Tensor, grid: torch.Tensor, interpolation="linear", b
 
     """
     # Convert parameters
-    bound = ensure_tuple(bound)
-    interpolation = ensure_tuple(interpolation)
-    bound = [_C.BoundType.__members__[b] if isinstance(b, str) else _C.BoundType(b) for b in bound]
+    bound = [_C.BoundType.__members__[b] if isinstance(b, str) else _C.BoundType(b) for b in ensure_tuple(bound)]
     interpolation = [
-        _C.InterpolationType.__members__[i] if isinstance(i, str) else _C.InterpolationType(i) for i in interpolation
+        _C.InterpolationType.__members__[i] if isinstance(i, str) else _C.InterpolationType(i)
+        for i in ensure_tuple(interpolation)
     ]
 
     return _GridPull.apply(input, grid, interpolation, bound, extrapolate)
@@ -129,17 +129,15 @@ def forward(ctx, input, grid, shape, interpolation, bound, extrapolate):
 
     @staticmethod
     def backward(ctx, grad):
-        var = ctx.saved_variables
+        if not (ctx.needs_input_grad[0] or ctx.needs_input_grad[1]):
+            return None, None, None, None, None, None
+        var = ctx.saved_tensors
         opt = ctx.opt
-        grad_input = grad_grid = None
         grads = _C.grid_push_backward(grad, *var, *opt)
         if ctx.needs_input_grad[0]:
-            grad_input = grads[0]
-            if ctx.needs_input_grad[1]:
-                grad_grid = grads[1]
-        elif ctx.needs_input_grad[1]:
-            grad_grid = grads[0]
-        return grad_input, grad_grid, None, None, None, None
+            return grads[0], grads[1] if ctx.needs_input_grad[1] else None, None, None, None, None
+        if ctx.needs_input_grad[1]:
+            return None, grads[0], None, None, None, None
 
 
 def grid_push(
@@ -156,7 +154,9 @@ def grid_push(
         - 2 or 'quadratic'  or InterpolationType.quadratic
         - 3 or 'cubic'      or InterpolationType.cubic
         - 4 or 'fourth'     or InterpolationType.fourth
-        - etc.
+        - 5 or 'fifth'      or InterpolationType.fifth
+        - 6 or 'sixth'      or InterpolationType.sixth
+        - 7 or 'seventh'    or InterpolationType.seventh
 
     A list of values can be provided, in the order `[W, H, D]`,
     to specify dimension-specific interpolation orders.
@@ -164,14 +164,13 @@ def grid_push(
     `bound` can be an int, a string or a BoundType.
     Possible values are::
 
-        - 0 or 'replicate'  or BoundType.replicate
-        - 1 or 'dct1'       or BoundType.dct1
-        - 2 or 'dct2'       or BoundType.dct2
-        - 3 or 'dst1'       or BoundType.dst1
-        - 4 or 'dst2'       or BoundType.dst2
-        - 5 or 'dft'        or BoundType.dft
-        - 6 or 'sliding'    or BoundType.sliding [not implemented]
-        - 7 or 'zero'       or BoundType.zero
+        - 0 or 'replicate' or 'nearest'      or BoundType.replicate
+        - 1 or 'dct1'      or 'mirror'       or BoundType.dct1
+        - 2 or 'dct2'      or 'reflect'      or BoundType.dct2
+        - 3 or 'dst1'      or 'antimirror'   or BoundType.dst1
+        - 4 or 'dst2'      or 'antireflect'  or BoundType.dst2
+        - 5 or 'dft'       or 'wrap'         or BoundType.dft
+        - 7 or 'zero'                        or BoundType.zero
 
     A list of values can be provided, in the order `[W, H, D]`,
     to specify dimension-specific boundary conditions.
@@ -183,17 +182,19 @@ def grid_push(
         - `dct2` corresponds to Neumann boundary conditions (symmetric)
         - `dst2` corresponds to Dirichlet boundary conditions (antisymmetric)
 
-    See also:
+    See Also:
 
         - https://en.wikipedia.org/wiki/Discrete_cosine_transform
         - https://en.wikipedia.org/wiki/Discrete_sine_transform
+        - ``help(monai._C.BoundType)``
+        - ``help(monai._C.InterpolationType)``
 
     Args:
         input: Input image `(B, C, Wi, Hi, Di)`.
-        grid: Deformation field `(B, Wi, Hi, Di, 2|3)`.
+        grid: Deformation field `(B, Wi, Hi, Di, 1|2|3)`.
         shape: Shape of the source image.
         interpolation (int or list[int] , optional): Interpolation order.
-            Defaults to `1`.
+            Defaults to `'linear'`.
         bound (BoundType, or list[BoundType], optional): Boundary conditions.
             Defaults to `'zero'`.
         extrapolate: Extrapolate out-of-bound data.
@@ -204,11 +205,10 @@ def grid_push(
 
     """
     # Convert parameters
-    bound = ensure_tuple(bound)
-    interpolation = ensure_tuple(interpolation)
-    bound = [_C.BoundType.__members__[b] if isinstance(b, str) else _C.BoundType(b) for b in bound]
+    bound = [_C.BoundType.__members__[b] if isinstance(b, str) else _C.BoundType(b) for b in ensure_tuple(bound)]
     interpolation = [
-        _C.InterpolationType.__members__[i] if isinstance(i, str) else _C.InterpolationType(i) for i in interpolation
+        _C.InterpolationType.__members__[i] if isinstance(i, str) else _C.InterpolationType(i)
+        for i in ensure_tuple(interpolation)
     ]
 
     if shape is None:
@@ -230,12 +230,11 @@ def forward(ctx, grid, shape, interpolation, bound, extrapolate):
 
     @staticmethod
     def backward(ctx, grad):
-        var = ctx.saved_variables
-        opt = ctx.opt
-        grad_grid = None
         if ctx.needs_input_grad[0]:
-            grad_grid = _C.grid_count_backward(grad, *var, *opt)
-        return grad_grid, None, None, None, None
+            var = ctx.saved_tensors
+            opt = ctx.opt
+            return _C.grid_count_backward(grad, *var, *opt), None, None, None, None
+        return None, None, None, None, None
 
 
 def grid_count(grid: torch.Tensor, shape=None, interpolation="linear", bound="zero", extrapolate: bool = True):
@@ -252,7 +251,9 @@ def grid_count(grid: torch.Tensor, shape=None, interpolation="linear", bound="ze
         - 2 or 'quadratic'  or InterpolationType.quadratic
         - 3 or 'cubic'      or InterpolationType.cubic
         - 4 or 'fourth'     or InterpolationType.fourth
-        - etc.
+        - 5 or 'fifth'      or InterpolationType.fifth
+        - 6 or 'sixth'      or InterpolationType.sixth
+        - 7 or 'seventh'    or InterpolationType.seventh
 
     A list of values can be provided, in the order [W, H, D],
     to specify dimension-specific interpolation orders.
@@ -260,14 +261,13 @@ def grid_count(grid: torch.Tensor, shape=None, interpolation="linear", bound="ze
     `bound` can be an int, a string or a BoundType.
     Possible values are::
 
-        - 0 or 'replicate'  or BoundType.replicate
-        - 1 or 'dct1'       or BoundType.dct1
-        - 2 or 'dct2'       or BoundType.dct2
-        - 3 or 'dst1'       or BoundType.dst1
-        - 4 or 'dst2'       or BoundType.dst2
-        - 5 or 'dft'        or BoundType.dft
-        - 6 or 'sliding'    or BoundType.sliding [not implemented]
-        - 7 or 'zero'       or BoundType.zero
+        - 0 or 'replicate' or 'nearest'      or BoundType.replicate
+        - 1 or 'dct1'      or 'mirror'       or BoundType.dct1
+        - 2 or 'dct2'      or 'reflect'      or BoundType.dct2
+        - 3 or 'dst1'      or 'antimirror'   or BoundType.dst1
+        - 4 or 'dst2'      or 'antireflect'  or BoundType.dst2
+        - 5 or 'dft'       or 'wrap'         or BoundType.dft
+        - 7 or 'zero'                        or BoundType.zero
 
     A list of values can be provided, in the order [W, H, D],
     to specify dimension-specific boundary conditions.
@@ -283,12 +283,14 @@ def grid_count(grid: torch.Tensor, shape=None, interpolation="linear", bound="ze
 
         - https://en.wikipedia.org/wiki/Discrete_cosine_transform
         - https://en.wikipedia.org/wiki/Discrete_sine_transform
+        - ``help(monai._C.BoundType)``
+        - ``help(monai._C.InterpolationType)``
 
     Args:
         grid: Deformation field `(B, Wi, Hi, Di, 2|3)`.
         shape: shape of the source image.
         interpolation (int or list[int] , optional): Interpolation order.
-            Defaults to `1`.
+            Defaults to `'linear'`.
         bound (BoundType, or list[BoundType], optional): Boundary conditions.
             Defaults to `'zero'`.
         extrapolate (bool, optional): Extrapolate out-of-bound data.
@@ -299,11 +301,10 @@ def grid_count(grid: torch.Tensor, shape=None, interpolation="linear", bound="ze
 
     """
     # Convert parameters
-    bound = ensure_tuple(bound)
-    interpolation = ensure_tuple(interpolation)
-    bound = [_C.BoundType.__members__[b] if isinstance(b, str) else _C.BoundType(b) for b in bound]
+    bound = [_C.BoundType.__members__[b] if isinstance(b, str) else _C.BoundType(b) for b in ensure_tuple(bound)]
     interpolation = [
-        _C.InterpolationType.__members__[i] if isinstance(i, str) else _C.InterpolationType(i) for i in interpolation
+        _C.InterpolationType.__members__[i] if isinstance(i, str) else _C.InterpolationType(i)
+        for i in ensure_tuple(interpolation)
     ]
 
     if shape is None:
@@ -325,18 +326,15 @@ def forward(ctx, input, grid, interpolation, bound, extrapolate):
 
     @staticmethod
     def backward(ctx, grad):
-        var = ctx.saved_variables
+        if not (ctx.needs_input_grad[0] or ctx.needs_input_grad[1]):
+            return None, None, None, None, None
+        var = ctx.saved_tensors
         opt = ctx.opt
-        grad_input = grad_grid = None
-        if ctx.needs_input_grad[0] or ctx.needs_input_grad[1]:
-            grads = _C.grid_grad_backward(grad, *var, *opt)
-            if ctx.needs_input_grad[0]:
-                grad_input = grads[0]
-                if ctx.needs_input_grad[1]:
-                    grad_grid = grads[1]
-            elif ctx.needs_input_grad[1]:
-                grad_grid = grads[0]
-        return grad_input, grad_grid, None, None, None
+        grads = _C.grid_grad_backward(grad, *var, *opt)
+        if ctx.needs_input_grad[0]:
+            return grads[0], grads[1] if ctx.needs_input_grad[1] else None, None, None, None
+        if ctx.needs_input_grad[1]:
+            return None, grads[0], None, None, None
 
 
 def grid_grad(input: torch.Tensor, grid: torch.Tensor, interpolation="linear", bound="zero", extrapolate: bool = True):
@@ -351,7 +349,9 @@ def grid_grad(input: torch.Tensor, grid: torch.Tensor, interpolation="linear", b
         - 2 or 'quadratic'  or InterpolationType.quadratic
         - 3 or 'cubic'      or InterpolationType.cubic
         - 4 or 'fourth'     or InterpolationType.fourth
-        - etc.
+        - 5 or 'fifth'      or InterpolationType.fifth
+        - 6 or 'sixth'      or InterpolationType.sixth
+        - 7 or 'seventh'    or InterpolationType.seventh
 
     A list of values can be provided, in the order [W, H, D],
     to specify dimension-specific interpolation orders.
@@ -359,14 +359,13 @@ def grid_grad(input: torch.Tensor, grid: torch.Tensor, interpolation="linear", b
     `bound` can be an int, a string or a BoundType.
     Possible values are::
 
-        - 0 or 'replicate'  or BoundType.replicate
-        - 1 or 'dct1'       or BoundType.dct1
-        - 2 or 'dct2'       or BoundType.dct2
-        - 3 or 'dst1'       or BoundType.dst1
-        - 4 or 'dst2'       or BoundType.dst2
-        - 5 or 'dft'        or BoundType.dft
-        - 6 or 'sliding'    or BoundType.sliding [not implemented]
-        - 7 or 'zero'       or BoundType.zero
+        - 0 or 'replicate' or 'nearest'      or BoundType.replicate
+        - 1 or 'dct1'      or 'mirror'       or BoundType.dct1
+        - 2 or 'dct2'      or 'reflect'      or BoundType.dct2
+        - 3 or 'dst1'      or 'antimirror'   or BoundType.dst1
+        - 4 or 'dst2'      or 'antireflect'  or BoundType.dst2
+        - 5 or 'dft'       or 'wrap'         or BoundType.dft
+        - 7 or 'zero'                        or BoundType.zero
 
     A list of values can be provided, in the order [W, H, D],
     to specify dimension-specific boundary conditions.
@@ -378,30 +377,32 @@ def grid_grad(input: torch.Tensor, grid: torch.Tensor, interpolation="linear", b
         - `dct2` corresponds to Neumann boundary conditions (symmetric)
         - `dst2` corresponds to Dirichlet boundary conditions (antisymmetric)
 
-    See also:
+    See Also:
 
         - https://en.wikipedia.org/wiki/Discrete_cosine_transform
         - https://en.wikipedia.org/wiki/Discrete_sine_transform
+        - ``help(monai._C.BoundType)``
+        - ``help(monai._C.InterpolationType)``
+
 
     Args:
         input: Input image. `(B, C, Wi, Hi, Di)`.
         grid: Deformation field. `(B, Wo, Ho, Do, 2|3)`.
         interpolation (int or list[int] , optional): Interpolation order.
-            Defaults to `1`.
+            Defaults to `'linear'`.
         bound (BoundType, or list[BoundType], optional): Boundary conditions.
             Defaults to `'zero'`.
         extrapolate: Extrapolate out-of-bound data. Defaults to `True`.
 
     Returns:
-        output (torch.Tensor): Sampled gradients (B, C, Wo, Ho, Do, 2|3).
+        output (torch.Tensor): Sampled gradients (B, C, Wo, Ho, Do, 1|2|3).
 
     """
     # Convert parameters
-    bound = ensure_tuple(bound)
-    interpolation = ensure_tuple(interpolation)
-    bound = [_C.BoundType.__members__[b] if isinstance(b, str) else _C.BoundType(b) for b in bound]
+    bound = [_C.BoundType.__members__[b] if isinstance(b, str) else _C.BoundType(b) for b in ensure_tuple(bound)]
     interpolation = [
-        _C.InterpolationType.__members__[i] if isinstance(i, str) else _C.InterpolationType(i) for i in interpolation
+        _C.InterpolationType.__members__[i] if isinstance(i, str) else _C.InterpolationType(i)
+        for i in ensure_tuple(interpolation)
     ]
 
     return _GridGrad.apply(input, grid, interpolation, bound, extrapolate)
diff --git a/monai/networks/nets/__init__.py b/monai/networks/nets/__init__.py
index a9308de9d7..91f46debf6 100644
--- a/monai/networks/nets/__init__.py
+++ b/monai/networks/nets/__init__.py
@@ -13,15 +13,17 @@
 from .autoencoder import AutoEncoder
 from .basic_unet import BasicUNet, BasicUnet, Basicunet
 from .classifier import Classifier, Critic, Discriminator
-from .densenet import DenseNet, densenet121, densenet169, densenet201, densenet264
+from .densenet import DenseNet, DenseNet121, DenseNet169, DenseNet201, DenseNet264
 from .dynunet import DynUNet, DynUnet, Dynunet
+from .efficientnet import EfficientNet, EfficientNetBN, drop_connect, get_efficientnet_image_size
 from .fullyconnectednet import FullyConnectedNet, VarFullyConnectedNet
 from .generator import Generator
 from .highresnet import HighResBlock, HighResNet
-from .localnet import LocalNet
 from .regressor import Regressor
+from .regunet import GlobalNet, LocalNet, RegUNet
 from .segresnet import SegResNet, SegResNetVAE
-from .senet import SENet, se_resnet50, se_resnet101, se_resnet152, se_resnext50_32x4d, se_resnext101_32x4d, senet154
+from .senet import SENet, SENet154, SEResNet50, SEResNet101, SEResNet152, SEResNext50, SEResNext101
+from .torchvision_fc import TorchVisionFullyConvModel
 from .unet import UNet, Unet, unet
 from .varautoencoder import VarAutoEncoder
 from .vnet import VNet
diff --git a/monai/networks/nets/densenet.py b/monai/networks/nets/densenet.py
index ad1d1d6e5f..280bc6b0cb 100644
--- a/monai/networks/nets/densenet.py
+++ b/monai/networks/nets/densenet.py
@@ -196,28 +196,32 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
         return x
 
 
-model_urls = {
-    "densenet121": "https://download.pytorch.org/models/densenet121-a639ec97.pth",
-    "densenet169": "https://download.pytorch.org/models/densenet169-b2777c0a.pth",
-    "densenet201": "https://download.pytorch.org/models/densenet201-c1103571.pth",
-}
-
-
-def _load_state_dict(model, model_url, progress):
+def _load_state_dict(model, arch, progress):
     """
     This function is used to load pretrained models.
     Adapted from `PyTorch Hub 2D version
     <https://github.com/pytorch/vision/blob/master/torchvision/models/densenet.py>`_
     """
+    model_urls = {
+        "densenet121": "https://download.pytorch.org/models/densenet121-a639ec97.pth",
+        "densenet169": "https://download.pytorch.org/models/densenet169-b2777c0a.pth",
+        "densenet201": "https://download.pytorch.org/models/densenet201-c1103571.pth",
+    }
+    if arch in model_urls:
+        model_url = model_urls[arch]
+    else:
+        raise ValueError(
+            "only 'densenet121', 'densenet169' and 'densenet201' are supported to load pretrained weights."
+        )
     pattern = re.compile(
-        r"^(.*denselayer\d+\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$"
+        r"^(.*denselayer\d+)(\.(?:norm|relu|conv))\.((?:[12])\.(?:weight|bias|running_mean|running_var))$"
     )
 
     state_dict = load_state_dict_from_url(model_url, progress=progress)
     for key in list(state_dict.keys()):
         res = pattern.match(key)
         if res:
-            new_key = res.group(1) + res.group(2)
+            new_key = res.group(1) + ".layers" + res.group(2) + res.group(3)
             state_dict[new_key] = state_dict[key]
             del state_dict[key]
 
@@ -229,47 +233,84 @@ def _load_state_dict(model, model_url, progress):
     model.load_state_dict(model_dict)
 
 
-def densenet121(pretrained: bool = False, progress: bool = True, **kwargs) -> DenseNet:
-    """
-    when `spatial_dims = 2`, specify `pretrained = True` can load Imagenet pretrained weights achieved
-    from `PyTorch Hub 2D version
-    <https://github.com/pytorch/vision/blob/master/torchvision/models/densenet.py>`_
-    """
-    model = DenseNet(init_features=64, growth_rate=32, block_config=(6, 12, 24, 16), **kwargs)
-    if pretrained:
-        arch = "densenet121"
-        _load_state_dict(model, model_urls[arch], progress)
-    return model
+class DenseNet121(DenseNet):
+    def __init__(
+        self,
+        init_features: int = 64,
+        growth_rate: int = 32,
+        block_config: Sequence[int] = (6, 12, 24, 16),
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(DenseNet121, self).__init__(
+            init_features=init_features,
+            growth_rate=growth_rate,
+            block_config=block_config,
+            **kwargs,
+        )
+        if pretrained:
+            # it only worked when `spatial_dims` is 2
+            _load_state_dict(self, "densenet121", progress)
 
 
-def densenet169(pretrained: bool = False, progress: bool = True, **kwargs) -> DenseNet:
-    """
-    when `spatial_dims = 2`, specify `pretrained = True` can load Imagenet pretrained weights achieved
-    from `PyTorch Hub 2D version
-    <https://github.com/pytorch/vision/blob/master/torchvision/models/densenet.py>`_
-    """
-    model = DenseNet(init_features=64, growth_rate=32, block_config=(6, 12, 32, 32), **kwargs)
-    if pretrained:
-        arch = "densenet169"
-        _load_state_dict(model, model_urls[arch], progress)
-    return model
+class DenseNet169(DenseNet):
+    def __init__(
+        self,
+        init_features: int = 64,
+        growth_rate: int = 32,
+        block_config: Sequence[int] = (6, 12, 32, 32),
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(DenseNet169, self).__init__(
+            init_features=init_features,
+            growth_rate=growth_rate,
+            block_config=block_config,
+            **kwargs,
+        )
+        if pretrained:
+            # it only worked when `spatial_dims` is 2
+            _load_state_dict(self, "densenet169", progress)
 
 
-def densenet201(pretrained: bool = False, progress: bool = True, **kwargs) -> DenseNet:
-    """
-    when `spatial_dims = 2`, specify `pretrained = True` can load Imagenet pretrained weights achieved
-    from `PyTorch Hub 2D version
-    <https://github.com/pytorch/vision/blob/master/torchvision/models/densenet.py>`_
-    """
-    model = DenseNet(init_features=64, growth_rate=32, block_config=(6, 12, 48, 32), **kwargs)
-    if pretrained:
-        arch = "densenet201"
-        _load_state_dict(model, model_urls[arch], progress)
-    return model
-
-
-def densenet264(pretrained: bool = False, progress: bool = True, **kwargs) -> DenseNet:
-    model = DenseNet(init_features=64, growth_rate=32, block_config=(6, 12, 64, 48), **kwargs)
-    if pretrained:
-        print("Currently PyTorch Hub does not provide densenet264 pretrained models.")
-    return model
+class DenseNet201(DenseNet):
+    def __init__(
+        self,
+        init_features: int = 64,
+        growth_rate: int = 32,
+        block_config: Sequence[int] = (6, 12, 48, 32),
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(DenseNet201, self).__init__(
+            init_features=init_features,
+            growth_rate=growth_rate,
+            block_config=block_config,
+            **kwargs,
+        )
+        if pretrained:
+            # it only worked when `spatial_dims` is 2
+            _load_state_dict(self, "densenet201", progress)
+
+
+class DenseNet264(DenseNet):
+    def __init__(
+        self,
+        init_features: int = 64,
+        growth_rate: int = 32,
+        block_config: Sequence[int] = (6, 12, 48, 32),
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(DenseNet264, self).__init__(
+            init_features=init_features,
+            growth_rate=growth_rate,
+            block_config=block_config,
+            **kwargs,
+        )
+        if pretrained:
+            print("Currently PyTorch Hub does not provide densenet264 pretrained models.")
diff --git a/monai/networks/nets/dynunet.py b/monai/networks/nets/dynunet.py
index 7d0b3bff79..a69814f61c 100644
--- a/monai/networks/nets/dynunet.py
+++ b/monai/networks/nets/dynunet.py
@@ -91,7 +91,7 @@ class DynUNet(nn.Module):
             (1, 2, 8, 6). The last two will be interpolated into (1, 2, 32, 24), and the stacked tensor
             will has the shape (1, 3, 2, 8, 6).
             When calculating the loss, you can use torch.unbind to get all feature maps can compute the loss
-            one by one with the groud truth, then do a weighted average for all losses to achieve the final loss.
+            one by one with the ground truth, then do a weighted average for all losses to achieve the final loss.
             (To be added: a corresponding tutorial link)
 
         deep_supr_num: number of feature maps that will output during deep supervision head. The
diff --git a/monai/networks/nets/efficientnet.py b/monai/networks/nets/efficientnet.py
new file mode 100644
index 0000000000..65054c870f
--- /dev/null
+++ b/monai/networks/nets/efficientnet.py
@@ -0,0 +1,849 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+import operator
+import re
+from functools import reduce
+from typing import List, NamedTuple, Optional, Tuple, Type, Union
+
+import torch
+from torch import nn
+from torch.utils import model_zoo
+
+from monai.networks.layers.factories import Act, Conv, Norm, Pad, Pool
+
+__all__ = ["EfficientNetBN", "get_efficientnet_image_size", "drop_connect"]
+
+efficientnet_params = {
+    # model_name: (width_mult, depth_mult, image_size, dropout_rate, dropconnect_rate)
+    "efficientnet-b0": (1.0, 1.0, 224, 0.2, 0.2),
+    "efficientnet-b1": (1.0, 1.1, 240, 0.2, 0.2),
+    "efficientnet-b2": (1.1, 1.2, 260, 0.3, 0.2),
+    "efficientnet-b3": (1.2, 1.4, 300, 0.3, 0.2),
+    "efficientnet-b4": (1.4, 1.8, 380, 0.4, 0.2),
+    "efficientnet-b5": (1.6, 2.2, 456, 0.4, 0.2),
+    "efficientnet-b6": (1.8, 2.6, 528, 0.5, 0.2),
+    "efficientnet-b7": (2.0, 3.1, 600, 0.5, 0.2),
+}
+
+
+class MBConvBlock(nn.Module):
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        out_channels: int,
+        kernel_size: int,
+        stride: int,
+        image_size: List[int],
+        expand_ratio: int,
+        se_ratio: Optional[float],
+        id_skip: Optional[bool] = True,
+        batch_norm_momentum: float = 0.99,
+        batch_norm_epsilon: float = 1e-3,
+        drop_connect_rate: Optional[float] = 0.2,
+    ) -> None:
+        """
+        Mobile Inverted Residual Bottleneck Block.
+
+        Args:
+            spatial_dims: number of spatial dimensions.
+            in_channels: number of input channels.
+            out_classes: number of output channels.
+            kernel_size: size of the kernel for conv ops.
+            stride: stride to use for conv ops.
+            image_size: input image resolution.
+            expand_ratio: expansion ratio for inverted bottleneck.
+            se_ratio: squeeze-excitation ratio for se layers.
+            id_skip: whether to use skip connection.
+            batch_norm_momentum: momentum for batch norm.
+            batch_norm_epsilon: epsilon for batch norm.
+            drop_connect_rate: dropconnect rate for drop connection (individual weights) layers.
+
+        References:
+            [1] https://arxiv.org/abs/1704.04861 (MobileNet v1)
+            [2] https://arxiv.org/abs/1801.04381 (MobileNet v2)
+            [3] https://arxiv.org/abs/1905.02244 (MobileNet v3)
+        """
+        super().__init__()
+
+        # select the type of N-Dimensional layers to use
+        # these are based on spatial dims and selected from MONAI factories
+        conv_type = Conv["conv", spatial_dims]
+        batchnorm_type = Norm["batch", spatial_dims]
+        adaptivepool_type = Pool["adaptiveavg", spatial_dims]
+
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.id_skip = id_skip
+        self.stride = stride
+        self.expand_ratio = expand_ratio
+        self.drop_connect_rate = drop_connect_rate
+
+        if (se_ratio is not None) and (0.0 < se_ratio <= 1.0):
+            self.has_se = True
+            self.se_ratio = se_ratio
+        else:
+            self.has_se = False
+
+        bn_mom = 1.0 - batch_norm_momentum  # pytorch"s difference from tensorflow
+        bn_eps = batch_norm_epsilon
+
+        # Expansion phase (Inverted Bottleneck)
+        inp = in_channels  # number of input channels
+        oup = in_channels * expand_ratio  # number of output channels
+        if self.expand_ratio != 1:
+            self._expand_conv = conv_type(in_channels=inp, out_channels=oup, kernel_size=1, bias=False)
+            self._expand_conv_padding = _make_same_padder(self._expand_conv, image_size)
+
+            self._bn0 = batchnorm_type(num_features=oup, momentum=bn_mom, eps=bn_eps)
+        else:
+            # need to have the following to fix JIT error:
+            #   "Module 'MBConvBlock' has no attribute '_expand_conv'"
+
+            # FIXME: find a better way to bypass JIT error
+            self._expand_conv = nn.Identity()
+            self._expand_conv_padding = nn.Identity()
+            self._bn0 = nn.Identity()
+
+        # Depthwise convolution phase
+        self._depthwise_conv = conv_type(
+            in_channels=oup,
+            out_channels=oup,
+            groups=oup,  # groups makes it depthwise
+            kernel_size=kernel_size,
+            stride=self.stride,
+            bias=False,
+        )
+        self._depthwise_conv_padding = _make_same_padder(self._depthwise_conv, image_size)
+        self._bn1 = batchnorm_type(num_features=oup, momentum=bn_mom, eps=bn_eps)
+        image_size = _calculate_output_image_size(image_size, self.stride)
+
+        # Squeeze and Excitation layer, if desired
+        if self.has_se:
+            self._se_adaptpool = adaptivepool_type(1)
+            num_squeezed_channels = max(1, int(in_channels * self.se_ratio))
+            self._se_reduce = conv_type(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1)
+            self._se_reduce_padding = _make_same_padder(self._se_reduce, [1, 1])
+            self._se_expand = conv_type(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1)
+            self._se_expand_padding = _make_same_padder(self._se_expand, [1, 1])
+
+        # Pointwise convolution phase
+        final_oup = out_channels
+        self._project_conv = conv_type(in_channels=oup, out_channels=final_oup, kernel_size=1, bias=False)
+        self._project_conv_padding = _make_same_padder(self._project_conv, image_size)
+        self._bn2 = batchnorm_type(num_features=final_oup, momentum=bn_mom, eps=bn_eps)
+
+        # swish activation to use - using memory efficient swish by default
+        # can be switched to normal swish using self.set_swish() function call
+        self._swish = Act["memswish"]()
+
+    def forward(self, inputs: torch.Tensor):
+        """MBConvBlock"s forward function.
+
+        Args:
+            inputs: Input tensor.
+
+        Returns:
+            Output of this block after processing.
+        """
+        # Expansion and Depthwise Convolution
+        x = inputs
+        if self.expand_ratio != 1:
+            x = self._expand_conv(self._expand_conv_padding(x))
+            x = self._bn0(x)
+            x = self._swish(x)
+
+        x = self._depthwise_conv(self._depthwise_conv_padding(x))
+        x = self._bn1(x)
+        x = self._swish(x)
+
+        # Squeeze and Excitation
+        if self.has_se:
+            x_squeezed = self._se_adaptpool(x)
+            x_squeezed = self._se_reduce(self._se_reduce_padding(x_squeezed))
+            x_squeezed = self._swish(x_squeezed)
+            x_squeezed = self._se_expand(self._se_expand_padding(x_squeezed))
+            x = torch.sigmoid(x_squeezed) * x
+
+        # Pointwise Convolution
+        x = self._project_conv(self._project_conv_padding(x))
+        x = self._bn2(x)
+
+        # Skip connection and drop connect
+        if self.id_skip and self.stride == 1 and self.in_channels == self.out_channels:
+            # the combination of skip connection and drop connect brings about stochastic depth.
+            if self.drop_connect_rate:
+                x = drop_connect(x, p=self.drop_connect_rate, training=self.training)
+            x = x + inputs  # skip connection
+        return x
+
+    def set_swish(self, memory_efficient: bool = True) -> None:
+        """Sets swish function as memory efficient (for training) or standard (for export).
+
+        Args:
+            memory_efficient (bool): Whether to use memory-efficient version of swish.
+        """
+        self._swish = Act["memswish"]() if memory_efficient else Act["swish"](alpha=1.0)
+
+
+class EfficientNet(nn.Module):
+    def __init__(
+        self,
+        blocks_args_str: List[str],
+        spatial_dims: int = 2,
+        in_channels: int = 3,
+        num_classes: int = 1000,
+        width_coefficient: float = 1.0,
+        depth_coefficient: float = 1.0,
+        dropout_rate: float = 0.2,
+        image_size: int = 224,
+        batch_norm_momentum: float = 0.99,
+        batch_norm_epsilon: float = 1e-3,
+        drop_connect_rate: float = 0.2,
+        depth_divisor: int = 8,
+    ) -> None:
+        """
+        EfficientNet based on `Rethinking Model Scaling for Convolutional Neural Networks <https://arxiv.org/pdf/1905.11946.pdf>`_.
+        Adapted from `EfficientNet-PyTorch
+        <https://github.com/lukemelas/EfficientNet-PyTorch>`_.
+
+        Args:
+            blocks_args_str: block definitions.
+            spatial_dims: number of spatial dimensions.
+            in_channels: number of input channels.
+            num_classes: number of output classes.
+            width_coefficient: width multiplier coefficient (w in paper).
+            depth_coefficient: depth multiplier coefficient (d in paper).
+            dropout_rate: dropout rate for dropout layers.
+            image_size: input image resolution.
+            batch_norm_momentum: momentum for batch norm.
+            batch_norm_epsilon: epsilon for batch norm.
+            drop_connect_rate: dropconnect rate for drop connection (individual weights) layers.
+            depth_divisor: depth divisor for channel rounding.
+
+        Examples::
+
+            # for pretrained spatial 2D ImageNet
+            >>> image_size = get_efficientnet_image_size("efficientnet-b0")
+            >>> inputs = torch.rand(1, 3, image_size, image_size)
+            >>> model = EfficientNetBN("efficientnet-b0", pretrained=True)
+            >>> model.eval()
+            >>> outputs = model(inputs)
+
+            # create spatial 2D
+            >>> model = EfficientNetBN("efficientnet-b0", spatial_dims=2)
+
+            # create spatial 3D
+            >>> model = EfficientNetBN("efficientnet-b0", spatial_dims=3)
+
+            # create EfficientNetB7 for spatial 2D
+            >>> model = EfficientNetBN("efficientnet-b7", spatial_dims=2)
+
+        """
+        super().__init__()
+
+        if spatial_dims not in (1, 2, 3):
+            raise ValueError("spatial_dims can only be 1, 2 or 3.")
+
+        # select the type of N-Dimensional layers to use
+        # these are based on spatial dims and selected from MONAI factories
+        conv_type: Type[Union[nn.Conv1d, nn.Conv2d, nn.Conv3d]] = Conv["conv", spatial_dims]
+        batchnorm_type: Type[Union[nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d]] = Norm["batch", spatial_dims]
+        adaptivepool_type: Type[Union[nn.AdaptiveAvgPool1d, nn.AdaptiveAvgPool2d, nn.AdaptiveAvgPool3d]] = Pool[
+            "adaptiveavg", spatial_dims
+        ]
+
+        # decode blocks args into arguments for MBConvBlock
+        blocks_args = _decode_block_list(blocks_args_str)
+
+        # checks for successful decoding of blocks_args_str
+        if not isinstance(blocks_args, list):
+            raise ValueError("blocks_args must be a list")
+
+        if blocks_args == []:
+            raise ValueError("block_args must be non-empty")
+
+        self._blocks_args = blocks_args
+        self.num_classes = num_classes
+        self.in_channels = in_channels
+        self.drop_connect_rate = drop_connect_rate
+
+        # expand input image dimensions to list
+        current_image_size = [image_size] * spatial_dims
+
+        # parameters for batch norm
+        bn_mom = 1 - batch_norm_momentum  # 1 - bn_m to convert tensorflow's arg to pytorch bn compatible
+        bn_eps = batch_norm_epsilon
+
+        # Stem
+        stride = 2
+        out_channels = _round_filters(32, width_coefficient, depth_divisor)  # number of output channels
+        self._conv_stem = conv_type(self.in_channels, out_channels, kernel_size=3, stride=stride, bias=False)
+        self._conv_stem_padding = _make_same_padder(self._conv_stem, current_image_size)
+        self._bn0 = batchnorm_type(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
+        current_image_size = _calculate_output_image_size(current_image_size, stride)
+
+        # build MBConv blocks
+        num_blocks = 0
+        self._blocks = nn.Sequential()
+
+        # update baseline blocks to input/output filters and number of repeats based on width and depth multipliers.
+        for idx, block_args in enumerate(self._blocks_args):
+            block_args = block_args._replace(
+                input_filters=_round_filters(block_args.input_filters, width_coefficient, depth_divisor),
+                output_filters=_round_filters(block_args.output_filters, width_coefficient, depth_divisor),
+                num_repeat=_round_repeats(block_args.num_repeat, depth_coefficient),
+            )
+            self._blocks_args[idx] = block_args
+
+            # calculate the total number of blocks - needed for drop_connect estimation
+            num_blocks += block_args.num_repeat
+
+        # create and add MBConvBlocks to self._blocks
+        idx = 0  # block index counter
+        for block_args in self._blocks_args:
+            blk_drop_connect_rate = self.drop_connect_rate
+
+            # scale drop connect_rate
+            if blk_drop_connect_rate:
+                blk_drop_connect_rate *= float(idx) / num_blocks
+
+            # the first block needs to take care of stride and filter size increase.
+            self._blocks.add_module(
+                str(idx),
+                MBConvBlock(
+                    spatial_dims=spatial_dims,
+                    in_channels=block_args.input_filters,
+                    out_channels=block_args.output_filters,
+                    kernel_size=block_args.kernel_size,
+                    stride=block_args.stride,
+                    image_size=current_image_size,
+                    expand_ratio=block_args.expand_ratio,
+                    se_ratio=block_args.se_ratio,
+                    id_skip=block_args.id_skip,
+                    batch_norm_momentum=batch_norm_momentum,
+                    batch_norm_epsilon=batch_norm_epsilon,
+                    drop_connect_rate=blk_drop_connect_rate,
+                ),
+            )
+            idx += 1  # increment blocks index counter
+
+            current_image_size = _calculate_output_image_size(current_image_size, block_args.stride)
+            if block_args.num_repeat > 1:  # modify block_args to keep same output size
+                block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
+
+            # add remaining block repeated num_repeat times
+            for _ in range(block_args.num_repeat - 1):
+                blk_drop_connect_rate = self.drop_connect_rate
+
+                # scale drop connect_rate
+                if blk_drop_connect_rate:
+                    blk_drop_connect_rate *= float(idx) / num_blocks
+
+                # add blocks
+                self._blocks.add_module(
+                    str(idx),
+                    MBConvBlock(
+                        spatial_dims=spatial_dims,
+                        in_channels=block_args.input_filters,
+                        out_channels=block_args.output_filters,
+                        kernel_size=block_args.kernel_size,
+                        stride=block_args.stride,
+                        image_size=current_image_size,
+                        expand_ratio=block_args.expand_ratio,
+                        se_ratio=block_args.se_ratio,
+                        id_skip=block_args.id_skip,
+                        batch_norm_momentum=batch_norm_momentum,
+                        batch_norm_epsilon=batch_norm_epsilon,
+                        drop_connect_rate=blk_drop_connect_rate,
+                    ),
+                )
+                idx += 1  # increment blocks index counter
+
+        # sanity check to see if len(self._blocks) equal expected num_blocks
+        if len(self._blocks) != num_blocks:
+            raise ValueError("number of blocks created != num_blocks")
+
+        # Head
+        head_in_channels = block_args.output_filters
+        out_channels = _round_filters(1280, width_coefficient, depth_divisor)
+        self._conv_head = conv_type(head_in_channels, out_channels, kernel_size=1, bias=False)
+        self._conv_head_padding = _make_same_padder(self._conv_head, current_image_size)
+        self._bn1 = batchnorm_type(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
+
+        # final linear layer
+        self._avg_pooling = adaptivepool_type(1)
+        self._dropout = nn.Dropout(dropout_rate)
+        self._fc = nn.Linear(out_channels, self.num_classes)
+
+        # swish activation to use - using memory efficient swish by default
+        # can be switched to normal swish using self.set_swish() function call
+        self._swish = Act["memswish"]()
+
+        # initialize weights using Tensorflow's init method from official impl.
+        self._initialize_weights()
+
+    def set_swish(self, memory_efficient: bool = True) -> None:
+        """
+        Sets swish function as memory efficient (for training) or standard (for JIT export).
+
+        Args:
+            memory_efficient: whether to use memory-efficient version of swish.
+
+        """
+        self._swish = Act["memswish"]() if memory_efficient else Act["swish"](alpha=1.0)
+        for block in self._blocks:
+            block.set_swish(memory_efficient)
+
+    def forward(self, inputs: torch.Tensor):
+        """
+        Args:
+            inputs: input should have spatially N dimensions
+            ``(Batch, in_channels, dim_0[, dim_1, ..., dim_N])``, N is defined by `dimensions`.
+
+        Returns:
+            A torch Tensor of classification prediction in shape
+            ``(Batch, num_classes)``.
+        """
+        # Stem
+        x = self._conv_stem(self._conv_stem_padding(inputs))
+        x = self._swish(self._bn0(x))
+        # Blocks
+        x = self._blocks(x)
+        # Head
+        x = self._conv_head(self._conv_head_padding(x))
+        x = self._swish(self._bn1(x))
+
+        # Pooling and final linear layer
+        x = self._avg_pooling(x)
+
+        x = x.flatten(start_dim=1)
+        x = self._dropout(x)
+        x = self._fc(x)
+        return x
+
+    def _initialize_weights(self) -> None:
+        """
+        Args:
+            None, initializes weights for conv/linear/batchnorm layers
+            following weight init methods from
+            `official Tensorflow EfficientNet implementation
+            <https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py#L61>`_.
+            Adapted from `EfficientNet-PyTorch's init method
+            <https://github.com/rwightman/gen-efficientnet-pytorch/blob/master/geffnet/efficientnet_builder.py>`_.
+        """
+        for _, m in self.named_modules():
+            if isinstance(m, (nn.Conv1d, nn.Conv2d, nn.Conv3d)):
+                fan_out = reduce(operator.mul, m.kernel_size, 1) * m.out_channels
+                m.weight.data.normal_(0, math.sqrt(2.0 / fan_out))
+                if m.bias is not None:
+                    m.bias.data.zero_()
+            elif isinstance(m, (nn.BatchNorm1d, nn.BatchNorm2d, nn.BatchNorm3d)):
+                m.weight.data.fill_(1.0)
+                m.bias.data.zero_()
+            elif isinstance(m, nn.Linear):
+                fan_out = m.weight.size(0)
+                fan_in = 0
+                init_range = 1.0 / math.sqrt(fan_in + fan_out)
+                m.weight.data.uniform_(-init_range, init_range)
+                m.bias.data.zero_()
+
+
+class EfficientNetBN(EfficientNet):
+    def __init__(
+        self,
+        model_name: str,
+        pretrained: bool = True,
+        progress: bool = True,
+        spatial_dims: int = 2,
+        in_channels: int = 3,
+        num_classes: int = 1000,
+    ) -> None:
+        """
+        Generic wrapper around EfficientNet, used to initialize EfficientNet-B0 to EfficientNet-B7 models
+        model_name is mandatory argument as there is no EfficientNetBN itself,
+        it needs the N in [0, 1, 2, 3, 4, 5, 6, 7] to be a model
+
+        Args:
+            model_name: name of model to initialize, can be from [efficientnet-b0, ..., efficientnet-b7].
+            pretrained: whether to initialize pretrained ImageNet weights, only available for spatial_dims=2.
+            progress: whether to show download progress for pretrained weights download.
+            spatial_dims: number of spatial dimensions.
+            in_channels: number of input channels.
+            num_classes: number of output classes.
+
+        """
+        # block args for EfficientNet-B0 to EfficientNet-B7
+        blocks_args_str = [
+            "r1_k3_s11_e1_i32_o16_se0.25",
+            "r2_k3_s22_e6_i16_o24_se0.25",
+            "r2_k5_s22_e6_i24_o40_se0.25",
+            "r3_k3_s22_e6_i40_o80_se0.25",
+            "r3_k5_s11_e6_i80_o112_se0.25",
+            "r4_k5_s22_e6_i112_o192_se0.25",
+            "r1_k3_s11_e6_i192_o320_se0.25",
+        ]
+
+        # check if model_name is valid model
+        if model_name not in efficientnet_params.keys():
+            raise ValueError(
+                "invalid model_name {} found, must be one of {} ".format(
+                    model_name, ", ".join(efficientnet_params.keys())
+                )
+            )
+
+        # get network parameters
+        weight_coeff, depth_coeff, image_size, dropout_rate, dropconnect_rate = efficientnet_params[model_name]
+
+        # create model and initialize random weights
+        super(EfficientNetBN, self).__init__(
+            blocks_args_str=blocks_args_str,
+            spatial_dims=spatial_dims,
+            in_channels=in_channels,
+            num_classes=num_classes,
+            width_coefficient=weight_coeff,
+            depth_coefficient=depth_coeff,
+            dropout_rate=dropout_rate,
+            image_size=image_size,
+            drop_connect_rate=dropconnect_rate,
+        )
+
+        # attempt to load pretrained
+        is_default_model = (spatial_dims == 2) and (in_channels == 3)
+        loadable_from_file = pretrained and is_default_model
+
+        if loadable_from_file:
+            # skip loading fc layers for transfer learning applications
+            load_fc = num_classes == 1000
+
+            # only pretrained for when `spatial_dims` is 2
+            _load_state_dict(self, model_name, progress, load_fc)
+        else:
+            print(
+                "Skipping loading pretrained weights for non-default {}, pretrained={}, is_default_model={}".format(
+                    model_name, pretrained, is_default_model
+                )
+            )
+
+
+def get_efficientnet_image_size(model_name: str) -> int:
+    """
+    Get the input image size for a given efficientnet model.
+
+    Args:
+        model_name: name of model to initialize, can be from [efficientnet-b0, ..., efficientnet-b7].
+
+    Returns:
+        Image size for single spatial dimension as integer.
+
+    """
+    # check if model_name is valid model
+    if model_name not in efficientnet_params.keys():
+        raise ValueError(
+            "invalid model_name {} found, must be one of {} ".format(model_name, ", ".join(efficientnet_params.keys()))
+        )
+
+    # return input image size (all dims equal so only need to return for one dim)
+    _, _, res, _, _ = efficientnet_params[model_name]
+    return res
+
+
+def drop_connect(inputs: torch.Tensor, p: float, training: bool) -> torch.Tensor:
+    """
+    Drop connect layer that drops individual connections.
+    Differs from dropout as dropconnect drops connections instead of whole neurons as in dropout.
+
+    Based on `Deep Networks with Stochastic Depth <https://arxiv.org/pdf/1603.09382.pdf>`_.
+    Adapted from `Official Tensorflow EfficientNet utils
+    <https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/utils.py>`_.
+
+    This function is generalized for MONAI's N-Dimensional spatial activations
+    e.g. 1D activations [B, C, H], 2D activations [B, C, H, W] and 3D activations [B, C, H, W, D]
+
+    Args:
+        input: input tensor with [B, C, dim_1, dim_2, ..., dim_N] where N=spatial_dims.
+        p: probability to use for dropping connections.
+        training: whether in training or evaluation mode.
+
+    Returns:
+        output: output tensor after applying drop connection.
+    """
+    if p < 0.0 or p > 1.0:
+        raise ValueError("p must be in range of [0, 1], found {}".format(p))
+
+    # eval mode: drop_connect is switched off - so return input without modifying
+    if not training:
+        return inputs
+
+    # train mode: calculate and apply drop_connect
+    batch_size: int = inputs.shape[0]
+    keep_prob: float = 1 - p
+    num_dims: int = len(inputs.shape) - 2
+
+    # build dimensions for random tensor, use num_dims to populate appropriate spatial dims
+    random_tensor_shape: List[int] = [batch_size, 1] + [1] * num_dims
+
+    # generate binary_tensor mask according to probability (p for 0, 1-p for 1)
+    random_tensor: torch.Tensor = torch.rand(random_tensor_shape, dtype=inputs.dtype, device=inputs.device)
+    random_tensor += keep_prob
+
+    # round to form binary tensor
+    binary_tensor: torch.Tensor = torch.floor(random_tensor)
+
+    # drop connect using binary tensor
+    output: torch.Tensor = inputs / keep_prob * binary_tensor
+    return output
+
+
+def _load_state_dict(model: nn.Module, model_name: str, progress: bool, load_fc: bool) -> None:
+    url_map = {
+        "efficientnet-b0": "https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b0-355c32eb.pth",
+        "efficientnet-b1": "https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b1-f1951068.pth",
+        "efficientnet-b2": "https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b2-8bb594d6.pth",
+        "efficientnet-b3": "https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b3-5fb5a3c3.pth",
+        "efficientnet-b4": "https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b4-6ed6700e.pth",
+        "efficientnet-b5": "https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b5-b6417697.pth",
+        "efficientnet-b6": "https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b6-c76e70fd.pth",
+        "efficientnet-b7": "https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b7-dcc49843.pth",
+    }
+    # load state dict from url
+    model_url = url_map[model_name]
+    state_dict = model_zoo.load_url(model_url, progress=progress)
+
+    # load state dict into model parameters
+    if load_fc:  # load everything
+        ret = model.load_state_dict(state_dict, strict=False)
+        if ret.missing_keys:
+            raise ValueError("Found missing keys when loading pretrained weights: {}".format(ret.missing_keys))
+    else:  # skip final FC layers, for transfer learning cases
+        state_dict.pop("_fc.weight")
+        state_dict.pop("_fc.bias")
+        ret = model.load_state_dict(state_dict, strict=False)
+
+        # check if no other keys missing except FC layer parameters
+        if set(ret.missing_keys) != {"_fc.weight", "_fc.bias"}:
+            raise ValueError("Found missing keys when loading pretrained weights: {}".format(ret.missing_keys))
+
+    # check for any unexpected keys
+    if ret.unexpected_keys:
+        raise ValueError("Missing keys when loading pretrained weights: {}".format(ret.unexpected_keys))
+
+
+def _get_same_padding_conv_nd(
+    image_size: List[int], kernel_size: Tuple[int, ...], dilation: Tuple[int, ...], stride: Tuple[int, ...]
+) -> List[int]:
+    """
+    Helper for getting padding (nn.ConstantPadNd) to be used to get SAME padding
+    conv operations similar to Tensorflow's SAME padding.
+
+    This function is generalized for MONAI's N-Dimensional spatial operations (e.g. Conv1D, Conv2D, Conv3D)
+
+    Args:
+        image_size: input image/feature spatial size.
+        kernel_size: conv kernel's spatial size.
+        dilation: conv dilation rate for Atrous conv.
+        stride: stride for conv operation.
+
+    Returns:
+        paddings for ConstantPadNd padder to be used on input tensor to conv op.
+    """
+    # get number of spatial dimensions, corresponds to kernel size length
+    num_dims = len(kernel_size)
+
+    # additional checks to populate dilation and stride (in case they are single entry tuples)
+    if len(dilation) == 1:
+        dilation = dilation * num_dims
+
+    if len(stride) == 1:
+        stride = stride * num_dims
+
+    # equation to calculate (pad^+ + pad^-) size
+    _pad_size: List[int] = [
+        max((math.ceil(_i_s / _s) - 1) * _s + (_k_s - 1) * _d + 1 - _i_s, 0)
+        for _i_s, _k_s, _d, _s in zip(image_size, kernel_size, dilation, stride)
+    ]
+    # distribute paddings into pad^+ and pad^- following Tensorflow's same padding strategy
+    _paddings: List[Tuple[int, int]] = [(_p // 2, _p - _p // 2) for _p in _pad_size]
+
+    # unroll list of tuples to tuples, and then to list
+    # reversed as nn.ConstantPadNd expects paddings starting with last dimension
+    _paddings_ret: List[int] = [outer for inner in reversed(_paddings) for outer in inner]
+    return _paddings_ret
+
+
+def _make_same_padder(conv_op: Union[nn.Conv1d, nn.Conv2d, nn.Conv3d], image_size: List[int]):
+    """
+    Helper for initializing ConstantPadNd with SAME padding similar to Tensorflow.
+    Uses output of _get_same_padding_conv_nd() to get the padding size.
+
+    This function is generalized for MONAI's N-Dimensional spatial operations (e.g. Conv1D, Conv2D, Conv3D)
+
+    Args:
+        conv_op: nn.ConvNd operation to extract parameters for op from
+        image_size: input image/feature spatial size
+
+    Returns:
+        If padding required then nn.ConstandNd() padder initialized to paddings otherwise nn.Identity()
+    """
+    # calculate padding required
+    padding: List[int] = _get_same_padding_conv_nd(image_size, conv_op.kernel_size, conv_op.dilation, conv_op.stride)
+
+    # initialize and return padder
+    padder = Pad["constantpad", len(padding) // 2]
+    if sum(padding) > 0:
+        return padder(padding=padding, value=0.0)
+    else:
+        return nn.Identity()
+
+
+def _round_filters(filters: int, width_coefficient: Optional[float], depth_divisor: float) -> int:
+    """
+    Calculate and round number of filters based on width coefficient multiplier and depth divisor.
+
+    Args:
+        filters: number of input filters.
+        width_coefficient: width coefficient for model.
+        depth_divisor: depth divisor to use.
+
+    Returns:
+        new_filters: new number of filters after calculation.
+    """
+
+    if not width_coefficient:
+        return filters
+
+    multiplier: float = width_coefficient
+    divisor: float = depth_divisor
+    filters_float: float = filters * multiplier
+
+    # follow the formula transferred from official TensorFlow implementation
+    new_filters: float = max(divisor, int(filters_float + divisor / 2) // divisor * divisor)
+    if new_filters < 0.9 * filters_float:  # prevent rounding by more than 10%
+        new_filters += divisor
+    return int(new_filters)
+
+
+def _round_repeats(repeats: int, depth_coefficient: Optional[float]) -> int:
+    """
+    Re-calculate module's repeat number of a block based on depth coefficient multiplier.
+
+    Args:
+        repeats: number of original repeats.
+        depth_coefficient: depth coefficient for model.
+
+    Returns:
+        new repeat: new number of repeat after calculating.
+    """
+    if not depth_coefficient:
+        return repeats
+
+    # follow the formula transferred from official TensorFlow impl.
+    return int(math.ceil(depth_coefficient * repeats))
+
+
+def _calculate_output_image_size(input_image_size: List[int], stride: Union[int, Tuple[int]]):
+    """
+    Calculates the output image size when using _make_same_padder with a stride.
+    Required for static padding.
+
+    Args:
+        input_image_size: input image/feature spatial size.
+        stride: Conv2d operation"s stride.
+
+    Returns:
+        output_image_size: output image/feature spatial size.
+    """
+    # get number of spatial dimensions, corresponds to image spatial size length
+    num_dims = len(input_image_size)
+
+    # checks to extract integer stride in case tuple was received
+    if isinstance(stride, tuple):
+        all_strides_equal = all(stride[0] == s for s in stride)
+        if not all_strides_equal:
+            raise ValueError("unequal strides are not possible, got {}".format(stride))
+
+        stride = stride[0]
+
+    # return output image size
+    return [int(math.ceil(im_sz / stride)) for im_sz in input_image_size]
+
+
+def _decode_block_list(string_list: List[str]):
+    """
+    Decode a list of string notations to specify blocks inside the network.
+
+    Args:
+        string_list: a list of strings, each string is a notation of block.
+
+    Returns:
+        blocks_args: a list of BlockArgs namedtuples of block args.
+    """
+    # Parameters for an individual model block
+    # namedtuple with defaults for mypy help from:
+    # https://stackoverflow.com/a/53255358
+    class BlockArgs(NamedTuple):
+        num_repeat: int
+        kernel_size: int
+        stride: int
+        expand_ratio: int
+        input_filters: int
+        output_filters: int
+        id_skip: bool
+        se_ratio: Optional[float] = None
+
+    def _decode_block_string(block_string: str):
+        """
+        Get a block through a string notation of arguments.
+
+        Args:
+            block_string (str): A string notation of arguments.
+                                Examples: "r1_k3_s11_e1_i32_o16_se0.25".
+
+        Returns:
+            BlockArgs: namedtuple defined at the top of this function.
+        """
+        ops = block_string.split("_")
+        options = {}
+        for op in ops:
+            splits = re.split(r"(\d.*)", op)
+            if len(splits) >= 2:
+                key, value = splits[:2]
+                options[key] = value
+
+        # check stride
+        stride_check = (
+            ("s" in options and len(options["s"]) == 1)
+            or (len(options["s"]) == 2 and options["s"][0] == options["s"][1])
+            or (len(options["s"]) == 3 and options["s"][0] == options["s"][1] and options["s"][0] == options["s"][2])
+        )
+        if not stride_check:
+            raise ValueError("invalid stride option received")
+
+        return BlockArgs(
+            num_repeat=int(options["r"]),
+            kernel_size=int(options["k"]),
+            stride=int(options["s"][0]),
+            expand_ratio=int(options["e"]),
+            input_filters=int(options["i"]),
+            output_filters=int(options["o"]),
+            id_skip=("noskip" not in block_string),
+            se_ratio=float(options["se"]) if "se" in options else None,
+        )
+
+    # convert block strings into BlockArgs for each entry in string_list list
+    blocks_args: List[BlockArgs] = []
+    for current_string in string_list:
+        blocks_args.append(_decode_block_string(current_string))
+
+    # return blocks_args list, to be used for arguments of MBConv layers in EfficientNet
+    return blocks_args
diff --git a/monai/networks/nets/localnet.py b/monai/networks/nets/localnet.py
deleted file mode 100644
index e9df68104d..0000000000
--- a/monai/networks/nets/localnet.py
+++ /dev/null
@@ -1,129 +0,0 @@
-from typing import List, Optional, Tuple, Union
-
-import torch
-from torch import nn
-from torch.nn import functional as F
-
-from monai.networks.blocks.localnet_block import (
-    LocalNetDownSampleBlock,
-    LocalNetFeatureExtractorBlock,
-    LocalNetUpSampleBlock,
-    get_conv_block,
-)
-
-
-class LocalNet(nn.Module):
-    """
-    Reimplementation of LocalNet, based on:
-    `Weakly-supervised convolutional neural networks for multimodal image registration
-    <https://doi.org/10.1016/j.media.2018.07.002>`_.
-    `Label-driven weakly-supervised learning for multimodal deformable image registration
-    <https://arxiv.org/abs/1711.01666>`_.
-
-    Adapted from:
-        DeepReg (https://github.com/DeepRegNet/DeepReg)
-    """
-
-    def __init__(
-        self,
-        spatial_dims: int,
-        in_channels: int,
-        out_channels: int,
-        num_channel_initial: int,
-        extract_levels: List[int],
-        out_activation: Optional[Union[Tuple, str]],
-        out_initializer: str = "kaiming_uniform",
-    ) -> None:
-        """
-        Args:
-            spatial_dims: number of spatial dimensions.
-            in_channels: number of input channels.
-            out_channels: number of output channels.
-            num_channel_initial: number of initial channels.
-            extract_levels: number of extraction levels.
-            out_activation: activation to use at end layer.
-            out_initializer: initializer for extraction layers.
-        """
-        super(LocalNet, self).__init__()
-        self.extract_levels = extract_levels
-        self.extract_max_level = max(self.extract_levels)  # E
-        self.extract_min_level = min(self.extract_levels)  # D
-
-        num_channels = [
-            num_channel_initial * (2 ** level) for level in range(self.extract_max_level + 1)
-        ]  # level 0 to E
-
-        self.downsample_blocks = nn.ModuleList(
-            [
-                LocalNetDownSampleBlock(
-                    spatial_dims=spatial_dims,
-                    in_channels=in_channels if i == 0 else num_channels[i - 1],
-                    out_channels=num_channels[i],
-                    kernel_size=7 if i == 0 else 3,
-                )
-                for i in range(self.extract_max_level)
-            ]
-        )  # level 0 to self.extract_max_level - 1
-        self.conv3d_block = get_conv_block(
-            spatial_dims=spatial_dims, in_channels=num_channels[-2], out_channels=num_channels[-1]
-        )  # self.extract_max_level
-
-        self.upsample_blocks = nn.ModuleList(
-            [
-                LocalNetUpSampleBlock(
-                    spatial_dims=spatial_dims,
-                    in_channels=num_channels[level + 1],
-                    out_channels=num_channels[level],
-                )
-                for level in range(self.extract_max_level - 1, self.extract_min_level - 1, -1)
-            ]
-        )  # self.extract_max_level - 1 to self.extract_min_level
-
-        self.extract_layers = nn.ModuleList(
-            [
-                # if kernels are not initialized by zeros, with init NN, extract may be too large
-                LocalNetFeatureExtractorBlock(
-                    spatial_dims=spatial_dims,
-                    in_channels=num_channels[level],
-                    out_channels=out_channels,
-                    act=out_activation,
-                    initializer=out_initializer,
-                )
-                for level in self.extract_levels
-            ]
-        )
-
-    def forward(self, x) -> torch.Tensor:
-        image_size = x.shape[2:]
-        for size in image_size:
-            if size % (2 ** self.extract_max_level) != 0:
-                raise ValueError(
-                    f"given extract_max_level {self.extract_max_level}, "
-                    f"all input spatial dimension must be divisible by {2 ** self.extract_max_level}, "
-                    f"got input of size {image_size}"
-                )
-        mid_features = []  # 0 -> self.extract_max_level - 1
-        for downsample_block in self.downsample_blocks:
-            x, mid = downsample_block(x)
-            mid_features.append(mid)
-        x = self.conv3d_block(x)  # self.extract_max_level
-
-        decoded_features = [x]
-        for idx, upsample_block in enumerate(self.upsample_blocks):
-            x = upsample_block(x, mid_features[-idx - 1])
-            decoded_features.append(x)  # self.extract_max_level -> self.extract_min_level
-
-        output = torch.mean(
-            torch.stack(
-                [
-                    F.interpolate(
-                        extract_layer(decoded_features[self.extract_max_level - self.extract_levels[idx]]),
-                        size=image_size,
-                    )
-                    for idx, extract_layer in enumerate(self.extract_layers)
-                ],
-                dim=-1,
-            ),
-            dim=-1,
-        )
-        return output
diff --git a/monai/networks/nets/regunet.py b/monai/networks/nets/regunet.py
new file mode 100644
index 0000000000..57fa801c8b
--- /dev/null
+++ b/monai/networks/nets/regunet.py
@@ -0,0 +1,452 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List, Optional, Tuple, Union
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from monai.networks.blocks.regunet_block import (
+    RegistrationDownSampleBlock,
+    RegistrationExtractionBlock,
+    RegistrationResidualConvBlock,
+    get_conv_block,
+    get_deconv_block,
+)
+
+
+class RegUNet(nn.Module):
+    """
+    Class that implements an adapted UNet. This class also serve as the parent class of LocalNet and GlobalNet
+
+    Reference:
+        O. Ronneberger, P. Fischer, and T. Brox,
+        “U-net: Convolutional networks for biomedical image segmentation,”,
+        Lecture Notes in Computer Science, 2015, vol. 9351, pp. 234–241.
+        https://arxiv.org/abs/1505.04597
+
+    Adapted from:
+        DeepReg (https://github.com/DeepRegNet/DeepReg)
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        num_channel_initial: int,
+        depth: int,
+        out_kernel_initializer: Optional[str] = "kaiming_uniform",
+        out_activation: Optional[str] = None,
+        out_channels: int = 3,
+        extract_levels: Optional[Tuple[int]] = None,
+        pooling: bool = True,
+        concat_skip: bool = False,
+        encode_kernel_sizes: Union[int, List[int]] = 3,
+    ):
+        """
+        Args:
+            spatial_dims: number of spatial dims
+            in_channels: number of input channels
+            num_channel_initial: number of initial channels
+            depth: input is at level 0, bottom is at level depth.
+            out_kernel_initializer: kernel initializer for the last layer
+            out_activation: activation at the last layer
+            out_channels: number of channels for the output
+            extract_levels: list, which levels from net to extract. The maximum level must equal to ``depth``
+            pooling: for down-sampling, use non-parameterized pooling if true, otherwise use conv3d
+            concat_skip: when up-sampling, concatenate skipped tensor if true, otherwise use addition
+            encode_kernel_sizes: kernel size for down-sampling
+        """
+        super(RegUNet, self).__init__()
+        if not extract_levels:
+            extract_levels = (depth,)
+        if max(extract_levels) != depth:
+            raise AssertionError
+
+        # save parameters
+        self.spatial_dims = spatial_dims
+        self.in_channels = in_channels
+        self.num_channel_initial = num_channel_initial
+        self.depth = depth
+        self.out_kernel_initializer = out_kernel_initializer
+        self.out_activation = out_activation
+        self.out_channels = out_channels
+        self.extract_levels = extract_levels
+        self.pooling = pooling
+        self.concat_skip = concat_skip
+
+        if isinstance(encode_kernel_sizes, int):
+            encode_kernel_sizes = [encode_kernel_sizes] * (self.depth + 1)
+        if len(encode_kernel_sizes) != self.depth + 1:
+            raise AssertionError
+        self.encode_kernel_sizes: List[int] = encode_kernel_sizes
+
+        self.num_channels = [self.num_channel_initial * (2 ** d) for d in range(self.depth + 1)]
+        self.min_extract_level = min(self.extract_levels)
+
+        # init layers
+        # all lists start with d = 0
+        self.encode_convs = None
+        self.encode_pools = None
+        self.bottom_block = None
+        self.decode_deconvs = None
+        self.decode_convs = None
+        self.output_block = None
+
+        # build layers
+        self.build_layers()
+
+    def build_layers(
+        self,
+    ):
+        self.build_encode_layers()
+        self.build_decode_layers()
+
+    def build_encode_layers(self):
+        # encoding / down-sampling
+        self.encode_convs = nn.ModuleList(
+            [
+                self.build_conv_block(
+                    in_channels=self.in_channels if d == 0 else self.num_channels[d - 1],
+                    out_channels=self.num_channels[d],
+                    kernel_size=self.encode_kernel_sizes[d],
+                )
+                for d in range(self.depth)
+            ]
+        )
+        self.encode_pools = nn.ModuleList(
+            [
+                self.build_down_sampling_block(
+                    channels=self.num_channels[d],
+                )
+                for d in range(self.depth)
+            ]
+        )
+        self.bottom_block = self.build_bottom_block(
+            in_channels=self.num_channels[-2], out_channels=self.num_channels[-1]
+        )
+
+    def build_conv_block(
+        self,
+        in_channels,
+        out_channels,
+        kernel_size,
+    ):
+        return nn.Sequential(
+            get_conv_block(
+                spatial_dims=self.spatial_dims,
+                in_channels=in_channels,
+                out_channels=out_channels,
+                kernel_size=kernel_size,
+            ),
+            RegistrationResidualConvBlock(
+                spatial_dims=self.spatial_dims,
+                in_channels=out_channels,
+                out_channels=out_channels,
+                kernel_size=kernel_size,
+            ),
+        )
+
+    def build_down_sampling_block(
+        self,
+        channels: int,
+    ):
+        return RegistrationDownSampleBlock(spatial_dims=self.spatial_dims, channels=channels, pooling=self.pooling)
+
+    def build_bottom_block(self, in_channels: int, out_channels: int):
+        kernel_size = self.encode_kernel_sizes[self.depth]
+        return nn.Sequential(
+            get_conv_block(
+                spatial_dims=self.spatial_dims,
+                in_channels=in_channels,
+                out_channels=out_channels,
+                kernel_size=kernel_size,
+            ),
+            RegistrationResidualConvBlock(
+                spatial_dims=self.spatial_dims,
+                in_channels=out_channels,
+                out_channels=out_channels,
+                kernel_size=kernel_size,
+            ),
+        )
+
+    def build_decode_layers(self):
+        # decoding / up-sampling
+        # [depth - 1, depth - 2, ..., min_extract_level]
+        self.decode_deconvs = nn.ModuleList(
+            [
+                self.build_up_sampling_block(in_channels=self.num_channels[d + 1], out_channels=self.num_channels[d])
+                for d in range(self.depth - 1, self.min_extract_level - 1, -1)
+            ]
+        )
+        self.decode_convs = nn.ModuleList(
+            [
+                self.build_conv_block(
+                    in_channels=(2 * self.num_channels[d] if self.concat_skip else self.num_channels[d]),
+                    out_channels=self.num_channels[d],
+                    kernel_size=3,
+                )
+                for d in range(self.depth - 1, self.min_extract_level - 1, -1)
+            ]
+        )
+
+        # extraction
+        self.output_block = self.build_output_block()
+
+    def build_up_sampling_block(
+        self,
+        in_channels: int,
+        out_channels: int,
+    ) -> nn.Module:
+        return get_deconv_block(spatial_dims=self.spatial_dims, in_channels=in_channels, out_channels=out_channels)
+
+    def build_output_block(self) -> nn.Module:
+        return RegistrationExtractionBlock(
+            spatial_dims=self.spatial_dims,
+            extract_levels=self.extract_levels,
+            num_channels=self.num_channels,
+            out_channels=self.out_channels,
+            kernel_initializer=self.out_kernel_initializer,
+            activation=self.out_activation,
+        )
+
+    def forward(self, x):
+        """
+        Args:
+            x: Tensor in shape (batch, ``in_channels``, insize_1, insize_2, [insize_3])
+
+        Returns:
+            Tensor in shape (batch, ``out_channels``, insize_1, insize_2, [insize_3]), with the same spatial size as ``x``
+        """
+        image_size = x.shape[2:]
+        skips = []  # [0, ..., depth - 1]
+        encoded = x
+        for encode_conv, encode_pool in zip(self.encode_convs, self.encode_pools):
+            skip = encode_conv(encoded)
+            encoded = encode_pool(skip)
+            skips.append(skip)
+        decoded = self.bottom_block(encoded)
+
+        outs = [decoded]
+
+        # [depth - 1, ..., min_extract_level]
+        for i, (decode_deconv, decode_conv) in enumerate(zip(self.decode_deconvs, self.decode_convs)):
+            # [depth - 1, depth - 2, ..., min_extract_level]
+            decoded = decode_deconv(decoded)
+            if self.concat_skip:
+                decoded = torch.cat([decoded, skips[-i - 1]], dim=1)
+            else:
+                decoded = decoded + skips[-i - 1]
+            decoded = decode_conv(decoded)
+            outs.append(decoded)
+
+        out = self.output_block(outs, image_size=image_size)
+        return out
+
+
+class AffineHead(nn.Module):
+    def __init__(
+        self,
+        spatial_dims: int,
+        image_size: List[int],
+        decode_size: List[int],
+        in_channels: int,
+    ):
+        super(AffineHead, self).__init__()
+        self.spatial_dims = spatial_dims
+        if spatial_dims == 2:
+            in_features = in_channels * decode_size[0] * decode_size[1]
+            out_features = 6
+            out_init = torch.tensor([1, 0, 0, 0, 1, 0], dtype=torch.float)
+        elif spatial_dims == 3:
+            in_features = in_channels * decode_size[0] * decode_size[1] * decode_size[2]
+            out_features = 12
+            out_init = torch.tensor([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0], dtype=torch.float)
+        else:
+            raise ValueError(f"only support 2D/3D operation, got spatial_dims={spatial_dims}")
+
+        self.fc = nn.Linear(in_features=in_features, out_features=out_features)
+        self.grid = self.get_reference_grid(image_size)  # (spatial_dims, ...)
+
+        # init weight/bias
+        self.fc.weight.data.zero_()
+        self.fc.bias.data.copy_(out_init)
+
+    @staticmethod
+    def get_reference_grid(image_size: Union[Tuple[int], List[int]]) -> torch.Tensor:
+        mesh_points = [torch.arange(0, dim) for dim in image_size]
+        grid = torch.stack(torch.meshgrid(*mesh_points), dim=0)  # (spatial_dims, ...)
+        return grid.to(dtype=torch.float)
+
+    def affine_transform(self, theta: torch.Tensor):
+        # (spatial_dims, ...) -> (spatial_dims + 1, ...)
+        grid_padded = torch.cat([self.grid, torch.ones_like(self.grid[:1])])
+
+        # grid_warped[b,p,...] = sum_over_q(grid_padded[q,...] * theta[b,p,q]
+        if self.spatial_dims == 2:
+            grid_warped = torch.einsum("qij,bpq->bpij", grid_padded, theta.reshape(-1, 2, 3))
+        elif self.spatial_dims == 3:
+            grid_warped = torch.einsum("qijk,bpq->bpijk", grid_padded, theta.reshape(-1, 3, 4))
+        else:
+            raise ValueError(f"do not support spatial_dims={self.spatial_dims}")
+        return grid_warped
+
+    def forward(self, x: List[torch.Tensor], image_size: List[int]) -> torch.Tensor:
+        f = x[0]
+        self.grid = self.grid.to(device=f.device)
+        theta = self.fc(f.reshape(f.shape[0], -1))
+        out: torch.Tensor = self.affine_transform(theta) - self.grid
+        return out
+
+
+class GlobalNet(RegUNet):
+    """
+    Build GlobalNet for image registration.
+
+    Reference:
+        Hu, Yipeng, et al.
+        "Label-driven weakly-supervised learning
+        for multimodal deformable image registration,"
+        https://arxiv.org/abs/1711.01666
+    """
+
+    def __init__(
+        self,
+        image_size: List[int],
+        spatial_dims: int,
+        in_channels: int,
+        num_channel_initial: int,
+        depth: int,
+        out_kernel_initializer: Optional[str] = "kaiming_uniform",
+        out_activation: Optional[str] = None,
+        pooling: bool = True,
+        concat_skip: bool = False,
+        encode_kernel_sizes: Union[int, List[int]] = 3,
+    ):
+        for size in image_size:
+            if size % (2 ** depth) != 0:
+                raise ValueError(
+                    f"given depth {depth}, "
+                    f"all input spatial dimension must be divisible by {2 ** depth}, "
+                    f"got input of size {image_size}"
+                )
+        self.image_size = image_size
+        self.decode_size = [size // (2 ** depth) for size in image_size]
+        super().__init__(
+            spatial_dims=spatial_dims,
+            in_channels=in_channels,
+            num_channel_initial=num_channel_initial,
+            depth=depth,
+            out_kernel_initializer=out_kernel_initializer,
+            out_activation=out_activation,
+            out_channels=spatial_dims,
+            pooling=pooling,
+            concat_skip=concat_skip,
+            encode_kernel_sizes=encode_kernel_sizes,
+        )
+
+    def build_output_block(self):
+        return AffineHead(
+            spatial_dims=self.spatial_dims,
+            image_size=self.image_size,
+            decode_size=self.decode_size,
+            in_channels=self.num_channels[-1],
+        )
+
+
+class AdditiveUpSampleBlock(nn.Module):
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        out_channels: int,
+    ):
+        super(AdditiveUpSampleBlock, self).__init__()
+        self.deconv = get_deconv_block(spatial_dims=spatial_dims, in_channels=in_channels, out_channels=out_channels)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        output_size = (size * 2 for size in x.shape[2:])
+        deconved = self.deconv(x)
+        resized = F.interpolate(x, output_size)
+        resized = torch.sum(torch.stack(resized.split(split_size=resized.shape[1] // 2, dim=1), dim=-1), dim=-1)
+        out: torch.Tensor = deconved + resized
+        return out
+
+
+class LocalNet(RegUNet):
+    """
+    Reimplementation of LocalNet, based on:
+    `Weakly-supervised convolutional neural networks for multimodal image registration
+    <https://doi.org/10.1016/j.media.2018.07.002>`_.
+    `Label-driven weakly-supervised learning for multimodal deformable image registration
+    <https://arxiv.org/abs/1711.01666>`_.
+
+    Adapted from:
+        DeepReg (https://github.com/DeepRegNet/DeepReg)
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int,
+        in_channels: int,
+        num_channel_initial: int,
+        extract_levels: Tuple[int],
+        out_kernel_initializer: Optional[str] = "kaiming_uniform",
+        out_activation: Optional[str] = None,
+        out_channels: int = 3,
+        pooling: bool = True,
+        concat_skip: bool = False,
+    ):
+        """
+        Args:
+            spatial_dims: number of spatial dims
+            in_channels: number of input channels
+            num_channel_initial: number of initial channels
+            out_kernel_initializer: kernel initializer for the last layer
+            out_activation: activation at the last layer
+            out_channels: number of channels for the output
+            extract_levels: list, which levels from net to extract. The maximum level must equal to ``depth``
+            pooling: for down-sampling, use non-parameterized pooling if true, otherwise use conv3d
+            concat_skip: when up-sampling, concatenate skipped tensor if true, otherwise use addition
+        """
+        super().__init__(
+            spatial_dims=spatial_dims,
+            in_channels=in_channels,
+            num_channel_initial=num_channel_initial,
+            depth=max(extract_levels),
+            out_kernel_initializer=out_kernel_initializer,
+            out_activation=out_activation,
+            out_channels=out_channels,
+            pooling=pooling,
+            concat_skip=concat_skip,
+            encode_kernel_sizes=[7] + [3] * max(extract_levels),
+        )
+
+    def build_bottom_block(self, in_channels: int, out_channels: int):
+        kernel_size = self.encode_kernel_sizes[self.depth]
+        return get_conv_block(
+            spatial_dims=self.spatial_dims,
+            in_channels=in_channels,
+            out_channels=out_channels,
+            kernel_size=kernel_size,
+        )
+
+    def build_up_sampling_block(
+        self,
+        in_channels: int,
+        out_channels: int,
+    ) -> nn.Module:
+        if self._use_additive_upsampling:
+            return AdditiveUpSampleBlock(
+                spatial_dims=self.spatial_dims, in_channels=in_channels, out_channels=out_channels
+            )
+
+        return get_deconv_block(spatial_dims=self.spatial_dims, in_channels=in_channels, out_channels=out_channels)
diff --git a/monai/networks/nets/senet.py b/monai/networks/nets/senet.py
index 655ff203c7..1e04e02973 100644
--- a/monai/networks/nets/senet.py
+++ b/monai/networks/nets/senet.py
@@ -11,7 +11,7 @@
 
 import re
 from collections import OrderedDict
-from typing import Any, List, Optional, Tuple, Type, Union
+from typing import Any, List, Optional, Sequence, Tuple, Type, Union
 
 import torch
 import torch.nn as nn
@@ -66,7 +66,6 @@ class SENet(nn.Module):
             - For SE-ResNeXt models: False
         num_classes: number of outputs in `last_linear` layer.
             for all models: 1000
-
     """
 
     def __init__(
@@ -74,7 +73,7 @@ def __init__(
         spatial_dims: int,
         in_channels: int,
         block: Type[Union[SEBottleneck, SEResNetBottleneck, SEResNeXtBottleneck]],
-        layers: List[int],
+        layers: Sequence[int],
         groups: int,
         reduction: int,
         dropout_prob: Optional[float] = 0.2,
@@ -248,20 +247,26 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
         return x
 
 
-model_urls = {
-    "senet154": "http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth",
-    "se_resnet50": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth",
-    "se_resnet101": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth",
-    "se_resnet152": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth",
-    "se_resnext50_32x4d": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth",
-    "se_resnext101_32x4d": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth",
-}
-
-
-def _load_state_dict(model, model_url, progress):
+def _load_state_dict(model, arch, progress):
     """
     This function is used to load pretrained models.
     """
+    model_urls = {
+        "senet154": "http://data.lip6.fr/cadene/pretrainedmodels/senet154-c7b49a05.pth",
+        "se_resnet50": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnet50-ce0d4300.pth",
+        "se_resnet101": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnet101-7e38fcc6.pth",
+        "se_resnet152": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnet152-d17c99b7.pth",
+        "se_resnext50_32x4d": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnext50_32x4d-a260b3a4.pth",
+        "se_resnext101_32x4d": "http://data.lip6.fr/cadene/pretrainedmodels/se_resnext101_32x4d-3b2fe3d8.pth",
+    }
+    if arch in model_urls:
+        model_url = model_urls[arch]
+    else:
+        raise ValueError(
+            "only 'senet154', 'se_resnet50', 'se_resnet101',  'se_resnet152', 'se_resnext50_32x4d', "
+            + "and se_resnext101_32x4d are supported to load pretrained weights."
+        )
+
     pattern_conv = re.compile(r"^(layer[1-4]\.\d\.(?:conv)\d\.)(\w*)$")
     pattern_bn = re.compile(r"^(layer[1-4]\.\d\.)(?:bn)(\d\.)(\w*)$")
     pattern_se = re.compile(r"^(layer[1-4]\.\d\.)(?:se_module.fc1.)(\w*)$")
@@ -275,7 +280,7 @@ def _load_state_dict(model, model_url, progress):
         if pattern_conv.match(key):
             new_key = re.sub(pattern_conv, r"\1conv.\2", key)
         elif pattern_bn.match(key):
-            new_key = re.sub(pattern_bn, r"\1conv\2norm.\3", key)
+            new_key = re.sub(pattern_bn, r"\1conv\2adn.N.\3", key)
         elif pattern_se.match(key):
             state_dict[key] = state_dict[key].squeeze()
             new_key = re.sub(pattern_se, r"\1se_layer.fc.0.\2", key)
@@ -285,7 +290,7 @@ def _load_state_dict(model, model_url, progress):
         elif pattern_down_conv.match(key):
             new_key = re.sub(pattern_down_conv, r"\1project.conv.\2", key)
         elif pattern_down_bn.match(key):
-            new_key = re.sub(pattern_down_bn, r"\1project.norm.\2", key)
+            new_key = re.sub(pattern_down_bn, r"\1project.adn.N.\2", key)
         if new_key:
             state_dict[new_key] = state_dict[key]
             del state_dict[key]
@@ -298,167 +303,169 @@ def _load_state_dict(model, model_url, progress):
     model.load_state_dict(model_dict)
 
 
-def senet154(
-    spatial_dims: int,
-    in_channels: int,
-    num_classes: int,
-    pretrained: bool = False,
-    progress: bool = True,
-) -> SENet:
-    """
-    when `spatial_dims = 2`, specify `pretrained = True` can load Imagenet pretrained weights achieved
-    from `Cadene Hub 2D version
-    <https://github.com/Cadene/pretrained-models.pytorch/blob/master/pretrainedmodels/models/senet.py>`_.
-    """
-    model = SENet(
-        spatial_dims=spatial_dims,
-        in_channels=in_channels,
-        block=SEBottleneck,
-        layers=[3, 8, 36, 3],
-        groups=64,
-        reduction=16,
-        dropout_prob=0.2,
-        dropout_dim=1,
-        num_classes=num_classes,
-    )
-    if pretrained:
-        arch = "senet154"
-        _load_state_dict(model, model_urls[arch], progress)
-    return model
-
-
-def se_resnet50(
-    spatial_dims: int, in_channels: int, num_classes: int, pretrained: bool = False, progress: bool = True
-) -> SENet:
-    """
-    when `spatial_dims = 2`, specify `pretrained = True` can load Imagenet pretrained weights achieved
-    from `Cadene Hub 2D version
-    <https://github.com/Cadene/pretrained-models.pytorch/blob/master/pretrainedmodels/models/senet.py>`_.
-    """
-    model = SENet(
-        spatial_dims=spatial_dims,
-        in_channels=in_channels,
-        block=SEResNetBottleneck,
-        layers=[3, 4, 6, 3],
-        groups=1,
-        reduction=16,
-        dropout_prob=None,
-        inplanes=64,
-        input_3x3=False,
-        downsample_kernel_size=1,
-        num_classes=num_classes,
-    )
-    if pretrained:
-        arch = "se_resnet50"
-        _load_state_dict(model, model_urls[arch], progress)
-    return model
-
-
-def se_resnet101(
-    spatial_dims: int, in_channels: int, num_classes: int, pretrained: bool = False, progress: bool = True
-) -> SENet:
-    """
-    when `spatial_dims = 2`, specify `pretrained = True` can load Imagenet pretrained weights achieved
-    from `Cadene Hub 2D version
-    <https://github.com/Cadene/pretrained-models.pytorch/blob/master/pretrainedmodels/models/senet.py>`_.
-    """
-    model = SENet(
-        spatial_dims=spatial_dims,
-        in_channels=in_channels,
-        block=SEResNetBottleneck,
-        layers=[3, 4, 23, 3],
-        groups=1,
-        reduction=16,
-        dropout_prob=0.2,
-        dropout_dim=1,
-        inplanes=64,
-        input_3x3=False,
-        downsample_kernel_size=1,
-        num_classes=num_classes,
-    )
-    if pretrained:
-        arch = "se_resnet101"
-        _load_state_dict(model, model_urls[arch], progress)
-    return model
-
-
-def se_resnet152(
-    spatial_dims: int, in_channels: int, num_classes: int, pretrained: bool = False, progress: bool = True
-) -> SENet:
-    """
-    when `spatial_dims = 2`, specify `pretrained = True` can load Imagenet pretrained weights achieved
-    from `Cadene Hub 2D version
-    <https://github.com/Cadene/pretrained-models.pytorch/blob/master/pretrainedmodels/models/senet.py>`_.
-    """
-    model = SENet(
-        spatial_dims=spatial_dims,
-        in_channels=in_channels,
-        block=SEResNetBottleneck,
-        layers=[3, 8, 36, 3],
-        groups=1,
-        reduction=16,
-        dropout_prob=0.2,
-        dropout_dim=1,
-        inplanes=64,
-        input_3x3=False,
-        downsample_kernel_size=1,
-        num_classes=num_classes,
-    )
-    if pretrained:
-        arch = "se_resnet152"
-        _load_state_dict(model, model_urls[arch], progress)
-    return model
-
-
-def se_resnext50_32x4d(
-    spatial_dims: int, in_channels: int, num_classes: int, pretrained: bool = False, progress: bool = True
-) -> SENet:
-    """
-    when `spatial_dims = 2`, specify `pretrained = True` can load Imagenet pretrained weights achieved
-    from `Cadene Hub 2D version
-    <https://github.com/Cadene/pretrained-models.pytorch/blob/master/pretrainedmodels/models/senet.py>`_.
-    """
-    model = SENet(
-        spatial_dims=spatial_dims,
-        in_channels=in_channels,
-        block=SEResNeXtBottleneck,
-        layers=[3, 4, 6, 3],
-        groups=32,
-        reduction=16,
-        dropout_prob=None,
-        inplanes=64,
-        input_3x3=False,
-        downsample_kernel_size=1,
-        num_classes=num_classes,
-    )
-    if pretrained:
-        arch = "se_resnext50_32x4d"
-        _load_state_dict(model, model_urls[arch], progress)
-    return model
-
-
-def se_resnext101_32x4d(
-    spatial_dims: int, in_channels: int, num_classes: int, pretrained: bool = False, progress: bool = True
-) -> SENet:
-    """
-    when `spatial_dims = 2`, specify `pretrained = True` can load Imagenet pretrained weights achieved
-    from `Cadene Hub 2D version
-    <https://github.com/Cadene/pretrained-models.pytorch/blob/master/pretrainedmodels/models/senet.py>`_.
-    """
-    model = SENet(
-        spatial_dims=spatial_dims,
-        in_channels=in_channels,
-        block=SEResNeXtBottleneck,
-        layers=[3, 4, 23, 3],
-        groups=32,
-        reduction=16,
-        dropout_prob=None,
-        inplanes=64,
-        input_3x3=False,
-        downsample_kernel_size=1,
-        num_classes=num_classes,
-    )
-    if pretrained:
-        arch = "se_resnext101_32x4d"
-        _load_state_dict(model, model_urls[arch], progress)
-    return model
+class SENet154(SENet):
+    def __init__(
+        self,
+        layers: Sequence[int] = (3, 8, 36, 3),
+        groups: int = 64,
+        reduction: int = 16,
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(SENet154, self).__init__(
+            block=SEBottleneck,
+            layers=layers,
+            groups=groups,
+            reduction=reduction,
+            **kwargs,
+        )
+        if pretrained:
+            # it only worked when `spatial_dims` is 2
+            _load_state_dict(self, "senet154", progress)
+
+
+class SEResNet50(SENet):
+    def __init__(
+        self,
+        layers: Sequence[int] = (3, 4, 6, 3),
+        groups: int = 1,
+        reduction: int = 16,
+        dropout_prob: Optional[float] = None,
+        inplanes: int = 64,
+        downsample_kernel_size: int = 1,
+        input_3x3: bool = False,
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(SEResNet50, self).__init__(
+            block=SEResNetBottleneck,
+            layers=layers,
+            groups=groups,
+            reduction=reduction,
+            dropout_prob=dropout_prob,
+            inplanes=inplanes,
+            downsample_kernel_size=downsample_kernel_size,
+            input_3x3=input_3x3,
+            **kwargs,
+        )
+        if pretrained:
+            # it only worked when `spatial_dims` is 2
+            _load_state_dict(self, "se_resnet50", progress)
+
+
+class SEResNet101(SENet):
+    def __init__(
+        self,
+        layers: Sequence[int] = (3, 4, 23, 3),
+        groups: int = 1,
+        reduction: int = 16,
+        inplanes: int = 64,
+        downsample_kernel_size: int = 1,
+        input_3x3: bool = False,
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(SEResNet101, self).__init__(
+            block=SEResNetBottleneck,
+            layers=layers,
+            groups=groups,
+            reduction=reduction,
+            inplanes=inplanes,
+            downsample_kernel_size=downsample_kernel_size,
+            input_3x3=input_3x3,
+            **kwargs,
+        )
+        if pretrained:
+            # it only worked when `spatial_dims` is 2
+            _load_state_dict(self, "se_resnet101", progress)
+
+
+class SEResNet152(SENet):
+    def __init__(
+        self,
+        layers: Sequence[int] = (3, 8, 36, 3),
+        groups: int = 1,
+        reduction: int = 16,
+        inplanes: int = 64,
+        downsample_kernel_size: int = 1,
+        input_3x3: bool = False,
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(SEResNet152, self).__init__(
+            block=SEResNetBottleneck,
+            layers=layers,
+            groups=groups,
+            reduction=reduction,
+            inplanes=inplanes,
+            downsample_kernel_size=downsample_kernel_size,
+            input_3x3=input_3x3,
+            **kwargs,
+        )
+        if pretrained:
+            # it only worked when `spatial_dims` is 2
+            _load_state_dict(self, "se_resnet152", progress)
+
+
+class SEResNext50(SENet):
+    def __init__(
+        self,
+        layers: Sequence[int] = (3, 4, 6, 3),
+        groups: int = 32,
+        reduction: int = 16,
+        dropout_prob: Optional[float] = None,
+        inplanes: int = 64,
+        downsample_kernel_size: int = 1,
+        input_3x3: bool = False,
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(SEResNext50, self).__init__(
+            block=SEResNeXtBottleneck,
+            layers=layers,
+            groups=groups,
+            dropout_prob=dropout_prob,
+            reduction=reduction,
+            inplanes=inplanes,
+            downsample_kernel_size=downsample_kernel_size,
+            input_3x3=input_3x3,
+            **kwargs,
+        )
+        if pretrained:
+            # it only worked when `spatial_dims` is 2
+            _load_state_dict(self, "se_resnext50_32x4d", progress)
+
+
+class SEResNext101(SENet):
+    def __init__(
+        self,
+        layers: Sequence[int] = (3, 4, 23, 3),
+        groups: int = 32,
+        reduction: int = 16,
+        dropout_prob: Optional[float] = None,
+        inplanes: int = 64,
+        downsample_kernel_size: int = 1,
+        input_3x3: bool = False,
+        pretrained: bool = False,
+        progress: bool = True,
+        **kwargs,
+    ) -> None:
+        super(SEResNext101, self).__init__(
+            block=SEResNeXtBottleneck,
+            layers=layers,
+            groups=groups,
+            dropout_prob=dropout_prob,
+            reduction=reduction,
+            inplanes=inplanes,
+            downsample_kernel_size=downsample_kernel_size,
+            input_3x3=input_3x3,
+            **kwargs,
+        )
+        if pretrained:
+            # it only worked when `spatial_dims` is 2
+            _load_state_dict(self, "se_resnext101_32x4d", progress)
diff --git a/monai/networks/nets/torchvision_fc.py b/monai/networks/nets/torchvision_fc.py
new file mode 100644
index 0000000000..8b8a223b55
--- /dev/null
+++ b/monai/networks/nets/torchvision_fc.py
@@ -0,0 +1,71 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Tuple, Union
+
+import torch
+
+from monai.utils import optional_import
+
+models, _ = optional_import("torchvision.models")
+
+
+class TorchVisionFullyConvModel(torch.nn.Module):
+    """
+    Customize TorchVision models to replace fully connected layer by convolutional layer.
+
+    Args:
+        model_name: name of any torchvision with adaptive avg pooling and fully connected layer at the end.
+            ``resnet18`` (default), ``resnet34m``, ``resnet50``, ``resnet101``, ``resnet152``,
+            ``resnext50_32x4d``, ``resnext101_32x8d``, ``wide_resnet50_2``, ``wide_resnet101_2``.
+        n_classes: number of classes for the last classification layer. Default to 1.
+        pool_size: the kernel size for `AvgPool2d` to replace `AdaptiveAvgPool2d`. Default to (7, 7).
+        pool_stride: the stride for `AvgPool2d` to replace `AdaptiveAvgPool2d`. Default to 1.
+        pretrained: whether to use the imagenet pretrained weights. Default to False.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "resnet18",
+        n_classes: int = 1,
+        pool_size: Union[int, Tuple[int, int]] = (7, 7),
+        pool_stride: Union[int, Tuple[int, int]] = 1,
+        pretrained: bool = False,
+    ):
+        super().__init__()
+        model = getattr(models, model_name)(pretrained=pretrained)
+        layers = list(model.children())
+
+        # check if the model is compatible
+        if not str(layers[-1]).startswith("Linear"):
+            raise ValueError(f"Model ['{model_name}'] does not have a Linear layer at the end.")
+        if not str(layers[-2]).startswith("AdaptiveAvgPool2d"):
+            raise ValueError(f"Model ['{model_name}'] does not have a AdaptiveAvgPool2d layer next to the end.")
+
+        # remove the last Linear layer (fully connected) and the adaptive avg pooling
+        self.features = torch.nn.Sequential(*layers[:-2])
+
+        # add 7x7 avg pooling (in place of adaptive avg pooling)
+        self.pool = torch.nn.AvgPool2d(kernel_size=pool_size, stride=pool_stride)
+
+        # add 1x1 conv (it behaves like a FC layer)
+        self.fc = torch.nn.Conv2d(model.fc.in_features, n_classes, kernel_size=(1, 1))
+
+    def forward(self, x):
+        x = self.features(x)
+
+        # apply 2D avg pooling
+        x = self.pool(x)
+
+        # apply last 1x1 conv layer that act like a linear layer
+        x = self.fc(x)
+
+        return x
diff --git a/monai/networks/utils.py b/monai/networks/utils.py
index 847bfc97c2..c5989f174b 100644
--- a/monai/networks/utils.py
+++ b/monai/networks/utils.py
@@ -14,13 +14,11 @@
 
 import warnings
 from contextlib import contextmanager
-from typing import Any, Callable, Optional, Sequence, cast
+from typing import Any, Callable, Optional, Sequence
 
 import torch
 import torch.nn as nn
 
-from monai.utils import ensure_tuple_size
-
 __all__ = [
     "one_hot",
     "slice_channels",
@@ -50,13 +48,14 @@ def one_hot(labels: torch.Tensor, num_classes: int, dtype: torch.dtype = torch.f
 
     # if `dim` is bigger, add singleton dim at the end
     if labels.ndim < dim + 1:
-        shape = ensure_tuple_size(labels.shape, dim + 1, 1)
-        labels = labels.reshape(*shape)
+        shape = list(labels.shape) + [1] * (dim + 1 - len(labels.shape))
+        labels = torch.reshape(labels, shape)
 
     sh = list(labels.shape)
 
     if sh[dim] != 1:
-        raise AssertionError("labels should have a channel with length equals to one.")
+        raise AssertionError("labels should have a channel with length equal to one.")
+
     sh[dim] = num_classes
 
     o = torch.zeros(size=sh, dtype=dtype, device=labels.device)
@@ -72,9 +71,7 @@ def slice_channels(tensor: torch.Tensor, *slicevals: Optional[int]) -> torch.Ten
     return tensor[slices]
 
 
-def predict_segmentation(
-    logits: torch.Tensor, mutually_exclusive: bool = False, threshold: float = 0.0
-) -> torch.Tensor:
+def predict_segmentation(logits: torch.Tensor, mutually_exclusive: bool = False, threshold: float = 0.0) -> Any:
     """
     Given the logits from a network, computing the segmentation by thresholding all values above 0
     if multi-labels task, computing the `argmax` along the channel axis if multi-classes task,
@@ -87,10 +84,10 @@ def predict_segmentation(
         threshold: thresholding the prediction values if multi-labels task.
     """
     if not mutually_exclusive:
-        return (cast(torch.Tensor, logits >= threshold)).int()
+        return (logits >= threshold).int()
     if logits.shape[1] == 1:
         warnings.warn("single channel prediction, `mutually_exclusive=True` ignored, use threshold instead.")
-        return (cast(torch.Tensor, logits >= threshold)).int()
+        return (logits >= threshold).int()
     return logits.argmax(1, keepdim=True)
 
 
diff --git a/monai/optimizers/lr_finder.py b/monai/optimizers/lr_finder.py
index 9e753a1ced..49d4427b3d 100644
--- a/monai/optimizers/lr_finder.py
+++ b/monai/optimizers/lr_finder.py
@@ -1,3 +1,14 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import warnings
 from functools import partial
 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
diff --git a/monai/optimizers/lr_scheduler.py b/monai/optimizers/lr_scheduler.py
index aa9bf2a89b..c4488f6e07 100644
--- a/monai/optimizers/lr_scheduler.py
+++ b/monai/optimizers/lr_scheduler.py
@@ -1,3 +1,14 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from torch.optim import Optimizer
 from torch.optim.lr_scheduler import _LRScheduler
 
diff --git a/monai/transforms/__init__.py b/monai/transforms/__init__.py
index 6f7c2a4f61..f96194c262 100644
--- a/monai/transforms/__init__.py
+++ b/monai/transforms/__init__.py
@@ -10,7 +10,7 @@
 # limitations under the License.
 
 from .adaptors import FunctionSignature, adaptor, apply_alias, to_kwargs
-from .compose import Compose, MapTransform, Randomizable, Transform
+from .compose import Compose
 from .croppad.array import (
     BorderPad,
     BoundingRect,
@@ -25,6 +25,7 @@
     SpatialCrop,
     SpatialPad,
 )
+from .croppad.batch import PadListDataCollate
 from .croppad.dictionary import (
     BorderPadd,
     BorderPadD,
@@ -72,17 +73,20 @@
     MaskIntensity,
     NormalizeIntensity,
     RandAdjustContrast,
+    RandBiasField,
     RandGaussianNoise,
     RandGaussianSharpen,
     RandGaussianSmooth,
     RandHistogramShift,
     RandScaleIntensity,
     RandShiftIntensity,
+    RandStdShiftIntensity,
     SavitzkyGolaySmooth,
     ScaleIntensity,
     ScaleIntensityRange,
     ScaleIntensityRangePercentiles,
     ShiftIntensity,
+    StdShiftIntensity,
     ThresholdIntensity,
 )
 from .intensity.dictionary import (
@@ -104,6 +108,9 @@
     RandAdjustContrastd,
     RandAdjustContrastD,
     RandAdjustContrastDict,
+    RandBiasFieldd,
+    RandBiasFieldD,
+    RandBiasFieldDict,
     RandGaussianNoised,
     RandGaussianNoiseD,
     RandGaussianNoiseDict,
@@ -122,6 +129,9 @@
     RandShiftIntensityd,
     RandShiftIntensityD,
     RandShiftIntensityDict,
+    RandStdShiftIntensityd,
+    RandStdShiftIntensityD,
+    RandStdShiftIntensityDict,
     ScaleIntensityd,
     ScaleIntensityD,
     ScaleIntensityDict,
@@ -134,10 +144,14 @@
     ShiftIntensityd,
     ShiftIntensityD,
     ShiftIntensityDict,
+    StdShiftIntensityd,
+    StdShiftIntensityD,
+    StdShiftIntensityDict,
     ThresholdIntensityd,
     ThresholdIntensityD,
     ThresholdIntensityDict,
 )
+from .inverse import InvertibleTransform
 from .io.array import LoadImage, SaveImage
 from .io.dictionary import LoadImaged, LoadImageD, LoadImageDict, SaveImaged, SaveImageD, SaveImageDict
 from .post.array import (
@@ -146,6 +160,7 @@
     KeepLargestConnectedComponent,
     LabelToContour,
     MeanEnsemble,
+    ProbNMS,
     VoteEnsemble,
 )
 from .post.dictionary import (
@@ -155,6 +170,9 @@
     AsDiscreted,
     AsDiscreteD,
     AsDiscreteDict,
+    Decollated,
+    DecollateD,
+    DecollateDict,
     Ensembled,
     KeepLargestConnectedComponentd,
     KeepLargestConnectedComponentD,
@@ -165,6 +183,9 @@
     MeanEnsembled,
     MeanEnsembleD,
     MeanEnsembleDict,
+    ProbNMSd,
+    ProbNMSD,
+    ProbNMSDict,
     VoteEnsembled,
     VoteEnsembleD,
     VoteEnsembleDict,
@@ -178,6 +199,7 @@
     Rand3DElastic,
     RandAffine,
     RandAffineGrid,
+    RandAxisFlip,
     RandDeformGrid,
     RandFlip,
     RandRotate,
@@ -191,6 +213,9 @@
     Zoom,
 )
 from .spatial.dictionary import (
+    Affined,
+    AffineD,
+    AffineDict,
     Flipd,
     FlipD,
     FlipDict,
@@ -206,6 +231,9 @@
     RandAffined,
     RandAffineD,
     RandAffineDict,
+    RandAxisFlipd,
+    RandAxisFlipD,
+    RandAxisFlipDict,
     RandFlipd,
     RandFlipD,
     RandFlipDict,
@@ -234,6 +262,7 @@
     ZoomD,
     ZoomDict,
 )
+from .transform import MapTransform, Randomizable, RandomizableTransform, Transform, apply_transform
 from .utility.array import (
     AddChannel,
     AddExtremePointsChannel,
@@ -242,15 +271,19 @@
     CastToType,
     ConvertToMultiChannelBasedOnBratsClasses,
     DataStats,
+    EnsureChannelFirst,
     FgBgToIndices,
     Identity,
     LabelToMask,
     Lambda,
+    MapLabelValue,
+    RemoveRepeatedChannel,
     RepeatChannel,
     SimulateDelay,
     SplitChannel,
     SqueezeDim,
     ToNumpy,
+    ToPIL,
     TorchVision,
     ToTensor,
     Transpose,
@@ -286,6 +319,9 @@
     DeleteItemsd,
     DeleteItemsD,
     DeleteItemsDict,
+    EnsureChannelFirstd,
+    EnsureChannelFirstD,
+    EnsureChannelFirstDict,
     FgBgToIndicesd,
     FgBgToIndicesD,
     FgBgToIndicesDict,
@@ -298,9 +334,15 @@
     Lambdad,
     LambdaD,
     LambdaDict,
+    MapLabelValued,
+    MapLabelValueD,
+    MapLabelValueDict,
     RandLambdad,
     RandLambdaD,
     RandLambdaDict,
+    RemoveRepeatedChanneld,
+    RemoveRepeatedChannelD,
+    RemoveRepeatedChannelDict,
     RepeatChanneld,
     RepeatChannelD,
     RepeatChannelDict,
@@ -315,13 +357,21 @@
     SqueezeDimD,
     SqueezeDimDict,
     ToNumpyd,
+    ToNumpyD,
+    ToNumpyDict,
+    ToPILd,
+    ToPILD,
+    ToPILDict,
     TorchVisiond,
+    TorchVisionD,
+    TorchVisionDict,
     ToTensord,
     ToTensorD,
     ToTensorDict,
 )
 from .utils import (
-    apply_transform,
+    allow_missing_keys_mode,
+    convert_inverse_interp_mode,
     copypaste_arrays,
     create_control_grid,
     create_grid,
diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py
index 2d1fe4eccd..ce965b8b18 100644
--- a/monai/transforms/compose.py
+++ b/monai/transforms/compose.py
@@ -13,138 +13,26 @@
 """
 
 import warnings
-from abc import ABC, abstractmethod
-from typing import Any, Callable, Hashable, Optional, Sequence, Tuple, Union
+from typing import Any, Callable, Optional, Sequence, Union
 
 import numpy as np
 
-from monai.config import KeysCollection
-from monai.transforms.utils import apply_transform
-from monai.utils import MAX_SEED, ensure_tuple, get_seed
-
-__all__ = ["Transform", "Randomizable", "Compose", "MapTransform"]
-
-
-class Transform(ABC):
-    """
-    An abstract class of a ``Transform``.
-    A transform is callable that processes ``data``.
-
-    It could be stateful and may modify ``data`` in place,
-    the implementation should be aware of:
-
-        #. thread safety when mutating its own states.
-           When used from a multi-process context, transform's instance variables are read-only.
-        #. ``data`` content unused by this transform may still be used in the
-           subsequent transforms in a composed transform.
-        #. storing too much information in ``data`` may not scale.
-
-    See Also
-
-        :py:class:`monai.transforms.Compose`
-    """
-
-    @abstractmethod
-    def __call__(self, data: Any):
-        """
-        ``data`` is an element which often comes from an iteration over an
-        iterable, such as :py:class:`torch.utils.data.Dataset`. This method should
-        return an updated version of ``data``.
-        To simplify the input validations, most of the transforms assume that
-
-        - ``data`` is a Numpy ndarray, PyTorch Tensor or string
-        - the data shape can be:
-
-          #. string data without shape, `LoadImage` transform expects file paths
-          #. most of the pre-processing transforms expect: ``(num_channels, spatial_dim_1[, spatial_dim_2, ...])``,
-             except that `AddChannel` expects (spatial_dim_1[, spatial_dim_2, ...]) and
-             `AsChannelFirst` expects (spatial_dim_1[, spatial_dim_2, ...], num_channels)
-          #. most of the post-processing transforms expect
-             ``(batch_size, num_channels, spatial_dim_1[, spatial_dim_2, ...])``
-
-        - the channel dimension is not omitted even if number of channels is one
-
-        This method can optionally take additional arguments to help execute transformation operation.
-
-        Raises:
-            NotImplementedError: When the subclass does not override this method.
-
-        """
-        raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.")
-
-
-class Randomizable(ABC):
-    """
-    An interface for handling random state locally, currently based on a class variable `R`,
-    which is an instance of `np.random.RandomState`.
-    This is mainly for randomized data augmentation transforms. For example::
-
-        class RandShiftIntensity(Randomizable):
-            def randomize():
-                self._offset = self.R.uniform(low=0, high=100)
-            def __call__(self, img):
-                self.randomize()
-                return img + self._offset
-
-        transform = RandShiftIntensity()
-        transform.set_random_state(seed=0)
-
-    """
-
-    R: np.random.RandomState = np.random.RandomState()
-
-    def set_random_state(
-        self, seed: Optional[int] = None, state: Optional[np.random.RandomState] = None
-    ) -> "Randomizable":
-        """
-        Set the random state locally, to control the randomness, the derived
-        classes should use :py:attr:`self.R` instead of `np.random` to introduce random
-        factors.
-
-        Args:
-            seed: set the random state with an integer seed.
-            state: set the random state with a `np.random.RandomState` object.
-
-        Raises:
-            TypeError: When ``state`` is not an ``Optional[np.random.RandomState]``.
+from monai.transforms.inverse import InvertibleTransform
 
-        Returns:
-            a Randomizable instance.
-
-        """
-        if seed is not None:
-            _seed = id(seed) if not isinstance(seed, (int, np.integer)) else seed
-            _seed = _seed % MAX_SEED
-            self.R = np.random.RandomState(_seed)
-            return self
-
-        if state is not None:
-            if not isinstance(state, np.random.RandomState):
-                raise TypeError(f"state must be None or a np.random.RandomState but is {type(state).__name__}.")
-            self.R = state
-            return self
-
-        self.R = np.random.RandomState()
-        return self
-
-    @abstractmethod
-    def randomize(self, data: Any) -> None:
-        """
-        Within this method, :py:attr:`self.R` should be used, instead of `np.random`, to introduce random factors.
-
-        all :py:attr:`self.R` calls happen here so that we have a better chance to
-        identify errors of sync the random state.
-
-        This method can generate the random factors based on properties of the input data.
-
-        Raises:
-            NotImplementedError: When the subclass does not override this method.
+# For backwards compatibility (so this still works: from monai.transforms.compose import MapTransform)
+from monai.transforms.transform import (  # noqa: F401
+    MapTransform,
+    Randomizable,
+    RandomizableTransform,
+    Transform,
+    apply_transform,
+)
+from monai.utils import MAX_SEED, ensure_tuple, get_seed
 
-        """
-        raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.")
+__all__ = ["Compose"]
 
 
-class Compose(Randomizable, Transform):
+class Compose(Randomizable, InvertibleTransform):
     """
     ``Compose`` provides the ability to chain a series of calls together in a
     sequence. Each transform in the sequence must take a single argument and
@@ -256,66 +144,12 @@ def __call__(self, input_):
             input_ = apply_transform(_transform, input_)
         return input_
 
+    def inverse(self, data):
+        invertible_transforms = [t for t in self.flatten().transforms if isinstance(t, InvertibleTransform)]
+        if len(invertible_transforms) == 0:
+            warnings.warn("inverse has been called but no invertible transforms have been supplied")
 
-class MapTransform(Transform):
-    """
-    A subclass of :py:class:`monai.transforms.Transform` with an assumption
-    that the ``data`` input of ``self.__call__`` is a MutableMapping such as ``dict``.
-
-    The ``keys`` parameter will be used to get and set the actual data
-    item to transform.  That is, the callable of this transform should
-    follow the pattern:
-
-        .. code-block:: python
-
-            def __call__(self, data):
-                for key in self.keys:
-                    if key in data:
-                        # update output data with some_transform_function(data[key]).
-                    else:
-                        # do nothing or some exceptions handling.
-                return data
-
-    Raises:
-        ValueError: When ``keys`` is an empty iterable.
-        TypeError: When ``keys`` type is not in ``Union[Hashable, Iterable[Hashable]]``.
-
-    """
-
-    def __init__(self, keys: KeysCollection) -> None:
-        self.keys: Tuple[Hashable, ...] = ensure_tuple(keys)
-        if not self.keys:
-            raise ValueError("keys must be non empty.")
-        for key in self.keys:
-            if not isinstance(key, Hashable):
-                raise TypeError(f"keys must be one of (Hashable, Iterable[Hashable]) but is {type(keys).__name__}.")
-
-    @abstractmethod
-    def __call__(self, data):
-        """
-        ``data`` often comes from an iteration over an iterable,
-        such as :py:class:`torch.utils.data.Dataset`.
-
-        To simplify the input validations, this method assumes:
-
-        - ``data`` is a Python dictionary
-        - ``data[key]`` is a Numpy ndarray, PyTorch Tensor or string, where ``key`` is an element
-          of ``self.keys``, the data shape can be:
-
-          #. string data without shape, `LoadImaged` transform expects file paths
-          #. most of the pre-processing transforms expect: ``(num_channels, spatial_dim_1[, spatial_dim_2, ...])``,
-             except that `AddChanneld` expects (spatial_dim_1[, spatial_dim_2, ...]) and
-             `AsChannelFirstd` expects (spatial_dim_1[, spatial_dim_2, ...], num_channels)
-          #. most of the post-processing transforms expect
-             ``(batch_size, num_channels, spatial_dim_1[, spatial_dim_2, ...])``
-
-        - the channel dimension is not omitted even if number of channels is one
-
-        Raises:
-            NotImplementedError: When the subclass does not override this method.
-
-        returns:
-            An updated dictionary version of ``data`` by applying the transform.
-
-        """
-        raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.")
+        # loop backwards over transforms
+        for t in reversed(invertible_transforms):
+            data = apply_transform(t.inverse, data)
+        return data
diff --git a/monai/transforms/croppad/array.py b/monai/transforms/croppad/array.py
index b4444803a4..9da94a046c 100644
--- a/monai/transforms/croppad/array.py
+++ b/monai/transforms/croppad/array.py
@@ -20,7 +20,7 @@
 
 from monai.config import IndexSelection
 from monai.data.utils import get_random_patch, get_valid_patch_size
-from monai.transforms.compose import Randomizable, Transform
+from monai.transforms.transform import Randomizable, Transform
 from monai.transforms.utils import (
     generate_pos_neg_label_crop_centers,
     generate_spatial_bounding_box,
@@ -106,7 +106,7 @@ class BorderPad(Transform):
     Pad the input data by adding specified borders to every dimension.
 
     Args:
-        spatial_border: specified size for every spatial border. it can be 3 shapes:
+        spatial_border: specified size for every spatial border. Any -ve values will be set to 0. It can be 3 shapes:
 
             - single int number, pad all the borders with the same size.
             - length equals the length of image shape, pad every spatial dimension separately.
@@ -140,16 +140,16 @@ def __call__(self, img: np.ndarray, mode: Optional[Union[NumpyPadMode, str]] = N
                 See also: https://numpy.org/doc/1.18/reference/generated/numpy.pad.html
 
         Raises:
-            ValueError: When ``self.spatial_border`` contains a nonnegative int.
+            ValueError: When ``self.spatial_border`` does not contain ints.
             ValueError: When ``self.spatial_border`` length is not one of
                 [1, len(spatial_shape), 2*len(spatial_shape)].
 
         """
         spatial_shape = img.shape[1:]
         spatial_border = ensure_tuple(self.spatial_border)
-        for b in spatial_border:
-            if not isinstance(b, int) or b < 0:
-                raise ValueError(f"self.spatial_border must contain only nonnegative ints, got {spatial_border}.")
+        if not all(isinstance(b, int) for b in spatial_border):
+            raise ValueError(f"self.spatial_border must contain only ints, got {spatial_border}.")
+        spatial_border = tuple(max(0, b) for b in spatial_border)
 
         if len(spatial_border) == 1:
             data_pad_width = [(spatial_border[0], spatial_border[0]) for _ in range(len(spatial_shape))]
@@ -214,8 +214,11 @@ class SpatialCrop(Transform):
     """
     General purpose cropper to produce sub-volume region of interest (ROI).
     It can support to crop ND spatial (channel-first) data.
-    Either a spatial center and size must be provided, or alternatively,
-    if center and size are not provided, the start and end coordinates of the ROI must be provided.
+
+    The cropped region can be parameterised in various ways:
+        - a list of slices for each spatial dimension (allows for use of -ve indexing and `None`)
+        - a spatial center and size
+        - the start and end coordinates of the ROI
     """
 
     def __init__(
@@ -224,6 +227,7 @@ def __init__(
         roi_size: Union[Sequence[int], np.ndarray, None] = None,
         roi_start: Union[Sequence[int], np.ndarray, None] = None,
         roi_end: Union[Sequence[int], np.ndarray, None] = None,
+        roi_slices: Optional[Sequence[slice]] = None,
     ) -> None:
         """
         Args:
@@ -231,26 +235,37 @@ def __init__(
             roi_size: size of the crop ROI.
             roi_start: voxel coordinates for start of the crop ROI.
             roi_end: voxel coordinates for end of the crop ROI.
+            roi_slices: list of slices for each of the spatial dimensions.
         """
-        if roi_center is not None and roi_size is not None:
-            roi_center = np.asarray(roi_center, dtype=np.int16)
-            roi_size = np.asarray(roi_size, dtype=np.int16)
-            self.roi_start = np.maximum(roi_center - np.floor_divide(roi_size, 2), 0)
-            self.roi_end = np.maximum(self.roi_start + roi_size, self.roi_start)
+        if roi_slices:
+            if not all(s.step is None or s.step == 1 for s in roi_slices):
+                raise ValueError("Only slice steps of 1/None are currently supported")
+            self.slices = list(roi_slices)
         else:
-            if roi_start is None or roi_end is None:
-                raise ValueError("Please specify either roi_center, roi_size or roi_start, roi_end.")
-            self.roi_start = np.maximum(np.asarray(roi_start, dtype=np.int16), 0)
-            self.roi_end = np.maximum(np.asarray(roi_end, dtype=np.int16), self.roi_start)
-
-    def __call__(self, img: Union[np.ndarray, torch.Tensor]) -> np.ndarray:
+            if roi_center is not None and roi_size is not None:
+                roi_center = np.asarray(roi_center, dtype=np.int16)
+                roi_size = np.asarray(roi_size, dtype=np.int16)
+                roi_start_np = np.maximum(roi_center - np.floor_divide(roi_size, 2), 0)
+                roi_end_np = np.maximum(roi_start_np + roi_size, roi_start_np)
+            else:
+                if roi_start is None or roi_end is None:
+                    raise ValueError("Please specify either roi_center, roi_size or roi_start, roi_end.")
+                roi_start_np = np.maximum(np.asarray(roi_start, dtype=np.int16), 0)
+                roi_end_np = np.maximum(np.asarray(roi_end, dtype=np.int16), roi_start_np)
+            # Allow for 1D by converting back to np.array (since np.maximum will convert to int)
+            roi_start_np = roi_start_np if isinstance(roi_start_np, np.ndarray) else np.array([roi_start_np])
+            roi_end_np = roi_end_np if isinstance(roi_end_np, np.ndarray) else np.array([roi_end_np])
+            # convert to slices
+            self.slices = [slice(s, e) for s, e in zip(roi_start_np, roi_end_np)]
+
+    def __call__(self, img: Union[np.ndarray, torch.Tensor]):
         """
         Apply the transform to `img`, assuming `img` is channel-first and
         slicing doesn't apply to the channel dim.
         """
-        sd = min(len(self.roi_start), len(self.roi_end), len(img.shape[1:]))  # spatial dims
-        slices = [slice(None)] + [slice(s, e) for s, e in zip(self.roi_start[:sd], self.roi_end[:sd])]
-        return np.asarray(img[tuple(slices)])
+        sd = min(len(self.slices), len(img.shape[1:]))  # spatial dims
+        slices = [slice(None)] + self.slices[:sd]
+        return img[tuple(slices)]
 
 
 class CenterSpatialCrop(Transform):
@@ -276,7 +291,7 @@ def __call__(self, img: np.ndarray):
         return cropper(img)
 
 
-class RandSpatialCrop(Randomizable, Transform):
+class RandSpatialCrop(Randomizable):
     """
     Crop image with random size or specific size ROI. It can crop at a random position as center
     or at the image center. And allows to set the minimum size to limit the randomly generated ROI.
@@ -321,7 +336,7 @@ def __call__(self, img: np.ndarray):
         return cropper(img)
 
 
-class RandSpatialCropSamples(Randomizable, Transform):
+class RandSpatialCropSamples(Randomizable):
     """
     Crop image with random size or specific size ROI to generate a list of N samples.
     It can crop at a random position as center or at the image center. And allows to set
@@ -429,7 +444,7 @@ def __call__(self, img: np.ndarray):
         return cropped
 
 
-class RandWeightedCrop(Randomizable, Transform):
+class RandWeightedCrop(Randomizable):
     """
     Samples a list of `num_samples` image patches according to the provided `weight_map`.
 
@@ -481,7 +496,7 @@ def __call__(self, img: np.ndarray, weight_map: Optional[np.ndarray] = None) ->
         return results
 
 
-class RandCropByPosNegLabel(Randomizable, Transform):
+class RandCropByPosNegLabel(Randomizable):
     """
     Crop random fixed sized regions with the center being a foreground or background voxel
     based on the Pos Neg Ratio.
diff --git a/monai/transforms/croppad/batch.py b/monai/transforms/croppad/batch.py
new file mode 100644
index 0000000000..37ff8618fa
--- /dev/null
+++ b/monai/transforms/croppad/batch.py
@@ -0,0 +1,129 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+A collection of "vanilla" transforms for crop and pad operations acting on batches of data
+https://github.com/Project-MONAI/MONAI/wiki/MONAI_Design
+"""
+
+from copy import deepcopy
+from typing import Any, Dict, Hashable, Union
+
+import numpy as np
+import torch
+
+from monai.data.utils import list_data_collate
+from monai.transforms.compose import Compose
+from monai.transforms.croppad.array import CenterSpatialCrop, SpatialPad
+from monai.transforms.inverse import InvertibleTransform
+from monai.transforms.utility.array import ToTensor
+from monai.utils.enums import InverseKeys, Method, NumpyPadMode
+
+__all__ = [
+    "PadListDataCollate",
+]
+
+
+def replace_element(to_replace, batch, idx, key_or_idx):
+    # since tuple is immutable we'll have to recreate
+    if isinstance(batch[idx], tuple):
+        batch_idx_list = list(batch[idx])
+        batch_idx_list[key_or_idx] = to_replace
+        batch[idx] = tuple(batch_idx_list)
+    # else, replace
+    else:
+        batch[idx][key_or_idx] = to_replace
+    return batch
+
+
+class PadListDataCollate(InvertibleTransform):
+    """
+    Same as MONAI's ``list_data_collate``, except any tensors are centrally padded to match the shape of the biggest
+    tensor in each dimension. This transform is useful if some of the applied transforms generate batch data of
+    different sizes.
+
+    This can be used on both list and dictionary data. In the case of the dictionary data, this transform will be added
+    to the list of invertible transforms.
+
+    Note that normally, a user won't explicitly use the `__call__` method. Rather this would be passed to the `DataLoader`.
+    This means that `__call__` handles data as it comes out of a `DataLoader`, containing batch dimension. However, the
+    `inverse` operates on dictionaries containing images of shape `C,H,W,[D]`. This asymmetry is necessary so that we can
+    pass the inverse through multiprocessing.
+
+    Args:
+        batch: batch of data to pad-collate
+        method: padding method (see :py:class:`monai.transforms.SpatialPad`)
+        mode: padding mode (see :py:class:`monai.transforms.SpatialPad`)
+    """
+
+    def __init__(
+        self,
+        method: Union[Method, str] = Method.SYMMETRIC,
+        mode: Union[NumpyPadMode, str] = NumpyPadMode.CONSTANT,
+    ) -> None:
+        self.method = method
+        self.mode = mode
+
+    def __call__(self, batch: Any):
+        # data is either list of dicts or list of lists
+        is_list_of_dicts = isinstance(batch[0], dict)
+        # loop over items inside of each element in a batch
+        for key_or_idx in batch[0].keys() if is_list_of_dicts else range(len(batch[0])):
+            # calculate max size of each dimension
+            max_shapes = []
+            for elem in batch:
+                if not isinstance(elem[key_or_idx], (torch.Tensor, np.ndarray)):
+                    break
+                max_shapes.append(elem[key_or_idx].shape[1:])
+            # len > 0 if objects were arrays, else skip as no padding to be done
+            if len(max_shapes) == 0:
+                continue
+            max_shape = np.array(max_shapes).max(axis=0)
+            # If all same size, skip
+            if np.all(np.array(max_shapes).min(axis=0) == max_shape):
+                continue
+            # Do we need to convert output to Tensor?
+            output_to_tensor = isinstance(batch[0][key_or_idx], torch.Tensor)
+
+            # Use `SpatialPadd` or `SpatialPad` to match sizes
+            # Default params are central padding, padding with 0's
+            # If input is dictionary, use the dictionary version so that the transformation is recorded
+
+            padder = SpatialPad(max_shape, self.method, self.mode)  # type: ignore
+            transform = padder if not output_to_tensor else Compose([padder, ToTensor()])
+
+            for idx in range(len(batch)):
+                im = batch[idx][key_or_idx]
+                orig_size = im.shape[1:]
+                padded = transform(batch[idx][key_or_idx])
+                batch = replace_element(padded, batch, idx, key_or_idx)
+
+                # If we have a dictionary of data, append to list
+                if is_list_of_dicts:
+                    self.push_transform(batch[idx], key_or_idx, orig_size=orig_size)
+
+        # After padding, use default list collator
+        return list_data_collate(batch)
+
+    @staticmethod
+    def inverse(data: dict) -> Dict[Hashable, np.ndarray]:
+        if not isinstance(data, dict):
+            raise RuntimeError("Inverse can only currently be applied on dictionaries.")
+
+        d = deepcopy(data)
+        for key in d.keys():
+            transform_key = str(key) + InverseKeys.KEY_SUFFIX
+            if transform_key in d.keys():
+                transform = d[transform_key][-1]
+                if transform[InverseKeys.CLASS_NAME] == PadListDataCollate.__name__:
+                    d[key] = CenterSpatialCrop(transform["orig_size"])(d[key])
+                    # remove transform
+                    d[transform_key].pop()
+        return d
diff --git a/monai/transforms/croppad/dictionary.py b/monai/transforms/croppad/dictionary.py
index 1faed25605..750add4a28 100644
--- a/monai/transforms/croppad/dictionary.py
+++ b/monai/transforms/croppad/dictionary.py
@@ -15,13 +15,16 @@
 Class names are ended with 'd' to denote dictionary-based transforms.
 """
 
+from copy import deepcopy
+from enum import Enum
+from itertools import chain
+from math import floor
 from typing import Any, Callable, Dict, Hashable, List, Mapping, Optional, Sequence, Tuple, Union
 
 import numpy as np
 
 from monai.config import IndexSelection, KeysCollection
 from monai.data.utils import get_random_patch, get_valid_patch_size
-from monai.transforms.compose import MapTransform, Randomizable
 from monai.transforms.croppad.array import (
     BorderPad,
     BoundingRect,
@@ -31,13 +34,17 @@
     SpatialCrop,
     SpatialPad,
 )
+from monai.transforms.inverse import InvertibleTransform
+from monai.transforms.transform import MapTransform, Randomizable
 from monai.transforms.utils import (
     generate_pos_neg_label_crop_centers,
     generate_spatial_bounding_box,
     map_binary_to_indices,
     weighted_patch_samples,
 )
+from monai.utils import ImageMetaKey as Key
 from monai.utils import Method, NumpyPadMode, ensure_tuple, ensure_tuple_rep, fall_back_tuple
+from monai.utils.enums import InverseKeys
 
 __all__ = [
     "NumpyPadModeSequence",
@@ -82,7 +89,7 @@
 NumpyPadModeSequence = Union[Sequence[Union[NumpyPadMode, str]], NumpyPadMode, str]
 
 
-class SpatialPadd(MapTransform):
+class SpatialPadd(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.SpatialPad`.
     Performs padding to the data, symmetric for all sides or all on one side for each dimension.
@@ -94,6 +101,7 @@ def __init__(
         spatial_size: Union[Sequence[int], int],
         method: Union[Method, str] = Method.SYMMETRIC,
         mode: NumpyPadModeSequence = NumpyPadMode.CONSTANT,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -108,20 +116,42 @@ def __init__(
                 One of the listed string values or a user supplied function. Defaults to ``"constant"``.
                 See also: https://numpy.org/doc/1.18/reference/generated/numpy.pad.html
                 It also can be a sequence of string, each element corresponds to a key in ``keys``.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.mode = ensure_tuple_rep(mode, len(self.keys))
         self.padder = SpatialPad(spatial_size, method)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key, m in zip(self.keys, self.mode):
+        for key, m in self.key_iterator(d, self.mode):
+            self.push_transform(d, key, extra_info={"mode": m.value if isinstance(m, Enum) else m})
             d[key] = self.padder(d[key], mode=m)
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            orig_size = transform[InverseKeys.ORIG_SIZE]
+            if self.padder.method == Method.SYMMETRIC:
+                current_size = d[key].shape[1:]
+                roi_center = [floor(i / 2) if r % 2 == 0 else (i - 1) // 2 for r, i in zip(orig_size, current_size)]
+            else:
+                roi_center = [floor(r / 2) if r % 2 == 0 else (r - 1) // 2 for r in orig_size]
+
+            inverse_transform = SpatialCrop(roi_center, orig_size)
+            # Apply inverse transform
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
 
-class BorderPadd(MapTransform):
+class BorderPadd(MapTransform, InvertibleTransform):
     """
     Pad the input data by adding specified borders to every dimension.
     Dictionary-based wrapper of :py:class:`monai.transforms.BorderPad`.
@@ -132,6 +162,7 @@ def __init__(
         keys: KeysCollection,
         spatial_border: Union[Sequence[int], int],
         mode: NumpyPadModeSequence = NumpyPadMode.CONSTANT,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -153,27 +184,57 @@ def __init__(
                 One of the listed string values or a user supplied function. Defaults to ``"constant"``.
                 See also: https://numpy.org/doc/1.18/reference/generated/numpy.pad.html
                 It also can be a sequence of string, each element corresponds to a key in ``keys``.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.mode = ensure_tuple_rep(mode, len(self.keys))
         self.padder = BorderPad(spatial_border=spatial_border)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key, m in zip(self.keys, self.mode):
+        for key, m in self.key_iterator(d, self.mode):
+            self.push_transform(d, key, extra_info={"mode": m.value if isinstance(m, Enum) else m})
             d[key] = self.padder(d[key], mode=m)
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            orig_size = np.array(transform[InverseKeys.ORIG_SIZE])
+            roi_start = np.array(self.padder.spatial_border)
+            # Need to convert single value to [min1,min2,...]
+            if roi_start.size == 1:
+                roi_start = np.full((len(orig_size)), roi_start)
+            # need to convert [min1,max1,min2,...] to [min1,min2,...]
+            elif roi_start.size == 2 * orig_size.size:
+                roi_start = roi_start[::2]
+            roi_end = np.array(transform[InverseKeys.ORIG_SIZE]) + roi_start
+
+            inverse_transform = SpatialCrop(roi_start=roi_start, roi_end=roi_end)
+            # Apply inverse transform
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
 
-class DivisiblePadd(MapTransform):
+class DivisiblePadd(MapTransform, InvertibleTransform):
     """
     Pad the input data, so that the spatial sizes are divisible by `k`.
     Dictionary-based wrapper of :py:class:`monai.transforms.DivisiblePad`.
     """
 
     def __init__(
-        self, keys: KeysCollection, k: Union[Sequence[int], int], mode: NumpyPadModeSequence = NumpyPadMode.CONSTANT
+        self,
+        keys: KeysCollection,
+        k: Union[Sequence[int], int],
+        mode: NumpyPadModeSequence = NumpyPadMode.CONSTANT,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -187,26 +248,51 @@ def __init__(
                 One of the listed string values or a user supplied function. Defaults to ``"constant"``.
                 See also: https://numpy.org/doc/1.18/reference/generated/numpy.pad.html
                 It also can be a sequence of string, each element corresponds to a key in ``keys``.
+            allow_missing_keys: don't raise exception if key is missing.
 
         See also :py:class:`monai.transforms.SpatialPad`
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.mode = ensure_tuple_rep(mode, len(self.keys))
         self.padder = DivisiblePad(k=k)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key, m in zip(self.keys, self.mode):
+        for key, m in self.key_iterator(d, self.mode):
+            self.push_transform(d, key, extra_info={"mode": m.value if isinstance(m, Enum) else m})
             d[key] = self.padder(d[key], mode=m)
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            orig_size = np.array(transform[InverseKeys.ORIG_SIZE])
+            current_size = np.array(d[key].shape[1:])
+            roi_start = np.floor((current_size - orig_size) / 2)
+            roi_end = orig_size + roi_start
+            inverse_transform = SpatialCrop(roi_start=roi_start, roi_end=roi_end)
+            # Apply inverse transform
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
 
-class SpatialCropd(MapTransform):
+class SpatialCropd(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.SpatialCrop`.
-    Either a spatial center and size must be provided, or alternatively if center and size
-    are not provided, the start and end coordinates of the ROI must be provided.
+    General purpose cropper to produce sub-volume region of interest (ROI).
+    It can support to crop ND spatial (channel-first) data.
+
+    The cropped region can be parameterised in various ways:
+        - a list of slices for each spatial dimension (allows for use of -ve indexing and `None`)
+        - a spatial center and size
+        - the start and end coordinates of the ROI
     """
 
     def __init__(
@@ -216,6 +302,8 @@ def __init__(
         roi_size: Optional[Sequence[int]] = None,
         roi_start: Optional[Sequence[int]] = None,
         roi_end: Optional[Sequence[int]] = None,
+        roi_slices: Optional[Sequence[slice]] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -225,18 +313,42 @@ def __init__(
             roi_size: size of the crop ROI.
             roi_start: voxel coordinates for start of the crop ROI.
             roi_end: voxel coordinates for end of the crop ROI.
+            roi_slices: list of slices for each of the spatial dimensions.
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
-        self.cropper = SpatialCrop(roi_center, roi_size, roi_start, roi_end)
+        super().__init__(keys, allow_missing_keys)
+        self.cropper = SpatialCrop(roi_center, roi_size, roi_start, roi_end, roi_slices)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            self.push_transform(d, key)
             d[key] = self.cropper(d[key])
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            orig_size = np.array(transform[InverseKeys.ORIG_SIZE])
+            current_size = np.array(d[key].shape[1:])
+            # get required pad to start and end
+            pad_to_start = np.array([s.indices(o)[0] for s, o in zip(self.cropper.slices, orig_size)])
+            pad_to_end = orig_size - current_size - pad_to_start
+            # interleave mins and maxes
+            pad = list(chain(*zip(pad_to_start.tolist(), pad_to_end.tolist())))
+            inverse_transform = BorderPad(pad)
+            # Apply inverse transform
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
 
-class CenterSpatialCropd(MapTransform):
+class CenterSpatialCropd(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.CenterSpatialCrop`.
 
@@ -245,20 +357,46 @@ class CenterSpatialCropd(MapTransform):
             See also: monai.transforms.MapTransform
         roi_size: the size of the crop region e.g. [224,224,128]
             If its components have non-positive values, the corresponding size of input image will be used.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
-    def __init__(self, keys: KeysCollection, roi_size: Union[Sequence[int], int]) -> None:
-        super().__init__(keys)
+    def __init__(
+        self, keys: KeysCollection, roi_size: Union[Sequence[int], int], allow_missing_keys: bool = False
+    ) -> None:
+        super().__init__(keys, allow_missing_keys)
         self.cropper = CenterSpatialCrop(roi_size)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            orig_size = d[key].shape[1:]
             d[key] = self.cropper(d[key])
+            self.push_transform(d, key, orig_size=orig_size)
+        return d
+
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            orig_size = np.array(transform[InverseKeys.ORIG_SIZE])
+            current_size = np.array(d[key].shape[1:])
+            pad_to_start = np.floor((orig_size - current_size) / 2).astype(int)
+            # in each direction, if original size is even and current size is odd, += 1
+            pad_to_start[np.logical_and(orig_size % 2 == 0, current_size % 2 == 1)] += 1
+            pad_to_end = orig_size - current_size - pad_to_start
+            pad = list(chain(*zip(pad_to_start.tolist(), pad_to_end.tolist())))
+            inverse_transform = BorderPad(pad)
+            # Apply inverse transform
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
         return d
 
 
-class RandSpatialCropd(Randomizable, MapTransform):
+class RandSpatialCropd(Randomizable, MapTransform, InvertibleTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.RandSpatialCrop`.
     Crop image with random size or specific size ROI. It can crop at a random position as
@@ -274,6 +412,7 @@ class RandSpatialCropd(Randomizable, MapTransform):
         random_center: crop at random position as center or the image center.
         random_size: crop with random size or specific size ROI.
             The actual size is sampled from `randint(roi_size, img_size)`.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -282,8 +421,9 @@ def __init__(
         roi_size: Union[Sequence[int], int],
         random_center: bool = True,
         random_size: bool = True,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
         self.roi_size = roi_size
         self.random_center = random_center
         self.random_size = random_size
@@ -303,14 +443,48 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
         self.randomize(d[self.keys[0]].shape[1:])  # image shape from the first data key
         if self._size is None:
             raise AssertionError
-        for key in self.keys:
+        for key in self.key_iterator(d):
             if self.random_center:
+                self.push_transform(d, key, {"slices": [(i.start, i.stop) for i in self._slices[1:]]})  # type: ignore
                 d[key] = d[key][self._slices]
             else:
+                self.push_transform(d, key)
                 cropper = CenterSpatialCrop(self._size)
                 d[key] = cropper(d[key])
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            orig_size = transform[InverseKeys.ORIG_SIZE]
+            random_center = self.random_center
+            pad_to_start = np.empty((len(orig_size)), dtype=np.int32)
+            pad_to_end = np.empty((len(orig_size)), dtype=np.int32)
+            if random_center:
+                for i, _slice in enumerate(transform[InverseKeys.EXTRA_INFO]["slices"]):
+                    pad_to_start[i] = _slice[0]
+                    pad_to_end[i] = orig_size[i] - _slice[1]
+            else:
+                current_size = d[key].shape[1:]
+                for i, (o_s, c_s) in enumerate(zip(orig_size, current_size)):
+                    pad_to_start[i] = pad_to_end[i] = (o_s - c_s) / 2
+                    if o_s % 2 == 0 and c_s % 2 == 1:
+                        pad_to_start[i] += 1
+                    elif o_s % 2 == 1 and c_s % 2 == 0:
+                        pad_to_end[i] += 1
+            # interleave mins and maxes
+            pad = list(chain(*zip(pad_to_start.tolist(), pad_to_end.tolist())))
+            inverse_transform = BorderPad(pad)
+            # Apply inverse transform
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
 
 class RandSpatialCropSamplesd(Randomizable, MapTransform):
     """
@@ -318,7 +492,7 @@ class RandSpatialCropSamplesd(Randomizable, MapTransform):
     Crop image with random size or specific size ROI to generate a list of N samples.
     It can crop at a random position as center or at the image center. And allows to set
     the minimum size to limit the randomly generated ROI. Suppose all the expected fields
-    specified by `keys` have same shape.
+    specified by `keys` have same shape, and add `patch_index` to the corresponding meta data.
     It will return a list of dictionaries for all the cropped images.
 
     Args:
@@ -330,6 +504,10 @@ class RandSpatialCropSamplesd(Randomizable, MapTransform):
         random_center: crop at random position as center or the image center.
         random_size: crop with random size or specific size ROI.
             The actual size is sampled from `randint(roi_size, img_size)`.
+        meta_key_postfix: use `key_{postfix}` to to fetch the meta data according to the key data,
+            default is `meta_dict`, the meta data is a dictionary object.
+            used to add `patch_index` to the meta dict.
+        allow_missing_keys: don't raise exception if key is missing.
 
     Raises:
         ValueError: When ``num_samples`` is nonpositive.
@@ -343,12 +521,15 @@ def __init__(
         num_samples: int,
         random_center: bool = True,
         random_size: bool = True,
+        meta_key_postfix: str = "meta_dict",
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
         if num_samples < 1:
             raise ValueError(f"num_samples must be positive, got {num_samples}.")
         self.num_samples = num_samples
-        self.cropper = RandSpatialCropd(keys, roi_size, random_center, random_size)
+        self.cropper = RandSpatialCropd(keys, roi_size, random_center, random_size, allow_missing_keys)
+        self.meta_key_postfix = meta_key_postfix
 
     def set_random_state(
         self, seed: Optional[int] = None, state: Optional[np.random.RandomState] = None
@@ -361,10 +542,21 @@ def randomize(self, data: Optional[Any] = None) -> None:
         pass
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> List[Dict[Hashable, np.ndarray]]:
-        return [self.cropper(data) for _ in range(self.num_samples)]
-
-
-class CropForegroundd(MapTransform):
+        ret = []
+        d = dict(data)
+        for i in range(self.num_samples):
+            cropped = self.cropper(d)
+            # add `patch_index` to the meta data
+            for key in self.key_iterator(d):
+                meta_data_key = f"{key}_{self.meta_key_postfix}"
+                if meta_data_key not in cropped:
+                    cropped[meta_data_key] = {}  # type: ignore
+                cropped[meta_data_key][Key.PATCH_INDEX] = i
+            ret.append(cropped)
+        return ret
+
+
+class CropForegroundd(MapTransform, InvertibleTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.CropForeground`.
     Crop only the foreground object of the expected images.
@@ -386,6 +578,7 @@ def __init__(
         margin: int = 0,
         start_coord_key: str = "foreground_start_coord",
         end_coord_key: str = "foreground_end_coord",
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -398,8 +591,9 @@ def __init__(
             margin: add margin value to spatial dims of the bounding box, if only 1 value provided, use it for all dims.
             start_coord_key: key to record the start coordinate of spatial bounding box for foreground.
             end_coord_key: key to record the end coordinate of spatial bounding box for foreground.
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.source_key = source_key
         self.select_fn = select_fn
         self.channel_indices = ensure_tuple(channel_indices) if channel_indices is not None else None
@@ -415,10 +609,30 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
         d[self.start_coord_key] = np.asarray(box_start)
         d[self.end_coord_key] = np.asarray(box_end)
         cropper = SpatialCrop(roi_start=box_start, roi_end=box_end)
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            self.push_transform(d, key, extra_info={"box_start": box_start, "box_end": box_end})
             d[key] = cropper(d[key])
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            orig_size = np.array(transform[InverseKeys.ORIG_SIZE])
+            extra_info = transform[InverseKeys.EXTRA_INFO]
+            pad_to_start = np.array(extra_info["box_start"])
+            pad_to_end = orig_size - np.array(extra_info["box_end"])
+            # interleave mins and maxes
+            pad = list(chain(*zip(pad_to_start.tolist(), pad_to_end.tolist())))
+            inverse_transform = BorderPad(pad)
+            # Apply inverse transform
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
 
 class RandWeightedCropd(Randomizable, MapTransform):
     """
@@ -433,6 +647,7 @@ class RandWeightedCropd(Randomizable, MapTransform):
             If its components have non-positive values, the corresponding size of `img` will be used.
         num_samples: number of samples (image patches) to take in the returned list.
         center_coord_key: if specified, the actual sampling location will be stored with the corresponding key.
+        allow_missing_keys: don't raise exception if key is missing.
 
     See Also:
         :py:class:`monai.transforms.RandWeightedCrop`
@@ -445,8 +660,9 @@ def __init__(
         spatial_size: Union[Sequence[int], int],
         num_samples: int = 1,
         center_coord_key: Optional[str] = None,
+        allow_missing_keys: bool = False,
     ):
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
         self.spatial_size = ensure_tuple(spatial_size)
         self.w_key = w_key
         self.num_samples = int(num_samples)
@@ -464,22 +680,22 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> List[Dict[Hashable, n
         _spatial_size = fall_back_tuple(self.spatial_size, d[self.w_key].shape[1:])
 
         results: List[Dict[Hashable, np.ndarray]] = [{} for _ in range(self.num_samples)]
-        for key in data.keys():
-            if key in self.keys:
-                img = d[key]
-                if img.shape[1:] != d[self.w_key].shape[1:]:
-                    raise ValueError(
-                        f"data {key} and weight map {self.w_key} spatial shape mismatch: "
-                        f"{img.shape[1:]} vs {d[self.w_key].shape[1:]}."
-                    )
-                for i, center in enumerate(self.centers):
-                    cropper = SpatialCrop(roi_center=center, roi_size=_spatial_size)
-                    results[i][key] = cropper(img)
-                    if self.center_coord_key:
-                        results[i][self.center_coord_key] = center
-            else:
-                for i in range(self.num_samples):
-                    results[i][key] = data[key]
+        for key in self.key_iterator(d):
+            img = d[key]
+            if img.shape[1:] != d[self.w_key].shape[1:]:
+                raise ValueError(
+                    f"data {key} and weight map {self.w_key} spatial shape mismatch: "
+                    f"{img.shape[1:]} vs {d[self.w_key].shape[1:]}."
+                )
+            for i, center in enumerate(self.centers):
+                cropper = SpatialCrop(roi_center=center, roi_size=_spatial_size)
+                results[i][key] = cropper(img)
+                if self.center_coord_key:
+                    results[i][self.center_coord_key] = center
+        # fill in the extra keys with unmodified data
+        for key in set(data.keys()).difference(set(self.keys)):
+            for i in range(self.num_samples):
+                results[i][key] = data[key]
 
         return results
 
@@ -489,6 +705,8 @@ class RandCropByPosNegLabeld(Randomizable, MapTransform):
     Dictionary-based version :py:class:`monai.transforms.RandCropByPosNegLabel`.
     Crop random fixed sized regions with the center being a foreground or background voxel
     based on the Pos Neg Ratio.
+    Suppose all the expected fields specified by `keys` have same shape,
+    and add `patch_index` to the corresponding meta data.
     And will return a list of dictionaries for all the cropped images.
 
     Args:
@@ -514,6 +732,10 @@ class RandCropByPosNegLabeld(Randomizable, MapTransform):
             `image_threshold`, and randomly select crop centers based on them, need to provide `fg_indices_key`
             and `bg_indices_key` together, expect to be 1 dim array of spatial indices after flattening.
             a typical usage is to call `FgBgToIndicesd` transform first and cache the results.
+        meta_key_postfix: use `key_{postfix}` to to fetch the meta data according to the key data,
+            default is `meta_dict`, the meta data is a dictionary object.
+            used to add `patch_index` to the meta dict.
+        allow_missing_keys: don't raise exception if key is missing.
 
     Raises:
         ValueError: When ``pos`` or ``neg`` are negative.
@@ -533,8 +755,10 @@ def __init__(
         image_threshold: float = 0.0,
         fg_indices_key: Optional[str] = None,
         bg_indices_key: Optional[str] = None,
+        meta_key_postfix: str = "meta_dict",
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
         self.label_key = label_key
         self.spatial_size: Union[Tuple[int, ...], Sequence[int], int] = spatial_size
         if pos < 0 or neg < 0:
@@ -547,6 +771,7 @@ def __init__(
         self.image_threshold = image_threshold
         self.fg_indices_key = fg_indices_key
         self.bg_indices_key = bg_indices_key
+        self.meta_key_postfix = meta_key_postfix
         self.centers: Optional[List[List[np.ndarray]]] = None
 
     def randomize(
@@ -570,8 +795,8 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> List[Dict[Hashable, n
         d = dict(data)
         label = d[self.label_key]
         image = d[self.image_key] if self.image_key else None
-        fg_indices = d.get(self.fg_indices_key, None) if self.fg_indices_key is not None else None
-        bg_indices = d.get(self.bg_indices_key, None) if self.bg_indices_key is not None else None
+        fg_indices = d.get(self.fg_indices_key) if self.fg_indices_key is not None else None
+        bg_indices = d.get(self.bg_indices_key) if self.bg_indices_key is not None else None
 
         self.randomize(label, fg_indices, bg_indices, image)
         if not isinstance(self.spatial_size, tuple):
@@ -579,20 +804,26 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> List[Dict[Hashable, n
         if self.centers is None:
             raise AssertionError
         results: List[Dict[Hashable, np.ndarray]] = [{} for _ in range(self.num_samples)]
-        for key in data.keys():
-            if key in self.keys:
+
+        for i, center in enumerate(self.centers):
+            for key in self.key_iterator(d):
                 img = d[key]
-                for i, center in enumerate(self.centers):
-                    cropper = SpatialCrop(roi_center=tuple(center), roi_size=self.spatial_size)  # type: ignore
-                    results[i][key] = cropper(img)
-            else:
-                for i in range(self.num_samples):
-                    results[i][key] = data[key]
+                cropper = SpatialCrop(roi_center=tuple(center), roi_size=self.spatial_size)  # type: ignore
+                results[i][key] = cropper(img)
+            # fill in the extra keys with unmodified data
+            for key in set(data.keys()).difference(set(self.keys)):
+                results[i][key] = data[key]
+            # add `patch_index` to the meta data
+            for key in self.key_iterator(d):
+                meta_data_key = f"{key}_{self.meta_key_postfix}"
+                if meta_data_key not in results[i]:
+                    results[i][meta_data_key] = {}  # type: ignore
+                results[i][meta_data_key][Key.PATCH_INDEX] = i
 
         return results
 
 
-class ResizeWithPadOrCropd(MapTransform):
+class ResizeWithPadOrCropd(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.ResizeWithPadOrCrop`.
 
@@ -605,6 +836,8 @@ class ResizeWithPadOrCropd(MapTransform):
             ``"median"``, ``"minimum"``, ``"reflect"``, ``"symmetric"``, ``"wrap"``, ``"empty"``}
             One of the listed string values or a user supplied function for padding. Defaults to ``"constant"``.
             See also: https://numpy.org/doc/1.18/reference/generated/numpy.pad.html
+            It also can be a sequence of string, each element corresponds to a key in ``keys``.
+        allow_missing_keys: don't raise exception if key is missing.
 
     """
 
@@ -612,15 +845,61 @@ def __init__(
         self,
         keys: KeysCollection,
         spatial_size: Union[Sequence[int], int],
-        mode: Union[NumpyPadMode, str] = NumpyPadMode.CONSTANT,
+        mode: NumpyPadModeSequence = NumpyPadMode.CONSTANT,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
-        self.padcropper = ResizeWithPadOrCrop(spatial_size=spatial_size, mode=mode)
+        super().__init__(keys, allow_missing_keys)
+        self.mode = ensure_tuple_rep(mode, len(self.keys))
+        self.padcropper = ResizeWithPadOrCrop(spatial_size=spatial_size)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
-            d[key] = self.padcropper(d[key])
+        for key, m in self.key_iterator(d, self.mode):
+            orig_size = d[key].shape[1:]
+            d[key] = self.padcropper(d[key], mode=m)
+            self.push_transform(
+                d,
+                key,
+                orig_size=orig_size,
+                extra_info={
+                    "mode": m.value if isinstance(m, Enum) else m,
+                },
+            )
+        return d
+
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            orig_size = np.array(transform[InverseKeys.ORIG_SIZE])
+            current_size = np.array(d[key].shape[1:])
+            # Unfortunately, we can't just use ResizeWithPadOrCrop with original size because of odd/even rounding.
+            # Instead, we first pad any smaller dimensions, and then we crop any larger dimensions.
+
+            # First, do pad
+            if np.any((orig_size - current_size) > 0):
+                pad_to_start = np.floor((orig_size - current_size) / 2).astype(int)
+                # in each direction, if original size is even and current size is odd, += 1
+                pad_to_start[np.logical_and(orig_size % 2 == 0, current_size % 2 == 1)] += 1
+                pad_to_start[pad_to_start < 0] = 0
+                pad_to_end = orig_size - current_size - pad_to_start
+                pad_to_end[pad_to_end < 0] = 0
+                pad = list(chain(*zip(pad_to_start.tolist(), pad_to_end.tolist())))
+                d[key] = BorderPad(pad)(d[key])
+
+            # Next crop
+            if np.any((orig_size - current_size) < 0):
+                if self.padcropper.padder.method == Method.SYMMETRIC:
+                    roi_center = [floor(i / 2) if r % 2 == 0 else (i - 1) // 2 for r, i in zip(orig_size, current_size)]
+                else:
+                    roi_center = [floor(r / 2) if r % 2 == 0 else (r - 1) // 2 for r in orig_size]
+
+                d[key] = SpatialCrop(roi_center, orig_size)(d[key])
+
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
         return d
 
 
@@ -634,10 +913,17 @@ class BoundingRectd(MapTransform):
         bbox_key_postfix: the output bounding box coordinates will be
             written to the value of `{key}_{bbox_key_postfix}`.
         select_fn: function to select expected foreground, default is to select values > 0.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
-    def __init__(self, keys: KeysCollection, bbox_key_postfix: str = "bbox", select_fn: Callable = lambda x: x > 0):
-        super().__init__(keys=keys)
+    def __init__(
+        self,
+        keys: KeysCollection,
+        bbox_key_postfix: str = "bbox",
+        select_fn: Callable = lambda x: x > 0,
+        allow_missing_keys: bool = False,
+    ):
+        super().__init__(keys, allow_missing_keys)
         self.bbox = BoundingRect(select_fn=select_fn)
         self.bbox_key_postfix = bbox_key_postfix
 
@@ -646,7 +932,7 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
         See also: :py:class:`monai.transforms.utils.generate_spatial_bounding_box`.
         """
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             bbox = self.bbox(d[key])
             key_to_add = f"{key}_{self.bbox_key_postfix}"
             if key_to_add in d:
diff --git a/monai/transforms/intensity/array.py b/monai/transforms/intensity/array.py
index 87091f6237..62350d4ab0 100644
--- a/monai/transforms/intensity/array.py
+++ b/monai/transforms/intensity/array.py
@@ -14,7 +14,7 @@
 """
 
 from collections.abc import Iterable
-from typing import Any, Optional, Sequence, Tuple, Union
+from typing import Any, List, Optional, Sequence, Tuple, Union
 from warnings import warn
 
 import numpy as np
@@ -22,7 +22,7 @@
 
 from monai.config import DtypeLike
 from monai.networks.layers import GaussianFilter, HilbertTransform, SavitzkyGolayFilter
-from monai.transforms.compose import Randomizable, Transform
+from monai.transforms.transform import RandomizableTransform, Transform
 from monai.transforms.utils import rescale_array
 from monai.utils import PT_BEFORE_1_7, InvalidPyTorchVersionError, dtype_torch_to_numpy, ensure_tuple_size
 
@@ -30,6 +30,9 @@
     "RandGaussianNoise",
     "ShiftIntensity",
     "RandShiftIntensity",
+    "StdShiftIntensity",
+    "RandStdShiftIntensity",
+    "RandBiasField",
     "ScaleIntensity",
     "RandScaleIntensity",
     "NormalizeIntensity",
@@ -49,7 +52,7 @@
 ]
 
 
-class RandGaussianNoise(Randomizable, Transform):
+class RandGaussianNoise(RandomizableTransform):
     """
     Add Gaussian noise to image.
 
@@ -60,14 +63,13 @@ class RandGaussianNoise(Randomizable, Transform):
     """
 
     def __init__(self, prob: float = 0.1, mean: Union[Sequence[float], float] = 0.0, std: float = 0.1) -> None:
-        self.prob = prob
+        RandomizableTransform.__init__(self, prob)
         self.mean = mean
         self.std = std
-        self._do_transform = False
         self._noise = None
 
     def randomize(self, im_shape: Sequence[int]) -> None:
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
         self._noise = self.R.normal(self.mean, self.R.uniform(0, self.std), size=im_shape)
 
     def __call__(self, img: Union[torch.Tensor, np.ndarray]) -> Union[torch.Tensor, np.ndarray]:
@@ -101,7 +103,7 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         return np.asarray((img + self.offset), dtype=img.dtype)
 
 
-class RandShiftIntensity(Randomizable, Transform):
+class RandShiftIntensity(RandomizableTransform):
     """
     Randomly shift intensity with randomly picked offset.
     """
@@ -113,19 +115,18 @@ def __init__(self, offsets: Union[Tuple[float, float], float], prob: float = 0.1
                 if single number, offset value is picked from (-offsets, offsets).
             prob: probability of shift.
         """
+        RandomizableTransform.__init__(self, prob)
         if isinstance(offsets, (int, float)):
             self.offsets = (min(-offsets, offsets), max(-offsets, offsets))
         else:
             if len(offsets) != 2:
                 raise AssertionError("offsets should be a number or pair of numbers.")
             self.offsets = (min(offsets), max(offsets))
-
-        self.prob = prob
-        self._do_transform = False
+        self._offset = self.offsets[0]
 
     def randomize(self, data: Optional[Any] = None) -> None:
         self._offset = self.R.uniform(low=self.offsets[0], high=self.offsets[1])
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
 
     def __call__(self, img: np.ndarray) -> np.ndarray:
         """
@@ -138,6 +139,107 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         return shifter(img)
 
 
+class StdShiftIntensity(Transform):
+    """
+    Shift intensity for the image with a factor and the standard deviation of the image
+    by: ``v = v + factor * std(v)``.
+    This transform can focus on only non-zero values or the entire image,
+    and can also calculate the std on each channel separately.
+
+    Args:
+        factor: factor shift by ``v = v + factor * std(v)``.
+        nonzero: whether only count non-zero values.
+        channel_wise: if True, calculate on each channel separately. Please ensure
+            that the first dimension represents the channel of the image if True.
+        dtype: output data type, defaults to float32.
+    """
+
+    def __init__(
+        self,
+        factor: float,
+        nonzero: bool = False,
+        channel_wise: bool = False,
+        dtype: DtypeLike = np.float32,
+    ) -> None:
+        self.factor = factor
+        self.nonzero = nonzero
+        self.channel_wise = channel_wise
+        self.dtype = dtype
+
+    def _stdshift(self, img: np.ndarray) -> np.ndarray:
+        slices = (img != 0) if self.nonzero else np.ones(img.shape, dtype=bool)
+        if not np.any(slices):
+            return img
+        offset = self.factor * np.std(img[slices])
+        img[slices] = img[slices] + offset
+        return img
+
+    def __call__(self, img: np.ndarray) -> np.ndarray:
+        """
+        Apply the transform to `img`.
+        """
+        img = img.astype(self.dtype)
+        if self.channel_wise:
+            for i, d in enumerate(img):
+                img[i] = self._stdshift(d)
+        else:
+            img = self._stdshift(img)
+        return img
+
+
+class RandStdShiftIntensity(RandomizableTransform):
+    """
+    Shift intensity for the image with a factor and the standard deviation of the image
+    by: ``v = v + factor * std(v)`` where the `factor` is randomly picked.
+    """
+
+    def __init__(
+        self,
+        factors: Union[Tuple[float, float], float],
+        prob: float = 0.1,
+        nonzero: bool = False,
+        channel_wise: bool = False,
+        dtype: DtypeLike = np.float32,
+    ) -> None:
+        """
+        Args:
+            factors: if tuple, the randomly picked range is (min(factors), max(factors)).
+                If single number, the range is (-factors, factors).
+            prob: probability of std shift.
+            nonzero: whether only count non-zero values.
+            channel_wise: if True, calculate on each channel separately.
+            dtype: output data type, defaults to float32.
+
+        """
+        RandomizableTransform.__init__(self, prob)
+        if isinstance(factors, (int, float)):
+            self.factors = (min(-factors, factors), max(-factors, factors))
+        else:
+            if len(factors) != 2:
+                raise AssertionError("factors should be a number or pair of numbers.")
+            self.factors = (min(factors), max(factors))
+        self.factor = self.factors[0]
+        self.nonzero = nonzero
+        self.channel_wise = channel_wise
+        self.dtype = dtype
+
+    def randomize(self, data: Optional[Any] = None) -> None:
+        self.factor = self.R.uniform(low=self.factors[0], high=self.factors[1])
+        super().randomize(None)
+
+    def __call__(self, img: np.ndarray) -> np.ndarray:
+        """
+        Apply the transform to `img`.
+        """
+        self.randomize()
+        if not self._do_transform:
+            return img
+        shifter = StdShiftIntensity(
+            factor=self.factor, nonzero=self.nonzero, channel_wise=self.channel_wise, dtype=self.dtype
+        )
+        return shifter(img)
+
+
 class ScaleIntensity(Transform):
     """
     Scale the intensity of input image to the given value range (minv, maxv).
@@ -151,7 +253,8 @@ def __init__(
         Args:
             minv: minimum value of output data.
             maxv: maximum value of output data.
-            factor: factor scale by ``v = v * (1 + factor)``.
+            factor: factor scale by ``v = v * (1 + factor)``. In order to use
+                this parameter, please set `minv` and `maxv` into None.
         """
         self.minv = minv
         self.maxv = maxv
@@ -172,10 +275,10 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         raise ValueError("Incompatible values: minv=None or maxv=None and factor=None.")
 
 
-class RandScaleIntensity(Randomizable, Transform):
+class RandScaleIntensity(RandomizableTransform):
     """
     Randomly scale the intensity of input image by ``v = v * (1 + factor)`` where the `factor`
-    is randomly picked from (-factors[0], factors[0]).
+    is randomly picked.
     """
 
     def __init__(self, factors: Union[Tuple[float, float], float], prob: float = 0.1) -> None:
@@ -186,19 +289,18 @@ def __init__(self, factors: Union[Tuple[float, float], float], prob: float = 0.1
             prob: probability of scale.
 
         """
+        RandomizableTransform.__init__(self, prob)
         if isinstance(factors, (int, float)):
             self.factors = (min(-factors, factors), max(-factors, factors))
         else:
             if len(factors) != 2:
                 raise AssertionError("factors should be a number or pair of numbers.")
             self.factors = (min(factors), max(factors))
-
-        self.prob = prob
-        self._do_transform = False
+        self.factor = self.factors[0]
 
     def randomize(self, data: Optional[Any] = None) -> None:
         self.factor = self.R.uniform(low=self.factors[0], high=self.factors[1])
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
 
     def __call__(self, img: np.ndarray) -> np.ndarray:
         """
@@ -211,6 +313,97 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         return scaler(img)
 
 
+class RandBiasField(RandomizableTransform):
+    """
+    Random bias field augmentation for MR images.
+    The bias field is considered as a linear combination of smoothly varying basis (polynomial)
+    functions, as described in `Automated Model-Based Tissue Classification of MR Images of the Brain
+    <https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=811270>`_.
+    This implementation adapted from `NiftyNet
+    <https://github.com/NifTK/NiftyNet>`_.
+    Referred to `Longitudinal segmentation of age-related white matter hyperintensities
+    <https://www.sciencedirect.com/science/article/pii/S1361841517300257?via%3Dihub>`_.
+
+    Args:
+        degree: degree of freedom of the polynomials. The value should be no less than 1.
+            Defaults to 3.
+        coeff_range: range of the random coefficients. Defaults to (0.0, 0.1).
+        dtype: output data type, defaults to float32.
+        prob: probability to do random bias field.
+
+    """
+
+    def __init__(
+        self,
+        degree: int = 3,
+        coeff_range: Tuple[float, float] = (0.0, 0.1),
+        dtype: DtypeLike = np.float32,
+        prob: float = 1.0,
+    ) -> None:
+        RandomizableTransform.__init__(self, prob)
+        if degree < 1:
+            raise ValueError("degree should be no less than 1.")
+        self.degree = degree
+        self.coeff_range = coeff_range
+        self.dtype = dtype
+
+    def _generate_random_field(
+        self,
+        spatial_shape: Tuple[int, ...],
+        rank: int,
+        degree: int,
+        coeff: Tuple[int, ...],
+    ):
+        """
+        products of polynomials as bias field estimations
+        """
+        coeff_mat = np.zeros((degree + 1,) * rank)
+        coords = [np.linspace(-1.0, 1.0, dim, dtype=np.float32) for dim in spatial_shape]
+        if rank == 2:
+            coeff_mat[np.tril_indices(degree + 1)] = coeff
+            field = np.polynomial.legendre.leggrid2d(coords[0], coords[1], coeff_mat)
+        elif rank == 3:
+            pts: List[List[int]] = [[0, 0, 0]]
+            for i in range(degree + 1):
+                for j in range(degree + 1 - i):
+                    for k in range(degree + 1 - i - j):
+                        pts.append([i, j, k])
+            if len(pts) > 1:
+                pts = pts[1:]
+            np_pts = np.stack(pts)
+            coeff_mat[np_pts[:, 0], np_pts[:, 1], np_pts[:, 2]] = coeff
+            field = np.polynomial.legendre.leggrid3d(coords[0], coords[1], coords[2], coeff_mat)
+        else:
+            raise NotImplementedError("only supoprts 2D or 3D fields")
+        return field
+
+    def randomize(self, data: np.ndarray) -> None:
+        super().randomize(None)
+        self.spatial_shape = data.shape[1:]
+        self.rank = len(self.spatial_shape)
+        n_coeff = int(np.prod([(self.degree + k) / k for k in range(1, self.rank + 1)]))
+        self._coeff = self.R.uniform(*self.coeff_range, n_coeff)
+
+    def __call__(self, img: np.ndarray):
+        """
+        Apply the transform to `img`.
+        """
+        self.randomize(data=img)
+        if not self._do_transform:
+            return img
+        num_channels = img.shape[0]
+        _bias_fields = np.stack(
+            [
+                self._generate_random_field(
+                    spatial_shape=self.spatial_shape, rank=self.rank, degree=self.degree, coeff=self._coeff
+                )
+                for _ in range(num_channels)
+            ],
+            axis=0,
+        )
+        return (img * _bias_fields).astype(self.dtype)
+
+
 class NormalizeIntensity(Transform):
     """
     Normalize input based on provided args, using calculated mean and std if not provided.
@@ -225,7 +418,7 @@ class NormalizeIntensity(Transform):
         nonzero: whether only normalize non-zero values.
         channel_wise: if using calculated mean and std, calculate on each channel separately
             or calculate on the entire image directly.
-        dtype: output data type, defaut to float32.
+        dtype: output data type, defaults to float32.
     """
 
     def __init__(
@@ -243,7 +436,7 @@ def __init__(
         self.dtype = dtype
 
     def _normalize(self, img: np.ndarray, sub=None, div=None) -> np.ndarray:
-        slices = (img != 0) if self.nonzero else np.ones(img.shape, dtype=np.bool_)
+        slices = (img != 0) if self.nonzero else np.ones(img.shape, dtype=bool)
         if not np.any(slices):
             return img
 
@@ -370,7 +563,7 @@ def __call__(self, img: np.ndarray):
         return np.power(((img - img_min) / float(img_range + epsilon)), self.gamma) * img_range + img_min
 
 
-class RandAdjustContrast(Randomizable, Transform):
+class RandAdjustContrast(RandomizableTransform):
     """
     Randomly changes image intensity by gamma. Each pixel/voxel intensity is updated as::
 
@@ -383,7 +576,7 @@ class RandAdjustContrast(Randomizable, Transform):
     """
 
     def __init__(self, prob: float = 0.1, gamma: Union[Sequence[float], float] = (0.5, 4.5)) -> None:
-        self.prob = prob
+        RandomizableTransform.__init__(self, prob)
 
         if isinstance(gamma, (int, float)):
             if gamma <= 0.5:
@@ -396,11 +589,10 @@ def __init__(self, prob: float = 0.1, gamma: Union[Sequence[float], float] = (0.
                 raise AssertionError("gamma should be a number or pair of numbers.")
             self.gamma = (min(gamma), max(gamma))
 
-        self._do_transform = False
         self.gamma_value = None
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self.gamma_value = self.R.uniform(low=self.gamma[0], high=self.gamma[1])
 
     def __call__(self, img: np.ndarray) -> np.ndarray:
@@ -657,7 +849,7 @@ def __call__(self, img: np.ndarray):
         return gaussian_filter(input_data).squeeze(0).detach().numpy()
 
 
-class RandGaussianSmooth(Randomizable, Transform):
+class RandGaussianSmooth(RandomizableTransform):
     """
     Apply Gaussian smooth to the input data based on randomly selected `sigma` parameters.
 
@@ -679,15 +871,18 @@ def __init__(
         prob: float = 0.1,
         approx: str = "erf",
     ) -> None:
+        RandomizableTransform.__init__(self, prob)
         self.sigma_x = sigma_x
         self.sigma_y = sigma_y
         self.sigma_z = sigma_z
-        self.prob = prob
         self.approx = approx
-        self._do_transform = False
+
+        self.x = self.sigma_x[0]
+        self.y = self.sigma_y[0]
+        self.z = self.sigma_z[0]
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self.x = self.R.uniform(low=self.sigma_x[0], high=self.sigma_x[1])
         self.y = self.R.uniform(low=self.sigma_y[0], high=self.sigma_y[1])
         self.z = self.R.uniform(low=self.sigma_z[0], high=self.sigma_z[1])
@@ -748,7 +943,7 @@ def __call__(self, img: np.ndarray):
         return (blurred_f + self.alpha * (blurred_f - filter_blurred_f)).squeeze(0).detach().numpy()
 
 
-class RandGaussianSharpen(Randomizable, Transform):
+class RandGaussianSharpen(RandomizableTransform):
     """
     Sharpen images using the Gaussian Blur filter based on randomly selected `sigma1`, `sigma2` and `alpha`.
     The algorithm is :py:class:`monai.transforms.GaussianSharpen`.
@@ -782,6 +977,7 @@ def __init__(
         approx: str = "erf",
         prob: float = 0.1,
     ) -> None:
+        RandomizableTransform.__init__(self, prob)
         self.sigma1_x = sigma1_x
         self.sigma1_y = sigma1_y
         self.sigma1_z = sigma1_z
@@ -790,11 +986,9 @@ def __init__(
         self.sigma2_z = sigma2_z
         self.alpha = alpha
         self.approx = approx
-        self.prob = prob
-        self._do_transform = False
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self.x1 = self.R.uniform(low=self.sigma1_x[0], high=self.sigma1_x[1])
         self.y1 = self.R.uniform(low=self.sigma1_y[0], high=self.sigma1_y[1])
         self.z1 = self.R.uniform(low=self.sigma1_z[0], high=self.sigma1_z[1])
@@ -815,7 +1009,7 @@ def __call__(self, img: np.ndarray):
         return GaussianSharpen(sigma1=sigma1, sigma2=sigma2, alpha=self.a, approx=self.approx)(img)
 
 
-class RandHistogramShift(Randomizable, Transform):
+class RandHistogramShift(RandomizableTransform):
     """
     Apply random nonlinear transform to the image's intensity histogram.
 
@@ -827,6 +1021,7 @@ class RandHistogramShift(Randomizable, Transform):
     """
 
     def __init__(self, num_control_points: Union[Tuple[int, int], int] = 10, prob: float = 0.1) -> None:
+        RandomizableTransform.__init__(self, prob)
 
         if isinstance(num_control_points, int):
             if num_control_points <= 2:
@@ -838,11 +1033,9 @@ def __init__(self, num_control_points: Union[Tuple[int, int], int] = 10, prob: f
             if min(num_control_points) <= 2:
                 raise AssertionError("num_control_points should be greater than or equal to 3")
             self.num_control_points = (min(num_control_points), max(num_control_points))
-        self.prob = prob
-        self._do_transform = False
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
         num_control_point = self.R.randint(self.num_control_points[0], self.num_control_points[1] + 1)
         self.reference_control_points = np.linspace(0, 1, num_control_point)
         self.floating_control_points = np.copy(self.reference_control_points)
diff --git a/monai/transforms/intensity/dictionary.py b/monai/transforms/intensity/dictionary.py
index 1c9b31c120..a35e5c8ea6 100644
--- a/monai/transforms/intensity/dictionary.py
+++ b/monai/transforms/intensity/dictionary.py
@@ -22,19 +22,21 @@
 import torch
 
 from monai.config import DtypeLike, KeysCollection
-from monai.transforms.compose import MapTransform, Randomizable
 from monai.transforms.intensity.array import (
     AdjustContrast,
     GaussianSharpen,
     GaussianSmooth,
     MaskIntensity,
     NormalizeIntensity,
+    RandBiasField,
     ScaleIntensity,
     ScaleIntensityRange,
     ScaleIntensityRangePercentiles,
     ShiftIntensity,
+    StdShiftIntensity,
     ThresholdIntensity,
 )
+from monai.transforms.transform import MapTransform, RandomizableTransform
 from monai.utils import dtype_torch_to_numpy, ensure_tuple_rep, ensure_tuple_size
 
 __all__ = [
@@ -43,6 +45,9 @@
     "RandShiftIntensityd",
     "ScaleIntensityd",
     "RandScaleIntensityd",
+    "StdShiftIntensityd",
+    "RandStdShiftIntensityd",
+    "RandBiasFieldd",
     "NormalizeIntensityd",
     "ThresholdIntensityd",
     "ScaleIntensityRanged",
@@ -63,8 +68,14 @@
     "RandShiftIntensityDict",
     "ScaleIntensityD",
     "ScaleIntensityDict",
+    "StdShiftIntensityD",
+    "StdShiftIntensityDict",
     "RandScaleIntensityD",
     "RandScaleIntensityDict",
+    "RandStdShiftIntensityD",
+    "RandStdShiftIntensityDict",
+    "RandBiasFieldD",
+    "RandBiasFieldDict",
     "NormalizeIntensityD",
     "NormalizeIntensityDict",
     "ThresholdIntensityD",
@@ -92,7 +103,7 @@
 ]
 
 
-class RandGaussianNoised(Randomizable, MapTransform):
+class RandGaussianNoised(RandomizableTransform, MapTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.RandGaussianNoise`.
     Add Gaussian noise to image. This transform assumes all the expected fields have same shape.
@@ -103,20 +114,25 @@ class RandGaussianNoised(Randomizable, MapTransform):
         prob: Probability to add Gaussian noise.
         mean: Mean or “centre” of the distribution.
         std: Standard deviation (spread) of distribution.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
-        self, keys: KeysCollection, prob: float = 0.1, mean: Union[Sequence[float], float] = 0.0, std: float = 0.1
+        self,
+        keys: KeysCollection,
+        prob: float = 0.1,
+        mean: Union[Sequence[float], float] = 0.0,
+        std: float = 0.1,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
-        self.prob = prob
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
         self.mean = ensure_tuple_rep(mean, len(self.keys))
         self.std = std
-        self._do_transform = False
         self._noise: List[np.ndarray] = []
 
     def randomize(self, im_shape: Sequence[int]) -> None:
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
         self._noise.clear()
         for m in self.mean:
             self._noise.append(self.R.normal(m, self.R.uniform(0, self.std), size=im_shape))
@@ -130,7 +146,7 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
             raise AssertionError
         if not self._do_transform:
             return d
-        for noise, key in zip(self._noise, self.keys):
+        for key, noise in self.key_iterator(d, self._noise):
             dtype = dtype_torch_to_numpy(d[key].dtype) if isinstance(d[key], torch.Tensor) else d[key].dtype
             d[key] = d[key] + noise.astype(dtype)
         return d
@@ -141,29 +157,36 @@ class ShiftIntensityd(MapTransform):
     Dictionary-based wrapper of :py:class:`monai.transforms.ShiftIntensity`.
     """
 
-    def __init__(self, keys: KeysCollection, offset: float) -> None:
+    def __init__(self, keys: KeysCollection, offset: float, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             offset: offset value to shift the intensity of image.
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.shifter = ShiftIntensity(offset)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.shifter(d[key])
         return d
 
 
-class RandShiftIntensityd(Randomizable, MapTransform):
+class RandShiftIntensityd(RandomizableTransform, MapTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.RandShiftIntensity`.
     """
 
-    def __init__(self, keys: KeysCollection, offsets: Union[Tuple[float, float], float], prob: float = 0.1) -> None:
+    def __init__(
+        self,
+        keys: KeysCollection,
+        offsets: Union[Tuple[float, float], float],
+        prob: float = 0.1,
+        allow_missing_keys: bool = False,
+    ) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
@@ -172,8 +195,10 @@ def __init__(self, keys: KeysCollection, offsets: Union[Tuple[float, float], flo
                 if single number, offset value is picked from (-offsets, offsets).
             prob: probability of rotating.
                 (Default 0.1, with 10% probability it returns a rotated array.)
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
 
         if isinstance(offsets, (int, float)):
             self.offsets = (min(-offsets, offsets), max(-offsets, offsets))
@@ -181,13 +206,11 @@ def __init__(self, keys: KeysCollection, offsets: Union[Tuple[float, float], flo
             if len(offsets) != 2:
                 raise AssertionError("offsets should be a number or pair of numbers.")
             self.offsets = (min(offsets), max(offsets))
-
-        self.prob = prob
-        self._do_transform = False
+        self._offset = self.offsets[0]
 
     def randomize(self, data: Optional[Any] = None) -> None:
         self._offset = self.R.uniform(low=self.offsets[0], high=self.offsets[1])
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
@@ -195,7 +218,98 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
         if not self._do_transform:
             return d
         shifter = ShiftIntensity(self._offset)
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            d[key] = shifter(d[key])
+        return d
+
+
+class StdShiftIntensityd(MapTransform):
+    """
+    Dictionary-based wrapper of :py:class:`monai.transforms.StdShiftIntensity`.
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        factor: float,
+        nonzero: bool = False,
+        channel_wise: bool = False,
+        dtype: DtypeLike = np.float32,
+        allow_missing_keys: bool = False,
+    ) -> None:
+        """
+        Args:
+            keys: keys of the corresponding items to be transformed.
+                See also: :py:class:`monai.transforms.compose.MapTransform`
+            factor: factor shift by ``v = v + factor * std(v)``.
+            nonzero: whether only count non-zero values.
+            channel_wise: if True, calculate on each channel separately. Please ensure
+                that the first dimension represents the channel of the image if True.
+            dtype: output data type, defaults to float32.
+            allow_missing_keys: don't raise exception if key is missing.
+        """
+        super().__init__(keys, allow_missing_keys)
+        self.shifter = StdShiftIntensity(factor, nonzero, channel_wise, dtype)
+
+    def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = dict(data)
+        for key in self.key_iterator(d):
+            d[key] = self.shifter(d[key])
+        return d
+
+
+class RandStdShiftIntensityd(RandomizableTransform, MapTransform):
+    """
+    Dictionary-based version :py:class:`monai.transforms.RandStdShiftIntensity`.
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        factors: Union[Tuple[float, float], float],
+        prob: float = 0.1,
+        nonzero: bool = False,
+        channel_wise: bool = False,
+        dtype: DtypeLike = np.float32,
+        allow_missing_keys: bool = False,
+    ) -> None:
+        """
+        Args:
+            keys: keys of the corresponding items to be transformed.
+                See also: :py:class:`monai.transforms.compose.MapTransform`
+            factors: if tuple, the randomly picked range is (min(factors), max(factors)).
+                If single number, the range is (-factors, factors).
+            prob: probability of std shift.
+            nonzero: whether only count non-zero values.
+            channel_wise: if True, calculate on each channel separately.
+            dtype: output data type, defaults to float32.
+            allow_missing_keys: don't raise exception if key is missing.
+        """
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
+
+        if isinstance(factors, (int, float)):
+            self.factors = (min(-factors, factors), max(-factors, factors))
+        else:
+            if len(factors) != 2:
+                raise AssertionError("factors should be a number or pair of numbers.")
+            self.factors = (min(factors), max(factors))
+        self.factor = self.factors[0]
+        self.nonzero = nonzero
+        self.channel_wise = channel_wise
+        self.dtype = dtype
+
+    def randomize(self, data: Optional[Any] = None) -> None:
+        self.factor = self.R.uniform(low=self.factors[0], high=self.factors[1])
+        super().randomize(None)
+
+    def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = dict(data)
+        self.randomize()
+        if not self._do_transform:
+            return d
+        shifter = StdShiftIntensity(self.factor, self.nonzero, self.channel_wise, self.dtype)
+        for key in self.key_iterator(d):
             d[key] = shifter(d[key])
         return d
 
@@ -208,7 +322,12 @@ class ScaleIntensityd(MapTransform):
     """
 
     def __init__(
-        self, keys: KeysCollection, minv: float = 0.0, maxv: float = 1.0, factor: Optional[float] = None
+        self,
+        keys: KeysCollection,
+        minv: Optional[float] = 0.0,
+        maxv: Optional[float] = 1.0,
+        factor: Optional[float] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -216,25 +335,33 @@ def __init__(
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             minv: minimum value of output data.
             maxv: maximum value of output data.
-            factor: factor scale by ``v = v * (1 + factor)``.
+            factor: factor scale by ``v = v * (1 + factor)``. In order to use
+                this parameter, please set `minv` and `maxv` into None.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.scaler = ScaleIntensity(minv, maxv, factor)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.scaler(d[key])
         return d
 
 
-class RandScaleIntensityd(Randomizable, MapTransform):
+class RandScaleIntensityd(RandomizableTransform, MapTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.RandScaleIntensity`.
     """
 
-    def __init__(self, keys: KeysCollection, factors: Union[Tuple[float, float], float], prob: float = 0.1) -> None:
+    def __init__(
+        self,
+        keys: KeysCollection,
+        factors: Union[Tuple[float, float], float],
+        prob: float = 0.1,
+        allow_missing_keys: bool = False,
+    ) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
@@ -243,9 +370,11 @@ def __init__(self, keys: KeysCollection, factors: Union[Tuple[float, float], flo
                 if single number, factor value is picked from (-factors, factors).
             prob: probability of rotating.
                 (Default 0.1, with 10% probability it returns a rotated array.)
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
 
         if isinstance(factors, (int, float)):
             self.factors = (min(-factors, factors), max(-factors, factors))
@@ -253,13 +382,11 @@ def __init__(self, keys: KeysCollection, factors: Union[Tuple[float, float], flo
             if len(factors) != 2:
                 raise AssertionError("factors should be a number or pair of numbers.")
             self.factors = (min(factors), max(factors))
-
-        self.prob = prob
-        self._do_transform = False
+        self.factor = self.factors[0]
 
     def randomize(self, data: Optional[Any] = None) -> None:
         self.factor = self.R.uniform(low=self.factors[0], high=self.factors[1])
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
@@ -267,11 +394,55 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
         if not self._do_transform:
             return d
         scaler = ScaleIntensity(minv=None, maxv=None, factor=self.factor)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = scaler(d[key])
         return d
 
 
+class RandBiasFieldd(RandomizableTransform, MapTransform):
+    """
+    Dictionary-based version :py:class:`monai.transforms.RandBiasField`.
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        degree: int = 3,
+        coeff_range: Tuple[float, float] = (0.0, 0.1),
+        dtype: DtypeLike = np.float32,
+        prob: float = 1.0,
+        allow_missing_keys: bool = False,
+    ) -> None:
+        """
+        Args:
+            keys: keys of the corresponding items to be transformed.
+                See also: :py:class:`monai.transforms.compose.MapTransform`
+            degree: degree of freedom of the polynomials. The value should be no less than 1.
+                Defaults to 3.
+            coeff_range: range of the random coefficients. Defaults to (0.0, 0.1).
+            dtype: output data type, defaults to float32.
+            prob: probability to do random bias field.
+            allow_missing_keys: don't raise exception if key is missing.
+
+        """
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
+
+        self.rand_bias_field = RandBiasField(degree, coeff_range, dtype, prob)
+
+    def randomize(self, data: Optional[Any] = None) -> None:
+        super().randomize(None)
+
+    def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = dict(data)
+        self.randomize()
+        if not self._do_transform:
+            return d
+        for key in self.key_iterator(d):
+            d[key] = self.rand_bias_field(d[key])
+        return d
+
+
 class NormalizeIntensityd(MapTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.NormalizeIntensity`.
@@ -286,7 +457,8 @@ class NormalizeIntensityd(MapTransform):
         nonzero: whether only normalize non-zero values.
         channel_wise: if using calculated mean and std, calculate on each channel separately
             or calculate on the entire image directly.
-        dtype: output data type, defaut to float32.
+        dtype: output data type, defaults to float32.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -297,13 +469,14 @@ def __init__(
         nonzero: bool = False,
         channel_wise: bool = False,
         dtype: DtypeLike = np.float32,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.normalizer = NormalizeIntensity(subtrahend, divisor, nonzero, channel_wise, dtype)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.normalizer(d[key])
         return d
 
@@ -318,15 +491,23 @@ class ThresholdIntensityd(MapTransform):
         threshold: the threshold to filter intensity values.
         above: filter values above the threshold or below the threshold, default is True.
         cval: value to fill the remaining parts of the image, default is 0.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
-    def __init__(self, keys: KeysCollection, threshold: float, above: bool = True, cval: float = 0.0) -> None:
-        super().__init__(keys)
+    def __init__(
+        self,
+        keys: KeysCollection,
+        threshold: float,
+        above: bool = True,
+        cval: float = 0.0,
+        allow_missing_keys: bool = False,
+    ) -> None:
+        super().__init__(keys, allow_missing_keys)
         self.filter = ThresholdIntensity(threshold, above, cval)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.filter(d[key])
         return d
 
@@ -343,17 +524,25 @@ class ScaleIntensityRanged(MapTransform):
         b_min: intensity target range min.
         b_max: intensity target range max.
         clip: whether to perform clip after scaling.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
-        self, keys: KeysCollection, a_min: float, a_max: float, b_min: float, b_max: float, clip: bool = False
+        self,
+        keys: KeysCollection,
+        a_min: float,
+        a_max: float,
+        b_min: float,
+        b_max: float,
+        clip: bool = False,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.scaler = ScaleIntensityRange(a_min, a_max, b_min, b_max, clip)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.scaler(d[key])
         return d
 
@@ -369,20 +558,21 @@ class AdjustContrastd(MapTransform):
         keys: keys of the corresponding items to be transformed.
             See also: monai.transforms.MapTransform
         gamma: gamma value to adjust the contrast as function.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
-    def __init__(self, keys: KeysCollection, gamma: float) -> None:
-        super().__init__(keys)
+    def __init__(self, keys: KeysCollection, gamma: float, allow_missing_keys: bool = False) -> None:
+        super().__init__(keys, allow_missing_keys)
         self.adjuster = AdjustContrast(gamma)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.adjuster(d[key])
         return d
 
 
-class RandAdjustContrastd(Randomizable, MapTransform):
+class RandAdjustContrastd(RandomizableTransform, MapTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.RandAdjustContrast`.
     Randomly changes image intensity by gamma. Each pixel/voxel intensity is updated as:
@@ -395,13 +585,18 @@ class RandAdjustContrastd(Randomizable, MapTransform):
         prob: Probability of adjustment.
         gamma: Range of gamma values.
             If single number, value is picked from (0.5, gamma), default is (0.5, 4.5).
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
-        self, keys: KeysCollection, prob: float = 0.1, gamma: Union[Tuple[float, float], float] = (0.5, 4.5)
+        self,
+        keys: KeysCollection,
+        prob: float = 0.1,
+        gamma: Union[Tuple[float, float], float] = (0.5, 4.5),
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
-        self.prob: float = prob
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
 
         if isinstance(gamma, (int, float)):
             if gamma <= 0.5:
@@ -414,11 +609,10 @@ def __init__(
                 raise AssertionError("gamma should be a number or pair of numbers.")
             self.gamma = (min(gamma), max(gamma))
 
-        self._do_transform = False
         self.gamma_value: Optional[float] = None
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self.gamma_value = self.R.uniform(low=self.gamma[0], high=self.gamma[1])
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
@@ -429,7 +623,7 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
         if not self._do_transform:
             return d
         adjuster = AdjustContrast(self.gamma_value)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = adjuster(d[key])
         return d
 
@@ -447,6 +641,7 @@ class ScaleIntensityRangePercentilesd(MapTransform):
         b_max: intensity target range max.
         clip: whether to perform clip after scaling.
         relative: whether to scale to the corresponding percentiles of [b_min, b_max]
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -458,13 +653,14 @@ def __init__(
         b_max: float,
         clip: bool = False,
         relative: bool = False,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.scaler = ScaleIntensityRangePercentiles(lower, upper, b_min, b_max, clip, relative)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.scaler(d[key])
         return d
 
@@ -483,6 +679,7 @@ class MaskIntensityd(MapTransform):
             if None, will extract the mask data from input data based on `mask_key`.
         mask_key: the key to extract mask data from input dictionary, only works
             when `mask_data` is None.
+        allow_missing_keys: don't raise exception if key is missing.
 
     """
 
@@ -491,14 +688,15 @@ def __init__(
         keys: KeysCollection,
         mask_data: Optional[np.ndarray] = None,
         mask_key: Optional[str] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = MaskIntensity(mask_data)
         self.mask_key = mask_key if mask_data is None else None
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key], d[self.mask_key]) if self.mask_key is not None else self.converter(d[key])
         return d
 
@@ -515,21 +713,28 @@ class GaussianSmoothd(MapTransform):
             use it for all spatial dimensions.
         approx: discrete Gaussian kernel type, available options are "erf", "sampled", and "scalespace".
             see also :py:meth:`monai.networks.layers.GaussianFilter`.
+        allow_missing_keys: don't raise exception if key is missing.
 
     """
 
-    def __init__(self, keys: KeysCollection, sigma: Union[Sequence[float], float], approx: str = "erf") -> None:
-        super().__init__(keys)
+    def __init__(
+        self,
+        keys: KeysCollection,
+        sigma: Union[Sequence[float], float],
+        approx: str = "erf",
+        allow_missing_keys: bool = False,
+    ) -> None:
+        super().__init__(keys, allow_missing_keys)
         self.converter = GaussianSmooth(sigma, approx=approx)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
         return d
 
 
-class RandGaussianSmoothd(Randomizable, MapTransform):
+class RandGaussianSmoothd(RandomizableTransform, MapTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.GaussianSmooth`.
 
@@ -542,6 +747,7 @@ class RandGaussianSmoothd(Randomizable, MapTransform):
         approx: discrete Gaussian kernel type, available options are "erf", "sampled", and "scalespace".
             see also :py:meth:`monai.networks.layers.GaussianFilter`.
         prob: probability of Gaussian smooth.
+        allow_missing_keys: don't raise exception if key is missing.
 
     """
 
@@ -553,17 +759,17 @@ def __init__(
         sigma_z: Tuple[float, float] = (0.25, 1.5),
         approx: str = "erf",
         prob: float = 0.1,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
-        self.sigma_x = sigma_x
-        self.sigma_y = sigma_y
-        self.sigma_z = sigma_z
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
+        self.sigma_x, self.sigma_y, self.sigma_z = sigma_x, sigma_y, sigma_z
         self.approx = approx
-        self.prob = prob
-        self._do_transform = False
+
+        self.x, self.y, self.z = self.sigma_x[0], self.sigma_y[0], self.sigma_z[0]
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self.x = self.R.uniform(low=self.sigma_x[0], high=self.sigma_x[1])
         self.y = self.R.uniform(low=self.sigma_y[0], high=self.sigma_y[1])
         self.z = self.R.uniform(low=self.sigma_z[0], high=self.sigma_z[1])
@@ -573,7 +779,7 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
         self.randomize()
         if not self._do_transform:
             return d
-        for key in self.keys:
+        for key in self.key_iterator(d):
             sigma = ensure_tuple_size(tup=(self.x, self.y, self.z), dim=d[key].ndim - 1)
             d[key] = GaussianSmooth(sigma=sigma, approx=self.approx)(d[key])
         return d
@@ -595,6 +801,7 @@ class GaussianSharpend(MapTransform):
         alpha: weight parameter to compute the final result.
         approx: discrete Gaussian kernel type, available options are "erf", "sampled", and "scalespace".
             see also :py:meth:`monai.networks.layers.GaussianFilter`.
+        allow_missing_keys: don't raise exception if key is missing.
 
     """
 
@@ -605,18 +812,19 @@ def __init__(
         sigma2: Union[Sequence[float], float] = 1.0,
         alpha: float = 30.0,
         approx: str = "erf",
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = GaussianSharpen(sigma1, sigma2, alpha, approx=approx)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
         return d
 
 
-class RandGaussianSharpend(Randomizable, MapTransform):
+class RandGaussianSharpend(RandomizableTransform, MapTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.GaussianSharpen`.
 
@@ -636,6 +844,7 @@ class RandGaussianSharpend(Randomizable, MapTransform):
         approx: discrete Gaussian kernel type, available options are "erf", "sampled", and "scalespace".
             see also :py:meth:`monai.networks.layers.GaussianFilter`.
         prob: probability of Gaussian sharpen.
+        allow_missing_keys: don't raise exception if key is missing.
 
     """
 
@@ -651,8 +860,10 @@ def __init__(
         alpha: Tuple[float, float] = (10.0, 30.0),
         approx: str = "erf",
         prob: float = 0.1,
+        allow_missing_keys: bool = False,
     ):
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
         self.sigma1_x = sigma1_x
         self.sigma1_y = sigma1_y
         self.sigma1_z = sigma1_z
@@ -661,11 +872,9 @@ def __init__(
         self.sigma2_z = sigma2_z
         self.alpha = alpha
         self.approx = approx
-        self.prob = prob
-        self._do_transform = False
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self.x1 = self.R.uniform(low=self.sigma1_x[0], high=self.sigma1_x[1])
         self.y1 = self.R.uniform(low=self.sigma1_y[0], high=self.sigma1_y[1])
         self.z1 = self.R.uniform(low=self.sigma1_z[0], high=self.sigma1_z[1])
@@ -682,14 +891,14 @@ def __call__(self, data):
         self.randomize()
         if not self._do_transform:
             return d
-        for key in self.keys:
+        for key in self.key_iterator(d):
             sigma1 = ensure_tuple_size(tup=(self.x1, self.y1, self.z1), dim=d[key].ndim - 1)
             sigma2 = ensure_tuple_size(tup=(self.x2, self.y2, self.z2), dim=d[key].ndim - 1)
             d[key] = GaussianSharpen(sigma1=sigma1, sigma2=sigma2, alpha=self.a, approx=self.approx)(d[key])
         return d
 
 
-class RandHistogramShiftd(Randomizable, MapTransform):
+class RandHistogramShiftd(RandomizableTransform, MapTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.RandHistogramShift`.
     Apply random nonlinear transform the the image's intensity histogram.
@@ -701,12 +910,18 @@ class RandHistogramShiftd(Randomizable, MapTransform):
             a smaller number of control points allows for larger intensity shifts. if two values provided, number of
             control points selecting from range (min_value, max_value).
         prob: probability of histogram shift.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
-        self, keys: KeysCollection, num_control_points: Union[Tuple[int, int], int] = 10, prob: float = 0.1
+        self,
+        keys: KeysCollection,
+        num_control_points: Union[Tuple[int, int], int] = 10,
+        prob: float = 0.1,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
         if isinstance(num_control_points, int):
             if num_control_points <= 2:
                 raise AssertionError("num_control_points should be greater than or equal to 3")
@@ -717,11 +932,9 @@ def __init__(
             if min(num_control_points) <= 2:
                 raise AssertionError("num_control_points should be greater than or equal to 3")
             self.num_control_points = (min(num_control_points), max(num_control_points))
-        self.prob = prob
-        self._do_transform = False
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
         num_control_point = self.R.randint(self.num_control_points[0], self.num_control_points[1] + 1)
         self.reference_control_points = np.linspace(0, 1, num_control_point)
         self.floating_control_points = np.copy(self.reference_control_points)
@@ -735,7 +948,7 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
         self.randomize()
         if not self._do_transform:
             return d
-        for key in self.keys:
+        for key in self.key_iterator(d):
             img_min, img_max = d[key].min(), d[key].max()
             reference_control_points_scaled = self.reference_control_points * (img_max - img_min) + img_min
             floating_control_points_scaled = self.floating_control_points * (img_max - img_min) + img_min
@@ -747,6 +960,9 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
 RandGaussianNoiseD = RandGaussianNoiseDict = RandGaussianNoised
 ShiftIntensityD = ShiftIntensityDict = ShiftIntensityd
 RandShiftIntensityD = RandShiftIntensityDict = RandShiftIntensityd
+StdShiftIntensityD = StdShiftIntensityDict = StdShiftIntensityd
+RandStdShiftIntensityD = RandStdShiftIntensityDict = RandStdShiftIntensityd
+RandBiasFieldD = RandBiasFieldDict = RandBiasFieldd
 ScaleIntensityD = ScaleIntensityDict = ScaleIntensityd
 RandScaleIntensityD = RandScaleIntensityDict = RandScaleIntensityd
 NormalizeIntensityD = NormalizeIntensityDict = NormalizeIntensityd
diff --git a/monai/transforms/inverse.py b/monai/transforms/inverse.py
new file mode 100644
index 0000000000..3baef91717
--- /dev/null
+++ b/monai/transforms/inverse.py
@@ -0,0 +1,121 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, Hashable, Optional, Tuple
+
+import numpy as np
+import torch
+
+from monai.transforms.transform import RandomizableTransform, Transform
+from monai.utils.enums import InverseKeys
+
+__all__ = ["InvertibleTransform"]
+
+
+class InvertibleTransform(Transform):
+    """Classes for invertible transforms.
+
+    This class exists so that an ``invert`` method can be implemented. This allows, for
+    example, images to be cropped, rotated, padded, etc., during training and inference,
+    and after be returned to their original size before saving to file for comparison in
+    an external viewer.
+
+    When the ``__call__`` method is called, the transformation information for each key is
+    stored. If the transforms were applied to keys "image" and "label", there will be two
+    extra keys in the dictionary: "image_transforms" and "label_transforms". Each list
+    contains a list of the transforms applied to that key. When the ``inverse`` method is
+    called, the inverse is called on each key individually, which allows for different
+    parameters being passed to each label (e.g., different interpolation for image and
+    label).
+
+    When the ``inverse`` method is called, the inverse transforms are applied in a last-
+    in-first-out order. As the inverse is applied, its entry is removed from the list
+    detailing the applied transformations. That is to say that during the forward pass,
+    the list of applied transforms grows, and then during the inverse it shrinks back
+    down to an empty list.
+
+    The information in ``data[key_transform]`` will be compatible with the default collate
+    since it only stores strings, numbers and arrays.
+
+    We currently check that the ``id()`` of the transform is the same in the forward and
+    inverse directions. This is a useful check to ensure that the inverses are being
+    processed in the correct order. However, this may cause issues if the ``id()`` of the
+    object changes (such as multiprocessing on Windows). If you feel this issue affects
+    you, please raise a GitHub issue.
+
+    Note to developers: When converting a transform to an invertible transform, you need to:
+
+        #. Inherit from this class.
+        #. In ``__call__``, add a call to ``push_transform``.
+        #. Any extra information that might be needed for the inverse can be included with the
+           dictionary ``extra_info``. This dictionary should have the same keys regardless of
+           whether ``do_transform`` was `True` or `False` and can only contain objects that are
+           accepted in pytorch data loader's collate function (e.g., `None` is not allowed).
+        #. Implement an ``inverse`` method. Make sure that after performing the inverse,
+           ``pop_transform`` is called.
+
+    """
+
+    def push_transform(
+        self,
+        data: dict,
+        key: Hashable,
+        extra_info: Optional[dict] = None,
+        orig_size: Optional[Tuple] = None,
+    ) -> None:
+        """Append to list of applied transforms for that key."""
+        key_transform = str(key) + InverseKeys.KEY_SUFFIX
+        info = {
+            InverseKeys.CLASS_NAME: self.__class__.__name__,
+            InverseKeys.ID: id(self),
+            InverseKeys.ORIG_SIZE: orig_size or (data[key].shape[1:] if hasattr(data[key], "shape") else None),
+        }
+        if extra_info is not None:
+            info[InverseKeys.EXTRA_INFO] = extra_info
+        # If class is randomizable transform, store whether the transform was actually performed (based on `prob`)
+        if isinstance(self, RandomizableTransform):
+            info[InverseKeys.DO_TRANSFORM] = self._do_transform
+        # If this is the first, create list
+        if key_transform not in data:
+            data[key_transform] = []
+        data[key_transform].append(info)
+
+    def check_transforms_match(self, transform: dict) -> None:
+        """Check transforms are of same instance."""
+        if transform[InverseKeys.ID] == id(self):
+            return
+        # basic check if multiprocessing uses 'spawn' (objects get recreated so don't have same ID)
+        if (
+            torch.multiprocessing.get_start_method(allow_none=False) == "spawn"
+            and transform[InverseKeys.CLASS_NAME] == self.__class__.__name__
+        ):
+            return
+        raise RuntimeError("Should inverse most recently applied invertible transform first")
+
+    def get_most_recent_transform(self, data: dict, key: Hashable) -> dict:
+        """Get most recent transform."""
+        transform = dict(data[str(key) + InverseKeys.KEY_SUFFIX][-1])
+        self.check_transforms_match(transform)
+        return transform
+
+    def pop_transform(self, data: dict, key: Hashable) -> None:
+        """Remove most recent transform."""
+        data[str(key) + InverseKeys.KEY_SUFFIX].pop()
+
+    def inverse(self, data: dict) -> Dict[Hashable, np.ndarray]:
+        """
+        Inverse of ``__call__``.
+
+        Raises:
+            NotImplementedError: When the subclass does not override this method.
+
+        """
+        raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.")
diff --git a/monai/transforms/io/array.py b/monai/transforms/io/array.py
index 9c14f7a689..7a7fcb8cda 100644
--- a/monai/transforms/io/array.py
+++ b/monai/transforms/io/array.py
@@ -22,7 +22,7 @@
 from monai.data.image_reader import ImageReader, ITKReader, NibabelReader, NumpyReader, PILReader
 from monai.data.nifti_saver import NiftiSaver
 from monai.data.png_saver import PNGSaver
-from monai.transforms.compose import Transform
+from monai.transforms.transform import Transform
 from monai.utils import GridSampleMode, GridSamplePadMode
 from monai.utils import ImageMetaKey as Key
 from monai.utils import InterpolateMode, ensure_tuple, optional_import
@@ -33,6 +33,27 @@
 __all__ = ["LoadImage", "SaveImage"]
 
 
+def switch_endianness(data, old, new):
+    """
+    If any numpy arrays have `old` (e.g., ">"),
+    replace with `new` (e.g., "<").
+    """
+    if isinstance(data, np.ndarray):
+        if data.dtype.byteorder == old:
+            data = data.newbyteorder(new)
+    elif isinstance(data, tuple):
+        data = tuple(switch_endianness(x, old, new) for x in data)
+    elif isinstance(data, list):
+        data = [switch_endianness(x, old, new) for x in data]
+    elif isinstance(data, dict):
+        data = {k: switch_endianness(v, old, new) for k, v in data.items()}
+    elif isinstance(data, (bool, str, float, int, type(None))):
+        pass
+    else:
+        raise AssertionError(f"Unknown type: {type(data).__name__}")
+    return data
+
+
 class LoadImage(Transform):
     """
     Load image file or files from provided path based on reader.
@@ -57,7 +78,7 @@ def __init__(
             reader: register reader to load image file and meta data, if None, still can register readers
                 at runtime or use the default readers. If a string of reader name provided, will construct
                 a reader object with the `*args` and `**kwargs` parameters, supported reader name: "NibabelReader",
-                "PILReader", "ITKReader", "NumpyReader"
+                "PILReader", "ITKReader", "NumpyReader".
             image_only: if True return only the image volume, otherwise return image data array and header dict.
             dtype: if not None convert the loaded image to this data type.
             args: additional parameters for reader if providing a reader name.
@@ -123,7 +144,12 @@ def __call__(
                     break
 
         if reader is None:
-            raise RuntimeError(f"can not find suitable reader for this file: {filename}.")
+            raise RuntimeError(
+                f"can not find suitable reader for this file: {filename}. \
+                Please install dependency libraries: (nii, nii.gz) -> Nibabel, (png, jpg, bmp) -> PIL, \
+                (npz, npy) -> Numpy, others -> ITK. Refer to the installation instruction: \
+                https://docs.monai.io/en/latest/installation.html#installing-the-recommended-dependencies."
+            )
 
         img = reader.read(filename)
         img_array, meta_data = reader.get_data(img)
@@ -132,6 +158,9 @@ def __call__(
         if self.image_only:
             return img_array
         meta_data[Key.FILENAME_OR_OBJ] = ensure_tuple(filename)[0]
+        # make sure all elements in metadata are little endian
+        meta_data = switch_endianness(meta_data, ">", "<")
+
         return img_array, meta_data
 
 
@@ -141,6 +170,8 @@ class SaveImage(Transform):
     It can work for both numpy array and PyTorch Tensor in both pre-transform chain
     and post transform chain.
 
+    NB: image should include channel dimension: [B],C,H,W,[D].
+
     Args:
         output_dir: output image directory.
         output_postfix: a string appended to all output file names, default to `trans`.
@@ -176,6 +207,21 @@ class SaveImage(Transform):
             it's used for NIfTI format only.
         save_batch: whether the import image is a batch data, default to `False`.
             usually pre-transforms run for channel first data, while post-transforms run for batch data.
+        squeeze_end_dims: if True, any trailing singleton dimensions will be removed (after the channel
+            has been moved to the end). So if input is (C,H,W,D), this will be altered to (H,W,D,C), and
+            then if C==1, it will be saved as (H,W,D). If D also ==1, it will be saved as (H,W). If false,
+            image will always be saved as (H,W,D,C).
+            it's used for NIfTI format only.
+        data_root_dir: if not empty, it specifies the beginning parts of the input file's
+            absolute path. it's used to compute `input_file_rel_path`, the relative path to the file from
+            `data_root_dir` to preserve folder structure when saving in case there are files in different
+            folders with the same file names. for example:
+            input_file_name: /foo/bar/test1/image.nii,
+            output_postfix: seg
+            output_ext: nii.gz
+            output_dir: /output,
+            data_root_dir: /foo/bar,
+            output will be: /output/test1/image/image_seg.nii.gz
 
     """
 
@@ -191,6 +237,8 @@ def __init__(
         dtype: DtypeLike = np.float64,
         output_dtype: DtypeLike = np.float32,
         save_batch: bool = False,
+        squeeze_end_dims: bool = True,
+        data_root_dir: str = "",
     ) -> None:
         self.saver: Union[NiftiSaver, PNGSaver]
         if output_ext in (".nii.gz", ".nii"):
@@ -203,6 +251,8 @@ def __init__(
                 padding_mode=padding_mode,
                 dtype=dtype,
                 output_dtype=output_dtype,
+                squeeze_end_dims=squeeze_end_dims,
+                data_root_dir=data_root_dir,
             )
         elif output_ext == ".png":
             self.saver = PNGSaver(
@@ -212,6 +262,7 @@ def __init__(
                 resample=resample,
                 mode=InterpolateMode(mode),
                 scale=scale,
+                data_root_dir=data_root_dir,
             )
         else:
             raise ValueError(f"unsupported output extension: {output_ext}.")
@@ -219,6 +270,12 @@ def __init__(
         self.save_batch = save_batch
 
     def __call__(self, img: Union[torch.Tensor, np.ndarray], meta_data: Optional[Dict] = None):
+        """
+        Args:
+            img: target data content that save into file.
+            meta_data: key-value pairs of meta_data corresponding to the data.
+
+        """
         if self.save_batch:
             self.saver.save_batch(img, meta_data)
         else:
diff --git a/monai/transforms/io/dictionary.py b/monai/transforms/io/dictionary.py
index d3220aa682..413f83b62d 100644
--- a/monai/transforms/io/dictionary.py
+++ b/monai/transforms/io/dictionary.py
@@ -21,8 +21,8 @@
 
 from monai.config import DtypeLike, KeysCollection
 from monai.data.image_reader import ImageReader
-from monai.transforms.compose import MapTransform
 from monai.transforms.io.array import LoadImage, SaveImage
+from monai.transforms.transform import MapTransform
 from monai.utils import GridSampleMode, GridSamplePadMode, InterpolateMode
 
 __all__ = [
@@ -59,6 +59,8 @@ def __init__(
         dtype: DtypeLike = np.float32,
         meta_key_postfix: str = "meta_dict",
         overwriting: bool = False,
+        image_only: bool = False,
+        allow_missing_keys: bool = False,
         *args,
         **kwargs,
     ) -> None:
@@ -69,18 +71,21 @@ def __init__(
             reader: register reader to load image file and meta data, if None, still can register readers
                 at runtime or use the default readers. If a string of reader name provided, will construct
                 a reader object with the `*args` and `**kwargs` parameters, supported reader name: "NibabelReader",
-                "PILReader", "ITKReader", "NumpyReader"
+                "PILReader", "ITKReader", "NumpyReader".
             dtype: if not None convert the loaded image data to this data type.
             meta_key_postfix: use `key_{postfix}` to store the metadata of the nifti image,
                 default is `meta_dict`. The meta data is a dictionary object.
                 For example, load nifti file for `image`, store the metadata into `image_meta_dict`.
             overwriting: whether allow to overwrite existing meta data of same key.
                 default is False, which will raise exception if encountering existing key.
+            image_only: if True return dictionary containing just only the image volumes, otherwise return
+                dictionary containing image data array and header dict per input key.
+            allow_missing_keys: don't raise exception if key is missing.
             args: additional parameters for reader if providing a reader name.
             kwargs: additional parameters for reader if providing a reader name.
         """
-        super().__init__(keys)
-        self._loader = LoadImage(reader, False, dtype, *args, **kwargs)
+        super().__init__(keys, allow_missing_keys)
+        self._loader = LoadImage(reader, image_only, dtype, *args, **kwargs)
         if not isinstance(meta_key_postfix, str):
             raise TypeError(f"meta_key_postfix must be a str but is {type(meta_key_postfix).__name__}.")
         self.meta_key_postfix = meta_key_postfix
@@ -96,17 +101,22 @@ def __call__(self, data, reader: Optional[ImageReader] = None):
 
         """
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             data = self._loader(d[key], reader)
-            if not isinstance(data, (tuple, list)):
-                raise ValueError("loader must return a tuple or list.")
-            d[key] = data[0]
-            if not isinstance(data[1], dict):
-                raise ValueError("metadata must be a dict.")
-            key_to_add = f"{key}_{self.meta_key_postfix}"
-            if key_to_add in d and not self.overwriting:
-                raise KeyError(f"Meta data with key {key_to_add} already exists and overwriting=False.")
-            d[key_to_add] = data[1]
+            if self._loader.image_only:
+                if not isinstance(data, np.ndarray):
+                    raise ValueError("loader must return a numpy array (because image_only=True was used).")
+                d[key] = data
+            else:
+                if not isinstance(data, (tuple, list)):
+                    raise ValueError("loader must return a tuple or list (because image_only=False was used).")
+                d[key] = data[0]
+                if not isinstance(data[1], dict):
+                    raise ValueError("metadata must be a dict.")
+                key_to_add = f"{key}_{self.meta_key_postfix}"
+                if key_to_add in d and not self.overwriting:
+                    raise KeyError(f"Meta data with key {key_to_add} already exists and overwriting=False.")
+                d[key_to_add] = data[1]
         return d
 
 
@@ -114,13 +124,18 @@ class SaveImaged(MapTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.SaveImage`.
 
+    Note:
+        Image should include channel dimension: [B],C,H,W,[D].
+        If the data is a patch of big image, will append the patch index to filename.
+
     Args:
         keys: keys of the corresponding items to be transformed.
             See also: :py:class:`monai.transforms.compose.MapTransform`
         meta_key_postfix: `key_{postfix}` was used to store the metadata in `LoadImaged`.
-            So need the key to extract metadata to save images, default is `meta_dict`.
-            The meta data is a dictionary object, if no corresponding metadata, set to `None`.
-            For example, for data with key `image`, the metadata by default is in `image_meta_dict`.
+            so need the key to extract metadata to save images, default is `meta_dict`.
+            for example, for data with key `image`, the metadata by default is in `image_meta_dict`.
+            the meta data is a dictionary object which contains: filename, affine, original_shape, etc.
+            if no corresponding metadata, set to `None`.
         output_dir: output image directory.
         output_postfix: a string appended to all output file names, default to `trans`.
         output_ext: output file extension name, available extensions: `.nii.gz`, `.nii`, `.png`.
@@ -155,6 +170,22 @@ class SaveImaged(MapTransform):
             it's used for NIfTI format only.
         save_batch: whether the import image is a batch data, default to `False`.
             usually pre-transforms run for channel first data, while post-transforms run for batch data.
+        allow_missing_keys: don't raise exception if key is missing.
+        squeeze_end_dims: if True, any trailing singleton dimensions will be removed (after the channel
+            has been moved to the end). So if input is (C,H,W,D), this will be altered to (H,W,D,C), and
+            then if C==1, it will be saved as (H,W,D). If D also ==1, it will be saved as (H,W). If false,
+            image will always be saved as (H,W,D,C).
+            it's used for NIfTI format only.
+        data_root_dir: if not empty, it specifies the beginning parts of the input file's
+            absolute path. it's used to compute `input_file_rel_path`, the relative path to the file from
+            `data_root_dir` to preserve folder structure when saving in case there are files in different
+            folders with the same file names. for example:
+            input_file_name: /foo/bar/test1/image.nii,
+            output_postfix: seg
+            output_ext: nii.gz
+            output_dir: /output,
+            data_root_dir: /foo/bar,
+            output will be: /output/test1/image/image_seg.nii.gz
 
     """
 
@@ -172,8 +203,11 @@ def __init__(
         dtype: DtypeLike = np.float64,
         output_dtype: DtypeLike = np.float32,
         save_batch: bool = False,
+        allow_missing_keys: bool = False,
+        squeeze_end_dims: bool = True,
+        data_root_dir: str = "",
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.meta_key_postfix = meta_key_postfix
         self._saver = SaveImage(
             output_dir=output_dir,
@@ -186,11 +220,13 @@ def __init__(
             dtype=dtype,
             output_dtype=output_dtype,
             save_batch=save_batch,
+            squeeze_end_dims=squeeze_end_dims,
+            data_root_dir=data_root_dir,
         )
 
     def __call__(self, data):
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             meta_data = d[f"{key}_{self.meta_key_postfix}"] if self.meta_key_postfix is not None else None
             self._saver(img=d[key], meta_data=meta_data)
         return d
diff --git a/monai/transforms/post/array.py b/monai/transforms/post/array.py
index 0c60b0cc89..7ac0e6799c 100644
--- a/monai/transforms/post/array.py
+++ b/monai/transforms/post/array.py
@@ -21,7 +21,8 @@
 import torch.nn.functional as F
 
 from monai.networks import one_hot
-from monai.transforms.compose import Transform
+from monai.networks.layers import GaussianFilter
+from monai.transforms.transform import Transform
 from monai.transforms.utils import get_largest_connected_component_mask
 from monai.utils import ensure_tuple
 
@@ -86,9 +87,14 @@ def __call__(
         if other is not None and not callable(other):
             raise TypeError(f"other must be None or callable but is {type(other).__name__}.")
 
+        # convert to float as activation must operate on float tensor
+        img = img.float()
         if sigmoid or self.sigmoid:
             img = torch.sigmoid(img)
         if softmax or self.softmax:
+            # add channel dim if not existing
+            if img.ndimension() == 1:
+                img = img.unsqueeze(-1)
             img = torch.softmax(img, dim=1)
 
         act_func = self.other if other is None else other
@@ -417,3 +423,97 @@ def __call__(self, img: Union[Sequence[torch.Tensor], torch.Tensor]) -> torch.Te
             return torch.argmax(img_, dim=1, keepdim=has_ch_dim)
         # for One-Hot data, round the float number to 0 or 1
         return torch.round(img_)
+
+
+class ProbNMS(Transform):
+    """
+    Performs probability based non-maximum suppression (NMS) on the probabilities map via
+    iteratively selecting the coordinate with highest probability and then move it as well
+    as its surrounding values. The remove range is determined by the parameter `box_size`.
+    If multiple coordinates have the same highest probability, only one of them will be
+    selected.
+
+    Args:
+        spatial_dims: number of spatial dimensions of the input probabilities map.
+            Defaults to 2.
+        sigma: the standard deviation for gaussian filter.
+            It could be a single value, or `spatial_dims` number of values. Defaults to 0.0.
+        prob_threshold: the probability threshold, the function will stop searching if
+            the highest probability is no larger than the threshold. The value should be
+            no less than 0.0. Defaults to 0.5.
+        box_size: the box size (in pixel) to be removed around the the pixel with the maximum probability.
+            It can be an integer that defines the size of a square or cube,
+            or a list containing different values for each dimensions. Defaults to 48.
+
+    Return:
+        a list of selected lists, where inner lists contain probability and coordinates.
+        For example, for 3D input, the inner lists are in the form of [probability, x, y, z].
+
+    Raises:
+        ValueError: When ``prob_threshold`` is less than 0.0.
+        ValueError: When ``box_size`` is a list or tuple, and its length is not equal to `spatial_dims`.
+        ValueError: When ``box_size`` has a less than 1 value.
+
+    """
+
+    def __init__(
+        self,
+        spatial_dims: int = 2,
+        sigma: Union[Sequence[float], float, Sequence[torch.Tensor], torch.Tensor] = 0.0,
+        prob_threshold: float = 0.5,
+        box_size: Union[int, Sequence[int]] = 48,
+    ) -> None:
+        self.sigma = sigma
+        self.spatial_dims = spatial_dims
+        if self.sigma != 0:
+            self.filter = GaussianFilter(spatial_dims=spatial_dims, sigma=sigma)
+        if prob_threshold < 0:
+            raise ValueError("prob_threshold should be no less than 0.0.")
+        self.prob_threshold = prob_threshold
+        if isinstance(box_size, int):
+            self.box_size = np.asarray([box_size] * spatial_dims)
+        else:
+            if len(box_size) != spatial_dims:
+                raise ValueError("the sequence length of box_size should be the same as spatial_dims.")
+            self.box_size = np.asarray(box_size)
+        if self.box_size.min() <= 0:
+            raise ValueError("box_size should be larger than 0.")
+
+        self.box_lower_bd = self.box_size // 2
+        self.box_upper_bd = self.box_size - self.box_lower_bd
+
+    def __call__(
+        self,
+        prob_map: Union[np.ndarray, torch.Tensor],
+    ):
+        """
+        prob_map: the input probabilities map, it must have shape (H[, W, ...]).
+        """
+        if self.sigma != 0:
+            if not isinstance(prob_map, torch.Tensor):
+                prob_map = torch.as_tensor(prob_map, dtype=torch.float)
+            self.filter.to(prob_map)
+            prob_map = self.filter(prob_map)
+        else:
+            if not isinstance(prob_map, torch.Tensor):
+                prob_map = prob_map.copy()
+
+        if isinstance(prob_map, torch.Tensor):
+            prob_map = prob_map.detach().cpu().numpy()
+
+        prob_map_shape = prob_map.shape
+
+        outputs = []
+        while np.max(prob_map) > self.prob_threshold:
+            max_idx = np.unravel_index(prob_map.argmax(), prob_map_shape)
+            prob_max = prob_map[max_idx]
+            max_idx_arr = np.asarray(max_idx)
+            outputs.append([prob_max] + list(max_idx_arr))
+
+            idx_min_range = (max_idx_arr - self.box_lower_bd).clip(0, None)
+            idx_max_range = (max_idx_arr + self.box_upper_bd).clip(None, prob_map_shape)
+            # for each dimension, set values during index ranges to 0
+            slices = tuple(slice(idx_min_range[i], idx_max_range[i]) for i in range(self.spatial_dims))
+            prob_map[slices] = 0
+
+        return outputs
diff --git a/monai/transforms/post/dictionary.py b/monai/transforms/post/dictionary.py
index 60cda11a91..52bde4ab79 100644
--- a/monai/transforms/post/dictionary.py
+++ b/monai/transforms/post/dictionary.py
@@ -20,16 +20,18 @@
 import numpy as np
 import torch
 
+import monai.data
 from monai.config import KeysCollection
-from monai.transforms.compose import MapTransform
 from monai.transforms.post.array import (
     Activations,
     AsDiscrete,
     KeepLargestConnectedComponent,
     LabelToContour,
     MeanEnsemble,
+    ProbNMS,
     VoteEnsemble,
 )
+from monai.transforms.transform import MapTransform
 from monai.utils import ensure_tuple_rep
 
 __all__ = [
@@ -52,6 +54,9 @@
     "MeanEnsembleDict",
     "VoteEnsembleD",
     "VoteEnsembleDict",
+    "DecollateD",
+    "DecollateDict",
+    "Decollated",
 ]
 
 
@@ -67,6 +72,7 @@ def __init__(
         sigmoid: Union[Sequence[bool], bool] = False,
         softmax: Union[Sequence[bool], bool] = False,
         other: Optional[Union[Sequence[Callable], Callable]] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -79,9 +85,10 @@ def __init__(
             other: callable function to execute other activation layers,
                 for example: `other = lambda x: torch.tanh(x)`. it also can be a sequence of Callable, each
                 element corresponds to a key in ``keys``.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.sigmoid = ensure_tuple_rep(sigmoid, len(self.keys))
         self.softmax = ensure_tuple_rep(softmax, len(self.keys))
         self.other = ensure_tuple_rep(other, len(self.keys))
@@ -89,8 +96,8 @@ def __init__(
 
     def __call__(self, data: Mapping[Hashable, torch.Tensor]) -> Dict[Hashable, torch.Tensor]:
         d = dict(data)
-        for idx, key in enumerate(self.keys):
-            d[key] = self.converter(d[key], self.sigmoid[idx], self.softmax[idx], self.other[idx])
+        for key, sigmoid, softmax, other in self.key_iterator(d, self.sigmoid, self.softmax, self.other):
+            d[key] = self.converter(d[key], sigmoid, softmax, other)
         return d
 
 
@@ -107,6 +114,7 @@ def __init__(
         n_classes: Optional[Union[Sequence[int], int]] = None,
         threshold_values: Union[Sequence[bool], bool] = False,
         logit_thresh: Union[Sequence[float], float] = 0.5,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -122,9 +130,10 @@ def __init__(
                 it also can be a sequence of bool, each element corresponds to a key in ``keys``.
             logit_thresh: the threshold value for thresholding operation, default is 0.5.
                 it also can be a sequence of float, each element corresponds to a key in ``keys``.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.argmax = ensure_tuple_rep(argmax, len(self.keys))
         self.to_onehot = ensure_tuple_rep(to_onehot, len(self.keys))
         self.n_classes = ensure_tuple_rep(n_classes, len(self.keys))
@@ -134,14 +143,16 @@ def __init__(
 
     def __call__(self, data: Mapping[Hashable, torch.Tensor]) -> Dict[Hashable, torch.Tensor]:
         d = dict(data)
-        for idx, key in enumerate(self.keys):
+        for key, argmax, to_onehot, n_classes, threshold_values, logit_thresh in self.key_iterator(
+            d, self.argmax, self.to_onehot, self.n_classes, self.threshold_values, self.logit_thresh
+        ):
             d[key] = self.converter(
                 d[key],
-                self.argmax[idx],
-                self.to_onehot[idx],
-                self.n_classes[idx],
-                self.threshold_values[idx],
-                self.logit_thresh[idx],
+                argmax,
+                to_onehot,
+                n_classes,
+                threshold_values,
+                logit_thresh,
             )
         return d
 
@@ -157,6 +168,7 @@ def __init__(
         applied_labels: Union[Sequence[int], int],
         independent: bool = True,
         connectivity: Optional[int] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -171,14 +183,15 @@ def __init__(
             connectivity: Maximum number of orthogonal hops to consider a pixel/voxel as a neighbor.
                 Accepted values are ranging from  1 to input.ndim. If ``None``, a full
                 connectivity of ``input.ndim`` is used.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = KeepLargestConnectedComponent(applied_labels, independent, connectivity)
 
     def __call__(self, data: Mapping[Hashable, torch.Tensor]) -> Dict[Hashable, torch.Tensor]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
         return d
 
@@ -188,20 +201,21 @@ class LabelToContourd(MapTransform):
     Dictionary-based wrapper of :py:class:`monai.transforms.LabelToContour`.
     """
 
-    def __init__(self, keys: KeysCollection, kernel_type: str = "Laplace") -> None:
+    def __init__(self, keys: KeysCollection, kernel_type: str = "Laplace", allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             kernel_type: the method applied to do edge detection, default is "Laplace".
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = LabelToContour(kernel_type=kernel_type)
 
     def __call__(self, data: Mapping[Hashable, torch.Tensor]) -> Dict[Hashable, torch.Tensor]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
         return d
 
@@ -217,6 +231,7 @@ def __init__(
         keys: KeysCollection,
         ensemble: Callable[[Union[Sequence[torch.Tensor], torch.Tensor]], torch.Tensor],
         output_key: Optional[str] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -225,13 +240,14 @@ def __init__(
             output_key: the key to store ensemble result in the dictionary.
             ensemble: callable method to execute ensemble on specified data.
                 if only 1 key provided in `keys`, `output_key` can be None and use `keys` as default.
+            allow_missing_keys: don't raise exception if key is missing.
 
         Raises:
             TypeError: When ``ensemble`` is not ``callable``.
             ValueError: When ``len(keys) > 1`` and ``output_key=None``. Incompatible values.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         if not callable(ensemble):
             raise TypeError(f"ensemble must be callable but is {type(ensemble).__name__}.")
         self.ensemble = ensemble
@@ -245,7 +261,7 @@ def __call__(self, data: Mapping[Hashable, torch.Tensor]) -> Dict[Hashable, torc
         if len(self.keys) == 1:
             items = d[self.keys[0]]
         else:
-            items = [d[key] for key in self.keys]
+            items = [d[key] for key in self.key_iterator(d)]
         d[self.output_key] = self.ensemble(items)
 
         return d
@@ -306,9 +322,85 @@ def __init__(
         super().__init__(keys, ensemble, output_key)
 
 
+class Decollated(MapTransform):
+    """
+    Decollate a batch of data.
+
+    Note that unlike most MapTransforms, this will decollate all data, so keys are not needed.
+
+    Args:
+        batch_size: if not supplied, we try to determine it based on array lengths. Will raise an error if
+            it fails to determine it automatically.
+    """
+
+    def __init__(self, batch_size: Optional[int] = None) -> None:
+        super().__init__(None)
+        self.batch_size = batch_size
+
+    def __call__(self, data: dict) -> List[dict]:
+        return monai.data.decollate_batch(data, self.batch_size)
+
+
+class ProbNMSd(MapTransform):
+    """
+    Performs probability based non-maximum suppression (NMS) on the probabilities map via
+    iteratively selecting the coordinate with highest probability and then move it as well
+    as its surrounding values. The remove range is determined by the parameter `box_size`.
+    If multiple coordinates have the same highest probability, only one of them will be
+    selected.
+
+    Args:
+        spatial_dims: number of spatial dimensions of the input probabilities map.
+            Defaults to 2.
+        sigma: the standard deviation for gaussian filter.
+            It could be a single value, or `spatial_dims` number of values. Defaults to 0.0.
+        prob_threshold: the probability threshold, the function will stop searching if
+            the highest probability is no larger than the threshold. The value should be
+            no less than 0.0. Defaults to 0.5.
+        box_size: the box size (in pixel) to be removed around the the pixel with the maximum probability.
+            It can be an integer that defines the size of a square or cube,
+            or a list containing different values for each dimensions. Defaults to 48.
+
+    Return:
+        a list of selected lists, where inner lists contain probability and coordinates.
+        For example, for 3D input, the inner lists are in the form of [probability, x, y, z].
+
+    Raises:
+        ValueError: When ``prob_threshold`` is less than 0.0.
+        ValueError: When ``box_size`` is a list or tuple, and its length is not equal to `spatial_dims`.
+        ValueError: When ``box_size`` has a less than 1 value.
+
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        spatial_dims: int = 2,
+        sigma: Union[Sequence[float], float, Sequence[torch.Tensor], torch.Tensor] = 0.0,
+        prob_threshold: float = 0.5,
+        box_size: Union[int, Sequence[int]] = 48,
+        allow_missing_keys: bool = False,
+    ) -> None:
+        super().__init__(keys, allow_missing_keys)
+        self.prob_nms = ProbNMS(
+            spatial_dims=spatial_dims,
+            sigma=sigma,
+            prob_threshold=prob_threshold,
+            box_size=box_size,
+        )
+
+    def __call__(self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]):
+        d = dict(data)
+        for key in self.key_iterator(d):
+            d[key] = self.prob_nms(d[key])
+        return d
+
+
 ActivationsD = ActivationsDict = Activationsd
 AsDiscreteD = AsDiscreteDict = AsDiscreted
 KeepLargestConnectedComponentD = KeepLargestConnectedComponentDict = KeepLargestConnectedComponentd
 LabelToContourD = LabelToContourDict = LabelToContourd
 MeanEnsembleD = MeanEnsembleDict = MeanEnsembled
+ProbNMSD = ProbNMSDict = ProbNMSd
 VoteEnsembleD = VoteEnsembleDict = VoteEnsembled
+DecollateD = DecollateDict = Decollated
diff --git a/monai/transforms/spatial/array.py b/monai/transforms/spatial/array.py
index df10480188..c9fb4a7033 100644
--- a/monai/transforms/spatial/array.py
+++ b/monai/transforms/spatial/array.py
@@ -22,8 +22,8 @@
 from monai.config import USE_COMPILED, DtypeLike
 from monai.data.utils import compute_shape_offset, to_affine_nd, zoom_affine
 from monai.networks.layers import AffineTransform, GaussianFilter, grid_pull
-from monai.transforms.compose import Randomizable, Transform
 from monai.transforms.croppad.array import CenterSpatialCrop
+from monai.transforms.transform import Randomizable, RandomizableTransform, Transform
 from monai.transforms.utils import (
     create_control_grid,
     create_grid,
@@ -42,6 +42,7 @@
     ensure_tuple_rep,
     ensure_tuple_size,
     fall_back_tuple,
+    issequenceiterable,
     optional_import,
 )
 
@@ -58,6 +59,7 @@
     "RandRotate90",
     "RandRotate",
     "RandFlip",
+    "RandAxisFlip",
     "RandZoom",
     "AffineGrid",
     "RandAffineGrid",
@@ -69,6 +71,8 @@
     "Rand3DElastic",
 ]
 
+RandRange = Optional[Union[Sequence[Union[Tuple[float, float], float]], float]]
+
 
 class Spacing(Transform):
     """
@@ -125,6 +129,7 @@ def __call__(
         padding_mode: Optional[Union[GridSamplePadMode, str]] = None,
         align_corners: Optional[bool] = None,
         dtype: DtypeLike = None,
+        output_spatial_shape: Optional[np.ndarray] = None,
     ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
         """
         Args:
@@ -141,13 +146,16 @@ def __call__(
             dtype: data type for resampling computation. Defaults to ``self.dtype``.
                 If None, use the data type of input data. To be compatible with other modules,
                 the output data type is always ``np.float32``.
+            output_spatial_shape: specify the shape of the output data_array. This is typically useful for
+                the inverse of `Spacingd` where sometimes we could not compute the exact shape due to the quantization
+                error with the affines.
 
         Raises:
             ValueError: When ``data_array`` has no spatial dimensions.
             ValueError: When ``pixdim`` is nonpositive.
 
         Returns:
-            data_array (resampled into `self.pixdim`), original pixdim, current pixdim.
+            data_array (resampled into `self.pixdim`), original affine, current affine.
 
         """
         _dtype = dtype or self.dtype or data_array.dtype
@@ -191,7 +199,7 @@ def __call__(
             # AffineTransform requires a batch dim
             torch.as_tensor(np.ascontiguousarray(data_array).astype(_dtype)).unsqueeze(0),
             torch.as_tensor(np.ascontiguousarray(transform).astype(_dtype)),
-            spatial_size=output_shape,
+            spatial_size=output_shape if output_spatial_shape is None else output_spatial_shape,
         )
         output_data = np.asarray(output_data.squeeze(0).detach().cpu().numpy(), dtype=np.float32)  # type: ignore
         new_affine = to_affine_nd(affine, new_affine)
@@ -277,7 +285,7 @@ def __call__(
         ornt[:, 0] += 1  # skip channel dim
         ornt = np.concatenate([np.array([[0, 1]]), ornt])
         shape = data_array.shape[1:]
-        data_array = nib.orientations.apply_orientation(data_array, ornt)
+        data_array = np.ascontiguousarray(nib.orientations.apply_orientation(data_array, ornt))
         new_affine = affine_ @ nib.orientations.inv_ornt_aff(spatial_ornt, shape)
         new_affine = to_affine_nd(affine, new_affine)
         return data_array, affine, new_affine
@@ -313,7 +321,7 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
 
 class Resize(Transform):
     """
-    Resize the input image to given spatial size.
+    Resize the input image to given spatial size (with scaling, not cropping/padding).
     Implemented using :py:class:`torch.nn.functional.interpolate`.
 
     Args:
@@ -417,6 +425,7 @@ def __init__(
         self.padding_mode: GridSamplePadMode = GridSamplePadMode(padding_mode)
         self.align_corners = align_corners
         self.dtype = dtype
+        self.rotation_matrix: Optional[np.ndarray] = None
 
     def __call__(
         self,
@@ -478,8 +487,13 @@ def __call__(
             torch.as_tensor(np.ascontiguousarray(transform).astype(_dtype)),
             spatial_size=output_shape,
         )
+        self.rotation_matrix = transform
         return np.asarray(output.squeeze(0).detach().cpu().numpy(), dtype=np.float32)
 
+    def get_rotation_matrix(self) -> Optional[np.ndarray]:
+        """Get the most recently applied rotation matrix"""
+        return self.rotation_matrix
+
 
 class Zoom(Transform):
     """
@@ -586,7 +600,7 @@ def __init__(self, k: int = 1, spatial_axes: Tuple[int, int] = (0, 1)) -> None:
                 If axis is negative it counts from the last to the first axis.
         """
         self.k = k
-        spatial_axes_ = ensure_tuple(spatial_axes)
+        spatial_axes_: Tuple[int, int] = ensure_tuple(spatial_axes)  # type: ignore
         if len(spatial_axes_) != 2:
             raise ValueError("spatial_axes must be 2 int numbers to indicate the axes to rotate 90 degrees.")
         self.spatial_axes = spatial_axes_
@@ -601,7 +615,7 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         return result.astype(img.dtype)
 
 
-class RandRotate90(Randomizable, Transform):
+class RandRotate90(RandomizableTransform):
     """
     With probability `prob`, input arrays are rotated by 90 degrees
     in the plane specified by `spatial_axes`.
@@ -616,16 +630,15 @@ def __init__(self, prob: float = 0.1, max_k: int = 3, spatial_axes: Tuple[int, i
             spatial_axes: 2 int numbers, defines the plane to rotate with 2 spatial axes.
                 Default: (0, 1), this is the first two axis in spatial dimensions.
         """
-        self.prob = min(max(prob, 0.0), 1.0)
+        RandomizableTransform.__init__(self, prob)
         self.max_k = max_k
         self.spatial_axes = spatial_axes
 
-        self._do_transform = False
         self._rand_k = 0
 
     def randomize(self, data: Optional[Any] = None) -> None:
         self._rand_k = self.R.randint(self.max_k) + 1
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
 
     def __call__(self, img: np.ndarray) -> np.ndarray:
         """
@@ -639,7 +652,7 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         return rotator(img)
 
 
-class RandRotate(Randomizable, Transform):
+class RandRotate(RandomizableTransform):
     """
     Randomly rotate the input arrays.
 
@@ -679,6 +692,7 @@ def __init__(
         align_corners: bool = False,
         dtype: DtypeLike = np.float64,
     ) -> None:
+        RandomizableTransform.__init__(self, prob)
         self.range_x = ensure_tuple(range_x)
         if len(self.range_x) == 1:
             self.range_x = tuple(sorted([-self.range_x[0], self.range_x[0]]))
@@ -689,20 +703,18 @@ def __init__(
         if len(self.range_z) == 1:
             self.range_z = tuple(sorted([-self.range_z[0], self.range_z[0]]))
 
-        self.prob = prob
         self.keep_size = keep_size
         self.mode: GridSampleMode = GridSampleMode(mode)
         self.padding_mode: GridSamplePadMode = GridSamplePadMode(padding_mode)
         self.align_corners = align_corners
         self.dtype = dtype
 
-        self._do_transform = False
         self.x = 0.0
         self.y = 0.0
         self.z = 0.0
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self.x = self.R.uniform(low=self.range_x[0], high=self.range_x[1])
         self.y = self.R.uniform(low=self.range_y[0], high=self.range_y[1])
         self.z = self.R.uniform(low=self.range_z[0], high=self.range_z[1])
@@ -741,10 +753,10 @@ def __call__(
             align_corners=self.align_corners if align_corners is None else align_corners,
             dtype=dtype or self.dtype or img.dtype,
         )
-        return rotator(img)
+        return np.array(rotator(img))
 
 
-class RandFlip(Randomizable, Transform):
+class RandFlip(RandomizableTransform):
     """
     Randomly flips the image along axes. Preserves shape.
     See numpy.flip for additional details.
@@ -756,25 +768,52 @@ class RandFlip(Randomizable, Transform):
     """
 
     def __init__(self, prob: float = 0.1, spatial_axis: Optional[Union[Sequence[int], int]] = None) -> None:
-        self.prob = prob
+        RandomizableTransform.__init__(self, prob)
         self.flipper = Flip(spatial_axis=spatial_axis)
-        self._do_transform = False
-
-    def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
 
     def __call__(self, img: np.ndarray) -> np.ndarray:
         """
         Args:
             img: channel first array, must have shape: (num_channels, H[, W, ..., ]),
         """
-        self.randomize()
+        self.randomize(None)
         if not self._do_transform:
             return img
         return self.flipper(img)
 
 
-class RandZoom(Randomizable, Transform):
+class RandAxisFlip(RandomizableTransform):
+    """
+    Randomly select a spatial axis and flip along it.
+    See numpy.flip for additional details.
+    https://docs.scipy.org/doc/numpy/reference/generated/numpy.flip.html
+
+    Args:
+        prob: Probability of flipping.
+
+    """
+
+    def __init__(self, prob: float = 0.1) -> None:
+        RandomizableTransform.__init__(self, prob)
+        self._axis: Optional[int] = None
+
+    def randomize(self, data: np.ndarray) -> None:
+        super().randomize(None)
+        self._axis = self.R.randint(data.ndim - 1)
+
+    def __call__(self, img: np.ndarray) -> np.ndarray:
+        """
+        Args:
+            img: channel first array, must have shape: (num_channels, H[, W, ..., ]),
+        """
+        self.randomize(data=img)
+        if not self._do_transform:
+            return img
+        flipper = Flip(spatial_axis=self._axis)
+        return flipper(img)
+
+
+class RandZoom(RandomizableTransform):
     """
     Randomly zooms input arrays with given probability within given zoom range.
 
@@ -813,21 +852,20 @@ def __init__(
         align_corners: Optional[bool] = None,
         keep_size: bool = True,
     ) -> None:
+        RandomizableTransform.__init__(self, prob)
         self.min_zoom = ensure_tuple(min_zoom)
         self.max_zoom = ensure_tuple(max_zoom)
         if len(self.min_zoom) != len(self.max_zoom):
             raise AssertionError("min_zoom and max_zoom must have same length.")
-        self.prob = prob
         self.mode: InterpolateMode = InterpolateMode(mode)
         self.padding_mode: NumpyPadMode = NumpyPadMode(padding_mode)
         self.align_corners = align_corners
         self.keep_size = keep_size
 
-        self._do_transform = False
         self._zoom: Sequence[float] = [1.0]
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self._zoom = [self.R.uniform(l, h) for l, h in zip(self.min_zoom, self.max_zoom)]
 
     def __call__(
@@ -897,6 +935,9 @@ class AffineGrid(Transform):
         as_tensor_output: whether to output tensor instead of numpy array.
             defaults to True.
         device: device to store the output grid data.
+        affine: If applied, ignore the params (`rotate_params`, etc.) and use the
+            supplied matrix. Should be square with each side = num of image spatial
+            dimensions + 1.
 
     """
 
@@ -908,6 +949,7 @@ def __init__(
         scale_params: Optional[Union[Sequence[float], float]] = None,
         as_tensor_output: bool = True,
         device: Optional[torch.device] = None,
+        affine: Optional[Union[np.ndarray, torch.Tensor]] = None,
     ) -> None:
         self.rotate_params = rotate_params
         self.shear_params = shear_params
@@ -917,9 +959,13 @@ def __init__(
         self.as_tensor_output = as_tensor_output
         self.device = device
 
+        self.affine = affine
+
     def __call__(
-        self, spatial_size: Optional[Sequence[int]] = None, grid: Optional[Union[np.ndarray, torch.Tensor]] = None
-    ) -> Union[np.ndarray, torch.Tensor]:
+        self,
+        spatial_size: Optional[Sequence[int]] = None,
+        grid: Optional[Union[np.ndarray, torch.Tensor]] = None,
+    ) -> Tuple[Union[np.ndarray, torch.Tensor], Union[np.ndarray, torch.Tensor]]:
         """
         Args:
             spatial_size: output grid size.
@@ -935,60 +981,59 @@ def __call__(
             else:
                 raise ValueError("Incompatible values: grid=None and spatial_size=None.")
 
-        spatial_dims = len(grid.shape) - 1
-        affine = np.eye(spatial_dims + 1)
-        if self.rotate_params:
-            affine = affine @ create_rotate(spatial_dims, self.rotate_params)
-        if self.shear_params:
-            affine = affine @ create_shear(spatial_dims, self.shear_params)
-        if self.translate_params:
-            affine = affine @ create_translate(spatial_dims, self.translate_params)
-        if self.scale_params:
-            affine = affine @ create_scale(spatial_dims, self.scale_params)
-        affine = torch.as_tensor(np.ascontiguousarray(affine), device=self.device)
+        if self.affine is None:
+            spatial_dims = len(grid.shape) - 1
+            affine = np.eye(spatial_dims + 1)
+            if self.rotate_params:
+                affine = affine @ create_rotate(spatial_dims, self.rotate_params)
+            if self.shear_params:
+                affine = affine @ create_shear(spatial_dims, self.shear_params)
+            if self.translate_params:
+                affine = affine @ create_translate(spatial_dims, self.translate_params)
+            if self.scale_params:
+                affine = affine @ create_scale(spatial_dims, self.scale_params)
+        else:
+            affine = self.affine
+
+        if isinstance(affine, np.ndarray):
+            affine = torch.as_tensor(np.ascontiguousarray(affine))
 
         grid = torch.tensor(grid) if not isinstance(grid, torch.Tensor) else grid.detach().clone()
         if self.device:
+            affine = affine.to(self.device)
             grid = grid.to(self.device)
         grid = (affine.float() @ grid.reshape((grid.shape[0], -1)).float()).reshape([-1] + list(grid.shape[1:]))
         if grid is None or not isinstance(grid, torch.Tensor):
             raise ValueError("Unknown grid.")
-        if self.as_tensor_output:
-            return grid
-        return np.asarray(grid.cpu().numpy())
+        return grid if self.as_tensor_output else np.asarray(grid.cpu().numpy()), affine
 
 
-class RandAffineGrid(Randomizable, Transform):
+class RandAffineGrid(Randomizable):
     """
     Generate randomised affine grid.
     """
 
     def __init__(
         self,
-        rotate_range: Optional[Union[Sequence[float], float]] = None,
-        shear_range: Optional[Union[Sequence[float], float]] = None,
-        translate_range: Optional[Union[Sequence[float], float]] = None,
-        scale_range: Optional[Union[Sequence[float], float]] = None,
+        rotate_range: RandRange = None,
+        shear_range: RandRange = None,
+        translate_range: RandRange = None,
+        scale_range: RandRange = None,
         as_tensor_output: bool = True,
         device: Optional[torch.device] = None,
     ) -> None:
         """
         Args:
-            rotate_range: angle range in radians. rotate_range[0] with be used to generate the 1st rotation
-                parameter from `uniform[-rotate_range[0], rotate_range[0])`. Similarly, `rotate_range[1]` and
-                `rotate_range[2]` are used in 3D affine for the range of 2nd and 3rd axes.
-            shear_range: shear_range[0] with be used to generate the 1st shearing parameter from
-                `uniform[-shear_range[0], shear_range[0])`. Similarly, `shear_range[1]` to
-                `shear_range[N]` controls the range of the uniform distribution used to generate the 2nd to
-                N-th parameter.
-            translate_range : translate_range[0] with be used to generate the 1st shift parameter from
-                `uniform[-translate_range[0], translate_range[0])`. Similarly, `translate_range[1]`
-                to `translate_range[N]` controls the range of the uniform distribution used to generate
-                the 2nd to N-th parameter.
-            scale_range: scaling_range[0] with be used to generate the 1st scaling factor from
-                `uniform[-scale_range[0], scale_range[0]) + 1.0`. Similarly, `scale_range[1]` to
-                `scale_range[N]` controls the range of the uniform distribution used to generate the 2nd to
-                N-th parameter.
+            rotate_range: angle range in radians. If element `i` is iterable, then
+                `uniform[-rotate_range[i][0], rotate_range[i][1])` will be used to generate the rotation parameter
+                for the ith dimension. If not, `uniform[-rotate_range[i], rotate_range[i])` will be used. This can
+                be altered on a per-dimension basis. E.g., `((0,3), 1, ...)`: for dim0, rotation will be in range
+                `[0, 3]`, and for dim1 `[-1, 1]` will be used. Setting a single value will use `[-x, x]` for dim0
+                and nothing for the remaining dimensions.
+            shear_range: shear_range with format matching `rotate_range`.
+            translate_range: translate_range with format matching `rotate_range`.
+            scale_range: scaling_range with format matching `rotate_range`. A value of 1.0 is added to the result.
+                This allows 0 to correspond to no change (i.e., a scaling of 1).
             as_tensor_output: whether to output tensor instead of numpy array.
                 defaults to True.
             device: device to store the output grid data.
@@ -1011,19 +1056,29 @@ def __init__(
 
         self.as_tensor_output = as_tensor_output
         self.device = device
+        self.affine: Optional[Union[np.ndarray, torch.Tensor]] = None
+
+    def _get_rand_param(self, param_range, add_scalar: float = 0.0):
+        out_param = []
+        for f in param_range:
+            if issequenceiterable(f):
+                if len(f) != 2:
+                    raise ValueError("If giving range as [min,max], should only have two elements per dim.")
+                out_param.append(self.R.uniform(f[0], f[1]) + add_scalar)
+            elif f is not None:
+                out_param.append(self.R.uniform(-f, f) + add_scalar)
+        return out_param
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        if self.rotate_range:
-            self.rotate_params = [self.R.uniform(-f, f) for f in self.rotate_range if f is not None]
-        if self.shear_range:
-            self.shear_params = [self.R.uniform(-f, f) for f in self.shear_range if f is not None]
-        if self.translate_range:
-            self.translate_params = [self.R.uniform(-f, f) for f in self.translate_range if f is not None]
-        if self.scale_range:
-            self.scale_params = [self.R.uniform(-f, f) + 1.0 for f in self.scale_range if f is not None]
+        self.rotate_params = self._get_rand_param(self.rotate_range)
+        self.shear_params = self._get_rand_param(self.shear_range)
+        self.translate_params = self._get_rand_param(self.translate_range)
+        self.scale_params = self._get_rand_param(self.scale_range, 1.0)
 
     def __call__(
-        self, spatial_size: Optional[Sequence[int]] = None, grid: Optional[Union[np.ndarray, torch.Tensor]] = None
+        self,
+        spatial_size: Optional[Sequence[int]] = None,
+        grid: Optional[Union[np.ndarray, torch.Tensor]] = None,
     ) -> Union[np.ndarray, torch.Tensor]:
         """
         Args:
@@ -1042,10 +1097,15 @@ def __call__(
             as_tensor_output=self.as_tensor_output,
             device=self.device,
         )
-        return affine_grid(spatial_size, grid)
+        grid, self.affine = affine_grid(spatial_size, grid)
+        return grid
 
+    def get_transformation_matrix(self) -> Optional[Union[np.ndarray, torch.Tensor]]:
+        """Get the most recently applied transformation matrix"""
+        return self.affine
 
-class RandDeformGrid(Randomizable, Transform):
+
+class RandDeformGrid(Randomizable):
     """
     Generate random deformation grid.
     """
@@ -1251,7 +1311,7 @@ def __call__(
         spatial_size: Optional[Union[Sequence[int], int]] = None,
         mode: Optional[Union[GridSampleMode, str]] = None,
         padding_mode: Optional[Union[GridSamplePadMode, str]] = None,
-    ) -> Union[np.ndarray, torch.Tensor]:
+    ) -> Tuple[Union[np.ndarray, torch.Tensor], Union[np.ndarray, torch.Tensor]]:
         """
         Args:
             img: shape must be (num_channels, H, W[, D]),
@@ -1268,13 +1328,14 @@ def __call__(
                 See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
         """
         sp_size = fall_back_tuple(spatial_size or self.spatial_size, img.shape[1:])
-        grid = self.affine_grid(spatial_size=sp_size)
-        return self.resampler(
-            img=img, grid=grid, mode=mode or self.mode, padding_mode=padding_mode or self.padding_mode
+        grid, affine = self.affine_grid(spatial_size=sp_size)
+        return (
+            self.resampler(img=img, grid=grid, mode=mode or self.mode, padding_mode=padding_mode or self.padding_mode),
+            affine,
         )
 
 
-class RandAffine(Randomizable, Transform):
+class RandAffine(RandomizableTransform):
     """
     Random affine transform.
     """
@@ -1282,11 +1343,11 @@ class RandAffine(Randomizable, Transform):
     def __init__(
         self,
         prob: float = 0.1,
-        rotate_range: Optional[Union[Sequence[float], float]] = None,
-        shear_range: Optional[Union[Sequence[float], float]] = None,
-        translate_range: Optional[Union[Sequence[float], float]] = None,
-        scale_range: Optional[Union[Sequence[float], float]] = None,
-        spatial_size: Optional[Union[Sequence[float], float]] = None,
+        rotate_range: RandRange = None,
+        shear_range: RandRange = None,
+        translate_range: RandRange = None,
+        scale_range: RandRange = None,
+        spatial_size: Optional[Union[Sequence[int], int]] = None,
         mode: Union[GridSampleMode, str] = GridSampleMode.BILINEAR,
         padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.REFLECTION,
         as_tensor_output: bool = True,
@@ -1296,21 +1357,16 @@ def __init__(
         Args:
             prob: probability of returning a randomized affine grid.
                 defaults to 0.1, with 10% chance returns a randomized grid.
-            rotate_range: angle range in radians. rotate_range[0] with be used to generate the 1st rotation
-                parameter from `uniform[-rotate_range[0], rotate_range[0])`. Similarly, `rotate_range[1]` and
-                `rotate_range[2]` are used in 3D affine for the range of 2nd and 3rd axes.
-            shear_range: shear_range[0] with be used to generate the 1st shearing parameter from
-                `uniform[-shear_range[0], shear_range[0])`. Similarly, `shear_range[1]` to
-                `shear_range[N]` controls the range of the uniform distribution used to generate the 2nd to
-                N-th parameter.
-            translate_range : translate_range[0] with be used to generate the 1st shift parameter from
-                `uniform[-translate_range[0], translate_range[0])`. Similarly, `translate_range[1]`
-                to `translate_range[N]` controls the range of the uniform distribution used to generate
-                the 2nd to N-th parameter.
-            scale_range: scaling_range[0] with be used to generate the 1st scaling factor from
-                `uniform[-scale_range[0], scale_range[0]) + 1.0`. Similarly, `scale_range[1]` to
-                `scale_range[N]` controls the range of the uniform distribution used to generate the 2nd to
-                N-th parameter.
+            rotate_range: angle range in radians. If element `i` is iterable, then
+                `uniform[-rotate_range[i][0], rotate_range[i][1])` will be used to generate the rotation parameter
+                for the ith dimension. If not, `uniform[-rotate_range[i], rotate_range[i])` will be used. This can
+                be altered on a per-dimension basis. E.g., `((0,3), 1, ...)`: for dim0, rotation will be in range
+                `[0, 3]`, and for dim1 `[-1, 1]` will be used. Setting a single value will use `[-x, x]` for dim0
+                and nothing for the remaining dimensions.
+            shear_range: shear_range with format matching `rotate_range`.
+            translate_range: translate_range with format matching `rotate_range`.
+            scale_range: scaling_range with format matching `rotate_range`. A value of 1.0 is added to the result.
+                This allows 0 to correspond to no change (i.e., a scaling of 1).
             spatial_size: output image spatial size.
                 if `spatial_size` and `self.spatial_size` are not defined, or smaller than 1,
                 the transform will use the spatial size of `img`.
@@ -1331,6 +1387,7 @@ def __init__(
             - :py:class:`RandAffineGrid` for the random affine parameters configurations.
             - :py:class:`Affine` for the affine transformation parameters configurations.
         """
+        RandomizableTransform.__init__(self, prob)
 
         self.rand_affine_grid = RandAffineGrid(
             rotate_range=rotate_range,
@@ -1346,9 +1403,6 @@ def __init__(
         self.mode: GridSampleMode = GridSampleMode(mode)
         self.padding_mode: GridSamplePadMode = GridSamplePadMode(padding_mode)
 
-        self.do_transform = False
-        self.prob = prob
-
     def set_random_state(
         self, seed: Optional[int] = None, state: Optional[np.random.RandomState] = None
     ) -> "RandAffine":
@@ -1357,7 +1411,7 @@ def set_random_state(
         return self
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self.do_transform = self.R.rand() < self.prob
+        super().randomize(None)
         self.rand_affine_grid.randomize()
 
     def __call__(
@@ -1383,9 +1437,8 @@ def __call__(
                 See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
         """
         self.randomize()
-
         sp_size = fall_back_tuple(spatial_size or self.spatial_size, img.shape[1:])
-        if self.do_transform:
+        if self._do_transform:
             grid = self.rand_affine_grid(spatial_size=sp_size)
         else:
             grid = create_grid(spatial_size=sp_size)
@@ -1394,7 +1447,7 @@ def __call__(
         )
 
 
-class Rand2DElastic(Randomizable, Transform):
+class Rand2DElastic(RandomizableTransform):
     """
     Random elastic deformation and affine in 2D
     """
@@ -1404,11 +1457,11 @@ def __init__(
         spacing: Union[Tuple[float, float], float],
         magnitude_range: Tuple[float, float],
         prob: float = 0.1,
-        rotate_range: Optional[Union[Sequence[float], float]] = None,
-        shear_range: Optional[Union[Sequence[float], float]] = None,
-        translate_range: Optional[Union[Sequence[float], float]] = None,
-        scale_range: Optional[Union[Sequence[float], float]] = None,
-        spatial_size: Optional[Union[Sequence[int], int]] = None,
+        rotate_range: RandRange = None,
+        shear_range: RandRange = None,
+        translate_range: RandRange = None,
+        scale_range: RandRange = None,
+        spatial_size: Optional[Union[Tuple[int, int], int]] = None,
         mode: Union[GridSampleMode, str] = GridSampleMode.BILINEAR,
         padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.REFLECTION,
         as_tensor_output: bool = False,
@@ -1421,17 +1474,16 @@ def __init__(
             prob: probability of returning a randomized elastic transform.
                 defaults to 0.1, with 10% chance returns a randomized elastic transform,
                 otherwise returns a ``spatial_size`` centered area extracted from the input image.
-            rotate_range: angle range in radians. rotate_range[0] with be used to generate the 1st rotation
-                parameter from `uniform[-rotate_range[0], rotate_range[0])`.
-            shear_range: shear_range[0] with be used to generate the 1st shearing parameter from
-                `uniform[-shear_range[0], shear_range[0])`. Similarly, `shear_range[1]` controls
-                the range of the uniform distribution used to generate the 2nd parameter.
-            translate_range : translate_range[0] with be used to generate the 1st shift parameter from
-                `uniform[-translate_range[0], translate_range[0])`. Similarly, `translate_range[1]` controls
-                the range of the uniform distribution used to generate the 2nd parameter.
-            scale_range: scaling_range[0] with be used to generate the 1st scaling factor from
-                `uniform[-scale_range[0], scale_range[0]) + 1.0`. Similarly, `scale_range[1]` controls
-                the range of the uniform distribution used to generate the 2nd parameter.
+            rotate_range: angle range in radians. If element `i` is iterable, then
+                `uniform[-rotate_range[i][0], rotate_range[i][1])` will be used to generate the rotation parameter
+                for the ith dimension. If not, `uniform[-rotate_range[i], rotate_range[i])` will be used. This can
+                be altered on a per-dimension basis. E.g., `((0,3), 1, ...)`: for dim0, rotation will be in range
+                `[0, 3]`, and for dim1 `[-1, 1]` will be used. Setting a single value will use `[-x, x]` for dim0
+                and nothing for the remaining dimensions.
+            shear_range: shear_range with format matching `rotate_range`.
+            translate_range: translate_range with format matching `rotate_range`.
+            scale_range: scaling_range with format matching `rotate_range`. A value of 1.0 is added to the result.
+                This allows 0 to correspond to no change (i.e., a scaling of 1).
             spatial_size: specifying output image spatial size [h, w].
                 if `spatial_size` and `self.spatial_size` are not defined, or smaller than 1,
                 the transform will use the spatial size of `img`.
@@ -1452,6 +1504,7 @@ def __init__(
             - :py:class:`RandAffineGrid` for the random affine parameters configurations.
             - :py:class:`Affine` for the affine transformation parameters configurations.
         """
+        RandomizableTransform.__init__(self, prob)
         self.deform_grid = RandDeformGrid(
             spacing=spacing, magnitude_range=magnitude_range, as_tensor_output=True, device=device
         )
@@ -1468,8 +1521,6 @@ def __init__(
         self.spatial_size = spatial_size
         self.mode: GridSampleMode = GridSampleMode(mode)
         self.padding_mode: GridSamplePadMode = GridSamplePadMode(padding_mode)
-        self.prob = prob
-        self.do_transform = False
 
     def set_random_state(
         self, seed: Optional[int] = None, state: Optional[np.random.RandomState] = None
@@ -1480,7 +1531,7 @@ def set_random_state(
         return self
 
     def randomize(self, spatial_size: Sequence[int]) -> None:
-        self.do_transform = self.R.rand() < self.prob
+        super().randomize(None)
         self.deform_grid.randomize(spatial_size)
         self.rand_affine_grid.randomize()
 
@@ -1506,7 +1557,7 @@ def __call__(
         """
         sp_size = fall_back_tuple(spatial_size or self.spatial_size, img.shape[1:])
         self.randomize(spatial_size=sp_size)
-        if self.do_transform:
+        if self._do_transform:
             grid = self.deform_grid(spatial_size=sp_size)
             grid = self.rand_affine_grid(grid=grid)
             grid = torch.nn.functional.interpolate(  # type: ignore
@@ -1516,13 +1567,13 @@ def __call__(
                 mode=InterpolateMode.BICUBIC.value,
                 align_corners=False,
             )
-            grid = CenterSpatialCrop(roi_size=sp_size)(np.asarray(grid[0]))
+            grid = CenterSpatialCrop(roi_size=sp_size)(grid[0])
         else:
             grid = create_grid(spatial_size=sp_size)
         return self.resampler(img, grid, mode=mode or self.mode, padding_mode=padding_mode or self.padding_mode)
 
 
-class Rand3DElastic(Randomizable, Transform):
+class Rand3DElastic(RandomizableTransform):
     """
     Random elastic deformation and affine in 3D
     """
@@ -1532,11 +1583,11 @@ def __init__(
         sigma_range: Tuple[float, float],
         magnitude_range: Tuple[float, float],
         prob: float = 0.1,
-        rotate_range: Optional[Union[Sequence[float], float]] = None,
-        shear_range: Optional[Union[Sequence[float], float]] = None,
-        translate_range: Optional[Union[Sequence[float], float]] = None,
-        scale_range: Optional[Union[Sequence[float], float]] = None,
-        spatial_size: Optional[Union[Sequence[int], int]] = None,
+        rotate_range: RandRange = None,
+        shear_range: RandRange = None,
+        translate_range: RandRange = None,
+        scale_range: RandRange = None,
+        spatial_size: Optional[Union[Tuple[int, int, int], int]] = None,
         mode: Union[GridSampleMode, str] = GridSampleMode.BILINEAR,
         padding_mode: Union[GridSamplePadMode, str] = GridSamplePadMode.REFLECTION,
         as_tensor_output: bool = False,
@@ -1551,19 +1602,16 @@ def __init__(
             prob: probability of returning a randomized elastic transform.
                 defaults to 0.1, with 10% chance returns a randomized elastic transform,
                 otherwise returns a ``spatial_size`` centered area extracted from the input image.
-            rotate_range: angle range in radians. rotate_range[0] with be used to generate the 1st rotation
-                parameter from `uniform[-rotate_range[0], rotate_range[0])`. Similarly, `rotate_range[1]` and
-                `rotate_range[2]` are used in 3D affine for the range of 2nd and 3rd axes.
-            shear_range: shear_range[0] with be used to generate the 1st shearing parameter from
-                `uniform[-shear_range[0], shear_range[0])`. Similarly, `shear_range[1]` and `shear_range[2]`
-                controls the range of the uniform distribution used to generate the 2nd and 3rd parameters.
-            translate_range : translate_range[0] with be used to generate the 1st shift parameter from
-                `uniform[-translate_range[0], translate_range[0])`. Similarly, `translate_range[1]` and
-                `translate_range[2]` controls the range of the uniform distribution used to generate
-                the 2nd and 3rd parameters.
-            scale_range: scaling_range[0] with be used to generate the 1st scaling factor from
-                `uniform[-scale_range[0], scale_range[0]) + 1.0`. Similarly, `scale_range[1]` and `scale_range[2]`
-                controls the range of the uniform distribution used to generate the 2nd and 3rd parameters.
+            rotate_range: angle range in radians. If element `i` is iterable, then
+                `uniform[-rotate_range[i][0], rotate_range[i][1])` will be used to generate the rotation parameter
+                for the ith dimension. If not, `uniform[-rotate_range[i], rotate_range[i])` will be used. This can
+                be altered on a per-dimension basis. E.g., `((0,3), 1, ...)`: for dim0, rotation will be in range
+                `[0, 3]`, and for dim1 `[-1, 1]` will be used. Setting a single value will use `[-x, x]` for dim0
+                and nothing for the remaining dimensions.
+            shear_range: shear_range with format matching `rotate_range`.
+            translate_range: translate_range with format matching `rotate_range`.
+            scale_range: scaling_range with format matching `rotate_range`. A value of 1.0 is added to the result.
+                This allows 0 to correspond to no change (i.e., a scaling of 1).
             spatial_size: specifying output image spatial size [h, w, d].
                 if `spatial_size` and `self.spatial_size` are not defined, or smaller than 1,
                 the transform will use the spatial size of `img`.
@@ -1584,6 +1632,7 @@ def __init__(
             - :py:class:`RandAffineGrid` for the random affine parameters configurations.
             - :py:class:`Affine` for the affine transformation parameters configurations.
         """
+        RandomizableTransform.__init__(self, prob)
         self.rand_affine_grid = RandAffineGrid(rotate_range, shear_range, translate_range, scale_range, True, device)
         self.resampler = Resample(as_tensor_output=as_tensor_output, device=device)
 
@@ -1594,8 +1643,6 @@ def __init__(
         self.padding_mode: GridSamplePadMode = GridSamplePadMode(padding_mode)
         self.device = device
 
-        self.prob = prob
-        self.do_transform = False
         self.rand_offset = None
         self.magnitude = 1.0
         self.sigma = 1.0
@@ -1608,8 +1655,8 @@ def set_random_state(
         return self
 
     def randomize(self, grid_size: Sequence[int]) -> None:
-        self.do_transform = self.R.rand() < self.prob
-        if self.do_transform:
+        super().randomize(None)
+        if self._do_transform:
             self.rand_offset = self.R.uniform(-1.0, 1.0, [3] + list(grid_size)).astype(np.float32)
         self.magnitude = self.R.uniform(self.magnitude_range[0], self.magnitude_range[1])
         self.sigma = self.R.uniform(self.sigma_range[0], self.sigma_range[1])
@@ -1638,7 +1685,7 @@ def __call__(
         sp_size = fall_back_tuple(spatial_size or self.spatial_size, img.shape[1:])
         self.randomize(grid_size=sp_size)
         grid = create_grid(spatial_size=sp_size)
-        if self.do_transform:
+        if self._do_transform:
             if self.rand_offset is None:
                 raise AssertionError
             grid = torch.as_tensor(np.ascontiguousarray(grid), device=self.device)
diff --git a/monai/transforms/spatial/dictionary.py b/monai/transforms/spatial/dictionary.py
index e612a25ef8..cfadd2f6e3 100644
--- a/monai/transforms/spatial/dictionary.py
+++ b/monai/transforms/spatial/dictionary.py
@@ -15,16 +15,21 @@
 Class names are ended with 'd' to denote dictionary-based transforms.
 """
 
+from copy import deepcopy
+from enum import Enum
 from typing import Any, Dict, Hashable, Mapping, Optional, Sequence, Tuple, Union
 
 import numpy as np
 import torch
 
 from monai.config import DtypeLike, KeysCollection
+from monai.networks.layers import AffineTransform
 from monai.networks.layers.simplelayers import GaussianFilter
-from monai.transforms.compose import MapTransform, Randomizable
-from monai.transforms.croppad.array import CenterSpatialCrop
+from monai.transforms.croppad.array import CenterSpatialCrop, SpatialPad
+from monai.transforms.inverse import InvertibleTransform
 from monai.transforms.spatial.array import (
+    Affine,
+    AffineGrid,
     Flip,
     Orientation,
     Rand2DElastic,
@@ -36,6 +41,7 @@
     Spacing,
     Zoom,
 )
+from monai.transforms.transform import MapTransform, RandomizableTransform
 from monai.transforms.utils import create_grid
 from monai.utils import (
     GridSampleMode,
@@ -46,6 +52,10 @@
     ensure_tuple_rep,
     fall_back_tuple,
 )
+from monai.utils.enums import InverseKeys
+from monai.utils.module import optional_import
+
+nib, _ = optional_import("nibabel")
 
 __all__ = [
     "Spacingd",
@@ -53,11 +63,13 @@
     "Rotate90d",
     "RandRotate90d",
     "Resized",
+    "Affined",
     "RandAffined",
     "Rand2DElasticd",
     "Rand3DElasticd",
     "Flipd",
     "RandFlipd",
+    "RandAxisFlipd",
     "Rotated",
     "RandRotated",
     "Zoomd",
@@ -72,6 +84,8 @@
     "RandRotate90Dict",
     "ResizeD",
     "ResizeDict",
+    "AffineD",
+    "AffineDict",
     "RandAffineD",
     "RandAffineDict",
     "Rand2DElasticD",
@@ -82,6 +96,8 @@
     "FlipDict",
     "RandFlipD",
     "RandFlipDict",
+    "RandAxisFlipD",
+    "RandAxisFlipDict",
     "RotateD",
     "RotateDict",
     "RandRotateD",
@@ -98,7 +114,7 @@
 NumpyPadModeSequence = Union[Sequence[Union[NumpyPadMode, str]], NumpyPadMode, str]
 
 
-class Spacingd(MapTransform):
+class Spacingd(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.Spacing`.
 
@@ -122,6 +138,7 @@ def __init__(
         align_corners: Union[Sequence[bool], bool] = False,
         dtype: Optional[Union[Sequence[DtypeLike], DtypeLike]] = np.float64,
         meta_key_postfix: str = "meta_dict",
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -157,12 +174,13 @@ def __init__(
                 default is `meta_dict`, the meta data is a dictionary object.
                 For example, to handle key `image`,  read/write affine matrices from the
                 metadata `image_meta_dict` dictionary's `affine` field.
+            allow_missing_keys: don't raise exception if key is missing.
 
         Raises:
             TypeError: When ``meta_key_postfix`` is not a ``str``.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.spacing_transform = Spacing(pixdim, diagonal=diagonal)
         self.mode = ensure_tuple_rep(mode, len(self.keys))
         self.padding_mode = ensure_tuple_rep(padding_mode, len(self.keys))
@@ -176,24 +194,74 @@ def __call__(
         self, data: Mapping[Union[Hashable, str], Dict[str, np.ndarray]]
     ) -> Dict[Union[Hashable, str], Union[np.ndarray, Dict[str, np.ndarray]]]:
         d: Dict = dict(data)
-        for idx, key in enumerate(self.keys):
-            meta_data = d[f"{key}_{self.meta_key_postfix}"]
+        for key, mode, padding_mode, align_corners, dtype in self.key_iterator(
+            d, self.mode, self.padding_mode, self.align_corners, self.dtype
+        ):
+            meta_data_key = f"{key}_{self.meta_key_postfix}"
+            meta_data = d[meta_data_key]
             # resample array of each corresponding key
             # using affine fetched from d[affine_key]
-            d[key], _, new_affine = self.spacing_transform(
+            original_spatial_shape = d[key].shape[1:]
+            d[key], old_affine, new_affine = self.spacing_transform(
                 data_array=np.asarray(d[key]),
                 affine=meta_data["affine"],
-                mode=self.mode[idx],
-                padding_mode=self.padding_mode[idx],
-                align_corners=self.align_corners[idx],
-                dtype=self.dtype[idx],
+                mode=mode,
+                padding_mode=padding_mode,
+                align_corners=align_corners,
+                dtype=dtype,
+            )
+            self.push_transform(
+                d,
+                key,
+                extra_info={
+                    "meta_data_key": meta_data_key,
+                    "old_affine": old_affine,
+                    "mode": mode.value if isinstance(mode, Enum) else mode,
+                    "padding_mode": padding_mode.value if isinstance(padding_mode, Enum) else padding_mode,
+                    "align_corners": align_corners if align_corners is not None else "none",
+                },
+                orig_size=original_spatial_shape,
             )
             # set the 'affine' key
             meta_data["affine"] = new_affine
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key, dtype in self.key_iterator(d, self.dtype):
+            transform = self.get_most_recent_transform(d, key)
+            if self.spacing_transform.diagonal:
+                raise RuntimeError(
+                    "Spacingd:inverse not yet implemented for diagonal=True. "
+                    + "Please raise a github issue if you need this feature"
+                )
+            # Create inverse transform
+            meta_data = d[transform[InverseKeys.EXTRA_INFO]["meta_data_key"]]
+            old_affine = np.array(transform[InverseKeys.EXTRA_INFO]["old_affine"])
+            mode = transform[InverseKeys.EXTRA_INFO]["mode"]
+            padding_mode = transform[InverseKeys.EXTRA_INFO]["padding_mode"]
+            align_corners = transform[InverseKeys.EXTRA_INFO]["align_corners"]
+            orig_size = transform[InverseKeys.ORIG_SIZE]
+            orig_pixdim = np.sqrt(np.sum(np.square(old_affine), 0))[:-1]
+            inverse_transform = Spacing(orig_pixdim, diagonal=self.spacing_transform.diagonal)
+            # Apply inverse
+            d[key], _, new_affine = inverse_transform(
+                data_array=np.asarray(d[key]),
+                affine=meta_data["affine"],
+                mode=mode,
+                padding_mode=padding_mode,
+                align_corners=False if align_corners == "none" else align_corners,
+                dtype=dtype,
+                output_spatial_shape=orig_size,
+            )
+            meta_data["affine"] = new_affine
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
 
-class Orientationd(MapTransform):
+class Orientationd(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.Orientation`.
 
@@ -211,6 +279,7 @@ def __init__(
         as_closest_canonical: bool = False,
         labels: Optional[Sequence[Tuple[str, str]]] = tuple(zip("LPI", "RAS")),
         meta_key_postfix: str = "meta_dict",
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -227,6 +296,7 @@ def __init__(
                 default is `meta_dict`, the meta data is a dictionary object.
                 For example, to handle key `image`,  read/write affine matrices from the
                 metadata `image_meta_dict` dictionary's `affine` field.
+            allow_missing_keys: don't raise exception if key is missing.
 
         Raises:
             TypeError: When ``meta_key_postfix`` is not a ``str``.
@@ -235,7 +305,7 @@ def __init__(
             `nibabel.orientations.ornt2axcodes`.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.ornt_transform = Orientation(axcodes=axcodes, as_closest_canonical=as_closest_canonical, labels=labels)
         if not isinstance(meta_key_postfix, str):
             raise TypeError(f"meta_key_postfix must be a str but is {type(meta_key_postfix).__name__}.")
@@ -245,36 +315,82 @@ def __call__(
         self, data: Mapping[Union[Hashable, str], Dict[str, np.ndarray]]
     ) -> Dict[Union[Hashable, str], Union[np.ndarray, Dict[str, np.ndarray]]]:
         d: Dict = dict(data)
-        for key in self.keys:
-            meta_data = d[f"{key}_{self.meta_key_postfix}"]
-            d[key], _, new_affine = self.ornt_transform(d[key], affine=meta_data["affine"])
+        for key in self.key_iterator(d):
+            meta_data_key = f"{key}_{self.meta_key_postfix}"
+            meta_data = d[meta_data_key]
+            d[key], old_affine, new_affine = self.ornt_transform(d[key], affine=meta_data["affine"])
+            self.push_transform(d, key, extra_info={"meta_data_key": meta_data_key, "old_affine": old_affine})
+            d[meta_data_key]["affine"] = new_affine
+        return d
+
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            meta_data = d[transform[InverseKeys.EXTRA_INFO]["meta_data_key"]]
+            orig_affine = transform[InverseKeys.EXTRA_INFO]["old_affine"]
+            orig_axcodes = nib.orientations.aff2axcodes(orig_affine)
+            inverse_transform = Orientation(
+                axcodes=orig_axcodes,
+                as_closest_canonical=False,
+                labels=self.ornt_transform.labels,
+            )
+            # Apply inverse
+            d[key], _, new_affine = inverse_transform(d[key], affine=meta_data["affine"])
             meta_data["affine"] = new_affine
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
         return d
 
 
-class Rotate90d(MapTransform):
+class Rotate90d(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.Rotate90`.
     """
 
-    def __init__(self, keys: KeysCollection, k: int = 1, spatial_axes: Tuple[int, int] = (0, 1)) -> None:
+    def __init__(
+        self, keys: KeysCollection, k: int = 1, spatial_axes: Tuple[int, int] = (0, 1), allow_missing_keys: bool = False
+    ) -> None:
         """
         Args:
             k: number of times to rotate by 90 degrees.
             spatial_axes: 2 int numbers, defines the plane to rotate with 2 spatial axes.
                 Default: (0, 1), this is the first two axis in spatial dimensions.
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.rotator = Rotate90(k, spatial_axes)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            self.push_transform(d, key)
             d[key] = self.rotator(d[key])
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            _ = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            spatial_axes = self.rotator.spatial_axes
+            num_times_rotated = self.rotator.k
+            num_times_to_rotate = 4 - num_times_rotated
+            inverse_transform = Rotate90(num_times_to_rotate, spatial_axes)
+            # Might need to convert to numpy
+            if isinstance(d[key], torch.Tensor):
+                d[key] = torch.Tensor(d[key]).cpu().numpy()
+            # Apply inverse
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
 
-class RandRotate90d(Randomizable, MapTransform):
+        return d
+
+
+class RandRotate90d(RandomizableTransform, MapTransform, InvertibleTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.RandRotate90`.
     With probability `prob`, input arrays are rotated by 90 degrees
@@ -287,6 +403,7 @@ def __init__(
         prob: float = 0.1,
         max_k: int = 3,
         spatial_axes: Tuple[int, int] = (0, 1),
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -298,33 +415,53 @@ def __init__(
                 (Default 3)
             spatial_axes: 2 int numbers, defines the plane to rotate with 2 spatial axes.
                 Default: (0, 1), this is the first two axis in spatial dimensions.
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
 
-        self.prob = min(max(prob, 0.0), 1.0)
         self.max_k = max_k
         self.spatial_axes = spatial_axes
 
-        self._do_transform = False
         self._rand_k = 0
 
     def randomize(self, data: Optional[Any] = None) -> None:
         self._rand_k = self.R.randint(self.max_k) + 1
-        self._do_transform = self.R.random() < self.prob
+        super().randomize(None)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Mapping[Hashable, np.ndarray]:
         self.randomize()
-        if not self._do_transform:
-            return data
+        d = dict(data)
 
         rotator = Rotate90(self._rand_k, self.spatial_axes)
-        d = dict(data)
-        for key in self.keys:
-            d[key] = rotator(d[key])
+        for key in self.key_iterator(d):
+            if self._do_transform:
+                d[key] = rotator(d[key])
+            self.push_transform(d, key, extra_info={"rand_k": self._rand_k})
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Check if random transform was actually performed (based on `prob`)
+            if transform[InverseKeys.DO_TRANSFORM]:
+                # Create inverse transform
+                num_times_rotated = transform[InverseKeys.EXTRA_INFO]["rand_k"]
+                num_times_to_rotate = 4 - num_times_rotated
+                inverse_transform = Rotate90(num_times_to_rotate, self.spatial_axes)
+                # Might need to convert to numpy
+                if isinstance(d[key], torch.Tensor):
+                    d[key] = torch.Tensor(d[key]).cpu().numpy()
+                # Apply inverse
+                d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
 
-class Resized(MapTransform):
+        return d
+
+
+class Resized(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.Resize`.
 
@@ -343,6 +480,7 @@ class Resized(MapTransform):
             'linear', 'bilinear', 'bicubic' or 'trilinear'. Default: None.
             See also: https://pytorch.org/docs/stable/nn.functional.html#interpolate
             It also can be a sequence of bool or None, each element corresponds to a key in ``keys``.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -351,20 +489,155 @@ def __init__(
         spatial_size: Union[Sequence[int], int],
         mode: InterpolateModeSequence = InterpolateMode.AREA,
         align_corners: Union[Sequence[Optional[bool]], Optional[bool]] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.mode = ensure_tuple_rep(mode, len(self.keys))
         self.align_corners = ensure_tuple_rep(align_corners, len(self.keys))
         self.resizer = Resize(spatial_size=spatial_size)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for idx, key in enumerate(self.keys):
-            d[key] = self.resizer(d[key], mode=self.mode[idx], align_corners=self.align_corners[idx])
+        for key, mode, align_corners in self.key_iterator(d, self.mode, self.align_corners):
+            self.push_transform(
+                d,
+                key,
+                extra_info={
+                    "mode": mode.value if isinstance(mode, Enum) else mode,
+                    "align_corners": align_corners if align_corners is not None else "none",
+                },
+            )
+            d[key] = self.resizer(d[key], mode=mode, align_corners=align_corners)
+        return d
+
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            orig_size = transform[InverseKeys.ORIG_SIZE]
+            mode = transform[InverseKeys.EXTRA_INFO]["mode"]
+            align_corners = transform[InverseKeys.EXTRA_INFO]["align_corners"]
+            # Create inverse transform
+            inverse_transform = Resize(orig_size, mode, None if align_corners == "none" else align_corners)
+            # Apply inverse transform
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
         return d
 
 
-class RandAffined(Randomizable, MapTransform):
+class Affined(MapTransform, InvertibleTransform):
+    """
+    Dictionary-based wrapper of :py:class:`monai.transforms.Affine`.
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        rotate_params: Optional[Union[Sequence[float], float]] = None,
+        shear_params: Optional[Union[Sequence[float], float]] = None,
+        translate_params: Optional[Union[Sequence[float], float]] = None,
+        scale_params: Optional[Union[Sequence[float], float]] = None,
+        spatial_size: Optional[Union[Sequence[int], int]] = None,
+        mode: GridSampleModeSequence = GridSampleMode.BILINEAR,
+        padding_mode: GridSamplePadModeSequence = GridSamplePadMode.REFLECTION,
+        as_tensor_output: bool = False,
+        device: Optional[torch.device] = None,
+        allow_missing_keys: bool = False,
+    ) -> None:
+        """
+        Args:
+            keys: keys of the corresponding items to be transformed.
+            rotate_params: a rotation angle in radians, a scalar for 2D image, a tuple of 3 floats for 3D.
+                Defaults to no rotation.
+            shear_params: a tuple of 2 floats for 2D, a tuple of 6 floats for 3D. Defaults to no shearing.
+            translate_params: a tuple of 2 floats for 2D, a tuple of 3 floats for 3D. Translation is in
+                pixel/voxel relative to the center of the input image. Defaults to no translation.
+            scale_params: a tuple of 2 floats for 2D, a tuple of 3 floats for 3D. Defaults to no scaling.
+            spatial_size: output image spatial size.
+                if `spatial_size` and `self.spatial_size` are not defined, or smaller than 1,
+                the transform will use the spatial size of `img`.
+                if the components of the `spatial_size` are non-positive values, the transform will use the
+                corresponding components of img size. For example, `spatial_size=(32, -1)` will be adapted
+                to `(32, 64)` if the second spatial dimension size of img is `64`.
+            mode: {``"bilinear"``, ``"nearest"``}
+                Interpolation mode to calculate output values. Defaults to ``"bilinear"``.
+                See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+                It also can be a sequence of string, each element corresponds to a key in ``keys``.
+            padding_mode: {``"zeros"``, ``"border"``, ``"reflection"``}
+                Padding mode for outside grid values. Defaults to ``"reflection"``.
+                See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+                It also can be a sequence of string, each element corresponds to a key in ``keys``.
+            as_tensor_output: the computation is implemented using pytorch tensors, this option specifies
+                whether to convert it back to numpy arrays.
+            device: device on which the tensor will be allocated.
+            allow_missing_keys: don't raise exception if key is missing.
+
+        See also:
+            - :py:class:`monai.transforms.compose.MapTransform`
+            - :py:class:`RandAffineGrid` for the random affine parameters configurations.
+        """
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        self.affine = Affine(
+            rotate_params=rotate_params,
+            shear_params=shear_params,
+            translate_params=translate_params,
+            scale_params=scale_params,
+            spatial_size=spatial_size,
+            as_tensor_output=as_tensor_output,
+            device=device,
+        )
+        self.mode = ensure_tuple_rep(mode, len(self.keys))
+        self.padding_mode = ensure_tuple_rep(padding_mode, len(self.keys))
+
+    def __call__(
+        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
+    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
+        d = dict(data)
+        for key, mode, padding_mode in self.key_iterator(d, self.mode, self.padding_mode):
+            orig_size = d[key].shape[1:]
+            d[key], affine = self.affine(d[key], mode=mode, padding_mode=padding_mode)
+            self.push_transform(
+                d,
+                key,
+                orig_size=orig_size,
+                extra_info={
+                    "affine": affine,
+                    "mode": mode.value if isinstance(mode, Enum) else mode,
+                    "padding_mode": padding_mode.value if isinstance(padding_mode, Enum) else padding_mode,
+                },
+            )
+        return d
+
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            orig_size = transform[InverseKeys.ORIG_SIZE]
+            # Create inverse transform
+            fwd_affine = transform[InverseKeys.EXTRA_INFO]["affine"]
+            mode = transform[InverseKeys.EXTRA_INFO]["mode"]
+            padding_mode = transform[InverseKeys.EXTRA_INFO]["padding_mode"]
+            inv_affine = np.linalg.inv(fwd_affine)
+
+            affine_grid = AffineGrid(affine=inv_affine)
+            grid, _ = affine_grid(orig_size)  # type: ignore
+
+            # Apply inverse transform
+            out = self.affine.resampler(d[key], grid, mode, padding_mode)
+
+            # Convert to numpy
+            d[key] = out if isinstance(out, np.ndarray) else out.cpu().numpy()
+
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
+
+class RandAffined(RandomizableTransform, MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.RandAffine`.
     """
@@ -374,14 +647,15 @@ def __init__(
         keys: KeysCollection,
         spatial_size: Optional[Union[Sequence[int], int]] = None,
         prob: float = 0.1,
-        rotate_range: Optional[Union[Sequence[float], float]] = None,
-        shear_range: Optional[Union[Sequence[float], float]] = None,
-        translate_range: Optional[Union[Sequence[float], float]] = None,
-        scale_range: Optional[Union[Sequence[float], float]] = None,
+        rotate_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
+        shear_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
+        translate_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
+        scale_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
         mode: GridSampleModeSequence = GridSampleMode.BILINEAR,
         padding_mode: GridSamplePadModeSequence = GridSamplePadMode.REFLECTION,
         as_tensor_output: bool = True,
         device: Optional[torch.device] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -394,21 +668,16 @@ def __init__(
                 to `(32, 64)` if the second spatial dimension size of img is `64`.
             prob: probability of returning a randomized affine grid.
                 defaults to 0.1, with 10% chance returns a randomized grid.
-            rotate_range: angle range in radians. rotate_range[0] with be used to generate the 1st rotation
-                parameter from `uniform[-rotate_range[0], rotate_range[0])`. Similarly, `rotate_range[1]` and
-                `rotate_range[2]` are used in 3D affine for the range of 2nd and 3rd axes.
-            shear_range: shear_range[0] with be used to generate the 1st shearing parameter from
-                `uniform[-shear_range[0], shear_range[0])`. Similarly, `shear_range[1]` to
-                `shear_range[N]` controls the range of the uniform distribution used to generate the 2nd to
-                N-th parameter.
-            translate_range : translate_range[0] with be used to generate the 1st shift parameter from
-                `uniform[-translate_range[0], translate_range[0])`. Similarly, `translate_range[1]`
-                to `translate_range[N]` controls the range of the uniform distribution used to generate
-                the 2nd to N-th parameter.
-            scale_range: scaling_range[0] with be used to generate the 1st scaling factor from
-                `uniform[-scale_range[0], scale_range[0]) + 1.0`. Similarly, `scale_range[1]` to
-                `scale_range[N]` controls the range of the uniform distribution used to generate the 2nd to
-                N-th parameter.
+            rotate_range: angle range in radians. If element `i` is iterable, then
+                `uniform[-rotate_range[i][0], rotate_range[i][1])` will be used to generate the rotation parameter
+                for the ith dimension. If not, `uniform[-rotate_range[i], rotate_range[i])` will be used. This can
+                be altered on a per-dimension basis. E.g., `((0,3), 1, ...)`: for dim0, rotation will be in range
+                `[0, 3]`, and for dim1 `[-1, 1]` will be used. Setting a single value will use `[-x, x]` for dim0
+                and nothing for the remaining dimensions.
+            shear_range: shear_range with format matching `rotate_range`.
+            translate_range: translate_range with format matching `rotate_range`.
+            scale_range: scaling_range with format matching `rotate_range`. A value of 1.0 is added to the result.
+                This allows 0 to correspond to no change (i.e., a scaling of 1).
             mode: {``"bilinear"``, ``"nearest"``}
                 Interpolation mode to calculate output values. Defaults to ``"bilinear"``.
                 See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
@@ -420,14 +689,16 @@ def __init__(
             as_tensor_output: the computation is implemented using pytorch tensors, this option specifies
                 whether to convert it back to numpy arrays.
             device: device on which the tensor will be allocated.
+            allow_missing_keys: don't raise exception if key is missing.
 
         See also:
             - :py:class:`monai.transforms.compose.MapTransform`
             - :py:class:`RandAffineGrid` for the random affine parameters configurations.
         """
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
         self.rand_affine = RandAffine(
-            prob=prob,
+            prob=1.0,  # because probability handled in this class
             rotate_range=rotate_range,
             shear_range=shear_range,
             translate_range=translate_range,
@@ -447,6 +718,7 @@ def set_random_state(
         return self
 
     def randomize(self, data: Optional[Any] = None) -> None:
+        super().randomize(None)
         self.rand_affine.randomize()
 
     def __call__(
@@ -456,17 +728,55 @@ def __call__(
         self.randomize()
 
         sp_size = fall_back_tuple(self.rand_affine.spatial_size, data[self.keys[0]].shape[1:])
-        if self.rand_affine.do_transform:
+        if self._do_transform:
             grid = self.rand_affine.rand_affine_grid(spatial_size=sp_size)
+            affine = self.rand_affine.rand_affine_grid.get_transformation_matrix()
         else:
             grid = create_grid(spatial_size=sp_size)
+            # to be consistent with the self._do_transform case (dtype and device)
+            affine = torch.as_tensor(np.eye(len(sp_size) + 1), device=self.rand_affine.rand_affine_grid.device)
+
+        for key, mode, padding_mode in self.key_iterator(d, self.mode, self.padding_mode):
+            self.push_transform(
+                d,
+                key,
+                extra_info={
+                    "affine": affine,
+                    "mode": mode.value if isinstance(mode, Enum) else mode,
+                    "padding_mode": padding_mode.value if isinstance(padding_mode, Enum) else padding_mode,
+                },
+            )
+            d[key] = self.rand_affine.resampler(d[key], grid, mode=mode, padding_mode=padding_mode)
+        return d
+
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            orig_size = transform[InverseKeys.ORIG_SIZE]
+            # Create inverse transform
+            fwd_affine = transform[InverseKeys.EXTRA_INFO]["affine"]
+            mode = transform[InverseKeys.EXTRA_INFO]["mode"]
+            padding_mode = transform[InverseKeys.EXTRA_INFO]["padding_mode"]
+            inv_affine = np.linalg.inv(fwd_affine)
+
+            affine_grid = AffineGrid(affine=inv_affine)
+            grid, _ = affine_grid(orig_size)  # type: ignore
+
+            # Apply inverse transform
+            out = self.rand_affine.resampler(d[key], grid, mode, padding_mode)
+
+            # Convert to numpy
+            d[key] = out if isinstance(out, np.ndarray) else out.cpu().numpy()
+
+            # Remove the applied transform
+            self.pop_transform(d, key)
 
-        for idx, key in enumerate(self.keys):
-            d[key] = self.rand_affine.resampler(d[key], grid, mode=self.mode[idx], padding_mode=self.padding_mode[idx])
         return d
 
 
-class Rand2DElasticd(Randomizable, MapTransform):
+class Rand2DElasticd(RandomizableTransform, MapTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.Rand2DElastic`.
     """
@@ -476,16 +786,17 @@ def __init__(
         keys: KeysCollection,
         spacing: Union[Tuple[float, float], float],
         magnitude_range: Tuple[float, float],
-        spatial_size: Optional[Union[Sequence[int], int]] = None,
+        spatial_size: Optional[Union[Tuple[int, int], int]] = None,
         prob: float = 0.1,
-        rotate_range: Optional[Union[Sequence[float], float]] = None,
-        shear_range: Optional[Union[Sequence[float], float]] = None,
-        translate_range: Optional[Union[Sequence[float], float]] = None,
-        scale_range: Optional[Union[Sequence[float], float]] = None,
+        rotate_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
+        shear_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
+        translate_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
+        scale_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
         mode: GridSampleModeSequence = GridSampleMode.BILINEAR,
         padding_mode: GridSamplePadModeSequence = GridSamplePadMode.REFLECTION,
         as_tensor_output: bool = False,
         device: Optional[torch.device] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -502,17 +813,16 @@ def __init__(
             prob: probability of returning a randomized affine grid.
                 defaults to 0.1, with 10% chance returns a randomized grid,
                 otherwise returns a ``spatial_size`` centered area extracted from the input image.
-            rotate_range: angle range in radians. rotate_range[0] with be used to generate the 1st rotation
-                parameter from `uniform[-rotate_range[0], rotate_range[0])`.
-            shear_range: shear_range[0] with be used to generate the 1st shearing parameter from
-                `uniform[-shear_range[0], shear_range[0])`. Similarly, `shear_range[1]` controls
-                the range of the uniform distribution used to generate the 2nd parameter.
-            translate_range : translate_range[0] with be used to generate the 1st shift parameter from
-                `uniform[-translate_range[0], translate_range[0])`. Similarly, `translate_range[1]` controls
-                the range of the uniform distribution used to generate the 2nd parameter.
-            scale_range: scaling_range[0] with be used to generate the 1st scaling factor from
-                `uniform[-scale_range[0], scale_range[0]) + 1.0`. Similarly, `scale_range[1]` controls
-                the range of the uniform distribution used to generate the 2nd parameter.
+            rotate_range: angle range in radians. If element `i` is iterable, then
+                `uniform[-rotate_range[i][0], rotate_range[i][1])` will be used to generate the rotation parameter
+                for the ith dimension. If not, `uniform[-rotate_range[i], rotate_range[i])` will be used. This can
+                be altered on a per-dimension basis. E.g., `((0,3), 1, ...)`: for dim0, rotation will be in range
+                `[0, 3]`, and for dim1 `[-1, 1]` will be used. Setting a single value will use `[-x, x]` for dim0
+                and nothing for the remaining dimensions.
+            shear_range: shear_range with format matching `rotate_range`.
+            translate_range: translate_range with format matching `rotate_range`.
+            scale_range: scaling_range with format matching `rotate_range`. A value of 1.0 is added to the result.
+                This allows 0 to correspond to no change (i.e., a scaling of 1).
             mode: {``"bilinear"``, ``"nearest"``}
                 Interpolation mode to calculate output values. Defaults to ``"bilinear"``.
                 See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
@@ -524,16 +834,18 @@ def __init__(
             as_tensor_output: the computation is implemented using pytorch tensors, this option specifies
                 whether to convert it back to numpy arrays.
             device: device on which the tensor will be allocated.
+            allow_missing_keys: don't raise exception if key is missing.
 
         See also:
             - :py:class:`RandAffineGrid` for the random affine parameters configurations.
             - :py:class:`Affine` for the affine transformation parameters configurations.
         """
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
         self.rand_2d_elastic = Rand2DElastic(
             spacing=spacing,
             magnitude_range=magnitude_range,
-            prob=prob,
+            prob=1.0,  # because probability controlled by this class
             rotate_range=rotate_range,
             shear_range=shear_range,
             translate_range=translate_range,
@@ -553,6 +865,7 @@ def set_random_state(
         return self
 
     def randomize(self, spatial_size: Sequence[int]) -> None:
+        super().randomize(None)
         self.rand_2d_elastic.randomize(spatial_size)
 
     def __call__(
@@ -563,7 +876,7 @@ def __call__(
         sp_size = fall_back_tuple(self.rand_2d_elastic.spatial_size, data[self.keys[0]].shape[1:])
         self.randomize(spatial_size=sp_size)
 
-        if self.rand_2d_elastic.do_transform:
+        if self._do_transform:
             grid = self.rand_2d_elastic.deform_grid(spatial_size=sp_size)
             grid = self.rand_2d_elastic.rand_affine_grid(grid=grid)
             grid = torch.nn.functional.interpolate(  # type: ignore
@@ -577,14 +890,12 @@ def __call__(
         else:
             grid = create_grid(spatial_size=sp_size)
 
-        for idx, key in enumerate(self.keys):
-            d[key] = self.rand_2d_elastic.resampler(
-                d[key], grid, mode=self.mode[idx], padding_mode=self.padding_mode[idx]
-            )
+        for key, mode, padding_mode in self.key_iterator(d, self.mode, self.padding_mode):
+            d[key] = self.rand_2d_elastic.resampler(d[key], grid, mode=mode, padding_mode=padding_mode)
         return d
 
 
-class Rand3DElasticd(Randomizable, MapTransform):
+class Rand3DElasticd(RandomizableTransform, MapTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.Rand3DElastic`.
     """
@@ -594,16 +905,17 @@ def __init__(
         keys: KeysCollection,
         sigma_range: Tuple[float, float],
         magnitude_range: Tuple[float, float],
-        spatial_size: Optional[Union[Sequence[int], int]] = None,
+        spatial_size: Optional[Union[Tuple[int, int, int], int]] = None,
         prob: float = 0.1,
-        rotate_range: Optional[Union[Sequence[float], float]] = None,
-        shear_range: Optional[Union[Sequence[float], float]] = None,
-        translate_range: Optional[Union[Sequence[float], float]] = None,
-        scale_range: Optional[Union[Sequence[float], float]] = None,
+        rotate_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
+        shear_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
+        translate_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
+        scale_range: Optional[Union[Sequence[Union[Tuple[float, float], float]], float]] = None,
         mode: GridSampleModeSequence = GridSampleMode.BILINEAR,
         padding_mode: GridSamplePadModeSequence = GridSamplePadMode.REFLECTION,
         as_tensor_output: bool = False,
         device: Optional[torch.device] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -621,19 +933,16 @@ def __init__(
             prob: probability of returning a randomized affine grid.
                 defaults to 0.1, with 10% chance returns a randomized grid,
                 otherwise returns a ``spatial_size`` centered area extracted from the input image.
-            rotate_range: angle range in radians. rotate_range[0] with be used to generate the 1st rotation
-                parameter from `uniform[-rotate_range[0], rotate_range[0])`. Similarly, `rotate_range[1]` and
-                `rotate_range[2]` are used in 3D affine for the range of 2nd and 3rd axes.
-            shear_range: shear_range[0] with be used to generate the 1st shearing parameter from
-                `uniform[-shear_range[0], shear_range[0])`. Similarly, `shear_range[1]` and `shear_range[2]`
-                controls the range of the uniform distribution used to generate the 2nd and 3rd parameters.
-            translate_range : translate_range[0] with be used to generate the 1st shift parameter from
-                `uniform[-translate_range[0], translate_range[0])`. Similarly, `translate_range[1]` and
-                `translate_range[2]` controls the range of the uniform distribution used to generate
-                the 2nd and 3rd parameters.
-            scale_range: scaling_range[0] with be used to generate the 1st scaling factor from
-                `uniform[-scale_range[0], scale_range[0]) + 1.0`. Similarly, `scale_range[1]` and `scale_range[2]`
-                controls the range of the uniform distribution used to generate the 2nd and 3rd parameters.
+            rotate_range: angle range in radians. If element `i` is iterable, then
+                `uniform[-rotate_range[i][0], rotate_range[i][1])` will be used to generate the rotation parameter
+                for the ith dimension. If not, `uniform[-rotate_range[i], rotate_range[i])` will be used. This can
+                be altered on a per-dimension basis. E.g., `((0,3), 1, ...)`: for dim0, rotation will be in range
+                `[0, 3]`, and for dim1 `[-1, 1]` will be used. Setting a single value will use `[-x, x]` for dim0
+                and nothing for the remaining dimensions.
+            shear_range: shear_range with format matching `rotate_range`.
+            translate_range: translate_range with format matching `rotate_range`.
+            scale_range: scaling_range with format matching `rotate_range`. A value of 1.0 is added to the result.
+                This allows 0 to correspond to no change (i.e., a scaling of 1).
             mode: {``"bilinear"``, ``"nearest"``}
                 Interpolation mode to calculate output values. Defaults to ``"bilinear"``.
                 See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
@@ -645,16 +954,18 @@ def __init__(
             as_tensor_output: the computation is implemented using pytorch tensors, this option specifies
                 whether to convert it back to numpy arrays.
             device: device on which the tensor will be allocated.
+            allow_missing_keys: don't raise exception if key is missing.
 
         See also:
             - :py:class:`RandAffineGrid` for the random affine parameters configurations.
             - :py:class:`Affine` for the affine transformation parameters configurations.
         """
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
         self.rand_3d_elastic = Rand3DElastic(
             sigma_range=sigma_range,
             magnitude_range=magnitude_range,
-            prob=prob,
+            prob=1.0,  # because probability controlled by this class
             rotate_range=rotate_range,
             shear_range=shear_range,
             translate_range=translate_range,
@@ -674,6 +985,7 @@ def set_random_state(
         return self
 
     def randomize(self, grid_size: Sequence[int]) -> None:
+        super().randomize(None)
         self.rand_3d_elastic.randomize(grid_size)
 
     def __call__(
@@ -684,7 +996,7 @@ def __call__(
 
         self.randomize(grid_size=sp_size)
         grid = create_grid(spatial_size=sp_size)
-        if self.rand_3d_elastic.do_transform:
+        if self._do_transform:
             device = self.rand_3d_elastic.device
             grid = torch.tensor(grid).to(device)
             gaussian = GaussianFilter(spatial_dims=3, sigma=self.rand_3d_elastic.sigma, truncated=3.0).to(device)
@@ -692,14 +1004,12 @@ def __call__(
             grid[:3] += gaussian(offset)[0] * self.rand_3d_elastic.magnitude
             grid = self.rand_3d_elastic.rand_affine_grid(grid=grid)
 
-        for idx, key in enumerate(self.keys):
-            d[key] = self.rand_3d_elastic.resampler(
-                d[key], grid, mode=self.mode[idx], padding_mode=self.padding_mode[idx]
-            )
+        for key, mode, padding_mode in self.key_iterator(d, self.mode, self.padding_mode):
+            d[key] = self.rand_3d_elastic.resampler(d[key], grid, mode=mode, padding_mode=padding_mode)
         return d
 
 
-class Flipd(MapTransform):
+class Flipd(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.Flip`.
 
@@ -709,20 +1019,41 @@ class Flipd(MapTransform):
     Args:
         keys: Keys to pick data for transformation.
         spatial_axis: Spatial axes along which to flip over. Default is None.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
-    def __init__(self, keys: KeysCollection, spatial_axis: Optional[Union[Sequence[int], int]] = None) -> None:
-        super().__init__(keys)
+    def __init__(
+        self,
+        keys: KeysCollection,
+        spatial_axis: Optional[Union[Sequence[int], int]] = None,
+        allow_missing_keys: bool = False,
+    ) -> None:
+        super().__init__(keys, allow_missing_keys)
         self.flipper = Flip(spatial_axis=spatial_axis)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            self.push_transform(d, key)
             d[key] = self.flipper(d[key])
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            _ = self.get_most_recent_transform(d, key)
+            # Might need to convert to numpy
+            if isinstance(d[key], torch.Tensor):
+                d[key] = torch.Tensor(d[key]).cpu().numpy()
+            # Inverse is same as forward
+            d[key] = self.flipper(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
 
-class RandFlipd(Randomizable, MapTransform):
+        return d
+
+
+class RandFlipd(RandomizableTransform, MapTransform, InvertibleTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.RandFlip`.
 
@@ -733,6 +1064,7 @@ class RandFlipd(Randomizable, MapTransform):
         keys: Keys to pick data for transformation.
         prob: Probability of flipping.
         spatial_axis: Spatial axes along which to flip over. Default is None.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -740,28 +1072,91 @@ def __init__(
         keys: KeysCollection,
         prob: float = 0.1,
         spatial_axis: Optional[Union[Sequence[int], int]] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
         self.spatial_axis = spatial_axis
-        self.prob = prob
 
-        self._do_transform = False
         self.flipper = Flip(spatial_axis=spatial_axis)
 
-    def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+    def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        self.randomize(None)
+        d = dict(data)
+        for key in self.key_iterator(d):
+            if self._do_transform:
+                d[key] = self.flipper(d[key])
+            self.push_transform(d, key)
+        return d
+
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Check if random transform was actually performed (based on `prob`)
+            if transform[InverseKeys.DO_TRANSFORM]:
+                # Might need to convert to numpy
+                if isinstance(d[key], torch.Tensor):
+                    d[key] = torch.Tensor(d[key]).cpu().numpy()
+                # Inverse is same as forward
+                d[key] = self.flipper(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+        return d
+
+
+class RandAxisFlipd(RandomizableTransform, MapTransform, InvertibleTransform):
+    """
+    Dictionary-based version :py:class:`monai.transforms.RandAxisFlip`.
+
+    See `numpy.flip` for additional details.
+    https://docs.scipy.org/doc/numpy/reference/generated/numpy.flip.html
+
+    Args:
+        keys: Keys to pick data for transformation.
+        prob: Probability of flipping.
+        allow_missing_keys: don't raise exception if key is missing.
+
+    """
+
+    def __init__(self, keys: KeysCollection, prob: float = 0.1, allow_missing_keys: bool = False) -> None:
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
+        self._axis: Optional[int] = None
+
+    def randomize(self, data: np.ndarray) -> None:
+        super().randomize(None)
+        self._axis = self.R.randint(data.ndim - 1)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
-        self.randomize()
+        self.randomize(data=data[self.keys[0]])
+        flipper = Flip(spatial_axis=self._axis)
+
         d = dict(data)
-        if not self._do_transform:
-            return d
-        for key in self.keys:
-            d[key] = self.flipper(d[key])
+        for key in self.key_iterator(d):
+            if self._do_transform:
+                d[key] = flipper(d[key])
+            self.push_transform(d, key, extra_info={"axis": self._axis})
+        return d
+
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Check if random transform was actually performed (based on `prob`)
+            if transform[InverseKeys.DO_TRANSFORM]:
+                flipper = Flip(spatial_axis=transform[InverseKeys.EXTRA_INFO]["axis"])
+                # Might need to convert to numpy
+                if isinstance(d[key], torch.Tensor):
+                    d[key] = torch.Tensor(d[key]).cpu().numpy()
+                # Inverse is same as forward
+                d[key] = flipper(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
         return d
 
 
-class Rotated(MapTransform):
+class Rotated(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.Rotate`.
 
@@ -786,6 +1181,7 @@ class Rotated(MapTransform):
             If None, use the data type of input data. To be compatible with other modules,
             the output data type is always ``np.float32``.
             It also can be a sequence of dtype or None, each element corresponds to a key in ``keys``.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -797,8 +1193,9 @@ def __init__(
         padding_mode: GridSamplePadModeSequence = GridSamplePadMode.BORDER,
         align_corners: Union[Sequence[bool], bool] = False,
         dtype: Union[Sequence[DtypeLike], DtypeLike] = np.float64,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.rotator = Rotate(angle=angle, keep_size=keep_size)
 
         self.mode = ensure_tuple_rep(mode, len(self.keys))
@@ -808,18 +1205,62 @@ def __init__(
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for idx, key in enumerate(self.keys):
+        for key, mode, padding_mode, align_corners, dtype in self.key_iterator(
+            d, self.mode, self.padding_mode, self.align_corners, self.dtype
+        ):
+            orig_size = d[key].shape[1:]
             d[key] = self.rotator(
                 d[key],
-                mode=self.mode[idx],
-                padding_mode=self.padding_mode[idx],
-                align_corners=self.align_corners[idx],
-                dtype=self.dtype[idx],
+                mode=mode,
+                padding_mode=padding_mode,
+                align_corners=align_corners,
+                dtype=dtype,
+            )
+            rot_mat = self.rotator.get_rotation_matrix()
+            self.push_transform(
+                d,
+                key,
+                orig_size=orig_size,
+                extra_info={
+                    "rot_mat": rot_mat,
+                    "mode": mode.value if isinstance(mode, Enum) else mode,
+                    "padding_mode": padding_mode.value if isinstance(padding_mode, Enum) else padding_mode,
+                    "align_corners": align_corners if align_corners is not None else "none",
+                },
             )
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key, dtype in self.key_iterator(d, self.dtype):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            fwd_rot_mat = transform[InverseKeys.EXTRA_INFO]["rot_mat"]
+            mode = transform[InverseKeys.EXTRA_INFO]["mode"]
+            padding_mode = transform[InverseKeys.EXTRA_INFO]["padding_mode"]
+            align_corners = transform[InverseKeys.EXTRA_INFO]["align_corners"]
+            inv_rot_mat = np.linalg.inv(fwd_rot_mat)
+
+            xform = AffineTransform(
+                normalized=False,
+                mode=mode,
+                padding_mode=padding_mode,
+                align_corners=False if align_corners == "none" else align_corners,
+                reverse_indexing=True,
+            )
+            output = xform(
+                torch.as_tensor(np.ascontiguousarray(d[key]).astype(dtype)).unsqueeze(0),
+                torch.as_tensor(np.ascontiguousarray(inv_rot_mat).astype(dtype)),
+                spatial_size=transform[InverseKeys.ORIG_SIZE],
+            )
+            d[key] = np.asarray(output.squeeze(0).detach().cpu().numpy(), dtype=np.float32)
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
 
-class RandRotated(Randomizable, MapTransform):
+class RandRotated(RandomizableTransform, MapTransform, InvertibleTransform):
     """
     Dictionary-based version :py:class:`monai.transforms.RandRotate`
     Randomly rotates the input arrays.
@@ -851,6 +1292,7 @@ class RandRotated(Randomizable, MapTransform):
             If None, use the data type of input data. To be compatible with other modules,
             the output data type is always ``np.float32``.
             It also can be a sequence of dtype or None, each element corresponds to a key in ``keys``.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -865,8 +1307,10 @@ def __init__(
         padding_mode: GridSamplePadModeSequence = GridSamplePadMode.BORDER,
         align_corners: Union[Sequence[bool], bool] = False,
         dtype: Union[Sequence[DtypeLike], DtypeLike] = np.float64,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
         self.range_x = ensure_tuple(range_x)
         if len(self.range_x) == 1:
             self.range_x = tuple(sorted([-self.range_x[0], self.range_x[0]]))
@@ -877,20 +1321,18 @@ def __init__(
         if len(self.range_z) == 1:
             self.range_z = tuple(sorted([-self.range_z[0], self.range_z[0]]))
 
-        self.prob = prob
         self.keep_size = keep_size
         self.mode = ensure_tuple_rep(mode, len(self.keys))
         self.padding_mode = ensure_tuple_rep(padding_mode, len(self.keys))
         self.align_corners = ensure_tuple_rep(align_corners, len(self.keys))
         self.dtype = ensure_tuple_rep(dtype, len(self.keys))
 
-        self._do_transform = False
         self.x = 0.0
         self.y = 0.0
         self.z = 0.0
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self.x = self.R.uniform(low=self.range_x[0], high=self.range_x[1])
         self.y = self.R.uniform(low=self.range_y[0], high=self.range_y[1])
         self.z = self.R.uniform(low=self.range_z[0], high=self.range_z[1])
@@ -898,24 +1340,72 @@ def randomize(self, data: Optional[Any] = None) -> None:
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         self.randomize()
         d = dict(data)
-        if not self._do_transform:
-            return d
+        angle: Union[Sequence[float], float] = self.x if d[self.keys[0]].ndim == 3 else (self.x, self.y, self.z)
         rotator = Rotate(
-            angle=self.x if d[self.keys[0]].ndim == 3 else (self.x, self.y, self.z),
+            angle=angle,
             keep_size=self.keep_size,
         )
-        for idx, key in enumerate(self.keys):
-            d[key] = rotator(
-                d[key],
-                mode=self.mode[idx],
-                padding_mode=self.padding_mode[idx],
-                align_corners=self.align_corners[idx],
-                dtype=self.dtype[idx],
+        for key, mode, padding_mode, align_corners, dtype in self.key_iterator(
+            d, self.mode, self.padding_mode, self.align_corners, self.dtype
+        ):
+            orig_size = d[key].shape[1:]
+            if self._do_transform:
+                d[key] = rotator(
+                    d[key],
+                    mode=mode,
+                    padding_mode=padding_mode,
+                    align_corners=align_corners,
+                    dtype=dtype,
+                )
+                rot_mat = rotator.get_rotation_matrix()
+            else:
+                rot_mat = np.eye(d[key].ndim)
+            self.push_transform(
+                d,
+                key,
+                orig_size=orig_size,
+                extra_info={
+                    "rot_mat": rot_mat,
+                    "mode": mode.value if isinstance(mode, Enum) else mode,
+                    "padding_mode": padding_mode.value if isinstance(padding_mode, Enum) else padding_mode,
+                    "align_corners": align_corners if align_corners is not None else "none",
+                },
             )
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key, dtype in self.key_iterator(d, self.dtype):
+            transform = self.get_most_recent_transform(d, key)
+            # Check if random transform was actually performed (based on `prob`)
+            if transform[InverseKeys.DO_TRANSFORM]:
+                # Create inverse transform
+                fwd_rot_mat = transform[InverseKeys.EXTRA_INFO]["rot_mat"]
+                mode = transform[InverseKeys.EXTRA_INFO]["mode"]
+                padding_mode = transform[InverseKeys.EXTRA_INFO]["padding_mode"]
+                align_corners = transform[InverseKeys.EXTRA_INFO]["align_corners"]
+                inv_rot_mat = np.linalg.inv(fwd_rot_mat)
+
+                xform = AffineTransform(
+                    normalized=False,
+                    mode=mode,
+                    padding_mode=padding_mode,
+                    align_corners=False if align_corners == "none" else align_corners,
+                    reverse_indexing=True,
+                )
+                output = xform(
+                    torch.as_tensor(np.ascontiguousarray(d[key]).astype(dtype)).unsqueeze(0),
+                    torch.as_tensor(np.ascontiguousarray(inv_rot_mat).astype(dtype)),
+                    spatial_size=transform[InverseKeys.ORIG_SIZE],
+                )
+                d[key] = np.asarray(output.squeeze(0).detach().cpu().numpy(), dtype=np.float32)
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
 
-class Zoomd(MapTransform):
+
+class Zoomd(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.Zoom`.
 
@@ -937,6 +1427,7 @@ class Zoomd(MapTransform):
             See also: https://pytorch.org/docs/stable/nn.functional.html#interpolate
             It also can be a sequence of bool or None, each element corresponds to a key in ``keys``.
         keep_size: Should keep original size (pad if needed), default is True.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -947,8 +1438,9 @@ def __init__(
         padding_mode: NumpyPadModeSequence = NumpyPadMode.EDGE,
         align_corners: Union[Sequence[Optional[bool]], Optional[bool]] = None,
         keep_size: bool = True,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.mode = ensure_tuple_rep(mode, len(self.keys))
         self.padding_mode = ensure_tuple_rep(padding_mode, len(self.keys))
         self.align_corners = ensure_tuple_rep(align_corners, len(self.keys))
@@ -956,17 +1448,52 @@ def __init__(
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for idx, key in enumerate(self.keys):
+        for key, mode, padding_mode, align_corners in self.key_iterator(
+            d, self.mode, self.padding_mode, self.align_corners
+        ):
+            self.push_transform(
+                d,
+                key,
+                extra_info={
+                    "mode": mode.value if isinstance(mode, Enum) else mode,
+                    "padding_mode": padding_mode.value if isinstance(padding_mode, Enum) else padding_mode,
+                    "align_corners": align_corners if align_corners is not None else "none",
+                },
+            )
             d[key] = self.zoomer(
                 d[key],
-                mode=self.mode[idx],
-                padding_mode=self.padding_mode[idx],
-                align_corners=self.align_corners[idx],
+                mode=mode,
+                padding_mode=padding_mode,
+                align_corners=align_corners,
             )
         return d
 
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Create inverse transform
+            zoom = np.array(self.zoomer.zoom)
+            inverse_transform = Zoom(zoom=1 / zoom, keep_size=self.zoomer.keep_size)
+            mode = transform[InverseKeys.EXTRA_INFO]["mode"]
+            padding_mode = transform[InverseKeys.EXTRA_INFO]["padding_mode"]
+            align_corners = transform[InverseKeys.EXTRA_INFO]["align_corners"]
+            # Apply inverse
+            d[key] = inverse_transform(
+                d[key],
+                mode=mode,
+                padding_mode=padding_mode,
+                align_corners=None if align_corners == "none" else align_corners,
+            )
+            # Size might be out by 1 voxel so pad
+            d[key] = SpatialPad(transform[InverseKeys.ORIG_SIZE], mode="edge")(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
+        return d
+
 
-class RandZoomd(Randomizable, MapTransform):
+class RandZoomd(RandomizableTransform, MapTransform, InvertibleTransform):
     """
     Dict-based version :py:class:`monai.transforms.RandZoom`.
 
@@ -996,6 +1523,7 @@ class RandZoomd(Randomizable, MapTransform):
             See also: https://pytorch.org/docs/stable/nn.functional.html#interpolate
             It also can be a sequence of bool or None, each element corresponds to a key in ``keys``.
         keep_size: Should keep original size (pad if needed), default is True.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -1008,32 +1536,30 @@ def __init__(
         padding_mode: NumpyPadModeSequence = NumpyPadMode.EDGE,
         align_corners: Union[Sequence[Optional[bool]], Optional[bool]] = None,
         keep_size: bool = True,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
+        RandomizableTransform.__init__(self, prob)
         self.min_zoom = ensure_tuple(min_zoom)
         self.max_zoom = ensure_tuple(max_zoom)
         if len(self.min_zoom) != len(self.max_zoom):
             raise AssertionError("min_zoom and max_zoom must have same length.")
-        self.prob = prob
 
         self.mode = ensure_tuple_rep(mode, len(self.keys))
         self.padding_mode = ensure_tuple_rep(padding_mode, len(self.keys))
         self.align_corners = ensure_tuple_rep(align_corners, len(self.keys))
         self.keep_size = keep_size
 
-        self._do_transform = False
         self._zoom: Sequence[float] = [1.0]
 
     def randomize(self, data: Optional[Any] = None) -> None:
-        self._do_transform = self.R.random_sample() < self.prob
+        super().randomize(None)
         self._zoom = [self.R.uniform(l, h) for l, h in zip(self.min_zoom, self.max_zoom)]
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         # match the spatial dim of first item
         self.randomize()
         d = dict(data)
-        if not self._do_transform:
-            return d
 
         img_dims = data[self.keys[0]].ndim
         if len(self._zoom) == 1:
@@ -1043,13 +1569,52 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
             # if 2 zoom factors provided for 3D data, use the first factor for H and W dims, second factor for D dim
             self._zoom = ensure_tuple_rep(self._zoom[0], img_dims - 2) + ensure_tuple(self._zoom[-1])
         zoomer = Zoom(self._zoom, keep_size=self.keep_size)
-        for idx, key in enumerate(self.keys):
-            d[key] = zoomer(
-                d[key],
-                mode=self.mode[idx],
-                padding_mode=self.padding_mode[idx],
-                align_corners=self.align_corners[idx],
+        for key, mode, padding_mode, align_corners in self.key_iterator(
+            d, self.mode, self.padding_mode, self.align_corners
+        ):
+            self.push_transform(
+                d,
+                key,
+                extra_info={
+                    "zoom": self._zoom,
+                    "mode": mode.value if isinstance(mode, Enum) else mode,
+                    "padding_mode": padding_mode.value if isinstance(padding_mode, Enum) else padding_mode,
+                    "align_corners": align_corners if align_corners is not None else "none",
+                },
             )
+            if self._do_transform:
+                d[key] = zoomer(
+                    d[key],
+                    mode=mode,
+                    padding_mode=padding_mode,
+                    align_corners=align_corners,
+                )
+        return d
+
+    def inverse(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            transform = self.get_most_recent_transform(d, key)
+            # Check if random transform was actually performed (based on `prob`)
+            if transform[InverseKeys.DO_TRANSFORM]:
+                # Create inverse transform
+                zoom = np.array(transform[InverseKeys.EXTRA_INFO]["zoom"])
+                mode = transform[InverseKeys.EXTRA_INFO]["mode"]
+                padding_mode = transform[InverseKeys.EXTRA_INFO]["padding_mode"]
+                align_corners = transform[InverseKeys.EXTRA_INFO]["align_corners"]
+                inverse_transform = Zoom(zoom=1 / zoom, keep_size=self.keep_size)
+                # Apply inverse
+                d[key] = inverse_transform(
+                    d[key],
+                    mode=mode,
+                    padding_mode=padding_mode,
+                    align_corners=None if align_corners == "none" else align_corners,
+                )
+                # Size might be out by 1 voxel so pad
+                d[key] = SpatialPad(transform[InverseKeys.ORIG_SIZE], mode="edge")(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+
         return d
 
 
@@ -1058,11 +1623,13 @@ def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.nda
 Rotate90D = Rotate90Dict = Rotate90d
 RandRotate90D = RandRotate90Dict = RandRotate90d
 ResizeD = ResizeDict = Resized
+AffineD = AffineDict = Affined
 RandAffineD = RandAffineDict = RandAffined
 Rand2DElasticD = Rand2DElasticDict = Rand2DElasticd
 Rand3DElasticD = Rand3DElasticDict = Rand3DElasticd
 FlipD = FlipDict = Flipd
 RandFlipD = RandFlipDict = RandFlipd
+RandAxisFlipD = RandAxisFlipDict = RandAxisFlipd
 RotateD = RotateDict = Rotated
 RandRotateD = RandRotateDict = RandRotated
 ZoomD = ZoomDict = Zoomd
diff --git a/monai/transforms/transform.py b/monai/transforms/transform.py
new file mode 100644
index 0000000000..ff5f021739
--- /dev/null
+++ b/monai/transforms/transform.py
@@ -0,0 +1,311 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+A collection of generic interfaces for MONAI transforms.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Callable, Dict, Generator, Hashable, Iterable, List, Optional, Tuple
+
+import numpy as np
+import torch
+
+from monai import transforms
+from monai.config import KeysCollection
+from monai.utils import MAX_SEED, ensure_tuple
+
+__all__ = ["apply_transform", "Randomizable", "RandomizableTransform", "Transform", "MapTransform"]
+
+
+def apply_transform(transform: Callable, data, map_items: bool = True):
+    """
+    Transform `data` with `transform`.
+    If `data` is a list or tuple and `map_data` is True, each item of `data` will be transformed
+    and this method returns a list of outcomes.
+    otherwise transform will be applied once with `data` as the argument.
+
+    Args:
+        transform: a callable to be used to transform `data`
+        data: an object to be transformed.
+        map_items: whether to apply transform to each item in `data`,
+            if `data` is a list or tuple. Defaults to True.
+
+    Raises:
+        Exception: When ``transform`` raises an exception.
+
+    """
+    try:
+        if isinstance(data, (list, tuple)) and map_items:
+            return [transform(item) for item in data]
+        return transform(data)
+    except Exception as e:
+
+        if not isinstance(transform, transforms.compose.Compose):
+            # log the input data information of exact transform in the transform chain
+            datastats = transforms.utility.array.DataStats(data_shape=False, value_range=False)
+            datastats._logger.info("input data information of the runtime error transform:")
+            if isinstance(data, (list, tuple)):
+                data = data[0]
+
+            def _log_stats(data, prefix: Optional[str] = "Data"):
+                if isinstance(data, (np.ndarray, torch.Tensor)):
+                    # log data type, shape, range for array
+                    datastats(img=data, data_shape=True, value_range=True, prefix=prefix)  # type: ignore
+                else:
+                    # log data type and value for other meta data
+                    datastats(img=data, data_value=True, prefix=prefix)
+
+            if isinstance(data, dict):
+                for k, v in data.items():
+                    _log_stats(data=v, prefix=k)
+            else:
+                _log_stats(data=data)
+        raise RuntimeError(f"applying transform {transform}") from e
+
+
+class Randomizable(ABC):
+    """
+    An interface for handling random state locally, currently based on a class variable `R`,
+    which is an instance of `np.random.RandomState`.
+    """
+
+    R: np.random.RandomState = np.random.RandomState()
+
+    def set_random_state(
+        self, seed: Optional[int] = None, state: Optional[np.random.RandomState] = None
+    ) -> "Randomizable":
+        """
+        Set the random state locally, to control the randomness, the derived
+        classes should use :py:attr:`self.R` instead of `np.random` to introduce random
+        factors.
+
+        Args:
+            seed: set the random state with an integer seed.
+            state: set the random state with a `np.random.RandomState` object.
+
+        Raises:
+            TypeError: When ``state`` is not an ``Optional[np.random.RandomState]``.
+
+        Returns:
+            a Randomizable instance.
+
+        """
+        if seed is not None:
+            _seed = id(seed) if not isinstance(seed, (int, np.integer)) else seed
+            _seed = _seed % MAX_SEED
+            self.R = np.random.RandomState(_seed)
+            return self
+
+        if state is not None:
+            if not isinstance(state, np.random.RandomState):
+                raise TypeError(f"state must be None or a np.random.RandomState but is {type(state).__name__}.")
+            self.R = state
+            return self
+
+        self.R = np.random.RandomState()
+        return self
+
+    def randomize(self, data: Any) -> None:
+        """
+        Within this method, :py:attr:`self.R` should be used, instead of `np.random`, to introduce random factors.
+
+        all :py:attr:`self.R` calls happen here so that we have a better chance to
+        identify errors of sync the random state.
+
+        This method can generate the random factors based on properties of the input data.
+
+        Raises:
+            NotImplementedError: When the subclass does not override this method.
+        """
+        raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.")
+
+
+class Transform(ABC):
+    """
+    An abstract class of a ``Transform``.
+    A transform is callable that processes ``data``.
+
+    It could be stateful and may modify ``data`` in place,
+    the implementation should be aware of:
+
+        #. thread safety when mutating its own states.
+           When used from a multi-process context, transform's instance variables are read-only.
+        #. ``data`` content unused by this transform may still be used in the
+           subsequent transforms in a composed transform.
+        #. storing too much information in ``data`` may not scale.
+
+    See Also
+
+        :py:class:`monai.transforms.Compose`
+    """
+
+    @abstractmethod
+    def __call__(self, data: Any):
+        """
+        ``data`` is an element which often comes from an iteration over an
+        iterable, such as :py:class:`torch.utils.data.Dataset`. This method should
+        return an updated version of ``data``.
+        To simplify the input validations, most of the transforms assume that
+
+        - ``data`` is a Numpy ndarray, PyTorch Tensor or string
+        - the data shape can be:
+
+          #. string data without shape, `LoadImage` transform expects file paths
+          #. most of the pre-processing transforms expect: ``(num_channels, spatial_dim_1[, spatial_dim_2, ...])``,
+             except that `AddChannel` expects (spatial_dim_1[, spatial_dim_2, ...]) and
+             `AsChannelFirst` expects (spatial_dim_1[, spatial_dim_2, ...], num_channels)
+          #. most of the post-processing transforms expect
+             ``(batch_size, num_channels, spatial_dim_1[, spatial_dim_2, ...])``
+
+        - the channel dimension is not omitted even if number of channels is one
+
+        This method can optionally take additional arguments to help execute transformation operation.
+
+        Raises:
+            NotImplementedError: When the subclass does not override this method.
+
+        """
+        raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.")
+
+
+class RandomizableTransform(Randomizable, Transform):
+    """
+    An interface for handling random state locally, currently based on a class variable `R`,
+    which is an instance of `np.random.RandomState`.
+    This class introduces a randomized flag `_do_transform`, is mainly for randomized data augmentation transforms.
+    For example:
+
+    .. code-block:: python
+
+        from monai.transforms import RandomizableTransform
+
+        class RandShiftIntensity100(RandomizableTransform):
+            def randomize(self):
+                super().randomize(None)
+                self._offset = self.R.uniform(low=0, high=100)
+
+            def __call__(self, img):
+                self.randomize()
+                if not self._do_transform:
+                    return img
+                return img + self._offset
+
+        transform = RandShiftIntensity()
+        transform.set_random_state(seed=0)
+        print(transform(10))
+
+    """
+
+    def __init__(self, prob: float = 1.0, do_transform: bool = True):
+        self._do_transform = do_transform
+        self.prob = min(max(prob, 0.0), 1.0)
+
+    def randomize(self, data: Any) -> None:
+        """
+        Within this method, :py:attr:`self.R` should be used, instead of `np.random`, to introduce random factors.
+
+        all :py:attr:`self.R` calls happen here so that we have a better chance to
+        identify errors of sync the random state.
+
+        This method can generate the random factors based on properties of the input data.
+        """
+        self._do_transform = self.R.rand() < self.prob
+
+
+class MapTransform(Transform):
+    """
+    A subclass of :py:class:`monai.transforms.Transform` with an assumption
+    that the ``data`` input of ``self.__call__`` is a MutableMapping such as ``dict``.
+
+    The ``keys`` parameter will be used to get and set the actual data
+    item to transform.  That is, the callable of this transform should
+    follow the pattern:
+
+        .. code-block:: python
+
+            def __call__(self, data):
+                for key in self.keys:
+                    if key in data:
+                        # update output data with some_transform_function(data[key]).
+                    else:
+                        # raise exception unless allow_missing_keys==True.
+                return data
+
+    Raises:
+        ValueError: When ``keys`` is an empty iterable.
+        TypeError: When ``keys`` type is not in ``Union[Hashable, Iterable[Hashable]]``.
+
+    """
+
+    def __init__(self, keys: KeysCollection, allow_missing_keys: bool = False) -> None:
+        self.keys: Tuple[Hashable, ...] = ensure_tuple(keys)
+        self.allow_missing_keys = allow_missing_keys
+        if not self.keys:
+            raise ValueError("keys must be non empty.")
+        for key in self.keys:
+            if not isinstance(key, Hashable):
+                raise TypeError(f"keys must be one of (Hashable, Iterable[Hashable]) but is {type(keys).__name__}.")
+
+    @abstractmethod
+    def __call__(self, data):
+        """
+        ``data`` often comes from an iteration over an iterable,
+        such as :py:class:`torch.utils.data.Dataset`.
+
+        To simplify the input validations, this method assumes:
+
+        - ``data`` is a Python dictionary
+        - ``data[key]`` is a Numpy ndarray, PyTorch Tensor or string, where ``key`` is an element
+          of ``self.keys``, the data shape can be:
+
+          #. string data without shape, `LoadImaged` transform expects file paths
+          #. most of the pre-processing transforms expect: ``(num_channels, spatial_dim_1[, spatial_dim_2, ...])``,
+             except that `AddChanneld` expects (spatial_dim_1[, spatial_dim_2, ...]) and
+             `AsChannelFirstd` expects (spatial_dim_1[, spatial_dim_2, ...], num_channels)
+          #. most of the post-processing transforms expect
+             ``(batch_size, num_channels, spatial_dim_1[, spatial_dim_2, ...])``
+
+        - the channel dimension is not omitted even if number of channels is one
+
+        Raises:
+            NotImplementedError: When the subclass does not override this method.
+
+        returns:
+            An updated dictionary version of ``data`` by applying the transform.
+
+        """
+        raise NotImplementedError(f"Subclass {self.__class__.__name__} must implement this method.")
+
+    def key_iterator(
+        self,
+        data: Dict[Hashable, Any],
+        *extra_iterables: Optional[Iterable],
+    ) -> Generator:
+        """
+        Iterate across keys and optionally extra iterables. If key is missing, exception is raised if
+        `allow_missing_keys==False` (default). If `allow_missing_keys==True`, key is skipped.
+
+        Args:
+            data: data that the transform will be applied to
+            extra_iterables: anything else to be iterated through
+        """
+        # if no extra iterables given, create a dummy list of Nones
+        ex_iters = extra_iterables if extra_iterables else [[None] * len(self.keys)]
+
+        # loop over keys and any extra iterables
+        _ex_iters: List[Any]
+        for key, *_ex_iters in zip(self.keys, *ex_iters):
+            # all normal, yield (what we yield depends on whether extra iterables were given)
+            if key in data.keys():
+                yield (key,) + tuple(_ex_iters) if extra_iterables else key
+            # if missing keys not allowed, raise
+            elif not self.allow_missing_keys:
+                raise KeyError(f"Key was missing ({key}) and allow_missing_keys==False")
diff --git a/monai/transforms/utility/array.py b/monai/transforms/utility/array.py
index c0ae40de59..6232b15d02 100644
--- a/monai/transforms/utility/array.py
+++ b/monai/transforms/utility/array.py
@@ -14,23 +14,29 @@
 """
 
 import logging
+import sys
 import time
-from typing import Callable, List, Optional, Sequence, Tuple, Union
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
 import torch
 
 from monai.config import DtypeLike, NdarrayTensor
-from monai.transforms.compose import Randomizable, Transform
+from monai.transforms.transform import Randomizable, Transform
 from monai.transforms.utils import extreme_points_to_image, get_extreme_points, map_binary_to_indices
 from monai.utils import ensure_tuple, min_version, optional_import
 
+PILImageImage, has_pil = optional_import("PIL.Image", name="Image")
+pil_image_fromarray, _ = optional_import("PIL.Image", name="fromarray")
+
 __all__ = [
     "Identity",
     "AsChannelFirst",
     "AsChannelLast",
     "AddChannel",
+    "EnsureChannelFirst",
     "RepeatChannel",
+    "RemoveRepeatedChannel",
     "SplitChannel",
     "CastToType",
     "ToTensor",
@@ -45,6 +51,7 @@
     "ConvertToMultiChannelBasedOnBratsClasses",
     "AddExtremePointsChannel",
     "TorchVision",
+    "MapLabelValue",
 ]
 
 
@@ -139,6 +146,31 @@ def __call__(self, img: NdarrayTensor):
         return img[None]
 
 
+class EnsureChannelFirst(Transform):
+    """
+    Automatically adjust or add the channel dimension of input data to ensure `channel_first` shape.
+    It extracts the `original_channel_dim` info from provided meta_data dictionary.
+    Typical values of `original_channel_dim` can be: "no_channel", 0, -1.
+    Convert the data to `channel_first` based on the `original_channel_dim` information.
+
+    """
+
+    def __call__(self, img: np.ndarray, meta_dict: Optional[Dict] = None):
+        """
+        Apply the transform to `img`.
+        """
+        if not isinstance(meta_dict, dict):
+            raise ValueError("meta_dict must be a dictionary data.")
+
+        channel_dim = meta_dict.get("original_channel_dim", None)
+
+        if channel_dim is None:
+            raise ValueError("meta_dict must contain `original_channel_dim` information.")
+        if channel_dim == "no_channel":
+            return AddChannel()(img)
+        return AsChannelFirst(channel_dim=channel_dim)(img)
+
+
 class RepeatChannel(Transform):
     """
     Repeat channel data to construct expected input shape for models.
@@ -161,6 +193,32 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         return np.repeat(img, self.repeats, 0)
 
 
+class RemoveRepeatedChannel(Transform):
+    """
+    RemoveRepeatedChannel data to undo RepeatChannel
+    The `repeats` count specifies the deletion of the origin data, for example:
+    ``RemoveRepeatedChannel(repeats=2)([[1, 2], [1, 2], [3, 4], [3, 4]])`` generates: ``[[1, 2], [3, 4]]``
+
+    Args:
+        repeats: the number of repetitions to be deleted for each element.
+    """
+
+    def __init__(self, repeats: int) -> None:
+        if repeats <= 0:
+            raise AssertionError("repeats count must be greater than 0.")
+
+        self.repeats = repeats
+
+    def __call__(self, img: np.ndarray) -> np.ndarray:
+        """
+        Apply the transform to `img`, assuming `img` is a "channel-first" array.
+        """
+        if np.shape(img)[0] < 2:
+            raise AssertionError("Image must have more than one channel")
+
+        return np.array(img[:: self.repeats, :])
+
+
 class SplitChannel(Transform):
     """
     Split Numpy array or PyTorch Tensor data according to the channel dim.
@@ -238,7 +296,7 @@ class ToTensor(Transform):
     Converts the input image to a tensor without applying any other transformations.
     """
 
-    def __call__(self, img: Union[np.ndarray, torch.Tensor]) -> torch.Tensor:
+    def __call__(self, img) -> torch.Tensor:
         """
         Apply the transform to `img` and make it contiguous.
         """
@@ -252,7 +310,7 @@ class ToNumpy(Transform):
     Converts the input data to numpy array, can support list or tuple of numbers and PyTorch Tensor.
     """
 
-    def __call__(self, img: Union[List, Tuple, np.ndarray, torch.Tensor]) -> np.ndarray:
+    def __call__(self, img) -> np.ndarray:
         """
         Apply the transform to `img` and make it contiguous.
         """
@@ -261,6 +319,22 @@ def __call__(self, img: Union[List, Tuple, np.ndarray, torch.Tensor]) -> np.ndar
         return np.ascontiguousarray(img)
 
 
+class ToPIL(Transform):
+    """
+    Converts the input image (in the form of NumPy array or PyTorch Tensor) to PIL image
+    """
+
+    def __call__(self, img):
+        """
+        Apply the transform to `img` and make it contiguous.
+        """
+        if isinstance(img, PILImageImage):
+            return img
+        if isinstance(img, torch.Tensor):
+            img = img.detach().cpu().numpy()
+        return pil_image_fromarray(img)
+
+
 class Transpose(Transform):
     """
     Transposes the input image based on the given `indices` dimension ordering.
@@ -314,6 +388,7 @@ class DataStats(Transform):
     def __init__(
         self,
         prefix: str = "Data",
+        data_type: bool = True,
         data_shape: bool = True,
         value_range: bool = True,
         data_value: bool = False,
@@ -323,6 +398,7 @@ def __init__(
         """
         Args:
             prefix: will be printed in format: "{prefix} statistics".
+            data_type: whether to show the type of input data.
             data_shape: whether to show the shape of input data.
             value_range: whether to show the value range of input data.
             data_value: whether to show the raw value of input data.
@@ -330,6 +406,7 @@ def __init__(
             additional_info: user can define callable function to extract additional info from input data.
             logger_handler: add additional handler to output data: save to file, etc.
                 add existing python logging handlers: https://docs.python.org/3/library/logging.handlers.html
+                the handler should have a logging level of at least `INFO`.
 
         Raises:
             TypeError: When ``additional_info`` is not an ``Optional[Callable]``.
@@ -338,6 +415,7 @@ def __init__(
         if not isinstance(prefix, str):
             raise AssertionError("prefix must be a string.")
         self.prefix = prefix
+        self.data_type = data_type
         self.data_shape = data_shape
         self.value_range = value_range
         self.data_value = data_value
@@ -345,8 +423,11 @@ def __init__(
             raise TypeError(f"additional_info must be None or callable but is {type(additional_info).__name__}.")
         self.additional_info = additional_info
         self.output: Optional[str] = None
-        logging.basicConfig(level=logging.NOTSET)
         self._logger = logging.getLogger("DataStats")
+        self._logger.setLevel(logging.INFO)
+        console = logging.StreamHandler(sys.stdout)  # always stdout
+        console.setLevel(logging.INFO)
+        self._logger.addHandler(console)
         if logger_handler is not None:
             self._logger.addHandler(logger_handler)
 
@@ -354,6 +435,7 @@ def __call__(
         self,
         img: NdarrayTensor,
         prefix: Optional[str] = None,
+        data_type: Optional[bool] = None,
         data_shape: Optional[bool] = None,
         value_range: Optional[bool] = None,
         data_value: Optional[bool] = None,
@@ -364,6 +446,8 @@ def __call__(
         """
         lines = [f"{prefix or self.prefix} statistics:"]
 
+        if self.data_type if data_type is None else data_type:
+            lines.append(f"Type: {type(img)}")
         if self.data_shape if data_shape is None else data_shape:
             lines.append(f"Shape: {img.shape}")
         if self.value_range if value_range is None else value_range:
@@ -380,7 +464,7 @@ def __call__(
             lines.append(f"Additional info: {additional_info(img)}")
         separator = "\n"
         self.output = f"{separator.join(lines)}"
-        self._logger.debug(self.output)
+        self._logger.info(self.output)
 
         return img
 
@@ -569,6 +653,10 @@ class ConvertToMultiChannelBasedOnBratsClasses(Transform):
     """
 
     def __call__(self, img: np.ndarray) -> np.ndarray:
+        # if img has channel dim, squeeze it
+        if img.ndim == 4 and img.shape[0] == 1:
+            img = np.squeeze(img, axis=0)
+
         result = []
         # merge labels 1 (tumor non-enh) and 4 (tumor enh) to TC
         result.append(np.logical_or(img == 1, img == 4))
@@ -576,10 +664,10 @@ def __call__(self, img: np.ndarray) -> np.ndarray:
         result.append(np.logical_or(np.logical_or(img == 1, img == 4), img == 2))
         # label 4 is ET
         result.append(img == 4)
-        return np.stack(result, axis=0).astype(np.float32)
+        return np.stack(result, axis=0)
 
 
-class AddExtremePointsChannel(Transform, Randomizable):
+class AddExtremePointsChannel(Randomizable):
     """
     Add extreme points of label to the image as a new channel. This transform generates extreme
     point from label and applies a gaussian filter. The pixel values in points image are rescaled
@@ -668,3 +756,46 @@ def __call__(self, img: torch.Tensor):
 
         """
         return self.trans(img)
+
+
+class MapLabelValue:
+    """
+    Utility to map label values to another set of values.
+    For example, map [3, 2, 1] to [0, 1, 2], [1, 2, 3] -> [0.5, 1.5, 2.5], ["label3", "label2", "label1"] -> [0, 1, 2],
+    [3.5, 2.5, 1.5] -> ["label0", "label1", "label2"], etc.
+    The label data must be numpy array or array-like data and the output data will be numpy array.
+
+    """
+
+    def __init__(self, orig_labels: Sequence, target_labels: Sequence, dtype: DtypeLike = np.float32) -> None:
+        """
+        Args:
+            orig_labels: original labels that map to others.
+            target_labels: expected label values, 1: 1 map to the `orig_labels`.
+            dtype: convert the output data to dtype, default to float32.
+
+        """
+        if len(orig_labels) != len(target_labels):
+            raise ValueError("orig_labels and target_labels must have the same length.")
+        if all([o == z for o, z in zip(orig_labels, target_labels)]):
+            raise ValueError("orig_labels and target_labels are exactly the same, should be different to map.")
+
+        self.orig_labels = orig_labels
+        self.target_labels = target_labels
+        self.dtype = dtype
+
+    def __call__(self, img: np.ndarray):
+        img = np.asarray(img)
+        img_flat = img.flatten()
+        try:
+            out_flat = np.copy(img_flat).astype(self.dtype)
+        except ValueError:
+            # can't copy unchanged labels as the expected dtype is not supported, must map all the label values
+            out_flat = np.zeros(shape=img_flat.shape, dtype=self.dtype)
+
+        for o, t in zip(self.orig_labels, self.target_labels):
+            if o == t:
+                continue
+            np.place(out_flat, img_flat == o, t)
+
+        return out_flat.reshape(img.shape)
diff --git a/monai/transforms/utility/dictionary.py b/monai/transforms/utility/dictionary.py
index f374b82d76..67da9ceb35 100644
--- a/monai/transforms/utility/dictionary.py
+++ b/monai/transforms/utility/dictionary.py
@@ -17,13 +17,15 @@
 
 import copy
 import logging
+from copy import deepcopy
 from typing import Any, Callable, Dict, Hashable, List, Mapping, Optional, Sequence, Tuple, Union
 
 import numpy as np
 import torch
 
 from monai.config import DtypeLike, KeysCollection, NdarrayTensor
-from monai.transforms.compose import MapTransform, Randomizable
+from monai.transforms.inverse import InvertibleTransform
+from monai.transforms.transform import MapTransform, Randomizable
 from monai.transforms.utility.array import (
     AddChannel,
     AsChannelFirst,
@@ -31,15 +33,19 @@
     CastToType,
     ConvertToMultiChannelBasedOnBratsClasses,
     DataStats,
+    EnsureChannelFirst,
     FgBgToIndices,
     Identity,
     LabelToMask,
     Lambda,
+    MapLabelValue,
+    RemoveRepeatedChannel,
     RepeatChannel,
     SimulateDelay,
     SplitChannel,
     SqueezeDim,
     ToNumpy,
+    ToPIL,
     TorchVision,
     ToTensor,
 )
@@ -51,11 +57,14 @@
     "AsChannelFirstd",
     "AsChannelLastd",
     "AddChanneld",
+    "EnsureChannelFirstd",
     "RepeatChanneld",
+    "RemoveRepeatedChanneld",
     "SplitChanneld",
     "CastToTyped",
     "ToTensord",
     "ToNumpyd",
+    "ToPILd",
     "DeleteItemsd",
     "SelectItemsd",
     "SqueezeDimd",
@@ -70,6 +79,7 @@
     "ConvertToMultiChannelBasedOnBratsClassesd",
     "AddExtremePointsChanneld",
     "TorchVisiond",
+    "MapLabelValued",
     "IdentityD",
     "IdentityDict",
     "AsChannelFirstD",
@@ -78,10 +88,14 @@
     "AsChannelLastDict",
     "AddChannelD",
     "AddChannelDict",
+    "EnsureChannelFirstD",
+    "EnsureChannelFirstDict",
     "RandLambdaD",
     "RandLambdaDict",
     "RepeatChannelD",
     "RepeatChannelDict",
+    "RemoveRepeatedChannelD",
+    "RemoveRepeatedChannelDict",
     "SplitChannelD",
     "SplitChannelDict",
     "CastToTypeD",
@@ -112,6 +126,8 @@
     "AddExtremePointsChannelDict",
     "TorchVisionD",
     "TorchVisionDict",
+    "MapLabelValueD",
+    "MapLabelValueDict",
 ]
 
 
@@ -120,21 +136,22 @@ class Identityd(MapTransform):
     Dictionary-based wrapper of :py:class:`monai.transforms.Identity`.
     """
 
-    def __init__(self, keys: KeysCollection) -> None:
+    def __init__(self, keys: KeysCollection, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.identity = Identity()
 
     def __call__(
         self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
     ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.identity(d[key])
         return d
 
@@ -144,19 +161,20 @@ class AsChannelFirstd(MapTransform):
     Dictionary-based wrapper of :py:class:`monai.transforms.AsChannelFirst`.
     """
 
-    def __init__(self, keys: KeysCollection, channel_dim: int = -1) -> None:
+    def __init__(self, keys: KeysCollection, channel_dim: int = -1, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             channel_dim: which dimension of input image is the channel, default is the last dimension.
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = AsChannelFirst(channel_dim=channel_dim)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
         return d
 
@@ -166,19 +184,20 @@ class AsChannelLastd(MapTransform):
     Dictionary-based wrapper of :py:class:`monai.transforms.AsChannelLast`.
     """
 
-    def __init__(self, keys: KeysCollection, channel_dim: int = 0) -> None:
+    def __init__(self, keys: KeysCollection, channel_dim: int = 0, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             channel_dim: which dimension of input image is the channel, default is the first dimension.
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = AsChannelLast(channel_dim=channel_dim)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
         return d
 
@@ -188,40 +207,91 @@ class AddChanneld(MapTransform):
     Dictionary-based wrapper of :py:class:`monai.transforms.AddChannel`.
     """
 
-    def __init__(self, keys: KeysCollection) -> None:
+    def __init__(self, keys: KeysCollection, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.adder = AddChannel()
 
     def __call__(self, data: Mapping[Hashable, NdarrayTensor]) -> Dict[Hashable, NdarrayTensor]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.adder(d[key])
         return d
 
 
+class EnsureChannelFirstd(MapTransform):
+    """
+    Dictionary-based wrapper of :py:class:`monai.transforms.EnsureChannelFirst`.
+    """
+
+    def __init__(self, keys: KeysCollection, meta_key_postfix: str = "meta_dict") -> None:
+        """
+        Args:
+            keys: keys of the corresponding items to be transformed.
+                See also: :py:class:`monai.transforms.compose.MapTransform`
+            meta_key_postfix: `key_{postfix}` was used to store the metadata in `LoadImaged`.
+                So need the key to extract metadata for channel dim information, default is `meta_dict`.
+                For example, for data with key `image`, metadata by default is in `image_meta_dict`.
+
+        """
+        super().__init__(keys)
+        self.adjuster = EnsureChannelFirst()
+        self.meta_key_postfix = meta_key_postfix
+
+    def __call__(self, data) -> Dict[Hashable, np.ndarray]:
+        d = dict(data)
+        for key in self.keys:
+            d[key] = self.adjuster(d[key], d[f"{key}_{self.meta_key_postfix}"])
+        return d
+
+
 class RepeatChanneld(MapTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.RepeatChannel`.
     """
 
-    def __init__(self, keys: KeysCollection, repeats: int) -> None:
+    def __init__(self, keys: KeysCollection, repeats: int, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             repeats: the number of repetitions for each element.
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.repeater = RepeatChannel(repeats)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            d[key] = self.repeater(d[key])
+        return d
+
+
+class RemoveRepeatedChanneld(MapTransform):
+    """
+    Dictionary-based wrapper of :py:class:`monai.transforms.RemoveRepeatedChannel`.
+    """
+
+    def __init__(self, keys: KeysCollection, repeats: int, allow_missing_keys: bool = False) -> None:
+        """
+        Args:
+            keys: keys of the corresponding items to be transformed.
+                See also: :py:class:`monai.transforms.compose.MapTransform`
+            repeats: the number of repetitions for each element.
+            allow_missing_keys: don't raise exception if key is missing.
+        """
+        super().__init__(keys, allow_missing_keys)
+        self.repeater = RemoveRepeatedChannel(repeats)
+
+    def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = dict(data)
+        for key in self.key_iterator(d):
             d[key] = self.repeater(d[key])
         return d
 
@@ -238,6 +308,7 @@ def __init__(
         keys: KeysCollection,
         output_postfixes: Optional[Sequence[str]] = None,
         channel_dim: Optional[int] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -251,9 +322,10 @@ def __init__(
                 to automatically select: if data is numpy array, channel_dim is 0 as
                 `numpy array` is used in the pre transforms, if PyTorch Tensor, channel_dim
                 is 1 as in most of the cases `Tensor` is uses in the post transforms.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.output_postfixes = output_postfixes
         self.splitter = SplitChannel(channel_dim=channel_dim)
 
@@ -261,7 +333,7 @@ def __call__(
         self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
     ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             rets = self.splitter(d[key])
             postfixes: Sequence = list(range(len(rets))) if self.output_postfixes is None else self.output_postfixes
             if len(postfixes) != len(rets):
@@ -283,6 +355,7 @@ def __init__(
         self,
         keys: KeysCollection,
         dtype: Union[Sequence[Union[DtypeLike, torch.dtype]], DtypeLike, torch.dtype] = np.float32,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -291,9 +364,10 @@ def __init__(
             dtype: convert image to this data type, default is `np.float32`.
                 it also can be a sequence of dtypes or torch.dtype,
                 each element corresponds to a key in ``keys``.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        MapTransform.__init__(self, keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
         self.dtype = ensure_tuple_rep(dtype, len(self.keys))
         self.converter = CastToType()
 
@@ -301,54 +375,86 @@ def __call__(
         self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
     ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
         d = dict(data)
-        for idx, key in enumerate(self.keys):
-            d[key] = self.converter(d[key], dtype=self.dtype[idx])
+        for key, dtype in self.key_iterator(d, self.dtype):
+            d[key] = self.converter(d[key], dtype=dtype)
 
         return d
 
 
-class ToTensord(MapTransform):
+class ToTensord(MapTransform, InvertibleTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.ToTensor`.
     """
 
-    def __init__(self, keys: KeysCollection) -> None:
+    def __init__(self, keys: KeysCollection, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = ToTensor()
 
-    def __call__(
-        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
-    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
+    def __call__(self, data: Mapping[Hashable, Any]) -> Dict[Hashable, Any]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            self.push_transform(d, key)
             d[key] = self.converter(d[key])
         return d
 
+    def inverse(self, data: Mapping[Hashable, Any]) -> Dict[Hashable, Any]:
+        d = deepcopy(dict(data))
+        for key in self.key_iterator(d):
+            # Create inverse transform
+            inverse_transform = ToNumpy()
+            # Apply inverse
+            d[key] = inverse_transform(d[key])
+            # Remove the applied transform
+            self.pop_transform(d, key)
+        return d
+
 
 class ToNumpyd(MapTransform):
     """
     Dictionary-based wrapper of :py:class:`monai.transforms.ToNumpy`.
     """
 
-    def __init__(self, keys: KeysCollection) -> None:
+    def __init__(self, keys: KeysCollection, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = ToNumpy()
 
-    def __call__(
-        self, data: Mapping[Hashable, Union[np.ndarray, torch.Tensor]]
-    ) -> Dict[Hashable, Union[np.ndarray, torch.Tensor]]:
+    def __call__(self, data: Mapping[Hashable, Any]) -> Dict[Hashable, Any]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
+            d[key] = self.converter(d[key])
+        return d
+
+
+class ToPILd(MapTransform):
+    """
+    Dictionary-based wrapper of :py:class:`monai.transforms.ToNumpy`.
+    """
+
+    def __init__(self, keys: KeysCollection, allow_missing_keys: bool = False) -> None:
+        """
+        Args:
+            keys: keys of the corresponding items to be transformed.
+                See also: :py:class:`monai.transforms.compose.MapTransform`
+            allow_missing_keys: don't raise exception if key is missing.
+        """
+        super().__init__(keys, allow_missing_keys)
+        self.converter = ToPIL()
+
+    def __call__(self, data: Mapping[Hashable, Any]) -> Dict[Hashable, Any]:
+        d = dict(data)
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
         return d
 
@@ -360,7 +466,7 @@ class DeleteItemsd(MapTransform):
     """
 
     def __call__(self, data):
-        return {key: val for key, val in data.items() if key not in self.keys}
+        return {key: val for key, val in data.items() if key not in self.key_iterator(data)}
 
 
 class SelectItemsd(MapTransform):
@@ -370,7 +476,7 @@ class SelectItemsd(MapTransform):
     """
 
     def __call__(self, data):
-        result = {key: val for key, val in data.items() if key in self.keys}
+        result = {key: data[key] for key in self.key_iterator(data)}
         return result
 
 
@@ -379,19 +485,20 @@ class SqueezeDimd(MapTransform):
     Dictionary-based wrapper of :py:class:`monai.transforms.SqueezeDim`.
     """
 
-    def __init__(self, keys: KeysCollection, dim: int = 0) -> None:
+    def __init__(self, keys: KeysCollection, dim: int = 0, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             dim: dimension to be squeezed. Default: 0 (the first dimension)
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = SqueezeDim(dim=dim)
 
     def __call__(self, data: Mapping[Hashable, NdarrayTensor]) -> Dict[Hashable, NdarrayTensor]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
         return d
 
@@ -405,11 +512,13 @@ def __init__(
         self,
         keys: KeysCollection,
         prefix: Union[Sequence[str], str] = "Data",
+        data_type: Union[Sequence[bool], bool] = True,
         data_shape: Union[Sequence[bool], bool] = True,
         value_range: Union[Sequence[bool], bool] = True,
         data_value: Union[Sequence[bool], bool] = False,
         additional_info: Optional[Union[Sequence[Callable], Callable]] = None,
         logger_handler: Optional[logging.Handler] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
         """
         Args:
@@ -417,6 +526,8 @@ def __init__(
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             prefix: will be printed in format: "{prefix} statistics".
                 it also can be a sequence of string, each element corresponds to a key in ``keys``.
+            data_type: whether to show the type of input data.
+                it also can be a sequence of bool, each element corresponds to a key in ``keys``.
             data_shape: whether to show the shape of input data.
                 it also can be a sequence of bool, each element corresponds to a key in ``keys``.
             value_range: whether to show the value range of input data.
@@ -429,10 +540,13 @@ def __init__(
                 corresponds to a key in ``keys``.
             logger_handler: add additional handler to output data: save to file, etc.
                 add existing python logging handlers: https://docs.python.org/3/library/logging.handlers.html
+                the handler should have a logging level of at least `INFO`.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.prefix = ensure_tuple_rep(prefix, len(self.keys))
+        self.data_type = ensure_tuple_rep(data_type, len(self.keys))
         self.data_shape = ensure_tuple_rep(data_shape, len(self.keys))
         self.value_range = ensure_tuple_rep(value_range, len(self.keys))
         self.data_value = ensure_tuple_rep(data_value, len(self.keys))
@@ -442,14 +556,17 @@ def __init__(
 
     def __call__(self, data: Mapping[Hashable, NdarrayTensor]) -> Dict[Hashable, NdarrayTensor]:
         d = dict(data)
-        for idx, key in enumerate(self.keys):
+        for key, prefix, data_type, data_shape, value_range, data_value, additional_info in self.key_iterator(
+            d, self.prefix, self.data_type, self.data_shape, self.value_range, self.data_value, self.additional_info
+        ):
             d[key] = self.printer(
                 d[key],
-                self.prefix[idx],
-                self.data_shape[idx],
-                self.value_range[idx],
-                self.data_value[idx],
-                self.additional_info[idx],
+                prefix,
+                data_type,
+                data_shape,
+                value_range,
+                data_value,
+                additional_info,
             )
         return d
 
@@ -459,23 +576,26 @@ class SimulateDelayd(MapTransform):
     Dictionary-based wrapper of :py:class:`monai.transforms.SimulateDelay`.
     """
 
-    def __init__(self, keys: KeysCollection, delay_time: Union[Sequence[float], float] = 0.0) -> None:
+    def __init__(
+        self, keys: KeysCollection, delay_time: Union[Sequence[float], float] = 0.0, allow_missing_keys: bool = False
+    ) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             delay_time: The minimum amount of time, in fractions of seconds, to accomplish this identity task.
                 It also can be a sequence of string, each element corresponds to a key in ``keys``.
+            allow_missing_keys: don't raise exception if key is missing.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.delay_time = ensure_tuple_rep(delay_time, len(self.keys))
         self.delayer = SimulateDelay()
 
     def __call__(self, data: Mapping[Hashable, NdarrayTensor]) -> Dict[Hashable, NdarrayTensor]:
         d = dict(data)
-        for idx, key in enumerate(self.keys):
-            d[key] = self.delayer(d[key], delay_time=self.delay_time[idx])
+        for key, delay_time in self.key_iterator(d, self.delay_time):
+            d[key] = self.delayer(d[key], delay_time=delay_time)
         return d
 
 
@@ -486,7 +606,9 @@ class CopyItemsd(MapTransform):
 
     """
 
-    def __init__(self, keys: KeysCollection, times: int, names: KeysCollection) -> None:
+    def __init__(
+        self, keys: KeysCollection, times: int, names: KeysCollection, allow_missing_keys: bool = False
+    ) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
@@ -496,13 +618,14 @@ def __init__(self, keys: KeysCollection, times: int, names: KeysCollection) -> N
             names: the names corresponding to the newly copied data,
                 the length should match `len(keys) x times`. for example, if keys is ["img", "seg"]
                 and times is 2, names can be: ["img_1", "seg_1", "img_2", "seg_2"].
+            allow_missing_keys: don't raise exception if key is missing.
 
         Raises:
             ValueError: When ``times`` is nonpositive.
             ValueError: When ``len(names)`` is not ``len(keys) * times``. Incompatible values.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         if times < 1:
             raise ValueError(f"times must be positive, got {times}.")
         self.times = times
@@ -521,13 +644,14 @@ def __call__(self, data):
 
         """
         d = dict(data)
-        for key, new_key in zip(self.keys * self.times, self.names):
+        for new_key in self.names:
             if new_key in d:
                 raise KeyError(f"Key {new_key} already exists in data.")
-            if isinstance(d[key], torch.Tensor):
-                d[new_key] = d[key].detach().clone()
-            else:
-                d[new_key] = copy.deepcopy(d[key])
+            for key in self.key_iterator(d):
+                if isinstance(d[key], torch.Tensor):
+                    d[new_key] = d[key].detach().clone()
+                else:
+                    d[new_key] = copy.deepcopy(d[key])
         return d
 
 
@@ -538,21 +662,16 @@ class ConcatItemsd(MapTransform):
 
     """
 
-    def __init__(self, keys: KeysCollection, name: str, dim: int = 0) -> None:
+    def __init__(self, keys: KeysCollection, name: str, dim: int = 0, allow_missing_keys: bool = False) -> None:
         """
         Args:
             keys: keys of the corresponding items to be concatenated together.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             name: the name corresponding to the key to store the concatenated data.
             dim: on which dimension to concatenate the items, default is 0.
-
-        Raises:
-            ValueError: When insufficient keys are given (``len(self.keys) < 2``).
-
+            allow_missing_keys: don't raise exception if key is missing.
         """
-        super().__init__(keys)
-        if len(self.keys) < 2:
-            raise ValueError("Concatenation requires at least 2 keys.")
+        super().__init__(keys, allow_missing_keys)
         self.name = name
         self.dim = dim
 
@@ -566,7 +685,7 @@ def __call__(self, data):
         d = dict(data)
         output = []
         data_type = None
-        for key in self.keys:
+        for key in self.key_iterator(d):
             if data_type is None:
                 data_type = type(d[key])
             elif not isinstance(d[key], data_type):
@@ -602,6 +721,7 @@ class Lambdad(MapTransform):
             each element corresponds to a key in ``keys``.
         overwrite: whether to overwrite the original data in the input dictionary with lamdbda function output.
             default to True. it also can be a sequence of bool, each element corresponds to a key in ``keys``.
+        allow_missing_keys: don't raise exception if key is missing.
     """
 
     def __init__(
@@ -609,17 +729,18 @@ def __init__(
         keys: KeysCollection,
         func: Union[Sequence[Callable], Callable],
         overwrite: Union[Sequence[bool], bool] = True,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.func = ensure_tuple_rep(func, len(self.keys))
         self.overwrite = ensure_tuple_rep(overwrite, len(self.keys))
         self._lambd = Lambda()
 
     def __call__(self, data):
         d = dict(data)
-        for idx, key in enumerate(self.keys):
-            ret = self._lambd(d[key], func=self.func[idx])
-            if self.overwrite[idx]:
+        for key, func, overwrite in self.key_iterator(d, self.func, self.overwrite):
+            ret = self._lambd(d[key], func=func)
+            if overwrite:
                 d[key] = ret
         return d
 
@@ -657,6 +778,7 @@ class LabelToMaskd(MapTransform):
             `select_labels` is the expected channel indices.
         merge_channels: whether to use `np.any()` to merge the result on channel dim.
             if yes, will return a single channel mask with binary data.
+        allow_missing_keys: don't raise exception if key is missing.
 
     """
 
@@ -665,13 +787,14 @@ def __init__(  # pytype: disable=annotation-type-mismatch
         keys: KeysCollection,
         select_labels: Union[Sequence[int], int],
         merge_channels: bool = False,
+        allow_missing_keys: bool = False,
     ) -> None:  # pytype: disable=annotation-type-mismatch
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.converter = LabelToMask(select_labels=select_labels, merge_channels=merge_channels)
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
 
         return d
@@ -693,6 +816,7 @@ class FgBgToIndicesd(MapTransform):
         image_threshold: if enabled image_key, use ``image > image_threshold`` to determine
             the valid image content area and select background only in this area.
         output_shape: expected shape of output indices. if not None, unravel indices to specified shape.
+        allow_missing_keys: don't raise exception if key is missing.
 
     """
 
@@ -704,8 +828,9 @@ def __init__(
         image_key: Optional[str] = None,
         image_threshold: float = 0.0,
         output_shape: Optional[Sequence[int]] = None,
+        allow_missing_keys: bool = False,
     ) -> None:
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.fg_postfix = fg_postfix
         self.bg_postfix = bg_postfix
         self.image_key = image_key
@@ -714,7 +839,7 @@ def __init__(
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
         image = d[self.image_key] if self.image_key else None
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[str(key) + self.fg_postfix], d[str(key) + self.bg_postfix] = self.converter(d[key], image)
 
         return d
@@ -731,13 +856,13 @@ class ConvertToMultiChannelBasedOnBratsClassesd(MapTransform):
     and ET (Enhancing tumor).
     """
 
-    def __init__(self, keys: KeysCollection):
-        super().__init__(keys)
+    def __init__(self, keys: KeysCollection, allow_missing_keys: bool = False):
+        super().__init__(keys, allow_missing_keys)
         self.converter = ConvertToMultiChannelBasedOnBratsClasses()
 
     def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.converter(d[key])
         return d
 
@@ -757,6 +882,7 @@ class AddExtremePointsChanneld(Randomizable, MapTransform):
             use it for all spatial dimensions.
         rescale_min: minimum value of output data.
         rescale_max: maximum value of output data.
+        allow_missing_keys: don't raise exception if key is missing.
 
     """
 
@@ -769,8 +895,9 @@ def __init__(
         sigma: Union[Sequence[float], float, Sequence[torch.Tensor], torch.Tensor] = 3.0,
         rescale_min: float = -1.0,
         rescale_max: float = 1.0,
+        allow_missing_keys: bool = False,
     ):
-        super().__init__(keys)
+        MapTransform.__init__(self, keys, allow_missing_keys)
         self.background = background
         self.pert = pert
         self.points: List[Tuple[int, ...]] = []
@@ -791,17 +918,16 @@ def __call__(self, data):
         # Generate extreme points
         self.randomize(label[0, :])
 
-        for key in data.keys():
-            if key in self.keys:
-                img = d[key]
-                points_image = extreme_points_to_image(
-                    points=self.points,
-                    label=label,
-                    sigma=self.sigma,
-                    rescale_min=self.rescale_min,
-                    rescale_max=self.rescale_max,
-                )
-                d[key] = np.concatenate([img, points_image], axis=0)
+        for key in self.key_iterator(d):
+            img = d[key]
+            points_image = extreme_points_to_image(
+                points=self.points,
+                label=label,
+                sigma=self.sigma,
+                rescale_min=self.rescale_min,
+                rescale_max=self.rescale_max,
+            )
+            d[key] = np.concatenate([img, points_image], axis=0)
         return d
 
 
@@ -812,34 +938,72 @@ class TorchVisiond(MapTransform):
     data to be dict of PyTorch Tensors, users can easily call `ToTensord` transform to convert Numpy to Tensor.
     """
 
-    def __init__(self, keys: KeysCollection, name: str, *args, **kwargs) -> None:
+    def __init__(self, keys: KeysCollection, name: str, allow_missing_keys: bool = False, *args, **kwargs) -> None:
         """
         Args:
             keys: keys of the corresponding items to be transformed.
                 See also: :py:class:`monai.transforms.compose.MapTransform`
             name: The transform name in TorchVision package.
+            allow_missing_keys: don't raise exception if key is missing.
             args: parameters for the TorchVision transform.
             kwargs: parameters for the TorchVision transform.
 
         """
-        super().__init__(keys)
+        super().__init__(keys, allow_missing_keys)
         self.trans = TorchVision(name, *args, **kwargs)
 
     def __call__(self, data: Mapping[Hashable, torch.Tensor]) -> Dict[Hashable, torch.Tensor]:
         d = dict(data)
-        for key in self.keys:
+        for key in self.key_iterator(d):
             d[key] = self.trans(d[key])
         return d
 
 
+class MapLabelValued(MapTransform):
+    """
+    Dictionary-based wrapper of :py:class:`monai.transforms.MapLabelValue`.
+    """
+
+    def __init__(
+        self,
+        keys: KeysCollection,
+        orig_labels: Sequence,
+        target_labels: Sequence,
+        dtype: DtypeLike = np.float32,
+        allow_missing_keys: bool = False,
+    ) -> None:
+        """
+        Args:
+            keys: keys of the corresponding items to be transformed.
+                See also: :py:class:`monai.transforms.compose.MapTransform`
+            orig_labels: original labels that map to others.
+            target_labels: expected label values, 1: 1 map to the `orig_labels`.
+            dtype: convert the output data to dtype, default to float32.
+            allow_missing_keys: don't raise exception if key is missing.
+
+        """
+        super().__init__(keys, allow_missing_keys)
+        self.mapper = MapLabelValue(orig_labels=orig_labels, target_labels=target_labels, dtype=dtype)
+
+    def __call__(self, data: Mapping[Hashable, np.ndarray]) -> Dict[Hashable, np.ndarray]:
+        d = dict(data)
+        for key in self.key_iterator(d):
+            d[key] = self.mapper(d[key])
+        return d
+
+
 IdentityD = IdentityDict = Identityd
 AsChannelFirstD = AsChannelFirstDict = AsChannelFirstd
 AsChannelLastD = AsChannelLastDict = AsChannelLastd
 AddChannelD = AddChannelDict = AddChanneld
+EnsureChannelFirstD = EnsureChannelFirstDict = EnsureChannelFirstd
+RemoveRepeatedChannelD = RemoveRepeatedChannelDict = RemoveRepeatedChanneld
 RepeatChannelD = RepeatChannelDict = RepeatChanneld
 SplitChannelD = SplitChannelDict = SplitChanneld
 CastToTypeD = CastToTypeDict = CastToTyped
 ToTensorD = ToTensorDict = ToTensord
+ToNumpyD = ToNumpyDict = ToNumpyd
+ToPILD = ToPILDict = ToPILd
 DeleteItemsD = DeleteItemsDict = DeleteItemsd
 SqueezeDimD = SqueezeDimDict = SqueezeDimd
 DataStatsD = DataStatsDict = DataStatsd
@@ -855,3 +1019,4 @@ def __call__(self, data: Mapping[Hashable, torch.Tensor]) -> Dict[Hashable, torc
 AddExtremePointsChannelD = AddExtremePointsChannelDict = AddExtremePointsChanneld
 TorchVisionD = TorchVisionDict = TorchVisiond
 RandLambdaD = RandLambdaDict = RandLambdad
+MapLabelValueD = MapLabelValueDict = MapLabelValued
diff --git a/monai/transforms/utils.py b/monai/transforms/utils.py
index 9a84eb00d9..3e73beb305 100644
--- a/monai/transforms/utils.py
+++ b/monai/transforms/utils.py
@@ -12,6 +12,7 @@
 import itertools
 import random
 import warnings
+from contextlib import contextmanager
 from typing import Callable, List, Optional, Sequence, Tuple, Union
 
 import numpy as np
@@ -19,7 +20,20 @@
 
 from monai.config import DtypeLike, IndexSelection
 from monai.networks.layers import GaussianFilter
-from monai.utils import ensure_tuple, ensure_tuple_rep, ensure_tuple_size, fall_back_tuple, min_version, optional_import
+from monai.transforms.compose import Compose
+from monai.transforms.transform import MapTransform
+from monai.utils import (
+    GridSampleMode,
+    InterpolateMode,
+    InverseKeys,
+    ensure_tuple,
+    ensure_tuple_rep,
+    ensure_tuple_size,
+    fall_back_tuple,
+    issequenceiterable,
+    min_version,
+    optional_import,
+)
 
 measure, _ = optional_import("skimage.measure", "0.14.2", min_version)
 
@@ -37,7 +51,6 @@
     "map_binary_to_indices",
     "weighted_patch_samples",
     "generate_pos_neg_label_crop_centers",
-    "apply_transform",
     "create_grid",
     "create_control_grid",
     "create_rotate",
@@ -49,6 +62,8 @@
     "get_extreme_points",
     "extreme_points_to_image",
     "map_spatial_axes",
+    "allow_missing_keys_mode",
+    "convert_inverse_interp_mode",
 ]
 
 
@@ -363,31 +378,6 @@ def _correct_centers(
     return centers
 
 
-def apply_transform(transform: Callable, data, map_items: bool = True):
-    """
-    Transform `data` with `transform`.
-    If `data` is a list or tuple and `map_data` is True, each item of `data` will be transformed
-    and this method returns a list of outcomes.
-    otherwise transform will be applied once with `data` as the argument.
-
-    Args:
-        transform: a callable to be used to transform `data`
-        data: an object to be transformed.
-        map_items: whether to apply transform to each item in `data`,
-            if `data` is a list or tuple. Defaults to True.
-
-    Raises:
-        Exception: When ``transform`` raises an exception.
-
-    """
-    try:
-        if isinstance(data, (list, tuple)) and map_items:
-            return [transform(item) for item in data]
-        return transform(data)
-    except Exception as e:
-        raise RuntimeError(f"applying transform {transform}") from e
-
-
 def create_grid(
     spatial_size: Sequence[int],
     spacing: Optional[Sequence[float]] = None,
@@ -716,7 +706,7 @@ def map_spatial_axes(
             The default `None` will convert to all the spatial axes of the image.
             If axis is negative it counts from the last to the first axis.
             If axis is a tuple of ints.
-        channel_first: the image data is channel first or channel last, defaut to channel first.
+        channel_first: the image data is channel first or channel last, default to channel first.
 
     """
     if spatial_axes is None:
@@ -730,3 +720,84 @@ def map_spatial_axes(
                 spatial_axes_.append(a - 1 if a < 0 else a)
 
     return spatial_axes_
+
+
+@contextmanager
+def allow_missing_keys_mode(transform: Union[MapTransform, Compose, Tuple[MapTransform], Tuple[Compose]]):
+    """Temporarily set all MapTransforms to not throw an error if keys are missing. After, revert to original states.
+
+    Args:
+        transform: either MapTransform or a Compose
+
+    Example:
+
+    .. code-block:: python
+
+        data = {"image": np.arange(16, dtype=float).reshape(1, 4, 4)}
+        t = SpatialPadd(["image", "label"], 10, allow_missing_keys=False)
+        _ = t(data)  # would raise exception
+        with allow_missing_keys_mode(t):
+            _ = t(data)  # OK!
+    """
+    # If given a sequence of transforms, Compose them to get a single list
+    if issequenceiterable(transform):
+        transform = Compose(transform)
+
+    # Get list of MapTransforms
+    transforms = []
+    if isinstance(transform, MapTransform):
+        transforms = [transform]
+    elif isinstance(transform, Compose):
+        # Only keep contained MapTransforms
+        transforms = [t for t in transform.flatten().transforms if isinstance(t, MapTransform)]
+    if len(transforms) == 0:
+        raise TypeError(
+            "allow_missing_keys_mode expects either MapTransform(s) or Compose(s) containing MapTransform(s)"
+        )
+
+    # Get the state of each `allow_missing_keys`
+    orig_states = [t.allow_missing_keys for t in transforms]
+
+    try:
+        # Set all to True
+        for t in transforms:
+            t.allow_missing_keys = True
+        yield
+    finally:
+        # Revert
+        for t, o_s in zip(transforms, orig_states):
+            t.allow_missing_keys = o_s
+
+
+def convert_inverse_interp_mode(trans_info: List, mode: str = "nearest", align_corners: Optional[bool] = None):
+    """
+    Change the interpolation mode when inverting spatial transforms, default to "nearest".
+    This function modifies trans_info's `InverseKeys.EXTRA_INFO`.
+
+    See also: :py:class:`monai.transform.inverse.InvertibleTransform`
+
+    Args:
+        trans_info: transforms inverse information list, contains context of every invertible transform.
+        mode: target interpolation mode to convert, default to "nearest" as it's usually used to save the mode output.
+        align_corners: target align corner value in PyTorch interpolation API, need to align with the `mode`.
+
+    """
+    interp_modes = [i.value for i in InterpolateMode] + [i.value for i in GridSampleMode]
+
+    # set to string for DataLoader collation
+    align_corners_ = "none" if align_corners is None else align_corners
+
+    for item in ensure_tuple(trans_info):
+        if InverseKeys.EXTRA_INFO in item:
+            orig_mode = item[InverseKeys.EXTRA_INFO].get("mode", None)
+            if orig_mode is not None:
+                if orig_mode[0] in interp_modes:
+                    item[InverseKeys.EXTRA_INFO]["mode"] = [mode for _ in range(len(mode))]
+                elif orig_mode in interp_modes:
+                    item[InverseKeys.EXTRA_INFO]["mode"] = mode
+            if "align_corners" in item[InverseKeys.EXTRA_INFO]:
+                if issequenceiterable(item[InverseKeys.EXTRA_INFO]["align_corners"]):
+                    item[InverseKeys.EXTRA_INFO]["align_corners"] = [align_corners_ for _ in range(len(mode))]
+                else:
+                    item[InverseKeys.EXTRA_INFO]["align_corners"] = align_corners_
+    return trans_info
diff --git a/monai/utils/__init__.py b/monai/utils/__init__.py
index 1e17d44029..d622ce96ae 100644
--- a/monai/utils/__init__.py
+++ b/monai/utils/__init__.py
@@ -17,9 +17,12 @@
     Average,
     BlendMode,
     ChannelMatching,
+    CommonKeys,
+    ForwardMode,
     GridSampleMode,
     GridSamplePadMode,
     InterpolateMode,
+    InverseKeys,
     LossReduction,
     Method,
     MetricReduction,
@@ -30,6 +33,7 @@
     UpsampleMode,
     Weight,
 )
+from .jupyter_utils import StatusMembers, ThreadContainer
 from .misc import (
     MAX_SEED,
     ImageMetaKey,
diff --git a/monai/utils/aliases.py b/monai/utils/aliases.py
index e8192897b8..2b7b29eeb5 100644
--- a/monai/utils/aliases.py
+++ b/monai/utils/aliases.py
@@ -58,7 +58,7 @@ def resolve_name(name):
     """
     # attempt to resolve an alias
     with alias_lock:
-        obj = GlobalAliases.get(name, None)
+        obj = GlobalAliases.get(name)
 
     if name in GlobalAliases and obj is None:
         raise AssertionError
diff --git a/monai/utils/enums.py b/monai/utils/enums.py
index d1d2d3bcce..9f753ca700 100644
--- a/monai/utils/enums.py
+++ b/monai/utils/enums.py
@@ -28,6 +28,8 @@
     "ChannelMatching",
     "SkipMode",
     "Method",
+    "InverseKeys",
+    "CommonKeys",
 ]
 
 
@@ -52,13 +54,19 @@ class NumpyPadMode(Enum):
 class GridSampleMode(Enum):
     """
     See also: https://pytorch.org/docs/stable/nn.functional.html#grid-sample
+
+    interpolation mode of `torch.nn.functional.grid_sample`
+
+    Note:
+        (documentation from `torch.nn.functional.grid_sample`)
+        `mode='bicubic'` supports only 4-D input.
+        When `mode='bilinear'` and the input is 5-D, the interpolation mode used internally will actually be trilinear.
+        However, when the input is 4-D, the interpolation mode will legitimately be bilinear.
     """
 
     NEAREST = "nearest"
     BILINEAR = "bilinear"
-    QUADRATIC = "quadratic"
-    CUBIC = "cubic"
-    FOURTH = "fourth"
+    BICUBIC = "bicubic"
 
 
 class InterpolateMode(Enum):
@@ -214,3 +222,40 @@ class Method(Enum):
 
     SYMMETRIC = "symmetric"
     END = "end"
+
+
+class ForwardMode(Enum):
+    """
+    See also: :py:class:`monai.transforms.engines.evaluator.Evaluator`
+    """
+
+    TRAIN = "train"
+    EVAL = "eval"
+
+
+class InverseKeys:
+    """Extra meta data keys used for inverse transforms."""
+
+    CLASS_NAME = "class"
+    ID = "id"
+    ORIG_SIZE = "orig_size"
+    EXTRA_INFO = "extra_info"
+    DO_TRANSFORM = "do_transforms"
+    KEY_SUFFIX = "_transforms"
+
+
+class CommonKeys:
+    """
+    A set of common keys for dictionary based supervised training process.
+    `IMAGE` is the input image data.
+    `LABEL` is the training or evaluation label of segmentation or classification task.
+    `PRED` is the prediction data of model output.
+    `LOSS` is the loss value of current iteration.
+    `INFO` is some useful information during training or evaluation, like loss value, etc.
+
+    """
+
+    IMAGE = "image"
+    LABEL = "label"
+    PRED = "pred"
+    LOSS = "loss"
diff --git a/monai/utils/jupyter_utils.py b/monai/utils/jupyter_utils.py
new file mode 100644
index 0000000000..efe5519239
--- /dev/null
+++ b/monai/utils/jupyter_utils.py
@@ -0,0 +1,345 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This set of utility function is meant to make using Jupyter notebooks easier with MONAI. Plotting functions using
+Matplotlib produce common plots for metrics and images.
+"""
+
+from enum import Enum
+from threading import RLock, Thread
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+try:
+    import matplotlib.pyplot as plt
+
+    has_matplotlib = True
+except ImportError:
+    has_matplotlib = False
+
+try:
+    from ignite.engine import Engine, Events
+
+    has_ignite = True
+except ImportError:
+    Engine = object
+    Events = object
+    has_ignite = False
+
+LOSS_NAME = "loss"
+
+
+def plot_metric_graph(
+    ax,
+    title: str,
+    graphmap: Dict[str, Union[List[float], Tuple[List[float], List[float]]]],
+    yscale: str = "log",
+    avg_keys: Tuple[str] = (LOSS_NAME,),
+    window_fraction: int = 20,
+):
+    """
+    Plot metrics on a single graph with running averages plotted for selected keys. The values in `graphmap`
+    should be lists of (timepoint, value) pairs as stored in MetricLogger objects.
+
+    Args:
+        ax: Axes object to plot into
+        title: graph title
+        graphmap: dictionary of named graph values, which are lists of values or (index, value) pairs
+        yscale: scale for y-axis compatible with `Axes.set_yscale`
+        avg_keys: tuple of keys in `graphmap` to provide running average plots for
+        window_fraction: what fraction of the graph value length to use as the running average window
+    """
+    from matplotlib.ticker import MaxNLocator
+
+    for n, v in graphmap.items():
+        if len(v) > 0:
+            if isinstance(v[0], (tuple, list)):  # values are (x,y) pairs
+                inds, vals = zip(*v)  # separate values into list of indices in X dimension and values
+            else:
+                inds, vals = tuple(range(len(v))), tuple(v)  # values are without indices, make indices for them
+
+            ax.plot(inds, vals, label=f"{n} = {vals[-1]:.5g}")
+
+            # if requested compute and plot a running average for the values using a fractional window size
+            if n in avg_keys and len(v) > window_fraction:
+                window = len(v) // window_fraction
+                kernel = np.ones((window,)) / window
+                ra = np.convolve((vals[0],) * (window - 1) + vals, kernel, mode="valid")
+
+                ax.plot(inds, ra, label=f"{n} Avg = {ra[-1]:.5g}")
+
+    ax.set_title(title)
+    ax.set_yscale(yscale)
+    ax.axis("on")
+    ax.legend(bbox_to_anchor=(1, 1), loc=1, borderaxespad=0.0)
+    ax.grid(True, "both", "both")
+    ax.xaxis.set_major_locator(MaxNLocator(integer=True))
+
+
+def plot_metric_images(
+    fig,
+    title: str,
+    graphmap: Dict[str, Union[List[float], Tuple[List[float], List[float]]]],
+    imagemap: Dict[str, np.ndarray],
+    yscale: str = "log",
+    avg_keys: Tuple[str] = (LOSS_NAME,),
+    window_fraction: int = 20,
+) -> List:
+    """
+    Plot metric graph data with images below into figure `fig`. The intended use is for the graph data to be
+    metrics from a training run and the images to be the batch and output from the last iteration. This uses
+    `plot_metric_graph` to plot the metric graph.
+
+    Args:
+        fig: Figure object to plot into, reuse from previous plotting for flicker-free refreshing
+        title: graph title
+        graphmap: dictionary of named graph values, which are lists of values or (index, value) pairs
+        imagemap: dictionary of named images to show with metric plot
+        yscale: for metric plot, scale for y-axis compatible with `Axes.set_yscale`
+        avg_keys: for metric plot, tuple of keys in `graphmap` to provide running average plots for
+        window_fraction: for metric plot, what fraction of the graph value length to use as the running average window
+
+    Returns:
+        list of Axes objects for graph followed by images
+    """
+    gridshape = (4, max(1, len(imagemap)))
+
+    graph = plt.subplot2grid(gridshape, (0, 0), colspan=gridshape[1], fig=fig)
+
+    plot_metric_graph(graph, title, graphmap, yscale, avg_keys, window_fraction)
+
+    axes = [graph]
+    for i, n in enumerate(imagemap):
+        im = plt.subplot2grid(gridshape, (1, i), rowspan=2, fig=fig)
+
+        if imagemap[n].shape[0] == 3:
+            im.imshow(imagemap[n].transpose([1, 2, 0]))
+        else:
+            im.imshow(np.squeeze(imagemap[n]), cmap="gray")
+
+        im.set_title("%s\n%.3g -> %.3g" % (n, imagemap[n].min(), imagemap[n].max()))
+        im.axis("off")
+        axes.append(im)
+
+    return axes
+
+
+def tensor_to_images(name: str, tensor: torch.Tensor):
+    """
+    Return an tuple of images derived from the given tensor. The `name` value indices which key from the
+    output or batch value the tensor was stored as, or is "Batch" or "Output" if these were single tensors
+    instead of dictionaries. Returns a tuple of 2D images of shape HW, or 3D images of shape CHW where C is
+    color channels RGB or RGBA. This allows multiple images to be created from a single tensor, ie. to show
+    each channel separately.
+    """
+    if tensor.ndim == 4 and tensor.shape[2] > 2 and tensor.shape[3] > 2:
+        return tuple(tensor[0].cpu().data.numpy())
+    if tensor.ndim == 5 and tensor.shape[3] > 2 and tensor.shape[4] > 2:
+        dmid = tensor.shape[2] // 2
+        return tuple(tensor[0, :, dmid].cpu().data.numpy())
+
+    return ()
+
+
+def plot_engine_status(
+    engine: Engine,
+    logger,
+    title: str = "Training Log",
+    yscale: str = "log",
+    avg_keys: Tuple[str] = (LOSS_NAME,),
+    window_fraction: int = 20,
+    image_fn: Optional[Callable] = tensor_to_images,
+    fig=None,
+) -> Tuple:
+    """
+    Plot the status of the given Engine with its logger. The plot will consist of a graph of loss values and metrics
+    taken from the logger, and images taken from the `output` and `batch` members of `engine.state`. The images are
+    converted to Numpy arrays suitable for input to `Axes.imshow` using `image_fn`, if this is None then no image
+    plotting is done.
+
+    Args:
+        engine: Engine to extract images from
+        logger: MetricLogger to extract loss and metric data from
+        title: graph title
+        yscale: for metric plot, scale for y-axis compatible with `Axes.set_yscale`
+        avg_keys: for metric plot, tuple of keys in `graphmap` to provide running average plots for
+        window_fraction: for metric plot, what fraction of the graph value length to use as the running average window
+        image_fn: callable converting tensors keyed to a name in the Engine to a tuple of images to plot
+        fig: Figure object to plot into, reuse from previous plotting for flicker-free refreshing
+
+    Returns:
+        Figure object (or `fig` if given), list of Axes objects for graph and images
+    """
+    if fig is not None:
+        fig.clf()
+    else:
+        fig = plt.Figure(figsize=(20, 10), tight_layout=True, facecolor="white")
+
+    graphmap = {LOSS_NAME: logger.loss}
+    graphmap.update(logger.metrics)
+
+    imagemap = {}
+
+    if image_fn is not None and engine.state is not None and engine.state.batch is not None:
+        for src in (engine.state.batch, engine.state.output):
+            if isinstance(src, dict):
+                for k, v in src.items():
+                    if isinstance(v, torch.Tensor):
+                        images = image_fn(k, v)
+
+                        for i, im in enumerate(images):
+                            imagemap[f"{k}_{i}"] = im
+            else:
+                label = "Batch" if src is engine.state.batch else "Output"
+                images = image_fn(label, src)
+
+                for i, im in enumerate(images):
+                    imagemap[f"{label}_{i}"] = im
+
+    axes = plot_metric_images(fig, title, graphmap, imagemap, yscale, avg_keys, window_fraction)
+
+    axes[0].axhline(logger.loss[-1][1], c="k", ls=":")  # draw dotted horizontal line at last loss value
+
+    return fig, axes
+
+
+def _get_loss_from_output(output: Union[Dict[str, torch.Tensor], torch.Tensor]) -> float:
+    """Returns a single value from the network output, which is a dict or tensor."""
+    if isinstance(output, dict):
+        return output["loss"].item()
+    return output.item()
+
+
+class StatusMembers(Enum):
+    """
+    Named members of the status dictionary, others may be present for named metric values.
+    """
+
+    STATUS = "Status"
+    EPOCHS = "Epochs"
+    ITERS = "Iters"
+    LOSS = "Loss"
+
+
+class ThreadContainer(Thread):
+    """
+    Contains a running `Engine` object within a separate thread from main thread in a Jupyter notebook. This
+    allows an engine to begin a run in the background and allow the starting notebook cell to complete. A
+    user can thus start a run and then navigate away from the notebook without concern for loosing connection
+    with the running cell. All output is acquired through methods which synchronize with the running engine
+    using an internal `lock` member, acquiring this lock allows the engine to be inspected while it's prevented
+    from starting the next iteration.
+
+    Args:
+        engine: wrapped `Engine` object, when the container is started its `run` method is called
+        loss_transform: callable to convert an output dict into a single numeric value
+        metric_transform: callable to convert a named metric value into a single numeric value
+        status_format: format string for status key-value pairs.
+    """
+
+    def __init__(
+        self,
+        engine: Engine,
+        loss_transform: Callable = _get_loss_from_output,
+        metric_transform: Callable = lambda name, value: value,
+        status_format: str = "{}: {:.4}",
+    ):
+        super().__init__()
+        self.lock = RLock()
+        self.engine = engine
+        self._status_dict: Dict[str, Any] = {}
+        self.loss_transform = loss_transform
+        self.metric_transform = metric_transform
+        self.fig = None
+        self.status_format = status_format
+
+        self.engine.add_event_handler(Events.ITERATION_COMPLETED, self._update_status)
+
+    def run(self):
+        """Calls the `run` method of the wrapped engine."""
+        self.engine.run()
+
+    def stop(self):
+        """Stop the engine and join the thread."""
+        self.engine.terminate()
+        self.join()
+
+    def _update_status(self):
+        """Called as an event, updates the internal status dict at the end of iterations."""
+        with self.lock:
+            state = self.engine.state
+            stats = {
+                StatusMembers.EPOCHS.value: 0,
+                StatusMembers.ITERS.value: 0,
+                StatusMembers.LOSS.value: float("nan"),
+            }
+
+            if state is not None:
+                if state.max_epochs >= 1:
+                    epoch = f"{state.epoch}/{state.max_epochs}"
+                else:
+                    epoch = str(state.epoch)
+
+                if state.epoch_length is not None:
+                    iters = f"{state.iteration % state.epoch_length}/{state.epoch_length}"
+                else:
+                    iters = str(state.iteration)
+
+                stats[StatusMembers.EPOCHS.value] = epoch
+                stats[StatusMembers.ITERS.value] = iters
+                stats[StatusMembers.LOSS.value] = self.loss_transform(state.output)
+
+                metrics = state.metrics or {}
+                for m, v in metrics.items():
+                    v = self.metric_transform(m, v)
+                    if v is not None:
+                        stats[m].append(v)
+
+            self._status_dict.update(stats)
+
+    @property
+    def status_dict(self) -> Dict[str, str]:
+        """A dictionary containing status information, current loss, and current metric values."""
+        with self.lock:
+            stats = {StatusMembers.STATUS.value: "Running" if self.is_alive() else "Stopped"}
+            stats.update(self._status_dict)
+            return stats
+
+    def status(self) -> str:
+        """Returns a status string for the current state of the engine."""
+        stats = self.status_dict
+
+        msgs = [stats.pop(StatusMembers.STATUS.value), "Iters: " + str(stats.pop(StatusMembers.ITERS.value))]
+
+        for key, val in stats.items():
+            if isinstance(val, float):
+                msg = self.status_format.format(key, val)
+            else:
+                msg = f"{key}: {val}"
+
+            msgs.append(msg)
+
+        return ", ".join(msgs)
+
+    def plot_status(self, logger, plot_func: Callable = plot_engine_status):
+        """
+        Generate a plot of the current status of the contained engine whose loss and metrics were tracked by `logger`.
+        The function `plot_func` must accept arguments `title`, `engine`, `logger`, and `fig` which are the plot title,
+        `self.engine`, `logger`, and `self.fig` respectively. The return value must be a figure object (stored in
+        `self.fig`) and a list of Axes objects for the plots in the figure. Only the figure is returned by this method,
+        which holds the internal lock during the plot generation.
+        """
+        with self.lock:
+            self.fig, axes = plot_func(title=self.status(), engine=self.engine, logger=logger, fig=self.fig)
+            return self.fig
diff --git a/monai/utils/misc.py b/monai/utils/misc.py
index f9346340cf..bd8e46d8b5 100644
--- a/monai/utils/misc.py
+++ b/monai/utils/misc.py
@@ -339,13 +339,13 @@ def copy_to_device(
 
     if hasattr(obj, "to"):
         return obj.to(device, non_blocking=non_blocking)
-    elif isinstance(obj, tuple):
+    if isinstance(obj, tuple):
         return tuple(copy_to_device(o, device, non_blocking) for o in obj)
-    elif isinstance(obj, list):
+    if isinstance(obj, list):
         return [copy_to_device(o, device, non_blocking) for o in obj]
-    elif isinstance(obj, dict):
+    if isinstance(obj, dict):
         return {k: copy_to_device(o, device, non_blocking) for k, o in obj.items()}
-    elif verbose:
+    if verbose:
         fn_name = cast(types.FrameType, inspect.currentframe()).f_code.co_name
         warnings.warn(f"{fn_name} called with incompatible type: " + f"{type(obj)}. Data will be returned unchanged.")
 
@@ -358,3 +358,4 @@ class ImageMetaKey:
     """
 
     FILENAME_OR_OBJ = "filename_or_obj"
+    PATCH_INDEX = "patch_index"
diff --git a/monai/utils/module.py b/monai/utils/module.py
index 0e11a6531d..b51b2820a8 100644
--- a/monai/utils/module.py
+++ b/monai/utils/module.py
@@ -11,6 +11,7 @@
 
 import inspect
 import sys
+import warnings
 from importlib import import_module
 from pkgutil import walk_packages
 from re import match
@@ -95,17 +96,21 @@ def min_version(the_module, min_version_str: str = "") -> bool:
     Returns True if the module's version is greater or equal to the 'min_version'.
     When min_version_str is not provided, it always returns True.
     """
-    if min_version_str:
-        mod_version = tuple(int(x) for x in the_module.__version__.split(".")[:2])
-        required = tuple(int(x) for x in min_version_str.split(".")[:2])
-        return mod_version >= required
-    return True  # always valid version
+    if not min_version_str or not hasattr(the_module, "__version__"):
+        return True  # always valid version
+
+    mod_version = tuple(int(x) for x in the_module.__version__.split(".")[:2])
+    required = tuple(int(x) for x in min_version_str.split(".")[:2])
+    return mod_version >= required
 
 
 def exact_version(the_module, version_str: str = "") -> bool:
     """
     Returns True if the module's __version__ matches version_str
     """
+    if not hasattr(the_module, "__version__"):
+        warnings.warn(f"{the_module} has no attribute __version__ in exact_version check.")
+        return False
     return bool(the_module.__version__ == version_str)
 
 
@@ -250,21 +255,11 @@ def has_option(obj, keywords: Union[str, Sequence[str]]) -> bool:
 def get_package_version(dep_name, default="NOT INSTALLED or UNKNOWN VERSION."):
     """
     Try to load package and get version. If not found, return `default`.
-
-    If the package was already loaded, leave it. If wasn't previously loaded, unload it.
     """
-    dep_ver = default
-    dep_already_loaded = dep_name not in sys.modules
-
     dep, has_dep = optional_import(dep_name)
-    if has_dep:
-        if hasattr(dep, "__version__"):
-            dep_ver = dep.__version__
-        # if not previously loaded, unload it
-        if not dep_already_loaded:
-            del dep
-            del sys.modules[dep_name]
-    return dep_ver
+    if has_dep and hasattr(dep, "__version__"):
+        return dep.__version__
+    return default
 
 
 def get_torch_version_tuple():
diff --git a/monai/utils/state_cacher.py b/monai/utils/state_cacher.py
index 66e9080724..65a6118670 100644
--- a/monai/utils/state_cacher.py
+++ b/monai/utils/state_cacher.py
@@ -1,3 +1,14 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import copy
 import os
 import tempfile
diff --git a/monai/visualize/class_activation_maps.py b/monai/visualize/class_activation_maps.py
index a917bcf800..c63e8e51d9 100644
--- a/monai/visualize/class_activation_maps.py
+++ b/monai/visualize/class_activation_maps.py
@@ -17,9 +17,8 @@
 import torch.nn as nn
 import torch.nn.functional as F
 
-from monai.networks.utils import eval_mode, train_mode
 from monai.transforms import ScaleIntensity
-from monai.utils import ensure_tuple
+from monai.utils import ensure_tuple, get_torch_version_tuple
 from monai.visualize.visualizer import default_upsampler
 
 __all__ = ["CAM", "GradCAM", "GradCAMpp", "ModelWithHooks", "default_normalizer"]
@@ -74,7 +73,13 @@ def __init__(
                 continue
             _registered.append(name)
             if self.register_backward:
-                mod.register_backward_hook(self.backward_hook(name))
+                if get_torch_version_tuple() < (1, 8):
+                    mod.register_backward_hook(self.backward_hook(name))
+                else:
+                    if "inplace" in mod.__dict__ and mod.__dict__["inplace"]:
+                        # inplace=True causes errors for register_full_backward_hook
+                        mod.__dict__["inplace"] = False
+                    mod.register_full_backward_hook(self.backward_hook(name))
             if self.register_forward:
                 mod.register_forward_hook(self.forward_hook(name))
         if len(_registered) != len(self.target_layers):
@@ -110,26 +115,24 @@ def get_layer(self, layer_id: Union[str, Callable]):
                     return mod
         raise NotImplementedError(f"Could not find {layer_id}.")
 
-    def class_score(self, logits, class_idx=None):
-        if class_idx is not None:
-            return logits[:, class_idx].squeeze(), class_idx
-        class_idx = logits.max(1)[-1]
-        return logits[:, class_idx].squeeze(), class_idx
+    def class_score(self, logits, class_idx):
+        return logits[:, class_idx].squeeze()
 
     def __call__(self, x, class_idx=None, retain_graph=False):
-        # Use train_mode if grad is required, else eval_mode
-        mode = train_mode if self.register_backward else eval_mode
-        with mode(self.model):
-            logits = self.model(x)
-            acti, grad = None, None
-            if self.register_forward:
-                acti = tuple(self.activations[layer] for layer in self.target_layers)
-            if self.register_backward:
-                score, class_idx = self.class_score(logits, class_idx)
-                self.model.zero_grad()
-                self.score, self.class_idx = score, class_idx
-                score.sum().backward(retain_graph=retain_graph)
-                grad = tuple(self.gradients[layer] for layer in self.target_layers)
+        train = self.model.training
+        self.model.eval()
+        logits = self.model(x)
+        self.class_idx = logits.max(1)[-1] if class_idx is None else class_idx
+        acti, grad = None, None
+        if self.register_forward:
+            acti = tuple(self.activations[layer] for layer in self.target_layers)
+        if self.register_backward:
+            self.score = self.class_score(logits, self.class_idx)
+            self.model.zero_grad()
+            self.score.sum().backward(retain_graph=retain_graph)
+            grad = tuple(self.gradients[layer] for layer in self.target_layers)
+        if train:
+            self.model.train()
         return logits, acti, grad
 
     def get_wrapped_net(self):
@@ -173,6 +176,17 @@ def feature_map_size(self, input_size, device="cpu", layer_idx=-1):
         return self.compute_map(torch.zeros(*input_size, device=device), layer_idx=layer_idx).shape
 
     def compute_map(self, x, class_idx=None, layer_idx=-1):
+        """
+        Compute the actual feature map with input tensor `x`.
+
+        Args:
+            x: input to `nn_module`.
+            class_idx: index of the class to be visualized. Default to `None` (computing `class_idx` from `argmax`)
+            layer_idx: index of the target layer if there are multiple target layers. Defaults to -1.
+
+        Returns:
+            activation maps (raw outputs without upsampling/post-processing.)
+        """
         raise NotImplementedError()
 
     def _upsample_and_post_process(self, acti_map, x):
@@ -191,16 +205,20 @@ def __call__(self):
 class CAM(CAMBase):
     """
     Compute class activation map from the last fully-connected layers before the spatial pooling.
+    This implementation is based on:
+
+        Zhou et al., Learning Deep Features for Discriminative Localization. CVPR '16,
+        https://arxiv.org/abs/1512.04150
 
     Examples
 
     .. code-block:: python
 
         # densenet 2d
-        from monai.networks.nets import densenet121
+        from monai.networks.nets import DenseNet121
         from monai.visualize import CAM
 
-        model_2d = densenet121(spatial_dims=2, in_channels=1, out_channels=3)
+        model_2d = DenseNet121(spatial_dims=2, in_channels=1, out_channels=3)
         cam = CAM(nn_module=model_2d, target_layers="class_layers.relu", fc_layers="class_layers.out")
         result = cam(x=torch.rand((1, 1, 48, 64)))
 
@@ -212,6 +230,12 @@ class CAM(CAMBase):
         cam = CAM(nn_module=model_2d, target_layers="layer4", fc_layers="last_linear")
         result = cam(x=torch.rand((2, 3, 48, 64)))
 
+    N.B.: To help select the target layer, it may be useful to list all layers:
+
+    .. code-block:: python
+
+        for name, _ in model.named_modules(): print(name)
+
     See Also:
 
         - :py:class:`monai.visualize.class_activation_maps.GradCAM`
@@ -249,9 +273,6 @@ def __init__(
         self.fc_layers = fc_layers
 
     def compute_map(self, x, class_idx=None, layer_idx=-1):
-        """
-        Compute the actual feature map with input tensor `x`.
-        """
         logits, acti, _ = self.nn_module(x)
         acti = acti[layer_idx]
         if class_idx is None:
@@ -292,10 +313,10 @@ class GradCAM(CAMBase):
     .. code-block:: python
 
         # densenet 2d
-        from monai.networks.nets import densenet121
+        from monai.networks.nets import DenseNet121
         from monai.visualize import GradCAM
 
-        model_2d = densenet121(spatial_dims=2, in_channels=1, out_channels=3)
+        model_2d = DenseNet121(spatial_dims=2, in_channels=1, out_channels=3)
         cam = GradCAM(nn_module=model_2d, target_layers="class_layers.relu")
         result = cam(x=torch.rand((1, 1, 48, 64)))
 
@@ -307,6 +328,12 @@ class GradCAM(CAMBase):
         cam = GradCAM(nn_module=model_2d, target_layers="layer4")
         result = cam(x=torch.rand((2, 3, 48, 64)))
 
+    N.B.: To help select the target layer, it may be useful to list all layers:
+
+    .. code-block:: python
+
+        for name, _ in model.named_modules(): print(name)
+
     See Also:
 
         - :py:class:`monai.visualize.class_activation_maps.CAM`
@@ -314,9 +341,6 @@ class GradCAM(CAMBase):
     """
 
     def compute_map(self, x, class_idx=None, retain_graph=False, layer_idx=-1):
-        """
-        Compute the actual feature map with input tensor `x`.
-        """
         _, acti, grad = self.nn_module(x, class_idx=class_idx, retain_graph=retain_graph)
         acti, grad = acti[layer_idx], grad[layer_idx]
         b, c, *spatial = grad.shape
@@ -356,9 +380,6 @@ class GradCAMpp(GradCAM):
     """
 
     def compute_map(self, x, class_idx=None, retain_graph=False, layer_idx=-1):
-        """
-        Compute the actual feature map with input tensor `x`.
-        """
         _, acti, grad = self.nn_module(x, class_idx=class_idx, retain_graph=retain_graph)
         acti, grad = acti[layer_idx], grad[layer_idx]
         b, c, *spatial = grad.shape
diff --git a/monai/visualize/occlusion_sensitivity.py b/monai/visualize/occlusion_sensitivity.py
index 5863614965..ec010e32b4 100644
--- a/monai/visualize/occlusion_sensitivity.py
+++ b/monai/visualize/occlusion_sensitivity.py
@@ -122,20 +122,20 @@ class OcclusionSensitivity:
     .. code-block:: python
 
         # densenet 2d
-        from monai.networks.nets import densenet121
+        from monai.networks.nets import DenseNet121
         from monai.visualize import OcclusionSensitivity
 
-        model_2d = densenet121(spatial_dims=2, in_channels=1, out_channels=3)
+        model_2d = DenseNet121(spatial_dims=2, in_channels=1, out_channels=3)
         occ_sens = OcclusionSensitivity(nn_module=model_2d)
-        occ_map, most_probable_class = occ_sens(x=torch.rand((1, 1, 48, 64)), class_idx=None, b_box=[-1, -1, 2, 40, 1, 62])
+        occ_map, most_probable_class = occ_sens(x=torch.rand((1, 1, 48, 64)), b_box=[-1, -1, 2, 40, 1, 62])
 
         # densenet 3d
         from monai.networks.nets import DenseNet
         from monai.visualize import OcclusionSensitivity
 
         model_3d = DenseNet(spatial_dims=3, in_channels=1, out_channels=3, init_features=2, growth_rate=2, block_config=(6,))
-        occ_sens = OcclusionSensitivity(nn_module=model_3d, n_batch=10, stride=2)
-        occ_map, most_probable_class = occ_sens(torch.rand(1, 1, 6, 6, 6), class_idx=1, b_box=[-1, -1, 2, 3, -1, -1, -1, -1])
+        occ_sens = OcclusionSensitivity(nn_module=model_3d, n_batch=10, stride=3)
+        occ_map, most_probable_class = occ_sens(torch.rand(1, 1, 6, 6, 6), b_box=[-1, -1, 1, 3, -1, -1, -1, -1])
 
     See Also:
 
@@ -152,7 +152,7 @@ def __init__(
         upsampler: Optional[Callable] = default_upsampler,
         verbose: bool = True,
     ) -> None:
-        """Occlusion sensitivitiy constructor.
+        """Occlusion sensitivity constructor.
 
         Args:
             nn_module: Classification model to use for inference
@@ -187,7 +187,7 @@ def _compute_occlusion_sensitivity(self, x, b_box):
         # Get the number of prediction classes
         num_classes = self.nn_module(x).numel()
 
-        #  If pad val not supplied, get the mean of the image
+        # If pad val not supplied, get the mean of the image
         pad_val = x.mean() if self.pad_val is None else self.pad_val
 
         # List containing a batch of images to be inferred
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 2a43e63d73..9ba05fb769 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,9 +1,9 @@
 # Full requirements for developments
 -r requirements-min.txt
-pytorch-ignite==0.4.2
+pytorch-ignite==0.4.4
 gdown>=3.6.4
 scipy
-itk>=5.0
+itk>=5.0, <=5.1.2
 nibabel
 pillow
 tensorboard
@@ -26,7 +26,9 @@ mypy>=0.790
 ninja
 torchvision
 psutil
-Sphinx==3.3.0
+Sphinx==3.5.3
 recommonmark==0.6.0
 sphinx-autodoc-typehints==1.11.1
-sphinx-rtd-theme==0.5.0
+sphinx-rtd-theme==0.5.2
+cucim==0.18.2
+openslide-python==1.1.2
diff --git a/requirements-min.txt b/requirements-min.txt
index 3a5585de8d..5db219c840 100644
--- a/requirements-min.txt
+++ b/requirements-min.txt
@@ -1,5 +1,5 @@
 # Requirements for minimal tests
 -r requirements.txt
 setuptools>=50.3.0
-coverage
+coverage>=5.5
 parameterized
diff --git a/runtests.sh b/runtests.sh
index 76692e731b..0e87a3a4e5 100755
--- a/runtests.sh
+++ b/runtests.sh
@@ -36,9 +36,7 @@ doQuickTests=false
 doNetTests=false
 doDryRun=false
 doZooTests=false
-
-doUnitTests=true
-
+doUnitTests=false
 doBlackFormat=false
 doBlackFix=false
 doIsortFormat=false
@@ -55,16 +53,17 @@ PY_EXE=${MONAI_PY_EXE:-$(which python)}
 
 function print_usage {
     echo "runtests.sh [--codeformat] [--autofix] [--black] [--isort] [--flake8] [--clangformat] [--pytype] [--mypy]"
-    echo "            [--nounittests] [--coverage] [--quick] [--net] [--dryrun] [-j number] [--clean] [--help] [--version]"
+    echo "            [--unittests] [--coverage] [--quick] [--net] [--dryrun] [-j number] [--clean] [--help] [--version]"
     echo ""
     echo "MONAI unit testing utilities."
     echo ""
     echo "Examples:"
-    echo "./runtests.sh --codeformat --coverage     # run full tests (${green}recommended before making pull requests${noColor})."
-    echo "./runtests.sh --codeformat --nounittests  # run coding style and static type checking."
-    echo "./runtests.sh --quick                     # run minimal unit tests, for quick verification during code developments."
-    echo "./runtests.sh --autofix --nounittests     # run automatic code formatting using \"isort\" and \"black\"."
-    echo "./runtests.sh --clean                     # clean up temporary files and run \"${PY_EXE} setup.py develop --uninstall\"."
+    echo "./runtests.sh -f -u --net --coverage  # run style checks, full tests, print code coverage (${green}recommended for pull requests${noColor})."
+    echo "./runtests.sh -f -u                   # run style checks and unit tests."
+    echo "./runtests.sh -f                      # run coding style and static type checking."
+    echo "./runtests.sh --quick --unittests     # run minimal unit tests, for quick verification during code developments."
+    echo "./runtests.sh --autofix               # run automatic code formatting using \"isort\" and \"black\"."
+    echo "./runtests.sh --clean                 # clean up temporary files and run \"${PY_EXE} setup.py develop --uninstall\"."
     echo ""
     echo "Code style check options:"
     echo "    --black           : perform \"black\" code format checks"
@@ -79,11 +78,11 @@ function print_usage {
     echo "    -j, --jobs        : number of parallel jobs to run \"pytype\" (default $NUM_PARALLEL)"
     echo ""
     echo "MONAI unit testing options:"
-    echo "    --nounittests     : skip doing unit testing (i.e. only format lint testers)"
-    echo "    --coverage        : peforms coverage analysis of code for tests run"
-    echo "    -q, --quick       : disable long running tests"
-    echo "    --net             : perform training/inference/eval integration testing"
-    echo "    --list_tests      : list tests and exit"
+    echo "    -u, --unittests   : perform unit testing"
+    echo "    --coverage        : report testing code coverage, to be used with \"--net\", \"--unittests\""
+    echo "    -q, --quick       : skip long running unit tests and integration tests"
+    echo "    --net             : perform integration testing"
+    echo "    --list_tests      : list unit tests and exit"
     echo ""
     echo "Misc. options:"
     echo "    --dryrun          : display the commands to the screen without running"
@@ -92,7 +91,7 @@ function print_usage {
     echo "    -h, --help        : show this help message and exit"
     echo "    -v, --version     : show MONAI and system version information and exit"
     echo ""
-    echo "${separator}For bug reports, questions, and discussions, please file an issue at:"
+    echo "${separator}For bug reports and feature requests, please file an issue at:"
     echo "    https://github.com/Project-MONAI/MONAI/issues/new/choose"
     echo ""
     echo "To choose an alternative python executable, set the environmental variable, \"MONAI_PY_EXE\"."
@@ -139,6 +138,9 @@ function clang_format {
 }
 
 function clean_py {
+    # remove coverage history
+    ${cmdPrefix}${PY_EXE} -m coverage erase
+
     # uninstall the development package
     echo "Uninstalling MONAI development files..."
     ${cmdPrefix}${PY_EXE} setup.py develop --user --uninstall
@@ -150,7 +152,7 @@ function clean_py {
     find ${TO_CLEAN}/monai -type f -name "*.py[co]" -delete
     find ${TO_CLEAN}/monai -type f -name "*.so" -delete
     find ${TO_CLEAN}/monai -type d -name "__pycache__" -delete
-    find ${TO_CLEAN} -maxdepth 1 -type f -name ".coverage" -delete
+    find ${TO_CLEAN} -maxdepth 1 -type f -name ".coverage.*" -delete
 
     find ${TO_CLEAN} -depth -maxdepth 1 -type d -name ".eggs" -exec rm -r "{}" +
     find ${TO_CLEAN} -depth -maxdepth 1 -type d -name "monai.egg-info" -exec rm -r "{}" +
@@ -159,6 +161,7 @@ function clean_py {
     find ${TO_CLEAN} -depth -maxdepth 1 -type d -name ".mypy_cache" -exec rm -r "{}" +
     find ${TO_CLEAN} -depth -maxdepth 1 -type d -name ".pytype" -exec rm -r "{}" +
     find ${TO_CLEAN} -depth -maxdepth 1 -type d -name ".coverage" -exec rm -r "{}" +
+    find ${TO_CLEAN} -depth -maxdepth 1 -type d -name "__pycache__" -exec rm -r "{}" +
 }
 
 function torch_validate {
@@ -172,7 +175,7 @@ function print_error_msg() {
 
 function print_style_fail_msg() {
     echo "${red}Check failed!${noColor}"
-    echo "Please run auto style fixes: ${green}./runtests.sh --autofix --nounittests${noColor}"
+    echo "Please run auto style fixes: ${green}./runtests.sh --autofix${noColor}"
 }
 
 function is_pip_installed() {
@@ -219,8 +222,8 @@ do
         --dryrun)
             doDryRun=true
         ;;
-        --nou*)  # allow --nounittest | --nounittests | --nounittesting  etc.
-            doUnitTests=false
+        -u|--u*)  # allow --unittest | --unittests | --unittesting  etc.
+            doUnitTests=true
         ;;
         -f|--codeformat)
             doBlackFormat=true
@@ -267,6 +270,10 @@ do
             print_version
             exit 1
         ;;
+        --nou*)  # allow --nounittest | --nounittests | --nounittesting  etc.
+            print_error_msg "nounittest option is deprecated, no unit tests is the default setting"
+            print_usage
+        ;;
         *)
             print_error_msg "Incorrect commandline provided, invalid key: $key"
             print_usage
@@ -492,12 +499,11 @@ then
     export QUICKTEST=True
 fi
 
-# set command and clear previous coverage data
+# set coverage command
 if [ $doCoverage = true ]
 then
     echo "${separator}${blue}coverage${noColor}"
-    cmd="${PY_EXE} -m coverage run -a --source ."
-    ${cmdPrefix}${PY_EXE} -m coverage erase
+    cmd="${PY_EXE} -m coverage run --append"
 fi
 
 # # download test data if needed
@@ -510,7 +516,7 @@ if [ $doUnitTests = true ]
 then
     echo "${separator}${blue}unittests${noColor}"
     torch_validate
-    ${cmdPrefix}${cmd} ./tests/runner.py
+    ${cmdPrefix}${cmd} ./tests/runner.py -p "test_((?!integration).)"
 fi
 
 # network training/inference/eval integration tests
@@ -536,5 +542,6 @@ fi
 if [ $doCoverage = true ]
 then
     echo "${separator}${blue}coverage${noColor}"
-    ${cmdPrefix}${PY_EXE} -m coverage report --skip-covered -m
+    ${cmdPrefix}${PY_EXE} -m coverage combine --append .coverage/
+    ${cmdPrefix}${PY_EXE} -m coverage report
 fi
diff --git a/setup.cfg b/setup.cfg
index ea61eadd92..702c8638c1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -8,6 +8,12 @@ long_description = file:README.md
 long_description_content_type = text/markdown; charset=UTF-8
 platforms = OS Independent
 license = Apache License 2.0
+license_files =
+    LICENSE
+project_urls =
+    Documentation=https://docs.monai.io/
+    Bug Tracker=https://github.com/Project-MONAI/MONAI/issues
+    Source Code=https://github.com/Project-MONAI/MONAI
 
 [options]
 python_requires = >= 3.6
@@ -27,11 +33,13 @@ all =
     scikit-image>=0.14.2
     pillow
     tensorboard
-    pytorch-ignite==0.4.2
+    pytorch-ignite==0.4.4
     gdown>=3.6.4
     torchvision
-    itk>=5.0
+    itk>=5.0, <=5.1.2
     tqdm>=4.47.0
+    cucim==0.18.2
+    openslide-python==1.1.2
 nibabel =
     nibabel
 skimage =
@@ -43,28 +51,32 @@ tensorboard =
 gdown =
     gdown>=3.6.4
 ignite =
-    pytorch-ignite==0.4.2
+    pytorch-ignite==0.4.4
 torchvision =
     torchvision
 itk =
-    itk>=5.0
+    itk>=5.0, <=5.1.2
 tqdm =
     tqdm>=4.47.0
 lmdb =
     lmdb
 psutil =
     psutil
+cucim =
+    cucim==0.18.2
+openslide =
+    openslide-python==1.1.2
 
 [flake8]
 select = B,C,E,F,N,P,T4,W,B9
-max-line-length = 120
+max_line_length = 120
 # C408 ignored because we like the dict keyword argument syntax
 # E501 is not flexible enough, we're using B950 instead
 ignore =
     E203,E305,E402,E501,E721,E741,F821,F841,F999,W503,W504,C408,E302,W291,E303,
     # N812 lowercase 'torch.nn.functional' imported as non lowercase 'F'
     N812
-per-file-ignores = __init__.py: F401
+per_file_ignores = __init__.py: F401
 exclude = *.pyi,.git,.eggs,monai/_version.py,versioneer.py,venv,.venv,_version.py
 
 [isort]
@@ -145,3 +157,21 @@ precise_return = True
 protocols = True
 # Experimental: Only load submodules that are explicitly imported.
 strict_import = False
+
+[coverage:run]
+concurrency = multiprocessing
+source = .
+data_file = .coverage/.coverage
+omit = setup.py
+
+[coverage:report]
+exclude_lines =
+    pragma: no cover
+    # Don't complain if tests don't hit code:
+    raise NotImplementedError
+    if __name__ == .__main__.:
+show_missing = True
+skip_covered = True
+
+[coverage:xml]
+output = coverage.xml
diff --git a/setup.py b/setup.py
index 9b20df845a..426866428c 100644
--- a/setup.py
+++ b/setup.py
@@ -26,6 +26,7 @@
 FORCE_CUDA = os.getenv("FORCE_CUDA", "0") == "1"  # flag ignored if BUILD_MONAI is False
 
 BUILD_CPP = BUILD_CUDA = False
+TORCH_VERSION = 0
 try:
     import torch
 
@@ -35,14 +36,13 @@
     BUILD_CPP = True
     from torch.utils.cpp_extension import CUDA_HOME, CUDAExtension
 
-    BUILD_CUDA = (torch.cuda.is_available() and (CUDA_HOME is not None)) or FORCE_CUDA
+    BUILD_CUDA = (CUDA_HOME is not None) if torch.cuda.is_available() else FORCE_CUDA
 
     _pt_version = pkg_resources.parse_version(torch.__version__).release  # type: ignore[attr-defined]
     if _pt_version is None or len(_pt_version) < 3:
         raise AssertionError("unknown torch version")
     TORCH_VERSION = int(_pt_version[0]) * 10000 + int(_pt_version[1]) * 100 + int(_pt_version[2])
 except (ImportError, TypeError, AssertionError, AttributeError) as e:
-    TORCH_VERSION = 0
     warnings.warn(f"extension build skipped: {e}")
 finally:
     if not RUN_BUILD:
diff --git a/tests/min_tests.py b/tests/min_tests.py
index 999a1aeaa0..49decc1e6a 100644
--- a/tests/min_tests.py
+++ b/tests/min_tests.py
@@ -33,6 +33,7 @@ def run_testsuit():
         "test_cachedataset_parallel",
         "test_dataset",
         "test_detect_envelope",
+        "test_efficientnet",
         "test_iterable_dataset",
         "test_ensemble_evaluator",
         "test_handler_checkpoint_loader",
@@ -42,9 +43,12 @@ def run_testsuit():
         "test_handler_confusion_matrix",
         "test_handler_confusion_matrix_dist",
         "test_handler_hausdorff_distance",
+        "test_handler_garbage_collector",
         "test_handler_mean_dice",
+        "test_handler_prob_map_producer",
         "test_handler_rocauc",
         "test_handler_rocauc_dist",
+        "test_handler_parameter_scheduler",
         "test_handler_segmentation_saver",
         "test_handler_smartcache",
         "test_handler_stats",
@@ -93,6 +97,7 @@ def run_testsuit():
         "test_smartcachedataset",
         "test_spacing",
         "test_spacingd",
+        "test_senet",
         "test_surface_distance",
         "test_zoom",
         "test_zoom_affine",
@@ -109,6 +114,11 @@ def run_testsuit():
         "test_deepgrow_dataset",
         "test_save_image",
         "test_save_imaged",
+        "test_ensure_channel_first",
+        "test_ensure_channel_firstd",
+        "test_handler_early_stop",
+        "test_handler_transform_inverter",
+        "test_testtimeaugmentation",
     ]
     assert sorted(exclude_cases) == sorted(set(exclude_cases)), f"Duplicated items in {exclude_cases}"
 
diff --git a/tests/runner.py b/tests/runner.py
index b5d1de5fc1..b340d60719 100644
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -10,8 +10,10 @@
 # limitations under the License.
 
 import argparse
+import glob
 import inspect
 import os
+import re
 import sys
 import time
 import unittest
@@ -62,7 +64,7 @@ def print_results(results, discovery_time, thresh, status):
     print("Remember to check above times for any errors!")
 
 
-def parse_args(default_pattern):
+def parse_args():
     parser = argparse.ArgumentParser(description="Runner for MONAI unittests with timing.")
     parser.add_argument(
         "-s", action="store", dest="path", default=".", help="Directory to start discovery (default: '%(default)s')"
@@ -71,7 +73,7 @@ def parse_args(default_pattern):
         "-p",
         action="store",
         dest="pattern",
-        default=default_pattern,
+        default="test_*.py",
         help="Pattern to match tests (default: '%(default)s')",
     )
     parser.add_argument(
@@ -111,11 +113,8 @@ def get_default_pattern(loader):
 
 if __name__ == "__main__":
 
-    loader = unittest.TestLoader()
-    default_pattern = get_default_pattern(loader)
-
     # Parse input arguments
-    args = parse_args(default_pattern)
+    args = parse_args()
 
     # If quick is desired, set environment variable
     if args.quick:
@@ -123,9 +122,17 @@ def get_default_pattern(loader):
 
     # Get all test names (optionally from some path with some pattern)
     with PerfContext() as pc:
-        tests = loader.discover(args.path, args.pattern)
+        # the files are searched from `tests/` folder, starting with `test_`
+        files = glob.glob(os.path.join(os.path.dirname(__file__), "test_*.py"))
+        cases = []
+        for test_module in {os.path.basename(f)[:-3] for f in files}:
+            if re.match(args.pattern, test_module):
+                cases.append(f"tests.{test_module}")
+            else:
+                print(f"monai test runner: excluding tests.{test_module}")
+        tests = unittest.TestLoader().loadTestsFromNames(cases)
     discovery_time = pc.total_time
-    print(f"time to discover tests: {discovery_time}s")
+    print(f"time to discover tests: {discovery_time}s, total cases: {tests.countTestCases()}.")
 
     test_runner = unittest.runner.TextTestRunner(
         resultclass=TimeLoggingTestResult, verbosity=args.verbosity, failfast=args.failfast
diff --git a/tests/test_activations.py b/tests/test_activations.py
index 1614642d6d..5ed9ec2046 100644
--- a/tests/test_activations.py
+++ b/tests/test_activations.py
@@ -48,6 +48,15 @@
 ]
 
 TEST_CASE_5 = [
+    "memswish",
+    torch.tensor([[[[-10, -8, -6, -4, -2], [0, 2, 4, 6, 8]]]], dtype=torch.float32),
+    torch.tensor(
+        [[[[-4.54e-04, -2.68e-03, -1.48e-02, -7.19e-02, -2.38e-01], [0.00e00, 1.76e00, 3.93e00, 5.99e00, 8.00e00]]]]
+    ),
+    (1, 1, 2, 5),
+]
+
+TEST_CASE_6 = [
     "mish",
     torch.tensor([[[[-10, -8, -6, -4, -2], [0, 2, 4, 6, 8]]]], dtype=torch.float32),
     torch.tensor(
@@ -64,7 +73,7 @@ def test_value_shape(self, input_param, img, out, expected_shape):
         torch.testing.assert_allclose(result, out)
         self.assertTupleEqual(result.shape, expected_shape)
 
-    @parameterized.expand([TEST_CASE_4, TEST_CASE_5])
+    @parameterized.expand([TEST_CASE_4, TEST_CASE_5, TEST_CASE_6])
     def test_monai_activations_value_shape(self, input_param, img, out, expected_shape):
         act = Act[input_param]()
         result = act(img)
diff --git a/tests/test_affine.py b/tests/test_affine.py
index 934473fc5c..1b6c19596b 100644
--- a/tests/test_affine.py
+++ b/tests/test_affine.py
@@ -78,12 +78,9 @@ class TestAffine(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
     def test_affine(self, input_param, input_data, expected_val):
         g = Affine(**input_param)
-        result = g(**input_data)
+        result, _ = g(**input_data)
         self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
-        if isinstance(result, torch.Tensor):
-            np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
-        else:
-            np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
+        np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_affine_grid.py b/tests/test_affine_grid.py
index 2906cd18b6..24772b9a21 100644
--- a/tests/test_affine_grid.py
+++ b/tests/test_affine_grid.py
@@ -92,7 +92,7 @@ class TestAffineGrid(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
     def test_affine_grid(self, input_param, input_data, expected_val):
         g = AffineGrid(**input_param)
-        result = g(**input_data)
+        result, _ = g(**input_data)
         self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
         if isinstance(result, torch.Tensor):
             np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
diff --git a/tests/test_affined.py b/tests/test_affined.py
new file mode 100644
index 0000000000..850f12905d
--- /dev/null
+++ b/tests/test_affined.py
@@ -0,0 +1,101 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.transforms import Affined
+
+TEST_CASES = [
+    [
+        dict(keys="img", padding_mode="zeros", as_tensor_output=False, spatial_size=(-1, 0), device=None),
+        {"img": np.arange(9).reshape((1, 3, 3))},
+        np.arange(9).reshape(1, 3, 3),
+    ],
+    [
+        dict(keys="img", padding_mode="zeros", as_tensor_output=False, device=None),
+        {"img": np.arange(4).reshape((1, 2, 2))},
+        np.arange(4).reshape(1, 2, 2),
+    ],
+    [
+        dict(keys="img", padding_mode="zeros", spatial_size=(4, 4), as_tensor_output=False, device=None),
+        {"img": np.arange(4).reshape((1, 2, 2))},
+        np.array([[[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 2.0, 3.0, 0.0], [0.0, 0.0, 0.0, 0.0]]]),
+    ],
+    [
+        dict(
+            keys="img",
+            rotate_params=[np.pi / 2],
+            padding_mode="zeros",
+            spatial_size=(4, 4),
+            as_tensor_output=False,
+            device=None,
+        ),
+        {"img": np.arange(4).reshape((1, 2, 2))},
+        np.array([[[0.0, 0.0, 0.0, 0.0], [0.0, 2.0, 0.0, 0.0], [0.0, 3.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]]),
+    ],
+    [
+        dict(keys="img", padding_mode="zeros", spatial_size=(-1, 0, 0), as_tensor_output=False, device=None),
+        {"img": np.arange(27).reshape((1, 3, 3, 3))},
+        np.arange(27).reshape(1, 3, 3, 3),
+    ],
+    [
+        dict(keys="img", padding_mode="zeros", spatial_size=(4, 4, 4), as_tensor_output=False, device=None),
+        {"img": np.arange(8).reshape((1, 2, 2, 2))},
+        np.array(
+            [
+                [
+                    [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+                    [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 2.0, 3.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+                    [[0.0, 0.0, 0.0, 0.0], [0.0, 4.0, 5.0, 0.0], [0.0, 6.0, 7.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+                    [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+                ]
+            ]
+        ),
+    ],
+    [
+        dict(
+            keys="img",
+            rotate_params=[np.pi / 2],
+            padding_mode="zeros",
+            spatial_size=(4, 4, 4),
+            as_tensor_output=False,
+            device=None,
+        ),
+        {"img": np.arange(8).reshape((1, 2, 2, 2))},
+        np.array(
+            [
+                [
+                    [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+                    [[0.0, 0.0, 0.0, 0.0], [0.0, 2.0, 0.0, 0.0], [0.0, 3.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+                    [[0.0, 0.0, 0.0, 0.0], [0.0, 6.0, 4.0, 0.0], [0.0, 7.0, 5.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+                    [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]],
+                ]
+            ]
+        ),
+    ],
+]
+
+
+class TestAffined(unittest.TestCase):
+    @parameterized.expand(TEST_CASES)
+    def test_affine(self, input_param, input_data, expected_val):
+        g = Affined(**input_param)
+        result = g(input_data)["img"]
+        self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected_val, torch.Tensor))
+        np.testing.assert_allclose(result, expected_val, rtol=1e-4, atol=1e-4)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_bending_energy.py b/tests/test_bending_energy.py
index f2b9a41cae..8f1fb43535 100644
--- a/tests/test_bending_energy.py
+++ b/tests/test_bending_energy.py
@@ -17,30 +17,32 @@
 
 from monai.losses.deform import BendingEnergyLoss
 
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
 TEST_CASES = [
     [
         {},
-        {"pred": torch.ones((1, 3, 5, 5, 5))},
+        {"pred": torch.ones((1, 3, 5, 5, 5), device=device)},
         0.0,
     ],
     [
         {},
-        {"pred": torch.arange(0, 5)[None, None, None, None, :].expand(1, 3, 5, 5, 5)},
+        {"pred": torch.arange(0, 5, device=device)[None, None, None, None, :].expand(1, 3, 5, 5, 5)},
         0.0,
     ],
     [
         {},
-        {"pred": torch.arange(0, 5)[None, None, None, None, :].expand(1, 3, 5, 5, 5) ** 2},
+        {"pred": torch.arange(0, 5, device=device)[None, None, None, None, :].expand(1, 3, 5, 5, 5) ** 2},
         4.0,
     ],
     [
         {},
-        {"pred": torch.arange(0, 5)[None, None, None, :].expand(1, 3, 5, 5) ** 2},
+        {"pred": torch.arange(0, 5, device=device)[None, None, None, :].expand(1, 3, 5, 5) ** 2},
         4.0,
     ],
     [
         {},
-        {"pred": torch.arange(0, 5)[None, None, :].expand(1, 3, 5) ** 2},
+        {"pred": torch.arange(0, 5, device=device)[None, None, :].expand(1, 3, 5) ** 2},
         4.0,
     ],
 ]
@@ -56,19 +58,19 @@ def test_ill_shape(self):
         loss = BendingEnergyLoss()
         # not in 3-d, 4-d, 5-d
         with self.assertRaisesRegex(ValueError, ""):
-            loss.forward(torch.ones((1, 3)))
+            loss.forward(torch.ones((1, 3), device=device))
         with self.assertRaisesRegex(ValueError, ""):
-            loss.forward(torch.ones((1, 3, 5, 5, 5, 5)))
+            loss.forward(torch.ones((1, 3, 5, 5, 5, 5), device=device))
         # spatial_dim < 5
         with self.assertRaisesRegex(ValueError, ""):
-            loss.forward(torch.ones((1, 3, 4, 5, 5)))
+            loss.forward(torch.ones((1, 3, 4, 5, 5), device=device))
         with self.assertRaisesRegex(ValueError, ""):
             loss.forward(torch.ones((1, 3, 5, 4, 5)))
         with self.assertRaisesRegex(ValueError, ""):
             loss.forward(torch.ones((1, 3, 5, 5, 4)))
 
     def test_ill_opts(self):
-        pred = torch.rand(1, 3, 5, 5, 5)
+        pred = torch.rand(1, 3, 5, 5, 5).to(device=device)
         with self.assertRaisesRegex(ValueError, ""):
             BendingEnergyLoss(reduction="unknown")(pred)
         with self.assertRaisesRegex(ValueError, ""):
diff --git a/tests/test_bilateral_approx_cpu.py b/tests/test_bilateral_approx_cpu.py
index 2b6088a56f..96d60fb22c 100644
--- a/tests/test_bilateral_approx_cpu.py
+++ b/tests/test_bilateral_approx_cpu.py
@@ -14,6 +14,7 @@
 import numpy as np
 import torch
 from parameterized import parameterized
+from torch.autograd import gradcheck
 
 from monai.networks.layers.filtering import BilateralFilter
 from tests.utils import skip_if_no_cpp_extension
@@ -376,6 +377,23 @@ def test_cpu_approx(self, test_case_description, sigmas, input, expected):
         # Ensure result are as expected
         np.testing.assert_allclose(output, expected, atol=1e-5)
 
+    @parameterized.expand(TEST_CASES)
+    def test_cpu_approx_backwards(self, test_case_description, sigmas, input, expected):
+
+        # Params to determine the implementation to test
+        device = torch.device("cpu")
+        fast_approx = True
+
+        # Prepare input tensor
+        input_tensor = torch.from_numpy(np.array(input)).to(dtype=torch.float, device=device)
+        input_tensor.requires_grad = True
+
+        # Prepare args
+        args = (input_tensor, *sigmas, fast_approx)
+
+        # Run grad check
+        gradcheck(BilateralFilter.apply, args, raise_exception=False)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_bilateral_approx_cuda.py b/tests/test_bilateral_approx_cuda.py
index fdaba26f72..c95fbed1f5 100644
--- a/tests/test_bilateral_approx_cuda.py
+++ b/tests/test_bilateral_approx_cuda.py
@@ -14,6 +14,7 @@
 import numpy as np
 import torch
 from parameterized import parameterized
+from torch.autograd import gradcheck
 
 from monai.networks.layers.filtering import BilateralFilter
 from tests.utils import skip_if_no_cpp_extension, skip_if_no_cuda
@@ -381,6 +382,23 @@ def test_cuda_approx(self, test_case_description, sigmas, input, expected):
         # Ensure result are as expected
         np.testing.assert_allclose(output, expected, atol=1e-2)
 
+    @parameterized.expand(TEST_CASES)
+    def test_cpu_approx_backwards(self, test_case_description, sigmas, input, expected):
+
+        # Params to determine the implementation to test
+        device = torch.device("cuda")
+        fast_approx = True
+
+        # Prepare input tensor
+        input_tensor = torch.from_numpy(np.array(input)).to(dtype=torch.float, device=device)
+        input_tensor.requires_grad = True
+
+        # Prepare args
+        args = (input_tensor, *sigmas, fast_approx)
+
+        # Run grad check
+        gradcheck(BilateralFilter.apply, args, raise_exception=False)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_bilateral_precise.py b/tests/test_bilateral_precise.py
index db2ee88239..df4d0c2500 100644
--- a/tests/test_bilateral_precise.py
+++ b/tests/test_bilateral_precise.py
@@ -14,6 +14,7 @@
 import numpy as np
 import torch
 from parameterized import parameterized
+from torch.autograd import gradcheck
 
 from monai.networks.layers.filtering import BilateralFilter
 from tests.utils import skip_if_no_cpp_extension, skip_if_no_cuda
@@ -361,9 +362,9 @@
 
 
 @skip_if_no_cpp_extension
-class BilateralFilterTestCaseCpuPrecised(unittest.TestCase):
+class BilateralFilterTestCaseCpuPrecise(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
-    def test_cpu_precised(self, test_case_description, sigmas, input, expected):
+    def test_cpu_precise(self, test_case_description, sigmas, input, expected):
 
         # Params to determine the implementation to test
         device = torch.device("cpu")
@@ -376,12 +377,29 @@ def test_cpu_precised(self, test_case_description, sigmas, input, expected):
         # Ensure result are as expected
         np.testing.assert_allclose(output, expected, atol=1e-5)
 
+    @parameterized.expand(TEST_CASES)
+    def test_cpu_precise_backwards(self, test_case_description, sigmas, input, expected):
+
+        # Params to determine the implementation to test
+        device = torch.device("cpu")
+        fast_approx = False
+
+        # Prepare input tensor
+        input_tensor = torch.from_numpy(np.array(input)).to(dtype=torch.float, device=device)
+        input_tensor.requires_grad = True
+
+        # Prepare args
+        args = (input_tensor, *sigmas, fast_approx)
+
+        # Run grad check
+        gradcheck(BilateralFilter.apply, args, raise_exception=False)
+
 
 @skip_if_no_cuda
 @skip_if_no_cpp_extension
-class BilateralFilterTestCaseCudaPrecised(unittest.TestCase):
+class BilateralFilterTestCaseCudaPrecise(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
-    def test_cuda_precised(self, test_case_description, sigmas, input, expected):
+    def test_cuda_precise(self, test_case_description, sigmas, input, expected):
 
         # Skip this test
         if not torch.cuda.is_available():
@@ -398,6 +416,23 @@ def test_cuda_precised(self, test_case_description, sigmas, input, expected):
         # Ensure result are as expected
         np.testing.assert_allclose(output, expected, atol=1e-5)
 
+    @parameterized.expand(TEST_CASES)
+    def test_cuda_precise_backwards(self, test_case_description, sigmas, input, expected):
+
+        # Params to determine the implementation to test
+        device = torch.device("cuda")
+        fast_approx = False
+
+        # Prepare input tensor
+        input_tensor = torch.from_numpy(np.array(input)).to(dtype=torch.float, device=device)
+        input_tensor.requires_grad = True
+
+        # Prepare args
+        args = (input_tensor, *sigmas, fast_approx)
+
+        # Run grad check
+        gradcheck(BilateralFilter.apply, args, raise_exception=False)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_cachedataset.py b/tests/test_cachedataset.py
index 2b8931704a..91e2558e89 100644
--- a/tests/test_cachedataset.py
+++ b/tests/test_cachedataset.py
@@ -51,10 +51,14 @@ def test_shape(self, transform, expected_shape):
             dataset = CacheDataset(data=test_data, transform=transform, cache_rate=0.5)
             data1 = dataset[0]
             data2 = dataset[1]
+            data3 = dataset[0:-1]
+            data4 = dataset[-1]
+            self.assertEqual(len(data3), 1)
 
         if transform is None:
             self.assertEqual(data1["image"], os.path.join(tempdir, "test_image1.nii.gz"))
             self.assertEqual(data2["label"], os.path.join(tempdir, "test_label2.nii.gz"))
+            self.assertEqual(data4["image"], os.path.join(tempdir, "test_image2.nii.gz"))
         else:
             self.assertTupleEqual(data1["image"].shape, expected_shape)
             self.assertTupleEqual(data1["label"].shape, expected_shape)
@@ -62,6 +66,8 @@ def test_shape(self, transform, expected_shape):
             self.assertTupleEqual(data2["image"].shape, expected_shape)
             self.assertTupleEqual(data2["label"].shape, expected_shape)
             self.assertTupleEqual(data2["extra"].shape, expected_shape)
+            for d in data3:
+                self.assertTupleEqual(d["image"].shape, expected_shape)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_center_spatial_crop.py b/tests/test_center_spatial_crop.py
index c03ec24e18..3e828176a5 100644
--- a/tests/test_center_spatial_crop.py
+++ b/tests/test_center_spatial_crop.py
@@ -12,6 +12,7 @@
 import unittest
 
 import numpy as np
+import torch
 from parameterized import parameterized
 
 from monai.transforms import CenterSpatialCrop
@@ -26,9 +27,15 @@
     np.array([[[1, 2], [2, 3]]]),
 ]
 
+TEST_CASE_3 = [
+    {"roi_size": [2, 2, 2]},
+    torch.randint(0, 2, size=[3, 3, 3, 3], device="cuda" if torch.cuda.is_available() else "cpu"),
+    (3, 2, 2, 2),
+]
+
 
 class TestCenterSpatialCrop(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_0, TEST_CASE_1])
+    @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_3])
     def test_shape(self, input_param, input_data, expected_shape):
         result = CenterSpatialCrop(**input_param)(input_data)
         np.testing.assert_allclose(result.shape, expected_shape)
diff --git a/tests/test_compose.py b/tests/test_compose.py
index c049044a97..97b044af8f 100644
--- a/tests/test_compose.py
+++ b/tests/test_compose.py
@@ -13,7 +13,8 @@
 import unittest
 
 from monai.data import DataLoader, Dataset
-from monai.transforms import AddChannel, Compose, Randomizable
+from monai.transforms import AddChannel, Compose
+from monai.transforms.transform import Randomizable
 from monai.utils import set_determinism
 
 
@@ -102,6 +103,9 @@ class _RandomClass(Randomizable):
             def randomize(self, foo1, foo2):
                 pass
 
+            def __call__(self, data):
+                pass
+
         c = Compose([_RandomClass(), _RandomClass()])
         with self.assertWarns(Warning):
             c.randomize()
@@ -167,6 +171,9 @@ def test_flatten_and_len(self):
         # test len
         self.assertEqual(len(t1), 8)
 
+    def test_backwards_compatible_imports(self):
+        from monai.transforms.compose import MapTransform, RandomizableTransform, Transform  # noqa: F401
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_compute_froc.py b/tests/test_compute_froc.py
new file mode 100644
index 0000000000..70de836dd9
--- /dev/null
+++ b/tests/test_compute_froc.py
@@ -0,0 +1,101 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.metrics import compute_fp_tp_probs, compute_froc_curve_data, compute_froc_score
+
+TEST_CASE_1 = [
+    {
+        "probs": torch.tensor([1, 0.6, 0.8]),
+        "y_coord": torch.tensor([0, 2, 3]),
+        "x_coord": torch.tensor([3, 0, 1]),
+        "evaluation_mask": np.array([[0, 0, 1, 1], [2, 2, 0, 0], [0, 3, 3, 0], [0, 3, 3, 3]]),
+        "labels_to_exclude": [2],
+        "resolution_level": 0,
+    },
+    np.array([0.6]),
+    np.array([1, 0, 0.8]),
+    2,
+]
+
+TEST_CASE_2 = [
+    {
+        "probs": torch.tensor([1, 0.6, 0.8]),
+        "y_coord": torch.tensor([0, 2, 3]),
+        "x_coord": torch.tensor([3, 0, 1]),
+        "evaluation_mask": np.array([[0, 0, 1, 1], [2, 2, 0, 0], [0, 3, 3, 0], [0, 3, 3, 3]]),
+        "resolution_level": 0,
+    },
+    np.array([0.6]),
+    np.array([1, 0, 0.8]),
+    3,
+]
+
+TEST_CASE_3 = [
+    {
+        "probs": torch.tensor([1, 0.6, 0.8]),
+        "y_coord": torch.tensor([0, 4, 6]),
+        "x_coord": torch.tensor([6, 0, 2]),
+        "evaluation_mask": np.array([[0, 0, 1, 1], [2, 2, 0, 0], [0, 3, 3, 0], [0, 3, 3, 3]]),
+        "resolution_level": 1,
+    },
+    np.array([0.6]),
+    np.array([1, 0, 0.8]),
+    3,
+]
+
+TEST_CASE_4 = [
+    {
+        "fp_probs": np.array([0.8, 0.6]),
+        "tp_probs": np.array([1, 1, 0, 0, 0.8, 0.8, 0]),
+        "num_targets": 4,
+        "num_images": 2,
+    },
+    (0.25, 0.5, 1, 2, 4, 8),
+    0.95833333,
+]
+
+TEST_CASE_5 = [
+    {
+        "fp_probs": torch.tensor([0.8, 0.6]),
+        "tp_probs": torch.tensor([1, 1, 0, 0, 0.8, 0.8, 0]),
+        "num_targets": 4,
+        "num_images": 2,
+    },
+    (0.25),
+    0.75,
+]
+
+
+class TestComputeFpTp(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
+    def test_value(self, input_data, expected_fp, expected_tp, expected_num):
+        fp_probs, tp_probs, num_tumors = compute_fp_tp_probs(**input_data)
+        np.testing.assert_allclose(fp_probs, expected_fp, rtol=1e-5)
+        np.testing.assert_allclose(tp_probs, expected_tp, rtol=1e-5)
+        np.testing.assert_equal(num_tumors, expected_num)
+
+
+class TestComputeFrocScore(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_4, TEST_CASE_5])
+    def test_value(self, input_data, thresholds, expected_score):
+        fps_per_image, total_sensitivity = compute_froc_curve_data(**input_data)
+        score = compute_froc_score(fps_per_image, total_sensitivity, thresholds)
+        np.testing.assert_allclose(score, expected_score, rtol=1e-5)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_compute_roc_auc.py b/tests/test_compute_roc_auc.py
index 612bd375ac..10141ce0a7 100644
--- a/tests/test_compute_roc_auc.py
+++ b/tests/test_compute_roc_auc.py
@@ -16,71 +16,78 @@
 from parameterized import parameterized
 
 from monai.metrics import compute_roc_auc
+from monai.transforms import Activations, AsDiscrete
 
 TEST_CASE_1 = [
-    {
-        "y_pred": torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5]]),
-        "y": torch.tensor([[0], [1], [0], [1]]),
-        "to_onehot_y": True,
-        "softmax": True,
-    },
+    torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5]]),
+    torch.tensor([[0], [1], [0], [1]]),
+    True,
+    True,
+    "macro",
     0.75,
 ]
 
-TEST_CASE_2 = [{"y_pred": torch.tensor([[0.5], [0.5], [0.2], [8.3]]), "y": torch.tensor([[0], [1], [0], [1]])}, 0.875]
+TEST_CASE_2 = [
+    torch.tensor([[0.5], [0.5], [0.2], [8.3]]),
+    torch.tensor([[0], [1], [0], [1]]),
+    False,
+    False,
+    "macro",
+    0.875,
+]
 
-TEST_CASE_3 = [{"y_pred": torch.tensor([[0.5], [0.5], [0.2], [8.3]]), "y": torch.tensor([0, 1, 0, 1])}, 0.875]
+TEST_CASE_3 = [
+    torch.tensor([[0.5], [0.5], [0.2], [8.3]]),
+    torch.tensor([0, 1, 0, 1]),
+    False,
+    False,
+    "macro",
+    0.875,
+]
 
-TEST_CASE_4 = [{"y_pred": torch.tensor([0.5, 0.5, 0.2, 8.3]), "y": torch.tensor([0, 1, 0, 1])}, 0.875]
+TEST_CASE_4 = [
+    torch.tensor([0.5, 0.5, 0.2, 8.3]),
+    torch.tensor([0, 1, 0, 1]),
+    False,
+    False,
+    "macro",
+    0.875,
+]
 
 TEST_CASE_5 = [
-    {
-        "y_pred": torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5]]),
-        "y": torch.tensor([[0], [1], [0], [1]]),
-        "to_onehot_y": True,
-        "softmax": True,
-        "average": "none",
-    },
+    torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5]]),
+    torch.tensor([[0], [1], [0], [1]]),
+    True,
+    True,
+    "none",
     [0.75, 0.75],
 ]
 
 TEST_CASE_6 = [
-    {
-        "y_pred": torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5], [0.1, 0.5]]),
-        "y": torch.tensor([[1, 0], [0, 1], [0, 0], [1, 1], [0, 1]]),
-        "softmax": True,
-        "average": "weighted",
-    },
+    torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5], [0.1, 0.5]]),
+    torch.tensor([[1, 0], [0, 1], [0, 0], [1, 1], [0, 1]]),
+    True,
+    False,
+    "weighted",
     0.56667,
 ]
 
 TEST_CASE_7 = [
-    {
-        "y_pred": torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5], [0.1, 0.5]]),
-        "y": torch.tensor([[1, 0], [0, 1], [0, 0], [1, 1], [0, 1]]),
-        "softmax": True,
-        "average": "micro",
-    },
+    torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5], [0.1, 0.5]]),
+    torch.tensor([[1, 0], [0, 1], [0, 0], [1, 1], [0, 1]]),
+    True,
+    False,
+    "micro",
     0.62,
 ]
 
-TEST_CASE_8 = [
-    {
-        "y_pred": torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5]]),
-        "y": torch.tensor([[0], [1], [0], [1]]),
-        "to_onehot_y": True,
-        "other_act": lambda x: torch.log_softmax(x, dim=1),
-    },
-    0.75,
-]
-
 
 class TestComputeROCAUC(unittest.TestCase):
-    @parameterized.expand(
-        [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]
-    )
-    def test_value(self, input_data, expected_value):
-        result = compute_roc_auc(**input_data)
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7])
+    def test_value(self, y_pred, y, softmax, to_onehot, average, expected_value):
+        y_pred = Activations(softmax=softmax)(y_pred)
+        y = AsDiscrete(to_onehot=to_onehot, n_classes=2)(y)
+        result = compute_roc_auc(y_pred=y_pred, y=y, average=average)
         np.testing.assert_allclose(expected_value, result, rtol=1e-5)
 
 
diff --git a/tests/test_concat_itemsd.py b/tests/test_concat_itemsd.py
index 520833fc88..9c51e1efea 100644
--- a/tests/test_concat_itemsd.py
+++ b/tests/test_concat_itemsd.py
@@ -38,6 +38,20 @@ def test_numpy_values(self):
         np.testing.assert_allclose(result["img1"], np.array([[0, 1], [1, 2]]))
         np.testing.assert_allclose(result["cat_img"], np.array([[1, 2], [2, 3], [1, 2], [2, 3]]))
 
+    def test_single_numpy(self):
+        input_data = {"img": np.array([[0, 1], [1, 2]])}
+        result = ConcatItemsd(keys="img", name="cat_img")(input_data)
+        result["cat_img"] += 1
+        np.testing.assert_allclose(result["img"], np.array([[0, 1], [1, 2]]))
+        np.testing.assert_allclose(result["cat_img"], np.array([[1, 2], [2, 3]]))
+
+    def test_single_tensor(self):
+        input_data = {"img": torch.tensor([[0, 1], [1, 2]])}
+        result = ConcatItemsd(keys="img", name="cat_img")(input_data)
+        result["cat_img"] += 1
+        torch.testing.assert_allclose(result["img"], torch.tensor([[0, 1], [1, 2]]))
+        torch.testing.assert_allclose(result["cat_img"], torch.tensor([[1, 2], [2, 3]]))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_convert_to_multi_channel.py b/tests/test_convert_to_multi_channel.py
index ea27371ac7..2f7a38e6e4 100644
--- a/tests/test_convert_to_multi_channel.py
+++ b/tests/test_convert_to_multi_channel.py
@@ -16,17 +16,29 @@
 
 from monai.transforms import ConvertToMultiChannelBasedOnBratsClasses
 
-TEST_CASE = [
+TEST_CASE_1 = [
     np.array([[0, 1, 2], [1, 2, 4], [0, 1, 4]]),
     np.array([[[0, 1, 0], [1, 0, 1], [0, 1, 1]], [[0, 1, 1], [1, 1, 1], [0, 1, 1]], [[0, 0, 0], [0, 0, 1], [0, 0, 1]]]),
 ]
 
+TEST_CASE_2 = [
+    np.array([[[[0, 1], [1, 2]], [[2, 4], [4, 4]]]]),
+    np.array(
+        [
+            [[[0, 1], [1, 0]], [[0, 1], [1, 1]]],
+            [[[0, 1], [1, 1]], [[1, 1], [1, 1]]],
+            [[[0, 0], [0, 0]], [[0, 1], [1, 1]]],
+        ]
+    ),
+]
+
 
 class TestConvertToMultiChannel(unittest.TestCase):
-    @parameterized.expand([TEST_CASE])
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
     def test_type_shape(self, data, expected_result):
         result = ConvertToMultiChannelBasedOnBratsClasses()(data)
         np.testing.assert_equal(result, expected_result)
+        self.assertEqual(f"{result.dtype}", "bool")
 
 
 if __name__ == "__main__":
diff --git a/tests/test_crf_cpu.py b/tests/test_crf_cpu.py
new file mode 100644
index 0000000000..41ae75f4b4
--- /dev/null
+++ b/tests/test_crf_cpu.py
@@ -0,0 +1,450 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.networks.blocks import CRF
+from tests.utils import skip_if_no_cpp_extension
+
+TEST_CASES = [
+    [
+        # Case Description
+        "2 batche(s), 1 dimension(s), 2 classe(s), 1 channel(s)",
+        # Parameters
+        [
+            1.0,  # bilateral_weight
+            0.3,  # gaussian_weight
+            5.0,  # bilateral_spatial_sigma
+            0.5,  # bilateral_color_sigma
+            5.0,  # gaussian_spatial_sigma
+            1.0,  # update_factor
+            1,  # compatibility_kernel_range
+            5,  # iterations
+        ],
+        # Input
+        [
+            # Batch 0
+            [
+                # Class 0
+                [0.8, 0.9, 0.6, 0.2, 0.3],
+                # Class 1
+                [0.1, 0.3, 0.5, 0.8, 0.7],
+            ],
+            # Batch 1
+            [
+                # Class 0
+                [0.8, 0.9, 0.6, 0.2, 0.3],
+                # Class 1
+                [0.1, 0.3, 0.5, 0.8, 0.7],
+            ],
+        ],
+        # Features
+        [
+            # Batch 0
+            [
+                # Channel 0
+                [1, 1, 1, 0.5, 0],
+            ],
+            # Batch 1
+            [
+                # Channel 0
+                [1, 1, 0.5, 0, 0],
+            ],
+        ],
+        # Expected
+        [
+            # Batch 0
+            [
+                # Class 0
+                [0.976472, 0.973789, 0.951958, 0.882982, 0.876651],
+                # Class 1
+                [0.023528, 0.026211, 0.048042, 0.117018, 0.123349],
+            ],
+            # Batch 1
+            [
+                # Class 0
+                [0.963642, 0.946892, 0.858650, 0.633639, 0.617334],
+                # Class 1
+                [0.036358, 0.053108, 0.141350, 0.366361, 0.382666],
+            ],
+        ],
+    ],
+    [
+        # Case Description
+        "1 batche(s), 2 dimension(s), 3 classe(s), 2 channel(s)",
+        # Parameters
+        [
+            1.0,  # bilateral_weight
+            0.3,  # gaussian_weight
+            5.0,  # bilateral_spatial_sigma
+            0.5,  # bilateral_color_sigma
+            5.0,  # gaussian_spatial_sigma
+            1.0,  # update_factor
+            1,  # compatibility_kernel_range
+            5,  # iterations
+        ],
+        # Input
+        [
+            # Batch 0
+            [
+                # Class 0
+                [
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 1.0, 1.0],
+                    [0.0, 0.0, 0.0, 1.0, 1.0],
+                ],
+                # Class 1
+                [
+                    [1.0, 1.0, 0.0, 0.0, 0.0],
+                    [1.0, 1.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                ],
+                # Class 2
+                [
+                    [0.0, 0.0, 0.0, 1.0, 1.0],
+                    [0.0, 0.0, 1.0, 1.0, 1.0],
+                    [0.0, 1.0, 1.0, 1.0, 0.0],
+                    [1.0, 1.0, 1.0, 0.0, 0.0],
+                    [1.0, 1.0, 0.0, 0.0, 0.0],
+                ],
+            ],
+        ],
+        # Features
+        [
+            # Batch 0
+            [
+                # Channel 0
+                [
+                    [1.0, 1.0, 0.0, 0.0, 0.0],
+                    [1.0, 1.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                ],
+                # Channel 1
+                [
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 1.0, 1.0],
+                    [0.0, 0.0, 0.0, 1.0, 1.0],
+                ],
+            ],
+        ],
+        # Expected
+        [
+            # Batch 0
+            [
+                # Class 0
+                [
+                    [0.000008, 0.000008, 0.000008, 0.000003, 0.000003],
+                    [0.000008, 0.000008, 0.000003, 0.000003, 0.000003],
+                    [0.000008, 0.000003, 0.000003, 0.000003, 0.000008],
+                    [0.000003, 0.000003, 0.000003, 0.000023, 0.000023],
+                    [0.000003, 0.000003, 0.000008, 0.000023, 0.000023],
+                ],
+                # Class 1
+                [
+                    [0.000023, 0.000023, 0.000008, 0.000003, 0.000003],
+                    [0.000023, 0.000023, 0.000003, 0.000003, 0.000003],
+                    [0.000008, 0.000003, 0.000003, 0.000003, 0.000008],
+                    [0.000003, 0.000003, 0.000003, 0.000008, 0.000008],
+                    [0.000003, 0.000003, 0.000008, 0.000008, 0.000008],
+                ],
+                # Class 2
+                [
+                    [0.999969, 0.999969, 0.999983, 0.999994, 0.999994],
+                    [0.999969, 0.999969, 0.999994, 0.999994, 0.999994],
+                    [0.999983, 0.999994, 0.999994, 0.999994, 0.999983],
+                    [0.999994, 0.999994, 0.999994, 0.999969, 0.999969],
+                    [0.999994, 0.999994, 0.999983, 0.999969, 0.999969],
+                ],
+            ],
+        ],
+    ],
+    [
+        # Case Description
+        "1 batche(s), 3 dimension(s), 2 classe(s), 1 channel(s)",
+        # Parameters
+        [
+            1.0,  # bilateral_weight
+            0.3,  # gaussian_weight
+            5.0,  # bilateral_spatial_sigma
+            0.1,  # bilateral_color_sigma
+            5.0,  # gaussian_spatial_sigma
+            1.0,  # update_factor
+            1,  # compatibility_kernel_range
+            2,  # iterations
+        ],
+        # Input
+        [
+            # Batch 0
+            [
+                # Class 0
+                [
+                    # Slice 0
+                    [
+                        [1.0, 1.0, 0.0, 0.0, 0.0],
+                        [1.0, 1.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 1
+                    [
+                        [1.0, 1.0, 0.0, 0.0, 0.0],
+                        [1.0, 1.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 2
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 3
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 4
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                ],
+                # Class 1
+                [
+                    # Slice 0
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 1
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 2
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 3
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 1.0, 1.0],
+                        [0.0, 0.0, 0.0, 1.0, 1.0],
+                    ],
+                    # Slice 4
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 1.0, 1.0],
+                        [0.0, 0.0, 0.0, 1.0, 1.0],
+                    ],
+                ],
+            ],
+        ],
+        # Features
+        [
+            # Batch 0
+            [
+                # Channel 0
+                [
+                    # Slice 0
+                    [
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 1
+                    [
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 2
+                    [
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.8, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                    ],
+                    # Slice 3
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                    ],
+                    # Slice 4
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                    ],
+                ],
+            ],
+        ],
+        # Expected
+        [
+            # Batch 0
+            [
+                # Class 0
+                [
+                    # Slice 0
+                    [
+                        [1.000000, 1.000000, 1.000000, 0.999808, 0.721122],
+                        [1.000000, 1.000000, 1.000000, 0.999758, 0.666025],
+                        [1.000000, 1.000000, 0.999979, 0.995894, 0.577459],
+                        [0.999787, 0.999739, 0.995725, 0.934170, 0.488704],
+                        [0.691645, 0.641028, 0.560127, 0.481718, 0.431180],
+                    ],
+                    # Slice 1
+                    [
+                        [1.000000, 1.000000, 1.000000, 0.999743, 0.650416],
+                        [1.000000, 1.000000, 0.999999, 0.992747, 0.108034],
+                        [1.000000, 0.999999, 0.998541, 0.402370, 0.007122],
+                        [0.999711, 0.992109, 0.391941, 0.003358, 0.000440],
+                        [0.615523, 0.097120, 0.006599, 0.000427, 0.000365],
+                    ],
+                    # Slice 2
+                    [
+                        [1.000000, 1.000000, 0.999975, 0.995241, 0.543122],
+                        [1.000000, 0.999998, 0.998394, 0.381981, 0.006586],
+                        [0.999973, 0.998313, 0.370238, 0.000596, 0.000034],
+                        [0.994611, 0.361317, 0.000573, 0.000001, 0.000000],
+                        [0.505392, 0.005862, 0.000032, 0.000000, 0.000000],
+                    ],
+                    # Slice 3
+                    [
+                        [0.999692, 0.999639, 0.994364, 0.919713, 0.446683],
+                        [0.999626, 0.990123, 0.347190, 0.002895, 0.000390],
+                        [0.993872, 0.336665, 0.000525, 0.000001, 0.000000],
+                        [0.910704, 0.002676, 0.000001, 0.000000, 0.000000],
+                        [0.413964, 0.000354, 0.000000, 0.000000, 0.000000],
+                    ],
+                    # Slice 4
+                    [
+                        [0.574496, 0.533306, 0.469335, 0.419446, 0.390403],
+                        [0.524093, 0.072180, 0.005087, 0.000354, 0.000318],
+                        [0.449362, 0.004875, 0.000028, 0.000000, 0.000000],
+                        [0.393431, 0.000331, 0.000000, 0.000000, 0.000000],
+                        [0.362417, 0.000295, 0.000000, 0.000000, 0.000000],
+                    ],
+                ],
+                # Class 1
+                [
+                    # Slice 0
+                    [
+                        [0.000000, 0.000000, 0.000000, 0.000192, 0.278878],
+                        [0.000000, 0.000000, 0.000000, 0.000242, 0.333975],
+                        [0.000000, 0.000000, 0.000021, 0.004106, 0.422541],
+                        [0.000213, 0.000261, 0.004275, 0.065830, 0.511296],
+                        [0.308355, 0.358972, 0.439873, 0.518282, 0.568820],
+                    ],
+                    # Slice 1
+                    [
+                        [0.000000, 0.000000, 0.000000, 0.000257, 0.349584],
+                        [0.000000, 0.000000, 0.000001, 0.007253, 0.891966],
+                        [0.000000, 0.000001, 0.001459, 0.597630, 0.992878],
+                        [0.000289, 0.007891, 0.608059, 0.996642, 0.999560],
+                        [0.384477, 0.902880, 0.993401, 0.999573, 0.999635],
+                    ],
+                    # Slice 2
+                    [
+                        [0.000000, 0.000000, 0.000025, 0.004759, 0.456878],
+                        [0.000000, 0.000002, 0.001606, 0.618019, 0.993414],
+                        [0.000027, 0.001687, 0.629762, 0.999404, 0.999966],
+                        [0.005389, 0.638683, 0.999427, 0.999999, 1.000000],
+                        [0.494608, 0.994138, 0.999968, 1.000000, 1.000000],
+                    ],
+                    # Slice 3
+                    [
+                        [0.000308, 0.000361, 0.005636, 0.080287, 0.553317],
+                        [0.000374, 0.009877, 0.652810, 0.997105, 0.999610],
+                        [0.006128, 0.663335, 0.999475, 0.999999, 1.000000],
+                        [0.089296, 0.997324, 0.999999, 1.000000, 1.000000],
+                        [0.586036, 0.999646, 1.000000, 1.000000, 1.000000],
+                    ],
+                    # Slice 4
+                    [
+                        [0.425504, 0.466694, 0.530665, 0.580554, 0.609597],
+                        [0.475907, 0.927820, 0.994913, 0.999646, 0.999682],
+                        [0.550638, 0.995125, 0.999972, 1.000000, 1.000000],
+                        [0.606569, 0.999669, 1.000000, 1.000000, 1.000000],
+                        [0.637583, 0.999705, 1.000000, 1.000000, 1.000000],
+                    ],
+                ],
+            ],
+        ],
+    ],
+]
+
+
+@skip_if_no_cpp_extension
+class CRFTestCaseCpu(unittest.TestCase):
+    @parameterized.expand(TEST_CASES)
+    def test(self, test_case_description, params, input, features, expected):
+
+        # Create input tensors
+        input_tensor = torch.from_numpy(np.array(input)).to(dtype=torch.float, device=torch.device("cpu"))
+        feature_tensor = torch.from_numpy(np.array(features)).to(dtype=torch.float, device=torch.device("cpu"))
+
+        # apply filter
+        crf = CRF(*params)
+        output = crf(input_tensor, feature_tensor).cpu().numpy()
+
+        # Ensure result are as expected
+        np.testing.assert_allclose(output, expected, atol=1e-4)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_crf_cuda.py b/tests/test_crf_cuda.py
new file mode 100644
index 0000000000..6c64807039
--- /dev/null
+++ b/tests/test_crf_cuda.py
@@ -0,0 +1,451 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.networks.blocks import CRF
+from tests.utils import skip_if_no_cpp_extension, skip_if_no_cuda
+
+TEST_CASES = [
+    [
+        # Case Description
+        "2 batche(s), 1 dimension(s), 2 classe(s), 1 channel(s)",
+        # Parameters
+        [
+            1.0,  # bilateral_weight
+            0.3,  # gaussian_weight
+            5.0,  # bilateral_spatial_sigma
+            0.5,  # bilateral_color_sigma
+            5.0,  # gaussian_spatial_sigma
+            1.0,  # update_factor
+            1,  # compatibility_kernel_range
+            5,  # iterations
+        ],
+        # Input
+        [
+            # Batch 0
+            [
+                # Class 0
+                [0.8, 0.9, 0.6, 0.2, 0.3],
+                # Class 1
+                [0.1, 0.3, 0.5, 0.8, 0.7],
+            ],
+            # Batch 1
+            [
+                # Class 0
+                [0.8, 0.9, 0.6, 0.2, 0.3],
+                # Class 1
+                [0.1, 0.3, 0.5, 0.8, 0.7],
+            ],
+        ],
+        # Features
+        [
+            # Batch 0
+            [
+                # Channel 0
+                [1, 1, 1, 0.5, 0],
+            ],
+            # Batch 1
+            [
+                # Channel 0
+                [1, 1, 0.5, 0, 0],
+            ],
+        ],
+        # Expected
+        [
+            # Batch 0
+            [
+                # Class 0
+                [0.965345, 0.961201, 0.920527, 0.772525, 0.711900],
+                # Class 1
+                [0.034655, 0.038799, 0.079473, 0.227475, 0.288100],
+            ],
+            # Batch 1
+            [
+                # Class 0
+                [0.897615, 0.816166, 0.500186, 0.158644, 0.133245],
+                # Class 1
+                [0.102385, 0.183834, 0.499814, 0.841356, 0.866755],
+            ],
+        ],
+    ],
+    [
+        # Case Description
+        "1 batche(s), 2 dimension(s), 3 classe(s), 2 channel(s)",
+        # Parameters
+        [
+            1.0,  # bilateral_weight
+            0.3,  # gaussian_weight
+            5.0,  # bilateral_spatial_sigma
+            0.5,  # bilateral_color_sigma
+            5.0,  # gaussian_spatial_sigma
+            1.0,  # update_factor
+            1,  # compatibility_kernel_range
+            5,  # iterations
+        ],
+        # Input
+        [
+            # Batch 0
+            [
+                # Class 0
+                [
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 1.0, 1.0],
+                    [0.0, 0.0, 0.0, 1.0, 1.0],
+                ],
+                # Class 1
+                [
+                    [1.0, 1.0, 0.0, 0.0, 0.0],
+                    [1.0, 1.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                ],
+                # Class 2
+                [
+                    [0.0, 0.0, 0.0, 0.5, 1.0],
+                    [0.0, 0.0, 0.5, 1.0, 0.5],
+                    [0.0, 0.5, 1.0, 0.5, 0.0],
+                    [0.5, 1.0, 0.5, 0.0, 0.0],
+                    [1.0, 0.5, 0.0, 0.0, 0.0],
+                ],
+            ],
+        ],
+        # Features
+        [
+            # Batch 0
+            [
+                # Channel 0
+                [
+                    [1.0, 1.0, 0.0, 0.0, 0.0],
+                    [1.0, 1.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                ],
+                # Channel 1
+                [
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 0.0, 0.0],
+                    [0.0, 0.0, 0.0, 1.0, 1.0],
+                    [0.0, 0.0, 0.0, 1.0, 1.0],
+                ],
+            ],
+        ],
+        # Expected
+        [
+            # Batch 0
+            [
+                # Class 0
+                [
+                    [0.001529, 0.000798, 0.000323, 0.000093, 0.000053],
+                    [0.001365, 0.000966, 0.000422, 0.000178, 0.000281],
+                    [0.001405, 0.001007, 0.002425, 0.013078, 0.064707],
+                    [0.001239, 0.001263, 0.033857, 0.665830, 0.951172],
+                    [0.001534, 0.004486, 0.263298, 0.973852, 0.999018],
+                ],
+                # Class 1
+                [
+                    [0.230989, 0.025518, 0.000764, 0.000057, 0.000029],
+                    [0.037540, 0.008348, 0.000381, 0.000055, 0.000075],
+                    [0.001987, 0.000665, 0.000363, 0.000499, 0.001170],
+                    [0.000187, 0.000143, 0.000805, 0.001361, 0.000533],
+                    [0.000131, 0.000286, 0.002139, 0.000410, 0.000069],
+                ],
+                # Class 2
+                [
+                    [0.767482, 0.973685, 0.998913, 0.999850, 0.999919],
+                    [0.961095, 0.990687, 0.999197, 0.999768, 0.999644],
+                    [0.996608, 0.998328, 0.997212, 0.986423, 0.934124],
+                    [0.998574, 0.998594, 0.965337, 0.332809, 0.048295],
+                    [0.998334, 0.995228, 0.734563, 0.025738, 0.000912],
+                ],
+            ],
+        ],
+    ],
+    [
+        # Case Description
+        "1 batche(s), 3 dimension(s), 2 classe(s), 1 channel(s)",
+        # Parameters
+        [
+            1.0,  # bilateral_weight
+            0.3,  # gaussian_weight
+            5.0,  # bilateral_spatial_sigma
+            0.1,  # bilateral_color_sigma
+            5.0,  # gaussian_spatial_sigma
+            1.0,  # update_factor
+            1,  # compatibility_kernel_range
+            2,  # iterations
+        ],
+        # Input
+        [
+            # Batch 0
+            [
+                # Class 0
+                [
+                    # Slice 0
+                    [
+                        [1.0, 1.0, 0.0, 0.0, 0.0],
+                        [1.0, 1.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 1
+                    [
+                        [1.0, 1.0, 0.0, 0.0, 0.0],
+                        [1.0, 1.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 2
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 3
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 4
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                ],
+                # Class 1
+                [
+                    # Slice 0
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 1
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 2
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 3
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 1.0, 1.0],
+                        [0.0, 0.0, 0.0, 1.0, 1.0],
+                    ],
+                    # Slice 4
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 1.0, 1.0],
+                        [0.0, 0.0, 0.0, 1.0, 1.0],
+                    ],
+                ],
+            ],
+        ],
+        # Features
+        [
+            # Batch 0
+            [
+                # Channel 0
+                [
+                    # Slice 0
+                    [
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 1
+                    [
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                    ],
+                    # Slice 2
+                    [
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.5, 0.0, 0.0],
+                        [0.5, 0.5, 0.8, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                    ],
+                    # Slice 3
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                    ],
+                    # Slice 4
+                    [
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 0.0, 0.0, 0.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                        [0.0, 0.0, 1.0, 1.0, 1.0],
+                    ],
+                ],
+            ],
+        ],
+        # Expected
+        [
+            # Batch 0
+            [
+                # Class 0
+                [
+                    # Slice 0
+                    [
+                        [1.000000, 1.000000, 1.000000, 0.999884, 0.769625],
+                        [1.000000, 1.000000, 1.000000, 0.999851, 0.714004],
+                        [1.000000, 1.000000, 0.999988, 0.997150, 0.614165],
+                        [0.999862, 0.999832, 0.996976, 0.945058, 0.497088],
+                        [0.720345, 0.672450, 0.590360, 0.490120, 0.416671],
+                    ],
+                    # Slice 1
+                    [
+                        [1.000000, 1.000000, 1.000000, 0.999848, 0.707997],
+                        [1.000000, 1.000000, 1.000000, 0.997064, 0.127893],
+                        [1.000000, 1.000000, 0.999469, 0.591574, 0.007791],
+                        [0.999812, 0.996663, 0.582521, 0.006041, 0.000427],
+                        [0.637809, 0.107586, 0.007432, 0.000437, 0.000333],
+                    ],
+                    # Slice 2
+                    [
+                        [1.000000, 1.000000, 0.999987, 0.996994, 0.600095],
+                        [1.000000, 1.000000, 0.999441, 0.575839, 0.007303],
+                        [0.999986, 0.999411, 0.587268, 0.001117, 0.000033],
+                        [0.996210, 0.550023, 0.001114, 0.000001, 0.000000],
+                        [0.520757, 0.006334, 0.000034, 0.000000, 0.000000],
+                    ],
+                    # Slice 3
+                    [
+                        [0.999834, 0.999807, 0.996617, 0.940887, 0.482334],
+                        [0.999799, 0.996410, 0.553696, 0.005287, 0.000376],
+                        [0.996193, 0.546801, 0.001047, 0.000001, 0.000000],
+                        [0.930515, 0.005142, 0.000001, 0.000000, 0.000000],
+                        [0.430705, 0.000371, 0.000000, 0.000000, 0.000000],
+                    ],
+                    # Slice 4
+                    [
+                        [0.665227, 0.627316, 0.550517, 0.467839, 0.406319],
+                        [0.617408, 0.098325, 0.006247, 0.000359, 0.000278],
+                        [0.524800, 0.006229, 0.000030, 0.000000, 0.000000],
+                        [0.443054, 0.000372, 0.000000, 0.000000, 0.000000],
+                        [0.388126, 0.000305, 0.000000, 0.000000, 0.000000],
+                    ],
+                ],
+                # Class 1
+                [
+                    # Slice 0
+                    [
+                        [0.000000, 0.000000, 0.000000, 0.000116, 0.230375],
+                        [0.000000, 0.000000, 0.000000, 0.000149, 0.285996],
+                        [0.000000, 0.000000, 0.000012, 0.002850, 0.385835],
+                        [0.000138, 0.000168, 0.003024, 0.054942, 0.502912],
+                        [0.279655, 0.327550, 0.409640, 0.509880, 0.583329],
+                    ],
+                    # Slice 1
+                    [
+                        [0.000000, 0.000000, 0.000000, 0.000152, 0.292003],
+                        [0.000000, 0.000000, 0.000000, 0.002936, 0.872107],
+                        [0.000000, 0.000000, 0.000531, 0.408426, 0.992209],
+                        [0.000188, 0.003337, 0.417479, 0.993959, 0.999574],
+                        [0.362191, 0.892414, 0.992568, 0.999564, 0.999667],
+                    ],
+                    # Slice 2
+                    [
+                        [0.000000, 0.000000, 0.000013, 0.003006, 0.399905],
+                        [0.000000, 0.000000, 0.000559, 0.424161, 0.992697],
+                        [0.000014, 0.000589, 0.412732, 0.998884, 0.999967],
+                        [0.003790, 0.449977, 0.998886, 0.999999, 1.000000],
+                        [0.479243, 0.993666, 0.999966, 1.000000, 1.000000],
+                    ],
+                    # Slice 3
+                    [
+                        [0.000166, 0.000193, 0.003383, 0.059113, 0.517666],
+                        [0.000201, 0.003590, 0.446304, 0.994713, 0.999624],
+                        [0.003807, 0.453199, 0.998953, 0.999999, 1.000000],
+                        [0.069485, 0.994858, 0.999999, 1.000000, 1.000000],
+                        [0.569295, 0.999629, 1.000000, 1.000000, 1.000000],
+                    ],
+                    # Slice 4
+                    [
+                        [0.334773, 0.372684, 0.449483, 0.532161, 0.593681],
+                        [0.382592, 0.901675, 0.993753, 0.999641, 0.999722],
+                        [0.475200, 0.993771, 0.999970, 1.000000, 1.000000],
+                        [0.556946, 0.999628, 1.000000, 1.000000, 1.000000],
+                        [0.611874, 0.999695, 1.000000, 1.000000, 1.000000],
+                    ],
+                ],
+            ],
+        ],
+    ],
+]
+
+
+@skip_if_no_cpp_extension
+@skip_if_no_cuda
+class CRFTestCaseCuda(unittest.TestCase):
+    @parameterized.expand(TEST_CASES)
+    def test(self, test_case_description, params, input, features, expected):
+
+        # Create input tensors
+        input_tensor = torch.from_numpy(np.array(input)).to(dtype=torch.float, device=torch.device("cuda"))
+        feature_tensor = torch.from_numpy(np.array(features)).to(dtype=torch.float, device=torch.device("cuda"))
+
+        # apply filter
+        crf = CRF(*params)
+        output = crf(input_tensor, feature_tensor).cpu().numpy()
+
+        # Ensure result are as expected
+        np.testing.assert_allclose(output, expected, atol=5e-2, rtol=5e-2)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_cuimage_reader.py b/tests/test_cuimage_reader.py
new file mode 100644
index 0000000000..2cbfaec113
--- /dev/null
+++ b/tests/test_cuimage_reader.py
@@ -0,0 +1,141 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+from unittest import skipUnless
+
+import numpy as np
+from numpy.testing import assert_array_equal
+from parameterized import parameterized
+
+from monai.apps.utils import download_url
+from monai.data.image_reader import WSIReader
+from monai.utils import optional_import
+
+_, has_cim = optional_import("cucim")
+PILImage, has_pil = optional_import("PIL.Image")
+
+FILE_URL = "http://openslide.cs.cmu.edu/download/openslide-testdata/Generic-TIFF/CMU-1.tiff"
+FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + os.path.basename(FILE_URL))
+
+HEIGHT = 32914
+WIDTH = 46000
+
+TEST_CASE_0 = [FILE_PATH, (3, HEIGHT, WIDTH)]
+
+TEST_CASE_1 = [
+    FILE_PATH,
+    {"location": (HEIGHT // 2, WIDTH // 2), "size": (2, 1), "level": 0},
+    np.array([[[246], [246]], [[246], [246]], [[246], [246]]]),
+]
+
+TEST_CASE_2 = [
+    FILE_PATH,
+    {"location": (0, 0), "size": (2, 1), "level": 2},
+    np.array([[[239], [239]], [[239], [239]], [[239], [239]]]),
+]
+
+TEST_CASE_3 = [
+    FILE_PATH,
+    {
+        "location": (0, 0),
+        "size": (8, 8),
+        "level": 2,
+        "grid_shape": (2, 1),
+        "patch_size": 2,
+    },
+    np.array(
+        [
+            [[[239, 239], [239, 239]], [[239, 239], [239, 239]], [[239, 239], [239, 239]]],
+            [[[242, 242], [242, 243]], [[242, 242], [242, 243]], [[242, 242], [242, 243]]],
+        ]
+    ),
+]
+
+TEST_CASE_4 = [
+    FILE_PATH,
+    {
+        "location": (0, 0),
+        "size": (8, 8),
+        "level": 2,
+        "grid_shape": (2, 1),
+        "patch_size": 1,
+    },
+    np.array([[[[239]], [[239]], [[239]]], [[[243]], [[243]], [[243]]]]),
+]
+
+TEST_CASE_RGB_0 = [
+    np.ones((3, 2, 2), dtype=np.uint8),  # CHW
+]
+
+TEST_CASE_RGB_1 = [
+    np.ones((3, 100, 100), dtype=np.uint8),  # CHW
+]
+
+
+class TestCuCIMReader(unittest.TestCase):
+    @skipUnless(has_cim, "Requires CuCIM")
+    def setUp(self):
+        download_url(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
+
+    @parameterized.expand([TEST_CASE_0])
+    def test_read_whole_image(self, file_path, expected_shape):
+        reader = WSIReader("cuCIM")
+        img_obj = reader.read(file_path)
+        img = reader.get_data(img_obj)[0]
+        self.assertTupleEqual(img.shape, expected_shape)
+
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
+    def test_read_region(self, file_path, patch_info, expected_img):
+        reader = WSIReader("cuCIM")
+        img_obj = reader.read(file_path)
+        img = reader.get_data(img_obj, **patch_info)[0]
+        self.assertTupleEqual(img.shape, expected_img.shape)
+        self.assertIsNone(assert_array_equal(img, expected_img))
+
+    @parameterized.expand([TEST_CASE_3, TEST_CASE_4])
+    def test_read_patches(self, file_path, patch_info, expected_img):
+        reader = WSIReader("cuCIM")
+        img_obj = reader.read(file_path)
+        img = reader.get_data(img_obj, **patch_info)[0]
+        self.assertTupleEqual(img.shape, expected_img.shape)
+        self.assertIsNone(assert_array_equal(img, expected_img))
+
+    @parameterized.expand([TEST_CASE_RGB_0, TEST_CASE_RGB_1])
+    @skipUnless(has_pil, "Requires PIL")
+    def test_read_rgba(self, img_expected):
+        image = {}
+        reader = WSIReader("cuCIM")
+        for mode in ["RGB", "RGBA"]:
+            file_path = self.create_rgba_image(img_expected, "temp_cu_tiff_image", mode=mode)
+            img_obj = reader.read(file_path)
+            image[mode], _ = reader.get_data(img_obj)
+
+        self.assertIsNone(assert_array_equal(image["RGB"], img_expected))
+        self.assertIsNone(assert_array_equal(image["RGBA"], img_expected))
+
+    def create_rgba_image(self, array: np.ndarray, filename_prefix: str, mode: str):
+        file_path = os.path.join(os.path.dirname(__file__), "testing_data", f"{filename_prefix}_{mode}.tiff")
+
+        if mode == "RGBA":
+            array = np.concatenate([array, 255 * np.ones_like(array[0])[np.newaxis]]).astype(np.uint8)
+
+        img_rgb = array.transpose(1, 2, 0)
+
+        image = PILImage.fromarray(img_rgb, mode=mode)
+        image.save(file_path)
+
+        return file_path
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_data_stats.py b/tests/test_data_stats.py
index e7334eb52c..073620ad43 100644
--- a/tests/test_data_stats.py
+++ b/tests/test_data_stats.py
@@ -23,6 +23,7 @@
 TEST_CASE_1 = [
     {
         "prefix": "test data",
+        "data_type": False,
         "data_shape": False,
         "value_range": False,
         "data_value": False,
@@ -36,58 +37,80 @@
 TEST_CASE_2 = [
     {
         "prefix": "test data",
-        "data_shape": True,
+        "data_type": True,
+        "data_shape": False,
         "value_range": False,
         "data_value": False,
         "additional_info": None,
         "logger_handler": None,
     },
     np.array([[0, 1], [1, 2]]),
-    "test data statistics:\nShape: (2, 2)",
+    "test data statistics:\nType: <class 'numpy.ndarray'>",
 ]
 
 TEST_CASE_3 = [
     {
         "prefix": "test data",
+        "data_type": True,
         "data_shape": True,
-        "value_range": True,
+        "value_range": False,
         "data_value": False,
         "additional_info": None,
         "logger_handler": None,
     },
     np.array([[0, 1], [1, 2]]),
-    "test data statistics:\nShape: (2, 2)\nValue range: (0, 2)",
+    "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)",
 ]
 
 TEST_CASE_4 = [
     {
         "prefix": "test data",
+        "data_type": True,
         "data_shape": True,
         "value_range": True,
-        "data_value": True,
+        "data_value": False,
         "additional_info": None,
         "logger_handler": None,
     },
     np.array([[0, 1], [1, 2]]),
-    "test data statistics:\nShape: (2, 2)\nValue range: (0, 2)\nValue: [[0 1]\n [1 2]]",
+    "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)\nValue range: (0, 2)",
 ]
 
 TEST_CASE_5 = [
     {
         "prefix": "test data",
+        "data_type": True,
         "data_shape": True,
         "value_range": True,
         "data_value": True,
-        "additional_info": np.mean,
+        "additional_info": None,
         "logger_handler": None,
     },
     np.array([[0, 1], [1, 2]]),
-    "test data statistics:\nShape: (2, 2)\nValue range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0",
+    "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)\nValue range: (0, 2)\nValue: [[0 1]\n [1 2]]",
 ]
 
 TEST_CASE_6 = [
     {
         "prefix": "test data",
+        "data_type": True,
+        "data_shape": True,
+        "value_range": True,
+        "data_value": True,
+        "additional_info": np.mean,
+        "logger_handler": None,
+    },
+    np.array([[0, 1], [1, 2]]),
+    (
+        "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)\n"
+        "Value range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0"
+    ),
+]
+
+TEST_CASE_7 = [
+    {
+        "prefix": "test data",
+        "data_type": True,
         "data_shape": True,
         "value_range": True,
         "data_value": True,
@@ -96,31 +119,34 @@
     },
     torch.tensor([[0, 1], [1, 2]]),
     (
-        "test data statistics:\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n"
+        "test data statistics:\nType: <class 'torch.Tensor'>\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n"
         "Value: tensor([[0, 1],\n        [1, 2]])\nAdditional info: 1.0"
     ),
 ]
 
-TEST_CASE_7 = [
+TEST_CASE_8 = [
     np.array([[0, 1], [1, 2]]),
-    "test data statistics:\nShape: (2, 2)\nValue range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0\n",
+    "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)\nValue range: (0, 2)\n"
+    "Value: [[0 1]\n [1 2]]\nAdditional info: 1.0\n",
 ]
 
 
 class TestDataStats(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6])
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7])
     def test_value(self, input_param, input_data, expected_print):
         transform = DataStats(**input_param)
         _ = transform(input_data)
         self.assertEqual(transform.output, expected_print)
 
-    @parameterized.expand([TEST_CASE_7])
+    @parameterized.expand([TEST_CASE_8])
     def test_file(self, input_data, expected_print):
         with tempfile.TemporaryDirectory() as tempdir:
             filename = os.path.join(tempdir, "test_data_stats.log")
             handler = logging.FileHandler(filename, mode="w")
+            handler.setLevel(logging.INFO)
             input_param = {
                 "prefix": "test data",
+                "data_type": True,
                 "data_shape": True,
                 "value_range": True,
                 "data_value": True,
@@ -129,8 +155,9 @@ def test_file(self, input_data, expected_print):
             }
             transform = DataStats(**input_param)
             _ = transform(input_data)
-            handler.stream.close()
-            transform._logger.removeHandler(handler)
+            for h in transform._logger.handlers[:]:
+                h.close()
+                transform._logger.removeHandler(h)
             with open(filename, "r") as f:
                 content = f.read()
             self.assertEqual(content, expected_print)
diff --git a/tests/test_data_statsd.py b/tests/test_data_statsd.py
index a5fae3d66d..7ac346b275 100644
--- a/tests/test_data_statsd.py
+++ b/tests/test_data_statsd.py
@@ -24,10 +24,12 @@
     {
         "keys": "img",
         "prefix": "test data",
+        "data_type": False,
         "data_shape": False,
         "value_range": False,
         "data_value": False,
         "additional_info": None,
+        "logger_handler": None,
     },
     {"img": np.array([[0, 1], [1, 2]])},
     "test data statistics:",
@@ -37,101 +39,143 @@
     {
         "keys": "img",
         "prefix": "test data",
-        "data_shape": True,
+        "data_type": True,
+        "data_shape": False,
         "value_range": False,
         "data_value": False,
         "additional_info": None,
+        "logger_handler": None,
     },
     {"img": np.array([[0, 1], [1, 2]])},
-    "test data statistics:\nShape: (2, 2)",
+    "test data statistics:\nType: <class 'numpy.ndarray'>",
 ]
 
 TEST_CASE_3 = [
     {
         "keys": "img",
         "prefix": "test data",
+        "data_type": True,
         "data_shape": True,
-        "value_range": True,
+        "value_range": False,
         "data_value": False,
         "additional_info": None,
+        "logger_handler": None,
     },
     {"img": np.array([[0, 1], [1, 2]])},
-    "test data statistics:\nShape: (2, 2)\nValue range: (0, 2)",
+    "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)",
 ]
 
 TEST_CASE_4 = [
     {
         "keys": "img",
         "prefix": "test data",
+        "data_type": True,
         "data_shape": True,
         "value_range": True,
-        "data_value": True,
+        "data_value": False,
         "additional_info": None,
+        "logger_handler": None,
     },
     {"img": np.array([[0, 1], [1, 2]])},
-    "test data statistics:\nShape: (2, 2)\nValue range: (0, 2)\nValue: [[0 1]\n [1 2]]",
+    "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)\nValue range: (0, 2)",
 ]
 
 TEST_CASE_5 = [
     {
         "keys": "img",
         "prefix": "test data",
+        "data_type": True,
         "data_shape": True,
         "value_range": True,
         "data_value": True,
-        "additional_info": np.mean,
+        "additional_info": None,
+        "logger_handler": None,
     },
     {"img": np.array([[0, 1], [1, 2]])},
-    "test data statistics:\nShape: (2, 2)\nValue range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0",
+    "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)\nValue range: (0, 2)\nValue: [[0 1]\n [1 2]]",
 ]
 
 TEST_CASE_6 = [
     {
         "keys": "img",
         "prefix": "test data",
+        "data_type": True,
+        "data_shape": True,
+        "value_range": True,
+        "data_value": True,
+        "additional_info": np.mean,
+        "logger_handler": None,
+    },
+    {"img": np.array([[0, 1], [1, 2]])},
+    (
+        "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)\n"
+        "Value range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0"
+    ),
+]
+
+TEST_CASE_7 = [
+    {
+        "keys": "img",
+        "prefix": "test data",
+        "data_type": True,
         "data_shape": True,
         "value_range": True,
         "data_value": True,
         "additional_info": lambda x: torch.mean(x.float()),
+        "logger_handler": None,
     },
     {"img": torch.tensor([[0, 1], [1, 2]])},
     (
-        "test data statistics:\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n"
+        "test data statistics:\nType: <class 'torch.Tensor'>\nShape: torch.Size([2, 2])\nValue range: (0, 2)\n"
         "Value: tensor([[0, 1],\n        [1, 2]])\nAdditional info: 1.0"
     ),
 ]
 
-TEST_CASE_7 = [
+TEST_CASE_8 = [
     {
         "keys": ("img", "affine"),
         "prefix": ("image", "affine"),
+        "data_type": True,
         "data_shape": True,
         "value_range": (True, False),
         "data_value": (False, True),
         "additional_info": (np.mean, None),
     },
     {"img": np.array([[0, 1], [1, 2]]), "affine": np.eye(2, 2)},
-    "affine statistics:\nShape: (2, 2)\nValue: [[1. 0.]\n [0. 1.]]",
+    "affine statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)\nValue: [[1. 0.]\n [0. 1.]]",
 ]
 
-TEST_CASE_8 = [
+TEST_CASE_9 = [
     {"img": np.array([[0, 1], [1, 2]])},
-    "test data statistics:\nShape: (2, 2)\nValue range: (0, 2)\nValue: [[0 1]\n [1 2]]\nAdditional info: 1.0\n",
+    "test data statistics:\nType: <class 'numpy.ndarray'>\nShape: (2, 2)\nValue range: (0, 2)\n"
+    "Value: [[0 1]\n [1 2]]\nAdditional info: 1.0\n",
 ]
 
 
 class TestDataStatsd(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7])
+    @parameterized.expand(
+        [
+            TEST_CASE_1,
+            TEST_CASE_2,
+            TEST_CASE_3,
+            TEST_CASE_4,
+            TEST_CASE_5,
+            TEST_CASE_6,
+            TEST_CASE_7,
+            TEST_CASE_8,
+        ]
+    )
     def test_value(self, input_param, input_data, expected_print):
         transform = DataStatsd(**input_param)
         _ = transform(input_data)
         self.assertEqual(transform.printer.output, expected_print)
 
-    @parameterized.expand([TEST_CASE_8])
+    @parameterized.expand([TEST_CASE_9])
     def test_file(self, input_data, expected_print):
         with tempfile.TemporaryDirectory() as tempdir:
             filename = os.path.join(tempdir, "test_stats.log")
             handler = logging.FileHandler(filename, mode="w")
+            handler.setLevel(logging.INFO)
             input_param = {
                 "keys": "img",
                 "prefix": "test data",
@@ -143,8 +187,10 @@ def test_file(self, input_data, expected_print):
             }
             transform = DataStatsd(**input_param)
             _ = transform(input_data)
-            handler.stream.close()
-            transform.printer._logger.removeHandler(handler)
+            for h in transform.printer._logger.handlers[:]:
+                h.close()
+                transform.printer._logger.removeHandler(h)
+            del handler
             with open(filename, "r") as f:
                 content = f.read()
             self.assertEqual(content, expected_print)
diff --git a/tests/test_dataloader.py b/tests/test_dataloader.py
index 072a4a01c0..53e7c89f67 100644
--- a/tests/test_dataloader.py
+++ b/tests/test_dataloader.py
@@ -12,9 +12,27 @@
 import sys
 import unittest
 
-from monai.data import CacheDataset, DataLoader
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.data import CacheDataset, DataLoader, Dataset
 from monai.transforms import Compose, DataStatsd, SimulateDelayd
 
+TEST_CASE_1 = [
+    [
+        {"image": np.asarray([1, 2, 3])},
+        {"image": np.asarray([4, 5])},
+    ]
+]
+
+TEST_CASE_2 = [
+    [
+        {"label": torch.as_tensor([[3], [2]])},
+        {"label": np.asarray([[1], [2]])},
+    ]
+]
+
 
 class TestDataLoader(unittest.TestCase):
     def test_values(self):
@@ -37,6 +55,14 @@ def test_values(self):
             self.assertEqual(d["label"][0], "spleen_label_19.nii.gz")
             self.assertEqual(d["label"][1], "spleen_label_31.nii.gz")
 
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
+    def test_exception(self, datalist):
+        dataset = Dataset(data=datalist, transform=None)
+        dataloader = DataLoader(dataset=dataset, batch_size=2, num_workers=0)
+        with self.assertRaisesRegex((TypeError, RuntimeError), "Collate error on the key"):
+            for _ in dataloader:
+                pass
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
index 2e92b15977..491b777550 100644
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -66,6 +66,8 @@ def test_shape(self, expected_shape):
             dataset = Dataset(data=test_data, transform=LoadImaged(keys=["image", "label", "extra"]))
             data1_simple = dataset[0]
             data2_simple = dataset[1]
+            data3_simple = dataset[-1]
+            data4_simple = dataset[[0, 1]]
 
             self.assertTupleEqual(data1_simple["image"].shape, expected_shape)
             self.assertTupleEqual(data1_simple["label"].shape, expected_shape)
@@ -73,6 +75,17 @@ def test_shape(self, expected_shape):
             self.assertTupleEqual(data2_simple["image"].shape, expected_shape)
             self.assertTupleEqual(data2_simple["label"].shape, expected_shape)
             self.assertTupleEqual(data2_simple["extra"].shape, expected_shape)
+            self.assertTupleEqual(data3_simple["image"].shape, expected_shape)
+            self.assertTupleEqual(data3_simple["label"].shape, expected_shape)
+            self.assertTupleEqual(data3_simple["extra"].shape, expected_shape)
+            self.assertTupleEqual(data4_simple[0]["image"].shape, expected_shape)
+            self.assertTupleEqual(data4_simple[1]["label"].shape, expected_shape)
+            self.assertTupleEqual(data4_simple[-1]["extra"].shape, expected_shape)
+
+            data4_list = dataset[0:1]
+            self.assertEqual(len(data4_list), 1)
+            for d in data4_list:
+                self.assertTupleEqual(d["image"].shape, expected_shape)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_decollate.py b/tests/test_decollate.py
new file mode 100644
index 0000000000..5b78bbbcf6
--- /dev/null
+++ b/tests/test_decollate.py
@@ -0,0 +1,95 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import unittest
+from enum import Enum
+from typing import List, Tuple
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.data import CacheDataset, DataLoader, create_test_image_2d
+from monai.data.utils import decollate_batch
+from monai.transforms import AddChanneld, Compose, LoadImaged, RandFlipd, SpatialPadd, ToTensord
+from monai.transforms.post.dictionary import Decollated
+from monai.transforms.spatial.dictionary import RandAffined, RandRotate90d
+from monai.utils import optional_import, set_determinism
+from monai.utils.enums import InverseKeys
+from tests.utils import make_nifti_image
+
+_, has_nib = optional_import("nibabel")
+
+KEYS = ["image"]
+
+TESTS: List[Tuple] = []
+TESTS.append((SpatialPadd(KEYS, 150), RandFlipd(KEYS, prob=1.0, spatial_axis=1)))
+TESTS.append((RandRotate90d(KEYS, prob=0.0, max_k=1),))
+TESTS.append((RandAffined(KEYS, prob=0.0, translate_range=10),))
+
+
+class TestDeCollate(unittest.TestCase):
+    def setUp(self) -> None:
+        set_determinism(seed=0)
+
+        im = create_test_image_2d(100, 101)[0]
+        self.data = [{"image": make_nifti_image(im) if has_nib else im} for _ in range(6)]
+
+    def tearDown(self) -> None:
+        set_determinism(None)
+
+    def check_match(self, in1, in2):
+        if isinstance(in1, dict):
+            self.assertTrue(isinstance(in2, dict))
+            for (k1, v1), (k2, v2) in zip(in1.items(), in2.items()):
+                if isinstance(k1, Enum) and isinstance(k2, Enum):
+                    k1, k2 = k1.value, k2.value
+                self.check_match(k1, k2)
+                # Transform ids won't match for windows with multiprocessing, so don't check values
+                if k1 == InverseKeys.ID and sys.platform in ["darwin", "win32"]:
+                    continue
+                self.check_match(v1, v2)
+        elif isinstance(in1, (list, tuple)):
+            for l1, l2 in zip(in1, in2):
+                self.check_match(l1, l2)
+        elif isinstance(in1, (str, int)):
+            self.assertEqual(in1, in2)
+        elif isinstance(in1, (torch.Tensor, np.ndarray)):
+            np.testing.assert_array_equal(in1, in2)
+        else:
+            raise RuntimeError(f"Not sure how to compare types. type(in1): {type(in1)}, type(in2): {type(in2)}")
+
+    @parameterized.expand(TESTS)
+    def test_decollation(self, *transforms):
+
+        batch_size = 2
+        num_workers = 2
+
+        t_compose = Compose([AddChanneld(KEYS), Compose(transforms), ToTensord(KEYS)])
+        # If nibabel present, read from disk
+        if has_nib:
+            t_compose = Compose([LoadImaged("image"), t_compose])
+
+        dataset = CacheDataset(self.data, t_compose, progress=False)
+        loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
+
+        for b, batch_data in enumerate(loader):
+            decollated_1 = decollate_batch(batch_data)
+            decollated_2 = Decollated()(batch_data)
+
+            for decollated in [decollated_1, decollated_2]:
+                for i, d in enumerate(decollated):
+                    self.check_match(dataset[b * batch_size + i], d)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_deepgrow_dataset.py b/tests/test_deepgrow_dataset.py
index e871c328a6..147d8e7099 100644
--- a/tests/test_deepgrow_dataset.py
+++ b/tests/test_deepgrow_dataset.py
@@ -10,47 +10,96 @@
 # limitations under the License.
 
 import os
+import shutil
 import tempfile
 import unittest
 
 import nibabel as nib
 import numpy as np
+from parameterized import parameterized
 
 from monai.apps.deepgrow.dataset import create_dataset
+from monai.utils import set_determinism
+
+TEST_CASE_1 = [{"dimension": 2, "pixdim": (1, 1)}, {"length": 3}, 9, 1]
+
+TEST_CASE_2 = [{"dimension": 2, "pixdim": (1, 1), "limit": 1}, {"length": 3}, 3, 1]
+
+TEST_CASE_3 = [{"dimension": 2, "pixdim": (1, 1)}, {"length": 1}, 3, 1]
+
+TEST_CASE_4 = [{"dimension": 3, "pixdim": (1, 1, 1)}, {"length": 1}, 1, 1]
+
+TEST_CASE_5 = [{"dimension": 3, "pixdim": (1, 1, 1)}, {"length": 1, "image_channel": 4}, 1, 1]
+
+TEST_CASE_6 = [{"dimension": 2, "pixdim": (1, 1)}, {"length": 1, "image_channel": 4}, 3, 1]
+
+TEST_CASE_7 = [
+    {"dimension": 2, "pixdim": (1, 1), "label_key": None},
+    {"length": 1, "image_channel": 4, "with_label": False},
+    40,
+    None,
+]
+
+TEST_CASE_8 = [
+    {"dimension": 3, "pixdim": (1, 1, 1), "label_key": None},
+    {"length": 1, "image_channel": 4, "with_label": False},
+    1,
+    None,
+]
 
 
 class TestCreateDataset(unittest.TestCase):
-    def _create_data(self, tempdir):
+    def setUp(self):
+        set_determinism(1)
+        self.tempdir = tempfile.mkdtemp()
+
+    def _create_data(self, length=1, image_channel=1, with_label=True):
         affine = np.eye(4)
-        image = np.random.randint(0, 2, size=(128, 128, 40))
-        image_file = os.path.join(tempdir, "image1.nii.gz")
-        nib.save(nib.Nifti1Image(image, affine), image_file)
-
-        label = np.zeros((128, 128, 40))
-        label[0][1][0] = 1
-        label[0][1][1] = 1
-        label[0][0][2] = 1
-        label[0][1][2] = 1
-        label_file = os.path.join(tempdir, "label1.nii.gz")
-        nib.save(nib.Nifti1Image(label, affine), label_file)
-
-        return [{"image": image_file, "label": label_file}]
-
-    def test_create_dataset_2d(self):
-        with tempfile.TemporaryDirectory() as tempdir:
-            datalist = self._create_data(tempdir)
-            output_dir = os.path.join(tempdir, "2d")
-            deepgrow_datalist = create_dataset(datalist=datalist, output_dir=output_dir, dimension=2, pixdim=(1, 1))
-            self.assertEqual(len(deepgrow_datalist), 3)
-            self.assertEqual(deepgrow_datalist[0]["region"], 1)
-
-    def test_create_dataset_3d(self):
-        with tempfile.TemporaryDirectory() as tempdir:
-            datalist = self._create_data(tempdir)
-            output_dir = os.path.join(tempdir, "3d")
-            deepgrow_datalist = create_dataset(datalist=datalist, output_dir=output_dir, dimension=3, pixdim=(1, 1, 1))
-            self.assertEqual(len(deepgrow_datalist), 1)
-            self.assertEqual(deepgrow_datalist[0]["region"], 1)
+        datalist = []
+        for i in range(length):
+            if image_channel == 1:
+                image = np.random.randint(0, 2, size=(128, 128, 40))
+            else:
+                image = np.random.randint(0, 2, size=(128, 128, 40, image_channel))
+            image_file = os.path.join(self.tempdir, f"image{i}.nii.gz")
+            nib.save(nib.Nifti1Image(image, affine), image_file)
+
+            if with_label:
+                # 3 slices has label
+                label = np.zeros((128, 128, 40))
+                label[0][1][0] = 1
+                label[0][1][1] = 1
+                label[0][0][2] = 1
+                label[0][1][2] = 1
+                label_file = os.path.join(self.tempdir, f"label{i}.nii.gz")
+                nib.save(nib.Nifti1Image(label, affine), label_file)
+                datalist.append({"image": image_file, "label": label_file})
+            else:
+                datalist.append({"image": image_file})
+
+        return datalist
+
+    @parameterized.expand(
+        [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8]
+    )
+    def test_create_dataset(self, args, data_args, expected_length, expected_region):
+        datalist = self._create_data(**data_args)
+        deepgrow_datalist = create_dataset(datalist=datalist, output_dir=self.tempdir, **args)
+        self.assertEqual(len(deepgrow_datalist), expected_length)
+        if expected_region is not None:
+            self.assertEqual(deepgrow_datalist[0]["region"], expected_region)
+
+    def test_invalid_dim(self):
+        with self.assertRaises(ValueError):
+            create_dataset(datalist=self._create_data(), output_dir=self.tempdir, dimension=4, pixdim=(1, 1, 1, 1))
+
+    def test_empty_datalist(self):
+        with self.assertRaises(ValueError):
+            create_dataset(datalist=[], output_dir=self.tempdir, dimension=3, pixdim=(1, 1, 1))
+
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+        set_determinism(None)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_deepgrow_transforms.py b/tests/test_deepgrow_transforms.py
index f534813832..2d57ed9325 100644
--- a/tests/test_deepgrow_transforms.py
+++ b/tests/test_deepgrow_transforms.py
@@ -15,12 +15,17 @@
 from parameterized import parameterized
 
 from monai.apps.deepgrow.transforms import (
+    AddGuidanceFromPointsd,
     AddGuidanceSignald,
     AddInitialSeedPointd,
     AddRandomGuidanced,
+    Fetch2DSliced,
     FindAllValidSlicesd,
     FindDiscrepancyRegionsd,
+    ResizeGuidanced,
+    RestoreLabeld,
     SpatialCropForegroundd,
+    SpatialCropGuidanced,
 )
 
 IMAGE = np.array([[[[1, 0, 2, 0, 1], [0, 1, 2, 1, 0], [2, 2, 3, 2, 2], [0, 1, 2, 1, 0], [1, 0, 2, 0, 1]]]])
@@ -76,6 +81,76 @@
     "probability": [1.0],
 }
 
+DATA_5 = {
+    "image": np.arange(25).reshape((1, 5, 5)),
+    "image_meta_dict": {"spatial_shape": [5, 5, 1]},
+    "foreground": [[2, 2, 0]],
+    "background": [],
+}
+
+DATA_6 = {
+    "image": np.arange(25).reshape((1, 5, 5)),
+    "image_meta_dict": {"spatial_shape": [5, 2, 1]},
+    "foreground": [[2, 1, 0]],
+    "background": [[1, 0, 0]],
+}
+
+DATA_7 = {
+    "image": np.arange(500).reshape((5, 10, 10)),
+    "image_meta_dict": {"spatial_shape": [20, 20, 10]},
+    "foreground": [[10, 14, 6], [10, 14, 8]],
+    "background": [[10, 16, 8]],
+    "slice": 6,
+}
+
+DATA_8 = {
+    "image": np.arange(500).reshape((1, 5, 10, 10)),
+    "image_meta_dict": {"spatial_shape": [20, 20, 10]},
+    "guidance": [[[3, 5, 7], [4, 5, 7]], [[4, 5, 8]]],
+}
+
+DATA_9 = {
+    "image": np.arange(1000).reshape((1, 5, 10, 20)),
+    "image_meta_dict": {"foreground_cropped_shape": (1, 10, 20, 40)},
+    "guidance": [[[6, 10, 14], [8, 10, 14]], [[8, 10, 16]]],
+}
+
+DATA_10 = {
+    "image": np.arange(9).reshape((1, 1, 3, 3)),
+    "image_meta_dict": {
+        "spatial_shape": [3, 3, 1],
+        "foreground_start_coord": np.array([0, 0, 0]),
+        "foreground_end_coord": np.array([1, 3, 3]),
+        "foreground_original_shape": (1, 1, 3, 3),
+        "foreground_cropped_shape": (1, 1, 3, 3),
+        "original_affine": np.array(
+            [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]]
+        ),
+    },
+    "pred": np.array([[[[1, 2, 3], [4, 5, 6], [7, 8, 9]]]]),
+}
+
+DATA_11 = {
+    "image": np.arange(500).reshape((1, 5, 10, 10)),
+    "image_meta_dict": {
+        "spatial_shape": [20, 20, 10],
+        "foreground_start_coord": np.array([2, 2, 2]),
+        "foreground_end_coord": np.array([4, 4, 4]),
+        "foreground_original_shape": (1, 5, 10, 10),
+        "foreground_cropped_shape": (1, 2, 2, 2),
+        "original_affine": np.array(
+            [[[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 0.0]]]
+        ),
+    },
+    "pred": np.array([[[[1, 2], [3, 4]], [[5, 6], [7, 8]]]]),
+}
+
+DATA_12 = {
+    "image": np.arange(27).reshape(3, 3, 3),
+    "image_meta_dict": {},
+    "guidance": [[0, 0, 0], [0, 1, 1], 1],
+}
+
 FIND_SLICE_TEST_CASE_1 = [
     {"label": "label", "sids": "sids"},
     DATA_1,
@@ -159,6 +234,118 @@
     np.array([[[[1, 0, 2, 2], [1, 0, 1, 3]], [[-1, -1, -1, -1], [-1, -1, -1, -1]]]]),
 ]
 
+ADD_GUIDANCE_FROM_POINTS_TEST_CASE_1 = [
+    {"ref_image": "image", "dimensions": 3, "guidance": "guidance", "depth_first": True},
+    DATA_5,
+    [[0, 2, 2]],
+    [],
+]
+
+ADD_GUIDANCE_FROM_POINTS_TEST_CASE_2 = [
+    {"ref_image": "image", "dimensions": 3, "guidance": "guidance", "depth_first": True},
+    DATA_6,
+    [[0, 2, 2]],
+    [[0, 1, 0]],
+]
+
+ADD_GUIDANCE_FROM_POINTS_TEST_CASE_3 = [
+    {"ref_image": "image", "dimensions": 3, "guidance": "guidance", "depth_first": True},
+    DATA_7,
+    [[3, 5, 7], [4, 5, 7]],
+    [[4, 5, 8]],
+]
+
+ADD_GUIDANCE_FROM_POINTS_TEST_CASE_4 = [
+    {"ref_image": "image", "dimensions": 2, "guidance": "guidance", "depth_first": True},
+    DATA_6,
+    [[2, 2]],
+    [[1, 0]],
+]
+
+ADD_GUIDANCE_FROM_POINTS_TEST_CASE_5 = [
+    {"ref_image": "image", "dimensions": 2, "guidance": "guidance", "depth_first": True, "slice_key": "slice"},
+    DATA_7,
+    [[5, 7]],
+    [],
+]
+
+ADD_GUIDANCE_FROM_POINTS_TEST_CASE_6 = [
+    {"ref_image": "image", "dimensions": 2, "guidance": "guidance", "depth_first": True},
+    DATA_5,
+    [[2, 2]],
+    [],
+]
+
+SPATIAL_CROP_GUIDANCE_TEST_CASE_1 = [
+    {"keys": ["image"], "guidance": "guidance", "spatial_size": [1, 4, 4], "margin": 0},
+    DATA_8,
+    np.array([[[[357, 358]], [[457, 458]]]]),
+]
+
+SPATIAL_CROP_GUIDANCE_TEST_CASE_2 = [
+    {"keys": ["image"], "guidance": "guidance", "spatial_size": [2, 2], "margin": 1},
+    DATA_8,
+    np.array(
+        [
+            [
+                [[246, 247, 248, 249], [256, 257, 258, 259], [266, 267, 268, 269]],
+                [[346, 347, 348, 349], [356, 357, 358, 359], [366, 367, 368, 369]],
+                [[446, 447, 448, 449], [456, 457, 458, 459], [466, 467, 468, 469]],
+            ]
+        ]
+    ),
+]
+
+SPATIAL_CROP_GUIDANCE_TEST_CASE_3 = [
+    {"keys": ["image"], "guidance": "guidance", "spatial_size": [3, 3], "margin": 0},
+    DATA_8,
+    np.array(
+        [
+            [
+                [[47, 48, 49], [57, 58, 59], [67, 68, 69]],
+                [[147, 148, 149], [157, 158, 159], [167, 168, 169]],
+                [[247, 248, 249], [257, 258, 259], [267, 268, 269]],
+                [[347, 348, 349], [357, 358, 359], [367, 368, 369]],
+                [[447, 448, 449], [457, 458, 459], [467, 468, 469]],
+            ]
+        ]
+    ),
+]
+
+RESIZE_GUIDANCE_TEST_CASE_1 = [
+    {"ref_image": "image", "guidance": "guidance"},
+    DATA_9,
+    [[[3, 5, 7], [4, 5, 7]], [[4, 5, 8]]],
+]
+
+RESTORE_LABEL_TEST_CASE_1 = [
+    {"keys": ["pred"], "ref_image": "image", "mode": "nearest"},
+    DATA_10,
+    np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]]]),
+]
+
+RESULT = np.zeros((10, 20, 20))
+RESULT[4:8, 4:8, 4:8] = np.array(
+    [
+        [[1.0, 1.0, 2.0, 2.0], [1.0, 1.0, 2.0, 2.0], [3.0, 3.0, 4.0, 4.0], [3.0, 3.0, 4.0, 4.0]],
+        [[1.0, 1.0, 2.0, 2.0], [1.0, 1.0, 2.0, 2.0], [3.0, 3.0, 4.0, 4.0], [3.0, 3.0, 4.0, 4.0]],
+        [[5.0, 5.0, 6.0, 6.0], [5.0, 5.0, 6.0, 6.0], [7.0, 7.0, 8.0, 8.0], [7.0, 7.0, 8.0, 8.0]],
+        [[5.0, 5.0, 6.0, 6.0], [5.0, 5.0, 6.0, 6.0], [7.0, 7.0, 8.0, 8.0], [7.0, 7.0, 8.0, 8.0]],
+    ],
+)
+
+RESTORE_LABEL_TEST_CASE_2 = [
+    {"keys": ["pred"], "ref_image": "image", "mode": "nearest"},
+    DATA_11,
+    RESULT,
+]
+
+FETCH_2D_SLICE_TEST_CASE_1 = [
+    {"keys": ["image"], "guidance": "guidance"},
+    DATA_12,
+    np.array([[9, 10, 11], [12, 13, 14], [15, 16, 17]]),
+]
+
 
 class TestFindAllValidSlicesd(unittest.TestCase):
     @parameterized.expand([FIND_SLICE_TEST_CASE_1, FIND_SLICE_TEST_CASE_2])
@@ -220,5 +407,52 @@ def test_correct_results(self, arguments, input_data, expected_result):
         np.testing.assert_allclose(result[arguments["guidance"]], expected_result, rtol=1e-5)
 
 
+class TestAddGuidanceFromPointsd(unittest.TestCase):
+    @parameterized.expand(
+        [
+            ADD_GUIDANCE_FROM_POINTS_TEST_CASE_1,
+            ADD_GUIDANCE_FROM_POINTS_TEST_CASE_2,
+            ADD_GUIDANCE_FROM_POINTS_TEST_CASE_3,
+            ADD_GUIDANCE_FROM_POINTS_TEST_CASE_4,
+            ADD_GUIDANCE_FROM_POINTS_TEST_CASE_5,
+            ADD_GUIDANCE_FROM_POINTS_TEST_CASE_6,
+        ]
+    )
+    def test_correct_results(self, arguments, input_data, expected_pos, expected_neg):
+        result = AddGuidanceFromPointsd(**arguments)(input_data)
+        self.assertEqual(result[arguments["guidance"]][0], expected_pos)
+        self.assertEqual(result[arguments["guidance"]][1], expected_neg)
+
+
+class TestSpatialCropGuidanced(unittest.TestCase):
+    @parameterized.expand(
+        [SPATIAL_CROP_GUIDANCE_TEST_CASE_1, SPATIAL_CROP_GUIDANCE_TEST_CASE_2, SPATIAL_CROP_GUIDANCE_TEST_CASE_3]
+    )
+    def test_correct_results(self, arguments, input_data, expected_result):
+        result = SpatialCropGuidanced(**arguments)(input_data)
+        np.testing.assert_allclose(result["image"], expected_result)
+
+
+class TestResizeGuidanced(unittest.TestCase):
+    @parameterized.expand([RESIZE_GUIDANCE_TEST_CASE_1])
+    def test_correct_results(self, arguments, input_data, expected_result):
+        result = ResizeGuidanced(**arguments)(input_data)
+        self.assertEqual(result[arguments["guidance"]], expected_result)
+
+
+class TestRestoreLabeld(unittest.TestCase):
+    @parameterized.expand([RESTORE_LABEL_TEST_CASE_1, RESTORE_LABEL_TEST_CASE_2])
+    def test_correct_results(self, arguments, input_data, expected_result):
+        result = RestoreLabeld(**arguments)(input_data)
+        np.testing.assert_allclose(result["pred"], expected_result)
+
+
+class TestFetch2DSliced(unittest.TestCase):
+    @parameterized.expand([FETCH_2D_SLICE_TEST_CASE_1])
+    def test_correct_results(self, arguments, input_data, expected_result):
+        result = Fetch2DSliced(**arguments)(input_data)
+        np.testing.assert_allclose(result["image"], expected_result)
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_densenet.py b/tests/test_densenet.py
index 876689314a..c934841598 100644
--- a/tests/test_densenet.py
+++ b/tests/test_densenet.py
@@ -10,13 +10,24 @@
 # limitations under the License.
 
 import unittest
+from typing import TYPE_CHECKING
+from unittest import skipUnless
 
 import torch
 from parameterized import parameterized
 
 from monai.networks import eval_mode
-from monai.networks.nets import densenet121, densenet169, densenet201, densenet264
-from tests.utils import skip_if_quick, test_pretrained_networks, test_script_save
+from monai.networks.nets import DenseNet121, DenseNet169, DenseNet201, DenseNet264
+from monai.utils import optional_import
+from tests.utils import skip_if_quick, test_script_save
+
+if TYPE_CHECKING:
+    import torchvision
+
+    has_torchvision = True
+else:
+    torchvision, has_torchvision = optional_import("torchvision")
+
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
@@ -40,37 +51,55 @@
 
 TEST_CASES = []
 for case in [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3]:
-    for model in [densenet121, densenet169, densenet201, densenet264]:
+    for model in [DenseNet121, DenseNet169, DenseNet201, DenseNet264]:
         TEST_CASES.append([model, *case])
 
 
-TEST_SCRIPT_CASES = [[model, *TEST_CASE_1] for model in [densenet121, densenet169, densenet201, densenet264]]
+TEST_SCRIPT_CASES = [[model, *TEST_CASE_1] for model in [DenseNet121, DenseNet169, DenseNet201, DenseNet264]]
 
 
 TEST_PRETRAINED_2D_CASE_1 = [  # 4-channel 2D, batch 2
-    densenet121,
+    DenseNet121,
     {"pretrained": True, "progress": True, "spatial_dims": 2, "in_channels": 2, "out_channels": 3},
-    (2, 2, 32, 64),
-    (2, 3),
+    (1, 2, 32, 64),
+    (1, 3),
 ]
 
 TEST_PRETRAINED_2D_CASE_2 = [  # 4-channel 2D, batch 2
-    densenet121,
-    {"pretrained": True, "progress": False, "spatial_dims": 2, "in_channels": 2, "out_channels": 3},
-    (2, 2, 32, 64),
-    (2, 3),
+    DenseNet121,
+    {"pretrained": True, "progress": False, "spatial_dims": 2, "in_channels": 2, "out_channels": 1},
+    (1, 2, 32, 64),
+    (1, 1),
+]
+
+TEST_PRETRAINED_2D_CASE_3 = [
+    DenseNet121,
+    {"pretrained": True, "progress": False, "spatial_dims": 2, "in_channels": 3, "out_channels": 1},
+    (1, 3, 32, 32),
 ]
 
 
 class TestPretrainedDENSENET(unittest.TestCase):
     @parameterized.expand([TEST_PRETRAINED_2D_CASE_1, TEST_PRETRAINED_2D_CASE_2])
     @skip_if_quick
-    def test_121_3d_shape_pretrain(self, model, input_param, input_shape, expected_shape):
-        net = test_pretrained_networks(model, input_param, device)
+    def test_121_2d_shape_pretrain(self, model, input_param, input_shape, expected_shape):
+        net = model(**input_param).to(device)
         with eval_mode(net):
             result = net.forward(torch.randn(input_shape).to(device))
             self.assertEqual(result.shape, expected_shape)
 
+    @parameterized.expand([TEST_PRETRAINED_2D_CASE_3])
+    @skipUnless(has_torchvision, "Requires `torchvision` package.")
+    def test_pretrain_consistency(self, model, input_param, input_shape):
+        example = torch.randn(input_shape).to(device)
+        net = model(**input_param).to(device)
+        with eval_mode(net):
+            result = net.features.forward(example)
+        torchvision_net = torchvision.models.densenet121(pretrained=True).to(device)
+        with eval_mode(torchvision_net):
+            expected_result = torchvision_net.features.forward(example)
+        self.assertTrue(torch.all(result == expected_result))
+
 
 class TestDENSENET(unittest.TestCase):
     @parameterized.expand(TEST_CASES)
diff --git a/tests/test_detect_envelope.py b/tests/test_detect_envelope.py
index 47b3a66305..ded0290de2 100644
--- a/tests/test_detect_envelope.py
+++ b/tests/test_detect_envelope.py
@@ -156,7 +156,7 @@ def test_no_fft_module_error(self):
 @SkipIfAtLeastPyTorchVersion((1, 7))
 class TestDetectEnvelopeInvalidPyTorch(unittest.TestCase):
     def test_invalid_pytorch_error(self):
-        with self.assertRaisesRegexp(InvalidPyTorchVersionError, "version"):
+        with self.assertRaisesRegex(InvalidPyTorchVersionError, "version"):
             DetectEnvelope()
 
 
diff --git a/tests/test_dice_ce_loss.py b/tests/test_dice_ce_loss.py
index 443d9a9baf..3423e1425b 100644
--- a/tests/test_dice_ce_loss.py
+++ b/tests/test_dice_ce_loss.py
@@ -16,6 +16,7 @@
 from parameterized import parameterized
 
 from monai.losses import DiceCELoss
+from tests.utils import SkipIfBeforePyTorchVersion, test_script_save
 
 TEST_CASES = [
     [  # shape: (2, 2, 3), (2, 1, 3)
@@ -42,6 +43,20 @@
         },
         0.2088,
     ],
+    [  # shape: (2, 2, 3), (2, 1, 3) lambda_dice: 1.0, lambda_ce: 2.0
+        {
+            "include_background": False,
+            "to_onehot_y": True,
+            "ce_weight": torch.tensor([1.0, 1.0]),
+            "lambda_dice": 1.0,
+            "lambda_ce": 2.0,
+        },
+        {
+            "input": torch.tensor([[[100.0, 100.0, 0.0], [0.0, 0.0, 1.0]], [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]]]),
+            "target": torch.tensor([[[0.0, 0.0, 1.0]], [[0.0, 1.0, 0.0]]]),
+        },
+        0.4176,
+    ],
     [  # shape: (2, 2, 3), (2, 1, 3), do not include class 0
         {"include_background": False, "to_onehot_y": True, "ce_weight": torch.tensor([0.0, 1.0])},
         {
@@ -64,6 +79,12 @@ def test_ill_shape(self):
         with self.assertRaisesRegex(ValueError, ""):
             loss(torch.ones((1, 2, 3)), torch.ones((1, 1, 2, 3)))
 
+    @SkipIfBeforePyTorchVersion((1, 7, 0))
+    def test_script(self):
+        loss = DiceCELoss()
+        test_input = torch.ones(2, 1, 8, 8)
+        test_script_save(loss, test_input, test_input)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_dice_focal_loss.py b/tests/test_dice_focal_loss.py
new file mode 100644
index 0000000000..4bab68131c
--- /dev/null
+++ b/tests/test_dice_focal_loss.py
@@ -0,0 +1,80 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+
+from monai.losses import DiceFocalLoss, DiceLoss, FocalLoss
+from tests.utils import SkipIfBeforePyTorchVersion, test_script_save
+
+
+class TestDiceFocalLoss(unittest.TestCase):
+    def test_result_onehot_target_include_bg(self):
+        size = [3, 3, 5, 5]
+        label = torch.randint(low=0, high=2, size=size)
+        pred = torch.randn(size)
+        for reduction in ["sum", "mean", "none"]:
+            common_params = {
+                "include_background": True,
+                "to_onehot_y": False,
+                "reduction": reduction,
+            }
+            for focal_weight in [None, torch.tensor([1.0, 1.0, 2.0]), (3, 2.0, 1)]:
+                for lambda_focal in [0.5, 1.0, 1.5]:
+                    dice_focal = DiceFocalLoss(
+                        focal_weight=focal_weight, gamma=1.0, lambda_focal=lambda_focal, **common_params
+                    )
+                    dice = DiceLoss(**common_params)
+                    focal = FocalLoss(weight=focal_weight, gamma=1.0, **common_params)
+                    result = dice_focal(pred, label)
+                    expected_val = dice(pred, label) + lambda_focal * focal(pred, label)
+                    np.testing.assert_allclose(result, expected_val)
+
+    def test_result_no_onehot_no_bg(self):
+        size = [3, 3, 5, 5]
+        label = torch.randint(low=0, high=2, size=size)
+        label = torch.argmax(label, dim=1, keepdim=True)
+        pred = torch.randn(size)
+        for reduction in ["sum", "mean", "none"]:
+            common_params = {
+                "include_background": False,
+                "to_onehot_y": True,
+                "reduction": reduction,
+            }
+            for focal_weight in [2.0, torch.tensor([1.0, 2.0]), (2.0, 1)]:
+                for lambda_focal in [0.5, 1.0, 1.5]:
+                    dice_focal = DiceFocalLoss(focal_weight=focal_weight, lambda_focal=lambda_focal, **common_params)
+                    dice = DiceLoss(**common_params)
+                    focal = FocalLoss(weight=focal_weight, **common_params)
+                    result = dice_focal(pred, label)
+                    expected_val = dice(pred, label) + lambda_focal * focal(pred, label)
+                    np.testing.assert_allclose(result, expected_val)
+
+    def test_ill_shape(self):
+        loss = DiceFocalLoss()
+        with self.assertRaisesRegex(ValueError, ""):
+            loss(torch.ones((1, 2, 3)), torch.ones((1, 1, 2, 3)))
+
+    def test_ill_lambda(self):
+        with self.assertRaisesRegex(ValueError, ""):
+            loss = DiceFocalLoss(lambda_dice=-1.0)
+
+    @SkipIfBeforePyTorchVersion((1, 7, 0))
+    def test_script(self):
+        loss = DiceFocalLoss()
+        test_input = torch.ones(2, 1, 8, 8)
+        test_script_save(loss, test_input, test_input)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_dice_loss.py b/tests/test_dice_loss.py
index aa4a7cbc34..ef0a51eb15 100644
--- a/tests/test_dice_loss.py
+++ b/tests/test_dice_loss.py
@@ -16,6 +16,7 @@
 from parameterized import parameterized
 
 from monai.losses import DiceLoss
+from tests.utils import SkipIfBeforePyTorchVersion, test_script_save
 
 TEST_CASES = [
     [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
@@ -195,6 +196,12 @@ def test_input_warnings(self):
             loss = DiceLoss(to_onehot_y=True)
             loss.forward(chn_input, chn_target)
 
+    @SkipIfBeforePyTorchVersion((1, 7, 0))
+    def test_script(self):
+        loss = DiceLoss()
+        test_input = torch.ones(2, 1, 8, 8)
+        test_script_save(loss, test_input, test_input)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_distcall.py b/tests/test_distcall.py
new file mode 100644
index 0000000000..1830a85654
--- /dev/null
+++ b/tests/test_distcall.py
@@ -0,0 +1,29 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from tests.utils import DistCall, DistTestCase
+
+
+class DistributedCallTest(DistTestCase):
+    def test_constructor(self):
+        with self.assertRaises(ValueError):
+            DistCall(nnodes=1, nproc_per_node=0)
+        with self.assertRaises(ValueError):
+            DistCall(nnodes=0, nproc_per_node=0)
+        with self.assertRaises(ValueError):
+            DistCall(nnodes=0, nproc_per_node=1)
+        _ = DistCall(nnodes=1, nproc_per_node=1)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_distributed_sampler.py b/tests/test_distributed_sampler.py
index d0054885eb..0a439874bd 100644
--- a/tests/test_distributed_sampler.py
+++ b/tests/test_distributed_sampler.py
@@ -24,6 +24,7 @@ def test_even(self):
         data = [1, 2, 3, 4, 5]
         sampler = DistributedSampler(dataset=data, shuffle=False)
         samples = np.array([data[i] for i in list(sampler)])
+        self.assertEqual(dist.get_rank(), sampler.rank)
         if dist.get_rank() == 0:
             np.testing.assert_allclose(samples, np.array([1, 3, 5]))
 
@@ -35,6 +36,7 @@ def test_uneven(self):
         data = [1, 2, 3, 4, 5]
         sampler = DistributedSampler(dataset=data, shuffle=False, even_divisible=False)
         samples = np.array([data[i] for i in list(sampler)])
+        self.assertEqual(dist.get_rank(), sampler.rank)
         if dist.get_rank() == 0:
             np.testing.assert_allclose(samples, np.array([1, 3, 5]))
 
diff --git a/tests/test_distributed_weighted_random_sampler.py b/tests/test_distributed_weighted_random_sampler.py
new file mode 100644
index 0000000000..b8e088fdcf
--- /dev/null
+++ b/tests/test_distributed_weighted_random_sampler.py
@@ -0,0 +1,62 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+import torch.distributed as dist
+
+from monai.data import DistributedWeightedRandomSampler
+from tests.utils import DistCall, DistTestCase
+
+
+class DistributedWeightedRandomSamplerTest(DistTestCase):
+    @DistCall(nnodes=1, nproc_per_node=2)
+    def test_sampling(self):
+        data = [1, 2, 3, 4, 5]
+        weights = [1, 2, 3, 4, 5]
+        sampler = DistributedWeightedRandomSampler(
+            weights=weights,
+            dataset=data,
+            shuffle=False,
+            generator=torch.Generator().manual_seed(0),
+        )
+        samples = np.array([data[i] for i in list(sampler)])
+
+        if dist.get_rank() == 0:
+            np.testing.assert_allclose(samples, np.array([5, 5, 5]))
+
+        if dist.get_rank() == 1:
+            np.testing.assert_allclose(samples, np.array([1, 4, 4]))
+
+    @DistCall(nnodes=1, nproc_per_node=2)
+    def test_num_samples(self):
+        data = [1, 2, 3, 4, 5]
+        weights = [1, 2, 3, 4, 5]
+        sampler = DistributedWeightedRandomSampler(
+            weights=weights,
+            num_samples_per_rank=5,
+            dataset=data,
+            shuffle=False,
+            generator=torch.Generator().manual_seed(123),
+        )
+        samples = np.array([data[i] for i in list(sampler)])
+
+        if dist.get_rank() == 0:
+            np.testing.assert_allclose(samples, np.array([3, 1, 5, 1, 5]))
+
+        if dist.get_rank() == 1:
+            np.testing.assert_allclose(samples, np.array([4, 2, 4, 2, 4]))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_dvf2ddf.py b/tests/test_dvf2ddf.py
index 0ee8ba6c30..cc3323cf13 100644
--- a/tests/test_dvf2ddf.py
+++ b/tests/test_dvf2ddf.py
@@ -1,3 +1,14 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import unittest
 
 import numpy as np
@@ -10,16 +21,16 @@
 from monai.utils import set_determinism
 
 TEST_CASES = [
-    [{"spatial_dims": 2, "num_steps": 1}, {"dvf": torch.zeros(1, 2, 2, 2)}, torch.zeros(1, 2, 2, 2)],
+    [{"num_steps": 1}, {"dvf": torch.zeros(1, 2, 2, 2)}, torch.zeros(1, 2, 2, 2)],
     [
-        {"spatial_dims": 3, "num_steps": 1},
+        {"num_steps": 1},
         {"dvf": torch.ones(1, 3, 2, 2, 2)},
         torch.tensor([[[1.0000, 0.7500], [0.7500, 0.6250]], [[0.7500, 0.6250], [0.6250, 0.5625]]])
         .reshape(1, 1, 2, 2, 2)
         .expand(-1, 3, -1, -1, -1),
     ],
     [
-        {"spatial_dims": 3, "num_steps": 2},
+        {"num_steps": 2},
         {"dvf": torch.ones(1, 3, 2, 2, 2)},
         torch.tensor([[[0.9175, 0.6618], [0.6618, 0.5306]], [[0.6618, 0.5306], [0.5306, 0.4506]]])
         .reshape(1, 1, 2, 2, 2)
@@ -43,7 +54,7 @@ def test_value(self, input_param, input_data, expected_val):
 
     def test_gradient(self):
         network = nn.Conv2d(in_channels=1, out_channels=2, kernel_size=1)
-        dvf2ddf = DVF2DDF(spatial_dims=2, num_steps=1)
+        dvf2ddf = DVF2DDF(num_steps=1)
         optimizer = SGD(network.parameters(), lr=0.01)
         x = torch.ones((1, 1, 5, 5))
         x = network(x)
diff --git a/tests/test_efficientnet.py b/tests/test_efficientnet.py
new file mode 100644
index 0000000000..7ef56c52a9
--- /dev/null
+++ b/tests/test_efficientnet.py
@@ -0,0 +1,308 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+from typing import TYPE_CHECKING
+from unittest import skipUnless
+
+import torch
+from parameterized import parameterized
+
+from monai.networks import eval_mode
+from monai.networks.nets import EfficientNetBN, drop_connect, get_efficientnet_image_size
+from monai.utils import optional_import
+from tests.utils import skip_if_quick, test_pretrained_networks, test_script_save
+
+if TYPE_CHECKING:
+    import torchvision
+
+    has_torchvision = True
+else:
+    torchvision, has_torchvision = optional_import("torchvision")
+
+if TYPE_CHECKING:
+    import PIL
+
+    has_pil = True
+else:
+    PIL, has_pil = optional_import("PIL")
+
+
+def get_model_names():
+    return ["efficientnet-b{}".format(d) for d in range(8)]
+
+
+def get_expected_model_shape(model_name):
+    model_input_shapes = {
+        "efficientnet-b0": 224,
+        "efficientnet-b1": 240,
+        "efficientnet-b2": 260,
+        "efficientnet-b3": 300,
+        "efficientnet-b4": 380,
+        "efficientnet-b5": 456,
+        "efficientnet-b6": 528,
+        "efficientnet-b7": 600,
+    }
+    return model_input_shapes[model_name]
+
+
+def make_shape_cases(models, spatial_dims, batches, pretrained, in_channels=3, num_classes=1000):
+    ret_tests = []
+    for spatial_dim in spatial_dims:  # selected spatial_dims
+        for batch in batches:  # check single batch as well as multiple batch input
+            for model in models:  # selected models
+                for is_pretrained in pretrained:  # pretrained or not pretrained
+                    kwargs = {
+                        "model_name": model,
+                        "pretrained": is_pretrained,
+                        "progress": False,
+                        "spatial_dims": spatial_dim,
+                        "in_channels": in_channels,
+                        "num_classes": num_classes,
+                    }
+                    ret_tests.append(
+                        [
+                            kwargs,
+                            (
+                                batch,
+                                in_channels,
+                            )
+                            + (get_expected_model_shape(model),) * spatial_dim,
+                            (batch, num_classes),
+                        ]
+                    )
+    return ret_tests
+
+
+# create list of selected models to speed up redundant tests
+# only test the models B0, B3, B7
+SEL_MODELS = [get_model_names()[i] for i in [0, 3, 7]]
+
+# pretrained=False cases
+# 1D models are cheap so do test for all models in 1D
+CASES_1D = make_shape_cases(
+    models=get_model_names(), spatial_dims=[1], batches=[1, 4], pretrained=[False], in_channels=3, num_classes=1000
+)
+
+# 2D and 3D models are expensive so use selected models
+CASES_2D = make_shape_cases(
+    models=SEL_MODELS, spatial_dims=[2], batches=[1, 4], pretrained=[False], in_channels=3, num_classes=1000
+)
+CASES_3D = make_shape_cases(
+    models=[SEL_MODELS[0]], spatial_dims=[3], batches=[1], pretrained=[False], in_channels=3, num_classes=1000
+)
+
+# pretrained=True cases
+# tabby kitty test with pretrained model
+# needs 'testing_data/kitty_test.jpg'
+# image from: https://commons.wikimedia.org/wiki/File:Tabby_cat_with_blue_eyes-3336579.jpg
+CASES_KITTY_TRAINED = [
+    (
+        {
+            "model_name": "efficientnet-b0",
+            "pretrained": True,
+            "progress": False,
+            "spatial_dims": 2,
+            "in_channels": 3,
+            "num_classes": 1000,
+        },
+        os.path.join(os.path.dirname(__file__), "testing_data", "kitty_test.jpg"),
+        282,  # ~ tiger cat
+    ),
+    (
+        {
+            "model_name": "efficientnet-b3",
+            "pretrained": True,
+            "progress": False,
+            "spatial_dims": 2,
+            "in_channels": 3,
+            "num_classes": 1000,
+        },
+        os.path.join(os.path.dirname(__file__), "testing_data", "kitty_test.jpg"),
+        282,  # ~ tiger cat
+    ),
+    (
+        {
+            "model_name": "efficientnet-b7",
+            "pretrained": True,
+            "progress": False,
+            "spatial_dims": 2,
+            "in_channels": 3,
+            "num_classes": 1000,
+        },
+        os.path.join(os.path.dirname(__file__), "testing_data", "kitty_test.jpg"),
+        282,  # ~ tiger cat
+    ),
+]
+
+# varying num_classes and in_channels
+CASES_VARIATIONS = []
+
+# change num_classes test
+# 10 classes
+# 2D
+CASES_VARIATIONS.extend(
+    make_shape_cases(
+        models=SEL_MODELS, spatial_dims=[2], batches=[1], pretrained=[False, True], in_channels=3, num_classes=10
+    )
+)
+# 3D
+CASES_VARIATIONS.extend(
+    make_shape_cases(
+        models=[SEL_MODELS[0]], spatial_dims=[3], batches=[1], pretrained=[False], in_channels=3, num_classes=10
+    )
+)
+
+# change in_channels test
+# 1 channel
+# 2D
+CASES_VARIATIONS.extend(
+    make_shape_cases(
+        models=SEL_MODELS, spatial_dims=[2], batches=[1], pretrained=[False, True], in_channels=1, num_classes=1000
+    )
+)
+# 8 channel
+# 2D
+CASES_VARIATIONS.extend(
+    make_shape_cases(
+        models=SEL_MODELS, spatial_dims=[2], batches=[1], pretrained=[False, True], in_channels=8, num_classes=1000
+    )
+)
+# 3D
+CASES_VARIATIONS.extend(
+    make_shape_cases(
+        models=[SEL_MODELS[0]], spatial_dims=[3], batches=[1], pretrained=[False], in_channels=1, num_classes=1000
+    )
+)
+
+
+class TestEFFICIENTNET(unittest.TestCase):
+    @parameterized.expand(CASES_1D + CASES_2D + CASES_3D + CASES_VARIATIONS)
+    def test_shape(self, input_param, input_shape, expected_shape):
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        print(input_param)
+
+        # initialize model
+        net = EfficientNetBN(**input_param).to(device)
+
+        # run inference with random tensor
+        with eval_mode(net):
+            result = net(torch.randn(input_shape).to(device))
+
+        # check output shape
+        self.assertEqual(result.shape, expected_shape)
+
+    @parameterized.expand(CASES_1D + CASES_2D)
+    def test_non_default_shapes(self, input_param, input_shape, expected_shape):
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        print(input_param)
+
+        # initialize model
+        net = EfficientNetBN(**input_param).to(device)
+
+        # override input shape with different variations
+        num_dims = len(input_shape) - 2
+        non_default_sizes = [128, 256, 512]
+        for candidate_size in non_default_sizes:
+            input_shape = input_shape[0:2] + (candidate_size,) * num_dims
+            print(input_shape)
+            # run inference with random tensor
+            with eval_mode(net):
+                result = net(torch.randn(input_shape).to(device))
+
+            # check output shape
+            self.assertEqual(result.shape, expected_shape)
+
+    @parameterized.expand(CASES_KITTY_TRAINED)
+    @skip_if_quick
+    @skipUnless(has_torchvision, "Requires `torchvision` package.")
+    @skipUnless(has_pil, "Requires `pillow` package.")
+    def test_kitty_pretrained(self, input_param, image_path, expected_label):
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+
+        # open image
+        image_size = get_efficientnet_image_size(input_param["model_name"])
+        img = PIL.Image.open(image_path)
+
+        # defin ImageNet transform
+        tfms = torchvision.transforms.Compose(
+            [
+                torchvision.transforms.Resize(image_size),
+                torchvision.transforms.CenterCrop(image_size),
+                torchvision.transforms.ToTensor(),
+                torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
+            ]
+        )
+
+        # preprocess and prepare image tensor
+        img = tfms(img).unsqueeze(0).to(device)
+
+        # initialize a pretrained model
+        net = test_pretrained_networks(EfficientNetBN, input_param, device)
+
+        # run inference
+        with eval_mode(net):
+            result = net(img)
+        pred_label = torch.argmax(result, dim=-1)
+
+        # check output
+        self.assertEqual(pred_label, expected_label)
+
+    def test_drop_connect_layer(self):
+        p_list = [float(d + 1) / 10.0 for d in range(9)]
+        # testing 1D, 2D and 3D shape
+        for rand_tensor_shape in [(512, 16, 4), (384, 16, 4, 4), (256, 16, 4, 4, 4)]:
+
+            # test validation mode, out tensor == in tensor
+            training = False
+            for p in p_list:
+                in_tensor = torch.rand(rand_tensor_shape) + 0.1
+                out_tensor = drop_connect(in_tensor, p, training=training)
+                self.assertTrue(torch.equal(out_tensor, in_tensor))
+
+            # test training mode, sum((out tensor * (1.0 - p)) != in tensor)/out_tensor.size() == p
+            # use tolerance of 0.175 to account for rounding errors due to finite set in/out
+            tol = 0.175
+            training = True
+            for p in p_list:
+                in_tensor = torch.rand(rand_tensor_shape) + 0.1
+                out_tensor = drop_connect(in_tensor, p, training=training)
+
+                p_calculated = 1.0 - torch.sum(torch.isclose(in_tensor, out_tensor * (1.0 - p))) / float(
+                    in_tensor.numel()
+                )
+                p_calculated = p_calculated.cpu().numpy()
+
+                self.assertTrue(abs(p_calculated - p) < tol)
+
+    def test_ill_arg(self):
+        with self.assertRaises(ValueError):
+            # wrong spatial_dims
+            EfficientNetBN(model_name="efficientnet-b0", spatial_dims=4)
+            # wrong model_name
+            EfficientNetBN(model_name="efficientnet-b10", spatial_dims=3)
+
+    def test_func_get_efficientnet_input_shape(self):
+        for model in get_model_names():
+            result_shape = get_efficientnet_image_size(model_name=model)
+            expected_shape = get_expected_model_shape(model)
+            self.assertEqual(result_shape, expected_shape)
+
+    def test_script(self):
+        net = EfficientNetBN(model_name="efficientnet-b0", spatial_dims=2, in_channels=3, num_classes=1000)
+        net.set_swish(memory_efficient=False)  # at the moment custom memory efficient swish is not exportable with jit
+        test_data = torch.randn(1, 3, 224, 224)
+        test_script_save(net, test_data)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_ensemble_evaluator.py b/tests/test_ensemble_evaluator.py
index 9cc977d876..28a2d4f941 100644
--- a/tests/test_ensemble_evaluator.py
+++ b/tests/test_ensemble_evaluator.py
@@ -12,7 +12,7 @@
 import unittest
 
 import torch
-from ignite.engine import Events
+from ignite.engine import EventEnum, Events
 
 from monai.engines import EnsembleEvaluator
 
@@ -44,11 +44,17 @@ def forward(self, x):
         net3 = TestNet(lambda x: x + 4)
         net4 = TestNet(lambda x: x + 5)
 
+        class CustomEvents(EventEnum):
+            FOO_EVENT = "foo_event"
+            BAR_EVENT = "bar_event"
+
         val_engine = EnsembleEvaluator(
             device=device,
             val_data_loader=val_loader,
             networks=[net0, net1, net2, net3, net4],
             pred_keys=["pred0", "pred1", "pred2", "pred3", "pred4"],
+            event_names=["bwd_event", "opt_event", CustomEvents],
+            event_to_attr={CustomEvents.FOO_EVENT: "foo", "opt_event": "opt"},
         )
 
         @val_engine.on(Events.ITERATION_COMPLETED)
@@ -57,6 +63,21 @@ def run_post_transform(engine):
                 expected_value = engine.state.iteration + i
                 torch.testing.assert_allclose(engine.state.output[f"pred{i}"], torch.tensor([[expected_value]]))
 
+        @val_engine.on(Events.EPOCH_COMPLETED)
+        def trigger_custom_event():
+            val_engine.fire_event(CustomEvents.FOO_EVENT)
+            val_engine.fire_event(CustomEvents.BAR_EVENT)
+            val_engine.fire_event("bwd_event")
+            val_engine.fire_event("opt_event")
+
+        @val_engine.on(CustomEvents.FOO_EVENT)
+        def do_foo_op():
+            self.assertEqual(val_engine.state.foo, 0)
+
+        @val_engine.on("opt_event")
+        def do_bar_op():
+            self.assertEqual(val_engine.state.opt, 0)
+
         val_engine.run()
 
 
diff --git a/tests/test_ensure_channel_first.py b/tests/test_ensure_channel_first.py
new file mode 100644
index 0000000000..ff656f2e24
--- /dev/null
+++ b/tests/test_ensure_channel_first.py
@@ -0,0 +1,86 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import unittest
+
+import itk
+import nibabel as nib
+import numpy as np
+from parameterized import parameterized
+from PIL import Image
+
+from monai.data import ITKReader
+from monai.transforms import EnsureChannelFirst, LoadImage
+
+TEST_CASE_1 = [{"image_only": False}, ["test_image.nii.gz"], None]
+
+TEST_CASE_2 = [{"image_only": False}, ["test_image.nii.gz"], -1]
+
+TEST_CASE_3 = [
+    {"image_only": False},
+    ["test_image.nii.gz", "test_image2.nii.gz", "test_image3.nii.gz"],
+    None,
+]
+
+TEST_CASE_4 = [{"reader": ITKReader(), "image_only": False}, ["test_image.nii.gz"], None]
+
+TEST_CASE_5 = [{"reader": ITKReader(), "image_only": False}, ["test_image.nii.gz"], -1]
+
+TEST_CASE_6 = [
+    {"reader": ITKReader(), "image_only": False},
+    ["test_image.nii.gz", "test_image2.nii.gz", "test_image3.nii.gz"],
+    None,
+]
+
+TEST_CASE_7 = [
+    {"image_only": False, "reader": ITKReader(pixel_type=itk.UC)},
+    "tests/testing_data/CT_DICOM",
+    None,
+]
+
+
+class TestEnsureChannelFirst(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6])
+    def test_load_nifti(self, input_param, filenames, original_channel_dim):
+        if original_channel_dim is None:
+            test_image = np.random.rand(128, 128, 128)
+        elif original_channel_dim == -1:
+            test_image = np.random.rand(128, 128, 128, 1)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            for i, name in enumerate(filenames):
+                filenames[i] = os.path.join(tempdir, name)
+                nib.save(nib.Nifti1Image(test_image, np.eye(4)), filenames[i])
+            result, header = LoadImage(**input_param)(filenames)
+            result = EnsureChannelFirst()(result, header)
+            self.assertEqual(result.shape[0], len(filenames))
+
+    @parameterized.expand([TEST_CASE_7])
+    def test_itk_dicom_series_reader(self, input_param, filenames, original_channel_dim):
+        result, header = LoadImage(**input_param)(filenames)
+        result = EnsureChannelFirst()(result, header)
+        self.assertEqual(result.shape[0], 1)
+
+    def test_load_png(self):
+        spatial_size = (256, 256, 3)
+        test_image = np.random.randint(0, 256, size=spatial_size)
+        with tempfile.TemporaryDirectory() as tempdir:
+            filename = os.path.join(tempdir, "test_image.png")
+            Image.fromarray(test_image.astype("uint8")).save(filename)
+            result, header = LoadImage(image_only=False)(filename)
+            result = EnsureChannelFirst()(result, header)
+            self.assertEqual(result.shape[0], 3)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_ensure_channel_firstd.py b/tests/test_ensure_channel_firstd.py
new file mode 100644
index 0000000000..a5298f4453
--- /dev/null
+++ b/tests/test_ensure_channel_firstd.py
@@ -0,0 +1,62 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import unittest
+
+import nibabel as nib
+import numpy as np
+from parameterized import parameterized
+from PIL import Image
+
+from monai.transforms import EnsureChannelFirstd, LoadImaged
+
+TEST_CASE_1 = [{"keys": "img"}, ["test_image.nii.gz"], None]
+
+TEST_CASE_2 = [{"keys": "img"}, ["test_image.nii.gz"], -1]
+
+TEST_CASE_3 = [
+    {"keys": "img"},
+    ["test_image.nii.gz", "test_image2.nii.gz", "test_image3.nii.gz"],
+    None,
+]
+
+
+class TestEnsureChannelFirstd(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
+    def test_load_nifti(self, input_param, filenames, original_channel_dim):
+        if original_channel_dim is None:
+            test_image = np.random.rand(128, 128, 128)
+        elif original_channel_dim == -1:
+            test_image = np.random.rand(128, 128, 128, 1)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            for i, name in enumerate(filenames):
+                filenames[i] = os.path.join(tempdir, name)
+                nib.save(nib.Nifti1Image(test_image, np.eye(4)), filenames[i])
+            result = LoadImaged(**input_param)({"img": filenames})
+            result = EnsureChannelFirstd(**input_param)(result)
+            self.assertEqual(result["img"].shape[0], len(filenames))
+
+    def test_load_png(self):
+        spatial_size = (256, 256, 3)
+        test_image = np.random.randint(0, 256, size=spatial_size)
+        with tempfile.TemporaryDirectory() as tempdir:
+            filename = os.path.join(tempdir, "test_image.png")
+            Image.fromarray(test_image.astype("uint8")).save(filename)
+            result = LoadImaged(keys="img")({"img": filename})
+            result = EnsureChannelFirstd(keys="img")(result)
+            self.assertEqual(result["img"].shape[0], 3)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_enum_bound_interp.py b/tests/test_enum_bound_interp.py
new file mode 100644
index 0000000000..f788f8ba17
--- /dev/null
+++ b/tests/test_enum_bound_interp.py
@@ -0,0 +1,73 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from monai.utils import optional_import
+from tests.utils import skip_if_no_cpp_extension
+
+b, _ = optional_import("monai._C", name="BoundType")
+p, _ = optional_import("monai._C", name="InterpolationType")
+
+
+@skip_if_no_cpp_extension
+class TestEnumBoundInterp(unittest.TestCase):
+    def test_bound(self):
+        self.assertEqual(str(b.replicate), "BoundType.replicate")
+        self.assertEqual(str(b.nearest), "BoundType.replicate")
+        self.assertEqual(str(b.dct1), "BoundType.dct1")
+        self.assertEqual(str(b.mirror), "BoundType.dct1")
+        self.assertEqual(str(b.dct2), "BoundType.dct2")
+        self.assertEqual(str(b.reflect), "BoundType.dct2")
+        self.assertEqual(str(b.dst1), "BoundType.dst1")
+        self.assertEqual(str(b.antimirror), "BoundType.dst1")
+        self.assertEqual(str(b.dst2), "BoundType.dst2")
+        self.assertEqual(str(b.antireflect), "BoundType.dst2")
+        self.assertEqual(str(b.dft), "BoundType.dft")
+        self.assertEqual(str(b.wrap), "BoundType.dft")
+        self.assertEqual(str(b.zero), "BoundType.zero")
+
+        self.assertEqual(int(b.replicate), 0)
+        self.assertEqual(int(b.nearest), 0)
+        self.assertEqual(int(b.dct1), 1)
+        self.assertEqual(int(b.mirror), 1)
+        self.assertEqual(int(b.dct2), 2)
+        self.assertEqual(int(b.reflect), 2)
+        self.assertEqual(int(b.dst1), 3)
+        self.assertEqual(int(b.antimirror), 3)
+        self.assertEqual(int(b.dst2), 4)
+        self.assertEqual(int(b.antireflect), 4)
+        self.assertEqual(int(b.dft), 5)
+        self.assertEqual(int(b.wrap), 5)
+        self.assertEqual(int(b.zero), 7)
+
+    def test_interp(self):
+        self.assertEqual(str(p.nearest), "InterpolationType.nearest")
+        self.assertEqual(str(p.linear), "InterpolationType.linear")
+        self.assertEqual(str(p.quadratic), "InterpolationType.quadratic")
+        self.assertEqual(str(p.cubic), "InterpolationType.cubic")
+        self.assertEqual(str(p.fourth), "InterpolationType.fourth")
+        self.assertEqual(str(p.fifth), "InterpolationType.fifth")
+        self.assertEqual(str(p.sixth), "InterpolationType.sixth")
+        self.assertEqual(str(p.seventh), "InterpolationType.seventh")
+
+        self.assertEqual(int(p.nearest), 0)
+        self.assertEqual(int(p.linear), 1)
+        self.assertEqual(int(p.quadratic), 2)
+        self.assertEqual(int(p.cubic), 3)
+        self.assertEqual(int(p.fourth), 4)
+        self.assertEqual(int(p.fifth), 5)
+        self.assertEqual(int(p.sixth), 6)
+        self.assertEqual(int(p.seventh), 7)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_file_basename.py b/tests/test_file_basename.py
index 21039d3d15..cb7ee77e62 100644
--- a/tests/test_file_basename.py
+++ b/tests/test_file_basename.py
@@ -36,6 +36,15 @@ def test_value(self):
             expected = os.path.join(output_tmp, "bar", "test", "test")
             self.assertEqual(result, expected)
 
+            result = create_file_basename(
+                postfix="",
+                input_file_name=os.path.join("foo", "bar", "data", "test.txt"),
+                folder_path=output_tmp,
+                data_root_dir=os.path.join("foo", "bar"),
+            )
+            expected = os.path.join(output_tmp, "data", "test", "test")
+            self.assertEqual(result, expected)
+
             result = create_file_basename("", os.path.join("foo", "bar", "test.txt"), output_tmp, "bar")
             expected = os.path.join(tempdir, "foo", "bar", "test", "test")
             self.assertEqual(result, expected)
@@ -48,10 +57,18 @@ def test_value(self):
             expected = os.path.join(output_tmp, "test", "test")
             self.assertEqual(result, expected)
 
+            result = create_file_basename("", "test.txt", output_tmp, "foo", 5)
+            expected = os.path.join(output_tmp, "test", "test_5")
+            self.assertEqual(result, expected)
+
             result = create_file_basename("post", "test.tar.gz", output_tmp, "foo")
             expected = os.path.join(output_tmp, "test", "test_post")
             self.assertEqual(result, expected)
 
+            result = create_file_basename("post", "test.tar.gz", output_tmp, "foo", 8)
+            expected = os.path.join(output_tmp, "test", "test_post_8")
+            self.assertEqual(result, expected)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_focal_loss.py b/tests/test_focal_loss.py
index d06e2b4c36..66665774ef 100644
--- a/tests/test_focal_loss.py
+++ b/tests/test_focal_loss.py
@@ -16,12 +16,14 @@
 import torch.nn.functional as F
 
 from monai.losses import FocalLoss
+from monai.networks import one_hot
+from tests.utils import SkipIfBeforePyTorchVersion, test_script_save
 
 
 class TestFocalLoss(unittest.TestCase):
     def test_consistency_with_cross_entropy_2d(self):
         # For gamma=0 the focal loss reduces to the cross entropy loss
-        focal_loss = FocalLoss(gamma=0.0, reduction="mean")
+        focal_loss = FocalLoss(to_onehot_y=True, gamma=0.0, reduction="mean", weight=1.0)
         ce = nn.CrossEntropyLoss(reduction="mean")
         max_error = 0
         class_num = 10
@@ -35,7 +37,30 @@ def test_consistency_with_cross_entropy_2d(self):
                 x = x.cuda()
                 l = l.cuda()
             output0 = focal_loss(x, l)
-            output1 = ce(x, l[:, 0])
+            output1 = ce(x, l[:, 0]) / class_num
+            a = float(output0.cpu().detach())
+            b = float(output1.cpu().detach())
+            if abs(a - b) > max_error:
+                max_error = abs(a - b)
+        self.assertAlmostEqual(max_error, 0.0, places=3)
+
+    def test_consistency_with_cross_entropy_2d_onehot_label(self):
+        # For gamma=0 the focal loss reduces to the cross entropy loss
+        focal_loss = FocalLoss(to_onehot_y=False, gamma=0.0, reduction="mean")
+        ce = nn.CrossEntropyLoss(reduction="mean")
+        max_error = 0
+        class_num = 10
+        batch_size = 128
+        for _ in range(100):
+            # Create a random tensor of shape (batch_size, class_num, 8, 4)
+            x = torch.rand(batch_size, class_num, 8, 4, requires_grad=True)
+            # Create a random batch of classes
+            l = torch.randint(low=0, high=class_num, size=(batch_size, 1, 8, 4))
+            if torch.cuda.is_available():
+                x = x.cuda()
+                l = l.cuda()
+            output0 = focal_loss(x, one_hot(l, num_classes=class_num))
+            output1 = ce(x, l[:, 0]) / class_num
             a = float(output0.cpu().detach())
             b = float(output1.cpu().detach())
             if abs(a - b) > max_error:
@@ -44,7 +69,7 @@ def test_consistency_with_cross_entropy_2d(self):
 
     def test_consistency_with_cross_entropy_classification(self):
         # for gamma=0 the focal loss reduces to the cross entropy loss
-        focal_loss = FocalLoss(gamma=0.0, reduction="mean")
+        focal_loss = FocalLoss(to_onehot_y=True, gamma=0.0, reduction="mean")
         ce = nn.CrossEntropyLoss(reduction="mean")
         max_error = 0
         class_num = 10
@@ -59,7 +84,7 @@ def test_consistency_with_cross_entropy_classification(self):
                 x = x.cuda()
                 l = l.cuda()
             output0 = focal_loss(x, l)
-            output1 = ce(x, l[:, 0])
+            output1 = ce(x, l[:, 0]) / class_num
             a = float(output0.cpu().detach())
             b = float(output1.cpu().detach())
             if abs(a - b) > max_error:
@@ -74,7 +99,7 @@ def test_bin_seg_2d(self):
         pred_very_good = 1000 * F.one_hot(target, num_classes=2).permute(0, 3, 1, 2).float()
 
         # initialize the mean dice loss
-        loss = FocalLoss()
+        loss = FocalLoss(to_onehot_y=True)
 
         # focal loss for pred_very_good should be close to 0
         target = target.unsqueeze(1)  # shape (1, 1, H, W)
@@ -90,7 +115,7 @@ def test_empty_class_2d(self):
         pred_very_good = 1000 * F.one_hot(target, num_classes=num_classes).permute(0, 3, 1, 2).float()
 
         # initialize the mean dice loss
-        loss = FocalLoss()
+        loss = FocalLoss(to_onehot_y=True)
 
         # focal loss for pred_very_good should be close to 0
         target = target.unsqueeze(1)  # shape (1, 1, H, W)
@@ -105,7 +130,8 @@ def test_multi_class_seg_2d(self):
         target = target.unsqueeze(0)  # shape (1, H, W)
         pred_very_good = 1000 * F.one_hot(target, num_classes=num_classes).permute(0, 3, 1, 2).float()
         # initialize the mean dice loss
-        loss = FocalLoss()
+        loss = FocalLoss(to_onehot_y=True)
+        loss_onehot = FocalLoss(to_onehot_y=False)
 
         # focal loss for pred_very_good should be close to 0
         target_one_hot = F.one_hot(target, num_classes=num_classes).permute(0, 3, 1, 2)  # test one hot
@@ -114,7 +140,7 @@ def test_multi_class_seg_2d(self):
         focal_loss_good = float(loss(pred_very_good, target).cpu())
         self.assertAlmostEqual(focal_loss_good, 0.0, places=3)
 
-        focal_loss_good = float(loss(pred_very_good, target_one_hot).cpu())
+        focal_loss_good = float(loss_onehot(pred_very_good, target_one_hot).cpu())
         self.assertAlmostEqual(focal_loss_good, 0.0, places=3)
 
     def test_bin_seg_3d(self):
@@ -136,33 +162,46 @@ def test_bin_seg_3d(self):
         pred_very_good = 1000 * F.one_hot(target, num_classes=num_classes).permute(0, 4, 1, 2, 3).float()
 
         # initialize the mean dice loss
-        loss = FocalLoss()
+        loss = FocalLoss(to_onehot_y=True)
+        loss_onehot = FocalLoss(to_onehot_y=False)
 
         # focal loss for pred_very_good should be close to 0
         target = target.unsqueeze(1)  # shape (1, 1, H, W)
         focal_loss_good = float(loss(pred_very_good, target).cpu())
         self.assertAlmostEqual(focal_loss_good, 0.0, places=3)
 
-        focal_loss_good = float(loss(pred_very_good, target_one_hot).cpu())
+        focal_loss_good = float(loss_onehot(pred_very_good, target_one_hot).cpu())
         self.assertAlmostEqual(focal_loss_good, 0.0, places=3)
 
     def test_ill_opts(self):
         chn_input = torch.ones((1, 2, 3))
-        chn_target = torch.ones((1, 1, 3))
+        chn_target = torch.ones((1, 2, 3))
         with self.assertRaisesRegex(ValueError, ""):
             FocalLoss(reduction="unknown")(chn_input, chn_target)
-        with self.assertRaisesRegex(ValueError, ""):
-            FocalLoss(reduction=None)(chn_input, chn_target)
+        with self.assertRaisesRegex(TypeError, ""):
+            FocalLoss(other_act="tanh")(chn_input, chn_target)
 
     def test_ill_shape(self):
         chn_input = torch.ones((1, 2, 3))
         chn_target = torch.ones((1, 3))
-        with self.assertRaisesRegex(ValueError, ""):
+        with self.assertRaisesRegex(AssertionError, ""):
             FocalLoss(reduction="mean")(chn_input, chn_target)
-        chn_input = torch.ones((1, 1, 30))
-        chn_target = torch.ones((1, 1, 30))
-        with self.assertRaisesRegex(NotImplementedError, ""):
-            FocalLoss()(chn_input, chn_target)
+
+    def test_ill_class_weight(self):
+        chn_input = torch.ones((1, 4, 3, 3))
+        chn_target = torch.ones((1, 4, 3, 3))
+        with self.assertRaisesRegex(ValueError, ""):
+            FocalLoss(include_background=True, weight=(1.0, 1.0, 2.0))(chn_input, chn_target)
+        with self.assertRaisesRegex(ValueError, ""):
+            FocalLoss(include_background=False, weight=(1.0, 1.0, 1.0, 1.0))(chn_input, chn_target)
+        with self.assertRaisesRegex(ValueError, ""):
+            FocalLoss(include_background=False, weight=(1.0, 1.0, -1.0))(chn_input, chn_target)
+
+    @SkipIfBeforePyTorchVersion((1, 7, 0))
+    def test_script(self):
+        loss = FocalLoss()
+        test_input = torch.ones(2, 2, 8, 8)
+        test_script_save(loss, test_input, test_input)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_generalized_dice_loss.py b/tests/test_generalized_dice_loss.py
index e88253ccba..06446204fb 100644
--- a/tests/test_generalized_dice_loss.py
+++ b/tests/test_generalized_dice_loss.py
@@ -16,6 +16,7 @@
 from parameterized import parameterized
 
 from monai.losses import GeneralizedDiceLoss
+from tests.utils import SkipIfBeforePyTorchVersion, test_script_save
 
 TEST_CASES = [
     [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
@@ -178,6 +179,12 @@ def test_input_warnings(self):
             loss = GeneralizedDiceLoss(to_onehot_y=True)
             loss.forward(chn_input, chn_target)
 
+    @SkipIfBeforePyTorchVersion((1, 7, 0))
+    def test_script(self):
+        loss = GeneralizedDiceLoss()
+        test_input = torch.ones(2, 1, 8, 8)
+        test_script_save(loss, test_input, test_input)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_generalized_wasserstein_dice_loss.py b/tests/test_generalized_wasserstein_dice_loss.py
index 6865b53027..295a4a6d70 100644
--- a/tests/test_generalized_wasserstein_dice_loss.py
+++ b/tests/test_generalized_wasserstein_dice_loss.py
@@ -18,6 +18,7 @@
 import torch.optim as optim
 
 from monai.losses import GeneralizedWassersteinDiceLoss
+from tests.utils import SkipIfBeforePyTorchVersion, test_script_save
 
 
 class TestGeneralizedWassersteinDiceLoss(unittest.TestCase):
@@ -215,6 +216,18 @@ def forward(self, x):
             # check that the predicted segmentation has improved
             self.assertGreater(diff_start, diff_end)
 
+    @SkipIfBeforePyTorchVersion((1, 7, 0))
+    def test_script(self):
+        target = torch.tensor([[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]])
+
+        # add another dimension corresponding to the batch (batch size = 1 here)
+        target = target.unsqueeze(0)
+        pred_very_good = 1000 * F.one_hot(target, num_classes=2).permute(0, 3, 1, 2).float()
+
+        loss = GeneralizedWassersteinDiceLoss(dist_matrix=np.array([[0.0, 1.0], [1.0, 0.0]]), weighting_mode="default")
+
+        test_script_save(loss, pred_very_good, target)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_get_package_version.py b/tests/test_get_package_version.py
new file mode 100644
index 0000000000..beddb340ab
--- /dev/null
+++ b/tests/test_get_package_version.py
@@ -0,0 +1,31 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from monai.utils.module import get_package_version
+
+
+class TestGetVersion(unittest.TestCase):
+    def test_default(self):
+        output = get_package_version("42foobarnoexist")
+        self.assertTrue("UNKNOWN" in output)
+
+        output = get_package_version("numpy")
+        self.assertFalse("UNKNOWN" in output)
+
+    def test_msg(self):
+        output = get_package_version("42foobarnoexist", "test")
+        self.assertTrue("test" in output)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_global_mutual_information_loss.py b/tests/test_global_mutual_information_loss.py
index 252a70e85e..3373b59621 100644
--- a/tests/test_global_mutual_information_loss.py
+++ b/tests/test_global_mutual_information_loss.py
@@ -17,20 +17,30 @@
 
 from monai.losses.image_dissimilarity import GlobalMutualInformationLoss
 
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
 TEST_CASES = [
     [
         {},
         {
-            "pred": torch.arange(0, 3, dtype=torch.float)[None, :, None, None, None].expand(1, 3, 3, 3, 3).div(3),
-            "target": torch.arange(0, 3, dtype=torch.float)[None, :, None, None, None].expand(1, 3, 3, 3, 3).div(3),
+            "pred": torch.arange(0, 3, dtype=torch.float, device=device)[None, :, None, None, None]
+            .expand(1, 3, 3, 3, 3)
+            .div(3),
+            "target": torch.arange(0, 3, dtype=torch.float, device=device)[None, :, None, None, None]
+            .expand(1, 3, 3, 3, 3)
+            .div(3),
         },
         -1.0986018,
     ],
     [
         {},
         {
-            "pred": torch.arange(0, 3, dtype=torch.float)[None, :, None, None, None].expand(1, 3, 3, 3, 3).div(3),
-            "target": torch.arange(0, 3, dtype=torch.float)[None, :, None, None, None].expand(1, 3, 3, 3, 3).div(3)
+            "pred": torch.arange(0, 3, dtype=torch.float, device=device)[None, :, None, None, None]
+            .expand(1, 3, 3, 3, 3)
+            .div(3),
+            "target": torch.arange(0, 3, dtype=torch.float, device=device)[None, :, None, None, None]
+            .expand(1, 3, 3, 3, 3)
+            .div(3)
             ** 2,
         },
         -1.083999,
@@ -38,32 +48,35 @@
     [
         {},
         {
-            "pred": torch.arange(0, 3, dtype=torch.float)[None, :, None, None].expand(1, 3, 3, 3).div(3),
-            "target": torch.arange(0, 3, dtype=torch.float)[None, :, None, None].expand(1, 3, 3, 3).div(3) ** 2,
+            "pred": torch.arange(0, 3, dtype=torch.float, device=device)[None, :, None, None].expand(1, 3, 3, 3).div(3),
+            "target": torch.arange(0, 3, dtype=torch.float, device=device)[None, :, None, None]
+            .expand(1, 3, 3, 3)
+            .div(3)
+            ** 2,
         },
         -1.083999,
     ],
     [
         {},
         {
-            "pred": torch.arange(0, 3, dtype=torch.float)[None, :, None].expand(1, 3, 3).div(3),
-            "target": torch.arange(0, 3, dtype=torch.float)[None, :, None].expand(1, 3, 3).div(3) ** 2,
+            "pred": torch.arange(0, 3, dtype=torch.float, device=device)[None, :, None].expand(1, 3, 3).div(3),
+            "target": torch.arange(0, 3, dtype=torch.float, device=device)[None, :, None].expand(1, 3, 3).div(3) ** 2,
         },
         -1.083999,
     ],
     [
         {},
         {
-            "pred": torch.arange(0, 3, dtype=torch.float)[None, :].div(3),
-            "target": torch.arange(0, 3, dtype=torch.float)[None, :].div(3) ** 2,
+            "pred": torch.arange(0, 3, dtype=torch.float, device=device)[None, :].div(3),
+            "target": torch.arange(0, 3, dtype=torch.float, device=device)[None, :].div(3) ** 2,
         },
         -1.083999,
     ],
     [
         {},
         {
-            "pred": torch.arange(0, 3, dtype=torch.float).div(3),
-            "target": torch.arange(0, 3, dtype=torch.float).div(3) ** 2,
+            "pred": torch.arange(0, 3, dtype=torch.float, device=device).div(3),
+            "target": torch.arange(0, 3, dtype=torch.float, device=device).div(3) ** 2,
         },
         -1.1920927e-07,
     ],
@@ -79,13 +92,13 @@ def test_shape(self, input_param, input_data, expected_val):
     def test_ill_shape(self):
         loss = GlobalMutualInformationLoss()
         with self.assertRaisesRegex(ValueError, ""):
-            loss.forward(torch.ones((1, 2), dtype=torch.float), torch.ones((1, 3), dtype=torch.float))
+            loss.forward(torch.ones((1, 2), dtype=torch.float), torch.ones((1, 3), dtype=torch.float, device=device))
         with self.assertRaisesRegex(ValueError, ""):
-            loss.forward(torch.ones((1, 3, 3), dtype=torch.float), torch.ones((1, 3), dtype=torch.float))
+            loss.forward(torch.ones((1, 3, 3), dtype=torch.float), torch.ones((1, 3), dtype=torch.float, device=device))
 
     def test_ill_opts(self):
-        pred = torch.ones((1, 3, 3, 3, 3), dtype=torch.float)
-        target = torch.ones((1, 3, 3, 3, 3), dtype=torch.float)
+        pred = torch.ones((1, 3, 3, 3, 3), dtype=torch.float, device=device)
+        target = torch.ones((1, 3, 3, 3, 3), dtype=torch.float, device=device)
         with self.assertRaisesRegex(ValueError, ""):
             GlobalMutualInformationLoss(num_bins=0)(pred, target)
         with self.assertRaisesRegex(ValueError, ""):
diff --git a/tests/test_globalnet.py b/tests/test_globalnet.py
new file mode 100644
index 0000000000..32bc58f610
--- /dev/null
+++ b/tests/test_globalnet.py
@@ -0,0 +1,96 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.networks import eval_mode
+from monai.networks.blocks import Warp
+from monai.networks.nets import GlobalNet
+from monai.networks.nets.regunet import AffineHead
+from tests.utils import test_script_save
+
+TEST_CASES_AFFINE_TRANSFORM = [
+    [
+        {"spatial_dims": 3, "image_size": (2, 2, 2), "decode_size": (2, 2, 2), "in_channels": 1},
+        torch.ones(2, 12),
+        torch.tensor([[[1, 2], [2, 3]], [[2, 3], [3, 4]]]).unsqueeze(0).unsqueeze(0).expand(2, 3, 2, 2, 2),
+    ],
+    [
+        {"spatial_dims": 3, "image_size": (2, 2, 2), "decode_size": (2, 2, 2), "in_channels": 1},
+        torch.arange(1, 13).reshape(1, 12).to(torch.float),
+        torch.tensor(
+            [
+                [[[4.0, 7.0], [6.0, 9.0]], [[5.0, 8.0], [7.0, 10.0]]],
+                [[[8.0, 15.0], [14.0, 21.0]], [[13.0, 20.0], [19.0, 26.0]]],
+                [[[12.0, 23.0], [22.0, 33.0]], [[21.0, 32.0], [31.0, 42.0]]],
+            ]
+        ).unsqueeze(0),
+    ],
+]
+
+
+TEST_CASES_GLOBAL_NET = [
+    [
+        {
+            "image_size": (16, 16),
+            "spatial_dims": 2,
+            "in_channels": 1,
+            "num_channel_initial": 16,
+            "depth": 1,
+            "out_kernel_initializer": "kaiming_uniform",
+            "out_activation": None,
+            "pooling": True,
+            "concat_skip": True,
+            "encode_kernel_sizes": 3,
+        },
+        (1, 1, 16, 16),
+        (1, 2, 16, 16),
+    ]
+]
+
+
+class TestAffineHead(unittest.TestCase):
+    @parameterized.expand(TEST_CASES_AFFINE_TRANSFORM)
+    def test_shape(self, input_param, theta, expected_val):
+        layer = AffineHead(**input_param)
+        result = layer.affine_transform(theta)
+        np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
+
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+class TestGlobalNet(unittest.TestCase):
+    @parameterized.expand(TEST_CASES_GLOBAL_NET)
+    def test_shape(self, input_param, input_shape, expected_shape):
+        net = GlobalNet(**input_param).to(device)
+        warp_layer = Warp()
+        with eval_mode(net):
+            img = torch.randn(input_shape)
+            result = net(img.to(device))
+            warped = warp_layer(img.to(device), result)
+            self.assertEqual(result.shape, expected_shape)
+            # testing initial pred identity
+            np.testing.assert_allclose(warped.detach().cpu().numpy(), img.detach().cpu().numpy(), rtol=1e-4, atol=1e-4)
+
+    def test_script(self):
+        input_param, input_shape, _ = TEST_CASES_GLOBAL_NET[0]
+        net = GlobalNet(**input_param)
+        test_data = torch.randn(input_shape)
+        test_script_save(net, test_data)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_grid_dataset.py b/tests/test_grid_dataset.py
new file mode 100644
index 0000000000..6e0aa4023e
--- /dev/null
+++ b/tests/test_grid_dataset.py
@@ -0,0 +1,83 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import unittest
+
+import numpy as np
+
+from monai.data import DataLoader, GridPatchDataset, PatchIter
+from monai.transforms import RandShiftIntensity
+from monai.utils import set_determinism
+
+
+def identity_generator(x):
+    # simple transform that returns the input itself
+    for idx, item in enumerate(x):
+        yield item, idx
+
+
+class TestGridPatchDataset(unittest.TestCase):
+    def setUp(self):
+        set_determinism(seed=1234)
+
+    def tearDown(self):
+        set_determinism(None)
+
+    def test_shape(self):
+        test_dataset = ["vwxyz", "helloworld", "worldfoobar"]
+        result = GridPatchDataset(dataset=test_dataset, patch_iter=identity_generator, with_coordinates=False)
+        output = []
+        n_workers = 0 if sys.platform == "win32" else 2
+        for item in DataLoader(result, batch_size=3, num_workers=n_workers):
+            output.append("".join(item))
+        expected = ["vwx", "wor", "yzh", "ldf", "ell", "oob", "owo", "ar", "rld"]
+        self.assertEqual(sorted(output), sorted(expected))
+        self.assertEqual(len("".join(expected)), len("".join(test_dataset)))
+
+    def test_loading_array(self):
+        set_determinism(seed=1234)
+        # image dataset
+        images = [np.arange(16, dtype=float).reshape(1, 4, 4), np.arange(16, dtype=float).reshape(1, 4, 4)]
+        # image level
+        patch_intensity = RandShiftIntensity(offsets=1.0, prob=1.0)
+        patch_iter = PatchIter(patch_size=(2, 2), start_pos=(0, 0))
+        ds = GridPatchDataset(dataset=images, patch_iter=patch_iter, transform=patch_intensity)
+        # use the grid patch dataset
+        for item in DataLoader(ds, batch_size=2, shuffle=False, num_workers=0):
+            np.testing.assert_equal(tuple(item[0].shape), (2, 1, 2, 2))
+        np.testing.assert_allclose(
+            item[0],
+            np.array([[[[1.7413, 2.7413], [5.7413, 6.7413]]], [[[9.1419, 10.1419], [13.1419, 14.1419]]]]),
+            rtol=1e-5,
+        )
+        np.testing.assert_allclose(
+            item[1],
+            np.array([[[0, 1], [0, 2], [2, 4]], [[0, 1], [2, 4], [2, 4]]]),
+            rtol=1e-5,
+        )
+        if sys.platform != "win32":
+            for item in DataLoader(ds, batch_size=2, shuffle=False, num_workers=2):
+                np.testing.assert_equal(tuple(item[0].shape), (2, 1, 2, 2))
+            np.testing.assert_allclose(
+                item[0],
+                np.array([[[[2.3944, 3.3944], [6.3944, 7.3944]]], [[[10.6551, 11.6551], [14.6551, 15.6551]]]]),
+                rtol=1e-3,
+            )
+            np.testing.assert_allclose(
+                item[1],
+                np.array([[[0, 1], [0, 2], [2, 4]], [[0, 1], [2, 4], [2, 4]]]),
+                rtol=1e-5,
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_grid_pull.py b/tests/test_grid_pull.py
new file mode 100644
index 0000000000..9e4d2e8237
--- /dev/null
+++ b/tests/test_grid_pull.py
@@ -0,0 +1,94 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.networks.layers import grid_pull
+from monai.utils import optional_import
+from tests.testing_data.cpp_resample_answers import Expected_1D_GP_bwd, Expected_1D_GP_fwd
+from tests.utils import skip_if_no_cpp_extension
+
+BType, has_b_type = optional_import("monai._C", name="BoundType")
+PType, has_p_type = optional_import("monai._C", name="InterpolationType")
+
+
+def make_grid(shape, dtype=None, device=None, requires_grad=True):
+    ranges = [torch.arange(float(s), dtype=dtype, device=device, requires_grad=requires_grad) for s in shape]
+    grid = torch.stack(torch.meshgrid(*ranges), dim=-1)
+    return grid[None]
+
+
+# 1D combinations of bounds/interpolations
+bounds = set(BType.__members__.values()) if has_b_type else []
+interps = set(PType.__members__.values()) if has_p_type else []
+device = "cuda" if torch.cuda.is_available() else "cpu"
+TEST_1D_GP = []
+for bound in bounds:
+    for interp in interps:
+        if not Expected_1D_GP_fwd or not Expected_1D_GP_bwd:
+            break  # skip if the testing data are unavailable
+        expected_val = Expected_1D_GP_fwd.pop(0)
+
+        for input_g in (True, False):
+            for grid_g in (True, False):
+                expected_grad = Expected_1D_GP_bwd.pop(0)
+                test_case = [
+                    {
+                        "input": torch.arange(10, dtype=torch.float, requires_grad=input_g, device=device).reshape(
+                            (1, 1, 10)
+                        ),
+                        "grid": make_grid((20,), dtype=torch.float, device=device, requires_grad=grid_g) + 0.5,
+                        "interpolation": interp,
+                        "bound": bound,
+                    },
+                    {
+                        "val": torch.tensor([[expected_val]]),
+                        "device": device,
+                        "grad": torch.tensor(expected_grad),
+                    },
+                ]
+                TEST_1D_GP.append(test_case)
+
+
+@skip_if_no_cpp_extension
+class TestGridPull(unittest.TestCase):
+    @parameterized.expand(TEST_1D_GP, skip_on_empty=True)
+    def test_grid_pull(self, input_param, expected):
+        result = grid_pull(**input_param)
+        if input_param["input"].requires_grad:
+            input_param["input"].retain_grad()
+        if input_param["grid"].requires_grad:
+            input_param["grid"].retain_grad()
+        if input_param["input"].requires_grad or input_param["grid"].requires_grad:
+            result.sum().backward()
+
+        grads = []
+        if input_param["input"].requires_grad:
+            grads.append(input_param["input"].grad.view(-1))
+        if input_param["grid"].requires_grad:
+            grads.append(input_param["grid"].grad.view(-1))
+        if not grads:
+            grads = torch.tensor(0.0, device=result.device)
+        elif len(grads) == 1:
+            grads = grads[0]
+        else:
+            grads = torch.cat(grads, dim=0)
+        self.assertTrue("{}".format(result.device).startswith(expected["device"]))
+        np.testing.assert_allclose(result.detach().cpu().numpy(), expected["val"].cpu().numpy(), rtol=1e-4, atol=1e-4)
+        np.testing.assert_allclose(grads.detach().cpu().numpy(), expected["grad"].cpu().numpy(), rtol=1e-4, atol=1e-4)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_handler_checkpoint_loader.py b/tests/test_handler_checkpoint_loader.py
index 8b0f752ff4..a69193c98c 100644
--- a/tests/test_handler_checkpoint_loader.py
+++ b/tests/test_handler_checkpoint_loader.py
@@ -16,7 +16,7 @@
 
 import torch
 import torch.optim as optim
-from ignite.engine import Engine
+from ignite.engine import Engine, Events
 
 from monai.handlers import CheckpointLoader, CheckpointSaver
 
@@ -33,15 +33,30 @@ def test_one_save_one_load(self):
         data2["weight"] = torch.tensor([0.2])
         net2.load_state_dict(data2)
         with tempfile.TemporaryDirectory() as tempdir:
-            engine = Engine(lambda e, b: None)
-            CheckpointSaver(save_dir=tempdir, save_dict={"net": net1}, save_final=True).attach(engine)
-            engine.run([0] * 8, max_epochs=5)
-            path = tempdir + "/net_final_iteration=40.pt"
-            engine = Engine(lambda e, b: None)
-            CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
-            engine.run([0] * 8, max_epochs=1)
+            engine1 = Engine(lambda e, b: None)
+            CheckpointSaver(save_dir=tempdir, save_dict={"net": net1, "eng": engine1}, save_final=True).attach(engine1)
+            engine1.run([0] * 8, max_epochs=5)
+            path = tempdir + "/checkpoint_final_iteration=40.pt"
+            engine2 = Engine(lambda e, b: None)
+            CheckpointLoader(load_path=path, load_dict={"net": net2, "eng": engine2}, strict=True).attach(engine2)
+
+            @engine2.on(Events.STARTED)
+            def check_epoch(engine: Engine):
+                self.assertEqual(engine.state.epoch, 5)
+
+            engine2.run([0] * 8, max_epochs=8)
             torch.testing.assert_allclose(net2.state_dict()["weight"], torch.tensor([0.1]))
 
+            # test bad case with max_epochs smaller than current epoch
+            engine3 = Engine(lambda e, b: None)
+            CheckpointLoader(load_path=path, load_dict={"net": net2, "eng": engine3}, strict=True).attach(engine3)
+
+            try:
+                engine3.run([0] * 8, max_epochs=3)
+            except ValueError:
+                self.assertEqual(engine3.state.epoch, 5)
+                self.assertEqual(engine3.state.max_epochs, 5)
+
     def test_two_save_one_load(self):
         logging.basicConfig(stream=sys.stdout, level=logging.INFO)
         net1 = torch.nn.PReLU()
@@ -60,7 +75,7 @@ def test_two_save_one_load(self):
             engine.run([0] * 8, max_epochs=5)
             path = tempdir + "/checkpoint_final_iteration=40.pt"
             engine = Engine(lambda e, b: None)
-            CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
+            CheckpointLoader(load_path=path, load_dict={"net": net2}, strict=True).attach(engine)
             engine.run([0] * 8, max_epochs=1)
             torch.testing.assert_allclose(net2.state_dict()["weight"], torch.tensor([0.1]))
 
@@ -81,10 +96,80 @@ def test_save_single_device_load_multi_devices(self):
             engine.run([0] * 8, max_epochs=5)
             path = tempdir + "/net_final_iteration=40.pt"
             engine = Engine(lambda e, b: None)
-            CheckpointLoader(load_path=path, load_dict={"net": net2}).attach(engine)
+            CheckpointLoader(load_path=path, load_dict={"net": net2}, strict=True).attach(engine)
             engine.run([0] * 8, max_epochs=1)
             torch.testing.assert_allclose(net2.state_dict()["module.weight"].cpu(), torch.tensor([0.1]))
 
+    def test_partial_under_load(self):
+        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+        net1 = torch.nn.Sequential(*[torch.nn.PReLU(), torch.nn.PReLU()])
+        data1 = net1.state_dict()
+        data1["0.weight"] = torch.tensor([0.1])
+        data1["1.weight"] = torch.tensor([0.2])
+        net1.load_state_dict(data1)
+
+        net2 = torch.nn.Sequential(*[torch.nn.PReLU()])
+        data2 = net2.state_dict()
+        data2["0.weight"] = torch.tensor([0.3])
+        net2.load_state_dict(data2)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            engine = Engine(lambda e, b: None)
+            CheckpointSaver(save_dir=tempdir, save_dict={"net": net1}, save_final=True).attach(engine)
+            engine.run([0] * 8, max_epochs=5)
+            path = tempdir + "/net_final_iteration=40.pt"
+            engine = Engine(lambda e, b: None)
+            CheckpointLoader(load_path=path, load_dict={"net": net2}, strict=False).attach(engine)
+            engine.run([0] * 8, max_epochs=1)
+            torch.testing.assert_allclose(net2.state_dict()["0.weight"].cpu(), torch.tensor([0.1]))
+
+    def test_partial_over_load(self):
+        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+        net1 = torch.nn.Sequential(*[torch.nn.PReLU()])
+        data1 = net1.state_dict()
+        data1["0.weight"] = torch.tensor([0.1])
+        net1.load_state_dict(data1)
+
+        net2 = torch.nn.Sequential(*[torch.nn.PReLU(), torch.nn.PReLU()])
+        data2 = net2.state_dict()
+        data2["0.weight"] = torch.tensor([0.2])
+        data2["1.weight"] = torch.tensor([0.3])
+        net2.load_state_dict(data2)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            engine = Engine(lambda e, b: None)
+            CheckpointSaver(save_dir=tempdir, save_dict={"net": net1}, save_final=True).attach(engine)
+            engine.run([0] * 8, max_epochs=5)
+            path = tempdir + "/net_final_iteration=40.pt"
+            engine = Engine(lambda e, b: None)
+            CheckpointLoader(load_path=path, load_dict={"net": net2}, strict=False).attach(engine)
+            engine.run([0] * 8, max_epochs=1)
+            torch.testing.assert_allclose(net2.state_dict()["0.weight"].cpu(), torch.tensor([0.1]))
+
+    def test_strict_shape(self):
+        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+        net1 = torch.nn.Sequential(*[torch.nn.PReLU(num_parameters=5)])
+        data1 = net1.state_dict()
+        data1["0.weight"] = torch.tensor([1, 2, 3, 4, 5])
+        data1["new"] = torch.tensor(0.1)
+        net1.load_state_dict(data1, strict=False)
+
+        net2 = torch.nn.Sequential(*[torch.nn.PReLU(), torch.nn.PReLU()])
+        data2 = net2.state_dict()
+        data2["0.weight"] = torch.tensor([0.2])
+        data2["1.weight"] = torch.tensor([0.3])
+        net2.load_state_dict(data2)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            engine = Engine(lambda e, b: None)
+            CheckpointSaver(save_dir=tempdir, save_dict={"net": net1}, save_final=True).attach(engine)
+            engine.run([0] * 8, max_epochs=5)
+            path = tempdir + "/net_final_iteration=40.pt"
+            engine = Engine(lambda e, b: None)
+            CheckpointLoader(load_path=path, load_dict={"net": net2}, strict=False, strict_shape=False).attach(engine)
+            engine.run([0] * 8, max_epochs=1)
+            torch.testing.assert_allclose(net2.state_dict()["0.weight"].cpu(), torch.tensor([0.2]))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_handler_checkpoint_saver.py b/tests/test_handler_checkpoint_saver.py
index 5c2b750a57..14474054df 100644
--- a/tests/test_handler_checkpoint_saver.py
+++ b/tests/test_handler_checkpoint_saver.py
@@ -22,7 +22,20 @@
 
 from monai.handlers import CheckpointLoader, CheckpointSaver
 
-TEST_CASE_1 = [True, None, False, None, 1, None, False, True, 0, None, ["test_checkpoint_final_iteration=40.pt"]]
+TEST_CASE_1 = [
+    True,
+    None,
+    False,
+    None,
+    1,
+    None,
+    False,
+    False,
+    True,
+    0,
+    None,
+    ["test_checkpoint_final_iteration=40.pt"],
+]
 
 TEST_CASE_2 = [
     False,
@@ -33,6 +46,7 @@
     None,
     False,
     True,
+    False,
     0,
     None,
     ["test_checkpoint_key_metric=32.pt", "test_checkpoint_key_metric=40.pt"],
@@ -47,6 +61,7 @@
     None,
     False,
     True,
+    True,
     2,
     2,
     ["test_checkpoint_epoch=2.pt", "test_checkpoint_epoch=4.pt"],
@@ -61,20 +76,48 @@
     None,
     False,
     False,
+    False,
     10,
     2,
     ["test_checkpoint_iteration=30.pt", "test_checkpoint_iteration=40.pt"],
 ]
 
-TEST_CASE_5 = [True, None, False, None, 1, None, False, True, 0, None, ["test_checkpoint_final_iteration=40.pt"], True]
+TEST_CASE_5 = [
+    True,
+    None,
+    False,
+    None,
+    1,
+    None,
+    False,
+    False,
+    True,
+    0,
+    None,
+    ["test_checkpoint_final_iteration=40.pt"],
+    True,
+]
+
+TEST_CASE_6 = [True, "final_model.pt", False, None, 1, None, False, False, True, 0, None, ["final_model.pt"]]
 
-TEST_CASE_6 = [True, "final_model.pt", False, None, 1, None, False, True, 0, None, ["final_model.pt"]]
+TEST_CASE_7 = [False, None, True, "val_loss", 1, "model.pt", False, False, True, 0, None, ["model.pt"]]
 
-TEST_CASE_7 = [False, None, True, "val_loss", 1, "model.pt", False, True, 0, None, ["model.pt"]]
+TEST_CASE_8 = [False, None, True, "val_loss", 1, "model.pt", False, True, True, 0, None, ["model.pt"]]
 
 
 class TestHandlerCheckpointSaver(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7])
+    @parameterized.expand(
+        [
+            TEST_CASE_1,
+            TEST_CASE_2,
+            TEST_CASE_3,
+            TEST_CASE_4,
+            TEST_CASE_5,
+            TEST_CASE_6,
+            TEST_CASE_7,
+            TEST_CASE_8,
+        ]
+    )
     def test_file(
         self,
         save_final,
@@ -84,6 +127,7 @@ def test_file(
         key_metric_n_saved,
         key_metric_filename,
         key_metric_save_state,
+        key_metric_greater_or_equal,
         epoch_level,
         save_interval,
         n_saved,
@@ -117,6 +161,7 @@ def _train_func(engine, batch):
                 key_metric_n_saved,
                 key_metric_filename,
                 key_metric_save_state,
+                key_metric_greater_or_equal,
                 epoch_level,
                 save_interval,
                 n_saved,
diff --git a/tests/test_handler_early_stop.py b/tests/test_handler_early_stop.py
new file mode 100644
index 0000000000..efe8e89825
--- /dev/null
+++ b/tests/test_handler_early_stop.py
@@ -0,0 +1,66 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from ignite.engine import Engine, Events
+
+from monai.handlers import EarlyStopHandler
+
+
+class TestHandlerEarlyStop(unittest.TestCase):
+    def test_early_stop_train_loss(self):
+        def _train_func(engine, batch):
+            return {"loss": 1.5}
+
+        trainer = Engine(_train_func)
+        EarlyStopHandler(
+            patience=5,
+            score_function=lambda x: x.state.output["loss"],
+            trainer=trainer,
+            epoch_level=False,
+        ).attach(trainer)
+
+        trainer.run(range(4), max_epochs=2)
+        self.assertEqual(trainer.state.iteration, 6)
+        self.assertEqual(trainer.state.epoch, 2)
+
+    def test_early_stop_val_metric(self):
+        def _train_func(engine, batch):
+            pass
+
+        trainer = Engine(_train_func)
+        validator = Engine(_train_func)
+        validator.state.metrics["val_acc"] = 0.90
+
+        @trainer.on(Events.EPOCH_COMPLETED)
+        def run_validation(engine):
+            validator.state.metrics["val_acc"] += 0.01
+            validator.run(range(3))
+
+        handler = EarlyStopHandler(
+            patience=3,
+            score_function=lambda x: x.state.metrics["val_acc"],
+            trainer=None,
+            min_delta=0.1,
+            cumulative_delta=True,
+            epoch_level=True,
+        )
+        handler.attach(validator)
+        handler.set_trainer(trainer=trainer)
+
+        trainer.run(range(3), max_epochs=5)
+        self.assertEqual(trainer.state.iteration, 12)
+        self.assertEqual(trainer.state.epoch, 4)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_handler_garbage_collector.py b/tests/test_handler_garbage_collector.py
new file mode 100644
index 0000000000..3766283f40
--- /dev/null
+++ b/tests/test_handler_garbage_collector.py
@@ -0,0 +1,77 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gc
+import unittest
+from unittest import skipUnless
+
+import torch
+from ignite.engine import Engine
+from parameterized import parameterized
+
+from monai.data import Dataset
+from monai.handlers import GarbageCollector
+from monai.utils import exact_version, optional_import
+
+Events, has_ignite = optional_import("ignite.engine", "0.4.4", exact_version, "Events")
+
+
+TEST_CASE_0 = [[0, 1, 2], "epoch"]
+
+TEST_CASE_1 = [[0, 1, 2], "iteration"]
+
+TEST_CASE_2 = [[0, 1, 2], Events.EPOCH_COMPLETED]
+
+
+class TestHandlerGarbageCollector(unittest.TestCase):
+    @skipUnless(has_ignite, "Requires ignite")
+    @parameterized.expand(
+        [
+            TEST_CASE_0,
+            TEST_CASE_1,
+            TEST_CASE_2,
+        ]
+    )
+    def test_content(self, data, trigger_event):
+        # set up engine
+        gb_count_dict = {}
+
+        def _train_func(engine, batch):
+            # store garbage collection counts
+            if trigger_event == Events.EPOCH_COMPLETED or trigger_event.lower() == "epoch":
+                if engine.state.iteration % engine.state.epoch_length == 1:
+                    gb_count_dict[engine.state.epoch] = gc.get_count()
+            elif trigger_event.lower() == "iteration":
+                gb_count_dict[engine.state.iteration] = gc.get_count()
+
+        engine = Engine(_train_func)
+
+        # set up testing handler
+        dataset = Dataset(data, transform=None)
+        data_loader = torch.utils.data.DataLoader(dataset, batch_size=1)
+        GarbageCollector(trigger_event=trigger_event, log_level=30).attach(engine)
+
+        engine.run(data_loader, max_epochs=5)
+
+        first_count = 0
+        for iter, gb_count in gb_count_dict.items():
+            # At least one zero-generation object is collected
+            # self.assertGreaterEqual(gb_count[0], 0)
+            if iter > 1:
+                # Since we are collecting all objects from all generations manually at each call,
+                # starting from the second call, there shouldn't be any 1st and 2nd
+                # generation objects available to collect.
+                self.assertEqual(gb_count[1], first_count)
+                self.assertEqual(gb_count[2], first_count)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_handler_mean_dice.py b/tests/test_handler_mean_dice.py
index d15b549d86..648ffe91ae 100644
--- a/tests/test_handler_mean_dice.py
+++ b/tests/test_handler_mean_dice.py
@@ -39,8 +39,8 @@ def _val_func(engine, batch):
         y = torch.Tensor([[[0], [1]], [[0], [1]]])
         dice_metric.update([y_pred, y])
 
-        y_pred = torch.Tensor([[[0], [1]], [[1], [0]]])
-        y = torch.Tensor([[[0], [1]], [[1], [0]]])
+        y_pred = [torch.Tensor([[0], [1]]), torch.Tensor([[1], [0]])]
+        y = [torch.Tensor([[0], [1]]), torch.Tensor([[1], [0]])]
         dice_metric.update([y_pred, y])
 
         avg_dice = dice_metric.compute()
diff --git a/tests/test_handler_metric_logger.py b/tests/test_handler_metric_logger.py
new file mode 100644
index 0000000000..5812605cd7
--- /dev/null
+++ b/tests/test_handler_metric_logger.py
@@ -0,0 +1,60 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import torch
+
+from monai.utils import optional_import
+from tests.utils import SkipIfNoModule
+
+try:
+    _, has_ignite = optional_import("ignite")
+    from ignite.engine import Engine, Events
+
+    from monai.handlers import MetricLogger
+except ImportError:
+    has_ignite = False
+
+
+class TestHandlerMetricLogger(unittest.TestCase):
+    @SkipIfNoModule("ignite")
+    def test_metric_logging(self):
+        dummy_name = "dummy"
+
+        # set up engine
+        def _train_func(engine, batch):
+            return torch.tensor(0.0)
+
+        engine = Engine(_train_func)
+
+        # set up dummy metric
+        @engine.on(Events.EPOCH_COMPLETED)
+        def _update_metric(engine):
+            engine.state.metrics[dummy_name] = 1
+
+        # set up testing handler
+        handler = MetricLogger(loss_transform=lambda output: output.item())
+        handler.attach(engine)
+
+        engine.run(range(3), max_epochs=2)
+
+        expected_loss = [(1, 0.0), (2, 0.0), (3, 0.0), (4, 0.0), (5, 0.0), (6, 0.0)]
+        expected_metric = [(4, 1), (5, 1), (6, 1)]
+
+        self.assertSetEqual({dummy_name}, set(handler.metrics))
+
+        self.assertListEqual(expected_loss, handler.loss)
+        self.assertListEqual(expected_metric, handler.metrics[dummy_name])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_handler_metrics_saver_dist.py b/tests/test_handler_metrics_saver_dist.py
index 1b17d0adb4..0868ec5ff3 100644
--- a/tests/test_handler_metrics_saver_dist.py
+++ b/tests/test_handler_metrics_saver_dist.py
@@ -27,79 +27,82 @@
 class DistributedMetricsSaver(DistTestCase):
     @DistCall(nnodes=1, nproc_per_node=2)
     def test_content(self):
-        self._run()
-
-    def _run(self):
         with tempfile.TemporaryDirectory() as tempdir:
-            metrics_saver = MetricsSaver(
-                save_dir=tempdir,
-                metrics=["metric1", "metric2"],
-                metric_details=["metric3", "metric4"],
-                batch_transform=lambda x: x["image_meta_dict"],
-                summary_ops="*",
-            )
-
-            def _val_func(engine, batch):
-                pass
-
-            engine = Engine(_val_func)
-
-            if dist.get_rank() == 0:
-                data = [{"image_meta_dict": {"filename_or_obj": ["filepath1"]}}]
-
-                @engine.on(Events.EPOCH_COMPLETED)
-                def _save_metrics0(engine):
-                    engine.state.metrics = {"metric1": 1, "metric2": 2}
-                    engine.state.metric_details = {
-                        "metric3": torch.tensor([[1, 2]]),
-                        "metric4": torch.tensor([[5, 6]]),
-                    }
-
-            if dist.get_rank() == 1:
-                # different ranks have different data length
-                data = [
-                    {"image_meta_dict": {"filename_or_obj": ["filepath2"]}},
-                    {"image_meta_dict": {"filename_or_obj": ["filepath3"]}},
-                ]
-
-                @engine.on(Events.EPOCH_COMPLETED)
-                def _save_metrics1(engine):
-                    engine.state.metrics = {"metric1": 1, "metric2": 2}
-                    engine.state.metric_details = {
-                        "metric3": torch.tensor([[2, 3], [3, 4]]),
-                        "metric4": torch.tensor([[6, 7], [7, 8]]),
-                    }
-
-            metrics_saver.attach(engine)
-            engine.run(data, max_epochs=1)
-
-            if dist.get_rank() == 0:
-                # check the metrics.csv and content
-                self.assertTrue(os.path.exists(os.path.join(tempdir, "metrics.csv")))
-                with open(os.path.join(tempdir, "metrics.csv")) as f:
-                    f_csv = csv.reader(f)
-                    for i, row in enumerate(f_csv):
-                        self.assertEqual(row, [f"metric{i + 1}\t{i + 1}"])
-                self.assertTrue(os.path.exists(os.path.join(tempdir, "metric3_raw.csv")))
-                # check the metric_raw.csv and content
-                with open(os.path.join(tempdir, "metric3_raw.csv")) as f:
-                    f_csv = csv.reader(f)
-                    for i, row in enumerate(f_csv):
-                        if i > 0:
-                            self.assertEqual(row, [f"filepath{i}\t{float(i)}\t{float(i + 1)}\t{i + 0.5}"])
-                self.assertTrue(os.path.exists(os.path.join(tempdir, "metric3_summary.csv")))
-                # check the metric_summary.csv and content
-                with open(os.path.join(tempdir, "metric3_summary.csv")) as f:
-                    f_csv = csv.reader(f)
-                    for i, row in enumerate(f_csv):
-                        if i == 1:
-                            self.assertEqual(row, ["class0\t1.0000\t1.0000\t1.0000\t1.0000\t1.0000\t0.0000"])
-                        elif i == 2:
-                            self.assertEqual(row, ["class1\t2.0000\t2.0000\t2.0000\t2.0000\t2.0000\t0.0000"])
-                        elif i == 3:
-                            self.assertEqual(row, ["mean\t1.5000\t1.5000\t1.5000\t1.5000\t1.5000\t0.0000"])
-                self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_raw.csv")))
-                self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_summary.csv")))
+            self._run(tempdir)
+
+    def _run(self, tempdir):
+        fnames = ["aaa" * 300, "bbb" * 301, "ccc" * 302]
+
+        metrics_saver = MetricsSaver(
+            save_dir=tempdir,
+            metrics=["metric1", "metric2"],
+            metric_details=["metric3", "metric4"],
+            batch_transform=lambda x: x["image_meta_dict"],
+            summary_ops="*",
+        )
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+
+        if dist.get_rank() == 0:
+            data = [{"image_meta_dict": {"filename_or_obj": [fnames[0]]}}]
+
+            @engine.on(Events.EPOCH_COMPLETED)
+            def _save_metrics0(engine):
+                engine.state.metrics = {"metric1": 1, "metric2": 2}
+                engine.state.metric_details = {
+                    "metric3": torch.tensor([[1, 2]]),
+                    "metric4": torch.tensor([[5, 6]]),
+                }
+
+        if dist.get_rank() == 1:
+            # different ranks have different data length
+            data = [
+                {"image_meta_dict": {"filename_or_obj": [fnames[1]]}},
+                {"image_meta_dict": {"filename_or_obj": [fnames[2]]}},
+            ]
+
+            @engine.on(Events.EPOCH_COMPLETED)
+            def _save_metrics1(engine):
+                engine.state.metrics = {"metric1": 1, "metric2": 2}
+                engine.state.metric_details = {
+                    "metric3": torch.tensor([[2, 3], [3, 4]]),
+                    "metric4": torch.tensor([[6, 7], [7, 8]]),
+                }
+
+        metrics_saver.attach(engine)
+        engine.run(data, max_epochs=1)
+
+        if dist.get_rank() == 0:
+            # check the metrics.csv and content
+            self.assertTrue(os.path.exists(os.path.join(tempdir, "metrics.csv")))
+            with open(os.path.join(tempdir, "metrics.csv")) as f:
+                f_csv = csv.reader(f)
+                for i, row in enumerate(f_csv):
+                    self.assertEqual(row, [f"metric{i + 1}\t{i + 1}"])
+            self.assertTrue(os.path.exists(os.path.join(tempdir, "metric3_raw.csv")))
+            # check the metric_raw.csv and content
+            with open(os.path.join(tempdir, "metric3_raw.csv")) as f:
+                f_csv = csv.reader(f)
+                for i, row in enumerate(f_csv):
+                    if i > 0:
+                        expected = [f"{fnames[i-1]}\t{float(i)}\t{float(i + 1)}\t{i + 0.5}"]
+                        self.assertEqual(row, expected)
+            self.assertTrue(os.path.exists(os.path.join(tempdir, "metric3_summary.csv")))
+            # check the metric_summary.csv and content
+            with open(os.path.join(tempdir, "metric3_summary.csv")) as f:
+                f_csv = csv.reader(f)
+                for i, row in enumerate(f_csv):
+                    if i == 1:
+                        self.assertEqual(row, ["class0\t1.0000\t1.0000\t1.0000\t1.0000\t1.0000\t0.0000"])
+                    elif i == 2:
+                        self.assertEqual(row, ["class1\t2.0000\t2.0000\t2.0000\t2.0000\t2.0000\t0.0000"])
+                    elif i == 3:
+                        self.assertEqual(row, ["mean\t1.5000\t1.5000\t1.5000\t1.5000\t1.5000\t0.0000"])
+            self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_raw.csv")))
+            self.assertTrue(os.path.exists(os.path.join(tempdir, "metric4_summary.csv")))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_handler_parameter_scheduler.py b/tests/test_handler_parameter_scheduler.py
new file mode 100644
index 0000000000..5b3e845ace
--- /dev/null
+++ b/tests/test_handler_parameter_scheduler.py
@@ -0,0 +1,123 @@
+import unittest
+
+import torch
+from ignite.engine import Engine, Events
+from torch.nn import Module
+
+from monai.handlers.parameter_scheduler import ParamSchedulerHandler
+
+
+class ToyNet(Module):
+    def __init__(self, value):
+        super(ToyNet, self).__init__()
+        self.value = value
+
+    def forward(self, input):
+        return input
+
+    def get_value(self):
+        return self.value
+
+    def set_value(self, value):
+        self.value = value
+
+
+class TestHandlerParameterScheduler(unittest.TestCase):
+    def test_linear_scheduler(self):
+        # Testing step_constant
+        net = ToyNet(value=-1)
+        engine = Engine(lambda e, b: None)
+        ParamSchedulerHandler(
+            parameter_setter=net.set_value,
+            value_calculator="linear",
+            vc_kwargs={"initial_value": 0, "step_constant": 2, "step_max_value": 5, "max_value": 10},
+            epoch_level=True,
+            event=Events.EPOCH_COMPLETED,
+        ).attach(engine)
+        engine.run([0] * 8, max_epochs=2)
+        torch.testing.assert_allclose(net.get_value(), 0)
+
+        # Testing linear increase
+        net = ToyNet(value=-1)
+        engine = Engine(lambda e, b: None)
+        ParamSchedulerHandler(
+            parameter_setter=net.set_value,
+            value_calculator="linear",
+            vc_kwargs={"initial_value": 0, "step_constant": 2, "step_max_value": 5, "max_value": 10},
+            epoch_level=True,
+            event=Events.EPOCH_COMPLETED,
+        ).attach(engine)
+        engine.run([0] * 8, max_epochs=3)
+        torch.testing.assert_allclose(net.get_value(), 3.333333, atol=0.001, rtol=0.0)
+
+        # Testing max_value
+        net = ToyNet(value=-1)
+        engine = Engine(lambda e, b: None)
+        ParamSchedulerHandler(
+            parameter_setter=net.set_value,
+            value_calculator="linear",
+            vc_kwargs={"initial_value": 0, "step_constant": 2, "step_max_value": 5, "max_value": 10},
+            epoch_level=True,
+            event=Events.EPOCH_COMPLETED,
+        ).attach(engine)
+        engine.run([0] * 8, max_epochs=10)
+        torch.testing.assert_allclose(net.get_value(), 10)
+
+    def test_exponential_scheduler(self):
+        net = ToyNet(value=-1)
+        engine = Engine(lambda e, b: None)
+        ParamSchedulerHandler(
+            parameter_setter=net.set_value,
+            value_calculator="exponential",
+            vc_kwargs={"initial_value": 10, "gamma": 0.99},
+            epoch_level=True,
+            event=Events.EPOCH_COMPLETED,
+        ).attach(engine)
+        engine.run([0] * 8, max_epochs=2)
+        torch.testing.assert_allclose(net.get_value(), 10 * 0.99 * 0.99)
+
+    def test_step_scheduler(self):
+        net = ToyNet(value=-1)
+        engine = Engine(lambda e, b: None)
+        ParamSchedulerHandler(
+            parameter_setter=net.set_value,
+            value_calculator="step",
+            vc_kwargs={"initial_value": 10, "gamma": 0.99, "step_size": 5},
+            epoch_level=True,
+            event=Events.EPOCH_COMPLETED,
+        ).attach(engine)
+        engine.run([0] * 8, max_epochs=10)
+        torch.testing.assert_allclose(net.get_value(), 10 * 0.99 * 0.99)
+
+    def test_multistep_scheduler(self):
+        net = ToyNet(value=-1)
+        engine = Engine(lambda e, b: None)
+        ParamSchedulerHandler(
+            parameter_setter=net.set_value,
+            value_calculator="multistep",
+            vc_kwargs={"initial_value": 10, "gamma": 0.99, "milestones": [3, 6]},
+            epoch_level=True,
+            event=Events.EPOCH_COMPLETED,
+        ).attach(engine)
+        engine.run([0] * 8, max_epochs=10)
+        torch.testing.assert_allclose(net.get_value(), 10 * 0.99 * 0.99)
+
+    def test_custom_scheduler(self):
+        def custom_logic(initial_value, gamma, current_step):
+            return initial_value * gamma ** (current_step % 9)
+
+        net = ToyNet(value=-1)
+        engine = Engine(lambda e, b: None)
+        ParamSchedulerHandler(
+            parameter_setter=net.set_value,
+            value_calculator=custom_logic,
+            vc_kwargs={"initial_value": 10, "gamma": 0.99},
+            epoch_level=True,
+            event=Events.EPOCH_COMPLETED,
+        ).attach(engine)
+        engine.run([0] * 8, max_epochs=2)
+        torch.testing.assert_allclose(net.get_value(), 10 * 0.99 * 0.99)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_handler_prob_map_producer.py b/tests/test_handler_prob_map_producer.py
new file mode 100644
index 0000000000..4f719fccc0
--- /dev/null
+++ b/tests/test_handler_prob_map_producer.py
@@ -0,0 +1,96 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+
+import numpy as np
+import torch
+from ignite.engine import Engine
+from parameterized import parameterized
+from torch.utils.data import DataLoader
+
+from monai.apps.pathology.handlers import ProbMapProducer
+from monai.data.dataset import Dataset
+from monai.engines import Evaluator
+from monai.handlers import ValidationHandler
+
+TEST_CASE_0 = ["temp_image_inference_output_1", 2]
+TEST_CASE_1 = ["temp_image_inference_output_2", 9]
+TEST_CASE_2 = ["temp_image_inference_output_3", 1000]
+
+
+class TestDataset(Dataset):
+    def __init__(self, name, size):
+        self.data = [
+            {
+                "name": name,
+                "mask_shape": (size, size),
+                "mask_locations": [[i, i] for i in range(size)],
+                "level": 0,
+            }
+        ]
+        self.len = size
+
+    def __len__(self):
+        return self.len
+
+    def __getitem__(self, index):
+        return {
+            "name": self.data[0]["name"],
+            "mask_location": self.data[0]["mask_locations"][index],
+            "pred": index + 1,
+        }
+
+
+class TestEvaluator(Evaluator):
+    def _iteration(self, engine, batchdata):
+        return batchdata
+
+
+class TestHandlerProbMapGenerator(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASE_0,
+            TEST_CASE_1,
+            TEST_CASE_2,
+        ]
+    )
+    def test_prob_map_generator(self, name, size):
+        # set up dataset
+        dataset = TestDataset(name, size)
+        data_loader = DataLoader(dataset, batch_size=1)
+
+        # set up engine
+        def inference(enging, batch):
+            pass
+
+        engine = Engine(inference)
+
+        # add ProbMapGenerator() to evaluator
+        output_dir = os.path.join(os.path.dirname(__file__), "testing_data")
+        prob_map_gen = ProbMapProducer(output_dir=output_dir)
+
+        evaluator = TestEvaluator(torch.device("cpu:0"), data_loader, size, val_handlers=[prob_map_gen])
+
+        # set up validation handler
+        validation = ValidationHandler(interval=1, validator=None)
+        validation.attach(engine)
+        validation.set_validator(validator=evaluator)
+
+        engine.run(data_loader)
+
+        prob_map = np.load(os.path.join(output_dir, name + ".npy"))
+        self.assertListEqual(np.diag(prob_map).astype(int).tolist(), list(range(1, size + 1)))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_handler_rocauc.py b/tests/test_handler_rocauc.py
index 05f6eebce6..04e4d3edb3 100644
--- a/tests/test_handler_rocauc.py
+++ b/tests/test_handler_rocauc.py
@@ -15,18 +15,25 @@
 import torch
 
 from monai.handlers import ROCAUC
+from monai.transforms import Activations, AsDiscrete
 
 
 class TestHandlerROCAUC(unittest.TestCase):
     def test_compute(self):
-        auc_metric = ROCAUC(to_onehot_y=True, softmax=True)
+        auc_metric = ROCAUC()
+        act = Activations(softmax=True)
+        to_onehot = AsDiscrete(to_onehot=True, n_classes=2)
 
         y_pred = torch.Tensor([[0.1, 0.9], [0.3, 1.4]])
         y = torch.Tensor([[0], [1]])
+        y_pred = act(y_pred)
+        y = to_onehot(y)
         auc_metric.update([y_pred, y])
 
         y_pred = torch.Tensor([[0.2, 0.1], [0.1, 0.5]])
         y = torch.Tensor([[0], [1]])
+        y_pred = act(y_pred)
+        y = to_onehot(y)
         auc_metric.update([y_pred, y])
 
         auc = auc_metric.compute()
diff --git a/tests/test_handler_rocauc_dist.py b/tests/test_handler_rocauc_dist.py
index c5cf44162c..e768906158 100644
--- a/tests/test_handler_rocauc_dist.py
+++ b/tests/test_handler_rocauc_dist.py
@@ -17,23 +17,29 @@
 import torch.distributed as dist
 
 from monai.handlers import ROCAUC
+from monai.transforms import Activations, AsDiscrete
 from tests.utils import DistCall, DistTestCase
 
 
 class DistributedROCAUC(DistTestCase):
     @DistCall(nnodes=1, nproc_per_node=2, node_rank=0)
     def test_compute(self):
-        auc_metric = ROCAUC(to_onehot_y=True, softmax=True)
+        auc_metric = ROCAUC()
+        act = Activations(softmax=True)
+        to_onehot = AsDiscrete(to_onehot=True, n_classes=2)
+
         device = f"cuda:{dist.get_rank()}" if torch.cuda.is_available() else "cpu"
         if dist.get_rank() == 0:
             y_pred = torch.tensor([[0.1, 0.9], [0.3, 1.4]], device=device)
             y = torch.tensor([[0], [1]], device=device)
-            auc_metric.update([y_pred, y])
 
         if dist.get_rank() == 1:
             y_pred = torch.tensor([[0.2, 0.1], [0.1, 0.5], [0.3, 0.4]], device=device)
             y = torch.tensor([[0], [1], [1]], device=device)
-            auc_metric.update([y_pred, y])
+
+        y_pred = act(y_pred)
+        y = to_onehot(y)
+        auc_metric.update([y_pred, y])
 
         result = auc_metric.compute()
         np.testing.assert_allclose(0.66667, result, rtol=1e-4)
diff --git a/tests/test_handler_segmentation_saver.py b/tests/test_handler_segmentation_saver.py
index 1a2bbb7fbd..5449530b50 100644
--- a/tests/test_handler_segmentation_saver.py
+++ b/tests/test_handler_segmentation_saver.py
@@ -40,10 +40,15 @@ def _train_func(engine, batch):
             saver = SegmentationSaver(output_dir=tempdir, output_postfix="seg", output_ext=output_ext, scale=255)
             saver.attach(engine)
 
-            data = [{"filename_or_obj": ["testfile" + str(i) + ".nii.gz" for i in range(8)]}]
+            data = [
+                {
+                    "filename_or_obj": ["testfile" + str(i) + ".nii.gz" for i in range(8)],
+                    "patch_index": list(range(8)),
+                }
+            ]
             engine.run(data, max_epochs=1)
             for i in range(8):
-                filepath = os.path.join("testfile" + str(i), "testfile" + str(i) + "_seg" + output_ext)
+                filepath = os.path.join("testfile" + str(i), "testfile" + str(i) + "_seg" + f"_{i}" + output_ext)
                 self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
 
     @parameterized.expand([TEST_CASE_0, TEST_CASE_1])
diff --git a/tests/test_handler_smartcache.py b/tests/test_handler_smartcache.py
index 95f8e70fa4..cfe68e98e2 100644
--- a/tests/test_handler_smartcache.py
+++ b/tests/test_handler_smartcache.py
@@ -36,7 +36,7 @@ def _train_func(engine, batch):
         engine = Engine(_train_func)
 
         # set up testing handler
-        dataset = SmartCacheDataset(data, transform=None, replace_rate=0.2, cache_num=5)
+        dataset = SmartCacheDataset(data, transform=None, replace_rate=0.2, cache_num=5, shuffle=False)
         data_loader = torch.utils.data.DataLoader(dataset, batch_size=5)
         SmartCacheHandler(dataset).attach(engine)
 
diff --git a/tests/test_handler_stats.py b/tests/test_handler_stats.py
index d1602f802a..248be9f329 100644
--- a/tests/test_handler_stats.py
+++ b/tests/test_handler_stats.py
@@ -25,7 +25,8 @@
 class TestHandlerStats(unittest.TestCase):
     def test_metrics_print(self):
         log_stream = StringIO()
-        logging.basicConfig(stream=log_stream, level=logging.INFO)
+        log_handler = logging.StreamHandler(log_stream)
+        log_handler.setLevel(logging.INFO)
         key_to_handler = "test_logging"
         key_to_print = "testing_metric"
 
@@ -42,13 +43,14 @@ def _update_metric(engine):
             engine.state.metrics[key_to_print] = current_metric + 0.1
 
         # set up testing handler
-        stats_handler = StatsHandler(name=key_to_handler)
+        stats_handler = StatsHandler(name=key_to_handler, logger_handler=log_handler)
         stats_handler.attach(engine)
 
         engine.run(range(3), max_epochs=2)
 
         # check logging output
         output_str = log_stream.getvalue()
+        log_handler.close()
         grep = re.compile(f".*{key_to_handler}.*")
         has_key_word = re.compile(f".*{key_to_print}.*")
         for idx, line in enumerate(output_str.split("\n")):
@@ -58,7 +60,8 @@ def _update_metric(engine):
 
     def test_loss_print(self):
         log_stream = StringIO()
-        logging.basicConfig(stream=log_stream, level=logging.INFO)
+        log_handler = logging.StreamHandler(log_stream)
+        log_handler.setLevel(logging.INFO)
         key_to_handler = "test_logging"
         key_to_print = "myLoss"
 
@@ -69,13 +72,14 @@ def _train_func(engine, batch):
         engine = Engine(_train_func)
 
         # set up testing handler
-        stats_handler = StatsHandler(name=key_to_handler, tag_name=key_to_print)
+        stats_handler = StatsHandler(name=key_to_handler, tag_name=key_to_print, logger_handler=log_handler)
         stats_handler.attach(engine)
 
         engine.run(range(3), max_epochs=2)
 
         # check logging output
         output_str = log_stream.getvalue()
+        log_handler.close()
         grep = re.compile(f".*{key_to_handler}.*")
         has_key_word = re.compile(f".*{key_to_print}.*")
         for idx, line in enumerate(output_str.split("\n")):
@@ -85,7 +89,8 @@ def _train_func(engine, batch):
 
     def test_loss_dict(self):
         log_stream = StringIO()
-        logging.basicConfig(stream=log_stream, level=logging.INFO)
+        log_handler = logging.StreamHandler(log_stream)
+        log_handler.setLevel(logging.INFO)
         key_to_handler = "test_logging"
         key_to_print = "myLoss1"
 
@@ -96,13 +101,16 @@ def _train_func(engine, batch):
         engine = Engine(_train_func)
 
         # set up testing handler
-        stats_handler = StatsHandler(name=key_to_handler, output_transform=lambda x: {key_to_print: x})
+        stats_handler = StatsHandler(
+            name=key_to_handler, output_transform=lambda x: {key_to_print: x}, logger_handler=log_handler
+        )
         stats_handler.attach(engine)
 
         engine.run(range(3), max_epochs=2)
 
         # check logging output
         output_str = log_stream.getvalue()
+        log_handler.close()
         grep = re.compile(f".*{key_to_handler}.*")
         has_key_word = re.compile(f".*{key_to_print}.*")
         for idx, line in enumerate(output_str.split("\n")):
@@ -111,13 +119,13 @@ def _train_func(engine, batch):
                     self.assertTrue(has_key_word.match(line))
 
     def test_loss_file(self):
-        logging.basicConfig(level=logging.INFO)
         key_to_handler = "test_logging"
         key_to_print = "myLoss"
 
         with tempfile.TemporaryDirectory() as tempdir:
             filename = os.path.join(tempdir, "test_loss_stats.log")
             handler = logging.FileHandler(filename, mode="w")
+            handler.setLevel(logging.INFO)
 
             # set up engine
             def _train_func(engine, batch):
@@ -130,7 +138,7 @@ def _train_func(engine, batch):
             stats_handler.attach(engine)
 
             engine.run(range(3), max_epochs=2)
-            handler.stream.close()
+            handler.close()
             stats_handler.logger.removeHandler(handler)
             with open(filename, "r") as f:
                 output_str = f.read()
@@ -142,8 +150,6 @@ def _train_func(engine, batch):
                             self.assertTrue(has_key_word.match(line))
 
     def test_exception(self):
-        logging.basicConfig(level=logging.INFO)
-
         # set up engine
         def _train_func(engine, batch):
             raise RuntimeError("test exception.")
diff --git a/tests/test_handler_transform_inverter.py b/tests/test_handler_transform_inverter.py
new file mode 100644
index 0000000000..524f13a760
--- /dev/null
+++ b/tests/test_handler_transform_inverter.py
@@ -0,0 +1,141 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import unittest
+
+import numpy as np
+import torch
+from ignite.engine import Engine
+
+from monai.data import CacheDataset, DataLoader, create_test_image_3d
+from monai.engines.utils import IterationEvents
+from monai.handlers import TransformInverter
+from monai.transforms import (
+    AddChanneld,
+    CastToTyped,
+    Compose,
+    LoadImaged,
+    Orientationd,
+    RandAffined,
+    RandAxisFlipd,
+    RandFlipd,
+    RandRotate90d,
+    RandRotated,
+    RandZoomd,
+    ResizeWithPadOrCropd,
+    ScaleIntensityd,
+    Spacingd,
+    ToTensord,
+)
+from monai.utils.misc import set_determinism
+from tests.utils import make_nifti_image
+
+KEYS = ["image", "label"]
+
+
+class TestTransformInverter(unittest.TestCase):
+    def test_invert(self):
+        set_determinism(seed=0)
+        im_fname, seg_fname = [make_nifti_image(i) for i in create_test_image_3d(101, 100, 107, noise_max=100)]
+        transform = Compose(
+            [
+                LoadImaged(KEYS),
+                AddChanneld(KEYS),
+                Orientationd(KEYS, "RPS"),
+                Spacingd(KEYS, pixdim=(1.2, 1.01, 0.9), mode=["bilinear", "nearest"], dtype=np.float32),
+                ScaleIntensityd("image", minv=1, maxv=10),
+                RandFlipd(KEYS, prob=0.5, spatial_axis=[1, 2]),
+                RandAxisFlipd(KEYS, prob=0.5),
+                RandRotate90d(KEYS, spatial_axes=(1, 2)),
+                RandZoomd(KEYS, prob=0.5, min_zoom=0.5, max_zoom=1.1, keep_size=True),
+                RandRotated(KEYS, prob=0.5, range_x=np.pi, mode="bilinear", align_corners=True),
+                RandAffined(KEYS, prob=0.5, rotate_range=np.pi, mode="nearest"),
+                ResizeWithPadOrCropd(KEYS, 100),
+                ToTensord(KEYS),
+                CastToTyped(KEYS, dtype=torch.uint8),
+            ]
+        )
+        data = [{"image": im_fname, "label": seg_fname} for _ in range(12)]
+
+        # num workers = 0 for mac or gpu transforms
+        num_workers = 0 if sys.platform == "darwin" or torch.cuda.is_available() else 2
+
+        dataset = CacheDataset(data, transform=transform, progress=False)
+        loader = DataLoader(dataset, num_workers=num_workers, batch_size=5)
+
+        # set up engine
+        def _train_func(engine, batch):
+            self.assertTupleEqual(batch["image"].shape[1:], (1, 100, 100, 100))
+            engine.state.output = batch
+            engine.fire_event(IterationEvents.MODEL_COMPLETED)
+            return engine.state.output
+
+        engine = Engine(_train_func)
+        engine.register_events(*IterationEvents)
+
+        # set up testing handler
+        TransformInverter(
+            transform=transform,
+            loader=loader,
+            output_keys=["image", "label"],
+            batch_keys="label",
+            nearest_interp=True,
+            postfix="inverted1",
+            num_workers=0 if sys.platform == "darwin" or torch.cuda.is_available() else 2,
+        ).attach(engine)
+
+        # test different nearest interpolation values
+        TransformInverter(
+            transform=transform,
+            loader=loader,
+            output_keys=["image", "label"],
+            batch_keys="image",
+            nearest_interp=[True, False],
+            postfix="inverted2",
+            num_workers=0 if sys.platform == "darwin" or torch.cuda.is_available() else 2,
+        ).attach(engine)
+
+        engine.run(loader, max_epochs=1)
+        set_determinism(seed=None)
+        self.assertTupleEqual(engine.state.output["image"].shape, (2, 1, 100, 100, 100))
+        self.assertTupleEqual(engine.state.output["label"].shape, (2, 1, 100, 100, 100))
+        # check the nearest inerpolation mode
+        for i in engine.state.output["image_inverted1"] + engine.state.output["label_inverted1"]:
+            torch.testing.assert_allclose(i.to(torch.uint8).to(torch.float), i.to(torch.float))
+            self.assertTupleEqual(i.shape, (1, 100, 101, 107))
+        # check labels match
+        reverted = engine.state.output["label_inverted1"][-1].detach().cpu().numpy()[0].astype(np.int32)
+        original = LoadImaged(KEYS)(data[-1])["label"]
+        n_good = np.sum(np.isclose(reverted, original, atol=1e-3))
+        reverted_name = engine.state.output["label_meta_dict"]["filename_or_obj"][-1]
+        original_name = data[-1]["label"]
+        self.assertEqual(reverted_name, original_name)
+        print("invert diff", reverted.size - n_good)
+        # 25300: 2 workers (cpu, non-macos)
+        # 1812: 0 workers (gpu or macos)
+        # 1824: torch 1.5.1
+        self.assertTrue((reverted.size - n_good) in (25300, 1812, 1824), "diff. in 3 possible values")
+
+        # check the case that different items use different interpolation mode to invert transforms
+        for i in engine.state.output["image_inverted2"]:
+            # if the interpolation mode is nearest, accumulated diff should be smaller than 1
+            self.assertLess(torch.sum(i.to(torch.float) - i.to(torch.uint8).to(torch.float)).item(), 1.0)
+            self.assertTupleEqual(i.shape, (1, 100, 101, 107))
+
+        for i in engine.state.output["label_inverted2"]:
+            # if the interpolation mode is not nearest, accumulated diff should be greater than 10000
+            self.assertGreater(torch.sum(i.to(torch.float) - i.to(torch.uint8).to(torch.float)).item(), 10000.0)
+            self.assertTupleEqual(i.shape, (1, 100, 101, 107))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_handler_validation.py b/tests/test_handler_validation.py
index 11a51c7213..06f400109d 100644
--- a/tests/test_handler_validation.py
+++ b/tests/test_handler_validation.py
@@ -37,7 +37,7 @@ def _train_func(engine, batch):
         # set up testing handler
         val_data_loader = torch.utils.data.DataLoader(Dataset(data))
         evaluator = TestEvaluator(torch.device("cpu:0"), val_data_loader)
-        saver = ValidationHandler(evaluator, interval=2)
+        saver = ValidationHandler(interval=2, validator=evaluator)
         saver.attach(engine)
 
         engine.run(data, max_epochs=5)
diff --git a/tests/test_image_dataset.py b/tests/test_image_dataset.py
index d79a7d884c..ec2cf77cd8 100644
--- a/tests/test_image_dataset.py
+++ b/tests/test_image_dataset.py
@@ -17,12 +17,12 @@
 import numpy as np
 
 from monai.data import ImageDataset
-from monai.transforms import Randomizable
+from monai.transforms.transform import RandomizableTransform
 
 FILENAMES = ["test1.nii.gz", "test2.nii", "test3.nii.gz"]
 
 
-class RandTest(Randomizable):
+class RandTest(RandomizableTransform):
     """
     randomisable transform for testing.
     """
diff --git a/tests/test_integration_classification_2d.py b/tests/test_integration_classification_2d.py
index 4be59cba41..68493e4ffb 100644
--- a/tests/test_integration_classification_2d.py
+++ b/tests/test_integration_classification_2d.py
@@ -22,8 +22,19 @@
 from monai.apps import download_and_extract
 from monai.metrics import compute_roc_auc
 from monai.networks import eval_mode
-from monai.networks.nets import densenet121
-from monai.transforms import AddChannel, Compose, LoadImage, RandFlip, RandRotate, RandZoom, ScaleIntensity, ToTensor
+from monai.networks.nets import DenseNet121
+from monai.transforms import (
+    Activations,
+    AddChannel,
+    AsDiscrete,
+    Compose,
+    LoadImage,
+    RandFlip,
+    RandRotate,
+    RandZoom,
+    ScaleIntensity,
+    ToTensor,
+)
 from monai.utils import set_determinism
 from tests.testing_data.integration_answers import test_integration_value
 from tests.utils import DistTestCase, TimedCall, skip_if_quick
@@ -63,6 +74,8 @@ def run_training_test(root_dir, train_x, train_y, val_x, val_y, device="cuda:0",
     )
     train_transforms.set_random_state(1234)
     val_transforms = Compose([LoadImage(image_only=True), AddChannel(), ScaleIntensity(), ToTensor()])
+    act = Activations(softmax=True)
+    to_onehot = AsDiscrete(to_onehot=True, n_classes=len(np.unique(train_y)))
 
     # create train, val data loaders
     train_ds = MedNISTDataset(train_x, train_y, train_transforms)
@@ -71,7 +84,7 @@ def run_training_test(root_dir, train_x, train_y, val_x, val_y, device="cuda:0",
     val_ds = MedNISTDataset(val_x, val_y, val_transforms)
     val_loader = DataLoader(val_ds, batch_size=300, num_workers=num_workers)
 
-    model = densenet121(spatial_dims=2, in_channels=1, out_channels=len(np.unique(train_y))).to(device)
+    model = DenseNet121(spatial_dims=2, in_channels=1, out_channels=len(np.unique(train_y))).to(device)
     loss_function = torch.nn.CrossEntropyLoss()
     optimizer = torch.optim.Adam(model.parameters(), 1e-5)
     epoch_num = 4
@@ -110,10 +123,15 @@ def run_training_test(root_dir, train_x, train_y, val_x, val_y, device="cuda:0",
                     val_images, val_labels = val_data[0].to(device), val_data[1].to(device)
                     y_pred = torch.cat([y_pred, model(val_images)], dim=0)
                     y = torch.cat([y, val_labels], dim=0)
-                auc_metric = compute_roc_auc(y_pred, y, to_onehot_y=True, softmax=True)
-                metric_values.append(auc_metric)
+
+                # compute accuracy
                 acc_value = torch.eq(y_pred.argmax(dim=1), y)
                 acc_metric = acc_value.sum().item() / len(acc_value)
+                # compute AUC
+                y_pred = act(y_pred)
+                y = to_onehot(y)
+                auc_metric = compute_roc_auc(y_pred, y)
+                metric_values.append(auc_metric)
                 if auc_metric > best_metric:
                     best_metric = auc_metric
                     best_metric_epoch = epoch + 1
@@ -133,7 +151,7 @@ def run_inference_test(root_dir, test_x, test_y, device="cuda:0", num_workers=10
     val_ds = MedNISTDataset(test_x, test_y, val_transforms)
     val_loader = DataLoader(val_ds, batch_size=300, num_workers=num_workers)
 
-    model = densenet121(spatial_dims=2, in_channels=1, out_channels=len(np.unique(test_y))).to(device)
+    model = DenseNet121(spatial_dims=2, in_channels=1, out_channels=len(np.unique(test_y))).to(device)
 
     model_filename = os.path.join(root_dir, "best_metric_model.pth")
     model.load_state_dict(torch.load(model_filename))
diff --git a/tests/test_integration_sliding_window.py b/tests/test_integration_sliding_window.py
index c4d020276e..faec377586 100644
--- a/tests/test_integration_sliding_window.py
+++ b/tests/test_integration_sliding_window.py
@@ -84,7 +84,7 @@ def test_training(self):
             )
             output_image = nib.load(output_file).get_fdata()
             np.testing.assert_allclose(np.sum(output_image), 33621)
-            np.testing.assert_allclose(output_image.shape, (28, 25, 63, 1))
+            np.testing.assert_allclose(output_image.shape, (28, 25, 63))
 
 
 if __name__ == "__main__":
diff --git a/tests/test_integration_workflows.py b/tests/test_integration_workflows.py
index db7580bf86..00d097b2b6 100644
--- a/tests/test_integration_workflows.py
+++ b/tests/test_integration_workflows.py
@@ -160,7 +160,7 @@ def attach(self, engine):
             engine.add_event_handler(IterationEvents.FORWARD_COMPLETED, self._forward_completed)
             engine.add_event_handler(IterationEvents.LOSS_COMPLETED, self._loss_completed)
             engine.add_event_handler(IterationEvents.BACKWARD_COMPLETED, self._backward_completed)
-            engine.add_event_handler(IterationEvents.OPTIMIZER_COMPLETED, self._optimizer_completed)
+            engine.add_event_handler(IterationEvents.MODEL_COMPLETED, self._model_completed)
 
         def _forward_completed(self, engine):
             pass
@@ -171,7 +171,7 @@ def _loss_completed(self, engine):
         def _backward_completed(self, engine):
             pass
 
-        def _optimizer_completed(self, engine):
+        def _model_completed(self, engine):
             pass
 
     train_handlers = [
diff --git a/tests/test_integration_workflows_gan.py b/tests/test_integration_workflows_gan.py
index 73a9e69370..c54e8b01f2 100644
--- a/tests/test_integration_workflows_gan.py
+++ b/tests/test_integration_workflows_gan.py
@@ -145,7 +145,7 @@ def tearDown(self):
         set_determinism(seed=None)
         shutil.rmtree(self.data_dir)
 
-    @TimedCall(seconds=100, daemon=False)
+    @TimedCall(seconds=200, daemon=False)
     def test_training(self):
         torch.manual_seed(0)
 
diff --git a/tests/test_inverse.py b/tests/test_inverse.py
new file mode 100644
index 0000000000..306cd8a33d
--- /dev/null
+++ b/tests/test_inverse.py
@@ -0,0 +1,603 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+import sys
+import unittest
+from functools import partial
+from typing import TYPE_CHECKING, List, Tuple
+from unittest.case import skipUnless
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.data import CacheDataset, DataLoader, create_test_image_2d, create_test_image_3d
+from monai.data.inverse_batch_transform import BatchInverseTransform
+from monai.data.utils import decollate_batch
+from monai.networks.nets import UNet
+from monai.transforms import (
+    AddChanneld,
+    Affined,
+    BorderPadd,
+    CenterSpatialCropd,
+    Compose,
+    CropForegroundd,
+    DivisiblePadd,
+    Flipd,
+    InvertibleTransform,
+    LoadImaged,
+    Orientationd,
+    RandAffined,
+    RandAxisFlipd,
+    RandFlipd,
+    Randomizable,
+    RandRotate90d,
+    RandRotated,
+    RandSpatialCropd,
+    RandZoomd,
+    Resized,
+    ResizeWithPadOrCrop,
+    ResizeWithPadOrCropd,
+    Rotate90d,
+    Rotated,
+    Spacingd,
+    SpatialCropd,
+    SpatialPadd,
+    Zoomd,
+    allow_missing_keys_mode,
+    convert_inverse_interp_mode,
+)
+from monai.utils import first, get_seed, optional_import, set_determinism
+from monai.utils.enums import InverseKeys
+from tests.utils import make_nifti_image, make_rand_affine
+
+if TYPE_CHECKING:
+
+    has_nib = True
+else:
+    _, has_nib = optional_import("nibabel")
+
+KEYS = ["image", "label"]
+
+TESTS: List[Tuple] = []
+
+# For pad, start with odd/even images and add odd/even amounts
+for name in ("1D even", "1D odd"):
+    for val in (3, 4):
+        for t in (
+            partial(SpatialPadd, spatial_size=val, method="symmetric"),
+            partial(SpatialPadd, spatial_size=val, method="end"),
+            partial(BorderPadd, spatial_border=[val, val + 1]),
+            partial(DivisiblePadd, k=val),
+            partial(ResizeWithPadOrCropd, spatial_size=20 + val),
+            partial(CenterSpatialCropd, roi_size=10 + val),
+            partial(CropForegroundd, source_key="label"),
+            partial(SpatialCropd, roi_center=10, roi_size=10 + val),
+            partial(SpatialCropd, roi_center=11, roi_size=10 + val),
+            partial(SpatialCropd, roi_start=val, roi_end=17),
+            partial(SpatialCropd, roi_start=val, roi_end=16),
+            partial(RandSpatialCropd, roi_size=12 + val),
+            partial(ResizeWithPadOrCropd, spatial_size=21 - val),
+        ):
+            TESTS.append((t.func.__name__ + name, name, 0, t(KEYS)))  # type: ignore
+
+# non-sensical tests: crop bigger or pad smaller or -ve values
+for t in (
+    partial(DivisiblePadd, k=-3),
+    partial(CenterSpatialCropd, roi_size=-3),
+    partial(RandSpatialCropd, roi_size=-3),
+    partial(SpatialPadd, spatial_size=15),
+    partial(BorderPadd, spatial_border=[15, 16]),
+    partial(CenterSpatialCropd, roi_size=30),
+    partial(SpatialCropd, roi_center=10, roi_size=100),
+    partial(SpatialCropd, roi_start=3, roi_end=100),
+):
+    TESTS.append((t.func.__name__ + "bad 1D even", "1D even", 0, t(KEYS)))  # type: ignore
+
+TESTS.append(
+    (
+        "SpatialPadd (x2) 2d",
+        "2D",
+        0,
+        SpatialPadd(KEYS, spatial_size=[111, 113], method="end"),
+        SpatialPadd(KEYS, spatial_size=[118, 117]),
+    )
+)
+
+TESTS.append(
+    (
+        "SpatialPadd 3d",
+        "3D",
+        0,
+        SpatialPadd(KEYS, spatial_size=[112, 113, 116]),
+    )
+)
+
+TESTS.append(
+    (
+        "SpatialCropd 2d",
+        "2D",
+        0,
+        SpatialCropd(KEYS, [49, 51], [90, 89]),
+    )
+)
+
+TESTS.append(
+    (
+        "SpatialCropd 3d",
+        "3D",
+        0,
+        SpatialCropd(KEYS, roi_slices=[slice(s, e) for s, e in zip([None, None, -99], [None, -2, None])]),
+    )
+)
+
+TESTS.append(
+    (
+        "SpatialCropd 2d",
+        "2D",
+        0,
+        SpatialCropd(KEYS, [49, 51], [390, 89]),
+    )
+)
+
+TESTS.append(
+    (
+        "SpatialCropd 3d",
+        "3D",
+        0,
+        SpatialCropd(KEYS, [49, 51, 44], [90, 89, 93]),
+    )
+)
+
+TESTS.append(("RandSpatialCropd 2d", "2D", 0, RandSpatialCropd(KEYS, [96, 93], True, False)))
+
+TESTS.append(("RandSpatialCropd 3d", "3D", 0, RandSpatialCropd(KEYS, [96, 93, 92], False, False)))
+
+TESTS.append(
+    (
+        "BorderPadd 2d",
+        "2D",
+        0,
+        BorderPadd(KEYS, [3, 7, 2, 5]),
+    )
+)
+
+TESTS.append(
+    (
+        "BorderPadd 2d",
+        "2D",
+        0,
+        BorderPadd(KEYS, [3, 7]),
+    )
+)
+
+TESTS.append(
+    (
+        "BorderPadd 3d",
+        "3D",
+        0,
+        BorderPadd(KEYS, [4]),
+    )
+)
+
+TESTS.append(
+    (
+        "DivisiblePadd 2d",
+        "2D",
+        0,
+        DivisiblePadd(KEYS, k=4),
+    )
+)
+
+TESTS.append(
+    (
+        "DivisiblePadd 3d",
+        "3D",
+        0,
+        DivisiblePadd(KEYS, k=[4, 8, 11]),
+    )
+)
+
+
+TESTS.append(
+    (
+        "CenterSpatialCropd 2d",
+        "2D",
+        0,
+        CenterSpatialCropd(KEYS, roi_size=95),
+    )
+)
+
+TESTS.append(
+    (
+        "CenterSpatialCropd 3d",
+        "3D",
+        0,
+        CenterSpatialCropd(KEYS, roi_size=[95, 97, 98]),
+    )
+)
+
+TESTS.append(("CropForegroundd 2d", "2D", 0, CropForegroundd(KEYS, source_key="label", margin=2)))
+
+TESTS.append(("CropForegroundd 3d", "3D", 0, CropForegroundd(KEYS, source_key="label")))
+
+
+TESTS.append(("ResizeWithPadOrCropd 3d", "3D", 0, ResizeWithPadOrCropd(KEYS, [201, 150, 105])))
+
+TESTS.append(
+    (
+        "Flipd 3d",
+        "3D",
+        0,
+        Flipd(KEYS, [1, 2]),
+    )
+)
+
+TESTS.append(
+    (
+        "Flipd 3d",
+        "3D",
+        0,
+        Flipd(KEYS, [1, 2]),
+    )
+)
+
+TESTS.append(
+    (
+        "RandFlipd 3d",
+        "3D",
+        0,
+        RandFlipd(KEYS, 1, [1, 2]),
+    )
+)
+
+TESTS.append(
+    (
+        "RandAxisFlipd 3d",
+        "3D",
+        0,
+        RandAxisFlipd(KEYS, 1),
+    )
+)
+
+for acc in [True, False]:
+    TESTS.append(
+        (
+            "Orientationd 3d",
+            "3D",
+            0,
+            Orientationd(KEYS, "RAS", as_closest_canonical=acc),
+        )
+    )
+
+TESTS.append(
+    (
+        "Rotate90d 2d",
+        "2D",
+        0,
+        Rotate90d(KEYS),
+    )
+)
+
+TESTS.append(
+    (
+        "Rotate90d 3d",
+        "3D",
+        0,
+        Rotate90d(KEYS, k=2, spatial_axes=(1, 2)),
+    )
+)
+
+TESTS.append(
+    (
+        "RandRotate90d 3d",
+        "3D",
+        0,
+        RandRotate90d(KEYS, prob=1, spatial_axes=(1, 2)),
+    )
+)
+
+TESTS.append(("Spacingd 3d", "3D", 3e-2, Spacingd(KEYS, [0.5, 0.7, 0.9], diagonal=False)))
+
+TESTS.append(("Resized 2d", "2D", 2e-1, Resized(KEYS, [50, 47])))
+
+TESTS.append(("Resized 3d", "3D", 5e-2, Resized(KEYS, [201, 150, 78])))
+
+
+TESTS.append(
+    (
+        "Zoomd 1d",
+        "1D odd",
+        0,
+        Zoomd(KEYS, zoom=2, keep_size=False),
+    )
+)
+
+TESTS.append(
+    (
+        "Zoomd 2d",
+        "2D",
+        2e-1,
+        Zoomd(KEYS, zoom=0.9),
+    )
+)
+
+TESTS.append(
+    (
+        "Zoomd 3d",
+        "3D",
+        3e-2,
+        Zoomd(KEYS, zoom=[2.5, 1, 3], keep_size=False),
+    )
+)
+
+TESTS.append(("RandZoom 3d", "3D", 9e-2, RandZoomd(KEYS, 1, [0.5, 0.6, 0.9], [1.1, 1, 1.05], keep_size=True)))
+
+TESTS.append(
+    (
+        "RandRotated, prob 0",
+        "2D",
+        0,
+        RandRotated(KEYS, prob=0),
+    )
+)
+
+TESTS.append(
+    (
+        "Rotated 2d",
+        "2D",
+        8e-2,
+        Rotated(KEYS, random.uniform(np.pi / 6, np.pi), keep_size=True, align_corners=False),
+    )
+)
+
+TESTS.append(
+    (
+        "Rotated 3d",
+        "3D",
+        1e-1,
+        Rotated(KEYS, [random.uniform(np.pi / 6, np.pi) for _ in range(3)], True),  # type: ignore
+    )
+)
+
+TESTS.append(
+    (
+        "RandRotated 3d",
+        "3D",
+        1e-1,
+        RandRotated(KEYS, *[random.uniform(np.pi / 6, np.pi) for _ in range(3)], 1),  # type: ignore
+    )
+)
+
+TESTS.append(
+    (
+        "Affine 3d",
+        "3D",
+        1e-1,
+        Affined(
+            KEYS,
+            spatial_size=[155, 179, 192],
+            rotate_params=[np.pi / 6, -np.pi / 5, np.pi / 7],
+            shear_params=[0.5, 0.5],
+            translate_params=[10, 5, -4],
+            scale_params=[0.8, 1.3],
+        ),
+    )
+)
+
+TESTS.append(
+    (
+        "RandAffine 3d",
+        "3D",
+        1e-1,
+        RandAffined(
+            KEYS,
+            [155, 179, 192],
+            prob=1,
+            padding_mode="zeros",
+            rotate_range=[np.pi / 6, -np.pi / 5, np.pi / 7],
+            shear_range=[(0.5, 0.5)],
+            translate_range=[10, 5, -4],
+            scale_range=[(0.8, 1.2), (0.9, 1.3)],
+        ),
+    )
+)
+
+TESTS_COMPOSE_X2 = [(t[0] + " Compose", t[1], t[2], Compose(Compose(t[3:]))) for t in TESTS]
+
+TESTS = TESTS + TESTS_COMPOSE_X2  # type: ignore
+
+
+def no_collation(x):
+    return x
+
+
+class TestInverse(unittest.TestCase):
+    """Test inverse methods.
+
+    If tests are failing, the following function might be useful for displaying
+    `x`, `fx`, `f⁻¹fx` and `x - f⁻¹fx`.
+
+    .. code-block:: python
+
+        def plot_im(orig, fwd_bck, fwd):
+            import matplotlib.pyplot as plt
+            diff_orig_fwd_bck = orig - fwd_bck
+            ims_to_show = [orig, fwd, fwd_bck, diff_orig_fwd_bck]
+            titles = ["x", "fx", "f⁻¹fx", "x - f⁻¹fx"]
+            fig, axes = plt.subplots(1, 4, gridspec_kw={"width_ratios": [i.shape[1] for i in ims_to_show]})
+            vmin = min(np.array(i).min() for i in [orig, fwd_bck, fwd])
+            vmax = max(np.array(i).max() for i in [orig, fwd_bck, fwd])
+            for im, title, ax in zip(ims_to_show, titles, axes):
+                _vmin, _vmax = (vmin, vmax) if id(im) != id(diff_orig_fwd_bck) else (None, None)
+                im = np.squeeze(np.array(im))
+                while im.ndim > 2:
+                    im = im[..., im.shape[-1] // 2]
+                im_show = ax.imshow(np.squeeze(im), vmin=_vmin, vmax=_vmax)
+                ax.set_title(title, fontsize=25)
+                ax.axis("off")
+                fig.colorbar(im_show, ax=ax)
+            plt.show()
+
+    This can then be added to the exception:
+
+    .. code-block:: python
+
+        except AssertionError:
+            print(
+                f"Failed: {name}. Mean diff = {mean_diff} (expected <= {acceptable_diff}), unmodified diff: {unmodded_diff}"
+            )
+            if orig[0].ndim > 1:
+                plot_im(orig, fwd_bck, unmodified)
+    """
+
+    def setUp(self):
+        if not has_nib:
+            self.skipTest("nibabel required for test_inverse")
+
+        set_determinism(seed=0)
+
+        self.all_data = {}
+
+        affine = make_rand_affine()
+        affine[0] *= 2
+
+        for size in [10, 11]:
+            # pad 5 onto both ends so that cropping can be lossless
+            im_1d = np.pad(np.arange(size), 5)[None]
+            name = "1D even" if size % 2 == 0 else "1D odd"
+            self.all_data[name] = {
+                "image": np.array(im_1d, copy=True),
+                "label": np.array(im_1d, copy=True),
+                "other": np.array(im_1d, copy=True),
+            }
+
+        im_2d_fname, seg_2d_fname = [make_nifti_image(i) for i in create_test_image_2d(101, 100)]
+        im_3d_fname, seg_3d_fname = [make_nifti_image(i, affine) for i in create_test_image_3d(100, 101, 107)]
+
+        load_ims = Compose([LoadImaged(KEYS), AddChanneld(KEYS)])
+        self.all_data["2D"] = load_ims({"image": im_2d_fname, "label": seg_2d_fname})
+        self.all_data["3D"] = load_ims({"image": im_3d_fname, "label": seg_3d_fname})
+
+    def tearDown(self):
+        set_determinism(seed=None)
+
+    def check_inverse(self, name, keys, orig_d, fwd_bck_d, unmodified_d, acceptable_diff):
+        for key in keys:
+            orig = orig_d[key]
+            fwd_bck = fwd_bck_d[key]
+            if isinstance(fwd_bck, torch.Tensor):
+                fwd_bck = fwd_bck.cpu().numpy()
+            unmodified = unmodified_d[key]
+            if isinstance(orig, np.ndarray):
+                mean_diff = np.mean(np.abs(orig - fwd_bck))
+                resized = ResizeWithPadOrCrop(orig.shape[1:])(unmodified)
+                if isinstance(resized, torch.Tensor):
+                    resized = resized.detach().cpu().numpy()
+                unmodded_diff = np.mean(np.abs(orig - resized))
+                try:
+                    self.assertLessEqual(mean_diff, acceptable_diff)
+                except AssertionError:
+                    print(
+                        f"Failed: {name}. Mean diff = {mean_diff} (expected <= {acceptable_diff}), unmodified diff: {unmodded_diff}"
+                    )
+                    if orig[0].ndim == 1:
+                        print("orig", orig[0])
+                        print("fwd_bck", fwd_bck[0])
+                        print("unmod", unmodified[0])
+                    raise
+
+    @parameterized.expand(TESTS)
+    def test_inverse(self, _, data_name, acceptable_diff, *transforms):
+        name = _
+
+        data = self.all_data[data_name]
+
+        forwards = [data.copy()]
+
+        # Apply forwards
+        for t in transforms:
+            if isinstance(t, Randomizable):
+                t.set_random_state(seed=get_seed())
+            forwards.append(t(forwards[-1]))
+
+        # Apply inverses
+        fwd_bck = forwards[-1].copy()
+        for i, t in enumerate(reversed(transforms)):
+            if isinstance(t, InvertibleTransform):
+                fwd_bck = t.inverse(fwd_bck)
+                self.check_inverse(name, data.keys(), forwards[-i - 2], fwd_bck, forwards[-1], acceptable_diff)
+
+    # skip this test if multiprocessing uses 'spawn', as the check is only basic anyway
+    @skipUnless(torch.multiprocessing.get_start_method(allow_none=False) == "spawn", "requires spawn")
+    def test_fail(self):
+
+        t1 = SpatialPadd("image", [10, 5])
+        data = t1(self.all_data["2D"])
+
+        # Check that error is thrown when inverse are used out of order.
+        t2 = ResizeWithPadOrCropd("image", [10, 5])
+        with self.assertRaises(RuntimeError):
+            t2.inverse(data)
+
+    def test_inverse_inferred_seg(self):
+
+        test_data = []
+        for _ in range(20):
+            image, label = create_test_image_2d(100, 101)
+            test_data.append({"image": image, "label": label.astype(np.float32)})
+
+        batch_size = 10
+        # num workers = 0 for mac
+        num_workers = 2 if sys.platform != "darwin" else 0
+        transforms = Compose([AddChanneld(KEYS), SpatialPadd(KEYS, (150, 153)), CenterSpatialCropd(KEYS, (110, 99))])
+        num_invertible_transforms = sum(1 for i in transforms.transforms if isinstance(i, InvertibleTransform))
+
+        dataset = CacheDataset(test_data, transform=transforms, progress=False)
+        loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
+
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        model = UNet(
+            dimensions=2,
+            in_channels=1,
+            out_channels=1,
+            channels=(2, 4),
+            strides=(2,),
+        ).to(device)
+
+        data = first(loader)
+        labels = data["label"].to(device)
+        segs = model(labels).detach().cpu()
+        label_transform_key = "label" + InverseKeys.KEY_SUFFIX
+        segs_dict = {"label": segs, label_transform_key: data[label_transform_key]}
+
+        segs_dict_decollated = decollate_batch(segs_dict)
+        # inverse of individual segmentation
+        seg_dict = first(segs_dict_decollated)
+        # test to convert interpolation mode for 1 data of model output batch
+        convert_inverse_interp_mode(seg_dict, mode="nearest", align_corners=None)
+
+        with allow_missing_keys_mode(transforms):
+            inv_seg = transforms.inverse(seg_dict)["label"]
+        self.assertEqual(len(data["label_transforms"]), num_invertible_transforms)
+        self.assertEqual(len(seg_dict["label_transforms"]), num_invertible_transforms)
+        self.assertEqual(inv_seg.shape[1:], test_data[0]["label"].shape)
+
+        # Inverse of batch
+        batch_inverter = BatchInverseTransform(transforms, loader, collate_fn=no_collation)
+        with allow_missing_keys_mode(transforms):
+            inv_batch = batch_inverter(segs_dict)
+        self.assertEqual(inv_batch[0]["label"].shape[1:], test_data[0]["label"].shape)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_inverse_collation.py b/tests/test_inverse_collation.py
new file mode 100644
index 0000000000..c302e04017
--- /dev/null
+++ b/tests/test_inverse_collation.py
@@ -0,0 +1,132 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import unittest
+from typing import TYPE_CHECKING
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.data import CacheDataset, DataLoader, create_test_image_2d, create_test_image_3d, pad_list_data_collate
+from monai.transforms import (
+    AddChanneld,
+    Compose,
+    LoadImaged,
+    RandAffined,
+    RandAxisFlipd,
+    RandFlipd,
+    RandRotate90d,
+    RandRotated,
+    RandZoomd,
+    ResizeWithPadOrCropd,
+    ToTensord,
+)
+from monai.utils import optional_import, set_determinism
+from tests.utils import make_nifti_image
+
+if TYPE_CHECKING:
+
+    has_nib = True
+else:
+    _, has_nib = optional_import("nibabel")
+
+KEYS = ["image", "label"]
+
+TESTS_3D = [
+    (t.__class__.__name__ + (" pad_list_data_collate" if collate_fn else " default_collate"), t, collate_fn, 3)
+    for collate_fn in [None, pad_list_data_collate]
+    for t in [
+        RandFlipd(keys=KEYS, prob=0.5, spatial_axis=[1, 2]),
+        RandAxisFlipd(keys=KEYS, prob=0.5),
+        RandRotate90d(keys=KEYS, spatial_axes=(1, 2)),
+        RandZoomd(keys=KEYS, prob=0.5, min_zoom=0.5, max_zoom=1.1, keep_size=True),
+        RandRotated(keys=KEYS, prob=0.5, range_x=np.pi),
+        RandAffined(
+            keys=KEYS,
+            prob=0.5,
+            rotate_range=np.pi,
+            device=torch.device("cuda" if torch.cuda.is_available() else "cpu"),
+            as_tensor_output=False,
+        ),
+    ]
+]
+
+TESTS_2D = [
+    (t.__class__.__name__ + (" pad_list_data_collate" if collate_fn else " default_collate"), t, collate_fn, 2)
+    for collate_fn in [None, pad_list_data_collate]
+    for t in [
+        RandFlipd(keys=KEYS, prob=0.5, spatial_axis=[1]),
+        RandAxisFlipd(keys=KEYS, prob=0.5),
+        RandRotate90d(keys=KEYS, prob=0.5, spatial_axes=(0, 1)),
+        RandZoomd(keys=KEYS, prob=0.5, min_zoom=0.5, max_zoom=1.1, keep_size=True),
+        RandRotated(keys=KEYS, prob=0.5, range_x=np.pi),
+        RandAffined(
+            keys=KEYS,
+            prob=0.5,
+            rotate_range=np.pi,
+            device=torch.device("cuda" if torch.cuda.is_available() else "cpu"),
+            as_tensor_output=False,
+        ),
+    ]
+]
+
+
+class TestInverseCollation(unittest.TestCase):
+    """Test collation for of random transformations with prob == 0 and 1."""
+
+    def setUp(self):
+        if not has_nib:
+            self.skipTest("nibabel required for test_inverse")
+
+        set_determinism(seed=0)
+
+        b_size = 11
+        im_fname, seg_fname = [make_nifti_image(i) for i in create_test_image_3d(101, 100, 107)]
+        load_ims = Compose([LoadImaged(KEYS), AddChanneld(KEYS)])
+        self.data_3d = [load_ims({"image": im_fname, "label": seg_fname}) for _ in range(b_size)]
+
+        b_size = 8
+        im_fname, seg_fname = [make_nifti_image(i) for i in create_test_image_2d(62, 37, rad_max=10)]
+        load_ims = Compose([LoadImaged(KEYS), AddChanneld(KEYS)])
+        self.data_2d = [load_ims({"image": im_fname, "label": seg_fname}) for _ in range(b_size)]
+
+        self.batch_size = 7
+
+    def tearDown(self):
+        set_determinism(seed=None)
+
+    @parameterized.expand(TESTS_2D + TESTS_3D)
+    def test_collation(self, _, transform, collate_fn, ndim):
+        if ndim == 3:
+            data = self.data_3d
+        else:
+            data = self.data_2d
+        if collate_fn:
+            modified_transform = transform
+        else:
+            modified_transform = Compose([transform, ResizeWithPadOrCropd(KEYS, 100), ToTensord(KEYS)])
+
+        # num workers = 0 for mac or gpu transforms
+        num_workers = 0 if sys.platform == "darwin" or torch.cuda.is_available() else 2
+
+        dataset = CacheDataset(data, transform=modified_transform, progress=False)
+        loader = DataLoader(dataset, num_workers, batch_size=self.batch_size, collate_fn=collate_fn)
+
+        for item in loader:
+            np.testing.assert_array_equal(
+                item["image_transforms"][0]["do_transforms"], item["label_transforms"][0]["do_transforms"]
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_lesion_froc.py b/tests/test_lesion_froc.py
new file mode 100644
index 0000000000..2454de88fa
--- /dev/null
+++ b/tests/test_lesion_froc.py
@@ -0,0 +1,332 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+from unittest import skipUnless
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.apps.pathology.metrics import LesionFROC
+from monai.utils import optional_import
+
+_, has_cucim = optional_import("cucim")
+_, has_skimage = optional_import("skimage.measure")
+_, has_sp = optional_import("scipy.ndimage")
+PILImage, has_pil = optional_import("PIL.Image")
+
+
+def save_as_tif(filename, array):
+    array = array[::-1, ...]  # Upside-down
+    img = PILImage.fromarray(array)
+    if not filename.endswith(".tif"):
+        filename += ".tif"
+    img.save(os.path.join("tests", "testing_data", filename))
+
+
+def around(val, interval=3):
+    return slice(val - interval, val + interval)
+
+
+# mask and prediction image size
+HEIGHT = 101
+WIDTH = 800
+
+
+def prepare_test_data():
+    # -------------------------------------
+    # Ground Truth - Binary Masks
+    # -------------------------------------
+    # ground truth with no tumor
+    ground_truth = np.zeros((HEIGHT, WIDTH), dtype=np.uint8)
+    save_as_tif("temp_ground_truth_0", ground_truth)
+
+    # ground truth with one tumor
+    ground_truth[around(HEIGHT // 2), around(1 * WIDTH // 7)] = 1
+    save_as_tif("temp_ground_truth_1", ground_truth)
+
+    # ground truth with two tumors
+    ground_truth[around(HEIGHT // 2), around(2 * WIDTH // 7)] = 1
+    save_as_tif("temp_ground_truth_2", ground_truth)
+
+    # ground truth with three tumors
+    ground_truth[around(HEIGHT // 2), around(3 * WIDTH // 7)] = 1
+    save_as_tif("temp_ground_truth_3", ground_truth)
+
+    # ground truth with four tumors
+    ground_truth[around(HEIGHT // 2), around(4 * WIDTH // 7)] = 1
+    save_as_tif("temp_ground_truth_4", ground_truth)
+
+    # -------------------------------------
+    # predictions - Probability Maps
+    # -------------------------------------
+
+    # prediction with no tumor
+    prob_map = np.zeros((HEIGHT, WIDTH))
+    np.save("./tests/testing_data/temp_prob_map_0_0.npy", prob_map)
+
+    # prediction with one incorrect tumor
+    prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
+    np.save("./tests/testing_data/temp_prob_map_0_1.npy", prob_map)
+
+    # prediction with correct first tumors and an incorrect tumor
+    prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
+    np.save("./tests/testing_data/temp_prob_map_1_1.npy", prob_map)
+
+    # prediction with correct firt two tumors and an incorrect tumor
+    prob_map[HEIGHT // 2, 2 * WIDTH // 7] = 0.8
+    np.save("./tests/testing_data/temp_prob_map_2_1.npy", prob_map)
+
+    # prediction with two incorrect tumors
+    prob_map = np.zeros((HEIGHT, WIDTH))
+    prob_map[HEIGHT // 2, 5 * WIDTH // 7] = 0.6
+    prob_map[HEIGHT // 2, 6 * WIDTH // 7] = 0.4
+    np.save("./tests/testing_data/temp_prob_map_0_2.npy", prob_map)
+
+    # prediction with correct first tumors and two incorrect tumors
+    prob_map[HEIGHT // 2, 1 * WIDTH // 7] = 0.8
+    np.save("./tests/testing_data/temp_prob_map_1_2.npy", prob_map)
+
+
+TEST_CASE_0 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_0_0.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_0.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    np.nan,
+]
+
+
+TEST_CASE_1 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_0_0.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    0.0,
+]
+
+TEST_CASE_2 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    1.0,
+]
+
+TEST_CASE_3 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_2_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    1.0,
+]
+
+
+TEST_CASE_4 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_2_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    1.0,
+]
+
+TEST_CASE_5 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            }
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    0.5,
+]
+
+
+TEST_CASE_6 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    2.0 / 3.0,
+]
+
+TEST_CASE_7 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_3.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    0.4,
+]
+
+TEST_CASE_8 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_0_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_1.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_3.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    1.0 / 3.0,
+]
+
+TEST_CASE_9 = [
+    {
+        "data": [
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_0_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_4.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_1.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_3.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+            {
+                "prob_map": "./tests/testing_data/temp_prob_map_1_2.npy",
+                "tumor_mask": "./tests/testing_data/temp_ground_truth_2.tif",
+                "level": 0,
+                "pixel_spacing": 1,
+            },
+        ],
+        "grow_distance": 2,
+        "itc_diameter": 0,
+    },
+    2.0 / 9.0,
+]
+
+
+class TestEvaluateTumorFROC(unittest.TestCase):
+    @skipUnless(has_cucim, "Requires cucim")
+    @skipUnless(has_skimage, "Requires skimage")
+    @skipUnless(has_sp, "Requires scipy")
+    @skipUnless(has_pil, "Requires PIL")
+    def setUp(self):
+        prepare_test_data()
+
+    @parameterized.expand(
+        [
+            TEST_CASE_0,
+            TEST_CASE_1,
+            TEST_CASE_2,
+            TEST_CASE_3,
+            TEST_CASE_4,
+            TEST_CASE_5,
+            TEST_CASE_6,
+            TEST_CASE_7,
+            TEST_CASE_8,
+            TEST_CASE_9,
+        ]
+    )
+    def test_read_patches_cucim(self, input_parameters, expected):
+        froc = LesionFROC(**input_parameters)
+        froc_score = froc.evaluate()
+        if np.isnan(expected):
+            self.assertTrue(np.isnan(froc_score))
+        else:
+            self.assertAlmostEqual(froc_score, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_load_decathlon_datalist.py b/tests/test_load_decathlon_datalist.py
index 90b9d3ab03..fe7ff6f8a2 100644
--- a/tests/test_load_decathlon_datalist.py
+++ b/tests/test_load_decathlon_datalist.py
@@ -96,6 +96,31 @@ def test_seg_no_labels(self):
             result = load_decathlon_datalist(file_path, True, "test", tempdir)
             self.assertEqual(result[0]["image"], os.path.join(tempdir, "spleen_15.nii.gz"))
 
+    def test_additional_items(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            with open(os.path.join(tempdir, "mask31.txt"), "w") as f:
+                f.write("spleen31 mask")
+
+            test_data = {
+                "name": "Spleen",
+                "description": "Spleen Segmentation",
+                "labels": {"0": "background", "1": "spleen"},
+                "training": [
+                    {"image": "spleen_19.nii.gz", "label": "spleen_19.nii.gz", "mask": "spleen mask"},
+                    {"image": "spleen_31.nii.gz", "label": "spleen_31.nii.gz", "mask": "mask31.txt"},
+                ],
+                "test": ["spleen_15.nii.gz", "spleen_23.nii.gz"],
+            }
+            json_str = json.dumps(test_data)
+            file_path = os.path.join(tempdir, "test_data.json")
+            with open(file_path, "w") as json_file:
+                json_file.write(json_str)
+            result = load_decathlon_datalist(file_path, True, "training", tempdir)
+            self.assertEqual(result[0]["image"], os.path.join(tempdir, "spleen_19.nii.gz"))
+            self.assertEqual(result[0]["label"], os.path.join(tempdir, "spleen_19.nii.gz"))
+            self.assertEqual(result[1]["mask"], os.path.join(tempdir, "mask31.txt"))
+            self.assertEqual(result[0]["mask"], "spleen mask")
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_local_normalized_cross_correlation_loss.py b/tests/test_local_normalized_cross_correlation_loss.py
index cf8566a559..31954e727b 100644
--- a/tests/test_local_normalized_cross_correlation_loss.py
+++ b/tests/test_local_normalized_cross_correlation_loss.py
@@ -1,4 +1,4 @@
-# Copyright 2020 MONAI Consortium
+# Copyright 2020 - 2021 MONAI Consortium
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -17,60 +17,89 @@
 
 from monai.losses.image_dissimilarity import LocalNormalizedCrossCorrelationLoss
 
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
 TEST_CASES = [
     [
-        {"in_channels": 1, "ndim": 1, "kernel_type": "rectangular", "reduction": "sum"},
+        {"ndim": 1, "kernel_type": "rectangular", "reduction": "sum"},
         {
-            "pred": torch.arange(0, 3).reshape(1, 1, -1).to(torch.float),
-            "target": torch.arange(0, 3).reshape(1, 1, -1).to(torch.float),
+            "pred": torch.arange(0, 3).reshape(1, 1, -1).to(dtype=torch.float, device=device),
+            "target": torch.arange(0, 3).reshape(1, 1, -1).to(dtype=torch.float, device=device),
         },
         -1.0 * 3,
     ],
     [
-        {"in_channels": 1, "ndim": 1, "kernel_type": "rectangular"},
+        {"ndim": 1, "kernel_type": "rectangular"},
         {
-            "pred": torch.arange(0, 3).reshape(1, 1, -1).to(torch.float),
-            "target": torch.arange(0, 3).reshape(1, 1, -1).to(torch.float),
+            "pred": torch.arange(0, 3).reshape(1, 1, -1).to(dtype=torch.float, device=device),
+            "target": torch.arange(0, 3).reshape(1, 1, -1).to(dtype=torch.float, device=device),
         },
         -1.0,
     ],
     [
-        {"in_channels": 1, "ndim": 2, "kernel_type": "rectangular"},
+        {"ndim": 2, "kernel_type": "rectangular"},
         {
-            "pred": torch.arange(0, 3).reshape(1, 1, -1, 1).expand(1, 1, 3, 3).to(torch.float),
-            "target": torch.arange(0, 3).reshape(1, 1, -1, 1).expand(1, 1, 3, 3).to(torch.float),
+            "pred": torch.arange(0, 3).reshape(1, 1, -1, 1).expand(1, 1, 3, 3).to(dtype=torch.float, device=device),
+            "target": torch.arange(0, 3).reshape(1, 1, -1, 1).expand(1, 1, 3, 3).to(dtype=torch.float, device=device),
         },
         -1.0,
     ],
     [
-        {"in_channels": 1, "ndim": 3, "kernel_type": "rectangular"},
+        {"ndim": 3, "kernel_type": "rectangular"},
         {
-            "pred": torch.arange(0, 3).reshape(1, 1, -1, 1, 1).expand(1, 1, 3, 3, 3).to(torch.float),
-            "target": torch.arange(0, 3).reshape(1, 1, -1, 1, 1).expand(1, 1, 3, 3, 3).to(torch.float),
+            "pred": torch.arange(0, 3)
+            .reshape(1, 1, -1, 1, 1)
+            .expand(1, 1, 3, 3, 3)
+            .to(dtype=torch.float, device=device),
+            "target": torch.arange(0, 3)
+            .reshape(1, 1, -1, 1, 1)
+            .expand(1, 1, 3, 3, 3)
+            .to(dtype=torch.float, device=device),
         },
         -1.0,
     ],
     [
-        {"in_channels": 3, "ndim": 3, "kernel_type": "rectangular"},
+        {"ndim": 3, "kernel_type": "rectangular"},
         {
-            "pred": torch.arange(0, 3).reshape(1, 1, -1, 1, 1).expand(1, 3, 3, 3, 3).to(torch.float),
-            "target": torch.arange(0, 3).reshape(1, 1, -1, 1, 1).expand(1, 3, 3, 3, 3).to(torch.float) ** 2,
+            "pred": torch.arange(0, 3)
+            .reshape(1, 1, -1, 1, 1)
+            .expand(1, 3, 3, 3, 3)
+            .to(dtype=torch.float, device=device),
+            "target": torch.arange(0, 3)
+            .reshape(1, 1, -1, 1, 1)
+            .expand(1, 3, 3, 3, 3)
+            .to(dtype=torch.float, device=device)
+            ** 2,
         },
         -0.95801723,
     ],
     [
-        {"in_channels": 3, "ndim": 3, "kernel_type": "triangular", "kernel_size": 5},
+        {"ndim": 3, "kernel_type": "triangular", "kernel_size": 5},
         {
-            "pred": torch.arange(0, 5).reshape(1, 1, -1, 1, 1).expand(1, 3, 5, 5, 5).to(torch.float),
-            "target": torch.arange(0, 5).reshape(1, 1, -1, 1, 1).expand(1, 3, 5, 5, 5).to(torch.float) ** 2,
+            "pred": torch.arange(0, 5)
+            .reshape(1, 1, -1, 1, 1)
+            .expand(1, 3, 5, 5, 5)
+            .to(dtype=torch.float, device=device),
+            "target": torch.arange(0, 5)
+            .reshape(1, 1, -1, 1, 1)
+            .expand(1, 3, 5, 5, 5)
+            .to(dtype=torch.float, device=device)
+            ** 2,
         },
         -0.918672,
     ],
     [
-        {"in_channels": 3, "ndim": 3, "kernel_type": "gaussian"},
+        {"ndim": 3, "kernel_type": "gaussian"},
         {
-            "pred": torch.arange(0, 3).reshape(1, 1, -1, 1, 1).expand(1, 3, 3, 3, 3).to(torch.float),
-            "target": torch.arange(0, 3).reshape(1, 1, -1, 1, 1).expand(1, 3, 3, 3, 3).to(torch.float) ** 2,
+            "pred": torch.arange(0, 3)
+            .reshape(1, 1, -1, 1, 1)
+            .expand(1, 3, 3, 3, 3)
+            .to(dtype=torch.float, device=device),
+            "target": torch.arange(0, 3)
+            .reshape(1, 1, -1, 1, 1)
+            .expand(1, 3, 3, 3, 3)
+            .to(dtype=torch.float, device=device)
+            ** 2,
         },
         -0.95406944,
     ],
@@ -84,30 +113,39 @@ def test_shape(self, input_param, input_data, expected_val):
         np.testing.assert_allclose(result.detach().cpu().numpy(), expected_val, rtol=1e-5)
 
     def test_ill_shape(self):
-        loss = LocalNormalizedCrossCorrelationLoss(in_channels=3, ndim=3)
-        # in_channel unmatch
-        with self.assertRaisesRegex(ValueError, ""):
-            loss.forward(torch.ones((1, 2, 3, 3, 3), dtype=torch.float), torch.ones((1, 2, 3, 3, 3), dtype=torch.float))
+        loss = LocalNormalizedCrossCorrelationLoss(ndim=3)
         # ndim unmatch
         with self.assertRaisesRegex(ValueError, ""):
-            loss.forward(torch.ones((1, 3, 3, 3), dtype=torch.float), torch.ones((1, 3, 3, 3), dtype=torch.float))
+            loss.forward(
+                torch.ones((1, 3, 3, 3), dtype=torch.float, device=device),
+                torch.ones((1, 3, 3, 3), dtype=torch.float, device=device),
+            )
         # pred, target shape unmatch
         with self.assertRaisesRegex(ValueError, ""):
-            loss.forward(torch.ones((1, 3, 3, 3, 3), dtype=torch.float), torch.ones((1, 3, 4, 4, 4), dtype=torch.float))
+            loss.forward(
+                torch.ones((1, 3, 3, 3, 3), dtype=torch.float, device=device),
+                torch.ones((1, 3, 4, 4, 4), dtype=torch.float, device=device),
+            )
 
     def test_ill_opts(self):
         pred = torch.ones((1, 3, 3, 3, 3), dtype=torch.float)
         target = torch.ones((1, 3, 3, 3, 3), dtype=torch.float)
         with self.assertRaisesRegex(ValueError, ""):
-            LocalNormalizedCrossCorrelationLoss(in_channels=3, kernel_type="unknown")(pred, target)
+            LocalNormalizedCrossCorrelationLoss(kernel_type="unknown")(pred, target)
         with self.assertRaisesRegex(ValueError, ""):
-            LocalNormalizedCrossCorrelationLoss(in_channels=3, kernel_type=None)(pred, target)
+            LocalNormalizedCrossCorrelationLoss(kernel_type=None)(pred, target)
         with self.assertRaisesRegex(ValueError, ""):
-            LocalNormalizedCrossCorrelationLoss(in_channels=3, kernel_size=4)(pred, target)
+            LocalNormalizedCrossCorrelationLoss(kernel_size=4)(pred, target)
         with self.assertRaisesRegex(ValueError, ""):
-            LocalNormalizedCrossCorrelationLoss(in_channels=3, reduction="unknown")(pred, target)
+            LocalNormalizedCrossCorrelationLoss(reduction="unknown")(pred, target)
         with self.assertRaisesRegex(ValueError, ""):
-            LocalNormalizedCrossCorrelationLoss(in_channels=3, reduction=None)(pred, target)
+            LocalNormalizedCrossCorrelationLoss(reduction=None)(pred, target)
+
+
+#     def test_script(self):
+#         input_param, input_data, _ = TEST_CASES[0]
+#         loss = LocalNormalizedCrossCorrelationLoss(**input_param)
+#         test_script_save(loss, input_data["pred"], input_data["target"])
 
 
 if __name__ == "__main__":
diff --git a/tests/test_localnet.py b/tests/test_localnet.py
index 97a10d0c83..dc680f15f9 100644
--- a/tests/test_localnet.py
+++ b/tests/test_localnet.py
@@ -1,10 +1,21 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import unittest
 
 import torch
 from parameterized import parameterized
 
 from monai.networks import eval_mode
-from monai.networks.nets.localnet import LocalNet
+from monai.networks.nets.regunet import LocalNet
 from tests.utils import test_script_save
 
 device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -15,39 +26,36 @@
         {
             "spatial_dims": 2,
             "in_channels": 2,
-            "out_channels": 2,
             "num_channel_initial": 16,
-            "extract_levels": [0, 1, 2],
-            "out_activation": act,
+            "out_kernel_initializer": "kaiming_uniform",
+            "out_activation": None,
+            "out_channels": 2,
+            "extract_levels": (0, 1),
+            "pooling": False,
+            "concat_skip": True,
         },
         (1, 2, 16, 16),
         (1, 2, 16, 16),
     ]
-    for act in ["sigmoid", None]
 ]
 
-TEST_CASE_LOCALNET_3D = []
-for in_channels in [2, 3]:
-    for out_channels in [1, 3]:
-        for num_channel_initial in [4, 16, 32]:
-            for extract_levels in [[0, 1, 2], [0, 1, 2, 3], [0, 1, 2, 3, 4]]:
-                for out_activation in ["sigmoid", None]:
-                    for out_initializer in ["kaiming_uniform", "zeros"]:
-                        TEST_CASE_LOCALNET_3D.append(
-                            [
-                                {
-                                    "spatial_dims": 3,
-                                    "in_channels": in_channels,
-                                    "out_channels": out_channels,
-                                    "num_channel_initial": num_channel_initial,
-                                    "extract_levels": extract_levels,
-                                    "out_activation": out_activation,
-                                    "out_initializer": out_initializer,
-                                },
-                                (1, in_channels, 16, 16, 16),
-                                (1, out_channels, 16, 16, 16),
-                            ]
-                        )
+TEST_CASE_LOCALNET_3D = [
+    [
+        {
+            "spatial_dims": 3,
+            "in_channels": 2,
+            "num_channel_initial": 16,
+            "out_kernel_initializer": "zeros",
+            "out_activation": "sigmoid",
+            "out_channels": 2,
+            "extract_levels": (0, 1, 2, 3),
+            "pooling": True,
+            "concat_skip": False,
+        },
+        (1, 2, 16, 16, 16),
+        (1, 2, 16, 16, 16),
+    ]
+]
 
 
 class TestLocalNet(unittest.TestCase):
@@ -58,13 +66,6 @@ def test_shape(self, input_param, input_shape, expected_shape):
             result = net(torch.randn(input_shape).to(device))
             self.assertEqual(result.shape, expected_shape)
 
-    def test_ill_shape(self):
-        with self.assertRaisesRegex(ValueError, ""):
-            input_param, _, _ = TEST_CASE_LOCALNET_2D[0]
-            input_shape = (1, input_param["in_channels"], 17, 17)
-            net = LocalNet(**input_param).to(device)
-            net.forward(torch.randn(input_shape).to(device))
-
     def test_script(self):
         input_param, input_shape, _ = TEST_CASE_LOCALNET_2D[0]
         net = LocalNet(**input_param)
diff --git a/tests/test_localnet_block.py b/tests/test_localnet_block.py
index e6171aeae9..f4e857a0fa 100644
--- a/tests/test_localnet_block.py
+++ b/tests/test_localnet_block.py
@@ -1,3 +1,14 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import unittest
 
 import torch
diff --git a/tests/test_lr_finder.py b/tests/test_lr_finder.py
index 9ee9c8a4d0..5b730c2a77 100644
--- a/tests/test_lr_finder.py
+++ b/tests/test_lr_finder.py
@@ -13,6 +13,7 @@
 import random
 import sys
 import unittest
+from typing import TYPE_CHECKING
 
 import torch
 from torch.utils.data import DataLoader
@@ -23,7 +24,14 @@
 from monai.transforms import AddChanneld, Compose, LoadImaged, ScaleIntensityd, ToTensord
 from monai.utils import optional_import, set_determinism
 
-PILImage, has_pil = optional_import("PIL.Image")
+if TYPE_CHECKING:
+    import matplotlib.pyplot as plt
+
+    has_matplotlib = True
+    has_pil = True
+else:
+    plt, has_matplotlib = optional_import("matplotlib.pyplot")
+    _, has_pil = optional_import("PIL.Image")
 
 RAND_SEED = 42
 random.seed(RAND_SEED)
@@ -73,7 +81,14 @@ def test_lr_finder(self):
         lr_finder = LearningRateFinder(model, optimizer, loss_function, device=device)
         lr_finder.range_test(train_loader, val_loader=train_loader, end_lr=10, num_iter=5)
         print(lr_finder.get_steepest_gradient(0, 0)[0])
-        lr_finder.plot(0, 0)  # to inspect the loss-learning rate graph
+
+        if has_matplotlib:
+            ax = plt.subplot()
+            plt.show(block=False)
+            lr_finder.plot(0, 0, ax=ax)  # to inspect the loss-learning rate graph
+            plt.pause(3)
+            plt.close()
+
         lr_finder.reset()  # to reset the model and optimizer to their initial state
 
 
diff --git a/tests/test_map_label_value.py b/tests/test_map_label_value.py
new file mode 100644
index 0000000000..ff1d7d1eef
--- /dev/null
+++ b/tests/test_map_label_value.py
@@ -0,0 +1,88 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.transforms import MapLabelValue
+
+TEST_CASE_1 = [
+    {"orig_labels": [3, 2, 1], "target_labels": [0, 1, 2]},
+    np.array([[3, 1], [1, 2]]),
+    np.array([[0, 2], [2, 1]]),
+]
+
+TEST_CASE_2 = [
+    {"orig_labels": [3, 5, 8], "target_labels": [0, 1, 2]},
+    np.array([[[3], [5], [5], [8]]]),
+    np.array([[[0], [1], [1], [2]]]),
+]
+
+TEST_CASE_3 = [
+    {"orig_labels": [1, 2, 3], "target_labels": [0, 1, 2]},
+    np.array([3, 1, 1, 2]),
+    np.array([2, 0, 0, 1]),
+]
+
+TEST_CASE_4 = [
+    {"orig_labels": [1, 2, 3], "target_labels": [0.5, 1.5, 2.5]},
+    np.array([3, 1, 1, 2]),
+    np.array([2.5, 0.5, 0.5, 1.5]),
+]
+
+TEST_CASE_5 = [
+    {"orig_labels": [1.5, 2.5, 3.5], "target_labels": [0, 1, 2], "dtype": np.int8},
+    np.array([3.5, 1.5, 1.5, 2.5]),
+    np.array([2, 0, 0, 1]),
+]
+
+TEST_CASE_6 = [
+    {"orig_labels": ["label3", "label2", "label1"], "target_labels": [0, 1, 2]},
+    np.array([["label3", "label1"], ["label1", "label2"]]),
+    np.array([[0, 2], [2, 1]]),
+]
+
+TEST_CASE_7 = [
+    {"orig_labels": [3.5, 2.5, 1.5], "target_labels": ["label0", "label1", "label2"], "dtype": "str"},
+    np.array([[3.5, 1.5], [1.5, 2.5]]),
+    np.array([["label0", "label2"], ["label2", "label1"]]),
+]
+
+TEST_CASE_8 = [
+    {"orig_labels": ["label3", "label2", "label1"], "target_labels": ["label1", "label2", "label3"], "dtype": "str"},
+    np.array([["label3", "label1"], ["label1", "label2"]]),
+    np.array([["label1", "label3"], ["label3", "label2"]]),
+]
+
+
+class TestMapLabelValue(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASE_1,
+            TEST_CASE_2,
+            TEST_CASE_3,
+            TEST_CASE_4,
+            TEST_CASE_5,
+            TEST_CASE_6,
+            TEST_CASE_7,
+            TEST_CASE_8,
+        ]
+    )
+    def test_shape(self, input_param, input_data, expected_value):
+        result = MapLabelValue(**input_param)(input_data)
+        np.testing.assert_equal(result, expected_value)
+        self.assertTupleEqual(result.shape, expected_value.shape)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_map_label_valued.py b/tests/test_map_label_valued.py
new file mode 100644
index 0000000000..426ac28836
--- /dev/null
+++ b/tests/test_map_label_valued.py
@@ -0,0 +1,71 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.transforms import MapLabelValued
+
+TEST_CASE_1 = [
+    {"keys": "seg", "orig_labels": [3, 2, 1], "target_labels": [0, 1, 2]},
+    {"seg": np.array([[3, 1], [1, 2]])},
+    np.array([[0, 2], [2, 1]]),
+]
+
+TEST_CASE_2 = [
+    {"keys": "seg", "orig_labels": [3, 5, 8], "target_labels": [0, 1, 2]},
+    {"seg": np.array([[[3], [5], [5], [8]]])},
+    np.array([[[0], [1], [1], [2]]]),
+]
+
+TEST_CASE_3 = [
+    {"keys": "seg", "orig_labels": [1, 2, 3], "target_labels": [0, 1, 2]},
+    {"seg": np.array([3, 1, 1, 2])},
+    np.array([2, 0, 0, 1]),
+]
+
+TEST_CASE_4 = [
+    {"keys": "seg", "orig_labels": [1, 2, 3], "target_labels": [0.5, 1.5, 2.5]},
+    {"seg": np.array([3, 1, 1, 2])},
+    np.array([2.5, 0.5, 0.5, 1.5]),
+]
+
+TEST_CASE_5 = [
+    {"keys": "seg", "orig_labels": [1.5, 2.5, 3.5], "target_labels": [0, 1, 2], "dtype": np.int8},
+    {"seg": np.array([3.5, 1.5, 1.5, 2.5])},
+    np.array([2, 0, 0, 1]),
+]
+
+TEST_CASE_6 = [
+    {"keys": "seg", "orig_labels": ["label3", "label2", "label1"], "target_labels": [0, 1, 2]},
+    {"seg": np.array([["label3", "label1"], ["label1", "label2"]])},
+    np.array([[0, 2], [2, 1]]),
+]
+
+TEST_CASE_7 = [
+    {"keys": "seg", "orig_labels": [3.5, 2.5, 1.5], "target_labels": ["label0", "label1", "label2"], "dtype": "str"},
+    {"seg": np.array([[3.5, 1.5], [1.5, 2.5]])},
+    np.array([["label0", "label2"], ["label2", "label1"]]),
+]
+
+
+class TestMapLabelValued(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7])
+    def test_shape(self, input_param, input_data, expected_value):
+        result = MapLabelValued(**input_param)(input_data)
+        np.testing.assert_equal(result["seg"], expected_value)
+        self.assertTupleEqual(result["seg"].shape, expected_value.shape)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_masked_inference_wsi_dataset.py b/tests/test_masked_inference_wsi_dataset.py
new file mode 100644
index 0000000000..ed79b4f3a7
--- /dev/null
+++ b/tests/test_masked_inference_wsi_dataset.py
@@ -0,0 +1,251 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+from unittest import skipUnless
+
+import numpy as np
+from numpy.testing import assert_array_equal
+from parameterized import parameterized
+
+from monai.apps.pathology.datasets import MaskedInferenceWSIDataset
+from monai.apps.utils import download_url
+from monai.utils import optional_import
+from tests.utils import skip_if_quick
+
+_, has_cim = optional_import("cucim")
+_, has_osl = optional_import("openslide")
+
+FILE_URL = "http://openslide.cs.cmu.edu/download/openslide-testdata/Generic-TIFF/CMU-1.tiff"
+base_name, extension = os.path.splitext(os.path.basename(FILE_URL))
+FILE_NAME = "temp_" + base_name
+FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", FILE_NAME + extension)
+
+MASK1 = os.path.join(os.path.dirname(__file__), "testing_data", "temp_tissue_mask1.npy")
+MASK2 = os.path.join(os.path.dirname(__file__), "testing_data", "temp_tissue_mask2.npy")
+MASK4 = os.path.join(os.path.dirname(__file__), "testing_data", "temp_tissue_mask4.npy")
+
+HEIGHT = 32914
+WIDTH = 46000
+
+
+def prepare_data():
+
+    mask = np.zeros((WIDTH // 2, HEIGHT // 2))
+    mask[100, 100] = 1
+    np.save(MASK1, mask)
+    mask[100, 100:102] = 1
+    np.save(MASK2, mask)
+    mask[100:102, 100:102] = 1
+    np.save(MASK4, mask)
+
+
+TEST_CASE_0 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "mask": MASK1},
+        ],
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [100, 100],
+        },
+    ],
+]
+
+TEST_CASE_1 = [
+    {
+        "data": [{"image": FILE_PATH, "mask": MASK2}],
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [101, 100],
+        },
+    ],
+]
+
+TEST_CASE_2 = [
+    {
+        "data": [{"image": FILE_PATH, "mask": MASK4}],
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [100, 101],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [101, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [101, 101],
+        },
+    ],
+]
+
+TEST_CASE_3 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "mask": MASK1},
+        ],
+        "patch_size": 2,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array(
+                [
+                    [[243, 243], [243, 243]],
+                    [[243, 243], [243, 243]],
+                    [[243, 243], [243, 243]],
+                ],
+                dtype=np.uint8,
+            ),
+            "name": FILE_NAME,
+            "mask_location": [100, 100],
+        },
+    ],
+]
+
+TEST_CASE_4 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "mask": MASK1},
+            {"image": FILE_PATH, "mask": MASK2},
+        ],
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [101, 100],
+        },
+    ],
+]
+
+
+TEST_CASE_OPENSLIDE_0 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "mask": MASK1},
+        ],
+        "patch_size": 1,
+        "image_reader_name": "OpenSlide",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [100, 100],
+        },
+    ],
+]
+
+TEST_CASE_OPENSLIDE_1 = [
+    {
+        "data": [{"image": FILE_PATH, "mask": MASK2}],
+        "patch_size": 1,
+        "image_reader_name": "OpenSlide",
+    },
+    [
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [100, 100],
+        },
+        {
+            "image": np.array([[[243]], [[243]], [[243]]], dtype=np.uint8),
+            "name": FILE_NAME,
+            "mask_location": [101, 100],
+        },
+    ],
+]
+
+
+class TestMaskedInferenceWSIDataset(unittest.TestCase):
+    def setUp(self):
+        prepare_data()
+        download_url(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
+
+    @parameterized.expand(
+        [
+            TEST_CASE_0,
+            TEST_CASE_1,
+            TEST_CASE_2,
+            TEST_CASE_3,
+            TEST_CASE_4,
+        ]
+    )
+    @skipUnless(has_cim, "Requires CuCIM")
+    @skip_if_quick
+    def test_read_patches_cucim(self, input_parameters, expected):
+        dataset = MaskedInferenceWSIDataset(**input_parameters)
+        self.compare_samples_expected(dataset, expected)
+
+    @parameterized.expand(
+        [
+            TEST_CASE_OPENSLIDE_0,
+            TEST_CASE_OPENSLIDE_1,
+        ]
+    )
+    @skipUnless(has_osl, "Requires OpenSlide")
+    @skip_if_quick
+    def test_read_patches_openslide(self, input_parameters, expected):
+        dataset = MaskedInferenceWSIDataset(**input_parameters)
+        self.compare_samples_expected(dataset, expected)
+
+    def compare_samples_expected(self, dataset, expected):
+        for i in range(len(dataset)):
+            self.assertTupleEqual(dataset[i][0]["image"].shape, expected[i]["image"].shape)
+            self.assertIsNone(assert_array_equal(dataset[i][0]["image"], expected[i]["image"]))
+            self.assertEqual(dataset[i][0]["name"], expected[i]["name"])
+            self.assertListEqual(dataset[i][0]["mask_location"], expected[i]["mask_location"])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_mednistdataset.py b/tests/test_mednistdataset.py
index 0887734a7c..2e27f4ba95 100644
--- a/tests/test_mednistdataset.py
+++ b/tests/test_mednistdataset.py
@@ -18,6 +18,8 @@
 from monai.transforms import AddChanneld, Compose, LoadImaged, ScaleIntensityd, ToTensord
 from tests.utils import skip_if_quick
 
+MEDNIST_FULL_DATASET_LENGTH = 58954
+
 
 class TestMedNISTDataset(unittest.TestCase):
     @skip_if_quick
@@ -33,7 +35,7 @@ def test_values(self):
         )
 
         def _test_dataset(dataset):
-            self.assertEqual(len(dataset), 5986)
+            self.assertEqual(len(dataset), int(MEDNIST_FULL_DATASET_LENGTH * dataset.test_frac))
             self.assertTrue("image" in dataset[0])
             self.assertTrue("label" in dataset[0])
             self.assertTrue("image_meta_dict" in dataset[0])
@@ -56,6 +58,9 @@ def _test_dataset(dataset):
         _test_dataset(data)
         data = MedNISTDataset(root_dir=testing_dir, section="test", download=False)
         self.assertTupleEqual(data[0]["image"].shape, (64, 64))
+        # test same dataset length with different random seed
+        data = MedNISTDataset(root_dir=testing_dir, transform=transform, section="test", download=False, seed=42)
+        _test_dataset(data)
         shutil.rmtree(os.path.join(testing_dir, "MedNIST"))
         try:
             data = MedNISTDataset(root_dir=testing_dir, transform=transform, section="test", download=False)
diff --git a/tests/test_module_list.py b/tests/test_module_list.py
new file mode 100644
index 0000000000..25e1dfbd6f
--- /dev/null
+++ b/tests/test_module_list.py
@@ -0,0 +1,38 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import os
+import unittest
+
+import monai
+
+
+class TestAllImport(unittest.TestCase):
+    def test_public_api(self):
+        """
+        This is to check "monai.__all__" should be consistent with
+        the top-level folders except for "__pycache__" and "csrc" (cpp/cuda src)
+        """
+        base_folder = os.path.dirname(monai.__file__)
+        to_search = os.path.join(base_folder, "*", "")
+        subfolders = [os.path.basename(x[:-1]) for x in glob.glob(to_search)]
+        to_exclude = ("__pycache__", "csrc")
+        mod = []
+        for code_folder in subfolders:
+            if code_folder in to_exclude:
+                continue
+            mod.append(code_folder)
+        self.assertEqual(sorted(monai.__all__), sorted(mod))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_multi_scale.py b/tests/test_multi_scale.py
index 722ae7cfce..01a760db72 100644
--- a/tests/test_multi_scale.py
+++ b/tests/test_multi_scale.py
@@ -16,25 +16,33 @@
 
 from monai.losses import DiceLoss
 from monai.losses.multi_scale import MultiScaleLoss
+from tests.utils import SkipIfBeforePyTorchVersion, test_script_save
 
 dice_loss = DiceLoss(include_background=True, sigmoid=True, smooth_nr=1e-5, smooth_dr=1e-5)
+device = "cuda" if torch.cuda.is_available() else "cpu"
 
 TEST_CASES = [
     [
         {"loss": dice_loss, "scales": None, "kernel": "gaussian"},
-        {"y_pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]), "y_true": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]])},
+        {
+            "y_pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]], device=device),
+            "y_true": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]], device=device),
+        },
         0.307576,
     ],
     [
         {"loss": dice_loss, "scales": [0, 1], "kernel": "gaussian"},
-        {"y_pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]]), "y_true": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]])},
+        {
+            "y_pred": torch.tensor([[[[1.0, -1.0], [-1.0, 1.0]]]], device=device),
+            "y_true": torch.tensor([[[[1.0, 0.0], [1.0, 1.0]]]], device=device),
+        },
         0.463116,
     ],
     [
         {"loss": dice_loss, "scales": [0, 1, 2], "kernel": "cauchy"},
         {
-            "y_pred": torch.tensor([[[[[1.0, -1.0], [-1.0, 1.0]]]]]),
-            "y_true": torch.tensor([[[[[1.0, 0.0], [1.0, 1.0]]]]]),
+            "y_pred": torch.tensor([[[[[1.0, -1.0], [-1.0, 1.0]]]]], device=device),
+            "y_true": torch.tensor([[[[[1.0, 0.0], [1.0, 1.0]]]]], device=device),
         },
         0.715228,
     ],
@@ -51,9 +59,19 @@ def test_ill_opts(self):
         with self.assertRaisesRegex(ValueError, ""):
             MultiScaleLoss(loss=dice_loss, kernel="none")
         with self.assertRaisesRegex(ValueError, ""):
-            MultiScaleLoss(loss=dice_loss, scales=[-1])(torch.ones((1, 1, 3)), torch.ones((1, 1, 3)))
+            MultiScaleLoss(loss=dice_loss, scales=[-1])(
+                torch.ones((1, 1, 3), device=device), torch.ones((1, 1, 3), device=device)
+            )
         with self.assertRaisesRegex(ValueError, ""):
-            MultiScaleLoss(loss=dice_loss, scales=[-1], reduction="none")(torch.ones((1, 1, 3)), torch.ones((1, 1, 3)))
+            MultiScaleLoss(loss=dice_loss, scales=[-1], reduction="none")(
+                torch.ones((1, 1, 3), device=device), torch.ones((1, 1, 3), device=device)
+            )
+
+    @SkipIfBeforePyTorchVersion((1, 7, 0))
+    def test_script(self):
+        input_param, input_data, expected_val = TEST_CASES[0]
+        loss = MultiScaleLoss(**input_param)
+        test_script_save(loss, input_data["y_pred"], input_data["y_true"])
 
 
 if __name__ == "__main__":
diff --git a/tests/test_nifti_endianness.py b/tests/test_nifti_endianness.py
new file mode 100644
index 0000000000..57f26d2247
--- /dev/null
+++ b/tests/test_nifti_endianness.py
@@ -0,0 +1,80 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import unittest
+from typing import TYPE_CHECKING, List, Tuple
+from unittest.case import skipUnless
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.data import DataLoader, Dataset, create_test_image_2d
+from monai.data.image_reader import PILReader
+from monai.transforms import LoadImage, LoadImaged
+from monai.transforms.io.array import switch_endianness
+from monai.utils.module import optional_import
+
+if TYPE_CHECKING:
+    import nibabel as nib
+    from PIL import Image as PILImage
+
+    has_nib = True
+    has_pil = True
+else:
+    nib, has_nib = optional_import("nibabel")
+    PILImage, has_pil = optional_import("PIL.Image")
+
+TESTS: List[Tuple] = []
+for endianness in ["<", ">"]:
+    for use_array in [True, False]:
+        for image_only in [True, False]:
+            TESTS.append((endianness, use_array, image_only))
+
+
+class TestNiftiEndianness(unittest.TestCase):
+    def setUp(self):
+        self.im, _ = create_test_image_2d(100, 100)
+        self.fname = tempfile.NamedTemporaryFile(suffix=".nii.gz").name
+
+    @parameterized.expand(TESTS)
+    @skipUnless(has_nib, "Requires NiBabel")
+    def test_endianness(self, endianness, use_array, image_only):
+
+        hdr = nib.Nifti1Header(endianness=endianness)
+        nii = nib.Nifti1Image(self.im, np.eye(4), header=hdr)
+        nib.save(nii, self.fname)
+
+        data = [self.fname] if use_array else [{"image": self.fname}]
+        tr = LoadImage(image_only=image_only) if use_array else LoadImaged("image", image_only=image_only)
+        check_ds = Dataset(data, tr)
+        check_loader = DataLoader(check_ds, batch_size=1)
+        _ = next(iter(check_loader))
+
+    def test_switch(self):  # verify data types
+        for data in (np.zeros((2, 1)), ("test",), [24, 42], {"foo": "bar"}, True, 42):
+            output = switch_endianness(data, ">", "<")
+            self.assertEqual(type(data), type(output))
+
+    @skipUnless(has_pil, "Requires PIL")
+    def test_pil(self):
+        tempdir = tempfile.mkdtemp()
+        test_image = np.random.randint(0, 256, size=[128, 256])
+        filename = os.path.join(tempdir, "test_image.png")
+        PILImage.fromarray(test_image.astype("uint8")).save(filename)
+
+        loader = LoadImage(PILReader(converter=lambda image: image.convert("LA")))
+        _ = loader(filename)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_nifti_saver.py b/tests/test_nifti_saver.py
index 2e2bfd4254..f48374a61c 100644
--- a/tests/test_nifti_saver.py
+++ b/tests/test_nifti_saver.py
@@ -17,6 +17,7 @@
 import torch
 
 from monai.data import NiftiSaver
+from monai.transforms import LoadImage
 
 
 class TestNiftiSaver(unittest.TestCase):
@@ -72,6 +73,29 @@ def test_saved_3d_resize_content(self):
                 filepath = os.path.join("testfile" + str(i), "testfile" + str(i) + "_seg.nii.gz")
                 self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
 
+    def test_squeeze_end_dims(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+
+            for squeeze_end_dims in [False, True]:
+
+                saver = NiftiSaver(
+                    output_dir=tempdir,
+                    output_postfix="",
+                    output_ext=".nii.gz",
+                    dtype=np.float32,
+                    squeeze_end_dims=squeeze_end_dims,
+                )
+
+                fname = "testfile_squeeze"
+                meta_data = {"filename_or_obj": fname}
+
+                # 2d image w channel
+                saver.save(torch.randint(0, 255, (1, 2, 2)), meta_data)
+
+                im, meta = LoadImage()(os.path.join(tempdir, fname, fname + ".nii.gz"))
+                self.assertTrue(im.ndim == 2 if squeeze_end_dims else 4)
+                self.assertTrue(meta["dim"][0] == im.ndim)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_normalize_intensity.py b/tests/test_normalize_intensity.py
index ecf162e12f..dfb0de18fa 100644
--- a/tests/test_normalize_intensity.py
+++ b/tests/test_normalize_intensity.py
@@ -58,7 +58,7 @@
 class TestNormalizeIntensity(NumpyImageTestCase2D):
     def test_default(self):
         normalizer = NormalizeIntensity()
-        normalized = normalizer(self.imt)
+        normalized = normalizer(self.imt.copy())
         self.assertTrue(normalized.dtype == np.float32)
         expected = (self.imt - np.mean(self.imt)) / np.std(self.imt)
         np.testing.assert_allclose(normalized, expected, rtol=1e-5)
diff --git a/tests/test_npzdictitemdataset.py b/tests/test_npzdictitemdataset.py
new file mode 100644
index 0000000000..5ec52f45a2
--- /dev/null
+++ b/tests/test_npzdictitemdataset.py
@@ -0,0 +1,55 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import tempfile
+import unittest
+from io import BytesIO
+
+import numpy as np
+
+from monai.data import NPZDictItemDataset
+
+
+class TestNPZDictItemDataset(unittest.TestCase):
+    def test_load_stream(self):
+        dat0 = np.random.rand(10, 1, 4, 4)
+        dat1 = np.random.rand(10, 1, 4, 4)
+
+        npzfile = BytesIO()
+        npz = np.savez_compressed(npzfile, dat0=dat0, dat1=dat1)
+        npzfile.seek(0)
+
+        npzds = NPZDictItemDataset(npzfile, {"dat0": "images", "dat1": "seg"})
+
+        item = npzds[0]
+
+        np.testing.assert_allclose(item["images"].shape, (1, 4, 4))
+        np.testing.assert_allclose(item["seg"].shape, (1, 4, 4))
+
+    def test_load_file(self):
+        dat0 = np.random.rand(10, 1, 4, 4)
+        dat1 = np.random.rand(10, 1, 4, 4)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            npzfile = f"{tempdir}/test.npz"
+
+            npz = np.savez_compressed(npzfile, dat0=dat0, dat1=dat1)
+
+            npzds = NPZDictItemDataset(npzfile, {"dat0": "images", "dat1": "seg"})
+
+            item = npzds[0]
+
+            np.testing.assert_allclose(item["images"].shape, (1, 4, 4))
+            np.testing.assert_allclose(item["seg"].shape, (1, 4, 4))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_occlusion_sensitivity.py b/tests/test_occlusion_sensitivity.py
index 47a13d01e1..d58359a598 100644
--- a/tests/test_occlusion_sensitivity.py
+++ b/tests/test_occlusion_sensitivity.py
@@ -14,13 +14,13 @@
 import torch
 from parameterized import parameterized
 
-from monai.networks.nets import DenseNet, densenet121
+from monai.networks.nets import DenseNet, DenseNet121
 from monai.visualize import OcclusionSensitivity
 
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 out_channels_2d = 4
 out_channels_3d = 3
-model_2d = densenet121(spatial_dims=2, in_channels=1, out_channels=out_channels_2d).to(device)
+model_2d = DenseNet121(spatial_dims=2, in_channels=1, out_channels=out_channels_2d).to(device)
 model_3d = DenseNet(
     spatial_dims=3, in_channels=1, out_channels=out_channels_3d, init_features=2, growth_rate=2, block_config=(6,)
 ).to(device)
diff --git a/tests/test_openslide_reader.py b/tests/test_openslide_reader.py
new file mode 100644
index 0000000000..c0b395fd02
--- /dev/null
+++ b/tests/test_openslide_reader.py
@@ -0,0 +1,107 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+from unittest import skipUnless
+
+import numpy as np
+from numpy.testing import assert_array_equal
+from parameterized import parameterized
+
+from monai.apps.utils import download_url
+from monai.data.image_reader import WSIReader
+from monai.utils import optional_import
+
+_, has_osl = optional_import("openslide")
+
+
+FILE_URL = "http://openslide.cs.cmu.edu/download/openslide-testdata/Generic-TIFF/CMU-1.tiff"
+FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + os.path.basename(FILE_URL))
+
+HEIGHT = 32914
+WIDTH = 46000
+
+TEST_CASE_0 = [FILE_PATH, (3, HEIGHT, WIDTH)]
+
+TEST_CASE_1 = [
+    FILE_PATH,
+    {"location": (HEIGHT // 2, WIDTH // 2), "size": (2, 1), "level": 0},
+    np.array([[[246], [246]], [[246], [246]], [[246], [246]]]),
+]
+
+TEST_CASE_2 = [
+    FILE_PATH,
+    {"location": (0, 0), "size": (2, 1), "level": 2},
+    np.array([[[239], [239]], [[239], [239]], [[239], [239]]]),
+]
+
+TEST_CASE_3 = [
+    FILE_PATH,
+    {
+        "location": (0, 0),
+        "size": (8, 8),
+        "level": 2,
+        "grid_shape": (2, 1),
+        "patch_size": 2,
+    },
+    np.array(
+        [
+            [[[239, 239], [239, 239]], [[239, 239], [239, 239]], [[239, 239], [239, 239]]],
+            [[[242, 242], [242, 243]], [[242, 242], [242, 243]], [[242, 242], [242, 243]]],
+        ]
+    ),
+]
+
+TEST_CASE_4 = [
+    FILE_PATH,
+    {
+        "location": (0, 0),
+        "size": (8, 8),
+        "level": 2,
+        "grid_shape": (2, 1),
+        "patch_size": 1,
+    },
+    np.array([[[[239]], [[239]], [[239]]], [[[243]], [[243]], [[243]]]]),
+]
+
+
+class TestOpenSlideReader(unittest.TestCase):
+    @skipUnless(has_osl, "Requires OpenSlide")
+    def setUp(self):
+        download_url(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
+
+    @parameterized.expand([TEST_CASE_0])
+    def test_read_whole_image(self, file_path, expected_shape):
+        reader = WSIReader("OpenSlide")
+        img_obj = reader.read(file_path)
+        img = reader.get_data(img_obj)[0]
+        self.assertTupleEqual(img.shape, expected_shape)
+
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
+    def test_read_region(self, file_path, patch_info, expected_img):
+        reader = WSIReader("OpenSlide")
+        img_obj = reader.read(file_path)
+        img = reader.get_data(img_obj, **patch_info)[0]
+        self.assertTupleEqual(img.shape, expected_img.shape)
+        self.assertIsNone(assert_array_equal(img, expected_img))
+
+    @parameterized.expand([TEST_CASE_3, TEST_CASE_4])
+    def test_read_patches(self, file_path, patch_info, expected_img):
+        reader = WSIReader("OpenSlide")
+        img_obj = reader.read(file_path)
+        img = reader.get_data(img_obj, **patch_info)[0]
+        self.assertTupleEqual(img.shape, expected_img.shape)
+        self.assertIsNone(assert_array_equal(img, expected_img))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_pad_collation.py b/tests/test_pad_collation.py
new file mode 100644
index 0000000000..3835dc8895
--- /dev/null
+++ b/tests/test_pad_collation.py
@@ -0,0 +1,100 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+import unittest
+from typing import List, Tuple
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.data import CacheDataset, DataLoader
+from monai.data.utils import decollate_batch, pad_list_data_collate
+from monai.transforms import (
+    PadListDataCollate,
+    RandRotate,
+    RandRotate90,
+    RandRotate90d,
+    RandRotated,
+    RandSpatialCrop,
+    RandSpatialCropd,
+    RandZoom,
+    RandZoomd,
+)
+from monai.utils import set_determinism
+
+TESTS: List[Tuple] = []
+
+for pad_collate in [pad_list_data_collate, PadListDataCollate()]:
+    TESTS.append((dict, pad_collate, RandSpatialCropd("image", roi_size=[8, 7], random_size=True)))
+    TESTS.append((dict, pad_collate, RandRotated("image", prob=1, range_x=np.pi, keep_size=False)))
+    TESTS.append((dict, pad_collate, RandZoomd("image", prob=1, min_zoom=1.1, max_zoom=2.0, keep_size=False)))
+    TESTS.append((dict, pad_collate, RandRotate90d("image", prob=1, max_k=2)))
+
+    TESTS.append((list, pad_collate, RandSpatialCrop(roi_size=[8, 7], random_size=True)))
+    TESTS.append((list, pad_collate, RandRotate(prob=1, range_x=np.pi, keep_size=False)))
+    TESTS.append((list, pad_collate, RandZoom(prob=1, min_zoom=1.1, max_zoom=2.0, keep_size=False)))
+    TESTS.append((list, pad_collate, RandRotate90(prob=1, max_k=2)))
+
+
+class _Dataset(torch.utils.data.Dataset):
+    def __init__(self, images, labels, transforms):
+        self.images = images
+        self.labels = labels
+        self.transforms = transforms
+
+    def __len__(self):
+        return len(self.images)
+
+    def __getitem__(self, index):
+        return self.transforms(self.images[index]), self.labels[index]
+
+
+class TestPadCollation(unittest.TestCase):
+    def setUp(self) -> None:
+        set_determinism(seed=0)
+        # image is non square to throw rotation errors
+        im = np.arange(0, 10 * 9).reshape(1, 10, 9)
+        num_elements = 20
+        self.dict_data = [{"image": im} for _ in range(num_elements)]
+        self.list_data = [im for _ in range(num_elements)]
+        self.list_labels = [random.randint(0, 1) for _ in range(num_elements)]
+
+    def tearDown(self) -> None:
+        set_determinism(None)
+
+    @parameterized.expand(TESTS)
+    def test_pad_collation(self, t_type, collate_method, transform):
+
+        if t_type == dict:
+            dataset = CacheDataset(self.dict_data, transform, progress=False)
+        else:
+            dataset = _Dataset(self.list_data, self.list_labels, transform)
+
+        # Default collation should raise an error
+        loader_fail = DataLoader(dataset, batch_size=10)
+        with self.assertRaises(RuntimeError):
+            for _ in loader_fail:
+                pass
+
+        # Padded collation shouldn't
+        loader = DataLoader(dataset, batch_size=10, collate_fn=collate_method)
+        # check collation in forward direction
+        for data in loader:
+            if t_type == dict:
+                decollated_data = decollate_batch(data)
+                for d in decollated_data:
+                    PadListDataCollate.inverse(d)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_patch_wsi_dataset.py b/tests/test_patch_wsi_dataset.py
new file mode 100644
index 0000000000..7c34997872
--- /dev/null
+++ b/tests/test_patch_wsi_dataset.py
@@ -0,0 +1,163 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+from unittest import skipUnless
+
+import numpy as np
+from numpy.testing import assert_array_equal
+from parameterized import parameterized
+
+from monai.apps.pathology.datasets import PatchWSIDataset
+from monai.apps.utils import download_url
+from monai.utils import optional_import
+
+_, has_cim = optional_import("cucim")
+_, has_osl = optional_import("openslide")
+
+FILE_URL = "http://openslide.cs.cmu.edu/download/openslide-testdata/Generic-TIFF/CMU-1.tiff"
+FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + os.path.basename(FILE_URL))
+
+TEST_CASE_0 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "location": [0, 0], "label": [1]},
+        ],
+        "region_size": (1, 1),
+        "grid_shape": (1, 1),
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[1]]])},
+    ],
+]
+
+TEST_CASE_1 = [
+    {
+        "data": [{"image": FILE_PATH, "location": [10004, 20004], "label": [0, 0, 0, 1]}],
+        "region_size": (8, 8),
+        "grid_shape": (2, 2),
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {"image": np.array([[[247]], [[245]], [[248]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[245]], [[247]], [[244]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[1]]])},
+    ],
+]
+
+TEST_CASE_2 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "location": [0, 0], "label": [1]},
+        ],
+        "region_size": 1,
+        "grid_shape": 1,
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[1]]])},
+    ],
+]
+
+TEST_CASE_3 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "location": [0, 0], "label": [[[0, 1], [1, 0]]]},
+        ],
+        "region_size": 1,
+        "grid_shape": 1,
+        "patch_size": 1,
+        "image_reader_name": "cuCIM",
+    },
+    [
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[0, 1], [1, 0]]])},
+    ],
+]
+
+TEST_CASE_OPENSLIDE_0 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "location": [0, 0], "label": [1]},
+        ],
+        "region_size": (1, 1),
+        "grid_shape": (1, 1),
+        "patch_size": 1,
+        "image_reader_name": "OpenSlide",
+    },
+    [
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[1]]])},
+    ],
+]
+
+TEST_CASE_OPENSLIDE_1 = [
+    {
+        "data": [{"image": FILE_PATH, "location": [10004, 20004], "label": [0, 0, 0, 1]}],
+        "region_size": (8, 8),
+        "grid_shape": (2, 2),
+        "patch_size": 1,
+        "image_reader_name": "OpenSlide",
+    },
+    [
+        {"image": np.array([[[247]], [[245]], [[248]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[245]], [[247]], [[244]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[1]]])},
+    ],
+]
+
+
+class TestPatchWSIDataset(unittest.TestCase):
+    def setUp(self):
+        download_url(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
+
+    @parameterized.expand(
+        [
+            TEST_CASE_0,
+            TEST_CASE_1,
+            TEST_CASE_2,
+            TEST_CASE_3,
+        ]
+    )
+    @skipUnless(has_cim, "Requires CuCIM")
+    def test_read_patches_cucim(self, input_parameters, expected):
+        dataset = PatchWSIDataset(**input_parameters)
+        samples = dataset[0]
+        for i in range(len(samples)):
+            self.assertTupleEqual(samples[i]["label"].shape, expected[i]["label"].shape)
+            self.assertTupleEqual(samples[i]["image"].shape, expected[i]["image"].shape)
+            self.assertIsNone(assert_array_equal(samples[i]["label"], expected[i]["label"]))
+            self.assertIsNone(assert_array_equal(samples[i]["image"], expected[i]["image"]))
+
+    @parameterized.expand(
+        [
+            TEST_CASE_OPENSLIDE_0,
+            TEST_CASE_OPENSLIDE_1,
+        ]
+    )
+    @skipUnless(has_osl, "Requires OpenSlide")
+    def test_read_patches_openslide(self, input_parameters, expected):
+        dataset = PatchWSIDataset(**input_parameters)
+        samples = dataset[0]
+        for i in range(len(samples)):
+            self.assertTupleEqual(samples[i]["label"].shape, expected[i]["label"].shape)
+            self.assertTupleEqual(samples[i]["image"].shape, expected[i]["image"].shape)
+            self.assertIsNone(assert_array_equal(samples[i]["label"], expected[i]["label"]))
+            self.assertIsNone(assert_array_equal(samples[i]["image"], expected[i]["image"]))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_pathology_prob_nms.py b/tests/test_pathology_prob_nms.py
new file mode 100644
index 0000000000..223b136ea7
--- /dev/null
+++ b/tests/test_pathology_prob_nms.py
@@ -0,0 +1,57 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.apps.pathology.utils import PathologyProbNMS
+
+probs_map_2d = np.random.rand(100, 100).clip(0, 0.5)
+probs_map_2d[33, 33] = 0.7
+probs_map_2d[66, 66] = 0.9
+expected_2d = [[0.9, 133, 133], [0.7, 67, 67]]
+TEST_CASES_2D = [
+    {"spatial_dims": 2, "prob_threshold": 0.5, "box_size": [10, 10]},
+    {"resolution_level": 1},
+    probs_map_2d,
+    expected_2d,
+]
+
+probs_map_3d = torch.rand([50, 50, 50]).uniform_(0, 0.5)
+probs_map_3d[25, 25, 25] = 0.7
+probs_map_3d[45, 45, 45] = 0.9
+expected_3d = [[0.9, 91, 91, 91], [0.7, 51, 51, 51]]
+TEST_CASES_3D = [
+    {"spatial_dims": 3, "prob_threshold": 0.5, "box_size": (10, 10, 10)},
+    {"resolution_level": 1},
+    probs_map_3d,
+    expected_3d,
+]
+
+
+class TestPathologyProbNMS(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASES_2D,
+            TEST_CASES_3D,
+        ]
+    )
+    def test_output(self, class_args, call_args, probs_map, expected):
+        nms = PathologyProbNMS(**class_args)
+        output = nms(probs_map, **call_args)
+        np.testing.assert_allclose(output, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_persistentdataset.py b/tests/test_persistentdataset.py
index deed810f1a..5d94064cf4 100644
--- a/tests/test_persistentdataset.py
+++ b/tests/test_persistentdataset.py
@@ -98,26 +98,29 @@ def test_shape(self, transform, expected_shape):
             dataset_postcached = PersistentDataset(data=test_data, transform=transform, cache_dir=cache_dir)
             data1_postcached = dataset_postcached[0]
             data2_postcached = dataset_postcached[1]
-
-        if transform is None:
-            self.assertEqual(data1_precached["image"], os.path.join(tempdir, "test_image1.nii.gz"))
-            self.assertEqual(data2_precached["label"], os.path.join(tempdir, "test_label2.nii.gz"))
-            self.assertEqual(data1_postcached["image"], os.path.join(tempdir, "test_image1.nii.gz"))
-            self.assertEqual(data2_postcached["extra"], os.path.join(tempdir, "test_extra2.nii.gz"))
-        else:
-            self.assertTupleEqual(data1_precached["image"].shape, expected_shape)
-            self.assertTupleEqual(data1_precached["label"].shape, expected_shape)
-            self.assertTupleEqual(data1_precached["extra"].shape, expected_shape)
-            self.assertTupleEqual(data2_precached["image"].shape, expected_shape)
-            self.assertTupleEqual(data2_precached["label"].shape, expected_shape)
-            self.assertTupleEqual(data2_precached["extra"].shape, expected_shape)
-
-            self.assertTupleEqual(data1_postcached["image"].shape, expected_shape)
-            self.assertTupleEqual(data1_postcached["label"].shape, expected_shape)
-            self.assertTupleEqual(data1_postcached["extra"].shape, expected_shape)
-            self.assertTupleEqual(data2_postcached["image"].shape, expected_shape)
-            self.assertTupleEqual(data2_postcached["label"].shape, expected_shape)
-            self.assertTupleEqual(data2_postcached["extra"].shape, expected_shape)
+            data3_postcached = dataset_postcached[0:2]
+
+            if transform is None:
+                self.assertEqual(data1_precached["image"], os.path.join(tempdir, "test_image1.nii.gz"))
+                self.assertEqual(data2_precached["label"], os.path.join(tempdir, "test_label2.nii.gz"))
+                self.assertEqual(data1_postcached["image"], os.path.join(tempdir, "test_image1.nii.gz"))
+                self.assertEqual(data2_postcached["extra"], os.path.join(tempdir, "test_extra2.nii.gz"))
+            else:
+                self.assertTupleEqual(data1_precached["image"].shape, expected_shape)
+                self.assertTupleEqual(data1_precached["label"].shape, expected_shape)
+                self.assertTupleEqual(data1_precached["extra"].shape, expected_shape)
+                self.assertTupleEqual(data2_precached["image"].shape, expected_shape)
+                self.assertTupleEqual(data2_precached["label"].shape, expected_shape)
+                self.assertTupleEqual(data2_precached["extra"].shape, expected_shape)
+
+                self.assertTupleEqual(data1_postcached["image"].shape, expected_shape)
+                self.assertTupleEqual(data1_postcached["label"].shape, expected_shape)
+                self.assertTupleEqual(data1_postcached["extra"].shape, expected_shape)
+                self.assertTupleEqual(data2_postcached["image"].shape, expected_shape)
+                self.assertTupleEqual(data2_postcached["label"].shape, expected_shape)
+                self.assertTupleEqual(data2_postcached["extra"].shape, expected_shape)
+                for d in data3_postcached:
+                    self.assertTupleEqual(d["image"].shape, expected_shape)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_png_saver.py b/tests/test_png_saver.py
index 6aa50184df..dbc41dfd75 100644
--- a/tests/test_png_saver.py
+++ b/tests/test_png_saver.py
@@ -55,6 +55,25 @@ def test_saved_content_spatial_size(self):
                 filepath = os.path.join("testfile" + str(i), "testfile" + str(i) + "_seg.png")
                 self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
 
+    def test_saved_specified_root(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+
+            saver = PNGSaver(
+                output_dir=tempdir,
+                output_postfix="seg",
+                output_ext=".png",
+                scale=255,
+                data_root_dir="test",
+            )
+
+            meta_data = {
+                "filename_or_obj": [os.path.join("test", "testfile" + str(i), "image" + ".jpg") for i in range(8)]
+            }
+            saver.save_batch(torch.randint(1, 200, (8, 1, 2, 2)), meta_data)
+            for i in range(8):
+                filepath = os.path.join("testfile" + str(i), "image", "image" + "_seg.png")
+                self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_probnms.py b/tests/test_probnms.py
new file mode 100644
index 0000000000..e51d1017d8
--- /dev/null
+++ b/tests/test_probnms.py
@@ -0,0 +1,103 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.transforms.post.array import ProbNMS
+
+probs_map_1 = np.random.rand(100, 100).clip(0, 0.5)
+TEST_CASES_2D_1 = [{"spatial_dims": 2, "prob_threshold": 0.5, "box_size": 10}, probs_map_1, []]
+
+probs_map_2 = np.random.rand(100, 100).clip(0, 0.5)
+probs_map_2[33, 33] = 0.7
+probs_map_2[66, 66] = 0.9
+expected_2 = [[0.9, 66, 66], [0.7, 33, 33]]
+TEST_CASES_2D_2 = [
+    {"spatial_dims": 2, "prob_threshold": 0.5, "box_size": [10, 10]},
+    probs_map_2,
+    expected_2,
+]
+
+probs_map_3 = np.random.rand(100, 100).clip(0, 0.5)
+probs_map_3[56, 58] = 0.7
+probs_map_3[60, 66] = 0.8
+probs_map_3[66, 66] = 0.9
+expected_3 = [[0.9, 66, 66], [0.8, 60, 66]]
+TEST_CASES_2D_3 = [
+    {"spatial_dims": 2, "prob_threshold": 0.5, "box_size": (10, 20)},
+    probs_map_3,
+    expected_3,
+]
+
+probs_map_4 = np.random.rand(100, 100).clip(0, 0.5)
+probs_map_4[33, 33] = 0.7
+probs_map_4[66, 66] = 0.9
+expected_4 = [[0.9, 66, 66]]
+TEST_CASES_2D_4 = [
+    {"spatial_dims": 2, "prob_threshold": 0.8, "box_size": 10},
+    probs_map_4,
+    expected_4,
+]
+
+probs_map_5 = np.random.rand(100, 100).clip(0, 0.5)
+TEST_CASES_2D_5 = [{"spatial_dims": 2, "prob_threshold": 0.5, "sigma": 0.1}, probs_map_5, []]
+
+probs_map_6 = torch.as_tensor(np.random.rand(100, 100).clip(0, 0.5))
+TEST_CASES_2D_6 = [{"spatial_dims": 2, "prob_threshold": 0.5, "sigma": 0.1}, probs_map_6, []]
+
+probs_map_7 = torch.as_tensor(np.random.rand(100, 100).clip(0, 0.5))
+probs_map_7[33, 33] = 0.7
+probs_map_7[66, 66] = 0.9
+if torch.cuda.is_available():
+    probs_map_7 = probs_map_7.cuda()
+expected_7 = [[0.9, 66, 66], [0.7, 33, 33]]
+TEST_CASES_2D_7 = [
+    {"spatial_dims": 2, "prob_threshold": 0.5, "sigma": 0.1},
+    probs_map_7,
+    expected_7,
+]
+
+probs_map_3d = torch.rand([50, 50, 50]).uniform_(0, 0.5)
+probs_map_3d[25, 25, 25] = 0.7
+probs_map_3d[45, 45, 45] = 0.9
+expected_3d = [[0.9, 45, 45, 45], [0.7, 25, 25, 25]]
+TEST_CASES_3D = [
+    {"spatial_dims": 3, "prob_threshold": 0.5, "box_size": (10, 10, 10)},
+    probs_map_3d,
+    expected_3d,
+]
+
+
+class TestProbNMS(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASES_2D_1,
+            TEST_CASES_2D_2,
+            TEST_CASES_2D_3,
+            TEST_CASES_2D_4,
+            TEST_CASES_2D_5,
+            TEST_CASES_2D_6,
+            TEST_CASES_2D_7,
+            TEST_CASES_3D,
+        ]
+    )
+    def test_output(self, class_args, probs_map, expected):
+        nms = ProbNMS(**class_args)
+        output = nms(probs_map)
+        np.testing.assert_allclose(output, expected)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_probnmsd.py b/tests/test_probnmsd.py
new file mode 100644
index 0000000000..5b75d4310f
--- /dev/null
+++ b/tests/test_probnmsd.py
@@ -0,0 +1,103 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.transforms.post.dictionary import ProbNMSD
+
+probs_map_1 = np.random.rand(100, 100).clip(0, 0.5)
+TEST_CASES_2D_1 = [{"spatial_dims": 2, "prob_threshold": 0.5, "box_size": 10}, {"prob_map": probs_map_1}, []]
+
+probs_map_2 = np.random.rand(100, 100).clip(0, 0.5)
+probs_map_2[33, 33] = 0.7
+probs_map_2[66, 66] = 0.9
+expected_2 = [[0.9, 66, 66], [0.7, 33, 33]]
+TEST_CASES_2D_2 = [
+    {"spatial_dims": 2, "prob_threshold": 0.5, "box_size": [10, 10]},
+    {"prob_map": probs_map_2},
+    expected_2,
+]
+
+probs_map_3 = np.random.rand(100, 100).clip(0, 0.5)
+probs_map_3[56, 58] = 0.7
+probs_map_3[60, 66] = 0.8
+probs_map_3[66, 66] = 0.9
+expected_3 = [[0.9, 66, 66], [0.8, 60, 66]]
+TEST_CASES_2D_3 = [
+    {"spatial_dims": 2, "prob_threshold": 0.5, "box_size": (10, 20)},
+    {"prob_map": probs_map_3},
+    expected_3,
+]
+
+probs_map_4 = np.random.rand(100, 100).clip(0, 0.5)
+probs_map_4[33, 33] = 0.7
+probs_map_4[66, 66] = 0.9
+expected_4 = [[0.9, 66, 66]]
+TEST_CASES_2D_4 = [
+    {"spatial_dims": 2, "prob_threshold": 0.8, "box_size": 10},
+    {"prob_map": probs_map_4},
+    expected_4,
+]
+
+probs_map_5 = np.random.rand(100, 100).clip(0, 0.5)
+TEST_CASES_2D_5 = [{"spatial_dims": 2, "prob_threshold": 0.5, "sigma": 0.1}, {"prob_map": probs_map_5}, []]
+
+probs_map_6 = torch.as_tensor(np.random.rand(100, 100).clip(0, 0.5))
+TEST_CASES_2D_6 = [{"spatial_dims": 2, "prob_threshold": 0.5, "sigma": 0.1}, {"prob_map": probs_map_6}, []]
+
+probs_map_7 = torch.as_tensor(np.random.rand(100, 100).clip(0, 0.5))
+probs_map_7[33, 33] = 0.7
+probs_map_7[66, 66] = 0.9
+if torch.cuda.is_available():
+    probs_map_7 = probs_map_7.cuda()
+expected_7 = [[0.9, 66, 66], [0.7, 33, 33]]
+TEST_CASES_2D_7 = [
+    {"spatial_dims": 2, "prob_threshold": 0.5, "sigma": 0.1},
+    {"prob_map": probs_map_7},
+    expected_7,
+]
+
+probs_map_3d = torch.rand([50, 50, 50]).uniform_(0, 0.5)
+probs_map_3d[25, 25, 25] = 0.7
+probs_map_3d[45, 45, 45] = 0.9
+expected_3d = [[0.9, 45, 45, 45], [0.7, 25, 25, 25]]
+TEST_CASES_3D = [
+    {"spatial_dims": 3, "prob_threshold": 0.5, "box_size": (10, 10, 10)},
+    {"prob_map": probs_map_3d},
+    expected_3d,
+]
+
+
+class TestProbNMS(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASES_2D_1,
+            TEST_CASES_2D_2,
+            TEST_CASES_2D_3,
+            TEST_CASES_2D_4,
+            TEST_CASES_2D_5,
+            TEST_CASES_2D_6,
+            TEST_CASES_2D_7,
+            TEST_CASES_3D,
+        ]
+    )
+    def test_output(self, class_args, probs_map, expected):
+        nms = ProbNMSD(keys="prob_map", **class_args)
+        output = nms(probs_map)
+        np.testing.assert_allclose(output["prob_map"], expected)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_rand_affined.py b/tests/test_rand_affined.py
index 54d71ad8f7..ae2adbe3b3 100644
--- a/tests/test_rand_affined.py
+++ b/tests/test_rand_affined.py
@@ -145,6 +145,8 @@ def test_rand_affined(self, input_param, input_data, expected_val):
         res = g(input_data)
         for key in res:
             result = res[key]
+            if "_transforms" in key:
+                continue
             expected = expected_val[key] if isinstance(expected_val, dict) else expected_val
             self.assertEqual(isinstance(result, torch.Tensor), isinstance(expected, torch.Tensor))
             if isinstance(result, torch.Tensor):
diff --git a/tests/test_rand_axis_flip.py b/tests/test_rand_axis_flip.py
new file mode 100644
index 0000000000..0bc2eb130e
--- /dev/null
+++ b/tests/test_rand_axis_flip.py
@@ -0,0 +1,32 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+from monai.transforms import RandAxisFlip
+from tests.utils import NumpyImageTestCase2D
+
+
+class TestRandAxisFlip(NumpyImageTestCase2D):
+    def test_correct_results(self):
+        flip = RandAxisFlip(prob=1.0)
+        result = flip(self.imt[0])
+
+        expected = []
+        for channel in self.imt[0]:
+            expected.append(np.flip(channel, flip._axis))
+        self.assertTrue(np.allclose(np.stack(expected), result))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_rand_axis_flipd.py b/tests/test_rand_axis_flipd.py
new file mode 100644
index 0000000000..154d7813cb
--- /dev/null
+++ b/tests/test_rand_axis_flipd.py
@@ -0,0 +1,32 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+from monai.transforms import RandAxisFlipd
+from tests.utils import NumpyImageTestCase3D
+
+
+class TestRandAxisFlip(NumpyImageTestCase3D):
+    def test_correct_results(self):
+        flip = RandAxisFlipd(keys="img", prob=1.0)
+        result = flip({"img": self.imt[0]})
+
+        expected = []
+        for channel in self.imt[0]:
+            expected.append(np.flip(channel, flip._axis))
+        self.assertTrue(np.allclose(np.stack(expected), result["img"]))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_rand_crop_by_pos_neg_labeld.py b/tests/test_rand_crop_by_pos_neg_labeld.py
index 06e63c14e8..d52ba900ac 100644
--- a/tests/test_rand_crop_by_pos_neg_labeld.py
+++ b/tests/test_rand_crop_by_pos_neg_labeld.py
@@ -91,6 +91,8 @@ def test_type_shape(self, input_param, input_data, expected_type, expected_shape
         self.assertTupleEqual(result[0]["image"].shape, expected_shape)
         self.assertTupleEqual(result[0]["extral"].shape, expected_shape)
         self.assertTupleEqual(result[0]["label"].shape, expected_shape)
+        for i, item in enumerate(result):
+            self.assertEqual(item["image_meta_dict"]["patch_index"], i)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_rand_elastic_2d.py b/tests/test_rand_elastic_2d.py
index aa408f0fdc..fbfb7d5761 100644
--- a/tests/test_rand_elastic_2d.py
+++ b/tests/test_rand_elastic_2d.py
@@ -74,7 +74,7 @@
             "scale_range": [0.01, 0.02],
             "prob": 0.9,
             "as_tensor_output": False,
-            "device": None,
+            "device": "cuda" if torch.cuda.is_available() else "cpu",
             "spatial_size": (2, 2),
         },
         {"img": torch.arange(27).reshape((3, 3, 3))},
diff --git a/tests/test_rand_elastic_3d.py b/tests/test_rand_elastic_3d.py
index 8cd74c6be7..c63282d571 100644
--- a/tests/test_rand_elastic_3d.py
+++ b/tests/test_rand_elastic_3d.py
@@ -59,7 +59,7 @@
             "prob": 0.9,
             "rotate_range": [1, 1, 1],
             "as_tensor_output": False,
-            "device": None,
+            "device": "cuda" if torch.cuda.is_available() else "cpu",
             "spatial_size": (2, 2, 2),
         },
         {"img": torch.arange(27).reshape((1, 3, 3, 3)), "mode": "bilinear"},
diff --git a/tests/test_rand_lambdad.py b/tests/test_rand_lambdad.py
index 359da8857a..a450b67413 100644
--- a/tests/test_rand_lambdad.py
+++ b/tests/test_rand_lambdad.py
@@ -13,7 +13,7 @@
 
 import numpy as np
 
-from monai.transforms import Randomizable
+from monai.transforms.transform import Randomizable
 from monai.transforms.utility.dictionary import RandLambdad
 
 
diff --git a/tests/test_rand_rotate.py b/tests/test_rand_rotate.py
index 79f3036454..0ff8508a0f 100644
--- a/tests/test_rand_rotate.py
+++ b/tests/test_rand_rotate.py
@@ -52,7 +52,8 @@ def test_correct_results(self, degrees, keep_size, mode, padding_mode, align_cor
             self.imt[0, 0], -np.rad2deg(angle), (0, 1), not keep_size, order=_order, mode=_mode, prefilter=False
         )
         expected = np.stack(expected).astype(np.float32)
-        np.testing.assert_allclose(expected, rotated[0])
+        good = np.sum(np.isclose(expected, rotated[0], atol=1e-3))
+        self.assertLessEqual(np.abs(good - expected.size), 5, "diff at most 5 pixels")
 
 
 class TestRandRotate3D(NumpyImageTestCase3D):
diff --git a/tests/test_rand_rotated.py b/tests/test_rand_rotated.py
index 962ac5fc51..47b4b7107e 100644
--- a/tests/test_rand_rotated.py
+++ b/tests/test_rand_rotated.py
@@ -54,7 +54,8 @@ def test_correct_results(self, degrees, keep_size, mode, padding_mode, align_cor
             self.imt[0, 0], -np.rad2deg(angle), (0, 1), not keep_size, order=_order, mode=_mode, prefilter=False
         )
         expected = np.stack(expected).astype(np.float32)
-        self.assertTrue(np.allclose(expected, rotated["img"][0]))
+        good = np.sum(np.isclose(expected, rotated["img"][0], atol=1e-3))
+        self.assertLessEqual(np.abs(good - expected.size), 5, "diff at most 5 pixels")
 
 
 class TestRandRotated3D(NumpyImageTestCase3D):
diff --git a/tests/test_rand_spatial_crop_samplesd.py b/tests/test_rand_spatial_crop_samplesd.py
index afd7ab602c..09688f44b7 100644
--- a/tests/test_rand_spatial_crop_samplesd.py
+++ b/tests/test_rand_spatial_crop_samplesd.py
@@ -70,6 +70,9 @@ def test_shape(self, input_param, input_data, expected_shape, expected_last):
         for item, expected in zip(result, expected_shape):
             self.assertTupleEqual(item["img"].shape, expected)
             self.assertTupleEqual(item["seg"].shape, expected)
+        for i, item in enumerate(result):
+            self.assertEqual(item["img_meta_dict"]["patch_index"], i)
+            self.assertEqual(item["seg_meta_dict"]["patch_index"], i)
         np.testing.assert_allclose(item["img"], expected_last["img"])
         np.testing.assert_allclose(item["seg"], expected_last["seg"])
 
diff --git a/tests/test_rand_std_shift_intensity.py b/tests/test_rand_std_shift_intensity.py
new file mode 100644
index 0000000000..9aff50ab66
--- /dev/null
+++ b/tests/test_rand_std_shift_intensity.py
@@ -0,0 +1,32 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+from monai.transforms import RandStdShiftIntensity
+from tests.utils import NumpyImageTestCase2D
+
+
+class TestRandStdShiftIntensity(NumpyImageTestCase2D):
+    def test_value(self):
+        shifter = RandStdShiftIntensity(factors=1.0, prob=1.0)
+        shifter.set_random_state(seed=0)
+        result = shifter(self.imt)
+        np.random.seed(0)
+        factor = np.random.uniform(low=-1.0, high=1.0)
+        expected = self.imt + factor * np.std(self.imt)
+        np.testing.assert_allclose(result, expected, rtol=1e-5)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_rand_std_shift_intensityd.py b/tests/test_rand_std_shift_intensityd.py
new file mode 100644
index 0000000000..0cb6bd66be
--- /dev/null
+++ b/tests/test_rand_std_shift_intensityd.py
@@ -0,0 +1,33 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+from monai.transforms import RandStdShiftIntensityd
+from tests.utils import NumpyImageTestCase2D
+
+
+class TestRandStdShiftIntensityd(NumpyImageTestCase2D):
+    def test_value(self):
+        key = "img"
+        shifter = RandStdShiftIntensityd(keys=[key], factors=1.0, prob=1.0)
+        shifter.set_random_state(seed=0)
+        result = shifter({key: self.imt})
+        np.random.seed(0)
+        factor = np.random.uniform(low=-1.0, high=1.0)
+        expected = self.imt + factor * np.std(self.imt)
+        np.testing.assert_allclose(result[key], expected, rtol=1e-5)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_random_bias_field.py b/tests/test_random_bias_field.py
new file mode 100644
index 0000000000..16b4ab6917
--- /dev/null
+++ b/tests/test_random_bias_field.py
@@ -0,0 +1,66 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.transforms import RandBiasField
+
+TEST_CASES_2D = [{}, (3, 32, 32)]
+TEST_CASES_3D = [{}, (3, 32, 32, 32)]
+TEST_CASES_2D_ZERO_RANGE = [{"coeff_range": (0.0, 0.0)}, (3, 32, 32)]
+TEST_CASES_2D_ONES = [{"coeff_range": (1.0, 1.0)}, np.asarray([[[2, -2], [2, 10]]])]
+
+
+class TestRandBiasField(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASES_2D,
+            TEST_CASES_3D,
+        ]
+    )
+    def test_output_shape(self, class_args, img_shape):
+        for degree in [1, 2, 3]:
+            bias_field = RandBiasField(degree=degree, **class_args)
+            img = np.random.rand(*img_shape)
+            output = bias_field(img)
+            np.testing.assert_equal(output.shape, img_shape)
+            np.testing.assert_equal(output.dtype, bias_field.dtype)
+
+            img_zero = np.zeros([*img_shape])
+            output_zero = bias_field(img_zero)
+            np.testing.assert_equal(output_zero, img_zero)
+
+    @parameterized.expand([TEST_CASES_2D_ZERO_RANGE])
+    def test_zero_range(self, class_args, img_shape):
+        bias_field = RandBiasField(**class_args)
+        img = np.random.rand(*img_shape)
+        output = bias_field(img)
+        np.testing.assert_equal(output, np.zeros(img_shape))
+
+    @parameterized.expand([TEST_CASES_2D_ONES])
+    def test_one_range_input(self, class_args, expected):
+        bias_field = RandBiasField(**class_args)
+        img = np.ones([1, 2, 2])
+        output = bias_field(img)
+        np.testing.assert_equal(output, expected.astype(bias_field.dtype))
+
+    def test_zero_prob(self):
+        bias_field = RandBiasField(prob=0.0)
+        img = np.random.rand(3, 32, 32)
+        output = bias_field(img)
+        np.testing.assert_equal(output, img)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_random_bias_fieldd.py b/tests/test_random_bias_fieldd.py
new file mode 100644
index 0000000000..136eb41f2e
--- /dev/null
+++ b/tests/test_random_bias_fieldd.py
@@ -0,0 +1,65 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.transforms import RandBiasFieldd
+
+TEST_CASES_2D = [{}, (3, 32, 32)]
+TEST_CASES_3D = [{}, (3, 32, 32, 32)]
+TEST_CASES_2D_ZERO_RANGE = [{"coeff_range": (0.0, 0.0)}, (3, 32, 32)]
+TEST_CASES_2D_ONES = [{"coeff_range": (1.0, 1.0)}, np.asarray([[[2, -2], [2, 10]]])]
+
+
+class TestRandBiasFieldd(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASES_2D,
+            TEST_CASES_3D,
+        ]
+    )
+    def test_output_shape(self, class_args, img_shape):
+        key = "img"
+        bias_field = RandBiasFieldd(keys=[key], **class_args)
+        img = np.random.rand(*img_shape)
+        output = bias_field({key: img})
+        np.testing.assert_equal(output[key].shape, img_shape)
+        np.testing.assert_equal(output[key].dtype, bias_field.rand_bias_field.dtype)
+
+    @parameterized.expand([TEST_CASES_2D_ZERO_RANGE])
+    def test_zero_range(self, class_args, img_shape):
+        key = "img"
+        bias_field = RandBiasFieldd(keys=[key], **class_args)
+        img = np.random.rand(*img_shape)
+        output = bias_field({key: img})
+        np.testing.assert_equal(output[key], np.zeros(img_shape))
+
+    @parameterized.expand([TEST_CASES_2D_ONES])
+    def test_one_range_input(self, class_args, expected):
+        key = "img"
+        bias_field = RandBiasFieldd(keys=[key], **class_args)
+        img = np.ones([1, 2, 2])
+        output = bias_field({key: img})
+        np.testing.assert_equal(output[key], expected.astype(bias_field.rand_bias_field.dtype))
+
+    def test_zero_prob(self):
+        key = "img"
+        bias_field = RandBiasFieldd(keys=[key], prob=0.0)
+        img = np.random.rand(3, 32, 32)
+        output = bias_field({key: img})
+        np.testing.assert_equal(output[key], img)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_randomizable.py b/tests/test_randomizable.py
index a7a30124df..9972bded0f 100644
--- a/tests/test_randomizable.py
+++ b/tests/test_randomizable.py
@@ -13,7 +13,7 @@
 
 import numpy as np
 
-from monai.transforms import Randomizable
+from monai.transforms.transform import Randomizable
 
 
 class RandTest(Randomizable):
diff --git a/tests/test_reg_loss_integration.py b/tests/test_reg_loss_integration.py
index b512add2e9..b864a64647 100644
--- a/tests/test_reg_loss_integration.py
+++ b/tests/test_reg_loss_integration.py
@@ -22,17 +22,17 @@
     [BendingEnergyLoss, {}, ["pred"]],
     [
         LocalNormalizedCrossCorrelationLoss,
-        {"in_channels": 1, "kernel_size": 7, "kernel_type": "rectangular"},
+        {"kernel_size": 7, "kernel_type": "rectangular"},
         ["pred", "target"],
     ],
     [
         LocalNormalizedCrossCorrelationLoss,
-        {"in_channels": 1, "kernel_size": 5, "kernel_type": "triangular"},
+        {"kernel_size": 5, "kernel_type": "triangular"},
         ["pred", "target"],
     ],
     [
         LocalNormalizedCrossCorrelationLoss,
-        {"in_channels": 1, "kernel_size": 3, "kernel_type": "gaussian"},
+        {"kernel_size": 3, "kernel_type": "gaussian"},
         ["pred", "target"],
     ],
     [GlobalMutualInformationLoss, {"num_bins": 10}, ["pred", "target"]],
diff --git a/tests/test_regunet.py b/tests/test_regunet.py
new file mode 100644
index 0000000000..4dd968a1cf
--- /dev/null
+++ b/tests/test_regunet.py
@@ -0,0 +1,87 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import torch
+from parameterized import parameterized
+
+from monai.networks import eval_mode
+from monai.networks.nets.regunet import RegUNet
+from tests.utils import test_script_save
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+TEST_CASE_REGUNET_2D = [
+    [
+        {
+            "spatial_dims": 2,
+            "in_channels": 2,
+            "num_channel_initial": 16,
+            "depth": 3,
+            "out_kernel_initializer": "kaiming_uniform",
+            "out_activation": None,
+            "out_channels": 2,
+            "pooling": False,
+            "concat_skip": True,
+            "encode_kernel_sizes": 3,
+        },
+        (1, 2, 16, 16),
+        (1, 2, 16, 16),
+    ]
+]
+
+TEST_CASE_REGUNET_3D = [
+    [
+        {
+            "spatial_dims": 3,
+            "in_channels": 2,
+            "num_channel_initial": 16,
+            "depth": 3,
+            "out_kernel_initializer": "kaiming_uniform",
+            "out_activation": "sigmoid",
+            "out_channels": 2,
+            "extract_levels": (0, 1, 2, 3),
+            "pooling": True,
+            "concat_skip": False,
+            "encode_kernel_sizes": (3, 3, 3, 7),
+        },
+        (1, 2, 16, 16, 16),
+        (1, 2, 16, 16, 16),
+    ]
+]
+
+
+class TestREGUNET(unittest.TestCase):
+    @parameterized.expand(TEST_CASE_REGUNET_2D + TEST_CASE_REGUNET_3D)
+    def test_shape(self, input_param, input_shape, expected_shape):
+        net = RegUNet(**input_param).to(device)
+        with eval_mode(net):
+            result = net(torch.randn(input_shape).to(device))
+            self.assertEqual(result.shape, expected_shape)
+
+    def test_ill_shape(self):
+        with self.assertRaisesRegex(ValueError, ""):
+            input_param, _, _ = TEST_CASE_REGUNET_2D[0]
+            input_shape = (1, input_param["in_channels"], 17, 17)
+            net = RegUNet(**input_param).to(device)
+            net.forward(torch.randn(input_shape).to(device))
+
+    def test_script(self):
+        input_param, input_shape, _ = TEST_CASE_REGUNET_2D[0]
+        net = RegUNet(**input_param)
+        test_data = torch.randn(input_shape)
+        test_script_save(net, test_data)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_regunet_block.py b/tests/test_regunet_block.py
new file mode 100644
index 0000000000..9b96875432
--- /dev/null
+++ b/tests/test_regunet_block.py
@@ -0,0 +1,97 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import torch
+from parameterized import parameterized
+
+from monai.networks import eval_mode
+from monai.networks.blocks.regunet_block import (
+    RegistrationDownSampleBlock,
+    RegistrationExtractionBlock,
+    RegistrationResidualConvBlock,
+)
+
+TEST_CASE_RESIDUAL = [
+    [{"spatial_dims": 2, "in_channels": 1, "out_channels": 2, "num_layers": 1}, (1, 1, 5, 5), (1, 2, 5, 5)],
+    [{"spatial_dims": 3, "in_channels": 2, "out_channels": 2, "num_layers": 2}, (1, 2, 5, 5, 5), (1, 2, 5, 5, 5)],
+]
+
+TEST_CASE_DOWN_SAMPLE = [
+    [{"spatial_dims": 2, "channels": 1, "pooling": False}, (1, 1, 4, 4), (1, 1, 2, 2)],
+    [{"spatial_dims": 3, "channels": 2, "pooling": True}, (1, 2, 4, 4, 4), (1, 2, 2, 2, 2)],
+]
+
+TEST_CASE_EXTRACTION = [
+    [
+        {
+            "spatial_dims": 2,
+            "extract_levels": (0,),
+            "num_channels": [1],
+            "out_channels": 1,
+            "kernel_initializer": "kaiming_uniform",
+            "activation": None,
+        },
+        [(1, 1, 2, 2)],
+        (3, 3),
+        (1, 1, 3, 3),
+    ],
+    [
+        {
+            "spatial_dims": 3,
+            "extract_levels": (1, 2),
+            "num_channels": [1, 2, 3],
+            "out_channels": 1,
+            "kernel_initializer": "zeros",
+            "activation": "sigmoid",
+        },
+        [(1, 3, 2, 2, 2), (1, 2, 4, 4, 4), (1, 1, 8, 8, 8)],
+        (3, 3, 3),
+        (1, 1, 3, 3, 3),
+    ],
+]
+
+
+class TestRegistrationResidualConvBlock(unittest.TestCase):
+    @parameterized.expand(TEST_CASE_RESIDUAL)
+    def test_shape(self, input_param, input_shape, expected_shape):
+        net = RegistrationResidualConvBlock(**input_param)
+        with eval_mode(net):
+            x = net(torch.randn(input_shape))
+            self.assertEqual(x.shape, expected_shape)
+
+
+class TestRegistrationDownSampleBlock(unittest.TestCase):
+    @parameterized.expand(TEST_CASE_DOWN_SAMPLE)
+    def test_shape(self, input_param, input_shape, expected_shape):
+        net = RegistrationDownSampleBlock(**input_param)
+        with eval_mode(net):
+            x = net(torch.rand(input_shape))
+            self.assertEqual(x.shape, expected_shape)
+
+    def test_ill_shape(self):
+        net = RegistrationDownSampleBlock(spatial_dims=2, channels=2, pooling=True)
+        with self.assertRaises(ValueError):
+            net(torch.rand((1, 2, 3, 3)))
+
+
+class TestRegistrationExtractionBlock(unittest.TestCase):
+    @parameterized.expand(TEST_CASE_EXTRACTION)
+    def test_shape(self, input_param, input_shapes, image_size, expected_shape):
+        net = RegistrationExtractionBlock(**input_param)
+        with eval_mode(net):
+            x = net([torch.rand(input_shape) for input_shape in input_shapes], image_size)
+            self.assertEqual(x.shape, expected_shape)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_remove_repeated_channel.py b/tests/test_remove_repeated_channel.py
new file mode 100644
index 0000000000..070e0e2b8d
--- /dev/null
+++ b/tests/test_remove_repeated_channel.py
@@ -0,0 +1,30 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.transforms import RemoveRepeatedChannel
+
+TEST_CASE_1 = [{"repeats": 2}, np.array([[1, 2], [1, 2], [3, 4], [3, 4]]), (2, 2)]
+
+
+class TestRemoveRepeatedChannel(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1])
+    def test_shape(self, input_param, input_data, expected_shape):
+        result = RemoveRepeatedChannel(**input_param)(input_data)
+        self.assertEqual(result.shape, expected_shape)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_remove_repeated_channeld.py b/tests/test_remove_repeated_channeld.py
new file mode 100644
index 0000000000..46c68bbdc2
--- /dev/null
+++ b/tests/test_remove_repeated_channeld.py
@@ -0,0 +1,34 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+from parameterized import parameterized
+
+from monai.transforms import RemoveRepeatedChanneld
+
+TEST_CASE_1 = [
+    {"keys": ["img"], "repeats": 2},
+    {"img": np.array([[1, 2], [1, 2], [3, 4], [3, 4]]), "seg": np.array([[1, 2], [1, 2], [3, 4], [3, 4]])},
+    (2, 2),
+]
+
+
+class TestRemoveRepeatedChanneld(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1])
+    def test_shape(self, input_param, input_data, expected_shape):
+        result = RemoveRepeatedChanneld(**input_param)(input_data)
+        self.assertEqual(result["img"].shape, expected_shape)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_rotate.py b/tests/test_rotate.py
index 6e43ab90e7..436c952d4b 100644
--- a/tests/test_rotate.py
+++ b/tests/test_rotate.py
@@ -70,7 +70,8 @@ def test_correct_results(self, angle, keep_size, mode, padding_mode, align_corne
                 )
             )
         expected = np.stack(expected).astype(np.float32)
-        np.testing.assert_allclose(expected, rotated, atol=1e-1)
+        good = np.sum(np.isclose(expected, rotated, atol=1e-3))
+        self.assertLessEqual(np.abs(good - expected.size), 5, "diff at most 5 pixels")
 
 
 class TestRotate3D(NumpyImageTestCase3D):
@@ -102,7 +103,8 @@ def test_correct_results(self, angle, keep_size, mode, padding_mode, align_corne
                 )
             )
         expected = np.stack(expected).astype(np.float32)
-        np.testing.assert_allclose(expected, rotated, atol=1e-1)
+        n_good = np.sum(np.isclose(expected, rotated, atol=1e-3))
+        self.assertLessEqual(expected.size - n_good, 5, "diff at most 5 pixels")
 
     @parameterized.expand(TEST_CASES_SHAPE_3D)
     def test_correct_shape(self, angle, mode, padding_mode, align_corners):
diff --git a/tests/test_rotated.py b/tests/test_rotated.py
index 3353ae9fba..dd57786fff 100644
--- a/tests/test_rotated.py
+++ b/tests/test_rotated.py
@@ -52,13 +52,14 @@ def test_correct_results(self, angle, keep_size, mode, padding_mode, align_corne
         expected = scipy.ndimage.rotate(
             self.imt[0, 0], -np.rad2deg(angle), (0, 1), not keep_size, order=_order, mode=_mode, prefilter=False
         )
-        np.testing.assert_allclose(expected, rotated["img"][0], atol=1e-3)
+        good = np.sum(np.isclose(expected, rotated["img"][0], atol=1e-3))
+        self.assertLessEqual(np.abs(good - expected.size), 5, "diff at most 5 pixels")
 
         expected = scipy.ndimage.rotate(
             self.segn[0, 0], -np.rad2deg(angle), (0, 1), not keep_size, order=0, mode=_mode, prefilter=False
         )
         expected = np.stack(expected).astype(int)
-        self.assertLessEqual(np.count_nonzero(expected != rotated["seg"][0]), 20)
+        self.assertLessEqual(np.count_nonzero(expected != rotated["seg"][0]), 30)
 
 
 class TestRotated3D(NumpyImageTestCase3D):
@@ -78,13 +79,14 @@ def test_correct_results(self, angle, keep_size, mode, padding_mode, align_corne
         expected = scipy.ndimage.rotate(
             self.imt[0, 0], np.rad2deg(angle), (0, 2), not keep_size, order=_order, mode=_mode, prefilter=False
         )
-        np.testing.assert_allclose(expected.astype(np.float32), rotated["img"][0], atol=1e-3)
+        good = np.sum(np.isclose(expected.astype(np.float32), rotated["img"][0], atol=1e-3))
+        self.assertLessEqual(np.abs(good - expected.size), 5, "diff at most 5 voxels.")
 
         expected = scipy.ndimage.rotate(
             self.segn[0, 0], np.rad2deg(angle), (0, 2), not keep_size, order=0, mode=_mode, prefilter=False
         )
         expected = np.stack(expected).astype(int)
-        self.assertLessEqual(np.count_nonzero(expected != rotated["seg"][0]), 105)
+        self.assertLessEqual(np.count_nonzero(expected != rotated["seg"][0]), 110)
 
 
 class TestRotated3DXY(NumpyImageTestCase3D):
@@ -104,13 +106,14 @@ def test_correct_results(self, angle, keep_size, mode, padding_mode, align_corne
         expected = scipy.ndimage.rotate(
             self.imt[0, 0], -np.rad2deg(angle), (0, 1), not keep_size, order=_order, mode=_mode, prefilter=False
         )
-        np.testing.assert_allclose(expected, rotated["img"][0], atol=1e-3)
+        good = np.sum(np.isclose(expected, rotated["img"][0], atol=1e-3))
+        self.assertLessEqual(np.abs(good - expected.size), 5, "diff at most 5 voxels")
 
         expected = scipy.ndimage.rotate(
             self.segn[0, 0], -np.rad2deg(angle), (0, 1), not keep_size, order=0, mode=_mode, prefilter=False
         )
         expected = np.stack(expected).astype(int)
-        self.assertLessEqual(np.count_nonzero(expected != rotated["seg"][0]), 100)
+        self.assertLessEqual(np.count_nonzero(expected != rotated["seg"][0]), 110)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_saliency_inferer.py b/tests/test_saliency_inferer.py
new file mode 100644
index 0000000000..416b7170ae
--- /dev/null
+++ b/tests/test_saliency_inferer.py
@@ -0,0 +1,52 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import torch
+from parameterized import parameterized
+
+from monai.inferers import SaliencyInferer
+from monai.networks.nets import DenseNet
+from monai.visualize.visualizer import default_upsampler
+
+TEST_CASE_1 = ["CAM"]
+
+TEST_CASE_2 = ["GradCAM"]
+
+TEST_CASE_3 = ["GradCAMpp"]
+
+
+class TestSaliencyInferer(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
+    def test_shape(self, cam_name):
+        model = DenseNet(
+            spatial_dims=3, in_channels=1, out_channels=3, init_features=2, growth_rate=2, block_config=(6,)
+        )
+        device = "cuda:0" if torch.cuda.is_available() else "cpu"
+        model.to(device)
+        model.eval()
+
+        image = torch.rand((2, 1, 6, 6, 6), device=device)
+        target_layer = "class_layers.relu"
+        fc_layer = "class_layers.out"
+        if cam_name == "CAM":
+            inferer = SaliencyInferer(cam_name, target_layer, None, fc_layer, upsampler=default_upsampler)
+            result = inferer(inputs=image, network=model, layer_idx=-1)
+        else:
+            inferer = SaliencyInferer(cam_name, target_layer, None, upsampler=default_upsampler)
+            result = inferer(image, model, -1, retain_graph=False)
+
+        self.assertTupleEqual(result.shape, (2, 1, 6, 6, 6))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_save_imaged.py b/tests/test_save_imaged.py
index a6ebfe0d8d..d6536b3d51 100644
--- a/tests/test_save_imaged.py
+++ b/tests/test_save_imaged.py
@@ -87,9 +87,20 @@
     False,
 ]
 
+TEST_CASE_6 = [
+    {
+        "img": torch.randint(0, 255, (1, 2, 3, 4)),
+        "img_meta_dict": {"filename_or_obj": "testfile0.nii.gz"},
+        "patch_index": 6,
+    },
+    ".nii.gz",
+    False,
+    False,
+]
+
 
 class TestSaveImaged(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5])
+    @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6])
     def test_saved_content(self, test_data, output_ext, resample, save_batch):
         with tempfile.TemporaryDirectory() as tempdir:
             trans = SaveImaged(
@@ -106,7 +117,9 @@ def test_saved_content(self, test_data, output_ext, resample, save_batch):
                     filepath = os.path.join("testfile" + str(i), "testfile" + str(i) + "_trans" + output_ext)
                     self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
             else:
-                filepath = os.path.join("testfile0", "testfile0" + "_trans" + output_ext)
+                patch_index = test_data["img_meta_dict"].get("patch_index", None)
+                patch_index = f"_{patch_index}" if patch_index is not None else ""
+                filepath = os.path.join("testfile0", "testfile0" + "_trans" + patch_index + output_ext)
                 self.assertTrue(os.path.exists(os.path.join(tempdir, filepath)))
 
 
diff --git a/tests/test_seg_loss_integration.py b/tests/test_seg_loss_integration.py
index 2103119342..d2f991f160 100644
--- a/tests/test_seg_loss_integration.py
+++ b/tests/test_seg_loss_integration.py
@@ -18,23 +18,29 @@
 from parameterized import parameterized
 
 from monai.losses import DiceLoss, FocalLoss, GeneralizedDiceLoss, TverskyLoss
+from monai.networks import one_hot
 
 TEST_CASES = [
     [DiceLoss, {"to_onehot_y": True, "squared_pred": True, "smooth_nr": 1e-4, "smooth_dr": 1e-4}, {}],
     [DiceLoss, {"to_onehot_y": True, "squared_pred": True, "smooth_nr": 0, "smooth_dr": 1e-3}, {}],
+    [DiceLoss, {"to_onehot_y": False, "squared_pred": True, "smooth_nr": 0, "smooth_dr": 1e-3}, {}],
     [DiceLoss, {"to_onehot_y": True, "squared_pred": True, "batch": True}, {}],
     [DiceLoss, {"to_onehot_y": True, "sigmoid": True}, {}],
     [DiceLoss, {"to_onehot_y": True, "softmax": True}, {}],
-    [FocalLoss, {"gamma": 1.5, "weight": torch.tensor([1, 2])}, {}],
-    [FocalLoss, {"gamma": 1.5}, {}],
+    [FocalLoss, {"to_onehot_y": True, "gamma": 1.5, "weight": torch.tensor([1, 2])}, {}],
+    [FocalLoss, {"to_onehot_y": False, "gamma": 1.5, "weight": [1, 2]}, {}],
+    [FocalLoss, {"to_onehot_y": False, "gamma": 1.5, "weight": 1.0}, {}],
+    [FocalLoss, {"to_onehot_y": True, "gamma": 1.5}, {}],
     [GeneralizedDiceLoss, {"to_onehot_y": True, "softmax": True}, {}],
     [GeneralizedDiceLoss, {"to_onehot_y": True, "sigmoid": True}, {}],
     [GeneralizedDiceLoss, {"to_onehot_y": True, "sigmoid": True, "w_type": "simple"}, {}],
     [GeneralizedDiceLoss, {"to_onehot_y": True, "sigmoid": True, "w_type": "uniform"}, {}],
     [GeneralizedDiceLoss, {"to_onehot_y": True, "sigmoid": True, "w_type": "uniform", "batch": True}, {}],
+    [GeneralizedDiceLoss, {"to_onehot_y": False, "sigmoid": True, "w_type": "uniform", "batch": True}, {}],
     [TverskyLoss, {"to_onehot_y": True, "softmax": True, "alpha": 0.8, "beta": 0.2}, {}],
     [TverskyLoss, {"to_onehot_y": True, "softmax": True, "alpha": 0.8, "beta": 0.2, "batch": True}, {}],
     [TverskyLoss, {"to_onehot_y": True, "softmax": True, "alpha": 1.0, "beta": 0.0}, {}],
+    [TverskyLoss, {"to_onehot_y": False, "softmax": True, "alpha": 1.0, "beta": 0.0}, {}],
 ]
 
 
@@ -80,6 +86,8 @@ def test_convergence(self, loss_type, loss_args, forward_args):
         num_classes = 2
         num_voxels = 3 * 4 * 4
 
+        target_onehot = one_hot(target_seg, num_classes=num_classes)
+
         # define a one layer model
         class OnelayerNet(nn.Module):
             def __init__(self):
@@ -118,7 +126,10 @@ def forward(self, x):
             if init_output is None:
                 init_output = torch.argmax(output, 1).detach().cpu().numpy()
 
-            loss_val = loss(output, target_seg, **forward_args)
+            if loss_args["to_onehot_y"] is False:
+                loss_val = loss(output, target_onehot, **forward_args)
+            else:
+                loss_val = loss(output, target_seg, **forward_args)
 
             if iter_i % 10 == 0:
                 pred = torch.argmax(output, 1).detach().cpu().numpy()
diff --git a/tests/test_senet.py b/tests/test_senet.py
index 883d75d62d..1c6222d6a0 100644
--- a/tests/test_senet.py
+++ b/tests/test_senet.py
@@ -10,32 +10,36 @@
 # limitations under the License.
 
 import unittest
+from typing import TYPE_CHECKING
+from unittest import skipUnless
 
 import torch
 from parameterized import parameterized
 
 from monai.networks import eval_mode
-from monai.networks.nets import (
-    se_resnet50,
-    se_resnet101,
-    se_resnet152,
-    se_resnext50_32x4d,
-    se_resnext101_32x4d,
-    senet154,
-)
+from monai.networks.nets import SENet154, SEResNet50, SEResNet101, SEResNet152, SEResNext50, SEResNext101
+from monai.utils import optional_import
 from tests.utils import test_pretrained_networks, test_script_save
 
+if TYPE_CHECKING:
+    import pretrainedmodels
+
+    has_cadene_pretrain = True
+else:
+    pretrainedmodels, has_cadene_pretrain = optional_import("pretrainedmodels")
+
+
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 NET_ARGS = {"spatial_dims": 3, "in_channels": 2, "num_classes": 2}
-TEST_CASE_1 = [senet154, NET_ARGS]
-TEST_CASE_2 = [se_resnet50, NET_ARGS]
-TEST_CASE_3 = [se_resnet101, NET_ARGS]
-TEST_CASE_4 = [se_resnet152, NET_ARGS]
-TEST_CASE_5 = [se_resnext50_32x4d, NET_ARGS]
-TEST_CASE_6 = [se_resnext101_32x4d, NET_ARGS]
+TEST_CASE_1 = [SENet154, NET_ARGS]
+TEST_CASE_2 = [SEResNet50, NET_ARGS]
+TEST_CASE_3 = [SEResNet101, NET_ARGS]
+TEST_CASE_4 = [SEResNet152, NET_ARGS]
+TEST_CASE_5 = [SEResNext50, NET_ARGS]
+TEST_CASE_6 = [SEResNext101, NET_ARGS]
 
-TEST_CASE_PRETRAINED = [se_resnet50, {"spatial_dims": 2, "in_channels": 3, "num_classes": 2, "pretrained": True}]
+TEST_CASE_PRETRAINED_1 = [SEResNet50, {"spatial_dims": 2, "in_channels": 3, "num_classes": 2, "pretrained": True}]
 
 
 class TestSENET(unittest.TestCase):
@@ -56,11 +60,7 @@ def test_script(self, net, net_args):
 
 
 class TestPretrainedSENET(unittest.TestCase):
-    @parameterized.expand(
-        [
-            TEST_CASE_PRETRAINED,
-        ]
-    )
+    @parameterized.expand([TEST_CASE_PRETRAINED_1])
     def test_senet_shape(self, model, input_param):
         net = test_pretrained_networks(model, input_param, device)
         input_data = torch.randn(3, 3, 64, 64).to(device)
@@ -70,6 +70,21 @@ def test_senet_shape(self, model, input_param):
             result = net(input_data)
             self.assertEqual(result.shape, expected_shape)
 
+    @parameterized.expand([TEST_CASE_PRETRAINED_1])
+    @skipUnless(has_cadene_pretrain, "Requires `pretrainedmodels` package.")
+    def test_pretrain_consistency(self, model, input_param):
+        input_data = torch.randn(1, 3, 64, 64).to(device)
+        net = test_pretrained_networks(model, input_param, device)
+        with eval_mode(net):
+            result = net.features(input_data)
+        cadene_net = pretrainedmodels.se_resnet50().to(device)
+        with eval_mode(cadene_net):
+            expected_result = cadene_net.features(input_data)
+        # The difference between Cadene's senet and our version is that
+        # we use nn.Linear as the FC layer, but Cadene's version uses
+        # a conv layer with kernel size equals to 1. It may bring a little difference.
+        self.assertTrue(torch.allclose(result, expected_result, rtol=1e-5, atol=1e-5))
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_smartcache_patch_wsi_dataset.py b/tests/test_smartcache_patch_wsi_dataset.py
new file mode 100644
index 0000000000..876a30a3b8
--- /dev/null
+++ b/tests/test_smartcache_patch_wsi_dataset.py
@@ -0,0 +1,172 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import unittest
+from unittest import skipUnless
+
+import numpy as np
+from numpy.testing import assert_array_equal
+from parameterized import parameterized
+
+from monai.apps.pathology.datasets import SmartCachePatchWSIDataset
+from monai.apps.utils import download_url
+from monai.utils import optional_import
+
+_, has_cim = optional_import("cucim")
+
+FILE_URL = "http://openslide.cs.cmu.edu/download/openslide-testdata/Generic-TIFF/CMU-1.tiff"
+FILE_PATH = os.path.join(os.path.dirname(__file__), "testing_data", "temp_" + os.path.basename(FILE_URL))
+
+TEST_CASE_0 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "location": [0, 0], "label": [0]},
+            {"image": FILE_PATH, "location": [0, 0], "label": [1]},
+            {"image": FILE_PATH, "location": [0, 0], "label": [2]},
+            {"image": FILE_PATH, "location": [0, 0], "label": [3]},
+        ],
+        "region_size": (1, 1),
+        "grid_shape": (1, 1),
+        "patch_size": 1,
+        "transform": lambda x: x,
+        "image_reader_name": "cuCIM",
+        "replace_rate": 0.5,
+        "cache_num": 2,
+        "num_init_workers": 1,
+        "num_replace_workers": 1,
+    },
+    [
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[3]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[3]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[0]]])},
+    ],
+]
+
+TEST_CASE_1 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "location": [0, 0], "label": [[0, 0]]},
+            {"image": FILE_PATH, "location": [0, 0], "label": [[1, 1]]},
+            {"image": FILE_PATH, "location": [0, 0], "label": [[2, 2]]},
+        ],
+        "region_size": (1, 1),
+        "grid_shape": (1, 1),
+        "patch_size": 1,
+        "transform": lambda x: x,
+        "image_reader_name": "cuCIM",
+        "replace_rate": 0.5,
+        "cache_num": 2,
+        "num_init_workers": 1,
+        "num_replace_workers": 1,
+    },
+    [
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[0, 0]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[1, 1]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[1, 1]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[2, 2]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[2, 2]]])},
+        {"image": np.array([[[239]], [[239]], [[239]]], dtype=np.uint8), "label": np.array([[[0, 0]]])},
+    ],
+]
+
+TEST_CASE_2 = [
+    {
+        "data": [
+            {"image": FILE_PATH, "location": [10004, 20004], "label": [0, 0, 0, 0]},
+            {"image": FILE_PATH, "location": [10004, 20004], "label": [1, 1, 1, 1]},
+            {"image": FILE_PATH, "location": [10004, 20004], "label": [2, 2, 2, 2]},
+        ],
+        "region_size": (8, 8),
+        "grid_shape": (2, 2),
+        "patch_size": 1,
+        "transform": lambda x: x,
+        "image_reader_name": "cuCIM",
+        "replace_rate": 0.5,
+        "cache_num": 2,
+        "num_init_workers": 1,
+        "num_replace_workers": 1,
+    },
+    [
+        {"image": np.array([[[247]], [[245]], [[248]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[245]], [[247]], [[244]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[247]], [[245]], [[248]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[245]], [[247]], [[244]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[247]], [[245]], [[248]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[245]], [[247]], [[244]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[1]]])},
+        {"image": np.array([[[247]], [[245]], [[248]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[245]], [[247]], [[244]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[247]], [[245]], [[248]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[245]], [[247]], [[244]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[2]]])},
+        {"image": np.array([[[247]], [[245]], [[248]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[245]], [[247]], [[244]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[0]]])},
+        {"image": np.array([[[246]], [[246]], [[246]]], dtype=np.uint8), "label": np.array([[[0]]])},
+    ],
+]
+
+
+class TestSmartCachePatchWSIDataset(unittest.TestCase):
+    def setUp(self):
+        download_url(FILE_URL, FILE_PATH, "5a3cfd4fd725c50578ddb80b517b759f")
+
+    @parameterized.expand(
+        [
+            TEST_CASE_0,
+            TEST_CASE_1,
+            TEST_CASE_2,
+        ]
+    )
+    @skipUnless(has_cim, "Requires CuCIM")
+    def test_read_patches(self, input_parameters, expected):
+        dataset = SmartCachePatchWSIDataset(**input_parameters)
+        self.assertEqual(len(dataset), input_parameters["cache_num"])
+        total_num_samples = len(input_parameters["data"])
+        num_epochs = int(
+            np.ceil(total_num_samples / (input_parameters["cache_num"] * input_parameters["replace_rate"]))
+        )
+
+        dataset.start()
+        i = 0
+        for _ in range(num_epochs):
+            for j in range(len(dataset)):
+                samples = dataset[j]
+                n_patches = len(samples)
+                self.assert_samples_expected(samples, expected[i : i + n_patches])
+                i += n_patches
+            dataset.update_cache()
+        dataset.shutdown()
+
+    def assert_samples_expected(self, samples, expected):
+        for i in range(len(samples)):
+            self.assertTupleEqual(samples[i]["label"].shape, expected[i]["label"].shape)
+            self.assertTupleEqual(samples[i]["image"].shape, expected[i]["image"].shape)
+            self.assertIsNone(assert_array_equal(samples[i]["label"], expected[i]["label"]))
+            self.assertIsNone(assert_array_equal(samples[i]["image"], expected[i]["image"]))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_smartcachedataset.py b/tests/test_smartcachedataset.py
index 3d1a051a83..992e8ae43b 100644
--- a/tests/test_smartcachedataset.py
+++ b/tests/test_smartcachedataset.py
@@ -24,13 +24,15 @@
 
 TEST_CASE_2 = [0.1, 4, Compose([LoadImaged(keys=["image", "label", "extra"])])]
 
-TEST_CASE_3 = [0.1, 4, None]
+TEST_CASE_3 = [0.1, None, Compose([LoadImaged(keys=["image", "label", "extra"])])]
 
-TEST_CASE_4 = [0.5, 2, Compose([LoadImaged(keys=["image", "label", "extra"])])]
+TEST_CASE_4 = [0.1, 4, None]
+
+TEST_CASE_5 = [0.5, 2, Compose([LoadImaged(keys=["image", "label", "extra"])])]
 
 
 class TestSmartCacheDataset(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4])
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5])
     def test_shape(self, replace_rate, num_replace_workers, transform):
         test_image = nib.Nifti1Image(np.random.randint(0, 2, size=[8, 8, 8]), np.eye(4))
         with tempfile.TemporaryDirectory() as tempdir:
@@ -61,13 +63,39 @@ def test_shape(self, replace_rate, num_replace_workers, transform):
                 dataset.start()
                 for _ in range(3):
                     dataset.update_cache()
-                    self.assertIsNotNone(dataset._cache[15])
-                    if isinstance(dataset._cache[15]["image"], np.ndarray):
-                        np.testing.assert_allclose(dataset._cache[15]["image"], dataset._cache[15]["label"])
+                    self.assertIsNotNone(dataset[15])
+                    if isinstance(dataset[15]["image"], np.ndarray):
+                        np.testing.assert_allclose(dataset[15]["image"], dataset[15]["label"])
                     else:
-                        self.assertIsInstance(dataset._cache[15]["image"], str)
+                        self.assertIsInstance(dataset[15]["image"], str)
                 dataset.shutdown()
 
+    def test_shuffle(self):
+        test_data = [{"image": f"test_image{i}.nii.gz"} for i in range(20)]
+        dataset = SmartCacheDataset(
+            data=test_data,
+            transform=None,
+            replace_rate=0.1,
+            cache_num=16,
+            num_init_workers=4,
+            num_replace_workers=4,
+            shuffle=True,
+            seed=123,
+        )
+
+        dataset.start()
+        for i in range(3):
+            dataset.update_cache()
+
+            if i == 0:
+                self.assertEqual(dataset[15]["image"], "test_image18.nii.gz")
+            elif i == 1:
+                self.assertEqual(dataset[15]["image"], "test_image13.nii.gz")
+            else:
+                self.assertEqual(dataset[15]["image"], "test_image5.nii.gz")
+
+        dataset.shutdown()
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_spacingd.py b/tests/test_spacingd.py
index ec32563543..e4efe4241d 100644
--- a/tests/test_spacingd.py
+++ b/tests/test_spacingd.py
@@ -21,7 +21,7 @@ def test_spacingd_3d(self):
         data = {"image": np.ones((2, 10, 15, 20)), "image_meta_dict": {"affine": np.eye(4)}}
         spacing = Spacingd(keys="image", pixdim=(1, 2, 1.4))
         res = spacing(data)
-        self.assertEqual(("image", "image_meta_dict"), tuple(sorted(res)))
+        self.assertEqual(("image", "image_meta_dict", "image_transforms"), tuple(sorted(res)))
         np.testing.assert_allclose(res["image"].shape, (2, 10, 8, 15))
         np.testing.assert_allclose(res["image_meta_dict"]["affine"], np.diag([1, 2, 1.4, 1.0]))
 
@@ -29,7 +29,7 @@ def test_spacingd_2d(self):
         data = {"image": np.ones((2, 10, 20)), "image_meta_dict": {"affine": np.eye(3)}}
         spacing = Spacingd(keys="image", pixdim=(1, 2, 1.4))
         res = spacing(data)
-        self.assertEqual(("image", "image_meta_dict"), tuple(sorted(res)))
+        self.assertEqual(("image", "image_meta_dict", "image_transforms"), tuple(sorted(res)))
         np.testing.assert_allclose(res["image"].shape, (2, 10, 10))
         np.testing.assert_allclose(res["image_meta_dict"]["affine"], np.diag((1, 2, 1)))
 
@@ -49,7 +49,10 @@ def test_interp_all(self):
             ),
         )
         res = spacing(data)
-        self.assertEqual(("image", "image_meta_dict", "seg", "seg_meta_dict"), tuple(sorted(res)))
+        self.assertEqual(
+            ("image", "image_meta_dict", "image_transforms", "seg", "seg_meta_dict", "seg_transforms"),
+            tuple(sorted(res)),
+        )
         np.testing.assert_allclose(res["image"].shape, (2, 1, 46))
         np.testing.assert_allclose(res["image_meta_dict"]["affine"], np.diag((1, 0.2, 1, 1)))
 
@@ -69,7 +72,10 @@ def test_interp_sep(self):
             ),
         )
         res = spacing(data)
-        self.assertEqual(("image", "image_meta_dict", "seg", "seg_meta_dict"), tuple(sorted(res)))
+        self.assertEqual(
+            ("image", "image_meta_dict", "image_transforms", "seg", "seg_meta_dict", "seg_transforms"),
+            tuple(sorted(res)),
+        )
         np.testing.assert_allclose(res["image"].shape, (2, 1, 46))
         np.testing.assert_allclose(res["image_meta_dict"]["affine"], np.diag((1, 0.2, 1, 1)))
 
diff --git a/tests/test_spatial_crop.py b/tests/test_spatial_crop.py
index f3c904889f..c76915f0a3 100644
--- a/tests/test_spatial_crop.py
+++ b/tests/test_spatial_crop.py
@@ -12,6 +12,7 @@
 import unittest
 
 import numpy as np
+import torch
 from parameterized import parameterized
 
 from monai.transforms import SpatialCrop
@@ -39,6 +40,15 @@
         (3, 3, 3, 3),
         (3, 0, 3, 3),
     ],
+    [
+        {"roi_slices": [slice(s, e) for s, e in zip([-1, -2, 0], [None, None, 2])]},
+        (3, 3, 3, 3),
+        (3, 1, 2, 2),
+    ],
+]
+
+TEST_ERRORS = [
+    [{"roi_slices": [slice(s, e, 2) for s, e in zip([-1, -2, 0], [None, None, 2])]}],
 ]
 
 
@@ -49,6 +59,17 @@ def test_shape(self, input_param, input_shape, expected_shape):
         result = SpatialCrop(**input_param)(input_data)
         self.assertTupleEqual(result.shape, expected_shape)
 
+    @parameterized.expand(TEST_CASES)
+    def test_tensor_shape(self, input_param, input_shape, expected_shape):
+        input_data = torch.randint(0, 2, size=input_shape, device="cuda" if torch.cuda.is_available() else "cpu")
+        result = SpatialCrop(**input_param)(input_data)
+        self.assertTupleEqual(result.shape, expected_shape)
+
+    @parameterized.expand(TEST_ERRORS)
+    def test_error(self, input_param):
+        with self.assertRaises(ValueError):
+            SpatialCrop(**input_param)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_spatial_cropd.py b/tests/test_spatial_cropd.py
index 590dc83281..797c25d34b 100644
--- a/tests/test_spatial_cropd.py
+++ b/tests/test_spatial_cropd.py
@@ -16,33 +16,37 @@
 
 from monai.transforms import SpatialCropd
 
-TEST_CASE_1 = [
-    {"keys": ["img"], "roi_center": [1, 1, 1], "roi_size": [2, 2, 2]},
-    {"img": np.random.randint(0, 2, size=[3, 3, 3, 3])},
-    (3, 2, 2, 2),
-]
-
-TEST_CASE_2 = [
-    {"keys": ["img"], "roi_start": [0, 0, 0], "roi_end": [2, 2, 2]},
-    {"img": np.random.randint(0, 2, size=[3, 3, 3, 3])},
-    (3, 2, 2, 2),
-]
-
-TEST_CASE_3 = [
-    {"keys": ["img"], "roi_start": [0, 0], "roi_end": [2, 2]},
-    {"img": np.random.randint(0, 2, size=[3, 3, 3, 3])},
-    (3, 2, 2, 3),
-]
-
-TEST_CASE_4 = [
-    {"keys": ["img"], "roi_start": [0, 0, 0, 0, 0], "roi_end": [2, 2, 2, 2, 2]},
-    {"img": np.random.randint(0, 2, size=[3, 3, 3, 3])},
-    (3, 2, 2, 2),
+TEST_CASES = [
+    [
+        {"keys": ["img"], "roi_center": [1, 1, 1], "roi_size": [2, 2, 2]},
+        {"img": np.random.randint(0, 2, size=[3, 3, 3, 3])},
+        (3, 2, 2, 2),
+    ],
+    [
+        {"keys": ["img"], "roi_start": [0, 0, 0], "roi_end": [2, 2, 2]},
+        {"img": np.random.randint(0, 2, size=[3, 3, 3, 3])},
+        (3, 2, 2, 2),
+    ],
+    [
+        {"keys": ["img"], "roi_start": [0, 0], "roi_end": [2, 2]},
+        {"img": np.random.randint(0, 2, size=[3, 3, 3, 3])},
+        (3, 2, 2, 3),
+    ],
+    [
+        {"keys": ["img"], "roi_start": [0, 0, 0, 0, 0], "roi_end": [2, 2, 2, 2, 2]},
+        {"img": np.random.randint(0, 2, size=[3, 3, 3, 3])},
+        (3, 2, 2, 2),
+    ],
+    [
+        {"keys": ["img"], "roi_slices": [slice(s, e) for s, e in zip([-1, -2, 0], [None, None, 2])]},
+        {"img": np.random.randint(0, 2, size=[3, 3, 3, 3])},
+        (3, 1, 2, 2),
+    ],
 ]
 
 
 class TestSpatialCropd(unittest.TestCase):
-    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4])
+    @parameterized.expand(TEST_CASES)
     def test_shape(self, input_param, input_data, expected_shape):
         result = SpatialCropd(**input_param)(input_data)
         self.assertTupleEqual(result["img"].shape, expected_shape)
diff --git a/tests/test_std_shift_intensity.py b/tests/test_std_shift_intensity.py
new file mode 100644
index 0000000000..f317330435
--- /dev/null
+++ b/tests/test_std_shift_intensity.py
@@ -0,0 +1,66 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+from monai.transforms import ShiftIntensity, StdShiftIntensity
+from tests.utils import NumpyImageTestCase2D
+
+
+class TestStdShiftIntensity(NumpyImageTestCase2D):
+    def test_value(self):
+        factor = np.random.rand()
+        offset = np.std(self.imt) * factor
+        shifter = ShiftIntensity(offset=offset)
+        expected = shifter(self.imt)
+        std_shifter = StdShiftIntensity(factor=factor)
+        result = std_shifter(self.imt)
+        np.testing.assert_allclose(result, expected, rtol=1e-5)
+
+    def test_zerostd(self):
+        image = np.ones([2, 3, 3])
+        for nonzero in [True, False]:
+            for channel_wise in [True, False]:
+                factor = np.random.rand()
+                std_shifter = StdShiftIntensity(factor=factor, nonzero=nonzero, channel_wise=channel_wise)
+                result = std_shifter(image)
+                np.testing.assert_allclose(result, image, rtol=1e-5)
+
+    def test_nonzero(self):
+        image = np.asarray([[4.0, 0.0, 2.0], [0, 2, 4]])  # std = 1
+        factor = np.random.rand()
+        std_shifter = StdShiftIntensity(factor=factor, nonzero=True)
+        result = std_shifter(image)
+        expected = np.asarray([[4 + factor, 0, 2 + factor], [0, 2 + factor, 4 + factor]])
+        np.testing.assert_allclose(result, expected, rtol=1e-5)
+
+    def test_channel_wise(self):
+        image = np.stack((np.asarray([1.0, 2.0]), np.asarray([1.0, 1.0])))  # std: 0.5, 0
+        factor = np.random.rand()
+        std_shifter = StdShiftIntensity(factor=factor, channel_wise=True)
+        result = std_shifter(image)
+        expected = np.stack((np.asarray([1 + 0.5 * factor, 2 + 0.5 * factor]), np.asarray([1, 1])))
+        np.testing.assert_allclose(result, expected, rtol=1e-5)
+
+    def test_dtype(self):
+        trans_dtype = np.float32
+        for dtype in [int, np.float32, np.float64]:
+            image = np.random.rand(2, 2, 2).astype(dtype)
+            factor = np.random.rand()
+            std_shifter = StdShiftIntensity(factor=factor, dtype=trans_dtype)
+            result = std_shifter(image)
+            np.testing.assert_equal(result.dtype, trans_dtype)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_std_shift_intensityd.py b/tests/test_std_shift_intensityd.py
new file mode 100644
index 0000000000..4eb256f1e5
--- /dev/null
+++ b/tests/test_std_shift_intensityd.py
@@ -0,0 +1,71 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+from monai.transforms import ShiftIntensityd, StdShiftIntensityd
+from tests.utils import NumpyImageTestCase2D
+
+
+class TestStdShiftIntensityd(NumpyImageTestCase2D):
+    def test_value(self):
+        key = "img"
+        factor = np.random.rand()
+        offset = np.std(self.imt) * factor
+        shifter = ShiftIntensityd(keys=[key], offset=offset)
+        expected = shifter({key: self.imt})
+        std_shifter = StdShiftIntensityd(keys=[key], factor=factor)
+        result = std_shifter({key: self.imt})
+        np.testing.assert_allclose(result[key], expected[key], rtol=1e-5)
+
+    def test_zerostd(self):
+        key = "img"
+        image = np.ones([2, 3, 3])
+        for nonzero in [True, False]:
+            for channel_wise in [True, False]:
+                factor = np.random.rand()
+                std_shifter = StdShiftIntensityd(keys=[key], factor=factor, nonzero=nonzero, channel_wise=channel_wise)
+                result = std_shifter({key: image})
+                np.testing.assert_allclose(result[key], image, rtol=1e-5)
+
+    def test_nonzero(self):
+        key = "img"
+        image = np.asarray([[4.0, 0.0, 2.0], [0, 2, 4]])  # std = 1
+        factor = np.random.rand()
+        std_shifter = StdShiftIntensityd(keys=[key], factor=factor, nonzero=True)
+        result = std_shifter({key: image})
+        expected = np.asarray([[4 + factor, 0, 2 + factor], [0, 2 + factor, 4 + factor]])
+        np.testing.assert_allclose(result[key], expected, rtol=1e-5)
+
+    def test_channel_wise(self):
+        key = "img"
+        image = np.stack((np.asarray([1.0, 2.0]), np.asarray([1.0, 1.0])))  # std: 0.5, 0
+        factor = np.random.rand()
+        std_shifter = StdShiftIntensityd(keys=[key], factor=factor, channel_wise=True)
+        result = std_shifter({key: image})
+        expected = np.stack((np.asarray([1 + 0.5 * factor, 2 + 0.5 * factor]), np.asarray([1, 1])))
+        np.testing.assert_allclose(result[key], expected, rtol=1e-5)
+
+    def test_dtype(self):
+        key = "img"
+        trans_dtype = np.float32
+        for dtype in [int, np.float32, np.float64]:
+            image = np.random.rand(2, 2, 2).astype(dtype)
+            factor = np.random.rand()
+            std_shifter = StdShiftIntensityd(keys=[key], factor=factor, dtype=trans_dtype)
+            result = std_shifter({key: image})
+            np.testing.assert_equal(result[key].dtype, trans_dtype)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_testtimeaugmentation.py b/tests/test_testtimeaugmentation.py
new file mode 100644
index 0000000000..34e8c62bd5
--- /dev/null
+++ b/tests/test_testtimeaugmentation.py
@@ -0,0 +1,162 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from functools import partial
+from typing import TYPE_CHECKING
+
+import numpy as np
+import torch
+
+from monai.data import CacheDataset, DataLoader, create_test_image_2d
+from monai.data.test_time_augmentation import TestTimeAugmentation
+from monai.data.utils import pad_list_data_collate
+from monai.losses import DiceLoss
+from monai.networks.nets import UNet
+from monai.transforms import Activations, AddChanneld, AsDiscrete, Compose, CropForegroundd, DivisiblePadd, RandAffined
+from monai.transforms.croppad.dictionary import SpatialPadd
+from monai.transforms.spatial.dictionary import Rand2DElasticd, RandFlipd, Spacingd
+from monai.utils import optional_import, set_determinism
+
+if TYPE_CHECKING:
+    import tqdm
+
+    has_tqdm = True
+    has_nib = True
+else:
+    tqdm, has_tqdm = optional_import("tqdm")
+    _, has_nib = optional_import("nibabel")
+
+trange = partial(tqdm.trange, desc="training") if has_tqdm else range
+
+
+class TestTestTimeAugmentation(unittest.TestCase):
+    @staticmethod
+    def get_data(num_examples, input_size, include_label=True):
+        custom_create_test_image_2d = partial(
+            create_test_image_2d, *input_size, rad_max=7, num_seg_classes=1, num_objs=1
+        )
+        data = []
+        for _ in range(num_examples):
+            im, label = custom_create_test_image_2d()
+            d = {}
+            d["image"] = im
+            d["image_meta_dict"] = {"affine": np.eye(4)}
+            if include_label:
+                d["label"] = label
+                d["label_meta_dict"] = {"affine": np.eye(4)}
+            data.append(d)
+        return data[0] if num_examples == 1 else data
+
+    def setUp(self) -> None:
+        set_determinism(seed=0)
+
+    def tearDown(self) -> None:
+        set_determinism(None)
+
+    def test_test_time_augmentation(self):
+        input_size = (20, 20)
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+        keys = ["image", "label"]
+        num_training_ims = 10
+        train_data = self.get_data(num_training_ims, input_size)
+        test_data = self.get_data(1, input_size)
+
+        transforms = Compose(
+            [
+                AddChanneld(keys),
+                RandAffined(
+                    keys,
+                    prob=1.0,
+                    spatial_size=(30, 30),
+                    rotate_range=(np.pi / 3, np.pi / 3),
+                    translate_range=(3, 3),
+                    scale_range=((0.8, 1), (0.8, 1)),
+                    padding_mode="zeros",
+                    mode=("bilinear", "nearest"),
+                    as_tensor_output=False,
+                ),
+                CropForegroundd(keys, source_key="image"),
+                DivisiblePadd(keys, 4),
+            ]
+        )
+
+        train_ds = CacheDataset(train_data, transforms)
+        # output might be different size, so pad so that they match
+        train_loader = DataLoader(train_ds, batch_size=2, collate_fn=pad_list_data_collate)
+
+        model = UNet(2, 1, 1, channels=(6, 6), strides=(2, 2)).to(device)
+        loss_function = DiceLoss(sigmoid=True)
+        optimizer = torch.optim.Adam(model.parameters(), 1e-3)
+
+        num_epochs = 10
+        for _ in trange(num_epochs):
+            epoch_loss = 0
+
+            for batch_data in train_loader:
+                inputs, labels = batch_data["image"].to(device), batch_data["label"].to(device)
+                optimizer.zero_grad()
+                outputs = model(inputs)
+                loss = loss_function(outputs, labels)
+                loss.backward()
+                optimizer.step()
+                epoch_loss += loss.item()
+
+            epoch_loss /= len(train_loader)
+
+        post_trans = Compose(
+            [
+                Activations(sigmoid=True),
+                AsDiscrete(threshold_values=True),
+            ]
+        )
+
+        def inferrer_fn(x):
+            return post_trans(model(x))
+
+        tt_aug = TestTimeAugmentation(transforms, batch_size=5, num_workers=0, inferrer_fn=inferrer_fn, device=device)
+        mode, mean, std, vvc = tt_aug(test_data)
+        self.assertEqual(mode.shape, (1,) + input_size)
+        self.assertEqual(mean.shape, (1,) + input_size)
+        self.assertTrue(all(np.unique(mode) == (0, 1)))
+        self.assertEqual((mean.min(), mean.max()), (0.0, 1.0))
+        self.assertEqual(std.shape, (1,) + input_size)
+        self.assertIsInstance(vvc, float)
+
+    def test_fail_non_random(self):
+        transforms = Compose([AddChanneld("im"), SpatialPadd("im", 1)])
+        with self.assertRaises(RuntimeError):
+            TestTimeAugmentation(transforms, None, None, None)
+
+    def test_fail_random_but_not_invertible(self):
+        transforms = Compose([AddChanneld("im"), Rand2DElasticd("im", None, None)])
+        with self.assertRaises(RuntimeError):
+            TestTimeAugmentation(transforms, None, None, None)
+
+    def test_single_transform(self):
+        transforms = RandFlipd(["image", "label"])
+        tta = TestTimeAugmentation(transforms, batch_size=5, num_workers=0, inferrer_fn=lambda x: x)
+        tta(self.get_data(1, (20, 20)))
+
+    def test_image_no_label(self):
+        transforms = RandFlipd(["image"])
+        tta = TestTimeAugmentation(transforms, batch_size=5, num_workers=0, inferrer_fn=lambda x: x, label_key="image")
+        tta(self.get_data(1, (20, 20), include_label=False))
+
+    @unittest.skipUnless(has_nib, "Requires nibabel")
+    def test_requires_meta_dict(self):
+        transforms = Compose([RandFlipd("image"), Spacingd("image", (1, 1))])
+        tta = TestTimeAugmentation(transforms, batch_size=5, num_workers=0, inferrer_fn=lambda x: x, label_key="image")
+        tta(self.get_data(1, (20, 20), include_label=False))
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_thread_buffer.py b/tests/test_thread_buffer.py
index 07e5a779ca..1b3ebb910d 100644
--- a/tests/test_thread_buffer.py
+++ b/tests/test_thread_buffer.py
@@ -9,10 +9,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
 import time
 import unittest
 
-from monai.data import DataLoader, Dataset, ThreadBuffer
+from monai.data import DataLoader, Dataset, ThreadBuffer, ThreadDataLoader
 from monai.transforms import Compose, SimulateDelayd
 from monai.utils import PerfContext
 
@@ -40,6 +41,16 @@ def test_values(self):
             self.assertEqual(d["label"][0], "spleen_label_19.nii.gz")
             self.assertEqual(d["label"][1], "spleen_label_31.nii.gz")
 
+    def test_dataloader(self):
+        dataset = Dataset(data=self.datalist, transform=self.transform)
+        dataloader = ThreadDataLoader(dataset=dataset, batch_size=2, num_workers=0)
+
+        for d in dataloader:
+            self.assertEqual(d["image"][0], "spleen_19.nii.gz")
+            self.assertEqual(d["image"][1], "spleen_31.nii.gz")
+            self.assertEqual(d["label"][0], "spleen_label_19.nii.gz")
+            self.assertEqual(d["label"][1], "spleen_label_31.nii.gz")
+
     def test_time(self):
         dataset = Dataset(data=self.datalist * 2, transform=self.transform)  # contains data for 2 batches
         dataloader = DataLoader(dataset=dataset, batch_size=2, num_workers=0)
@@ -57,11 +68,13 @@ def test_time(self):
                 time.sleep(0.5)  # while "computation" is happening the next batch is being generated, saving 0.4 s
 
         buffered_time = pc.total_time
-
-        self.assertTrue(
-            buffered_time < unbuffered_time,
-            f"Buffered time {buffered_time} should be less than unbuffered time {unbuffered_time}",
-        )
+        if sys.platform == "darwin":  # skip macOS measure
+            print(f"darwin: Buffered time {buffered_time} vs unbuffered time {unbuffered_time}")
+        else:
+            self.assertTrue(
+                buffered_time < unbuffered_time,
+                f"Buffered time {buffered_time} should be less than unbuffered time {unbuffered_time}",
+            )
 
 
 if __name__ == "__main__":
diff --git a/tests/test_threadcontainer.py b/tests/test_threadcontainer.py
new file mode 100644
index 0000000000..68d7826a50
--- /dev/null
+++ b/tests/test_threadcontainer.py
@@ -0,0 +1,111 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import tempfile
+import time
+import unittest
+
+import torch
+
+from monai.data import DataLoader
+from monai.utils import optional_import, set_determinism
+from monai.utils.enums import CommonKeys
+from tests.utils import SkipIfNoModule
+
+try:
+    _, has_ignite = optional_import("ignite")
+
+    from monai.engines import SupervisedTrainer
+    from monai.handlers import MetricLogger
+    from monai.utils import ThreadContainer
+except ImportError:
+    has_ignite = False
+
+compare_images, _ = optional_import("matplotlib.testing.compare", name="compare_images")
+
+
+class TestThreadContainer(unittest.TestCase):
+    @SkipIfNoModule("ignite")
+    def test_container(self):
+        net = torch.nn.Conv2d(1, 1, 3, padding=1)
+
+        opt = torch.optim.Adam(net.parameters())
+
+        img = torch.rand(1, 16, 16)
+        data = {CommonKeys.IMAGE: img, CommonKeys.LABEL: img}
+        loader = DataLoader([data for _ in range(10)])
+
+        trainer = SupervisedTrainer(
+            device=torch.device("cpu"),
+            max_epochs=1,
+            train_data_loader=loader,
+            network=net,
+            optimizer=opt,
+            loss_function=torch.nn.L1Loss(),
+        )
+
+        con = ThreadContainer(trainer)
+        con.start()
+        time.sleep(1)  # wait for trainer to start
+
+        self.assertTrue(con.is_alive)
+        self.assertIsNotNone(con.status())
+        self.assertTrue(len(con.status_dict) > 0)
+
+        con.join()
+
+    @SkipIfNoModule("ignite")
+    @SkipIfNoModule("matplotlib")
+    def test_plot(self):
+        set_determinism(0)
+
+        testing_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testing_data")
+
+        net = torch.nn.Conv2d(1, 1, 3, padding=1)
+
+        opt = torch.optim.Adam(net.parameters())
+
+        img = torch.rand(1, 16, 16)
+
+        # a third non-image key is added to test that this is correctly ignored when plotting
+        data = {CommonKeys.IMAGE: img, CommonKeys.LABEL: img, "Not Image Data": ["This isn't an image"]}
+
+        loader = DataLoader([data] * 10)
+
+        trainer = SupervisedTrainer(
+            device=torch.device("cpu"),
+            max_epochs=1,
+            train_data_loader=loader,
+            network=net,
+            optimizer=opt,
+            loss_function=torch.nn.L1Loss(),
+        )
+
+        logger = MetricLogger()
+        logger.attach(trainer)
+
+        con = ThreadContainer(trainer)
+        con.start()
+        con.join()
+
+        fig = con.plot_status(logger)
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            tempimg = f"{tempdir}/threadcontainer_plot_test.png"
+            fig.savefig(tempimg)
+            comp = compare_images(f"{testing_dir}/threadcontainer_plot_test.png", tempimg, 1e-3)
+
+            self.assertIsNone(comp, comp)  # None indicates test passed
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_timedcall.py b/tests/test_timedcall.py
index e87d160743..de10abb8f7 100644
--- a/tests/test_timedcall.py
+++ b/tests/test_timedcall.py
@@ -10,13 +10,14 @@
 # limitations under the License.
 
 import multiprocessing
+import sys
 import time
 import unittest
 
 from tests.utils import TimedCall
 
 
-@TimedCall(seconds=10, force_quit=False)
+@TimedCall(seconds=10 if sys.platform == "linux" else 60, force_quit=False)
 def case_1_seconds(arg=None):
     time.sleep(1)
     return "good" if not arg else arg
diff --git a/tests/test_to_pil.py b/tests/test_to_pil.py
new file mode 100644
index 0000000000..ec63750ce4
--- /dev/null
+++ b/tests/test_to_pil.py
@@ -0,0 +1,64 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from typing import TYPE_CHECKING
+from unittest import skipUnless
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.transforms import ToPIL
+from monai.utils import optional_import
+
+if TYPE_CHECKING:
+    from PIL.Image import Image as PILImageImage
+    from PIL.Image import fromarray as pil_image_fromarray
+
+    has_pil = True
+else:
+    pil_image_fromarray, has_pil = optional_import("PIL.Image", name="fromarray")
+    PILImageImage, _ = optional_import("PIL.Image", name="Image")
+
+TEST_CASE_ARRAY_1 = [np.array([[1.0, 2.0], [3.0, 4.0]])]
+TEST_CASE_TENSOR_1 = [torch.tensor([[1.0, 2.0], [3.0, 4.0]])]
+
+
+class TestToPIL(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_ARRAY_1])
+    @skipUnless(has_pil, "Requires `pillow` package.")
+    def test_numpy_input(self, test_data):
+        self.assertTrue(isinstance(test_data, np.ndarray))
+        result = ToPIL()(test_data)
+        self.assertTrue(isinstance(result, PILImageImage))
+        np.testing.assert_allclose(np.array(result), test_data)
+
+    @parameterized.expand([TEST_CASE_TENSOR_1])
+    @skipUnless(has_pil, "Requires `pillow` package.")
+    def test_tensor_input(self, test_data):
+        self.assertTrue(isinstance(test_data, torch.Tensor))
+        result = ToPIL()(test_data)
+        self.assertTrue(isinstance(result, PILImageImage))
+        np.testing.assert_allclose(np.array(result), test_data.numpy())
+
+    @parameterized.expand([TEST_CASE_ARRAY_1])
+    @skipUnless(has_pil, "Requires `pillow` package.")
+    def test_pil_input(self, test_data):
+        test_data_pil = pil_image_fromarray(test_data)
+        self.assertTrue(isinstance(test_data_pil, PILImageImage))
+        result = ToPIL()(test_data_pil)
+        self.assertTrue(isinstance(result, PILImageImage))
+        np.testing.assert_allclose(np.array(result), test_data)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_to_pild.py b/tests/test_to_pild.py
new file mode 100644
index 0000000000..43778022ee
--- /dev/null
+++ b/tests/test_to_pild.py
@@ -0,0 +1,65 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from typing import TYPE_CHECKING
+from unittest import skipUnless
+
+import numpy as np
+import torch
+from parameterized import parameterized
+
+from monai.transforms import ToPILd
+from monai.utils import optional_import
+
+if TYPE_CHECKING:
+    from PIL.Image import Image as PILImageImage
+    from PIL.Image import fromarray as pil_image_fromarray
+
+    has_pil = True
+else:
+    pil_image_fromarray, has_pil = optional_import("PIL.Image", name="fromarray")
+    PILImageImage, _ = optional_import("PIL.Image", name="Image")
+
+TEST_CASE_ARRAY_1 = [{"keys": "image"}, {"image": np.array([[1.0, 2.0], [3.0, 4.0]])}]
+TEST_CASE__TENSOR_1 = [{"keys": "image"}, {"image": torch.tensor([[1.0, 2.0], [3.0, 4.0]])}]
+
+
+class TestToPIL(unittest.TestCase):
+    @parameterized.expand([TEST_CASE_ARRAY_1])
+    @skipUnless(has_pil, "Requires `pillow` package.")
+    def test_numpy_input(self, input_param, test_data):
+        self.assertTrue(isinstance(test_data[input_param["keys"]], np.ndarray))
+        result = ToPILd(**input_param)(test_data)[input_param["keys"]]
+        self.assertTrue(isinstance(result, PILImageImage))
+        np.testing.assert_allclose(np.array(result), test_data[input_param["keys"]])
+
+    @parameterized.expand([TEST_CASE__TENSOR_1])
+    @skipUnless(has_pil, "Requires `pillow` package.")
+    def test_tensor_input(self, input_param, test_data):
+        self.assertTrue(isinstance(test_data[input_param["keys"]], torch.Tensor))
+        result = ToPILd(**input_param)(test_data)[input_param["keys"]]
+        self.assertTrue(isinstance(result, PILImageImage))
+        np.testing.assert_allclose(np.array(result), test_data[input_param["keys"]].numpy())
+
+    @parameterized.expand([TEST_CASE_ARRAY_1])
+    @skipUnless(has_pil, "Requires `pillow` package.")
+    def test_pil_input(self, input_param, test_data):
+        input_array = test_data[input_param["keys"]]
+        test_data[input_param["keys"]] = pil_image_fromarray(input_array)
+        self.assertTrue(isinstance(test_data[input_param["keys"]], PILImageImage))
+        result = ToPILd(**input_param)(test_data)[input_param["keys"]]
+        self.assertTrue(isinstance(result, PILImageImage))
+        np.testing.assert_allclose(np.array(result), test_data[input_param["keys"]])
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_torchvision_fc_model.py b/tests/test_torchvision_fc_model.py
new file mode 100644
index 0000000000..2c65f0d32c
--- /dev/null
+++ b/tests/test_torchvision_fc_model.py
@@ -0,0 +1,106 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from unittest import skipUnless
+
+import torch
+from parameterized import parameterized
+
+from monai.networks import eval_mode
+from monai.networks.nets import TorchVisionFullyConvModel
+from monai.utils import optional_import
+
+_, has_tv = optional_import("torchvision")
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+TEST_CASE_0 = [
+    {"model_name": "resnet18", "n_classes": 1, "pretrained": False},
+    (2, 3, 224, 224),
+    (2, 1, 1, 1),
+]
+
+TEST_CASE_1 = [
+    {"model_name": "resnet18", "n_classes": 1, "pretrained": False},
+    (2, 3, 256, 256),
+    (2, 1, 2, 2),
+]
+
+TEST_CASE_2 = [
+    {"model_name": "resnet101", "n_classes": 5, "pretrained": False},
+    (2, 3, 256, 256),
+    (2, 5, 2, 2),
+]
+
+TEST_CASE_3 = [
+    {"model_name": "resnet101", "n_classes": 5, "pool_size": 6, "pretrained": False},
+    (2, 3, 224, 224),
+    (2, 5, 2, 2),
+]
+
+TEST_CASE_PRETRAINED_0 = [
+    {"model_name": "resnet18", "n_classes": 1, "pretrained": True},
+    (2, 3, 224, 224),
+    (2, 1, 1, 1),
+    -0.010419349186122417,
+]
+
+TEST_CASE_PRETRAINED_1 = [
+    {"model_name": "resnet18", "n_classes": 1, "pretrained": True},
+    (2, 3, 256, 256),
+    (2, 1, 2, 2),
+    -0.010419349186122417,
+]
+
+TEST_CASE_PRETRAINED_2 = [
+    {"model_name": "resnet18", "n_classes": 5, "pretrained": True},
+    (2, 3, 256, 256),
+    (2, 5, 2, 2),
+    -0.010419349186122417,
+]
+
+
+class TestTorchVisionFullyConvModel(unittest.TestCase):
+    @parameterized.expand(
+        [
+            TEST_CASE_0,
+            TEST_CASE_1,
+            TEST_CASE_2,
+            TEST_CASE_3,
+        ]
+    )
+    @skipUnless(has_tv, "Requires TorchVision.")
+    def test_without_pretrained(self, input_param, input_shape, expected_shape):
+        net = TorchVisionFullyConvModel(**input_param).to(device)
+        with eval_mode(net):
+            result = net.forward(torch.randn(input_shape).to(device))
+            self.assertEqual(result.shape, expected_shape)
+
+    @parameterized.expand(
+        [
+            TEST_CASE_PRETRAINED_0,
+            TEST_CASE_PRETRAINED_1,
+            TEST_CASE_PRETRAINED_2,
+        ]
+    )
+    @skipUnless(has_tv, "Requires TorchVision.")
+    def test_with_pretrained(self, input_param, input_shape, expected_shape, expected_value):
+        net = TorchVisionFullyConvModel(**input_param).to(device)
+        with eval_mode(net):
+            result = net.forward(torch.randn(input_shape).to(device))
+            value = next(net.parameters())[0, 0, 0, 0].item()
+            self.assertEqual(value, expected_value)
+            self.assertEqual(result.shape, expected_shape)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_tversky_loss.py b/tests/test_tversky_loss.py
index a1befa062d..0bc2ca2e70 100644
--- a/tests/test_tversky_loss.py
+++ b/tests/test_tversky_loss.py
@@ -16,6 +16,7 @@
 from parameterized import parameterized
 
 from monai.losses import TverskyLoss
+from tests.utils import SkipIfBeforePyTorchVersion, test_script_save
 
 TEST_CASES = [
     [  # shape: (1, 1, 2, 2), (1, 1, 2, 2)
@@ -183,6 +184,12 @@ def test_input_warnings(self):
             loss = TverskyLoss(to_onehot_y=True)
             loss.forward(chn_input, chn_target)
 
+    @SkipIfBeforePyTorchVersion((1, 7, 0))
+    def test_script(self):
+        loss = TverskyLoss()
+        test_input = torch.ones(2, 1, 8, 8)
+        test_script_save(loss, test_input, test_input)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_vis_cam.py b/tests/test_vis_cam.py
index d400c27f02..47c116cd5d 100644
--- a/tests/test_vis_cam.py
+++ b/tests/test_vis_cam.py
@@ -14,7 +14,7 @@
 import torch
 from parameterized import parameterized
 
-from monai.networks.nets import DenseNet, densenet121, se_resnet50
+from monai.networks.nets import DenseNet, DenseNet121, SEResNet50
 from monai.visualize import CAM
 
 # 2D
@@ -68,15 +68,15 @@ class TestClassActivationMap(unittest.TestCase):
     @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
     def test_shape(self, input_data, expected_shape):
         if input_data["model"] == "densenet2d":
-            model = densenet121(spatial_dims=2, in_channels=1, out_channels=3)
+            model = DenseNet121(spatial_dims=2, in_channels=1, out_channels=3)
         if input_data["model"] == "densenet3d":
             model = DenseNet(
                 spatial_dims=3, in_channels=1, out_channels=3, init_features=2, growth_rate=2, block_config=(6,)
             )
         if input_data["model"] == "senet2d":
-            model = se_resnet50(spatial_dims=2, in_channels=3, num_classes=4)
+            model = SEResNet50(spatial_dims=2, in_channels=3, num_classes=4)
         if input_data["model"] == "senet3d":
-            model = se_resnet50(spatial_dims=3, in_channels=3, num_classes=4)
+            model = SEResNet50(spatial_dims=3, in_channels=3, num_classes=4)
         device = "cuda:0" if torch.cuda.is_available() else "cpu"
         model.to(device)
         model.eval()
diff --git a/tests/test_vis_gradcam.py b/tests/test_vis_gradcam.py
index 2a7de0e70c..f8e49f486f 100644
--- a/tests/test_vis_gradcam.py
+++ b/tests/test_vis_gradcam.py
@@ -11,10 +11,11 @@
 
 import unittest
 
+import numpy as np
 import torch
 from parameterized import parameterized
 
-from monai.networks.nets import DenseNet, densenet121, se_resnet50
+from monai.networks.nets import DenseNet, DenseNet121, SEResNet50
 from monai.visualize import GradCAM
 
 # 2D
@@ -64,24 +65,28 @@ class TestGradientClassActivationMap(unittest.TestCase):
     @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
     def test_shape(self, input_data, expected_shape):
         if input_data["model"] == "densenet2d":
-            model = densenet121(spatial_dims=2, in_channels=1, out_channels=3)
+            model = DenseNet121(spatial_dims=2, in_channels=1, out_channels=3)
         if input_data["model"] == "densenet3d":
             model = DenseNet(
                 spatial_dims=3, in_channels=1, out_channels=3, init_features=2, growth_rate=2, block_config=(6,)
             )
         if input_data["model"] == "senet2d":
-            model = se_resnet50(spatial_dims=2, in_channels=3, num_classes=4)
+            model = SEResNet50(spatial_dims=2, in_channels=3, num_classes=4)
         if input_data["model"] == "senet3d":
-            model = se_resnet50(spatial_dims=3, in_channels=3, num_classes=4)
+            model = SEResNet50(spatial_dims=3, in_channels=3, num_classes=4)
         device = "cuda:0" if torch.cuda.is_available() else "cpu"
         model.to(device)
         model.eval()
         cam = GradCAM(nn_module=model, target_layers=input_data["target_layers"])
         image = torch.rand(input_data["shape"], device=device)
         result = cam(x=image, layer_idx=-1)
+        np.testing.assert_array_equal(cam.nn_module.class_idx.cpu(), model(image).max(1)[-1].cpu())
         fea_shape = cam.feature_map_size(input_data["shape"], device=device)
         self.assertTupleEqual(fea_shape, input_data["feature_shape"])
         self.assertTupleEqual(result.shape, expected_shape)
+        # check result is same whether class_idx=None is used or not
+        result2 = cam(x=image, layer_idx=-1, class_idx=model(image).max(1)[-1].cpu())
+        np.testing.assert_array_almost_equal(result, result2)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_vis_gradcampp.py b/tests/test_vis_gradcampp.py
index fce68ccde0..92a4b2ac7b 100644
--- a/tests/test_vis_gradcampp.py
+++ b/tests/test_vis_gradcampp.py
@@ -14,7 +14,7 @@
 import torch
 from parameterized import parameterized
 
-from monai.networks.nets import DenseNet, densenet121, se_resnet50
+from monai.networks.nets import DenseNet, DenseNet121, SEResNet50
 from monai.visualize import GradCAMpp
 
 # 2D
@@ -64,15 +64,15 @@ class TestGradientClassActivationMapPP(unittest.TestCase):
     @parameterized.expand([TEST_CASE_0, TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
     def test_shape(self, input_data, expected_shape):
         if input_data["model"] == "densenet2d":
-            model = densenet121(spatial_dims=2, in_channels=1, out_channels=3)
+            model = DenseNet121(spatial_dims=2, in_channels=1, out_channels=3)
         if input_data["model"] == "densenet3d":
             model = DenseNet(
                 spatial_dims=3, in_channels=1, out_channels=3, init_features=2, growth_rate=2, block_config=(6,)
             )
         if input_data["model"] == "senet2d":
-            model = se_resnet50(spatial_dims=2, in_channels=3, num_classes=4)
+            model = SEResNet50(spatial_dims=2, in_channels=3, num_classes=4)
         if input_data["model"] == "senet3d":
-            model = se_resnet50(spatial_dims=3, in_channels=3, num_classes=4)
+            model = SEResNet50(spatial_dims=3, in_channels=3, num_classes=4)
         device = "cuda:0" if torch.cuda.is_available() else "cpu"
         model.to(device)
         model.eval()
diff --git a/tests/test_warp.py b/tests/test_warp.py
index 69ae997e38..c6c79a369a 100644
--- a/tests/test_warp.py
+++ b/tests/test_warp.py
@@ -1,54 +1,109 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import unittest
 
 import numpy as np
 import torch
 from parameterized import parameterized
+from torch.autograd import gradcheck
 
+from monai.config.deviceconfig import USE_COMPILED
 from monai.networks.blocks.warp import Warp
+from monai.utils import GridSampleMode, GridSamplePadMode
+from tests.utils import SkipIfBeforePyTorchVersion
 
-LOW_POWER_TEST_CASES = [
+LOW_POWER_TEST_CASES = [  # run with BUILD_MONAI=1 to test csrc/resample, BUILD_MONAI=0 to test native grid_sample
     [
-        {"spatial_dims": 2, "mode": 0, "padding_mode": "zeros"},
+        {"mode": "nearest", "padding_mode": "zeros"},
         {"image": torch.arange(4).reshape((1, 1, 2, 2)).to(dtype=torch.float), "ddf": torch.zeros(1, 2, 2, 2)},
         torch.arange(4).reshape((1, 1, 2, 2)),
     ],
     [
-        {"spatial_dims": 2, "mode": 1, "padding_mode": "zeros"},
+        {"mode": "bilinear", "padding_mode": "zeros"},
         {"image": torch.arange(4).reshape((1, 1, 2, 2)).to(dtype=torch.float), "ddf": torch.ones(1, 2, 2, 2)},
         torch.tensor([[[[3, 0], [0, 0]]]]),
     ],
+    [
+        {"mode": "bilinear", "padding_mode": "border"},
+        {
+            "image": torch.arange(8).reshape((1, 1, 2, 2, 2)).to(dtype=torch.float),
+            "ddf": torch.ones(1, 3, 2, 2, 2) * -1,
+        },
+        torch.tensor([[[[[0, 0], [0, 0]], [[0, 0], [0, 0]]]]]),
+    ],
+    [
+        {"mode": "bilinear", "padding_mode": "reflection"},
+        {
+            "image": torch.arange(8).reshape((1, 1, 2, 2, 2)).to(dtype=torch.float),
+            "ddf": torch.ones(1, 3, 2, 2, 2) * -1,
+        },
+        torch.tensor([[[[[7.0, 6.0], [5.0, 4.0]], [[3.0, 2.0], [1.0, 0.0]]]]]),
+    ],
 ]
 
-HIGH_POWER_TEST_CASES = [
+CPP_TEST_CASES = [  # high order, BUILD_MONAI=1 to test csrc/resample
     [
-        {"spatial_dims": 3, "mode": 2, "padding_mode": "border"},
+        {"mode": 2, "padding_mode": "border"},
         {
             "image": torch.arange(8).reshape((1, 1, 2, 2, 2)).to(dtype=torch.float),
             "ddf": torch.ones(1, 3, 2, 2, 2) * -1,
         },
-        torch.tensor([[[[[0, 0], [0, 0]], [[0, 0], [0, 0]]]]]),
+        torch.tensor([[[[[0.0000, 0.1250], [0.2500, 0.3750]], [[0.5000, 0.6250], [0.7500, 0.8750]]]]]),
+    ],
+    [
+        {"mode": 2, "padding_mode": "reflection"},
+        {
+            "image": torch.arange(8).reshape((1, 1, 2, 2, 2)).to(dtype=torch.float),
+            "ddf": torch.ones(1, 3, 2, 2, 2) * -1,
+        },
+        torch.tensor([[[[[5.2500, 4.7500], [4.2500, 3.7500]], [[3.2500, 2.7500], [2.2500, 1.7500]]]]]),
     ],
     [
-        {"spatial_dims": 3, "mode": 3, "padding_mode": "reflection"},
+        {"mode": 2, "padding_mode": "zeros"},
+        {
+            "image": torch.arange(8).reshape((1, 1, 2, 2, 2)).to(dtype=torch.float),
+            "ddf": torch.ones(1, 3, 2, 2, 2) * -1,
+        },
+        torch.tensor([[[[[0.0000, 0.0020], [0.0039, 0.0410]], [[0.0078, 0.0684], [0.0820, 0.6699]]]]]),
+    ],
+    [
+        {"mode": 2, "padding_mode": 7},
+        {
+            "image": torch.arange(8).reshape((1, 1, 2, 2, 2)).to(dtype=torch.float),
+            "ddf": torch.ones(1, 3, 2, 2, 2) * -1,
+        },
+        torch.tensor([[[[[0.0000, 0.0020], [0.0039, 0.0410]], [[0.0078, 0.0684], [0.0820, 0.6699]]]]]),
+    ],
+    [
+        {"mode": 3, "padding_mode": "reflection"},
         {"image": torch.arange(8).reshape((1, 1, 2, 2, 2)).to(dtype=torch.float), "ddf": torch.ones(1, 3, 2, 2, 2)},
-        torch.tensor([[[[[7, 6], [5, 4]], [[3, 2], [1, 0]]]]]),
+        torch.tensor([[[[[4.6667, 4.3333], [4.0000, 3.6667]], [[3.3333, 3.0000], [2.6667, 2.3333]]]]]),
     ],
 ]
 
 TEST_CASES = LOW_POWER_TEST_CASES
-# if USE_COMPILED:
-#     TEST_CASES += HIGH_POWER_TEST_CASES
+if USE_COMPILED:
+    TEST_CASES += CPP_TEST_CASES
 
 
 class TestWarp(unittest.TestCase):
-    @parameterized.expand(TEST_CASES)
+    @parameterized.expand(TEST_CASES, skip_on_empty=True)
     def test_resample(self, input_param, input_data, expected_val):
         warp_layer = Warp(**input_param)
         result = warp_layer(**input_data)
         np.testing.assert_allclose(result.cpu().numpy(), expected_val.cpu().numpy(), rtol=1e-4, atol=1e-4)
 
     def test_ill_shape(self):
-        warp_layer = Warp(spatial_dims=2)
+        warp_layer = Warp()
         with self.assertRaisesRegex(ValueError, ""):
             warp_layer(
                 image=torch.arange(4).reshape((1, 1, 1, 2, 2)).to(dtype=torch.float), ddf=torch.zeros(1, 2, 2, 2)
@@ -60,9 +115,16 @@ def test_ill_shape(self):
         with self.assertRaisesRegex(ValueError, ""):
             warp_layer(image=torch.arange(4).reshape((1, 1, 2, 2)).to(dtype=torch.float), ddf=torch.zeros(1, 2, 3, 3))
 
-    def test_ill_opts(self):
-        with self.assertRaisesRegex(ValueError, ""):
-            Warp(spatial_dims=4)
+    @SkipIfBeforePyTorchVersion((1, 8))
+    def test_grad(self):
+        for b in GridSampleMode:
+            for p in GridSamplePadMode:
+                warp_layer = Warp(mode=b.value, padding_mode=p.value)
+                input_image = torch.rand((2, 3, 20, 20), dtype=torch.float64) * 10.0
+                ddf = torch.rand((2, 2, 20, 20), dtype=torch.float64) * 2.0
+                input_image.requires_grad = True
+                ddf.requires_grad = False  # Jacobian mismatch for output 0 with respect to input 1
+                gradcheck(warp_layer, (input_image, ddf), atol=1e-2, eps=1e-2)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_with_allow_missing_keys.py b/tests/test_with_allow_missing_keys.py
new file mode 100644
index 0000000000..68c5ad30c4
--- /dev/null
+++ b/tests/test_with_allow_missing_keys.py
@@ -0,0 +1,73 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import numpy as np
+
+from monai.transforms import Compose, SpatialPad, SpatialPadd, allow_missing_keys_mode
+
+
+class TestWithAllowMissingKeysMode(unittest.TestCase):
+    def setUp(self):
+        self.data = {"image": np.arange(16, dtype=float).reshape(1, 4, 4)}
+
+    def test_map_transform(self):
+        for amk in [True, False]:
+            t = SpatialPadd(["image", "label"], 10, allow_missing_keys=amk)
+            with allow_missing_keys_mode(t):
+                # check state is True
+                self.assertTrue(t.allow_missing_keys)
+                # and that transform works even though key is missing
+                _ = t(self.data)
+            # check it has returned to original state
+            self.assertEqual(t.allow_missing_keys, amk)
+            if not amk:
+                # should fail because amks==False and key is missing
+                with self.assertRaises(KeyError):
+                    _ = t(self.data)
+
+    def test_compose(self):
+        amks = [True, False, True]
+        t = Compose([SpatialPadd(["image", "label"], 10, allow_missing_keys=amk) for amk in amks])
+        with allow_missing_keys_mode(t):
+            # check states are all True
+            for _t in t.transforms:
+                self.assertTrue(_t.allow_missing_keys)
+            # and that transform works even though key is missing
+            _ = t(self.data)
+        # check they've returned to original state
+        for _t, amk in zip(t.transforms, amks):
+            self.assertEqual(_t.allow_missing_keys, amk)
+        # should fail because not all amks==True and key is missing
+        with self.assertRaises((KeyError, RuntimeError)):
+            _ = t(self.data)
+
+    def test_array_transform(self):
+        for t in [SpatialPad(10), Compose([SpatialPad(10)])]:
+            with self.assertRaises(TypeError):
+                with allow_missing_keys_mode(t):
+                    pass
+
+    def test_multiple(self):
+        orig_states = [True, False]
+        ts = [SpatialPadd(["image", "label"], 10, allow_missing_keys=i) for i in orig_states]
+        with allow_missing_keys_mode(ts):
+            for t in ts:
+                self.assertTrue(t.allow_missing_keys)
+                # and that transform works even though key is missing
+                _ = t(self.data)
+        for t, o_s in zip(ts, orig_states):
+            self.assertEqual(t.allow_missing_keys, o_s)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/tests/test_zipdataset.py b/tests/test_zipdataset.py
index 1bdb6458d3..710ca71fc2 100644
--- a/tests/test_zipdataset.py
+++ b/tests/test_zipdataset.py
@@ -52,6 +52,18 @@ def test_value(self, datasets, transform, expected_output, expected_length):
         self.assertEqual(test_dataset[0], expected_output)
         self.assertEqual(len(test_dataset), expected_length)
 
+    def test_slicing(self):
+        test_dataset = ZipDataset(datasets=[Dataset_(5), Dataset_(5), Dataset_(5)], transform=None)
+        subset = test_dataset[0:2]
+        self.assertEqual(subset[-1], (1, 1, 1))
+        self.assertEqual(len(subset), 2)
+
+    def test_sequence(self):
+        test_dataset = ZipDataset(datasets=[Dataset_(5), Dataset_(5), Dataset_(5)], transform=None)
+        subset = test_dataset[[1, 3, 4]]
+        self.assertEqual(subset[-1], (4, 4, 4))
+        self.assertEqual(len(subset), 3)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/testing_data/1D_BP_bwd.txt b/tests/testing_data/1D_BP_bwd.txt
new file mode 100644
index 0000000000..de43270e94
--- /dev/null
+++ b/tests/testing_data/1D_BP_bwd.txt
@@ -0,0 +1,224 @@
+0., 1., 1., 1., 1., 1., 1., 1., 1.,12., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., # InterpolationType.nearest BoundType.replicate
+0., 1., 1., 1., 1., 1., 1., 1., 1.,12., # InterpolationType.nearest BoundType.replicate
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.replicate
+0., # InterpolationType.nearest BoundType.replicate
+0.5, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,11.5, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.linear BoundType.replicate
+0.5, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,11.5, # InterpolationType.linear BoundType.replicate
+1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.linear BoundType.replicate
+0., # InterpolationType.linear BoundType.replicate
+0.5, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,11.5, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.quadratic BoundType.replicate
+0.5, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,11.5, # InterpolationType.quadratic BoundType.replicate
+1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.quadratic BoundType.replicate
+0., # InterpolationType.quadratic BoundType.replicate
+0.5208333 , 0.9791666 , 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994,11.5 , 0.875 , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.875 , 0.125 , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.cubic BoundType.replicate
+0.5208333 , 0.9791666 , 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994,11.5 , # InterpolationType.cubic BoundType.replicate
+0.875,1. ,1. ,1. ,1. ,1. ,1. ,1. ,0.875,0.125,0. ,0. ,0. ,0. ,0. ,0. ,0. ,0. ,0. ,0. , # InterpolationType.cubic BoundType.replicate
+0., # InterpolationType.cubic BoundType.replicate
+0.5416667 , 0.9583334 , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,11.5 , 0.8333334 , 1. , 1. , 1. , 1. , 1. , 0.9999999 , 1. , 0.833333 , 0.16666651, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.fourth BoundType.replicate
+0.5416667, 0.9583334, 1. , 1. , 1. , 1. , 1. , 1. , 1. ,11.5 , # InterpolationType.fourth BoundType.replicate
+0.8333334 ,1. ,1. ,1. ,1. ,1. ,0.9999999 ,1. ,0.833333 ,0.16666651,0. ,0. ,0. ,0. ,0. ,0. ,0. ,0. ,0. ,0. , # InterpolationType.fourth BoundType.replicate
+0., # InterpolationType.fourth BoundType.replicate
+5.6223959e-01,9.3802083e-01,9.9973959e-01,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.1499999e+01,7.9947913e-01,9.9739581e-01,1.0000000e+00,1.0000000e+00,9.9999994e-01,1.0000001e+00,9.9999976e-01,9.9739575e-01,7.9947948e-01,2.0052099e-01,2.6040077e-03,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.fifth BoundType.replicate
+0.5622396, 0.9380208, 0.9997396, 1. , 1. , 1. , 1. , 1. , 1. ,11.499999 , # InterpolationType.fifth BoundType.replicate
+0.7994791 ,0.9973958 ,1. ,1. ,0.99999994,1.0000001 ,0.99999976,0.99739575,0.7994795 ,0.20052099,0.00260401,0. ,0. ,0. ,0. ,0. ,0. ,0. ,0. ,0. , # InterpolationType.fifth BoundType.replicate
+0., # InterpolationType.fifth BoundType.replicate
+5.8194447e-01,9.1944444e-01,9.9861109e-01,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.1499997e+01,7.7499998e-01,9.9166673e-01,1.0000000e+00,1.0000000e+00,1.0000000e+00,9.9999982e-01,1.0000004e+00,9.9166673e-01,7.7499980e-01,2.2500010e-01,8.3333999e-03,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07, # InterpolationType.sixth BoundType.replicate
+0.58194447, 0.91944444, 0.9986111 , 1. , 1. , 1. , 1. , 1. , 1. ,11.499997 , # InterpolationType.sixth BoundType.replicate
+7.7499998e-01,9.9166673e-01,1.0000000e+00,1.0000000e+00,1.0000000e+00,9.9999982e-01,1.0000004e+00,9.9166673e-01,7.7499980e-01,2.2500010e-01,8.3333999e-03,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07,1.9371510e-07, # InterpolationType.sixth BoundType.replicate
+0., # InterpolationType.sixth BoundType.replicate
+6.0078436e-01,9.0259641e-01,9.9662077e-01,9.9999845e-01,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.0000000e+00,1.1500004e+01,7.5551212e-01,9.8430985e-01,9.9997836e-01,9.9999994e-01,1.0000000e+00,1.0000001e+00,9.9997842e-01,9.8431003e-01,7.5551212e-01,2.4448761e-01,1.5690181e-02,2.1788481e-05,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07, # InterpolationType.seventh BoundType.replicate
+0.60078436, 0.9025964 , 0.9966208 , 0.99999845, 1. , 1. , 1. , 1. , 1. ,11.500004 , # InterpolationType.seventh BoundType.replicate
+7.5551212e-01,9.8430985e-01,9.9997836e-01,9.9999994e-01,1.0000000e+00,1.0000001e+00,9.9997842e-01,9.8431003e-01,7.5551212e-01,2.4448761e-01,1.5690181e-02,2.1788481e-05,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07,3.3080869e-07, # InterpolationType.seventh BoundType.replicate
+0., # InterpolationType.seventh BoundType.replicate
+1.,3.,3.,2.,2.,2.,2.,2.,2.,1.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dct1
+1.,3.,3.,2.,2.,2.,2.,2.,2.,1., # InterpolationType.nearest BoundType.dct1
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dct1
+0., # InterpolationType.nearest BoundType.dct1
+1.5, 3. , 2.5, 2. , 2. , 2. , 2. , 2. , 2. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. , 1. , 1. , # InterpolationType.linear BoundType.dct1
+1.5,3. ,2.5,2. ,2. ,2. ,2. ,2. ,2. ,1. , # InterpolationType.linear BoundType.dct1
+1., 1., 1., 1., 1., 1., 1., 1., 1.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1., 1., 1., # InterpolationType.linear BoundType.dct1
+0., # InterpolationType.linear BoundType.dct1
+1.5, 3. , 2.5, 2. , 2. , 2. , 2. , 2. , 2. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. , 1. , 1. , # InterpolationType.quadratic BoundType.dct1
+1.5,3. ,2.5,2. ,2. ,2. ,2. ,2. ,2. ,1. , # InterpolationType.quadratic BoundType.dct1
+1., 1., 1., 1., 1., 1., 1., 1., 1.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1., 1., 1., # InterpolationType.quadratic BoundType.dct1
+0., # InterpolationType.quadratic BoundType.dct1
+1.5 , 2.9791667 , 2.5 , 2.0208333 , 1.9999999 , 1.9999999 , 1.9999999 , 1.9999999 , 1.9999999 , 0.99999994, 0.75 , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.75 ,-0.75 ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-0.75 , 0.75 , 1. , # InterpolationType.cubic BoundType.dct1
+1.5 ,2.9791667 ,2.5 ,2.0208333 ,1.9999999 ,1.9999999 ,1.9999999 ,1.9999999 ,1.9999999 ,0.99999994, # InterpolationType.cubic BoundType.dct1
+0.75, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.75,-0.75,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-0.75, 0.75, 1. , # InterpolationType.cubic BoundType.dct1
+0., # InterpolationType.cubic BoundType.dct1
+1.5 , 2.9583333 , 2.5 , 2.0416667 , 2. , 2. , 2. , 2. , 2. , 1. , 0.6666666 , 1. , 1. , 1. , 1. , 1. , 0.9999999 , 1. , 0.6666664 ,-0.66666675,-1. ,-1.0000001 ,-1.0000002 ,-1. ,-1.0000001 ,-1.0000001 ,-1. ,-0.6666667 , 0.6666666 , 1. , # InterpolationType.fourth BoundType.dct1
+1.5 ,2.9583333,2.5 ,2.0416667,2. ,2. ,2. ,2. ,2. ,1. , # InterpolationType.fourth BoundType.dct1
+0.6666666 , 1. , 1. , 1. , 1. , 1. , 0.9999999 , 1. , 0.6666664 ,-0.66666675,-1. ,-1.0000001 ,-1.0000002 ,-1. ,-1.0000001 ,-1.0000001 ,-1. ,-0.6666667 , 0.6666666 , 1. , # InterpolationType.fourth BoundType.dct1
+0., # InterpolationType.fourth BoundType.dct1
+1.4997395 , 2.9380207 , 2.5 , 2.061979 , 2.0002604 , 2. , 2. , 2. , 2. , 1. , 0.5989583 , 0.9947917 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.99479157, 0.5989587 ,-0.59895825,-0.9947917 ,-0.9999998 ,-1.0000002 ,-1. ,-0.9999998 ,-1. ,-0.9947917 ,-0.5989583 , 0.5989583 , 0.9947917 , # InterpolationType.fifth BoundType.dct1
+1.4997395,2.9380207,2.5 ,2.061979 ,2.0002604,2. ,2. ,2. ,2. ,1. , # InterpolationType.fifth BoundType.dct1
+0.5989583 , 0.9947917 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.99479157, 0.5989587 ,-0.59895825,-0.9947917 ,-0.9999998 ,-1.0000002 ,-1. ,-0.9999998 ,-1. ,-0.9947917 ,-0.5989583 , 0.5989583 , 0.9947917 , # InterpolationType.fifth BoundType.dct1
+0., # InterpolationType.fifth BoundType.dct1
+1.498611 , 2.919444 , 2.5 , 2.0805554 , 2.0013888 , 2. , 2. , 2. , 2. , 1. , 0.54999995, 0.9833334 , 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.9833334 , 0.5499998 ,-0.5499999 ,-0.9833334 ,-1.0000004 ,-1.0000001 ,-1.0000001 ,-1. ,-0.99999994,-0.98333335,-0.55 , 0.54999995, 0.9833334 , # InterpolationType.sixth BoundType.dct1
+1.498611 ,2.919444 ,2.5 ,2.0805554,2.0013888,2. ,2. ,2. ,2. ,1. , # InterpolationType.sixth BoundType.dct1
+0.54999995, 0.9833334 , 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.9833334 , 0.5499998 ,-0.5499999 ,-0.9833334 ,-1.0000004 ,-1.0000001 ,-1.0000001 ,-1. ,-0.99999994,-0.98333335,-0.55 , 0.54999995, 0.9833334 , # InterpolationType.sixth BoundType.dct1
+0., # InterpolationType.sixth BoundType.dct1
+1.4966209 , 2.9025953 , 2.5000002 , 2.097404 , 2.0033796 , 2.000002 , 2.0000002 , 2.0000002 , 2.0000002 , 1. , 0.5110243 , 0.9686197 , 0.99995667, 0.99999994, 1. , 1.0000001 , 0.9999567 , 0.96861994, 0.51102436,-0.5110245 ,-0.9686197 ,-0.99995685,-1. ,-1. ,-1.0000001 ,-0.99995655,-0.9686198 ,-0.5110243 , 0.5110243 , 0.9686197 , # InterpolationType.seventh BoundType.dct1
+1.4966209,2.9025953,2.5000002,2.097404 ,2.0033796,2.000002 ,2.0000002,2.0000002,2.0000002,1. , # InterpolationType.seventh BoundType.dct1
+0.5110243 , 0.9686197 , 0.99995667, 0.99999994, 1. , 1.0000001 , 0.9999567 , 0.96861994, 0.51102436,-0.5110245 ,-0.9686197 ,-0.99995685,-1. ,-1. ,-1.0000001 ,-0.99995655,-0.9686198 ,-0.5110243 , 0.5110243 , 0.9686197 , # InterpolationType.seventh BoundType.dct1
+0., # InterpolationType.seventh BoundType.dct1
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dct2
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.nearest BoundType.dct2
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dct2
+0., # InterpolationType.nearest BoundType.dct2
+2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1., 0., # InterpolationType.linear BoundType.dct2
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.linear BoundType.dct2
+1., 1., 1., 1., 1., 1., 1., 1., 1., 0.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1., 0., # InterpolationType.linear BoundType.dct2
+0., # InterpolationType.linear BoundType.dct2
+2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1., 0., # InterpolationType.quadratic BoundType.dct2
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.quadratic BoundType.dct2
+1., 1., 1., 1., 1., 1., 1., 1., 1., 0.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1.,-1., 0., # InterpolationType.quadratic BoundType.dct2
+0., # InterpolationType.quadratic BoundType.dct2
+1.9999999, 2. , 1.9999999, 1.9999999, 1.9999999, 1.9999999, 1.9999999, 1.9999999, 1.9999999, 2. , 0.875 , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.875 , 0. ,-0.875 ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-0.875 , 0. , # InterpolationType.cubic BoundType.dct2
+1.9999999,2. ,1.9999999,1.9999999,1.9999999,1.9999999,1.9999999,1.9999999,1.9999999,2. , # InterpolationType.cubic BoundType.dct2
+0.875, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.875, 0. ,-0.875,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-1. ,-0.875, 0. , # InterpolationType.cubic BoundType.dct2
+0., # InterpolationType.cubic BoundType.dct2
+2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 8.3333337e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 9.9999988e-01, 1.0000000e+00, 8.3333302e-01,-1.1920929e-07,-8.3333325e-01,-1.0000000e+00,-1.0000001e+00,-1.0000002e+00,-1.0000000e+00,-1.0000001e+00,-1.0000001e+00,-1.0000000e+00,-8.3333337e-01, 0., # InterpolationType.fourth BoundType.dct2
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.fourth BoundType.dct2
+8.3333337e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 9.9999988e-01, 1.0000000e+00, 8.3333302e-01,-1.1920929e-07,-8.3333325e-01,-1.0000000e+00,-1.0000001e+00,-1.0000002e+00,-1.0000000e+00,-1.0000001e+00,-1.0000001e+00,-1.0000000e+00,-8.3333337e-01, 0., # InterpolationType.fourth BoundType.dct2
+0., # InterpolationType.fourth BoundType.dct2
+2.0000000e+00, 1.9999999e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 7.9687500e-01, 9.9739581e-01, 1.0000000e+00, 1.0000000e+00, 9.9999994e-01, 1.0000001e+00, 9.9999976e-01, 9.9739575e-01, 7.9687530e-01, 1.6018748e-07,-7.9687524e-01,-9.9739569e-01,-9.9999982e-01,-1.0000002e+00,-1.0000000e+00,-9.9999982e-01,-1.0000000e+00,-9.9739587e-01,-7.9687494e-01, 5.1222742e-09, # InterpolationType.fifth BoundType.dct2
+2. ,1.9999999,2. ,2. ,2. ,2. ,2. ,2. ,2. ,2. , # InterpolationType.fifth BoundType.dct2
+7.9687500e-01, 9.9739581e-01, 1.0000000e+00, 1.0000000e+00, 9.9999994e-01, 1.0000001e+00, 9.9999976e-01, 9.9739575e-01, 7.9687530e-01, 1.6018748e-07,-7.9687524e-01,-9.9739569e-01,-9.9999982e-01,-1.0000002e+00,-1.0000000e+00,-9.9999982e-01,-1.0000000e+00,-9.9739587e-01,-7.9687494e-01, 5.1222742e-09, # InterpolationType.fifth BoundType.dct2
+0., # InterpolationType.fifth BoundType.dct2
+2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 2.0000000e+00, 7.6666665e-01, 9.9166673e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 9.9999982e-01, 1.0000004e+00, 9.9166673e-01, 7.6666647e-01, 5.9604645e-08,-7.6666659e-01,-9.9166662e-01,-1.0000004e+00,-1.0000001e+00,-1.0000001e+00,-1.0000000e+00,-9.9999994e-01,-9.9166667e-01,-7.6666665e-01, 1.8626451e-09, # InterpolationType.sixth BoundType.dct2
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.sixth BoundType.dct2
+7.6666665e-01, 9.9166673e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 9.9999982e-01, 1.0000004e+00, 9.9166673e-01, 7.6666647e-01, 5.9604645e-08,-7.6666659e-01,-9.9166662e-01,-1.0000004e+00,-1.0000001e+00,-1.0000001e+00,-1.0000000e+00,-9.9999994e-01,-9.9166667e-01,-7.6666665e-01, 1.8626451e-09, # InterpolationType.sixth BoundType.dct2
+0., # InterpolationType.sixth BoundType.dct2
+2.0000002e+00, 2.0000000e+00, 2.0000000e+00, 2.0000002e+00, 2.0000002e+00, 2.0000002e+00, 2.0000002e+00, 2.0000002e+00, 2.0000002e+00, 2.0000002e+00, 7.3982203e-01, 9.8428816e-01, 9.9997836e-01, 9.9999994e-01, 1.0000000e+00, 1.0000001e+00, 9.9997842e-01, 9.8428833e-01, 7.3982203e-01,-1.6936974e-07,-7.3982191e-01,-9.8428810e-01,-9.9997830e-01,-1.0000000e+00,-1.0000000e+00,-1.0000001e+00,-9.9997824e-01,-9.8428822e-01,-7.3982203e-01,-2.7284841e-09, # InterpolationType.seventh BoundType.dct2
+2.0000002,2. ,2. ,2.0000002,2.0000002,2.0000002,2.0000002,2.0000002,2.0000002,2.0000002, # InterpolationType.seventh BoundType.dct2
+7.3982203e-01, 9.8428816e-01, 9.9997836e-01, 9.9999994e-01, 1.0000000e+00, 1.0000001e+00, 9.9997842e-01, 9.8428833e-01, 7.3982203e-01,-1.6936974e-07,-7.3982191e-01,-9.8428810e-01,-9.9997830e-01,-1.0000000e+00,-1.0000000e+00,-1.0000001e+00,-9.9997824e-01,-9.8428822e-01,-7.3982203e-01,-2.7284841e-09, # InterpolationType.seventh BoundType.dct2
+0., # InterpolationType.seventh BoundType.dct2
+-1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., # InterpolationType.nearest BoundType.dst1
+-1., 0., 0., 0., 0., 0., 0., 0., 0., 0., # InterpolationType.nearest BoundType.dst1
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dst1
+0., # InterpolationType.nearest BoundType.dst1
+0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-9.,-9., 1., 1., 1., 1., 1., 1., 1., 1., 1., # InterpolationType.linear BoundType.dst1
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.linear BoundType.dst1
+1., 1., 1., 1., 1., 1., 1., 1., 1.,-9.,-9., 1., 1., 1., 1., 1., 1., 1., 1., 1., # InterpolationType.linear BoundType.dst1
+0., # InterpolationType.linear BoundType.dst1
+0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-9.,-9., 1., 1., 1., 1., 1., 1., 1., 1., 1., # InterpolationType.quadratic BoundType.dst1
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.quadratic BoundType.dst1
+1., 1., 1., 1., 1., 1., 1., 1., 1.,-9.,-9., 1., 1., 1., 1., 1., 1., 1., 1., 1., # InterpolationType.quadratic BoundType.dst1
+0., # InterpolationType.quadratic BoundType.dst1
+0., 0.,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08, 8.7500000e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00,-2.5000000e-01,-7.7500000e+00,-7.7500000e+00,-2.5000000e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 8.7500000e-01, # InterpolationType.cubic BoundType.dst1
+0., 0.,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08, # InterpolationType.cubic BoundType.dst1
+0.875, 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-0.25 ,-7.75 ,-7.75 ,-0.25 , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.875, # InterpolationType.cubic BoundType.dst1
+0., # InterpolationType.cubic BoundType.dst1
+0., 0.,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-1.1175871e-08, 8.3333337e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 9.9999988e-01, 1.0000000e+00,-6.6666698e-01,-7.3333335e+00,-7.3333335e+00,-6.6666675e-01, 1.0000000e+00, 1.0000001e+00, 1.0000002e+00, 1.0000000e+00, 1.0000001e+00, 1.0000001e+00, 1.0000000e+00, 8.3333337e-01, # InterpolationType.fourth BoundType.dst1
+0., 0.,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-1.1175871e-08, # InterpolationType.fourth BoundType.dst1
+0.8333334 , 1. , 1. , 1. , 1. , 1. , 0.9999999 , 1. ,-0.666667 ,-7.3333335 ,-7.3333335 ,-0.66666675, 1. , 1.0000001 , 1.0000002 , 1. , 1.0000001 , 1.0000001 , 1. , 0.8333334 , # InterpolationType.fourth BoundType.dst1
+0., # InterpolationType.fourth BoundType.dst1
+3.9872248e-09, 0., 1.1175871e-08, 7.1886461e-09, 7.1886461e-09, 7.1886461e-09, 7.1886461e-09, 7.1886461e-09, 7.1886461e-09, 7.1886461e-09, 7.9947913e-01, 9.9739581e-01, 1.0000000e+00, 1.0000000e+00, 9.9999994e-01, 1.0000001e+00, 9.9999976e-01, 9.7395825e-01,-1.0052080e+00,-6.9687500e+00,-6.9687500e+00,-1.0052083e+00, 9.7395819e-01, 9.9999982e-01, 1.0000002e+00, 1.0000000e+00, 9.9999982e-01, 1.0000000e+00, 9.9739587e-01, 7.9947913e-01, # InterpolationType.fifth BoundType.dst1
+3.9872248e-09,0.,1.1175871e-08,7.1886461e-09,7.1886461e-09,7.1886461e-09,7.1886461e-09,7.1886461e-09,7.1886461e-09,7.1886461e-09, # InterpolationType.fifth BoundType.dst1
+0.7994791 , 0.9973958 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.97395825,-1.005208 ,-6.96875 ,-6.96875 ,-1.0052083 , 0.9739582 , 0.9999998 , 1.0000002 , 1. , 0.9999998 , 1. , 0.9973959 , 0.7994791 , # InterpolationType.fifth BoundType.dst1
+0., # InterpolationType.fifth BoundType.dst1
+4.1094609e-08, 0.,-1.4901161e-08, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09,-2.6193447e-08, 7.7499998e-01, 9.9166673e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 9.9999982e-01, 1.0000004e+00, 9.1666675e-01,-1.2500002e+00,-6.6666665e+00,-6.6666665e+00,-1.2500000e+00, 9.1666681e-01, 1.0000004e+00, 1.0000001e+00, 1.0000001e+00, 1.0000000e+00, 9.9999994e-01, 9.9166667e-01, 7.7499998e-01, # InterpolationType.sixth BoundType.dst1
+4.1094609e-08, 0.,-1.4901161e-08, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09,-2.6193447e-08, # InterpolationType.sixth BoundType.dst1
+0.775 , 0.99166673, 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.91666675,-1.2500002 ,-6.6666665 ,-6.6666665 ,-1.25 , 0.9166668 , 1.0000004 , 1.0000001 , 1.0000001 , 1. , 0.99999994, 0.9916667 , 0.775 , # InterpolationType.sixth BoundType.dst1
+0., # InterpolationType.sixth BoundType.dst1
+-9.7788870e-09, 3.7846348e-10,-7.4505806e-09, 2.3283064e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 7.5553381e-01, 9.8430985e-01, 9.9997836e-01, 9.9999994e-01, 1.0000000e+00, 1.0000001e+00, 9.9978310e-01, 8.4309906e-01,-1.4446614e+00,-6.3982205e+00,-6.3982205e+00,-1.4446614e+00, 8.4309900e-01, 9.9978304e-01, 1.0000000e+00, 1.0000000e+00, 1.0000001e+00, 9.9997824e-01, 9.8430991e-01, 7.5553381e-01, # InterpolationType.seventh BoundType.dst1
+-9.7788870e-09, 3.7846348e-10,-7.4505806e-09, 2.3283064e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, # InterpolationType.seventh BoundType.dst1
+0.7555338 , 0.98430985, 0.99997836, 0.99999994, 1. , 1.0000001 , 0.9997831 , 0.84309906,-1.4446614 ,-6.3982205 ,-6.3982205 ,-1.4446614 , 0.843099 , 0.99978304, 1. , 1. , 1.0000001 , 0.99997824, 0.9843099 , 0.7555338 , # InterpolationType.seventh BoundType.dst1
+0., # InterpolationType.seventh BoundType.dst1
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dst2
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dst2
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dst2
+0., # InterpolationType.nearest BoundType.dst2
+ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-18., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., # InterpolationType.linear BoundType.dst2
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.linear BoundType.dst2
+ 1., 1., 1., 1., 1., 1., 1., 1., 1.,-18., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., # InterpolationType.linear BoundType.dst2
+0., # InterpolationType.linear BoundType.dst2
+ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-18., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., # InterpolationType.quadratic BoundType.dst2
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.quadratic BoundType.dst2
+ 1., 1., 1., 1., 1., 1., 1., 1., 1.,-18., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., # InterpolationType.quadratic BoundType.dst2
+0., # InterpolationType.quadratic BoundType.dst2
+0., 0.,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08, 9.3132257e-09, 8.7500000e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00,-1.3750000e+00,-1.3250000e+01,-1.3750000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 8.7500000e-01, 2.5000000e-01, # InterpolationType.cubic BoundType.dst2
+0., 0.,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08,-2.0489097e-08, 9.3132257e-09, # InterpolationType.cubic BoundType.dst2
+ 0.875, 1. , 1. , 1. , 1. , 1. , 1. , 1. , -1.375,-13.25 , -1.375, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.875, 0.25 , # InterpolationType.cubic BoundType.dst2
+0., # InterpolationType.cubic BoundType.dst2
+0., 0.,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-1.1175871e-08, 8.3333337e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 9.9999988e-01, 1.0000000e+00,-2.1666670e+00,-1.1666667e+01,-2.1666667e+00, 1.0000000e+00, 1.0000001e+00, 1.0000002e+00, 1.0000000e+00, 1.0000001e+00, 1.0000001e+00, 1.0000000e+00, 8.3333337e-01, 3.3333334e-01, # InterpolationType.fourth BoundType.dst2
+0., 0.,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-4.0978193e-08,-1.1175871e-08, # InterpolationType.fourth BoundType.dst2
+ 0.8333334 , 1. , 1. , 1. , 1. , 1. , 0.9999999 , 1. , -2.166667 ,-11.666667 , -2.1666667 , 1. , 1.0000001 , 1.0000002 , 1. , 1.0000001 , 1.0000001 , 1. , 0.8333334 , 0.33333334, # InterpolationType.fourth BoundType.dst2
+0., # InterpolationType.fourth BoundType.dst2
+0., 3.7252903e-09, 1.1175871e-08, 7.1886461e-09, 7.1886461e-09, 7.1886461e-09, 7.1886461e-09, 7.1886461e-09, 7.1886461e-09, 1.0913936e-08, 8.0208331e-01, 9.9739581e-01, 1.0000000e+00, 1.0000000e+00, 9.9999994e-01, 1.0000001e+00, 9.9999976e-01, 9.5052075e-01,-2.7604163e+00,-1.0380208e+01,-2.7604165e+00, 9.5052069e-01, 9.9999982e-01, 1.0000002e+00, 1.0000000e+00, 9.9999982e-01, 1.0000000e+00, 9.9739587e-01, 8.0208331e-01, 4.0104166e-01, # InterpolationType.fifth BoundType.dst2
+0.,3.7252903e-09,1.1175871e-08,7.1886461e-09,7.1886461e-09,7.1886461e-09,7.1886461e-09,7.1886461e-09,7.1886461e-09,1.0913936e-08, # InterpolationType.fifth BoundType.dst2
+ 0.8020833 , 0.9973958 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.95052075, -2.7604163 ,-10.380208 , -2.7604165 , 0.9505207 , 0.9999998 , 1.0000002 , 1. , 0.9999998 , 1. , 0.9973959 , 0.8020833 , 0.40104166, # InterpolationType.fifth BoundType.dst2
+0., # InterpolationType.fifth BoundType.dst2
+5.9604645e-08,-1.4901161e-08,-1.4901161e-08, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09,-1.1292286e-08, 7.8333330e-01, 9.9166673e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 9.9999982e-01, 1.0000004e+00, 8.4166676e-01,-3.1166668e+00,-9.4499998e+00,-3.1166666e+00, 8.4166652e-01, 1.0000004e+00, 1.0000001e+00, 1.0000001e+00, 1.0000000e+00, 9.9999994e-01, 9.9166667e-01, 7.8333330e-01, 4.5000002e-01, # InterpolationType.sixth BoundType.dst2
+5.9604645e-08,-1.4901161e-08,-1.4901161e-08, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09, 3.6088750e-09,-1.1292286e-08, # InterpolationType.sixth BoundType.dst2
+0.7833333 , 0.99166673, 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.84166676,-3.1166668 ,-9.45 ,-3.1166666 , 0.8416665 , 1.0000004 , 1.0000001 , 1.0000001 , 1. , 0.99999994, 0.9916667 , 0.7833333 , 0.45000002, # InterpolationType.sixth BoundType.dst2
+0., # InterpolationType.sixth BoundType.dst2
+0.,-7.4505806e-09,-6.9849193e-09, 2.3283064e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09,-5.0350764e-09, 7.7120221e-01, 9.8433155e-01, 9.9997836e-01, 9.9999994e-01, 1.0000000e+00, 1.0000001e+00, 9.9958777e-01, 7.0230043e-01,-3.3471570e+00,-8.7094622e+00,-3.3471570e+00, 7.0230043e-01, 9.9958777e-01, 1.0000000e+00, 1.0000000e+00, 1.0000001e+00, 9.9997824e-01, 9.8433161e-01, 7.7120221e-01, 4.8897570e-01, # InterpolationType.seventh BoundType.dst2
+0.,-7.4505806e-09,-6.9849193e-09, 2.3283064e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09, 1.9498430e-09,-5.0350764e-09, # InterpolationType.seventh BoundType.dst2
+0.7712022 , 0.98433155, 0.99997836, 0.99999994, 1. , 1.0000001 , 0.9995878 , 0.7023004 ,-3.347157 ,-8.709462 ,-3.347157 , 0.7023004 , 0.9995878 , 1. , 1. , 1.0000001 , 0.99997824, 0.9843316 , 0.7712022 , 0.4889757 , # InterpolationType.seventh BoundType.dst2
+0., # InterpolationType.seventh BoundType.dst2
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dft
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.nearest BoundType.dft
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.dft
+0., # InterpolationType.nearest BoundType.dft
+2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., # InterpolationType.linear BoundType.dft
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.linear BoundType.dft
+1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., # InterpolationType.linear BoundType.dft
+0., # InterpolationType.linear BoundType.dft
+2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., # InterpolationType.quadratic BoundType.dft
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.quadratic BoundType.dft
+1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., 1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., # InterpolationType.quadratic BoundType.dft
+0., # InterpolationType.quadratic BoundType.dft
+2. , 2. , 1.9999999, 1.9999999, 1.9999999, 1.9999999, 1.9999999, 1.9999999, 1.9999999, 2. ,-0.25 , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-0.25 ,-6.5 ,-0.25 , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-0.25 ,-6.5 , # InterpolationType.cubic BoundType.dft
+2. ,2. ,1.9999999,1.9999999,1.9999999,1.9999999,1.9999999,1.9999999,1.9999999,2. , # InterpolationType.cubic BoundType.dft
+-0.25, 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-0.25,-6.5 ,-0.25, 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-0.25,-6.5 , # InterpolationType.cubic BoundType.dft
+0., # InterpolationType.cubic BoundType.dft
+2. , 2. , 2. , 2. , 2. , 2. , 2. , 2. , 2. , 2. ,-0.6666666, 1. , 1. , 1. , 1. , 1. , 0.9999999, 1. ,-0.666667 ,-5.666667 ,-0.6666666, 1. , 1. , 1. , 1. , 1. , 0.9999999, 1. ,-0.666667 ,-5.666667 , # InterpolationType.fourth BoundType.dft
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.fourth BoundType.dft
+-0.6666666, 1. , 1. , 1. , 1. , 1. , 0.9999999, 1. ,-0.666667 ,-5.666667 ,-0.6666666, 1. , 1. , 1. , 1. , 1. , 0.9999999, 1. ,-0.666667 ,-5.666667 , # InterpolationType.fourth BoundType.dft
+0., # InterpolationType.fourth BoundType.dft
+2. , 2. , 2. , 2. , 2. , 2. , 2. , 2. , 2. , 2. ,-0.97916675, 0.9739583 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.97395825,-0.9791663 ,-4.989583 ,-0.97916675, 0.9739583 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.97395825,-0.9791663 ,-4.989583 , # InterpolationType.fifth BoundType.dft
+2.,2.,2.,2.,2.,2.,2.,2.,2.,2., # InterpolationType.fifth BoundType.dft
+-0.97916675, 0.9739583 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.97395825,-0.9791663 ,-4.989583 ,-0.97916675, 0.9739583 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.97395825,-0.9791663 ,-4.989583 , # InterpolationType.fifth BoundType.dft
+0., # InterpolationType.fifth BoundType.dft
+1.9999999 , 2. , 2. , 2. , 2. , 2. , 2. , 2. , 2. , 2. ,-1.1666667 , 0.9166667 , 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.91666675,-1.1666669 ,-4.4999995 ,-1.1666667 , 0.9166667 , 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.91666675,-1.1666669 ,-4.4999995 , # InterpolationType.sixth BoundType.dft
+1.9999999,2. ,2. ,2. ,2. ,2. ,2. ,2. ,2. ,2. , # InterpolationType.sixth BoundType.dft
+-1.1666667 , 0.9166667 , 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.91666675,-1.1666669 ,-4.4999995 ,-1.1666667 , 0.9166667 , 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.91666675,-1.1666669 ,-4.4999995 , # InterpolationType.sixth BoundType.dft
+0., # InterpolationType.sixth BoundType.dft
+2.0000002 , 2.0000002 , 2.0000002 , 2.0000002 , 2.0000002 , 2.0000002 , 2.0000002 , 2.0000002 , 2.0000002 , 2.0000002 ,-1.2879773 , 0.84331596, 0.999783 , 0.99999994, 1. , 1.0000001 , 0.9997831 , 0.8433161 ,-1.2879775 ,-4.110243 ,-1.2879773 , 0.84331596, 0.999783 , 0.99999994, 1. , 1.0000001 , 0.9997831 , 0.8433161 ,-1.2879775 ,-4.110243 , # InterpolationType.seventh BoundType.dft
+2.0000002,2.0000002,2.0000002,2.0000002,2.0000002,2.0000002,2.0000002,2.0000002,2.0000002,2.0000002, # InterpolationType.seventh BoundType.dft
+-1.2879773 , 0.84331596, 0.999783 , 0.99999994, 1. , 1.0000001 , 0.9997831 , 0.8433161 ,-1.2879775 ,-4.110243 ,-1.2879773 , 0.84331596, 0.999783 , 0.99999994, 1. , 1.0000001 , 0.9997831 , 0.8433161 ,-1.2879775 ,-4.110243 , # InterpolationType.seventh BoundType.dft
+0., # InterpolationType.seventh BoundType.dft
+0.,1.,1.,1.,1.,1.,1.,1.,1.,1.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.zero
+0.,1.,1.,1.,1.,1.,1.,1.,1.,1., # InterpolationType.nearest BoundType.zero
+0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0.,0., # InterpolationType.nearest BoundType.zero
+0., # InterpolationType.nearest BoundType.zero
+0.5, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-9. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.linear BoundType.zero
+0.5,1. ,1. ,1. ,1. ,1. ,1. ,1. ,1. ,1. , # InterpolationType.linear BoundType.zero
+1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., # InterpolationType.linear BoundType.zero
+0., # InterpolationType.linear BoundType.zero
+0.5, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-9. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.quadratic BoundType.zero
+0.5,1. ,1. ,1. ,1. ,1. ,1. ,1. ,1. ,1. , # InterpolationType.quadratic BoundType.zero
+1., 1., 1., 1., 1., 1., 1., 1., 1.,-9., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., # InterpolationType.quadratic BoundType.zero
+0., # InterpolationType.quadratic BoundType.zero
+0.5 , 0.9791666 , 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.99999994, 0.875 , 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-0.25 ,-6.625 ,-1.125 , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.cubic BoundType.zero
+0.5 ,0.9791666 ,0.99999994,0.99999994,0.99999994,0.99999994,0.99999994,0.99999994,0.99999994,0.99999994, # InterpolationType.cubic BoundType.zero
+0.875, 1. , 1. , 1. , 1. , 1. , 1. , 1. ,-0.25 ,-6.625,-1.125, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.cubic BoundType.zero
+0., # InterpolationType.cubic BoundType.zero
+0.5 , 0.9583334, 1. , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.8333334, 1. , 1. , 1. , 1. , 1. , 0.9999999, 1. ,-0.666667 ,-5.8333335,-1.5 , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.fourth BoundType.zero
+0.5 ,0.9583334,1. ,1. ,1. ,1. ,1. ,1. ,1. ,1. , # InterpolationType.fourth BoundType.zero
+0.8333334, 1. , 1. , 1. , 1. , 1. , 0.9999999, 1. ,-0.666667 ,-5.8333335,-1.5 , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.fourth BoundType.zero
+0., # InterpolationType.fourth BoundType.zero
+0.5 , 0.9380208 , 0.9997396 , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.7994791 , 0.9973958 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.97395825,-0.9817705 ,-5.190104 ,-1.7786459 ,-0.0234375 , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.fifth BoundType.zero
+0.5 ,0.9380208,0.9997396,1. ,1. ,1. ,1. ,1. ,1. ,1. , # InterpolationType.fifth BoundType.zero
+0.7994791 , 0.9973958 , 1. , 1. , 0.99999994, 1.0000001 , 0.99999976, 0.97395825,-0.9817705 ,-5.190104 ,-1.7786459 ,-0.0234375 , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.fifth BoundType.zero
+0., # InterpolationType.fifth BoundType.zero
+0.49999997, 0.91944444, 0.9986111 , 1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.775 , 0.99166673, 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.91666675,-1.1750002 ,-4.725 ,-1.9416667 ,-0.075 , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.sixth BoundType.zero
+0.49999997,0.91944444,0.9986111 ,1. ,1. ,1. ,1. ,1. ,1. ,1. , # InterpolationType.sixth BoundType.zero
+0.775 , 0.99166673, 1. , 1. , 1. , 0.9999998 , 1.0000004 , 0.91666675,-1.1750002 ,-4.725 ,-1.9416667 ,-0.075 , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , # InterpolationType.sixth BoundType.zero
+0., # InterpolationType.sixth BoundType.zero
+5.0000000e-01, 9.0259641e-01, 9.9662077e-01, 9.9999845e-01, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 1.0000000e+00, 7.5551212e-01, 9.8430985e-01, 9.9997836e-01, 9.9999994e-01, 1.0000000e+00, 1.0000001e+00, 9.9978310e-01, 8.4329438e-01,-1.3036675e+00,-4.3547311e+00,-2.0434895e+00,-1.4099392e-01,-1.9531250e-04, 0., 0., 0., 0., 0., 0., 0., # InterpolationType.seventh BoundType.zero
+0.5 ,0.9025964 ,0.9966208 ,0.99999845,1. ,1. ,1. ,1. ,1. ,1. , # InterpolationType.seventh BoundType.zero
+7.5551212e-01, 9.8430985e-01, 9.9997836e-01, 9.9999994e-01, 1.0000000e+00, 1.0000001e+00, 9.9978310e-01, 8.4329438e-01,-1.3036675e+00,-4.3547311e+00,-2.0434895e+00,-1.4099392e-01,-1.9531250e-04, 0., 0., 0., 0., 0., 0., 0., # InterpolationType.seventh BoundType.zero
+0., # InterpolationType.seventh BoundType.zero
diff --git a/tests/testing_data/1D_BP_fwd.txt b/tests/testing_data/1D_BP_fwd.txt
new file mode 100644
index 0000000000..a620d59dff
--- /dev/null
+++ b/tests/testing_data/1D_BP_fwd.txt
@@ -0,0 +1,56 @@
+1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, # InterpolationType.nearest BoundType.replicate
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, # InterpolationType.linear BoundType.replicate
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, # InterpolationType.quadratic BoundType.replicate
+0.5208, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.4792, 8.9792, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, # InterpolationType.cubic BoundType.replicate
+0.5417, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.4583, 8.9583, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, # InterpolationType.fourth BoundType.replicate
+0.5622, 1.5003, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4997, 8.4378, 8.9378, 8.9997, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, # InterpolationType.fifth BoundType.replicate
+0.5819, 1.5014, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4986, 8.4181, 8.9181, 8.9986, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, # InterpolationType.sixth BoundType.replicate
+0.6008, 1.5034, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4966, 8.3992, 8.8992, 8.9966, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, 9.0, # InterpolationType.seventh BoundType.replicate
+1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0, 1.0, 2.0, # InterpolationType.nearest BoundType.dct1
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 8.5, 7.5, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5, 0.5, 0.5, 1.5, # InterpolationType.linear BoundType.dct1
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 8.5, 7.5, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5, 0.5, 0.5, 1.5, # InterpolationType.quadratic BoundType.dct1
+0.5417, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.4583, 8.4583, 7.5, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5, 0.5417, 0.5417, 1.5, # InterpolationType.cubic BoundType.dct1
+0.5833, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.4167, 8.4167, 7.5, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5, 0.5833, 0.5833, 1.5, # InterpolationType.fourth BoundType.dct1
+0.6245, 1.5005, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4995, 8.3755, 8.3755, 7.4995, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5005, 0.6245, 0.6245, 1.5005, # InterpolationType.fifth BoundType.dct1
+0.6639, 1.5028, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4972, 8.3361, 8.3361, 7.4972, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5028, 0.6639, 0.6639, 1.5028, # InterpolationType.sixth BoundType.dct1
+0.7016, 1.5068, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4932, 8.2984, 8.2984, 7.4932, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5068, 0.7016, 0.7016, 1.5068, # InterpolationType.seventh BoundType.dct1
+1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0, 0.0, # InterpolationType.nearest BoundType.dct2
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.0, 8.5, 7.5, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5, 0.5, 0.0, # InterpolationType.linear BoundType.dct2
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.0, 8.5, 7.5, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5, 0.5, 0.0, # InterpolationType.quadratic BoundType.dct2
+0.5208, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.4792, 8.9583, 8.4792, 7.5, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5, 0.5208, 0.0417, # InterpolationType.cubic BoundType.dct2
+0.5417, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.4583, 8.9167, 8.4583, 7.5, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5, 0.5417, 0.0833, # InterpolationType.fourth BoundType.dct2
+0.5625, 1.5003, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4997, 8.4375, 8.8755, 8.4375, 7.4997, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5003, 0.5625, 0.1245, # InterpolationType.fifth BoundType.dct2
+0.5833, 1.5014, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4986, 8.4167, 8.8361, 8.4167, 7.4986, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5014, 0.5833, 0.1639, # InterpolationType.sixth BoundType.dct2
+0.6042, 1.5034, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4966, 8.3958, 8.7984, 8.3958, 7.4966, 6.5, 5.5, 4.5, 3.5, 2.5, 1.5034, 0.6042, 0.2016, # InterpolationType.seventh BoundType.dct2
+1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 0.0, -9.0, -8.0, -7.0, -6.0, -5.0, -4.0, -3.0, -2.0, -1.0, -0.0, # InterpolationType.nearest BoundType.dst1
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 4.5, -4.5, -8.5, -7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -0.5, # InterpolationType.linear BoundType.dst1
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 4.5, -4.5, -8.5, -7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -0.5, # InterpolationType.quadratic BoundType.dst1
+0.5208, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.2917, 4.2917, -4.2917, -8.2917, -7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -0.5208, # InterpolationType.cubic BoundType.dst1
+0.5417, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.0833, 4.0833, -4.0833, -8.0833, -7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -0.5417, # InterpolationType.fourth BoundType.dst1
+0.5622, 1.5003, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4974, 7.8776, 3.8802, -3.8802, -7.8776, -7.4974, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5003, -0.5622, # InterpolationType.fifth BoundType.dst1
+0.5819, 1.5014, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4861, 7.6806, 3.6944, -3.6944, -7.6806, -7.4861, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5014, -0.5819, # InterpolationType.sixth BoundType.dst1
+0.6008, 1.5034, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4662, 7.4922, 3.5260, -3.5260, -7.4922, -7.4662, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5034, -0.6008, # InterpolationType.seventh BoundType.dst1
+1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, -9.0, -8.0, -7.0, -6.0, -5.0, -4.0, -3.0, -2.0, -1.0, -0.0, 0.0, # InterpolationType.nearest BoundType.dst2
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 0.0, -8.5, -7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -0.5, 0.0, # InterpolationType.linear BoundType.dst2
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 0.0, -8.5, -7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -0.5, 0.0, # InterpolationType.quadratic BoundType.dst2
+5.2083e-01, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.1042, -1.6391e-07, -8.1042, -7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -5.2083e-01, 0.0, # InterpolationType.cubic BoundType.dst2
+5.4167e-01, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 7.7083, 1.4901e-07, -7.7083, -7.5, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5, -5.4167e-01, 0.0, # InterpolationType.fourth BoundType.dst2
+5.6198e-01, 1.5003, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4951, 7.3224, 1.2107e-07, -7.3224, -7.4951, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5003, -5.6198e-01, 5.2387e-10, # InterpolationType.fifth BoundType.dst2
+5.8056e-01, 1.5014, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4736, 6.9694, -1.0896e-07, -6.9694, -7.4736, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5014, -5.8056e-01, 2.3283e-10, # InterpolationType.sixth BoundType.dst2
+0.59740, 1.5034, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4358, 6.6493, 0.0, -6.6493, -7.4358, -6.5, -5.5, -4.5, -3.5, -2.5, -1.5034, -0.59740, 0.0, # InterpolationType.seventh BoundType.dst2
+1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 0.0, # InterpolationType.nearest BoundType.dft
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 4.5, 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 4.5, # InterpolationType.linear BoundType.dft
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 4.5, 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 4.5, # InterpolationType.quadratic BoundType.dft
+0.7083, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.2917, 4.5, 0.7083, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.2917, 4.5, # InterpolationType.cubic BoundType.dft
+0.9167, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.0833, 4.5, 0.9167, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.0833, 4.5, # InterpolationType.fourth BoundType.dft
+1.1198, 1.5026, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4974, 7.8802, 4.5, 1.1198, 1.5026, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4974, 7.8802, 4.5, # InterpolationType.fifth BoundType.dft
+1.3056, 1.5139, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4861, 7.6944, 4.5, 1.3056, 1.5139, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4861, 7.6944, 4.5, # InterpolationType.sixth BoundType.dft
+1.4740, 1.5338, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4662, 7.5260, 4.5, 1.4740, 1.5338, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4662, 7.5260, 4.5, # InterpolationType.seventh BoundType.dft
+1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, # InterpolationType.nearest BoundType.zero
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 4.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, # InterpolationType.linear BoundType.zero
+0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 4.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, # InterpolationType.quadratic BoundType.zero
+0.5208, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.2917, 4.4792, 0.1875, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, # InterpolationType.cubic BoundType.zero
+0.5417, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.0833, 4.4583, 0.3750, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, # InterpolationType.fourth BoundType.zero
+5.6224e-01, 1.5003, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4974, 7.8799, 4.4378, 5.5755e-01, 2.3438e-03, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, # InterpolationType.fifth BoundType.zero
+0.5819, 1.5014, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4861, 7.6931, 4.4181, 0.7236, 0.0125, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, # InterpolationType.sixth BoundType.zero
+6.0078e-01, 1.5034, 2.5, 3.5, 4.5, 5.5, 6.5, 7.4662, 7.5226, 4.3992, 8.7325e-01, 3.0411e-02, 1.3951e-05, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, # InterpolationType.seventh BoundType.zero
diff --git a/tests/testing_data/cpp_resample_answers.py b/tests/testing_data/cpp_resample_answers.py
new file mode 100644
index 0000000000..51ac6ccda9
--- /dev/null
+++ b/tests/testing_data/cpp_resample_answers.py
@@ -0,0 +1,41 @@
+# Copyright 2020 - 2021 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import csv
+import os
+import warnings
+from typing import List, Optional
+
+
+def _read_testing_data_answers(fname: Optional[str] = None, delimiter=",") -> List:
+    answers: List = []
+    if not fname:
+        return answers
+    # read answers from directory of the current file
+    pwd = os.path.dirname(os.path.abspath(__file__))
+    filename = os.path.join(pwd, fname)
+    if not os.path.isfile(filename):
+        warnings.warn("test data {} not found.".format(filename))
+        return answers
+    with open(filename) as f:
+        res_reader = csv.reader(f, delimiter=delimiter)
+        for r in res_reader:
+            res_row = []
+            for item in r:
+                if item.strip().startswith("#"):
+                    continue  # allow for some simple comments in the file
+                res_row.append(float(item))
+            answers.append(res_row)
+    return answers
+
+
+Expected_1D_GP_fwd: List = _read_testing_data_answers(fname="1D_BP_fwd.txt")
+Expected_1D_GP_bwd: List = _read_testing_data_answers(fname="1D_BP_bwd.txt")
diff --git a/tests/testing_data/kitty_test.jpg b/tests/testing_data/kitty_test.jpg
new file mode 100644
index 0000000000..f103760de5
Binary files /dev/null and b/tests/testing_data/kitty_test.jpg differ
diff --git a/tests/testing_data/threadcontainer_plot_test.png b/tests/testing_data/threadcontainer_plot_test.png
new file mode 100644
index 0000000000..af742a8812
Binary files /dev/null and b/tests/testing_data/threadcontainer_plot_test.png differ
diff --git a/tests/utils.py b/tests/utils.py
index 4597a18fbd..5fa67f3e49 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -16,6 +16,7 @@
 import queue
 import sys
 import tempfile
+import time
 import traceback
 import unittest
 import warnings
@@ -237,7 +238,11 @@ def __init__(
         """
         self.nnodes = int(nnodes)
         self.nproc_per_node = int(nproc_per_node)
-        self.node_rank = int(os.environ.get("NODE_RANK", "0")) if node_rank is None else node_rank
+        if self.nnodes < 1 or self.nproc_per_node < 1:
+            raise ValueError(
+                f"number of nodes and processes per node must be >= 1, got {self.nnodes} and {self.nproc_per_node}"
+            )
+        self.node_rank = int(os.environ.get("NODE_RANK", "0")) if node_rank is None else int(node_rank)
         self.master_addr = master_addr
         self.master_port = np.random.randint(10000, 20000) if master_port is None else master_port
 
@@ -269,6 +274,7 @@ def run_process(self, func, local_rank, args, kwargs, results):
             os.environ["RANK"] = str(self.nproc_per_node * self.node_rank + local_rank)
 
             if torch.cuda.is_available():
+                os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
                 torch.cuda.set_device(int(local_rank))
 
             dist.init_process_group(
@@ -279,6 +285,11 @@ def run_process(self, func, local_rank, args, kwargs, results):
                 rank=int(os.environ["RANK"]),
             )
             func(*args, **kwargs)
+            # the primary node lives longer to
+            # avoid _store_based_barrier, RuntimeError: Broken pipe
+            # as the TCP store daemon is on the rank 0
+            if int(os.environ["RANK"]) == 0:
+                time.sleep(0.1)
             results.put(True)
         except Exception as e:
             results.put(False)
@@ -286,11 +297,20 @@ def run_process(self, func, local_rank, args, kwargs, results):
         finally:
             os.environ.clear()
             os.environ.update(_env)
-            dist.destroy_process_group()
+            try:
+                dist.destroy_process_group()
+            except RuntimeError as e:
+                warnings.warn(f"While closing process group: {e}.")
 
     def __call__(self, obj):
         if not torch.distributed.is_available():
             return unittest.skipIf(True, "Skipping distributed tests because not torch.distributed.is_available()")(obj)
+        if torch.cuda.is_available() and torch.cuda.device_count() < self.nproc_per_node:
+            return unittest.skipIf(
+                True,
+                f"Skipping distributed tests because it requires {self.nnodes} devices "
+                f"but got {torch.cuda.device_count()}",
+            )(obj)
 
         _cache_original_func(obj)
 
@@ -549,13 +569,14 @@ def query_memory(n=2):
     """
     Find best n idle devices and return a string of device ids.
     """
-    bash_string = "nvidia-smi --query-gpu=utilization.gpu,temperature.gpu,memory.used --format=csv,noheader,nounits"
+    bash_string = "nvidia-smi --query-gpu=power.draw,temperature.gpu,memory.used --format=csv,noheader,nounits"
 
     try:
         p1 = Popen(bash_string.split(), stdout=PIPE)
         output, error = p1.communicate()
         free_memory = [x.split(",") for x in output.decode("utf-8").split("\n")[:-1]]
-        free_memory = np.asarray(free_memory, dtype=np.float).T
+        free_memory = np.asarray(free_memory, dtype=float).T
+        free_memory[1] += free_memory[0]  # combine 0/1 column measures
         ids = np.lexsort(free_memory)[:n]
     except (FileNotFoundError, TypeError, IndexError):
         ids = range(n) if isinstance(n, int) else []
diff --git a/versioneer.py b/versioneer.py
index 441b3d4c2d..9112ac66a5 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -1,4 +1,4 @@
-# Version: 0.18
+# Version: 0.19
 
 """The Versioneer - like a rocketeer, but for versions.
 
@@ -6,16 +6,12 @@
 ==============
 
 * like a rocketeer, but for versions!
-* https://github.com/warner/python-versioneer
+* https://github.com/python-versioneer/python-versioneer
 * Brian Warner
 * License: Public Domain
-* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy
-* [![Latest Version]
-(https://pypip.in/version/versioneer/badge.svg?style=flat)
-](https://pypi.python.org/pypi/versioneer/)
-* [![Build Status]
-(https://travis-ci.org/warner/python-versioneer.png?branch=master)
-](https://travis-ci.org/warner/python-versioneer)
+* Compatible with: Python 3.6, 3.7, 3.8, 3.9 and pypy3
+* [![Latest Version][pypi-image]][pypi-url]
+* [![Build Status][travis-image]][travis-url]
 
 This is a tool for managing a recorded version number in distutils-based
 python projects. The goal is to remove the tedious and error-prone "update
@@ -26,9 +22,10 @@
 
 ## Quick Install
 
-* `pip install versioneer` to somewhere to your $PATH
-* add a `[versioneer]` section to your setup.cfg (see below)
+* `pip install versioneer` to somewhere in your $PATH
+* add a `[versioneer]` section to your setup.cfg (see [Install](INSTALL.md))
 * run `versioneer install` in your source tree, commit the results
+* Verify version information with `python setup.py version`
 
 ## Version Identifiers
 
@@ -60,7 +57,7 @@
 for example `git describe --tags --dirty --always` reports things like
 "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the
 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has
-uncommitted changes.
+uncommitted changes).
 
 The version identifier is used for multiple purposes:
 
@@ -165,7 +162,7 @@
 
 Some situations are known to cause problems for Versioneer. This details the
 most significant ones. More can be found on Github
-[issues page](https://github.com/warner/python-versioneer/issues).
+[issues page](https://github.com/python-versioneer/python-versioneer/issues).
 
 ### Subprojects
 
@@ -193,9 +190,9 @@
 Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in
 some later version.
 
-[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking
+[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking
 this issue. The discussion in
-[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the
+[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the
 issue from the Versioneer side in more detail.
 [pip PR#3176](https://github.com/pypa/pip/pull/3176) and
 [pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve
@@ -223,22 +220,10 @@
 cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into
 a different virtualenv), so this can be surprising.
 
-[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes
+[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes
 this one, but upgrading to a newer version of setuptools should probably
 resolve it.
 
-### Unicode version strings
-
-While Versioneer works (and is continually tested) with both Python 2 and
-Python 3, it is not entirely consistent with bytes-vs-unicode distinctions.
-Newer releases probably generate unicode version strings on py2. It's not
-clear that this is wrong, but it may be surprising for applications when then
-write these strings to a network connection or include them in bytes-oriented
-APIs like cryptographic checksums.
-
-[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates
-this question.
-
 
 ## Updating Versioneer
 
@@ -264,6 +249,12 @@
 direction and include code from all supported VCS systems, reducing the
 number of intermediate scripts.
 
+## Similar projects
+
+* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time
+  dependency
+* [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of
+  versioneer
 
 ## License
 
@@ -273,14 +264,15 @@
 Dedication" license (CC0-1.0), as described in
 https://creativecommons.org/publicdomain/zero/1.0/ .
 
-"""
+[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg
+[pypi-url]: https://pypi.python.org/pypi/versioneer/
+[travis-image]:
+https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg
+[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer
 
-from __future__ import print_function
+"""
 
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
+import configparser
 import errno
 import json
 import os
@@ -340,9 +332,9 @@ def get_config_from_root(root):
     # configparser.NoOptionError (if it lacks "VCS="). See the docstring at
     # the top of versioneer.py for instructions on writing your setup.cfg .
     setup_cfg = os.path.join(root, "setup.cfg")
-    parser = configparser.SafeConfigParser()
+    parser = configparser.ConfigParser()
     with open(setup_cfg, "r") as f:
-        parser.readfp(f)
+        parser.read_file(f)
     VCS = parser.get("versioneer", "VCS")  # mandatory
 
     def get(parser, name):
@@ -373,7 +365,7 @@ class NotThisMethod(Exception):
 
 
 def register_vcs_handler(vcs, method):  # decorator
-    """Decorator to mark a method as the handler for a particular VCS."""
+    """Create decorator to mark a method as the handler of a VCS."""
 
     def decorate(f):
         """Store f in HANDLERS[vcs][method]."""
@@ -409,9 +401,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=
         if verbose:
             print("unable to find command, tried %s" % (commands,))
         return None, None
-    stdout = p.communicate()[0].strip()
-    if sys.version_info[0] >= 3:
-        stdout = stdout.decode()
+    stdout = p.communicate()[0].strip().decode()
     if p.returncode != 0:
         if verbose:
             print("unable to run %s (error)" % dispcmd)
@@ -422,7 +412,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=
 
 LONG_VERSION_PY[
     "git"
-] = '''
+] = r'''
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
 # feature). Distribution tarballs (built by setup.py sdist) and build
@@ -430,7 +420,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=
 # that just contains the computed version number.
 
 # This file is released into the public domain. Generated by
-# versioneer-0.18 (https://github.com/warner/python-versioneer)
+# versioneer-0.19 (https://github.com/python-versioneer/python-versioneer)
 
 """Git implementation of _version.py."""
 
@@ -481,7 +471,7 @@ class NotThisMethod(Exception):
 
 
 def register_vcs_handler(vcs, method):  # decorator
-    """Decorator to mark a method as the handler for a particular VCS."""
+    """Create decorator to mark a method as the handler of a VCS."""
     def decorate(f):
         """Store f in HANDLERS[vcs][method]."""
         if vcs not in HANDLERS:
@@ -517,9 +507,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
         if verbose:
             print("unable to find command, tried %%s" %% (commands,))
         return None, None
-    stdout = p.communicate()[0].strip()
-    if sys.version_info[0] >= 3:
-        stdout = stdout.decode()
+    stdout = p.communicate()[0].strip().decode()
     if p.returncode != 0:
         if verbose:
             print("unable to run %%s (error)" %% dispcmd)
@@ -589,6 +577,10 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         raise NotThisMethod("no keywords at all, weird")
     date = keywords.get("date")
     if date is not None:
+        # Use only the last line.  Previous lines may contain GPG signature
+        # information.
+        date = date.splitlines()[-1]
+
         # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant
         # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601
         # -like" string, which we must then edit to make compliant), because
@@ -724,6 +716,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     # commit date: see ISO-8601 comment in git_versions_from_keywords()
     date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"],
                        cwd=root)[0].strip()
+    # Use only the last line.  Previous lines may contain GPG signature
+    # information.
+    date = date.splitlines()[-1]
     pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
 
     return pieces
@@ -762,18 +757,18 @@ def render_pep440(pieces):
 
 
 def render_pep440_pre(pieces):
-    """TAG[.post.devDISTANCE] -- No -dirty.
+    """TAG[.post0.devDISTANCE] -- No -dirty.
 
     Exceptions:
-    1: no tags. 0.post.devDISTANCE
+    1: no tags. 0.post0.devDISTANCE
     """
     if pieces["closest-tag"]:
         rendered = pieces["closest-tag"]
         if pieces["distance"]:
-            rendered += ".post.dev%%d" %% pieces["distance"]
+            rendered += ".post0.dev%%d" %% pieces["distance"]
     else:
         # exception #1
-        rendered = "0.post.dev%%d" %% pieces["distance"]
+        rendered = "0.post0.dev%%d" %% pieces["distance"]
     return rendered
 
 
@@ -981,6 +976,10 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         raise NotThisMethod("no keywords at all, weird")
     date = keywords.get("date")
     if date is not None:
+        # Use only the last line.  Previous lines may contain GPG signature
+        # information.
+        date = date.splitlines()[-1]
+
         # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
         # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
         # -like" string, which we must then edit to make compliant), because
@@ -1117,6 +1116,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
 
     # commit date: see ISO-8601 comment in git_versions_from_keywords()
     date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip()
+    # Use only the last line.  Previous lines may contain GPG signature
+    # information.
+    date = date.splitlines()[-1]
     pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
 
     return pieces
@@ -1189,7 +1191,7 @@ def versions_from_parentdir(parentdir_prefix, root, verbose):
 
 
 SHORT_VERSION_PY = """
-# This file was generated by 'versioneer.py' (0.18) from
+# This file was generated by 'versioneer.py' (0.19) from
 # revision-control system data, or from the parent directory name of an
 # unpacked source archive. Distribution tarballs contain a pre-generated copy
 # of this file.
@@ -1263,18 +1265,18 @@ def render_pep440(pieces):
 
 
 def render_pep440_pre(pieces):
-    """TAG[.post.devDISTANCE] -- No -dirty.
+    """TAG[.post0.devDISTANCE] -- No -dirty.
 
     Exceptions:
-    1: no tags. 0.post.devDISTANCE
+    1: no tags. 0.post0.devDISTANCE
     """
     if pieces["closest-tag"]:
         rendered = pieces["closest-tag"]
         if pieces["distance"]:
-            rendered += ".post.dev%d" % pieces["distance"]
+            rendered += ".post0.dev%d" % pieces["distance"]
     else:
         # exception #1
-        rendered = "0.post.dev%d" % pieces["distance"]
+        rendered = "0.post0.dev%d" % pieces["distance"]
     return rendered
 
 
@@ -1310,7 +1312,7 @@ def render_pep440_old(pieces):
 
     The ".dev0" means dirty.
 
-    Eexceptions:
+    Exceptions:
     1: no tags. 0.postDISTANCE[.dev0]
     """
     if pieces["closest-tag"]:
@@ -1493,8 +1495,12 @@ def get_version():
     return get_versions()["version"]
 
 
-def get_cmdclass():
-    """Get the custom setuptools/distutils subclasses used by Versioneer."""
+def get_cmdclass(cmdclass=None):
+    """Get the custom setuptools/distutils subclasses used by Versioneer.
+
+    If the package uses a different cmdclass (e.g. one from numpy), it
+    should be provide as an argument.
+    """
     if "versioneer" in sys.modules:
         del sys.modules["versioneer"]
         # this fixes the "python setup.py develop" case (also 'install' and
@@ -1508,9 +1514,9 @@ def get_cmdclass():
         # parent is protected against the child's "import versioneer". By
         # removing ourselves from sys.modules here, before the child build
         # happens, we protect the child from the parent's versioneer too.
-        # Also see https://github.com/warner/python-versioneer/issues/52
+        # Also see https://github.com/python-versioneer/python-versioneer/issues/52
 
-    cmds = {}
+    cmds = {} if cmdclass is None else cmdclass.copy()
 
     # we add "version" to both distutils and setuptools
     from distutils.core import Command
@@ -1553,7 +1559,9 @@ def run(self):
     #  setup.py egg_info -> ?
 
     # we override different "build_py" commands for both environments
-    if "setuptools" in sys.modules:
+    if "build_py" in cmds:
+        _build_py = cmds["build_py"]
+    elif "setuptools" in sys.modules:
         from setuptools.command.build_py import build_py as _build_py
     else:
         from distutils.command.build_py import build_py as _build_py
@@ -1573,6 +1581,31 @@ def run(self):
 
     cmds["build_py"] = cmd_build_py
 
+    if "setuptools" in sys.modules:
+        from setuptools.command.build_ext import build_ext as _build_ext
+    else:
+        from distutils.command.build_ext import build_ext as _build_ext
+
+    class cmd_build_ext(_build_ext):
+        def run(self):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            versions = get_versions()
+            _build_ext.run(self)
+            if self.inplace:
+                # build_ext --inplace will only build extensions in
+                # build/lib<..> dir with no _version.py to write to.
+                # As in place builds will already have a _version.py
+                # in the module dir, we do not need to write one.
+                return
+            # now locate _version.py in the new build/ directory and replace
+            # it with an updated value
+            target_versionfile = os.path.join(self.build_lib, cfg.versionfile_source)
+            print("UPDATING %s" % target_versionfile)
+            write_to_version_file(target_versionfile, versions)
+
+    cmds["build_ext"] = cmd_build_ext
+
     if "cx_Freeze" in sys.modules:  # cx_freeze enabled?
         from cx_Freeze.dist import build_exe as _build_exe
 
@@ -1611,10 +1644,7 @@ def run(self):
         del cmds["build_py"]
 
     if "py2exe" in sys.modules:  # py2exe enabled?
-        try:
-            from py2exe.distutils_buildexe import py2exe as _py2exe  # py3
-        except ImportError:
-            from py2exe.build_exe import py2exe as _py2exe  # py2
+        from py2exe.distutils_buildexe import py2exe as _py2exe
 
         class cmd_py2exe(_py2exe):
             def run(self):
@@ -1643,7 +1673,9 @@ def run(self):
         cmds["py2exe"] = cmd_py2exe
 
     # we override different "sdist" commands for both environments
-    if "setuptools" in sys.modules:
+    if "sdist" in cmds:
+        _sdist = cmds["sdist"]
+    elif "setuptools" in sys.modules:
         from setuptools.command.sdist import sdist as _sdist
     else:
         from distutils.command.sdist import sdist as _sdist
@@ -1718,7 +1750,7 @@ def make_release_tree(self, base_dir, files):
 
 
 def do_setup():
-    """Main VCS-independent setup function for installing Versioneer."""
+    """Do main VCS-independent setup function for installing Versioneer."""
     root = get_root()
     try:
         cfg = get_config_from_root(root)