From d58c72132c7b26efb80245283b267cca0ccfa242 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 31 Oct 2022 09:49:33 +0000 Subject: [PATCH 01/13] =?UTF-8?q?=E2=9C=A8=20Update=20Lung=20model=20param?= =?UTF-8?q?eters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 ++ InnerEye/ML/configs/segmentation/Lung.py | 57 +++++++++++++----------- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index d41e8fa1c..aab1fc1de 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,7 @@ cifar-10-python.tar.gz .amltconfig amlt_job.yml + +## VS Code + +.vscode/launch.json diff --git a/InnerEye/ML/configs/segmentation/Lung.py b/InnerEye/ML/configs/segmentation/Lung.py index e5e743fe6..eb2107130 100644 --- a/InnerEye/ML/configs/segmentation/Lung.py +++ b/InnerEye/ML/configs/segmentation/Lung.py @@ -25,48 +25,51 @@ class Lung(SegmentationModelBase): def __init__(self, **kwargs: Any) -> None: fg_classes = ["spinalcord", "lung_r", "lung_l", "heart", "esophagus"] fg_display_names = ["SpinalCord", "Lung_R", "Lung_L", "Heart", "Esophagus"] + + azure_dataset_id = kwargs.pop("azure_dataset_id", LUNG_AZURE_DATASET_ID) + super().__init__( + adam_betas=(0.9, 0.999), architecture="UNet3D", - feature_channels=[32], - kernel_size=3, - azure_dataset_id=LUNG_AZURE_DATASET_ID, - crop_size=(64, 224, 224), - test_crop_size=(128, 512, 512), - image_channels=["ct"], - ground_truth_ids=fg_classes, - ground_truth_ids_display_names=fg_display_names, + azure_dataset_id=azure_dataset_id, + check_exclusive=False, + class_weights=equally_weighted_classes(fg_classes, background_weight=0.02), colours=[(255, 255, 255)] * len(fg_classes), + crop_size=(64, 256, 256), + feature_channels=[32], fill_holes=[False] * len(fg_classes), - roi_interpreted_types=["ORGAN"] * len(fg_classes), - largest_connected_component_foreground_classes=["lung_r", "lung_l", "heart"], - num_dataload_workers=2, - norm_method=PhotometricNormalizationMethod.CtWindow, - level=40, - window=400, - class_weights=equally_weighted_classes(fg_classes, background_weight=0.02), - train_batch_size=8, + ground_truth_ids_display_names=fg_display_names, + ground_truth_ids=fg_classes, + image_channels=["ct"], inference_batch_size=1, inference_stride_size=(64, 256, 256), - num_epochs=140, + kernel_size=3, + l_rate_polynomial_gamma=0.9, l_rate=1e-3, + largest_connected_component_foreground_classes=["lung_l", "lung_r", "heart"], + level=-500, + loss_type=SegmentationLoss.SoftDice, min_l_rate=1e-5, - l_rate_polynomial_gamma=0.9, - optimizer_type=OptimizerType.Adam, - opt_eps=1e-4, - adam_betas=(0.9, 0.999), momentum=0.9, - weight_decay=1e-4, + monitoring_interval_seconds=0, + norm_method=PhotometricNormalizationMethod.CtWindow, + num_dataload_workers=10, + num_epochs=300, + opt_eps=1e-4, + optimizer_type=OptimizerType.Adam, + roi_interpreted_types=["ORGAN"] * len(fg_classes), + test_crop_size=(128, 512, 512), + train_batch_size=10, use_mixed_precision=True, use_model_parallel=True, - monitoring_interval_seconds=0, - loss_type=SegmentationLoss.SoftDice, - check_exclusive=False, + weight_decay=1e-4, + window=2200, ) self.add_and_validate(kwargs) + def get_model_train_test_dataset_splits(self, dataset_df: pd.DataFrame) -> DatasetSplits: - # The first 24 subject IDs are the designated test subjects in this dataset. - test = list(map(str, range(0, 24))) + test = list(map(str, range(0, 9))) train_val = list(dataset_df[~dataset_df.subject.isin(test)].subject.unique()) val = list(map(str, numpy.random.choice(train_val, int(len(train_val) * 0.1), replace=False))) From b92ad8b6570c1524a9842bdb9c6f7aecd19db0fc Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 31 Oct 2022 09:53:48 +0000 Subject: [PATCH 02/13] =?UTF-8?q?=F0=9F=9A=A8=20Fix=20flake8=20warning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/ML/configs/segmentation/Lung.py | 1 - 1 file changed, 1 deletion(-) diff --git a/InnerEye/ML/configs/segmentation/Lung.py b/InnerEye/ML/configs/segmentation/Lung.py index eb2107130..268f373c1 100644 --- a/InnerEye/ML/configs/segmentation/Lung.py +++ b/InnerEye/ML/configs/segmentation/Lung.py @@ -67,7 +67,6 @@ def __init__(self, **kwargs: Any) -> None: ) self.add_and_validate(kwargs) - def get_model_train_test_dataset_splits(self, dataset_df: pd.DataFrame) -> DatasetSplits: test = list(map(str, range(0, 9))) train_val = list(dataset_df[~dataset_df.subject.isin(test)].subject.unique()) From 6b7aa6ec10844977cb50743257b2656631e624e7 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 31 Oct 2022 10:53:43 +0000 Subject: [PATCH 03/13] =?UTF-8?q?=F0=9F=93=9D=20Create=20doc=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/index.rst | 2 +- docs/source/md/lung_model.md | 3 +++ docs/source/rst/models.rst | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 docs/source/md/lung_model.md create mode 100644 docs/source/rst/models.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index e6936fa5b..c3270a2b2 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -22,7 +22,7 @@ InnerEye-DeepLearning Documentation md/debugging_and_monitoring.md md/model_diagnostics.md md/move_model.md - md/hippocampus_model.md + rst/models .. toctree:: :maxdepth: 1 diff --git a/docs/source/md/lung_model.md b/docs/source/md/lung_model.md new file mode 100644 index 000000000..cd0bf68c1 --- /dev/null +++ b/docs/source/md/lung_model.md @@ -0,0 +1,3 @@ +# Lung Model Goes Here + +Helo :) diff --git a/docs/source/rst/models.rst b/docs/source/rst/models.rst new file mode 100644 index 000000000..6bf7d34b7 --- /dev/null +++ b/docs/source/rst/models.rst @@ -0,0 +1,7 @@ +Pre-Trained Models +================== + +.. toctree:: + + ../md/hippocampus_model.md + ../md/lung_model.md From eda5fccf03c17ae4cf15e982bdc12b97ff3a3524 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 31 Oct 2022 11:48:59 +0000 Subject: [PATCH 04/13] =?UTF-8?q?=F0=9F=93=9D=20Create=20generic=20pre-tra?= =?UTF-8?q?ined=20model=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/md/hippocampus_model.md | 6 +- docs/source/md/lung_model.md | 18 +++++- docs/source/rst/models.rst | 97 ++++++++++++++++++++++++++++- 3 files changed, 114 insertions(+), 7 deletions(-) mode change 100644 => 100755 docs/source/rst/models.rst diff --git a/docs/source/md/hippocampus_model.md b/docs/source/md/hippocampus_model.md index e254fbc71..0fc770a52 100644 --- a/docs/source/md/hippocampus_model.md +++ b/docs/source/md/hippocampus_model.md @@ -1,4 +1,4 @@ -# Trained model for hippocampal segmentation +# Hippocampus Segmentation Model ## Purpose @@ -14,11 +14,11 @@ Please note that this model is intended for research purposes only. You are resp The following instructions assume you have completed the preceding setup steps in the [InnerEye README](https://github.com/microsoft/InnerEye-DeepLearning/), in particular, [Setting up Azure Machine Learning](setting_up_aml.md). -### Create an Azure ML Dataset +### Create an AzureML Dataset To evaluate this model on your own data, you will first need to register an [Azure ML Dataset](https://docs.microsoft.com/en-us/azure/machine-learning/v1/how-to-create-register-datasets). You can follow the instructions in the for [creating datasets](creating_dataset.md) in order to do this. -## Downloading the model +### Downloading the model The saved weights from the trained Hippocampus model can be downloaded along with the source code used to train it from [our GitHub releases page](https://github.com/microsoft/InnerEye-DeepLearning/releases/tag/v0.5). diff --git a/docs/source/md/lung_model.md b/docs/source/md/lung_model.md index cd0bf68c1..8d436bac6 100644 --- a/docs/source/md/lung_model.md +++ b/docs/source/md/lung_model.md @@ -1,3 +1,17 @@ -# Lung Model Goes Here +# Lung Segmentation Model -Helo :) +## Purpose + +## Connected Components + +## Model Card + +### Model Details + +### Limitations + +### Intended Uses + +### Training Data + +### Metrics diff --git a/docs/source/rst/models.rst b/docs/source/rst/models.rst old mode 100644 new mode 100755 index 6bf7d34b7..37d6511e8 --- a/docs/source/rst/models.rst +++ b/docs/source/rst/models.rst @@ -1,7 +1,100 @@ Pre-Trained Models ================== +InnerEye-DeepLearning currently has two pre-trained models avaiable for use +in segmentation tasks. This page describes how to set up and use these models. +For specific information on the models, please refer to the relevant model card: + .. toctree:: + :maxdepth: 1 + + ../md/hippocampus_model.md + ../md/lung_model.md + + +Terms of use +------------ + +Please note that all models provided by InnerEye-DeepLearning are intended for +research purposes only. You are responsible for the performance, the necessary testing, + and if needed any regulatory clearance for any of the models produced by this toolbox. + +Usage +----- + +The following instructions assume you have completed the preceding setup +steps in the `InnerEye +README `__, in +particular, `Setting up Azure Machine Learning `__. + +Create an AzureML Dataset +~~~~~~~~~~~~~~~~~~~~~~~~~ + +To evaluate pre-trained models on your own data, you will first need to register +an `Azure ML +Dataset `__. +You can follow the instructions in the for `creating +datasets `__ in order to do this. + +Downloading the models +~~~~~~~~~~~~~~~~~~~~~~ + +The saved weights for each model can be found in their respective :ref:`model cards`. +You will need to download the weights and source code for the model that you wish to use. + +Registering a model in Azure ML +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To evaluate the model in Azure ML, you must first `register an Azure ML +Model `__. +To register the pre-trained model in your AML Workspace, unpack the +source code downloaded in the previous step and follow InnerEye's +`instructions to upload models to Azure ML `__. + +Run the following from a folder that contains both the ``ENVIRONMENT/`` +and ``MODEL/`` folders (these exist inside the downloaded model files): + +.. code:: shell + + WORKSPACE="fill with your workspace name" + GROUP="fill with your resource group name" + SUBSCRIPTION="fill with your subscription ID" + + python InnerEye/Scripts/move_model.py \ + --action upload \ + --path . \ + --workspace_name $WORKSPACE \ + --resource_group $GROUP \ + --subscription_id $SUBSCRIPTION \ + --model_id Hippocampus:118 + +Evaluating the model +~~~~~~~~~~~~~~~~~~~~ + +You can evaluate the model either in Azure ML or locally using the +downloaded checkpoint files. These 2 scenarios are described in more +detail, along with instructions in `testing an existing +model `__. + +For example, to evaluate the model on your Dataset in Azure ML, run the +following from within the directory ``*/MODEL/final_ensemble_model/`` + +.. code:: shell + + CLUSTER="fill with your cluster name" + DATASET_ID="fill with your dataset name" + + python InnerEye/ML/runner.py \ + --azure_dataset_id $DATASET_ID \ + --model \ + --model_id : \ + --experiment_name \ + --azureml \ + --no-train \ + --cluster $CLUSTER + --restrict_subjects=0,0,+ + +Deploy with InnerEye Gateway +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - ../md/hippocampus_model.md - ../md/lung_model.md +To deploy a model using the InnerEye Gateway, see the instructions in the `Gateway Repo `__. From 32e77f8b777a6e563950e2a8a140e163784f3961 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 31 Oct 2022 11:54:47 +0000 Subject: [PATCH 05/13] =?UTF-8?q?=F0=9F=93=9D=20Update=20hippocampus=20mod?= =?UTF-8?q?el=20card?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/md/hippocampus_model.md | 61 +---------------------------- 1 file changed, 1 insertion(+), 60 deletions(-) diff --git a/docs/source/md/hippocampus_model.md b/docs/source/md/hippocampus_model.md index 0fc770a52..0fd5e0113 100644 --- a/docs/source/md/hippocampus_model.md +++ b/docs/source/md/hippocampus_model.md @@ -8,72 +8,13 @@ This documentation describes how to use our pre-trained model to segment the lef Please note that this model is intended for research purposes only. You are responsible for the performance, the necessary testing, and if needed any regulatory clearance for any of the models produced by this toolbox. ---- - ## Usage -The following instructions assume you have completed the preceding setup steps in the [InnerEye README](https://github.com/microsoft/InnerEye-DeepLearning/), in particular, [Setting up Azure Machine Learning](setting_up_aml.md). - -### Create an AzureML Dataset - -To evaluate this model on your own data, you will first need to register an [Azure ML Dataset](https://docs.microsoft.com/en-us/azure/machine-learning/v1/how-to-create-register-datasets). You can follow the instructions in the for [creating datasets](creating_dataset.md) in order to do this. - -### Downloading the model - -The saved weights from the trained Hippocampus model can be downloaded along with the source code used to train it from [our GitHub releases page](https://github.com/microsoft/InnerEye-DeepLearning/releases/tag/v0.5). - -### Registering a model in Azure ML - -To evaluate the model in Azure ML, you must first [register an Azure ML Model](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.model.model?view=azure-ml-py#remarks). To register the Hippocampus model in your AML Workspace, unpack the source code downloaded in the previous step and follow InnerEye's [instructions to upload models to Azure ML](move_model.md). - -Run the following from a folder that contains both the `ENVIRONMENT/` and `MODEL/` folders (these exist inside the downloaded model files): - -```shell -WORKSPACE="fill with your workspace name" -GROUP="fill with your resource group name" -SUBSCRIPTION="fill with your subscription ID" - -python InnerEye/Scripts/move_model.py \ - --action upload \ - --path . \ - --workspace_name $WORKSPACE \ - --resource_group $GROUP \ - --subscription_id $SUBSCRIPTION \ - --model_id Hippocampus:118 -``` - -### Evaluating the model - -You can evaluate the model either in Azure ML or locally using the downloaded checkpoint files. These 2 scenarios are described in more detail, along with instructions in [testing an existing model](building_models.md#testing-an-existing-model). - -For example, to evaluate the model on your Dataset in Azure ML, run the following from within the directory `*/MODEL/final_ensemble_model/` - -```shell -CLUSTER="fill with your cluster name" -DATASET_ID="fill with your dataset name" - -python InnerEye/ML/runner.py \ - --azure_dataset_id $DATASET_ID \ - --model Hippocampus \ - --model_id Hippocampus:111 \ - --experiment_name evaluate_hippocampus_model \ - --azureml \ - --no-train \ - --cluster $CLUSTER - --restrict_subjects=0,0,+ -``` - ### Connected components It is possible to apply connected components as a post-processing step, although by default this is disabled. To enable, update the property `largest_connected_component_foreground_classes` of the Hippocampus class in `InnerEye/ML/configs/segmentation/Hippocampus.py` -### Deploy with InnerEye Gateway - -To deploy this model, see the instructions in the [InnerEye README](https://github.com/microsoft/InnerEye-DeepLearning/). - ---- - -## Hippocampal Segmentation Model Card +## Model Card ### Model details From 50dae53569a310bf84632f00a69d996cedb27565 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 31 Oct 2022 13:46:14 +0000 Subject: [PATCH 06/13] =?UTF-8?q?=F0=9F=93=9D=20Update=20model=20cards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/md/hippocampus_model.md | 2 +- docs/source/md/lung_model.md | 39 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docs/source/md/hippocampus_model.md b/docs/source/md/hippocampus_model.md index 0fd5e0113..bf316cfb9 100644 --- a/docs/source/md/hippocampus_model.md +++ b/docs/source/md/hippocampus_model.md @@ -2,7 +2,7 @@ ## Purpose -This documentation describes how to use our pre-trained model to segment the left and right hippocampi from brain MRI scans. The model was trained on data from the [ADNI](https://adni.loni.usc.edu/) dataset (for more information see the model card below). This data is publicly available via their website, but users must sign a Data Use Agreement in order to gain access. We do not provide access to the data. The following description assumes the user has their own dataset to evaluate/ retrain the model on. +This documentation describes our pre-trained model for segmentation of the left and right hippocampi from brain MRI scans. The model was trained on data from the [ADNI](https://adni.loni.usc.edu/) dataset (for more information see the model card below). This data is publicly available via their website, but users must sign a Data Use Agreement in order to gain access. We do not provide access to the data. The following description assumes the user has their own dataset to evaluate/ retrain the model on. ## Terms of use diff --git a/docs/source/md/lung_model.md b/docs/source/md/lung_model.md index 8d436bac6..cfa64252a 100644 --- a/docs/source/md/lung_model.md +++ b/docs/source/md/lung_model.md @@ -2,16 +2,55 @@ ## Purpose +This model is designed to perform segmentation of CT scans of human torsos. It is trained to identify 5 key structures: left lung, right lung, heart, spinalcord and esophagus. + +## Terms of Use + +This model is intended for research purposes only. You are responsible for the performance, the necessary testing, and if needed any regulatory clearance for any of the models produced by this toolbox. + ## Connected Components +It is possible to apply connected components as a post-processing step, and by default this is performed on the 3 largest structures: both lungs and the heart. To alter this behaviour, update the property `largest_connected_component_foreground_classes` of the Lung class in `InnerEye/ML/configs/segmentation/Lung.py`. + ## Model Card ### Model Details +- Organisation: Biomedical Imaging Team at Microsoft Research, Cambridge UK. +- Model date: 31st October 2022. +- Model version: 1.0. +- Model type: ensemble of 3D UNet. Training details are as described in [this paper](https://jamanetwork.com/journals/jamanetworkopen/fullarticle/2773292). +- Training details: 5 fold ensemble model. Trained on the [LCTSC 2017 dataset](https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=24284539) (described in detail below). +- License: The model is released under MIT license as described [here](https://github.com/microsoft/InnerEye-DeepLearning/blob/main/LICENSE). +- Contact: innereyeinfo@microsoft.com. + ### Limitations +The dataset used for training contains only 60 scans, 10 of which are withheld for testing. This limited amount of training data means that the model may not yet generalise well to data samples from outside the dataset and underperforms on the smaller structures (esophagus and spinalcord). + ### Intended Uses +This model is intended for research purposes only. It is intended to be used as a starting-point for more challenging segmentation tasks or training using more thorough and comprehensive segmentation tasks. + ### Training Data +This model is trained on the [LCTSC 2017 dataset](https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=24284539). For a detailed description on this data, including the contouring guidelines, see [this page](https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=24284539#242845396723d79f9909442996e4dd0af5e56a30). + +To create the dataset used for training, we utilised the [InnerEye-CreateDataset tool](https://github.com/microsoft/InnerEye-CreateDataset). The dataset was processed using the following command: + +```shell + .\InnerEye.CreateDataset.Runner.exe dataset --datasetRootDirectory=C:\Users\a-phessey\Datasets\lung_dicom --niftiDatasetDirectory=lung_nifti --dicomDatasetDirectory=LCTSC --geoNorm 1 1 3 --groundTruthDescendingPriority esophagus spinalcord lung_r lung_l heart +``` + ### Metrics + +Metrics for the withheld test data (first 10 scans in the dataset), can be seen in the following table: + +| Structure | count | DiceNumeric_mean | DiceNumeric_std | DiceNumeric_min | DiceNumeric_max | HausdorffDistance_mm_mean | HausdorffDistance_mm_std | HausdorffDistance_mm_min | HausdorffDistance_mm_max | MeanDistance_mm_mean | MeanDistance_mm_std | MeanDistance_mm_min | MeanDistance_mm_max | +|---------------|---------|------------------|-----------------|-----------------|-----------------|---------------------------|--------------------------|--------------------------|--------------------------|----------------------|---------------------|---------------------|---------------------| +| lung_r | 10 | 0.982 | 0.010 | 0.957 | 0.990 | 11.107 | 3.382 | 6.782 | 16.734 | 0.366 | 0.213 | 0.177 | 0.902 | +| lung_l | 10 | 0.980 | 0.009 | 0.958 | 0.989 | 11.457 | 3.389 | 8.246 | 18.028 | 0.359 | 0.261 | 0.191 | 1.040 | +| heart | 10 | 0.953 | 0.015 | 0.902 | 0.953 | 13.384 | 3.101 | 8.186 | 17.975 | 1.983 | 0.574 | 1.519 | 3.301 | +| spinalcord | 10 | 0.848 | 0.044 | 0.763 | 0.894 | 29.362 | 21.370 | 10.050 | 78.473 | 1.888 | 2.029 | 0.631 | 6.710 | +| esophagus | 10 | 0.701 | 0.108 | 0.489 | 0.809 | 27.945 | 27.767 | 12.728 | 80.214 | 3.991 | 5.735 | 1.108 | 18.489 | +| | | | | | | | | | | | | | | From afb65849aeb5b65c7e6dfb68c5b6c0729e00186b Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 1 Nov 2022 14:09:56 +0000 Subject: [PATCH 07/13] =?UTF-8?q?=F0=9F=93=9D=20Fix=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/rst/models.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/rst/models.rst b/docs/source/rst/models.rst index 37d6511e8..f9086f8e9 100755 --- a/docs/source/rst/models.rst +++ b/docs/source/rst/models.rst @@ -66,7 +66,7 @@ and ``MODEL/`` folders (these exist inside the downloaded model files): --workspace_name $WORKSPACE \ --resource_group $GROUP \ --subscription_id $SUBSCRIPTION \ - --model_id Hippocampus:118 + --model_id : Evaluating the model ~~~~~~~~~~~~~~~~~~~~ From 548da0561653edc266183b9f0fc2b490989e9022 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 2 Nov 2022 11:57:14 +0000 Subject: [PATCH 08/13] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20Fit=20lung=20model?= =?UTF-8?q?=20onto=20smaller=20GPUs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- InnerEye/ML/configs/segmentation/Lung.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/InnerEye/ML/configs/segmentation/Lung.py b/InnerEye/ML/configs/segmentation/Lung.py index 268f373c1..646f408fe 100644 --- a/InnerEye/ML/configs/segmentation/Lung.py +++ b/InnerEye/ML/configs/segmentation/Lung.py @@ -35,7 +35,7 @@ def __init__(self, **kwargs: Any) -> None: check_exclusive=False, class_weights=equally_weighted_classes(fg_classes, background_weight=0.02), colours=[(255, 255, 255)] * len(fg_classes), - crop_size=(64, 256, 256), + crop_size=(64, 224, 224), feature_channels=[32], fill_holes=[False] * len(fg_classes), ground_truth_ids_display_names=fg_display_names, @@ -53,13 +53,13 @@ def __init__(self, **kwargs: Any) -> None: momentum=0.9, monitoring_interval_seconds=0, norm_method=PhotometricNormalizationMethod.CtWindow, - num_dataload_workers=10, + num_dataload_workers=2, num_epochs=300, opt_eps=1e-4, optimizer_type=OptimizerType.Adam, roi_interpreted_types=["ORGAN"] * len(fg_classes), test_crop_size=(128, 512, 512), - train_batch_size=10, + train_batch_size=3, use_mixed_precision=True, use_model_parallel=True, weight_decay=1e-4, From c2ad815162d164d22776229c0b4d0a7fb092440e Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 3 Nov 2022 09:48:13 +0000 Subject: [PATCH 09/13] Reduce test crop size for v100s --- InnerEye/ML/configs/segmentation/Lung.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InnerEye/ML/configs/segmentation/Lung.py b/InnerEye/ML/configs/segmentation/Lung.py index 646f408fe..95a16b321 100644 --- a/InnerEye/ML/configs/segmentation/Lung.py +++ b/InnerEye/ML/configs/segmentation/Lung.py @@ -58,7 +58,7 @@ def __init__(self, **kwargs: Any) -> None: opt_eps=1e-4, optimizer_type=OptimizerType.Adam, roi_interpreted_types=["ORGAN"] * len(fg_classes), - test_crop_size=(128, 512, 512), + test_crop_size=(64, 256, 256), train_batch_size=3, use_mixed_precision=True, use_model_parallel=True, From 0336e95915e9345690c6b7c21ff35369791fae4b Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 3 Nov 2022 13:11:32 +0000 Subject: [PATCH 10/13] Maximise test_crop_size --- InnerEye/ML/configs/segmentation/Lung.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InnerEye/ML/configs/segmentation/Lung.py b/InnerEye/ML/configs/segmentation/Lung.py index 95a16b321..26c76fff1 100644 --- a/InnerEye/ML/configs/segmentation/Lung.py +++ b/InnerEye/ML/configs/segmentation/Lung.py @@ -58,7 +58,7 @@ def __init__(self, **kwargs: Any) -> None: opt_eps=1e-4, optimizer_type=OptimizerType.Adam, roi_interpreted_types=["ORGAN"] * len(fg_classes), - test_crop_size=(64, 256, 256), + test_crop_size=(112, 512, 512), train_batch_size=3, use_mixed_precision=True, use_model_parallel=True, From 478d37b752da9adf0d21e15e081c5f9128cd3104 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 3 Nov 2022 14:24:24 +0000 Subject: [PATCH 11/13] =?UTF-8?q?=F0=9F=93=9D=20Update=20lung=20model=20sc?= =?UTF-8?q?ore=20card?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/md/lung_model.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/md/lung_model.md b/docs/source/md/lung_model.md index cfa64252a..142a6f581 100644 --- a/docs/source/md/lung_model.md +++ b/docs/source/md/lung_model.md @@ -48,9 +48,9 @@ Metrics for the withheld test data (first 10 scans in the dataset), can be seen | Structure | count | DiceNumeric_mean | DiceNumeric_std | DiceNumeric_min | DiceNumeric_max | HausdorffDistance_mm_mean | HausdorffDistance_mm_std | HausdorffDistance_mm_min | HausdorffDistance_mm_max | MeanDistance_mm_mean | MeanDistance_mm_std | MeanDistance_mm_min | MeanDistance_mm_max | |---------------|---------|------------------|-----------------|-----------------|-----------------|---------------------------|--------------------------|--------------------------|--------------------------|----------------------|---------------------|---------------------|---------------------| -| lung_r | 10 | 0.982 | 0.010 | 0.957 | 0.990 | 11.107 | 3.382 | 6.782 | 16.734 | 0.366 | 0.213 | 0.177 | 0.902 | -| lung_l | 10 | 0.980 | 0.009 | 0.958 | 0.989 | 11.457 | 3.389 | 8.246 | 18.028 | 0.359 | 0.261 | 0.191 | 1.040 | -| heart | 10 | 0.953 | 0.015 | 0.902 | 0.953 | 13.384 | 3.101 | 8.186 | 17.975 | 1.983 | 0.574 | 1.519 | 3.301 | -| spinalcord | 10 | 0.848 | 0.044 | 0.763 | 0.894 | 29.362 | 21.370 | 10.050 | 78.473 | 1.888 | 2.029 | 0.631 | 6.710 | -| esophagus | 10 | 0.701 | 0.108 | 0.489 | 0.809 | 27.945 | 27.767 | 12.728 | 80.214 | 3.991 | 5.735 | 1.108 | 18.489 | +| lung_l | 10 | 0.984 | 0.009 | 0.958 | 0.990 | 11.642 | 4.868 | 6.558 | 19.221 | 0.344 | 0.266 | 0.167 | 1.027 | +| lung_r | 10 | 0.983 | 0.009 | 0.960 | 0.991 | 10.764 | 3.307 | 6.325 | 16.156 | 0.345 | 0.200 | 0.160 | 0.797 | +| spinalcord | 10 | 0.860 | 0.050 | 0.756 | 0.912 | 27.213 | 22.015 | 12.000 | 81.398 | 1.750 | 2.167 | 0.552 | 7.209 | +| heart | 10 | 0.935 | 0.015 | 0.908 | 0.953 | 17.550 | 14.796 | 9.000 | 17.550 | 2.022 | 0.661 | 1.456 | 3.299 | +| esophagus | 10 | 0.728 | 0.128 | 0.509 | 0.891 | 23.503 | 25.679 | 6.173 | 72.008 | 3.207 | 4.333 | 0.409 | 13.991 | | | | | | | | | | | | | | | | From 32922050ace48c7323e20403cc1284ac7515b3dd Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 3 Nov 2022 15:07:10 +0000 Subject: [PATCH 12/13] =?UTF-8?q?=F0=9F=93=9D=20Add=20model=20download=20l?= =?UTF-8?q?inks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/md/hippocampus_model.md | 6 ++++-- docs/source/md/lung_model.md | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/source/md/hippocampus_model.md b/docs/source/md/hippocampus_model.md index bf316cfb9..ba4e3a68c 100644 --- a/docs/source/md/hippocampus_model.md +++ b/docs/source/md/hippocampus_model.md @@ -8,9 +8,11 @@ This documentation describes our pre-trained model for segmentation of the left Please note that this model is intended for research purposes only. You are responsible for the performance, the necessary testing, and if needed any regulatory clearance for any of the models produced by this toolbox. -## Usage +## Download -### Connected components +The hippocampus segmentation model can be downloaded from [this release](https://github.com/microsoft/InnerEye-DeepLearning/releases/tag/v0.5). + +## Connected components It is possible to apply connected components as a post-processing step, although by default this is disabled. To enable, update the property `largest_connected_component_foreground_classes` of the Hippocampus class in `InnerEye/ML/configs/segmentation/Hippocampus.py` diff --git a/docs/source/md/lung_model.md b/docs/source/md/lung_model.md index 142a6f581..80c3f000c 100644 --- a/docs/source/md/lung_model.md +++ b/docs/source/md/lung_model.md @@ -8,6 +8,11 @@ This model is designed to perform segmentation of CT scans of human torsos. It i This model is intended for research purposes only. You are responsible for the performance, the necessary testing, and if needed any regulatory clearance for any of the models produced by this toolbox. + +## Download + +The lung segmentation model can be downloaded from [this release](https://github.com/microsoft/InnerEye-DeepLearning/releases/tag/v0.8). + ## Connected Components It is possible to apply connected components as a post-processing step, and by default this is performed on the 3 largest structures: both lungs and the heart. To alter this behaviour, update the property `largest_connected_component_foreground_classes` of the Lung class in `InnerEye/ML/configs/segmentation/Lung.py`. From 06f957119271aaab606c7750f1b66c215f38914e Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 7 Nov 2022 10:35:01 +0000 Subject: [PATCH 13/13] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Address=20PR=20comme?= =?UTF-8?q?nts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 ---- docs/source/md/lung_model.md | 26 ++++++++++++++++---------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index aab1fc1de..d41e8fa1c 100644 --- a/.gitignore +++ b/.gitignore @@ -167,7 +167,3 @@ cifar-10-python.tar.gz .amltconfig amlt_job.yml - -## VS Code - -.vscode/launch.json diff --git a/docs/source/md/lung_model.md b/docs/source/md/lung_model.md index 80c3f000c..968a7f37a 100644 --- a/docs/source/md/lung_model.md +++ b/docs/source/md/lung_model.md @@ -4,11 +4,6 @@ This model is designed to perform segmentation of CT scans of human torsos. It is trained to identify 5 key structures: left lung, right lung, heart, spinalcord and esophagus. -## Terms of Use - -This model is intended for research purposes only. You are responsible for the performance, the necessary testing, and if needed any regulatory clearance for any of the models produced by this toolbox. - - ## Download The lung segmentation model can be downloaded from [this release](https://github.com/microsoft/InnerEye-DeepLearning/releases/tag/v0.8). @@ -29,9 +24,15 @@ It is possible to apply connected components as a post-processing step, and by d - License: The model is released under MIT license as described [here](https://github.com/microsoft/InnerEye-DeepLearning/blob/main/LICENSE). - Contact: innereyeinfo@microsoft.com. +### Terms of use + +Please note that all models provided by InnerEye-DeepLearning are intended for research purposes only. You are responsible for the performance, the necessary testing, and if needed any regulatory clearance for any of the models produced by this toolbox. + ### Limitations -The dataset used for training contains only 60 scans, 10 of which are withheld for testing. This limited amount of training data means that the model may not yet generalise well to data samples from outside the dataset and underperforms on the smaller structures (esophagus and spinalcord). +The dataset used for training contains only 60 scans, 10 of which are withheld for testing. This limited amount of training data means that the model underperforms on the smaller structures (esophagus and spinalcord) and may not yet generalise well to data samples from outside the dataset. + +Furthermore, the dataset description does not contain details on the population of patients used for creating the dataset. Therefore it is not possible to assess whether this model is suitable for use on a target population outside of the dataset. ### Intended Uses @@ -41,11 +42,16 @@ This model is intended for research purposes only. It is intended to be used as This model is trained on the [LCTSC 2017 dataset](https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=24284539). For a detailed description on this data, including the contouring guidelines, see [this page](https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=24284539#242845396723d79f9909442996e4dd0af5e56a30). -To create the dataset used for training, we utilised the [InnerEye-CreateDataset tool](https://github.com/microsoft/InnerEye-CreateDataset). The dataset was processed using the following command: +The following steps were carrried out to create the dataset used for training this model: + +1. Download the DICOM dataset from the above LCTSC 2017 link. +1. Use the [InnerEye-CreateDataset tool](https://github.com/microsoft/InnerEye-CreateDataset) to run the following command on the data: + + ```shell + .\InnerEye.CreateDataset.Runner.exe dataset --datasetRootDirectory= --niftiDatasetDirectory=lung_nifti --dicomDatasetDirectory=LCTSC --geoNorm 1 1 3 --groundTruthDescendingPriority esophagus spinalcord lung_r lung_l heart + ``` -```shell - .\InnerEye.CreateDataset.Runner.exe dataset --datasetRootDirectory=C:\Users\a-phessey\Datasets\lung_dicom --niftiDatasetDirectory=lung_nifti --dicomDatasetDirectory=LCTSC --geoNorm 1 1 3 --groundTruthDescendingPriority esophagus spinalcord lung_r lung_l heart -``` +1. Upload and register NIFTI dataset to Azure by following the [dataset creation](creating_dataset.md) guide. ### Metrics