Skip to content

Update auc api #156

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wyli merged 1 commit into from
Mar 24, 2021
Merged

Update auc api #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 23 additions & 16 deletions 2d_classification/mednist_tutorial.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 2,
"metadata": {
"tags": []
},
Expand All @@ -54,23 +54,23 @@
"name": "stdout",
"output_type": "stream",
"text": [
"MONAI version: 0.4.0+42.g58bc231\n",
"Numpy version: 1.19.5\n",
"Pytorch version: 1.7.1\n",
"MONAI flags: HAS_EXT = False, USE_COMPILED = False\n",
"MONAI rev id: 58bc231cb8a3eff6864cf857b2b4dc263d6738cc\n",
"MONAI version: 0+untagged.1.ge8e2f3e.dirty\n",
"Numpy version: 1.19.2\n",
"Pytorch version: 1.8.0a0+52ea372\n",
"MONAI flags: HAS_EXT = True, USE_COMPILED = False\n",
"MONAI rev id: e8e2f3e2efb6bf8072b7ded40cb57bca01e07773\n",
"\n",
"Optional dependencies:\n",
"Pytorch Ignite version: 0.4.2\n",
"Pytorch Ignite version: 0.4.4\n",
"Nibabel version: 3.2.1\n",
"scikit-image version: 0.18.1\n",
"Pillow version: 8.1.0\n",
"Tensorboard version: 2.4.0\n",
"scikit-image version: 0.15.0\n",
"Pillow version: 8.1.2\n",
"Tensorboard version: 1.15.0+nv\n",
"gdown version: 3.12.2\n",
"TorchVision version: 0.8.2\n",
"TorchVision version: 0.9.0a0\n",
"ITK version: 5.1.2\n",
"tqdm version: 4.51.0\n",
"lmdb version: 1.0.0\n",
"tqdm version: 4.53.0\n",
"lmdb version: 1.1.1\n",
"psutil version: 5.8.0\n",
"\n",
"For details about installing the optional dependencies, please visit:\n",
Expand Down Expand Up @@ -105,7 +105,9 @@
"from monai.metrics import compute_roc_auc\n",
"from monai.networks.nets import DenseNet121\n",
"from monai.transforms import (\n",
" Activations,\n",
" AddChannel,\n",
" AsDiscrete,\n",
" Compose,\n",
" LoadImage,\n",
" RandFlip,\n",
Expand Down Expand Up @@ -367,7 +369,10 @@
")\n",
"\n",
"val_transforms = Compose(\n",
" [LoadImage(image_only=True), AddChannel(), ScaleIntensity(), ToTensor()])"
" [LoadImage(image_only=True), AddChannel(), ScaleIntensity(), ToTensor()])\n",
"\n",
"act = Activations(softmax=True)\n",
"to_onehot = AsDiscrete(to_onehot=True, n_classes=num_class)"
]
},
{
Expand Down Expand Up @@ -1161,8 +1166,10 @@
" )\n",
" y_pred = torch.cat([y_pred, model(val_images)], dim=0)\n",
" y = torch.cat([y, val_labels], dim=0)\n",
" auc_metric = compute_roc_auc(\n",
" y_pred, y, to_onehot_y=True, softmax=True)\n",
" y_onehot = to_onehot(y)\n",
" y_pred_act = act(y_pred)\n",
" auc_metric = compute_roc_auc(y_pred_act, y_onehot)\n",
" del y_pred_act, y_onehot\n",
" metric_values.append(auc_metric)\n",
" acc_value = torch.eq(y_pred.argmax(dim=1), y)\n",
" acc_metric = acc_value.sum().item() / len(acc_value)\n",
Expand Down
9 changes: 6 additions & 3 deletions 3d_classification/ignite/densenet_training_dict.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

import monai
from monai.handlers import ROCAUC, StatsHandler, TensorBoardStatsHandler, stopping_fn_from_metric
from monai.transforms import AddChanneld, Compose, LoadImaged, RandRotate90d, Resized, ScaleIntensityd, ToTensord
from monai.transforms import Activations, AddChanneld, AsDiscrete, Compose, LoadImaged, RandRotate90d, Resized, ScaleIntensityd, ToTensord


def main():
Expand Down Expand Up @@ -121,10 +121,13 @@ def prepare_batch(batch, device=None, non_blocking=False):

metric_name = "Accuracy"
# add evaluation metric to the evaluator engine
val_metrics = {metric_name: Accuracy(), "AUC": ROCAUC(to_onehot_y=True, softmax=True)}
val_metrics = {metric_name: Accuracy(), "AUC": ROCAUC()}

post_label = AsDiscrete(to_onehot=True, n_classes=2)
post_pred = Activations(softmax=True)
# Ignite evaluator expects batch=(img, label) and returns output=(y_pred, y) at every iteration,
# user can add output_transform to return other values
evaluator = create_supervised_evaluator(net, val_metrics, device, True, prepare_batch=prepare_batch)
evaluator = create_supervised_evaluator(net, val_metrics, device, True, prepare_batch=prepare_batch, output_transform=lambda x, y, y_pred: (post_pred(y_pred), post_label(y))

# add stats event handler to print validation stats via evaluator
val_stats_handler = StatsHandler(
Expand Down
9 changes: 7 additions & 2 deletions 3d_classification/torch/densenet_training_dict.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import monai
from monai.metrics import compute_roc_auc
from monai.transforms import AddChanneld, Compose, LoadImaged, RandRotate90d, Resized, ScaleIntensityd, ToTensord
from monai.transforms import Activations, AddChanneld, AsDiscrete, Compose, LoadImaged, RandRotate90d, Resized, ScaleIntensityd, ToTensord


def main():
Expand Down Expand Up @@ -76,6 +76,8 @@ def main():
ToTensord(keys=["img"]),
]
)
act = Activations(softmax=True)
to_onehot = AsDiscrete(to_onehot=True, n_classes=2)

# Define dataset, data loader
check_ds = monai.data.Dataset(data=train_files, transform=train_transforms)
Expand Down Expand Up @@ -135,7 +137,10 @@ def main():

acc_value = torch.eq(y_pred.argmax(dim=1), y)
acc_metric = acc_value.sum().item() / len(acc_value)
auc_metric = compute_roc_auc(y_pred, y, to_onehot_y=True, softmax=True)
y_onehot = to_onehot(y)
y_pred_act = act(y_pred)
auc_metric = compute_roc_auc(y_pred_act, y_onehot)
del y_pred_act, y_onehot
if acc_metric > best_metric:
best_metric = acc_metric
best_metric_epoch = epoch + 1
Expand Down
34 changes: 20 additions & 14 deletions modules/interpretability/covid_classification.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,23 +37,23 @@
"name": "stdout",
"output_type": "stream",
"text": [
"MONAI version: 0.4.0+62.g8fef0e1\n",
"Numpy version: 1.19.5\n",
"Pytorch version: 1.7.1\n",
"MONAI flags: HAS_EXT = False, USE_COMPILED = False\n",
"MONAI rev id: 8fef0e12b4b79aa124b9ba85f8dd8f452baad1cf\n",
"MONAI version: 0+untagged.1.ge8e2f3e.dirty\n",
"Numpy version: 1.19.2\n",
"Pytorch version: 1.8.0a0+52ea372\n",
"MONAI flags: HAS_EXT = True, USE_COMPILED = False\n",
"MONAI rev id: e8e2f3e2efb6bf8072b7ded40cb57bca01e07773\n",
"\n",
"Optional dependencies:\n",
"Pytorch Ignite version: 0.4.2\n",
"Pytorch Ignite version: 0.4.4\n",
"Nibabel version: 3.2.1\n",
"scikit-image version: 0.18.1\n",
"Pillow version: 7.0.0\n",
"Tensorboard version: 2.2.0\n",
"scikit-image version: 0.15.0\n",
"Pillow version: 8.1.2\n",
"Tensorboard version: 1.15.0+nv\n",
"gdown version: 3.12.2\n",
"TorchVision version: 0.8.2\n",
"TorchVision version: 0.9.0a0\n",
"ITK version: 5.1.2\n",
"tqdm version: 4.51.0\n",
"lmdb version: 1.0.0\n",
"tqdm version: 4.53.0\n",
"lmdb version: 1.1.1\n",
"psutil version: 5.8.0\n",
"\n",
"For details about installing the optional dependencies, please visit:\n",
Expand Down Expand Up @@ -81,6 +81,7 @@
"from monai.networks.utils import eval_mode\n",
"from monai.networks.nets import DenseNet121\n",
"from monai.transforms import (\n",
" Activations, AsDiscrete,\n",
" Compose, LoadImage, Lambda, AddChannel,\n",
" ScaleIntensity, ToTensor, RandRotate,\n",
" RandFlip, Rand2DElastic, RandZoom, Resize,\n",
Expand Down Expand Up @@ -238,6 +239,9 @@
" ToTensor(),\n",
"])\n",
"\n",
"act = Activations(softmax=True)\n",
"to_onehot = AsDiscrete(to_onehot=True, n_classes=num_class)\n",
"\n",
"\n",
"all_files = glob(os.path.join(train_dir, \"*.png\"))\n",
"random.shuffle(all_files)\n",
Expand Down Expand Up @@ -1344,8 +1348,10 @@
" y_pred = torch.cat([y_pred, outputs], dim=0)\n",
" y = torch.cat([y, val_labels], dim=0)\n",
"\n",
" auc_metric = monai.metrics.compute_roc_auc(\n",
" y_pred, y, to_onehot_y=True, softmax=True)\n",
" y_onehot = to_onehot(y)\n",
" y_pred_act = act(y_pred)\n",
" auc_metric = monai.metrics.compute_roc_auc(y_pred_act, y_onehot)\n",
" del y_pred_act, y_onehot\n",
" auc.append(auc_metric)\n",
"\n",
" acc_value = torch.eq(y_pred.argmax(dim=1), y)\n",
Expand Down
38 changes: 22 additions & 16 deletions modules/learning_rate.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,23 +50,23 @@
"name": "stdout",
"output_type": "stream",
"text": [
"MONAI version: 0.4.0+66.g028a965.dirty\n",
"Numpy version: 1.19.5\n",
"Pytorch version: 1.7.1\n",
"MONAI flags: HAS_EXT = False, USE_COMPILED = False\n",
"MONAI rev id: 028a965b208e2ef4265d51ec9c7ec8bd43b83359\n",
"MONAI version: 0+untagged.1.ge8e2f3e.dirty\n",
"Numpy version: 1.19.2\n",
"Pytorch version: 1.8.0a0+52ea372\n",
"MONAI flags: HAS_EXT = True, USE_COMPILED = False\n",
"MONAI rev id: e8e2f3e2efb6bf8072b7ded40cb57bca01e07773\n",
"\n",
"Optional dependencies:\n",
"Pytorch Ignite version: 0.4.2\n",
"Pytorch Ignite version: 0.4.4\n",
"Nibabel version: 3.2.1\n",
"scikit-image version: 0.18.1\n",
"Pillow version: 8.1.0\n",
"Tensorboard version: 2.4.0\n",
"scikit-image version: 0.15.0\n",
"Pillow version: 8.1.2\n",
"Tensorboard version: 1.15.0+nv\n",
"gdown version: 3.12.2\n",
"TorchVision version: 0.8.2\n",
"TorchVision version: 0.9.0a0\n",
"ITK version: 5.1.2\n",
"tqdm version: 4.51.0\n",
"lmdb version: 1.0.0\n",
"tqdm version: 4.53.0\n",
"lmdb version: 1.1.1\n",
"psutil version: 5.8.0\n",
"\n",
"For details about installing the optional dependencies, please visit:\n",
Expand Down Expand Up @@ -101,6 +101,8 @@
"from monai.networks.utils import eval_mode\n",
"from monai.optimizers import LearningRateFinder\n",
"from monai.transforms import (\n",
" Activations,\n",
" AsDiscrete,\n",
" AddChanneld,\n",
" CenterSpatialCropd,\n",
" Compose,\n",
Expand Down Expand Up @@ -252,7 +254,10 @@
"print(len(train_ds))\n",
"print(len(val_ds))\n",
"print(train_ds[0][\"image\"].shape)\n",
"num_classes = train_ds.get_num_classes()"
"num_classes = train_ds.get_num_classes()\n",
"\n",
"act = Activations(softmax=True)\n",
"to_onehot = AsDiscrete(to_onehot=True, n_classes=num_classes)"
]
},
{
Expand Down Expand Up @@ -499,9 +504,10 @@
" y = torch.cat([y, val_labels], dim=0)\n",
"\n",
" for d in data:\n",
" auc_metric = compute_roc_auc(\n",
" data[d][\"y_pred\"], y, to_onehot_y=True, softmax=True\n",
" )\n",
" y_onehot = to_onehot(y)\n",
" y_pred_act = act(data[d][\"y_pred\"])\n",
" auc_metric = compute_roc_auc(y_pred_act, y_onehot)\n",
" del y_pred_act, y_onehot\n",
" data[d][\"auc\"][\"x\"].append(epoch + 1)\n",
" data[d][\"auc\"][\"y\"].append(auc_metric)\n",
"\n",
Expand Down