PCa16TileTwoStageClassifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import cv2\n",
    "import time\n",
    "import copy\n",
    "import pickle  # Log dictionary data\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sn\n",
    "import sklearn.metrics as metrics\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F # stateless functions\n",
    "import torchvision.transforms as T\n",
    "import torchvision.models as models\n",
    "\n",
    "import multiprocessing\n",
    "# We must import this explicitly, it is not imported by the top-level\n",
    "# multiprocessing module.\n",
    "import multiprocessing.pool\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import cohen_kappa_score,confusion_matrix\n",
    "from tqdm.auto import tqdm\n",
    "from torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\n",
    "from datetime import datetime\n",
    "from multiprocessing import Manager\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "import glob\n",
    "import cv2\n",
    "import time\n",
    "import copy\n",
    "import pickle  # Log dictionary data\n",
    "import skimage.io\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sn\n",
    "import sklearn.metrics as metrics\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F # stateless functions\n",
    "import torchvision.transforms as T\n",
    "import torchvision.models as models\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from tqdm.notebook import tqdm\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.metrics import cohen_kappa_score,confusion_matrix\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Configuration in Common\n",
    "\"\"\"\n",
    "class CFG:\n",
    "    debug = False\n",
    "    device = torch.device('cuda')\n",
    "    dtype = torch.float32\n",
    "    nfolds = 4\n",
    "    seed = 524\n",
    "    TRAIN = '../yi_data/panda-16x128x128-tiles-data/train/'\n",
    "    LABELS = '../data/train.csv'\n",
    "\n",
    "\"\"\"Configuration for Stage One\n",
    "\"\"\"\n",
    "class CFG1:\n",
    "    batch_size = 16    \n",
    "    epochs = 16\n",
    "    lr = 1e-4\n",
    "    model_name = 'resnet_bc'\n",
    "    num_classes = 1\n",
    "    nworkers = 2\n",
    "    threshold = .32\n",
    "    \n",
    "\"\"\"Configuration for Stage Two\n",
    "\"\"\"\n",
    "class CFG2:\n",
    "    batch_size = 16\n",
    "    epochs = 16\n",
    "    lr = 1e-4\n",
    "    model_name = 'resnet'\n",
    "    num_classes = 3\n",
    "    nworkers = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Datasets and Dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>data_provider</th>\n",
       "      <th>isup_grade</th>\n",
       "      <th>gleason_score</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0005f7aaab2800f6170c399693a96917</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000920ad0b612851f8e01bcc880d9b3d</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0018ae58b01bdadc8e347995b69f99aa</td>\n",
       "      <td>radboud</td>\n",
       "      <td>4</td>\n",
       "      <td>4+4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001c62abd11fa4b57bf7a6c603a11bb9</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>4</td>\n",
       "      <td>4+4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001d865e65ef5d2579c190a0e0350d8f</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           image_id data_provider  isup_grade gleason_score  \\\n",
       "0  0005f7aaab2800f6170c399693a96917    karolinska           0           0+0   \n",
       "1  000920ad0b612851f8e01bcc880d9b3d    karolinska           0           0+0   \n",
       "2  0018ae58b01bdadc8e347995b69f99aa       radboud           4           4+4   \n",
       "3  001c62abd11fa4b57bf7a6c603a11bb9    karolinska           4           4+4   \n",
       "4  001d865e65ef5d2579c190a0e0350d8f    karolinska           0           0+0   \n",
       "\n",
       "   split  \n",
       "0      2  \n",
       "1      3  \n",
       "2      1  \n",
       "3      1  \n",
       "4      0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(CFG.LABELS).set_index('image_id')\n",
    "files = sorted(set([p[:32] for p in os.listdir(CFG.TRAIN)]))\n",
    "train = train.loc[files].reset_index()\n",
    "\n",
    "if CFG.debug:\n",
    "    df = train.sample(n=50, random_state=CFG.seed).copy().reset_index(drop=True)\n",
    "else:\n",
    "    df = train.copy()\n",
    "\n",
    "# Generate train/validation sets containing the same distribution of isup_grade\n",
    "splits = StratifiedKFold(n_splits=CFG.nfolds, random_state=CFG.seed, shuffle=True)\n",
    "splits = list(splits.split(df,df.isup_grade))\n",
    "# Assign split index to training samples\n",
    "folds_splits = np.zeros(len(df)).astype(np.int)\n",
    "for i in range(CFG.nfolds):\n",
    "    folds_splits[splits[i][1]] = i\n",
    "df['split'] = folds_splits\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/yasufuminakama/panda-se-resnext50-regression-baseline\n",
    "class TrainDataset(Dataset):\n",
    "    \"\"\"Prostate Cancer Biopsy Dataset\"\"\"\n",
    "    \n",
    "    def __init__(self, df, labels, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            csv_file (string): Path to the csv file\n",
    "            root_dir (string): Path to the directory with all images\n",
    "            transform (callable, optional): Optional transform to be applied on an image sample\n",
    "        \"\"\"\n",
    "        # Shuffle dataframes with fixed seed; otherwise, validation set only get cancerous samples\n",
    "        self.df = df\n",
    "        self.labels = labels\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        # https://stackoverflow.com/questions/33369832/read-multiple-images-on-a-folder-in-opencv-python\n",
    "        tile_fns = [fn for fn in glob.glob(f\"{CFG.TRAIN}/{self.df['image_id'][idx]}_*.png\")]\n",
    "        # As we use cv2, the color channel is BGR. https://stackoverflow.com/questions/50963283/python-opencv-imshow-doesnt-need-convert-from-bgr-to-rgb\n",
    "        img_tiles = [Image.fromarray(cv2.imread(fn)) for fn in tile_fns]\n",
    "        if self.transform:\n",
    "            img_tiles = list(map(self.transform, img_tiles))\n",
    "            \n",
    "        label = self.labels[idx]\n",
    "\n",
    "        return img_tiles, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_transforms(phase):\n",
    "    assert phase in {'train', 'val'}\n",
    "    \n",
    "    if phase == 'train':\n",
    "        return T.Compose([\n",
    "            T.RandomHorizontalFlip(),\n",
    "            T.RandomVerticalFlip(),\n",
    "            T.RandomRotation(15, fill=255),\n",
    "            T.ToTensor(),\n",
    "            T.Normalize(\n",
    "                mean=[0.8776, 0.8186, 0.9090],\n",
    "                std=[0.1659, 0.2507, 0.1357],\n",
    "            ),\n",
    "        ])\n",
    "    else:\n",
    "        return T.Compose([\n",
    "            T.ToTensor(),\n",
    "            T.Normalize(\n",
    "                mean=[0.8776, 0.8186, 0.9090],\n",
    "                std=[0.1659, 0.2507, 0.1357],\n",
    "            ),\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ntransform = get_transforms(phase='train')\\ntrain_dataset = TrainDataset(df.reset_index(drop=True),\\n                             df.reset_index(drop=True)['isup_grade'],\\n                             transform = get_transforms(phase='train'))\\nimg, label = train_dataset[0]\\n#tiles = map(transform, tiles)\\n#print(list(tiles))\\n#print(label)\\n#print(tiles.shape)\\n#print(type(tiles[0]))\\nplt.imshow(img.permute(1,2,0))\\n#plt.show()\\n#plt.imshow(tiles[0].permute(1,2,0))\\n\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "transform = get_transforms(phase='train')\n",
    "train_dataset = TrainDataset(df.reset_index(drop=True),\n",
    "                             df.reset_index(drop=True)['isup_grade'],\n",
    "                             transform = get_transforms(phase='train'))\n",
    "img, label = train_dataset[0]\n",
    "#tiles = map(transform, tiles)\n",
    "#print(list(tiles))\n",
    "#print(label)\n",
    "#print(tiles.shape)\n",
    "#print(type(tiles[0]))\n",
    "plt.imshow(img.permute(1,2,0))\n",
    "#plt.show()\n",
    "#plt.imshow(tiles[0].permute(1,2,0))\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use fold idx as validation set\n",
    "def data_loader_stage1(fold_idx):\n",
    "    train_idx = df[df['split'] != fold_idx].index\n",
    "    val_idx = df[df['split'] == fold_idx].index\n",
    "\n",
    "    train_dataset = TrainDataset(df.loc[train_idx].reset_index(drop=True),\n",
    "                                 df.loc[train_idx].reset_index(drop=True)['isup_grade'],\n",
    "                                 transform = get_transforms(phase='train'))\n",
    "    val_dataset = TrainDataset(df.loc[val_idx].reset_index(drop=True),\n",
    "                               df.loc[val_idx].reset_index(drop=True)['isup_grade'],\n",
    "                               transform = get_transforms(phase='val'))\n",
    "    \n",
    "    train_loader = DataLoader(train_dataset, batch_size=CFG1.batch_size, shuffle=True, num_workers=CFG1.nworkers)\n",
    "    val_loader = DataLoader(val_dataset, batch_size=CFG1.batch_size, shuffle=False, num_workers=CFG1.nworkers)\n",
    "    return train_loader, val_loader\n",
    "\n",
    "def data_loader_stage2(fold_idx):\n",
    "    train_idx = df[(df['split'] != fold_idx) & (df['isup_grade'] > 2)].index\n",
    "    val_idx = df[(df['split'] == fold_idx) & (df['isup_grade'] > 2)].index\n",
    "\n",
    "    train_dataset = TrainDataset(df.loc[train_idx].reset_index(drop=True),\n",
    "                                 df.loc[train_idx].reset_index(drop=True)['isup_grade']-3,\n",
    "                                 transform = get_transforms(phase='train'))\n",
    "    val_dataset = TrainDataset(df.loc[val_idx].reset_index(drop=True),\n",
    "                               df.loc[val_idx].reset_index(drop=True)['isup_grade']-3,\n",
    "                               transform = get_transforms(phase='val'))\n",
    "    \n",
    "    train_loader = DataLoader(train_dataset, batch_size=CFG2.batch_size, shuffle=True, num_workers=CFG2.nworkers)\n",
    "    val_loader = DataLoader(val_dataset, batch_size=CFG2.batch_size, shuffle=False, num_workers=CFG2.nworkers)\n",
    "    return train_loader, val_loader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(stage, model, fold, dataloaders, criterion, optimizer, scheduler, num_epochs=25):\n",
    "    since = time.time()\n",
    "    \n",
    "    # Send the model to GPU/CPU\n",
    "    model = model.to(device=CFG.device)\n",
    "    \n",
    "    train_acc_history = []\n",
    "    val_acc_history = []\n",
    "    loss_history = []\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        \n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            preds, targets = [], []\n",
    "\n",
    "            if phase == 'train':\n",
    "                model.train()   # Set model to training phase\n",
    "            else:\n",
    "                model.eval()    # Set model to evaluate phase\n",
    "            \n",
    "            avg_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            \n",
    "            print(' ', end='', flush=True)  # To workaround tqdm issue in multiprocess\n",
    "            for inputs, labels in tqdm(dataloaders[phase],\n",
    "                                       desc='[{}-{}] {}/{}({:5s})'.format(stage, fold, epoch+1,num_epochs,phase)):\n",
    "                for tile_i in range(16):\n",
    "                    inputs[tile_i] = inputs[tile_i].to(device=CFG.device, dtype=CFG.dtype)\n",
    "                inputs = torch.cat([torch.cat([inputs[0],inputs[1],inputs[2],inputs[3]], 2).to(device=CFG.device, dtype=CFG.dtype),\n",
    "                                    torch.cat([inputs[4],inputs[5],inputs[6],inputs[7]], 2).to(device=CFG.device, dtype=CFG.dtype),\n",
    "                                    torch.cat([inputs[8],inputs[9],inputs[10],inputs[11]], 2).to(device=CFG.device, dtype=CFG.dtype),\n",
    "                                    torch.cat([inputs[12],inputs[13],inputs[14],inputs[15]], 2).to(device=CFG.device, dtype=CFG.dtype)],\n",
    "                                   3).to(device=CFG.device, dtype=CFG.dtype)\n",
    "                if stage == 1:\n",
    "                    labels = torch.where(labels<3, torch.tensor(0), torch.tensor(1)).to(device=CFG.device, dtype=CFG.dtype)\n",
    "                elif stage == 2:\n",
    "                    labels = labels.to(device=CFG.device, dtype=torch.long)\n",
    "                else:\n",
    "                    assert False\n",
    "                \n",
    "                # Zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                # Forward, track history if only in training\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    outputs = model(inputs).squeeze()\n",
    "                    #print(outputs.shape, labels.shape)\n",
    "                    loss = criterion(outputs, labels)\n",
    "                    #print(outputs)\n",
    "                    if stage == 1:\n",
    "                        pred = torch.where(outputs<CFG1.threshold,\n",
    "                                           torch.tensor(0).to(device=CFG.device),\n",
    "                                           torch.tensor(1).to(device=CFG.device)).to(device=CFG.device, dtype=torch.long)\n",
    "                        labels = labels.to(dtype=torch.long)\n",
    "                    #print(pred)\n",
    "                    #print(labels)\n",
    "                    elif stage == 2:\n",
    "                        pred = torch.argmax(outputs, 1)\n",
    "                        print(type(pred))\n",
    "                        #print(pred.shape)\n",
    "                    else:\n",
    "                        assert False\n",
    "\n",
    "                    # Backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                    \n",
    "                # Statistics\n",
    "                avg_loss += loss.item()*(inputs.size(0)/len(dataloaders[phase].dataset))  # len(dataloaders[phase].dataset) not len(dataloaders[phase])\n",
    "                running_corrects += torch.sum(pred == labels)\n",
    "                #print('correts: ', torch.sum(pred == labels))\n",
    "                preds.append(pred)\n",
    "                targets.append(labels)\n",
    "            \n",
    "            # End of epoch\n",
    "            with torch.no_grad():\n",
    "                epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n",
    "\n",
    "                if phase == 'val':\n",
    "                    val_acc_history.append(epoch_acc)\n",
    "                    # deep copy the model\n",
    "                    if epoch_acc > best_acc:\n",
    "                        best_acc = epoch_acc\n",
    "                        best_model_wts = copy.deepcopy(model.state_dict())\n",
    "                    # Apply lr_scheduler\n",
    "                    if scheduler is not None:\n",
    "                        scheduler.step(avg_loss)\n",
    "                else:\n",
    "                    train_acc_history.append(epoch_acc)\n",
    "                    loss_history.append(avg_loss)\n",
    "                print('[{}-{}] {} Loss: {:4f} Acc: {:4f}'.format(stage, fold, phase, avg_loss, epoch_acc))\n",
    "    \n",
    "    time_elapsed = time.time() - since\n",
    "    print('[{}-{}] Training complete in {:.0f}m {:0f}s'.format(stage, fold, time_elapsed//60, time_elapsed%60))\n",
    "    print('[{}-{}] Best val Acc: {:4f}'.format(stage, fold, best_acc))\n",
    "    print()\n",
    "    \n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model, loss_history, train_acc_history, val_acc_history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Two-Stage Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "def set_parameter_requires_grad(model, feature_extracting):\n",
    "    if feature_extracting:\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False\n",
    "            \n",
    "\n",
    "def initialize_model(model_name, num_classes, feature_extract=False, use_pretrained=False):\n",
    "    \"\"\"\n",
    "    Params:\n",
    "        feature_extract\n",
    "            True - fine tunning\n",
    "            False - fix the model\n",
    "    \"\"\"\n",
    "    model_ft = None\n",
    "    \n",
    "    if model_name == 'alexnet':\n",
    "        \"\"\"AlexNet\n",
    "        \"\"\"\n",
    "        model_ft = models.alexnet(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.classifier[6].in_features\n",
    "        model_ft.classifier[6] = nn.Linear(num_ftrs, num_classes)\n",
    "    elif model_name == 'resnet':\n",
    "        \"\"\"Resnet\n",
    "        \"\"\"\n",
    "        model_ft = models.resnet18(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(model_ft, feature_extract)\n",
    "        num_ftrs = model_ft.fc.in_features\n",
    "        model_ft.fc = nn.Linear(num_ftrs, num_classes)\n",
    "    elif model_name == 'resnet_bc':\n",
    "        \"\"\"Resnet\n",
    "        \"\"\"\n",
    "        res_model = models.resnet18(pretrained=use_pretrained)\n",
    "        set_parameter_requires_grad(res_model, feature_extract)\n",
    "        num_ftrs = res_model.fc.in_features\n",
    "        res_model.fc = nn.Linear(num_ftrs, num_classes)\n",
    "        model_ft = nn.Sequential(\n",
    "                    res_model,\n",
    "                    nn.Sigmoid(),  # Squeeze output within [0,1]\n",
    "        )\n",
    "    return model_ft\n",
    "\n",
    "\n",
    "def train_stage1_layer(fold):\n",
    "    model_ft = initialize_model(CFG1.model_name, CFG1.num_classes, use_pretrained=False)\n",
    "\n",
    "    optimizer = optim.Adam(model_ft.parameters(),\n",
    "                           lr=CFG1.lr)\n",
    "    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=2, verbose=True, eps=1e-06)\n",
    "    loader_train, loader_val = data_loader_stage1(fold)\n",
    "    return train_model(1, model_ft, fold,\n",
    "                       {'train': loader_train, 'val': loader_val},\n",
    "                       nn.BCELoss(), optimizer, scheduler, CFG1.epochs)\n",
    "\n",
    "def train_stage2_layer(fold):\n",
    "    model_ft = initialize_model(CFG2.model_name, CFG2.num_classes, use_pretrained=False)\n",
    "    optimizer = optim.Adam(model_ft.parameters(),\n",
    "                           lr=CFG2.lr)\n",
    "    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=2, verbose=True, eps=1e-06)\n",
    "    loader_train, loader_val = data_loader_stage2(fold)\n",
    "    return train_model(2, model_ft, fold,\n",
    "                      {'train': loader_train, 'val': loader_val},\n",
    "                      F.cross_entropy, optimizer, scheduler, CFG2.epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NoDaemonProcess(multiprocessing.Process):\n",
    "    # make 'daemon' attribute always return False\n",
    "    def _get_daemon(self):\n",
    "        return False\n",
    "    def _set_daemon(self, value):\n",
    "        pass\n",
    "    daemon = property(_get_daemon, _set_daemon)\n",
    "\n",
    "# We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool\n",
    "# because the latter is only a wrapper function, not a proper class.\n",
    "class MyPool(multiprocessing.pool.Pool):\n",
    "    Process = NoDaemonProcess\n",
    "\n",
    "def progressor(fold):\n",
    "    #print(f'stage{stage} fold{fold}')\n",
    "    best_model_1, loss_history_1, train_acc_history_1, val_acc_history_1 = train_stage1_layer(fold)\n",
    "    best_model_2, loss_history_2, train_acc_history_2, val_acc_history_2 = train_stage2_layer(fold)\n",
    "    return {f'stage1_model_{fold}': best_model_1.to('cpu'),  # Don't save model as cuda\n",
    "            f'stage1_loss_history_{fold}': loss_history_1,\n",
    "            f'stage1_train_acc_history_{fold}': train_acc_history_1,\n",
    "            f'stage1_val_acc_history_{fold}': val_acc_history_1,\n",
    "            f'stage2_model_{fold}': best_model_2.to('cpu'),  # Don't save model as cuda\n",
    "            f'stage2_loss_history_{fold}': loss_history_2,\n",
    "            f'stage2_train_acc_history_{fold}': train_acc_history_2,\n",
    "            f'stage2_val_acc_history_{fold}': val_acc_history_2}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "695e88bfbf464664bd7b839b8aabad0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 1/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd8ed8b1aa48403698cabd75a676c241",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 1/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f457f35d6ac6487984a294c643b1dac3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 1/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d86d3044aeb4dabaf2e26730c87f8b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 1/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.543454 Acc: 0.678458\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf5922b31d6c4b76928ad36844dd5232",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 1/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] train Loss: 0.548356 Acc: 0.688601\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "faa66b4f99a04530bafb278aba35b3b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 1/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.554381 Acc: 0.678331\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e9bc731fefb474285725b697e69b858",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 1/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.546584 Acc: 0.679346\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52b1311d6ab949c28aeaeb795bd5b536",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 1/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.650193 Acc: 0.703690\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2225eb7a69c2462aa969f58421997510",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 2/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.645951 Acc: 0.737923\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f45ce1908094c85946f461d2cee814a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 2/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 1.590324 Acc: 0.354507\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5d5778c178542d4bc123d3f41aea23e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 2/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.676281 Acc: 0.727273\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ffaebbb5398464ea084614c6c377183",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 2/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.486123 Acc: 0.734753\n",
      " \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5197cbf7bcb54f62b02da59e1b2593f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 2/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1-1] train Loss: 0.495147 Acc: 0.729301\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bea0cfd2cd1c47fab3765a98cee1065a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 2/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.490059 Acc: 0.728287\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3679f15b7ccd44f69450c5e7505a8afb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 2/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.502852 Acc: 0.730316\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a208e9de79834c2cb9e66e9edeccb091",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 2/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.778021 Acc: 0.603652\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f02f88dc1da14e1ba45424c07b02cf8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 3/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.682806 Acc: 0.696463\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a138b80f96d9463f9a42f34c69c90a6e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 3/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.485029 Acc: 0.761506\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31e4e49f514e4027816a4bcc50c8612e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 3/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 0.460301 Acc: 0.783948\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d5c3908cbaf045f3ad29c03a24c1ae44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 3/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.469993 Acc: 0.748447\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46cdbb0b04794ebbbb002ec836bb3b88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 3/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] train Loss: 0.469042 Acc: 0.749208\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e2d16f6f046481fbd2e37ec2c4416ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 3/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.473790 Acc: 0.741347\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb788f506b0441799f5ce35c9ef5146a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 3/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.465822 Acc: 0.754913\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9070c673e23848839cb7d04223397d4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 3/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.476795 Acc: 0.769494\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99c412bb437b47d683c32a0e478ec525",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 4/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.665413 Acc: 0.748193\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d019bcdd12714fbca2ac2a646feb6748",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 4/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 1.004331 Acc: 0.429060\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0aa92f8588014f94903343044b854570",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 4/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.521960 Acc: 0.773678\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e845e0663e324e81af2b4ab4f72ebd4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 4/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.462479 Acc: 0.752377\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa07da265b414a0ba54827dd7a584658",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 4/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] train Loss: 0.454251 Acc: 0.763662\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3afa96bea114b87a7a76e5a665a2cac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 4/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.468606 Acc: 0.745911\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "080a833b78d74d0bbdeb3958ed0a044b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 4/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.461631 Acc: 0.754660\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6da84a9cc54d4251a449992d7ef5fb37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 4/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.461755 Acc: 0.736402\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2431a280b45d45139675e18da2d1af31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 5/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch     4: reducing learning rate of group 0 to 1.0000e-05.\n",
      "[1-1] val Loss: 0.669679 Acc: 0.696843\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7099ccaa86f4a12819808c5f3109369",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 5/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 0.740657 Acc: 0.585394\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7ae41a4653c43faa00fc0b897d9ad54",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 5/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.468579 Acc: 0.734500\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ead4b7d0113b4016bfa55f2049bdf98a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 5/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.447828 Acc: 0.768607\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6990599bb6724929818a58910eac5f8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 5/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.459633 Acc: 0.763535\n",
      "\n",
      "[1-1] train Loss: 0.428622 Acc: 0.784709\n",
      "  "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8067f900aff44a98980ece8fa05f182d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 5/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9125d770ab4046fea3ae5e41cd8a871b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 5/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.454246 Acc: 0.759604\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f888f1dbb88d4cb59a0b683a1db0642a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 5/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.427856 Acc: 0.788132\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97771a7fc645446b93b1ed39110364ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 6/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.424286 Acc: 0.760365\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6a644422e904e5e8c15507e9b185da6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 6/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch     5: reducing learning rate of group 0 to 1.0000e-05.\n",
      "[1-2] val Loss: 0.633162 Acc: 0.560669\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "057aa22df49b4d299913dbeb5147e1af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 6/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.446447 Acc: 0.775960\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16e53dbf60b44ee99c0229d4ec7afd8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 6/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.446188 Acc: 0.765183\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff7295bc8b374d4a81505cc1e9afe61d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 6/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] train Loss: 0.424133 Acc: 0.783188\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9faba7619f945f484e9db8c47cff89f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 6/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.433180 Acc: 0.765183\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05b7be376b064323b2b55c1f6ef5f9de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 6/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.443315 Acc: 0.769874\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cf0bbba121b4c2f88f0e74efeba35ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 6/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.472500 Acc: 0.783188\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4067461e7cc649ad90e8f2ff9bad98b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 7/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.416565 Acc: 0.801065\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "901c47f471bf4f2e9a16cb4f19aa4280",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 7/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 0.415403 Acc: 0.763789\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3292ffd84324b59a0e596cda3e2a7c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 7/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.516315 Acc: 0.793838\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "609c1eba18af474aa0721030f066f300",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 7/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.443316 Acc: 0.774693\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c435795d1724bfca502a63812fb57eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 7/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.423890 Acc: 0.779257\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c442cf9160664a2e8b5327427c846568",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 7/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] train Loss: 0.418954 Acc: 0.789527\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6adda54f8987472a860403dcb4eaeb17",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 7/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.442222 Acc: 0.768226\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e13dda3f053c43ac83a8214f62f96ca5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 7/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.419949 Acc: 0.749715\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15a3a5ea7ed64c3495c5faa8a7191c63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 8/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 0.410773 Acc: 0.771776\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9168bbc7bf094d83abbff8cf350bb0c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 8/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.412950 Acc: 0.785850\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bdf26ad9bd294f999262d4518b979c74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 8/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.433577 Acc: 0.789654\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d0d74bf0ada4175991e3dc6f540526b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 8/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.440586 Acc: 0.769367\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8eb518705d3b4240bc6f0eb48bc1540e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 8/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.425242 Acc: 0.777736\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f2ac10f226b4772ba92b331eac2f326",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 8/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.435904 Acc: 0.775834\n",
      " \n",
      "[1-1] train Loss: 0.415560 Acc: 0.786484\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91dc5a7815854020ac3e9956962abd97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 8/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24f02e39a4c247fd8060b16592764e3b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 8/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.447853 Acc: 0.790034\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e75de34cedfb4363a210d58c973497bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 9/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 0.404266 Acc: 0.790415\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf77cb6d53074e87ae79b8899f079e80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 9/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.449181 Acc: 0.720046\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb75e828de1b41f9964149832f3fb9c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 9/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.419397 Acc: 0.771776\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4883530298e547759487fb35ae1fbdce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 9/16(train)', max=493.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.429841 Acc: 0.784836\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d498b3a9b1864eb89faabe11726c330a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 9/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.423202 Acc: 0.779257\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bddd3244f154dd3bedc3ed0541abaaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 9/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.431869 Acc: 0.780525\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c441456e2ae240d4b1c008fb34853213",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 9/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] train Loss: 0.413285 Acc: 0.788766\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5d9845e90284fe3becf1cb8fb1446a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 9/16(val  )', max=165.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.453479 Acc: 0.810574\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c03ea1a8fe27434992b5ba256a24a473",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 10/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.446730 Acc: 0.713960\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6475ef440474dea8cc1372341a3186b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 10/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 0.397431 Acc: 0.793838\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee7b2343a219424a8d3ffd4f16f4c2e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 10/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.409860 Acc: 0.776341\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4ca7fbab04546ec9500742337e9180d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 10/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.431197 Acc: 0.781286\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ed67163974045bcb94b4dbec13cfe1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 10/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.429446 Acc: 0.780778\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79e5b1b2e99c48f2a177fdca1a65aa49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 10/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.419218 Acc: 0.781666\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d43fcb03f0fc4ae0bb23207dc0988665",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 10/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] train Loss: 0.412746 Acc: 0.784836\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1117ed86d9814aebb3d1018cd5f65600",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 10/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch    10: reducing learning rate of group 0 to 1.0000e-05.\n",
      "[1-0] val Loss: 0.557006 Acc: 0.779003\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3271be9979ee4ab6ac1fe68b387744b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 11/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch    10: reducing learning rate of group 0 to 1.0000e-05.\n",
      "[1-3] val Loss: 0.454433 Acc: 0.784329\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94da06abdd6d4ea78cf31f5a7956c852",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 11/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 0.401167 Acc: 0.790034\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a5e65aa258a4e438bcbb3e990a7499e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 11/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.410110 Acc: 0.798783\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4a4a994ec6e49bda6784591f73fa1b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 11/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.424790 Acc: 0.778623\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b052764067d443fad82dc16b359d201",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 11/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.407112 Acc: 0.795486\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d8ea3dd6716489ebeba02d6c8ad321f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 11/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.405394 Acc: 0.784075\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "937b660ef8ba4b919b0696b24f88d387",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 11/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] train Loss: 0.420014 Acc: 0.786611\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9e2c824b2624d1ab376c21d78d9005d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 11/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] val Loss: 0.400215 Acc: 0.798402\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae2e72011c4a4bb8911fd6ca82b2757f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 12/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.402088 Acc: 0.807151\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0efc23b54437467280ca94173f4ddb05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 12/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.406450 Acc: 0.784329\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "894e969dd62c41ad9d87f53cc511a0d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 12/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] val Loss: 0.412598 Acc: 0.777102\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "315b915e6e644147b70ebe384cbeec05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 12/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-2] train Loss: 0.410206 Acc: 0.787879\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c403bb0609c248fcb73058d4f35fa408",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 12/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] train Loss: 0.404355 Acc: 0.793838\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b140c31aa934493bbc5b2812aefdf29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 12/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] train Loss: 0.400168 Acc: 0.793838\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71215c0b01004c73b93a91bad798def2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 12/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-1] train Loss: 0.408507 Acc: 0.790795\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b67316a3fae84141a0984c98a11788be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 12/16(val  )', max=165.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch    12: reducing learning rate of group 0 to 1.0000e-06.\n",
      "[1-2] val Loss: 0.412797 Acc: 0.799924\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a3f660dfc714e2a8e0667b9a12781ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-2] 13/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-0] val Loss: 0.400130 Acc: 0.804108\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8311ca6e06d4dc28f18995dfa580ebb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-0] 13/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[1-3] val Loss: 0.407824 Acc: 0.807912\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f54ac06ff6534a1daf75cf232cdde178",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-3] 13/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch    12: reducing learning rate of group 0 to 1.0000e-06.\n",
      "[1-1] val Loss: 0.409929 Acc: 0.780525\n",
      " "
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00b1f26e760d40e3b0ed3d3868124512",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='[1-1] 13/16(train)', max=493.0, style=ProgressStyle(descr…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "log_dict = {'batch_size_1': CFG1.batch_size,\n",
    "            'batch_size_2': CFG2.batch_size,\n",
    "            'epochs_1': CFG1.epochs,\n",
    "            'epochs_2': CFG2.epochs,\n",
    "            'learning_rate_1': CFG1.lr,\n",
    "            'learning_rate_2': CFG2.lr,\n",
    "            'model_1': CFG1.model_name,\n",
    "            'model_2': CFG2.model_name,\n",
    "            'nworkers_1': CFG1.nworkers,\n",
    "            'nworkers_2': CFG2.nworkers,\n",
    "            'nfolds': CFG.nfolds,\n",
    "            'random_seed': CFG.seed}\n",
    "\n",
    "result_list = list(MyPool(CFG.nfolds).map(progressor, range(CFG.nfolds)))\n",
    "\n",
    "# Accumulate result from each process\n",
    "for result in result_list:\n",
    "    log_dict.update(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrics\n",
    "## Analyze Stage One"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets, scores = [], []\n",
    "for fold in range(CFG.nfolds):\n",
    "    model_fd = log_dict[f'stage1_model_{fold}'].to(device=CFG.device, dtype=CFG.dtype)\n",
    "    _, loader_val = data_loader_stage1(fold)\n",
    "    for inputs, labels in tqdm(loader_val):\n",
    "        inputs = inputs.to(device=CFG.device, dtype=CFG.dtype)\n",
    "        labels = torch.where(labels<3, torch.tensor(0), torch.tensor(1)).to(device=CFG.device, dtype=CFG.dtype)\n",
    "                \n",
    "        # Forward, track history if only in training\n",
    "        with torch.no_grad():\n",
    "            outputs = model_fd(inputs).squeeze()\n",
    "        targets.append(labels)\n",
    "        scores.append(outputs)\n",
    "\n",
    "t1 = torch.cat(targets).cpu()\n",
    "s1 = torch.cat(scores).cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate the fpr and tpr for all thresholds of the classification\n",
    "fpr, tpr, threshold = metrics.roc_curve(t1, s1.view(-1))\n",
    "roc_auc = metrics.auc(fpr, tpr)\n",
    "log_dict['fpr'] = fpr\n",
    "log_dict['tpr'] = tpr\n",
    "log_dict['threshold'] = threshold\n",
    "\n",
    "\n",
    "plt.title('Receiver Operating Characteristic')\n",
    "plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n",
    "plt.legend(loc = 'lower right')\n",
    "plt.plot([0, 1], [0, 1],'r--')\n",
    "plt.xlim([0, 1])\n",
    "plt.ylim([0, 1])\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tn, fp, fn, tp = conf_mat.ravel()\n",
    "print(f'sensitivity: {tp/(tp+fn)}')\n",
    "print(f'specificity: {tn/(tn+fp)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze Stage Two"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "masks, preds, targets = [], [], []\n",
    "for fold in range(CFG.nfolds):\n",
    "    model_fd1 = log_dict[f'stage1_model_{fold}'].to(device=CFG.device, dtype=CFG.dtype)\n",
    "    model_fd2 = log_dict[f'stage2_model_{fold}'].to(device=CFG.device, dtype=CFG.dtype)\n",
    "    _, loader_val = data_loader_stage2(fold)\n",
    "    for inputs, labels in tqdm(loader_val):\n",
    "        inputs = inputs.to(device=CFG.device, dtype=CFG.dtype)\n",
    "                \n",
    "        # Forward, track history if only in training\n",
    "        with torch.no_grad():\n",
    "            probs = model_fd1(inputs).squeeze()\n",
    "            outputs = model_fd2(inputs)\n",
    "            mask = torch.where(probs<CFG1.threshold,\n",
    "                               torch.tensor(0).to(device=CFG.device),\n",
    "                               torch.tensor(1).to(device=CFG.device))\n",
    "            pred = torch.argmax(outputs, 1)\n",
    "        masks.append(mask)\n",
    "        targets.append(labels)\n",
    "        preds.append(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p2 = torch.cat(preds).cpu()\n",
    "t2 = torch.cat(targets).cpu()\n",
    "m2 = torch.cat(masks).cpu()\n",
    "\n",
    "kappa = cohen_kappa_score(t2[m2==1], p2[m2==1], weights='quadratic')\n",
    "print(f'Kappa: {kappa}')\n",
    "conf_mat = confusion_matrix(t2[m2==1], p2[m2==1])\n",
    "#plt.matshow()\n",
    "plt.figure(figsize=(14,7))\n",
    "isup_labels = list(range(3,6))\n",
    "sn.heatmap(conf_mat, annot=True, xticklabels=isup_labels, yticklabels=isup_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Log results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_file = f'{CFG1.model_name}-{CFG2.model_name}_{datetime.now().strftime(\"%m_%d_%Y_%H_%M\")}.pkl'\n",
    "with open(log_file, 'wb') as pkl_file:\n",
    "    pickle.dump(log_dict, pkl_file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}