ProstateCancerPipeline2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import cv2\n",
    "import time\n",
    "import copy\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F # stateless functions\n",
    "import torchvision.transforms as T\n",
    "import torchvision.models as models\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from tqdm.auto import tqdm\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.metrics import cohen_kappa_score,confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CFG:\n",
    "    batch_size = 16\n",
    "    debug = False\n",
    "    device = torch.device('cuda')\n",
    "    dtype = torch.float32\n",
    "    epochs = 30\n",
    "    nfolds = 4\n",
    "    seed = 524\n",
    "    TRAIN = '../yi_data/panda-16x128x128-tiles-data/train/'\n",
    "    LABELS = '../data/train.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>data_provider</th>\n",
       "      <th>isup_grade</th>\n",
       "      <th>gleason_score</th>\n",
       "      <th>split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0005f7aaab2800f6170c399693a96917</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000920ad0b612851f8e01bcc880d9b3d</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0018ae58b01bdadc8e347995b69f99aa</td>\n",
       "      <td>radboud</td>\n",
       "      <td>4</td>\n",
       "      <td>4+4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001c62abd11fa4b57bf7a6c603a11bb9</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>4</td>\n",
       "      <td>4+4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001d865e65ef5d2579c190a0e0350d8f</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           image_id data_provider  isup_grade gleason_score  \\\n",
       "0  0005f7aaab2800f6170c399693a96917    karolinska           0           0+0   \n",
       "1  000920ad0b612851f8e01bcc880d9b3d    karolinska           0           0+0   \n",
       "2  0018ae58b01bdadc8e347995b69f99aa       radboud           4           4+4   \n",
       "3  001c62abd11fa4b57bf7a6c603a11bb9    karolinska           4           4+4   \n",
       "4  001d865e65ef5d2579c190a0e0350d8f    karolinska           0           0+0   \n",
       "\n",
       "   split  \n",
       "0      2  \n",
       "1      3  \n",
       "2      1  \n",
       "3      1  \n",
       "4      0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(CFG.LABELS).set_index('image_id')\n",
    "files = sorted(set([p[:32] for p in os.listdir(CFG.TRAIN)]))\n",
    "train = train.loc[files].reset_index()\n",
    "\n",
    "if CFG.debug:\n",
    "    df = train.sample(n=200, random_state=CFG.seed).copy()\n",
    "else:\n",
    "    df = train.copy()\n",
    "\n",
    "# Generate train/validation sets containing the same distribution of isup_grade\n",
    "splits = StratifiedKFold(n_splits=CFG.nfolds, random_state=CFG.seed, shuffle=True)\n",
    "splits = list(splits.split(df,df.isup_grade))\n",
    "# Assign split index to training samples\n",
    "folds_splits = np.zeros(len(df)).astype(np.int)\n",
    "for i in range(CFG.nfolds):\n",
    "    folds_splits[splits[i][1]] = i\n",
    "df['split'] = folds_splits\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/yasufuminakama/panda-se-resnext50-regression-baseline\n",
    "class TrainDataset(Dataset):\n",
    "    \"\"\"Prostate Cancer Biopsy Dataset\"\"\"\n",
    "    \n",
    "    def __init__(self, df, labels, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            csv_file (string): Path to the csv file\n",
    "            root_dir (string): Path to the directory with all images\n",
    "            transform (callable, optional): Optional transform to be applied on an image sample\n",
    "        \"\"\"\n",
    "        # Shuffle dataframes with fixed seed; otherwise, validation set only get cancerous samples\n",
    "        self.df = df\n",
    "        self.labels = labels\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        # https://stackoverflow.com/questions/33369832/read-multiple-images-on-a-folder-in-opencv-python\n",
    "        img_fns = [fn for fn in glob.glob(f\"{CFG.TRAIN}/{self.df['image_id'][idx]}_*.png\")]\n",
    "        imgs = [cv2.imread(fn) for fn in img_fns]\n",
    "        # (D,W,H)\n",
    "        img = cv2.hconcat([cv2.vconcat([imgs[0], imgs[1], imgs[2], imgs[3]]),\n",
    "                           cv2.vconcat([imgs[4], imgs[5], imgs[6], imgs[7]]),\n",
    "                           cv2.vconcat([imgs[8], imgs[9], imgs[10], imgs[11]]),\n",
    "                           cv2.vconcat([imgs[12], imgs[13], imgs[14], imgs[15]])])\n",
    "        \n",
    "        if self.transform:\n",
    "            img = self.transform(img)\n",
    "            \n",
    "        label = torch.tensor(self.labels[idx])\n",
    "        return img, label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_transforms(phase):\n",
    "    assert phase in {'train', 'val'}\n",
    "    \n",
    "    if phase == 'train':\n",
    "        return T.Compose([\n",
    "            T.ToTensor(),\n",
    "            T.Normalize(\n",
    "                mean=[0.485, 0.456, 0.406],\n",
    "                std=[0.229, 0.224, 0.225],\n",
    "                #mean=[0.8776, 0.8186, 0.9090],\n",
    "                #std=[0.1659, 0.2507, 0.1357],\n",
    "            ),\n",
    "            # TODO: Add normalize\n",
    "        ])\n",
    "    else:\n",
    "        return T.Compose([\n",
    "            T.ToTensor(),\n",
    "            T.Normalize(\n",
    "                mean=[0.485, 0.456, 0.406],\n",
    "                std=[0.229, 0.224, 0.225],\n",
    "                #mean=[0.8776, 0.8186, 0.9090],\n",
    "                #std=[0.1659, 0.2507, 0.1357],\n",
    "            ),\n",
    "        ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use fold idx as validation set\n",
    "def data_loader(fold_idx):\n",
    "    train_idx = df[df['split'] != fold_idx].index\n",
    "    val_idx = df[df['split'] == fold_idx].index\n",
    "\n",
    "    train_dataset = TrainDataset(df.loc[train_idx].reset_index(drop=True),\n",
    "                                 df.loc[train_idx].reset_index(drop=True)['isup_grade'],\n",
    "                                 transform = get_transforms(phase='train'))\n",
    "    val_dataset = TrainDataset(df.loc[val_idx].reset_index(drop=True),\n",
    "                               df.loc[val_idx].reset_index(drop=True)['isup_grade'],\n",
    "                               transform = get_transforms(phase='train'))\n",
    "    \n",
    "    train_loader = DataLoader(train_dataset, batch_size=CFG.batch_size, shuffle=True, num_workers=4)\n",
    "    val_loader = DataLoader(val_dataset, batch_size=CFG.batch_size, shuffle=False, num_workers=4)\n",
    "    return train_loader, val_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, dataloaders, criterion, optimizer, scheduler, num_epochs=25):\n",
    "    since = time.time()\n",
    "    \n",
    "    # Send the model to GPU/CPU\n",
    "    model = model.to(device=CFG.device)\n",
    "    \n",
    "    train_acc_history = []\n",
    "    val_acc_history = []\n",
    "    loss_history = []\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "    \n",
    "    preds, targets = [], []\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        \n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                model.train()   # Set model to training phase\n",
    "            else:\n",
    "                model.eval()    # Set model to evaluate phase\n",
    "            \n",
    "            avg_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            \n",
    "            for inputs, labels in tqdm(dataloaders[phase],\n",
    "                                       desc='{}/{}({:5s})'.format(epoch+1,num_epochs,phase)):\n",
    "                inputs = inputs.to(device=CFG.device, dtype=CFG.dtype)\n",
    "                labels = labels.to(device=CFG.device, dtype=torch.long)\n",
    "                \n",
    "                # Zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "                \n",
    "                # Forward, track history if only in training\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    outputs = model(inputs)\n",
    "                    loss = criterion(outputs, labels)\n",
    "                    \n",
    "                    pred = torch.argmax(outputs, 1)\n",
    "\n",
    "                    # Backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                    \n",
    "                # Statistics\n",
    "                avg_loss += loss.item()*(inputs.size(0)/len(dataloaders[phase].dataset))  # len(dataloaders[phase].dataset) not len(dataloaders[phase])\n",
    "                running_corrects += torch.sum(pred == labels)\n",
    "                preds.append(pred)\n",
    "                targets.append(labels)\n",
    "            \n",
    "            # End of epoch\n",
    "            with torch.no_grad():\n",
    "                epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n",
    "\n",
    "                if phase == 'val':\n",
    "                    val_acc_history.append(epoch_acc)\n",
    "                    p = torch.cat(preds).cpu()\n",
    "                    t = torch.cat(targets).cpu()\n",
    "                    kappa = cohen_kappa_score(t, p, weights='quadratic')\n",
    "                    #print(confusion_matrix(t,p)) https://www.dataschool.io/simple-guide-to-confusion-matrix-terminology/\n",
    "                    # deep copy the model\n",
    "                    if epoch_acc > best_acc:\n",
    "                        best_acc = epoch_acc\n",
    "                        best_model_wts = copy.deepcopy(model.state_dict())\n",
    "                else:\n",
    "                    train_acc_history.append(epoch_acc)\n",
    "                    loss_history.append(avg_loss)\n",
    "                print('{} Loss: {:4f} Acc: {:4f} Kaap: {:4f}'.format(\n",
    "                          phase, avg_loss, epoch_acc, kappa if phase=='val' else 1))\n",
    "\n",
    "                if scheduler is not None and phase == 'train':\n",
    "                    scheduler.step()\n",
    "                \n",
    "        print()\n",
    "    \n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:0f}s'.format(time_elapsed//60, time_elapsed%60))\n",
    "    print('Best val Acc: {:4f}'.format(best_acc))\n",
    "    \n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model, loss_history, train_acc_history, val_acc_history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_parameter_requires_grad(model, feature_extracting):\n",
    "    if feature_extracting:\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False\n",
    "            \n",
    "\n",
    "def initialize_model(num_classes, feature_extract=False, use_pretrained=False):\n",
    "    model_ft = None\n",
    "    input_size = 0\n",
    "    \n",
    "    model_ft = models.alexnet(pretrained=use_pretrained)\n",
    "    set_parameter_requires_grad(model_ft, feature_extract)\n",
    "    num_ftrs = model_ft.classifier[6].in_features\n",
    "    model_ft.classifier[6] = nn.Linear(num_ftrs, num_classes)\n",
    "    #input_size = 128\n",
    "    \n",
    "    return model_ft, input_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96134538d268453084368b168eb20674",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='1/30(train)', max=493.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.689471 Acc: 0.260682 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63bad983714f4a448167fed14d92d61a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='1/30(val  )', max=165.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.651554 Acc: 0.270825 Kaap: 0.027224\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d134489fcefd429598223b3a19ffcb92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='2/30(train)', max=493.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.649438 Acc: 0.276658 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab4d18f8e54e44db8c2e668dfd3f256c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='2/30(val  )', max=165.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.643398 Acc: 0.248764 Kaap: 0.051294\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5314f062aec04888a2002992f7b32333",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='3/30(train)', max=493.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.633159 Acc: 0.282110 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb19f9cbd8224473a0d24d015ccaedad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='3/30(val  )', max=165.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.625439 Acc: 0.284899 Kaap: 0.071004\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be7dd0fa2b774b14ae6f4cf8e71ecde8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='4/30(train)', max=493.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.622249 Acc: 0.294409 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5bc7eebc5d404ac988737284df989899",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='4/30(val  )', max=165.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.601063 Acc: 0.294789 Kaap: 0.091495\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92b26b7a671e461ea2f8f2a6123bfda9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='5/30(train)', max=493.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.606653 Acc: 0.309623 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d7fe55d37744c60be6b4524f1687f0a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='5/30(val  )', max=165.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.616106 Acc: 0.310765 Kaap: 0.147699\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff6b7fc5cfc74cfb9799c86ed807af75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='6/30(train)', max=493.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.601978 Acc: 0.316977 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd3e84a32f83492c83b85da576e1fc4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='6/30(val  )', max=165.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.603312 Acc: 0.311525 Kaap: 0.180663\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "896153b4355f44d1a6b298ecfddac156",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='7/30(train)', max=493.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.590918 Acc: 0.323570 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e25812a6618a4c2cb848db4c81cc418d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='7/30(val  )', max=165.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.577080 Acc: 0.314949 Kaap: 0.215254\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0152263101c14a50a587439670987c20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='8/30(train)', max=493.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.583742 Acc: 0.325979 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d73fb993b074c78a98c378e429c7ac9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='8/30(val  )', max=165.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.572733 Acc: 0.320654 Kaap: 0.239272\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e50745123e14ac1bc97b39cadc141f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='9/30(train)', max=493.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.581650 Acc: 0.335869 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fd0bccde1c642c2aa4d43400a211356",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='9/30(val  )', max=165.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.579765 Acc: 0.327121 Kaap: 0.258320\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e45c3970485d4cbbade5c7365f23ecfe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='10/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.573671 Acc: 0.338025 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57288c13d6944a62872781e49868279e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='10/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.571658 Acc: 0.330164 Kaap: 0.276303\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f00b63bff2634a67a7539a6bfbf059f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='11/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.572064 Acc: 0.342462 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6942edb27454435282c1ed1b85cb2b91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='11/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.569866 Acc: 0.315329 Kaap: 0.289667\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c22a8f40fe054f4580bb9308fe3da3b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='12/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.554828 Acc: 0.352732 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba4ccb882728420fa8458b9752ca99fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='12/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.567808 Acc: 0.337771 Kaap: 0.303943\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "330b8c9610bf4a8cb78a732fd1771ba7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='13/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.553455 Acc: 0.354000 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2447a02228f642d78b3a9177e64f50f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='13/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.560858 Acc: 0.356029 Kaap: 0.315002\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e1b646939914185956dcc8b44af55c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='14/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.544473 Acc: 0.360086 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1566088b9b5a4132b9937a010b3d3976",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='14/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.566777 Acc: 0.341194 Kaap: 0.323786\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b200467cbcd4690ad2253d9dae5d584",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='15/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.531629 Acc: 0.365919 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "442e8851a23348d4a1e363e72b29d431",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='15/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.557456 Acc: 0.366679 Kaap: 0.332500\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb6e88d6865b4ca2874e6b564b9aebc6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='16/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.521355 Acc: 0.364397 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b029e6b756094b60b2bd93234454872f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='16/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.575457 Acc: 0.322176 Kaap: 0.339582\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "903318c987694bc09e61b50c4a358bdd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='17/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.508782 Acc: 0.374540 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "922219a5ffb24e4d9e4eb429c2b06dc9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='17/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.535290 Acc: 0.362495 Kaap: 0.348494\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbf1c8d3ce0243619c28613e4337456a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='18/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.498560 Acc: 0.377964 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1f2df3e89a94e89baf989188ca5b141",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='18/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.612779 Acc: 0.349943 Kaap: 0.356549\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab863f32c1aa4090ab8a025c3e52c7e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='19/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.490623 Acc: 0.387727 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f404331ed9f64e8fbaa2acf4aa5db8b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='19/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.488226 Acc: 0.395588 Kaap: 0.364320\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eed1f0478f7e48a093838a388c519811",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='20/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.479458 Acc: 0.391657 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd484b2572614cf58c6980538bfb2fbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='20/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.492096 Acc: 0.397490 Kaap: 0.371080\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "252c968e18bd4ca4bd22986720924dd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='21/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.465862 Acc: 0.400786 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a1938f8e79e4d77b026d8a3dccae868",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='21/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.478894 Acc: 0.391784 Kaap: 0.378316\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "792e9464ffca48379065f7066dfac3e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='22/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.451888 Acc: 0.403449 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08c68cf6b1254f0db16ec5fced72424f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='22/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.568908 Acc: 0.370483 Kaap: 0.384949\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eaa821e7dd2a499f91938a0800d9f9dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='23/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.444996 Acc: 0.411817 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e6032c54abf4ff2bef02ed70347da76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='23/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.472937 Acc: 0.401674 Kaap: 0.391689\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f034bd8d9554d2483d8091452583d84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='24/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.435347 Acc: 0.412324 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e185c4e650a4a57894aa25a5e2d8ae3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='24/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.471807 Acc: 0.400913 Kaap: 0.397578\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a38a6de93a3424a860934367d012c88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='25/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.431939 Acc: 0.416255 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94f2698b5c604e50a4c0722443147863",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='25/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.470376 Acc: 0.394827 Kaap: 0.403549\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba23f87478494f2fbe15563122d82f2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='26/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.413299 Acc: 0.423228 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcffbcc3280647f2b2a9eab7196a5d04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='26/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.492312 Acc: 0.401674 Kaap: 0.408834\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de9e28f632b540b691e1b435edea315c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='27/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.404800 Acc: 0.435527 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8192eb1fbb8149b7a2c50445c72fa81a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='27/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.489647 Acc: 0.403576 Kaap: 0.414785\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "314bcaf0bc0b4d898bbe6e94dc0a66b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='28/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.392579 Acc: 0.437555 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d92d3f848ba4b2baa2d932fa3b0e37b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='28/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.486726 Acc: 0.389121 Kaap: 0.420427\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40f0739d251f4d33b4f9a41284276eee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='29/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.382688 Acc: 0.439077 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efb1550e4b9141ba8da1903e43513659",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='29/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.465262 Acc: 0.399772 Kaap: 0.425659\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "290b72aff1d24b0fb96ffc7b7b17ec73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='30/30(train)', max=493.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "train Loss: 1.362116 Acc: 0.446304 Kaap: 1.000000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8967aa79e1904aaba9b6d9d62bf602e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='30/30(val  )', max=165.0, style=ProgressStyle(description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "val Loss: 1.488377 Acc: 0.386078 Kaap: 0.430801\n",
      "\n",
      "Training complete in 1324m 6.374866s\n",
      "Best val Acc: 0.403576\n",
      "[1.6894708464048018, 1.6494383671384387, 1.6331593877872899, 1.622249245900564, 1.6066531996408384, 1.6019782574892931, 1.5909179589647717, 1.5837415044292893, 1.581650170035064, 1.5736711711745701, 1.5720637920798493, 1.5548280147873161, 1.5534552395593184, 1.5444726234941226, 1.531629367126807, 1.521354815904609, 1.5087817000151755, 1.4985602027157388, 1.490622816618454, 1.4794579497063223, 1.4658620983562471, 1.451887565676639, 1.4449956507922825, 1.4353465667556295, 1.4319386354368775, 1.4132991425661867, 1.4048002519670435, 1.3925792793515996, 1.382687727746302, 1.3621163612477927]\n",
      "[tensor(0.2607, device='cuda:0', dtype=torch.float64), tensor(0.2767, device='cuda:0', dtype=torch.float64), tensor(0.2821, device='cuda:0', dtype=torch.float64), tensor(0.2944, device='cuda:0', dtype=torch.float64), tensor(0.3096, device='cuda:0', dtype=torch.float64), tensor(0.3170, device='cuda:0', dtype=torch.float64), tensor(0.3236, device='cuda:0', dtype=torch.float64), tensor(0.3260, device='cuda:0', dtype=torch.float64), tensor(0.3359, device='cuda:0', dtype=torch.float64), tensor(0.3380, device='cuda:0', dtype=torch.float64), tensor(0.3425, device='cuda:0', dtype=torch.float64), tensor(0.3527, device='cuda:0', dtype=torch.float64), tensor(0.3540, device='cuda:0', dtype=torch.float64), tensor(0.3601, device='cuda:0', dtype=torch.float64), tensor(0.3659, device='cuda:0', dtype=torch.float64), tensor(0.3644, device='cuda:0', dtype=torch.float64), tensor(0.3745, device='cuda:0', dtype=torch.float64), tensor(0.3780, device='cuda:0', dtype=torch.float64), tensor(0.3877, device='cuda:0', dtype=torch.float64), tensor(0.3917, device='cuda:0', dtype=torch.float64), tensor(0.4008, device='cuda:0', dtype=torch.float64), tensor(0.4034, device='cuda:0', dtype=torch.float64), tensor(0.4118, device='cuda:0', dtype=torch.float64), tensor(0.4123, device='cuda:0', dtype=torch.float64), tensor(0.4163, device='cuda:0', dtype=torch.float64), tensor(0.4232, device='cuda:0', dtype=torch.float64), tensor(0.4355, device='cuda:0', dtype=torch.float64), tensor(0.4376, device='cuda:0', dtype=torch.float64), tensor(0.4391, device='cuda:0', dtype=torch.float64), tensor(0.4463, device='cuda:0', dtype=torch.float64)]\n",
      "[tensor(0.2708, device='cuda:0', dtype=torch.float64), tensor(0.2488, device='cuda:0', dtype=torch.float64), tensor(0.2849, device='cuda:0', dtype=torch.float64), tensor(0.2948, device='cuda:0', dtype=torch.float64), tensor(0.3108, device='cuda:0', dtype=torch.float64), tensor(0.3115, device='cuda:0', dtype=torch.float64), tensor(0.3149, device='cuda:0', dtype=torch.float64), tensor(0.3207, device='cuda:0', dtype=torch.float64), tensor(0.3271, device='cuda:0', dtype=torch.float64), tensor(0.3302, device='cuda:0', dtype=torch.float64), tensor(0.3153, device='cuda:0', dtype=torch.float64), tensor(0.3378, device='cuda:0', dtype=torch.float64), tensor(0.3560, device='cuda:0', dtype=torch.float64), tensor(0.3412, device='cuda:0', dtype=torch.float64), tensor(0.3667, device='cuda:0', dtype=torch.float64), tensor(0.3222, device='cuda:0', dtype=torch.float64), tensor(0.3625, device='cuda:0', dtype=torch.float64), tensor(0.3499, device='cuda:0', dtype=torch.float64), tensor(0.3956, device='cuda:0', dtype=torch.float64), tensor(0.3975, device='cuda:0', dtype=torch.float64), tensor(0.3918, device='cuda:0', dtype=torch.float64), tensor(0.3705, device='cuda:0', dtype=torch.float64), tensor(0.4017, device='cuda:0', dtype=torch.float64), tensor(0.4009, device='cuda:0', dtype=torch.float64), tensor(0.3948, device='cuda:0', dtype=torch.float64), tensor(0.4017, device='cuda:0', dtype=torch.float64), tensor(0.4036, device='cuda:0', dtype=torch.float64), tensor(0.3891, device='cuda:0', dtype=torch.float64), tensor(0.3998, device='cuda:0', dtype=torch.float64), tensor(0.3861, device='cuda:0', dtype=torch.float64)]\n"
     ]
    }
   ],
   "source": [
    "model_ft, input_size = initialize_model(6)\n",
    "#print(model_ft)\n",
    "\n",
    "optimizer = optim.SGD(model_ft.parameters(),\n",
    "                      lr=3e-3,\n",
    "                      momentum=.9,\n",
    "                      nesterov=True)\n",
    "\n",
    "loader_train, loader_val = data_loader(0)()\n",
    "best_model, loss_history, train_acc_history, val_acc_history = train_model(model_ft, {'train': loader_train, 'val': loader_val}, F.cross_entropy, optimizer, None, CFG.epochs)\n",
    "print(loss_history)\n",
    "print(train_acc_history)\n",
    "print(val_acc_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEJCAYAAAB2T0usAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU5dXA8d/JAglr2JewBBVBBQUJigWVuuEGokVFa1+1tdi31rVFsK2K1lZfrXVf625RioigVkVFBVFE9kUQQcqSsIVAQgLZc94/7gSSzJ3kTjKTyUzO9/PhQ+bOnTvPwyX33Pss5xFVxRhjTNMWF+kCGGOMiTwLBsYYYywYGGOMsWBgjDEGCwbGGGOwYGCMMYYwBgMReUlEdovImgDvTxSRFb4/a0SkTETah6s8xhhjApNwzTMQkdOAfOA1VR1Qy76jgVtV9YywFMYYY0yNEsJ1YFWdLyJpHne/AnjTy44dO3bUtDSvhzXGGAOwdOnSParaKdD7YQsGXolIC+Bc4Hde9k9LS2PJkiXhLZQxxsQYEdlS0/sRDwbAaOArVd0baAcRmQBMAOjVq1fQXzBreSYPzVnP9pwCuqckM3FUP8YOTq1zgY0xJtY0htFE46mliUhVn1fVdFVN79Qp4FOOq1nLM7lj5moycwpQIDOngDtmrmbW8sx6FNkYY2JLRIOBiLQFTgdmh+s7HpqznoKSsirbCkrKeGjO+nB9pTHGRJ2wNROJyJvASKCjiGQAdwOJAKr6rG+3i4GPVfVAuMqxPacgqO3GGNMUhXM00RUe9nkFeCVcZQDonpJMpsuFv3tKcji/1hhjokpj6DMIq4mj+pGcGF9lW3JiPBNH9YtQiYwxpvFpDKOJwqpi1JCNJjLGmMBiPhiAExDs4m+MMYHFfDORMcaY2lkwMMYYY8HAGGNME+kzCIalrjDGNEUWDCqpSF1RMWO5InUFYAHBGBPTrJmoEktdYYxpqiwYVGKpK4wxTZU1E1USTOoK61swxsQSezKoxGvqCkuLbYyJNRYMKhk7OJX7LxlIakoyAqSmJHP/JQP97vitb8EYE2usmagaL6krgu1bsCYlY0xjZ08GdRAo/XWgvgVrUjLGNHYWDOogmLTY1qRkjIkG1kxUB8GkxbbhqsaYaGDBoI68psW2ldaMMdHAmonCLJgmpVnLMxn+wGf0mfwfhj/wmfUrGGMaTNieDETkJeBCYLeqDgiwz0jgUSAR2KOqp4erPJHitUkp2LxINkLJGBNKoqrhObDIaUA+8JpbMBCRFOBr4FxV3SoinVV1d23HTU9P1yVLloS+wBE2/IHPXJuTUlOS+WryGVW2VQ8c4DxtuM2JMMYYABFZqqrpgd4PWzORqs4H9tawy5XATFXd6tu/1kAQy4LpaLYRSsaYUItkB/LRQKKIfAG0Bh5T1dfcdhSRCcAEgF69ejVYARtSMB3NwQQOa04yxngRyQ7kBGAIcAEwCrhTRI5221FVn1fVdFVN79SpU0OWscEE09HsddKbTXgzxngVyWCQAXykqgdUdQ8wHzghguWJKK95kcB74Ai2OcnraCYb9WRM7IlkM9Fs4EkRSQCaAScDj0SwPBHnde6C1xFKwTYneRnNZKvBGRObwjm09E1gJNBRRDKAu3GGkKKqz6rqOhH5CFgFlAMvqOqacJUn1ngJHMH0Q9T0FFH5e7zuZ4yJLmELBqp6hYd9HgIeClcZmrqJo/q5DkF164fw+hQRroyt1tFtTGRZOooYFkwOJa9PEcGuBmdNT8ZEB0tHEePGDk7lq8ln8N8HLuCryWcEvLh67ZQOR8ZWmzdhTOTZk4EBvD9FhCNjq2V2NSbyLBiYQ4IZzRTKjK2W2dWYyLNmIhM24Wh6ApsPYUw42JOBCZtwND1Zp7Qx4RG2rKXhEqtZS403XrO7BpMF1pimoLaspfZkYKJKpOdDGBOrPPUZiMjNItJGHC+KyDIROSfchTOmOq9J+rzuB5bQzxjw3oH8S1XdD5wDdAKuBR4IW6mMCSCS8yGMiWVem4nE9/f5wMuqulJEpKYPGBMOkZwPAdacZGKX12CwVEQ+BvoAd4hIa5zkcsY0uEjNh7ARSiaWeW0m+hUwGRiqqgdxso9eG7ZSGdOAwrU+hDHRxOuTwSnAClU9ICJXAScCj4WvWMY0nHCsDwHWpGSii9dg8AxwgoicANwOvAi8BpweroIZ05BCvT6ENSmZaOO1mahUndlpF+EsXP8YziL2xjQZNkLJxDKvTwZ5InIH8AvgVBGJx7dqmTFNhY1QMrHMazC4HLgSZ77BThHpha1QZpogG6FkYpWnZiJV3QlMBdqKyIVAoaq+VtNnROQlEdktIq7rGovISBHJFZEVvj93BV16YxopG6Fkoo2nJwMRuQznSeALnAloT4jIRFWdUcPHXgGexOloDuRLVb3QW1GNiR42QslEG6/NRH/CmWOwG0BEOgGfAgGDgarOF5G0+hbQmGhlI5RMNPE6miiuIhD4ZAfx2ZqcIiIrReRDETkuBMczJqqEa4SSLexjguX1yeAjEZkDvOl7fTnwQT2/exnQW1XzReR8YBbQ121HEZkATADo1atXPb/WmMYjHCOU7AnC1IXnxW1E5GfAcJw+g/mq+o6Hz6QB76vqAA/7bgbSVXVPTfvZ4jamqbKFfUx9hGxxG1V9G3g7JKUCRKQrsEtVVUROwml2yg7V8Y2JNRNH9atyxw/uTUrWKW3qosZgICJ5gNujgwCqqm1q+OybwEigo4hkAHfjm6imqs8C44D/FZFSoAAYr9G2BqcxDchrk5J1Spu6sDWQjYkx1S/w4DxB3H/JQL8LvDUpNR22BrIxTUy40maY2GbBwJgYFOq0GWB9C7EuFHMFjDFRyus8h4qmp8ycApTDfQs2fyF2WDAwpgkbOziV+y8ZSGpKMoLTV+DWt2A5lGKfNRMZ08R5aVKyvoXYZ8HAGFOrYPoWwPoXopE1ExljahVMDiXrX4hO9mRgjKlVMMNVa+pfqL6/PUE0HhYMjDGeeB2uagn1opM1ExljQipQP0L17TZCqXGxYGCMCSmv/Qt1SahnazSEjzUTGWNCKtIJ9awfom4sGBhjQs5L/4LXlNzgvVPa+iHqzpqJjDER4XX2M3hvUrJ+iLqzJwNjTMSEOqGezZSuO3syMMY0el47pb2OZDL+LBgYYxo9r01KwcyUBhuhVJk1ExljooKXJqVgZkpbZ3NVFgyMMTHFaz+Epc2oKmzNRCLykojsFpE1tew3VETKRGRcuMpijDHVBZs2I9YT74Wzz+AV4NyadhCReOD/gDlhLIcxxvixtBlVhS0YqOp8YG8tu90IvA3sDlc5jDHGjaXNqCpio4lEJBW4GHjWw74TRGSJiCzJysoKf+GMMTHP6wilYIarRnOTUiQ7kB8FJqlqmYjUuKOqPg88D5Cenq4NUDZjTBMQqbQZjVEkg0E6MM0XCDoC54tIqarOimCZjDGmimCGqwbTpNTYRihFLBioap+Kn0XkFeB9CwTGmMYo1GkzGuMch3AOLX0TWAj0E5EMEfmViPxGRH4Tru80xphI8top3RhHKIXtyUBVrwhi32vCVQ5jjGkoXpuUGmNCPZuBbIwxIeSlSSmYhX0aiiWqM8aYBhZsQr2GYE8GxhjTwIIZoQQNM/LIgoExxkSA1xFKDTXyyJqJjDGmEWuokUcWDIwxphFrqJFHFgyMMaYRa6ilPC0YGGNMI9ZQI4+sA9kYYxqxYEce1ZWoRlcSUBHJArbU8eMdgT0hLE5jEGt1irX6QOzVKdbqA7FXJ7f69FbVToE+EHXBoD5EZImqpke6HKEUa3WKtfpA7NUp1uoDsVenutTH+gyMMcZYMDDGGNP0gsHzkS5AGMRanWKtPhB7dYq1+kDs1Sno+jSpPgNjjDHumtqTgTHGGBdNJhiIyLkisl5ENorI5EiXJxREZLOIrBaRFSKyJNLlCZaIvCQiu0VkTaVt7UXkExHZ4Pu7XSTLGKwAdZoiIpm+87RCRM6PZBmDISI9ReRzEVknIt+JyM2+7VF5nmqoTzSfoyQR+VZEVvrqdI9vex8RWeQ7R/8WkWY1HqcpNBOJSDzwA3A2kAEsBq5Q1bURLVg9ichmIF1Vo3J8tIicBuQDr6nqAN+2B4G9qvqAL2i3U9VJkSxnMALUaQqQr6p/j2TZ6kJEugHdVHWZiLQGlgJjgWuIwvNUQ30uI3rPkQAtVTVfRBKBBcDNwG3ATFWdJiLPAitV9ZlAx2kqTwYnARtVdZOqFgPTgIsiXKYmT1XnA3urbb4IeNX386s4v6hRI0Cdopaq7lDVZb6f84B1QCpRep5qqE/UUke+72Wi748CZwAzfNtrPUdNJRikAtsqvc4gyv8D+CjwsYgsFZEJkS5MiHRR1R3g/OICnSNcnlD5nYis8jUjRUWTSnUikgYMBhYRA+epWn0gis+RiMSLyApgN/AJ8COQo6qlvl1qveY1lWAgLttioX1suKqeCJwH3OBrojCNzzPAkcAgYAfwcGSLEzwRaQW8DdyiqvsjXZ76cqlPVJ8jVS1T1UFAD5yWkGPcdqvpGE0lGGQAPSu97gFsj1BZQkZVt/v+3g28g/OfINrt8rXrVrTv7o5weepNVXf5flnLgX8SZefJ1w79NjBVVWf6NkfteXKrT7SfowqqmgN8AQwDUkSkIhlprde8phIMFgN9fb3rzYDxwLsRLlO9iEhLXwcYItISOAdYU/OnosK7wNW+n68GZkewLCFRcdH0uZgoOk++zskXgXWq+o9Kb0XleQpUnyg/R51EJMX3czJwFk5fyOfAON9utZ6jJjGaCMA3VOxRIB54SVX/GuEi1YuIHIHzNABOKvI3oq1OIvImMBInw+Iu4G5gFjAd6AVsBS5V1ajpkA1Qp5E4zQ8KbAaur2hvb+xEZATwJbAaKPdt/iNOO3vUnaca6nMF0XuOjsfpII7HucGfrqr3+q4R04D2wHLgKlUtCnicphIMjDHGBNZUmomMMcbUwIKBMcYYCwbGGGMsGBhjjMEZhRJVOnbsqGlpaZEuhjHGRJWlS5fuqWkN5KgLBmlpaSxZEnUJOo0xps5mLc/koTnr2Z5TQPeUZCaO6sfYwcFl1BGRLTW9H3XBwBhjmpJZyzO5Y+ZqCkrKAMjMKeCOmasBgg4INbE+A2OMacQemrP+UCCoUFBSxkNz1of0e+zJwBhjGqHCkjK+3LCHzJwC1/e3B9heVzERDEpKSsjIyKCwsDDSRQm7pKQkevToQWJiYqSLYoxx4bV9322/kf06MXfdbj5eu5P5P+yhoKQMwT3daPeU5JCWOyaCQUZGBq1btyYtLQ0nD1VsUlWys7PJyMigT58+kS6OMU1GMBd4L+37bvvdNn0Fqs6Fv0ub5owb0oNzjuvCrtxC7pz9XZWmouTEeCaO6hfSOsZEMCgsLIz5QAAgInTo0IGsrKxIF8WYJsPrBb6wpIz7P1zn2r7/p1mrmb8hi/0FJeQWlLB8aw6l5VXv98sVWjVP4F/XnczxqW2Jizt8PUuIj6v3aKLaxEQwAGI+EFRoKvU0prEI1IE7eeYqXv9mC9n5RezJLya/qDTAEeBAURmLNu2lTXIibZMT/ALB4f1KGdQzxW/72MGpIb/4V2ejiUIgJyeHp59+OujPnX/++eTk5IShRMaYUAnUUVtYUk6z+DgG9khh3JAeTBzVj5Rk97681JRkvpp8Bh/efCrTJpxCaoD2/lD3AwQjZp4MghGKCRyVVQSD3/72t1W2l5WVER8fH/BzH3zwQZ2/0xgTXgeLS3n5q80B309NSebNCcP8tlVuUgL39v2Jo/p52q8hNblgEI4JHJMnT+bHH39k0KBBJCYm0qpVK7p168aKFStYu3YtY8eOZdu2bRQWFnLzzTczYYKzdn3FbOr8/HzOO+88RowYwddff01qaiqzZ88mOTlydwnGNFUlZeVMW7yNx+duICuviOO6t2bj7gMUlZYf2ifQhbviGlLbzabX/RpS1C1uk56ertXTUaxbt45jjnHWf77nve9Yuz3wet3Lt+ZQXFbut71ZfByDe/m31QEc270Nd48+LuAxN2/ezIUXXsiaNWv44osvuOCCC1izZs2hET979+6lffv2FBQUMHToUObNm0eHDh2qBIOjjjqKJUuWMGjQIC677DLGjBnDVVdd5fp9letrjKm7yq0E3VKSOPOYLsz/IYst2QdJ792OSef1Z2ha+5C3JkSCiCxV1fRA7ze5JwO3QFDT9ro46aSTqgz9fPzxx3nnHWeFym3btrFhwwY6dOhQ5TN9+vRh0KBBAAwZMoTNmzeHrDzGGH/VWwm25xTy+sItdG3TnBevTueM/p0PDdhoiA7cSIu5YFDTHTzA8Ac+c53Rl5qSzL+vPyUkZWjZsuWhn7/44gs+/fRTFi5cSIsWLRg5cqTr5LjmzZsf+jk+Pp6CgtDOLjTGVOU2DBQgLk4485guEShRZMVcMKhNODpuWrduTV5enut7ubm5tGvXjhYtWvD999/zzTff1Pl7jDG1C9SkU1auLNu6j0/X7eLTtbvYtd99bfgdObGfycBNkwsG4ei46dChA8OHD2fAgAEkJyfTpcvhu4pzzz2XZ599luOPP55+/foxbNiwGo5kjKkPtwEit89Yyb++2cymPQfZe6CYhDjh5CPasye/iNwC/7kBkRzeGUkx14HcFDS1+prYFUzHbG37lpSVM/yBz9id53/HLwIXndCdM4/pwun9OtEmKdEvcIDTSnD/JQNjsn/AOpCNMY1SMMO8A93xf/zdThLi4/hhVx6bsg4EHgii8Oj4wVU2NcbhnZFkwcAYExE1pXl4b+V2isvKKSkrp7RMWZmRQ0lZ1VaM4jLlgzU76dEumaO7tGZkv878e/FW9h0s8fuuQE0/TWGUkFcWDIwxDaqsXFmwMXCe/sKScnblFZIYH0difBzNE+P8AkEFARZMOuPQ6/5dWze6mb3RwoKBMSbkqrfv/+Gcozm6a2veWZbJ7JXbycorCpinPzUlmfdvPLXKtkBDwqvf8Yet6WfVdJh7L+RmQNsecOZdcPxl9TtmIxPWYCAi5wKPAfHAC6r6QID9xgFvAUNV1Va7N6YRqk9O/9umr0SBxHjhp/06c/HgVPKLSln07rPcwjS6yx62a0ceZTwjRv3W75gTR/VjwTtPe9o3qKYfLxf5VdPhvZugxBeMcrc5ryGmAkLYgoGIxANPAWcDGcBiEXlXVddW2681cBOwKFxlMcbUT6DOXlXltKM7sSO3kMycAnbkFPD3j3/w6wtQoG1yIl/8YSTtWjZzNq6azsWJL5BQ5ozr7yF7eCD+BRLiTwCqXmTHxn/FhR739cztIj/7d7B9BXTuDwf2wMFsWPLS4X0qlBQ4QaQ+waCRPW2E88ngJGCjqm4CEJFpwEXA2mr7/QV4EPhDGMvSqLRq1Yr8/PxIF8MYh4eLUqDO3lunr/T8NfsLSg4HAoC59x66uFdIKCuEd34Dn/8NEpMhoTkkJMH2ZSSUFfnvW9cLcsE++HCy/0W+rAi+eerw68QWUHLQ/Ri5GcF/b4VG+LQRzmCQCmyr9DoDOLnyDiIyGOipqu+LSMMFg0YWkY0JOa//xz1clP6750DAzl6Auy48lu4pyXRPSaJ7SjIXPbmATJdZvH4jegJdTLUMeqRDaSGUFjllK3WfLUzuNlj4NBxxOnQ+1plQ4Fb3gZfCnh/gh4/ghzmw9Rvne1wJ3LIKWnSEZi3gkQHO9/jtJrByGhx/ufNzMObeE56njXoIZzBw+9c51F8kInHAI8A1tR5IZAIwAaBXr171K1UYIvKkSZPo3bv3ofUMpkyZgogwf/589u3bR0lJCffddx8XXXRR/cpujBeuzR+/hXXvQeuucCDrcBNI1veg1cbmlxSgc/7Ee6Un8+aSHSzclA3AmLgF3J4w/VCb/YOll7G0zdn8ckTV9bgnjurvbURPm+6wP9O//G17ws9eqLot0AU5LgHm3OH83LIztEuDHcuhrORw3Wf9L3x0Bxzc42zrMhBG3ArLXoMDu12+vwekVLrOnHlX1X9PcJ5YWneHd66HFW/AhY9AhyP9j1VdYS4seTlwIMzdBqrBB5cQCNsMZBE5BZiiqqN8r+8AUNX7fa/bAj8CFe0lXYG9wJiaOpFrnYH84WTYuTpwwTIWO4+C1cU3hx5D3T/TdSCc59r3DcDy5cu55ZZbmDdvHgDHHnssH330ESkpKbRp04Y9e/YwbNgwNmzYgIjUu5nIZiCbGgW6cAIktXXueFt2hBYd0fX/cb1rA8jTZL6L70957xEkSwn9f3yJZCk+9H6BNmPNkPsYOuZ6v8/W2tmsCi+e7fw+VpaYDKMfr70Tt/K+vU6B/86DTV/Amrf9gxs4TU2j/gZHj3Iu9rUd0+37qz9tDBgHS1+CT+9xnlxOmwjDb4a1s/z3TTsVvnnaCQTFeU4wCfS003sEnP8gdKk56WawIjkDeTHQV0T6AJnAeODKijdVNRfoWKmgXwB/CPtoIrdAUNN2DwYPHszu3bvZvn07WVlZtGvXjm7dunHrrbcyf/584uLiyMzMZNeuXXTt2rXO32OMJwHbsgUmb62yZeeUo+hGlt+ee7U1hUeP5uR9S5H/PnHo45UlSzFDf3wC8A8GtY7oWfaaEwiOHQuZS2tvzqrYFqjpa/BVzp/VM9y/r7QIhv4quGNW39dt+9DroP+F8OEk+Pw+WPyC0x9RcT3J3eb0gag6/37HjoXhN8GeDe6B6LhLYP0H8OypcNKvYeQdkOy+zkqoeQoGIvI28BLwoapb2PWnqqUi8jtgDs7Q0pdU9TsRuRdYoqrv1rXQNarhDh4IfNfUtidc+586f+24ceOYMWMGO3fuZPz48UydOpWsrCyWLl1KYmIiaWlprqmrjQm1g8ldaVGww2/7/uZdmLNkGxt257N+Zx4/7MpjaPGlPJD4Ai0q3fEf1GbcU/ILHvv5/c6G/Cz4+1HuX1aXTtRda+HD2+GIkTDuZYjzuBR7oAtyZW17BPj97lH3Y9amdVe47FWnL+LNK/z7IrQMmrWC3yyA9r4mte6+1BhugejgXvjsL7DoOSe4nX0PxCU628LYz+n1yeAZ4FrgcRF5C3hFVb+v7UOq+gHwQbVtdwXYd6THstSPW/tfYrKzvR7Gjx/Pr3/9a/bs2cO8efOYPn06nTt3JjExkc8//5wtW7bUs+DGePN48UVM0merNDsf1Gb8Oe8S3p2ximYJcRzVqRXDjujA3HUjmVyMry8gm+3a4VBfwCGtOjk3S24X2Tbdgytc8UGYcS00bw0XP+89EHgVpt9vT44e5d5EBVB84HAgqBAoELVo7/RBnHg1fDARZt8AEnf42GEaeeQpGKjqp8Cnvnb+K4BPRGQb8E/gX6rqnwyksQrm0TAIxx13HHl5eaSmptKtWzd+/vOfM3r0aNLT0xk0aBD9+/cPQeGNqdn2nAJaFe1A42G3ptCJ3EMX+HfLR/DZ70+nV/sWJMQ7F2Fn/oDybvGIQ8dITozn/uqdvW4XWXA6cA9kQ8uqK/cF9OHtkLUefvEOtA7DAjJh+v32LNgnk5p0HwS/nAMPHQkFe6u+F4aRR577DESkA3AV8AtgOTAVGAFcDYwMWYkaQigeDV2sXn2447pjx44sXLjQdT+bY2BCSVX5ZtNeXlu4mcVrN/BF4hz+Uz6MG0tuqrJfakoyR3RqVWWb5/QNbhfZY8Y4beQvn+dc3NvWMut31Vuw/HU49fdw5E/rU+Wahen325NQP5nExTl9EG7qM8/Bhdc+g5lAf+B1YLSqVjRI/ltELH2EMQ2k8iidrm2TGNG3I6u25bJ+Vx4pLRJ5pvcCWu4o4jkurfK5mpK1eU7f4HaR7X++007+0ignIHTs6/7Z7B/h/Vug5zAY+UcvVY1O4XgyCeXTRg28Phk8qaqfub1R01AlY0ztgsn5M3nmKgpLnLbjHbmFvLUkg9SUJB4cdzxjjmpG0lPXwICfcd2R5zVMnv60EXDN+/CvnzkB4aq3D3eOVigtgreugfhEGPcixMd4fsxQP5k0UD+I17NyjIgsU9UcABFpB1yhqk+HtDTGNDHuOX9WkZ1fxJGdW/HfPQcO/fl6YzZlAeYFXZbeEz6+E0oL4PRJjO3UgHn6u53gtG2/NhZeGQ1XvAF9Tjv8/sd3ws5VMP7NkN/NNgkN1A/iNRj8WlUPJexQ1X0i8mug0QQDVUUiMGuvoUXbMqWmZu45f8r5y3/WHXrdunkCR3RqGTAQbM8phPzd8O0/nbQLnY4Oa5lddTgSfjUHXr/YeUoYep0z47mieePIs5wmJVM3DdAP4jUYxImIqO9K5MtI2qyWzzSYpKQksrOz6dChQ0wHBFUlOzubpKSkSBfFhMC+A8U15vx56zen0KdjSzq0bIaI1JzT/6vHoKwYTp8UziLXrE13uPZDeP4MZ7ZtZVu/cmbxWg6wRstrMJgDTBeRZ3HyC/0G+ChspQpSjx49yMjIICvLfyZlrElKSqJHD3vUjpTF7z5Hz2UP0Vmz2C2d2HbiRNd0DDXZuDuPFxdsZuaywKNBUlOSGZrWvsq2iaP6ueb8+fPp7WHuC07CNC/5ccKpRXsoL/bfHuEkbKZ2XoPBJJw55/+LM6n6Y+CFGj/RgBITE+nTp0/tOxpTD4vffY4BS//s5OcR6EoWbZf+mcXgIT9PEhee0J11O/KY/0MWzRLiuGRwKr07tODxuRs9LdMYaBjoeTufcBKznT4xbHUPyv7t7ttDPBTShJbXSWflOLOQnwlvcYxpvHose6hKojZw8vOkLn2QZ9qcRdvkxEN/lm/dx5Ofb6So1Bn5k5lTyHPzNtG6eTy/P/torjy5Fx1aNQcgff9c/6eNwee6lsFvGOj+HfD+SzDoCmh/RHgqHqwGGgppQsvrPIO+wP3AscChBmtVbST/+4ypH7fhnWcf24UlW/ax8MdsFm7K5h3Nck3M3pVs/u+jWrOzANA6KZEbz6w0Fn/VdIauvhsoOPS00XX13ZDWzluTyoJ/OLlvTmskTwUQ2ZQQps68NhO9DNyNs/7AT3HyFMVuT61pUtyGd946fQWo00GWECcM6plCDq1pT57f53dLR767ZxS5BSXsLywh92AJlz//jet37citlqxw7rqeBW8AABbfSURBVL11X+QkNxOWvgKDfu7k8W8sIp0SwtSJ12CQrKpzfSOKtgBTRORLnABhTKNUW2dvaVk5KzNyuXP2Gv81exVaNU/gmatOZEjvdrTIXEj5qwcoUyFeDg/xLNU4tg2ZyNDmCbRsnkB3nNW8UlOSA4/8qaymRU5qy/nz5cNOQU9rhCvGRjIlhKkTr8Gg0Lcy2QZfWupMoHP4imVMYF5m7Abq7P2koJiMHhfy1cY9LNq0l7yi0oDfc6ColFP7doIdq2DalcR1Oprl3S6n26qn6ax7OCjJtJKDDB16it9nA4388esYDtS+DvD4IGexlGG/dZZfrCxnm7MmwOCrqq7KZUwdec0fewvQArgJGIKTsO7qcBXKmEAqmnQycwpQKmbsrmbW8kxUlQNFpezaX0iPZQ+6dvYe892j3PPeWjbszmf0oO48deWJdGvrPm+je0oy7P0vTB0HzdvAVTMZfMltdJ2ykbh7cmg1aZ2zctjnf/X77NjBqdx/yUBSU5IRnCeF+y8Z6D8r+MRf+H9xYjKcdY+T6uGzv8ATJzoX/rJSZ6z+IwPg0QFQXgKd3PMNGROsWpe99E0we0BVG0UPlduyl6bpCDTxSnCWjS33/Xfe1PxK4lx6tVQh46bt9OzQ8tC26n0G4NzFP3xBd87/9mona+Qv57hfeL982Gkb/9Wn0DPAsqmBqDr5fHathaQ2zpDM6u3rW7520jlkLoHW3Zx1i8sqBblAyzQaU01ty17W+mSgqmXAEInlqb2m0SsrVz7+bmfAGbsK3PDTo7jjvP5MS/8h4HriItDzw6sh5/Dyj2538Q+NPoLzV97oDN28cnrgO/CTroeWneCze4Ov1Jq3YdsiOPdvcNtamJIDt66pemHv/RO47lO47DVnIfuyahO6Kjqbjaknr30Gy4HZvlXODlRsVNWZYSmVMT4Hi0uZsTSDlxb8l83ZB4kXcc3Rk5qSzO/PSHMWT1nzCrmt+9Js/xa/Bdyzep5Lry1z4alhzh34Sb+GuPiq4/dLi+GNS2Hnahj/BvQ8KXABm7eCEbfBnDtg0zw44nRvFSspgE/uhq4DndFANRGBYy+C8jL3920ylwkBr8GgPZANnFFpmwIWDExIVO8U/vVpfdi1v4g3Fm0lt6CEQT1TeGpUf4pKSvnTrO/8mnTuPC0FXrnAWWR9xK20PeNOFr//gm800R52S0dn1M+Y652ngvdvg48mwZoZMOYJ58JfMRQyMcm5WF/0NPRzn/xVRfovYeGT8Nl9TrZOLw/RXz8B+zPgkucgLt7bP5JN5jJhVGufQWNjfQaxx63NvsL5A7vyqxFHMKR3uyr7Vw4cDwzJ49QVv3cu4GOfdu6ia6MKq9+CDydBQY6zolR5pZFFcYnOsby2xS952Vm85crpzlq4Ndm/HZ4YAkedBZe/7u344HQeu03msj4D40FtfQaegoGIvIzzJFCFqv6yfsULngWD6BJoGGh5ubJ170HWbM9l8turyXcZ4tmlTXMW/fEs/4Oumn74Lj6pLRTmOgnaLp8KnYNca/rAHnh0IJQc9H+vbU+nDd+LshJ4Mt0ZdTRhXs0Lvc+8Hr6bCTd8679Iem0q190mc5kg1BYMvDYTvV/p5yTgYiBANioT64JZmav6zN7fv7WSx+duYHdekWsAqGz3/iL/jdXvjgtzQOLgJzcGHwgAWnb0nwFcIZi2+PhEGHkHvHM9rHsXjhvrvl/GUlg1DUbcGnwgAJvMZcLGa6K6tyu/FpE3gU/DUiLTqLld4G+fsYrFW7Lp3b4lWXlFZOUVsSe/mG82ZVNaXvWBsqxc2bbvIOOH9uK47m0YkNqWCa8tYXv1NA24zNYF9/QNWg7z/w5DrqlbpULVFj/wUljwCHz+NzhmtH9fgCp8NBladnYWhTemEfE66ay6voBNe2yCHpzzvV/bfnFZOVO/2cbfPvie17/ZwtKt+zhYXEppuTImbgELmt3EpuZXsqDZTYyJW0BpmfKXsQMYf1IvBqS25fZz+5OcWPXCGXAB94DpG+oxoubMu5y298rqklgtLh5++kfYs955gqluzduQ8a1z3Oat615eY8LAa9bSPKr2GezEWePANBH7DhQzbfE2Z4lFFwKsmnIOrZonHFptbsp9d3N7yQu08A3v7CF7eCDxBdonNgMuOPTZQHn6/Zqe9u9wLrjlLs1L9RlRE8rEav1HQ9fj4Yv7YeA4p/kIoPggfHKX896gK+teVmPCxGszkd3GNAFufQFHd2nNq19vZtaKTIpKy2meEMeo8vncnjCd7rKH7dqRB0svY2mbs2mdlHj4YMUHmRQ/leTSqpOkWkgxtyf+G7inyna/PP3V5WbAq6NB4iE+Hsoq9SeEIj1yqNri4+LgjDudeQrLX3eGnQJ8/Tjsz4RL/ul9KKkxDcjraKKLgc9UNdf3OgUYqaqzwlw+PzaaKDh17ewFiPOld0hOjOeSE1O5+idp5C5643ACOJ9iTSCr2+mkdu4I+zbDvi2Qv7OGUokz29arfVucQFCwD6562/mOxjyipiLNRM42uGm5k0LiiSFw9DnOTGJjIiBUo4nuVtV3Kl6oao6I3A00eDAw3rl19t4xczXg3ImXlJWTlVfEzv2F3Pv+d359AeUKbZIS+PL2M2jbIhGK8uD7B6FaArhmUkrqzrlQ0NPJq9/3LEhJcxZFL9jrX7AWNaRlri77R3h1DBTnw//MhtQTnRnBjeniX52I83Tw6oXwcD9nxBNAz5MjWy5jauA1GLh1NHv9rImQh+as97vAF5SUMXHGSu77zzqyDxRR+cFwTNwCv+afjUU9aLtsLWycC1u/cTJluhL/MfntevtPkkLg4B6YOQFG3V9zvv6sH+C1MU4+nqvfg27HB1X/iMrb4Qx5Laz0BPTZX5w8Ro05kJkmy+sFfYmI/AN4Cqcj+UZgadhKZWpVU/PPjtwC5q3PCpjUraRMOeuYznRpk0SXNkl0bducz6Y/xR/Lqnb2Ppb4tJNZ4VOcHDqn3AArpjoJ06pz68B165j96R+dZp4v/wEbP4Vz/8/paK2ewmHXWnjNN5P46vehy7HB/yNF0tx7nSGvlXldwcyYCPDaZ9ASuBOomA76MfBXVT0Q+FPhYX0G7u37zRLiGHFUB7bnFPL9TmdpxjiBC8X/bn9pm7P5arIvzVR+FmxZQMnM35JY5j8LtygxheY3LYLWXZ0NoUqJsGstvHujk5q57zlw5Bmw8CknaLTqDEX5Tlrnq9+Djn1rP15jMyUFl0n7BN1fYkyIhCQdRWNiwSBwTn+AU47owMh+nRjZrzP7v53KgKV3VunsLdRmZPb5GUd2TIbNXzlj4mvkcvEKVUqE8jL49nknX79f85PAOX9xZhZHo0cGBJjIFkSKC2NCKCQdyCLyCXCpqub4XrcDpqlqLRm5TFBquchuzylgji+nv1v7/nvlI3hzwjBn5/Iy2PCoX2dvkhRz5OY3YXtr6DXMGfOeNgLeutp94lag5p+QDMOMh2H/C1895rSxV6Gw6LnoDQZn3uX+BFXfIbDGhInXPoOOFYEAQFX3iYitgRxKq6ZTOvtGEsp8k7pyt1E6+0ay8wqZUfwTPv5uJyszcgG4OOEr/hpftX3/74nPc1nctzDtDcjeCHs3+S+EcojApM0QX+n0n3l35C5eeQGGoUZznv5QTmQzpgF4DQblItJLVbcCiEga7g2ipo4OfngXLcqqzu5NKCsk5eNb+YmmcX5iGSntymkVX0JC/k6Eqp2TzaSU4boEsvtDh6OcNMrLXnPG5lfXtkfVQACRvXjFap5+SypnoojXYPAnYIGIzPO9Pg2YEJ4iNU1JBe53x82klGPSepKU1AISmjt36yumuu4rCNyw6PCGLgOCu9uP1MXLmlSMiTiv6Sg+EpF0nACwApgNBMj7e5iInAs8BsQDL6jqA9Xe/w1wA1AG5AMTVHVtUDWIAZr1A6UaTzPxz7mTWd6RHtdWm9v33/ne7qSjpakiWsppTAzz2oF8HXAz0AMnGAwDFlJ1Gczqn4nHmZdwNpABLBaRd6td7N9Q1Wd9+48B/gF4WGcwRqiyf/4zNP9iCiXEowrNKwWEg9qMF5pdxZTqnwvmTjpamiqipZzGxCivKaxvBoYCW1T1p8BgwGXmURUnARtVdZOqFgPTgCrrEarq/kovW9KE+iE0N5MdT51Pm8/vYFF5fx46aip/0t+QUd6RchUyyjtyl05g0AUurXHHX+aM62/bExDnb1v60BhTD177DApVtVBEEJHmqvq9iLgkm68iFajclpEB+CVnEZEbgNuAZtTwpBHVqg0Zzet9FrJmBm3Livln2xs5538mc0/HVsxafiyXzzmz1qRygN1JG2NCymswyPBlKp0FfCIi+6h92Utx2ea2jvJTwFMiciXwZ+BqvwOJTMDXYd2rV5StqeMyZLT1qpfZop1ZfupUfnXGacTFOf9UtaZxNsaYMPHagXyx78cpIvI50Bb4qJaPZQA9K73uQc0BZBrwTIDvfx54HpwZyF7K3Fi4DRkF6JAUx9izTo9AiYwxxl/QmUdVdV7tewGwGOgrIn2ATGA8UGWJJxHpq6obfC8vADYQS4rySS6oPrPW0aJoVwMXxhhjAgtbGmpVLRWR3wFzcIaWvqSq34nIvcASVX0X+J2InAWUAPtwaSKKSns3UbzwOXTZ6zQPsMv28g5E+ZQqY0wMCeuaBKr6AfBBtW13Vfr55nB+f1i55RFq0YHyRc8hGz4mjjjeLzuZrdKd6+TdQ6kjoIYho8YYEyG2QE1duHQK68wJCEoObXm99GLWp45jwoXDYc8B7nqnK7foNLpLNtu1A48ynhFuQ0aNMSZCLBjUgVunsKDs1VZc2/4Vfn/eQG7q2xERYVDPFOC33oeMGmNMBFgwqINAeYRS5ADv3PjTQ0NFK9iQUWNMY2fBoBq35STPHdCVVRm5LN68l8Wb93JfeQd6xO3x++z28g70iHObXmGMMY2bBYNKqi8nmZlTwG3TV/D76VDmm93Qt3MrpnMmt+q/qyzba53Cxpho1jSCgcdlGh+as77KusIA5QqtmifwyOWDGNK7He2T49n3+J/I25dMfnkyXWWfdQobY6Je7AeD6gu4525zXkOVgLC/sCTgusIHiko5+9guzouFT9MuZzWL0//OLd8daZ3CxpiYEPvBYO69VVM9g/N67r2HgsHCH7P5w1srAx6ie0qy88O+LfDZX6DvKIZeeB1fjbb+AWNMbPCawjp6BVpHNzeDwpIy7nt/LVe+8A3NEuK45ay+JCfGV9ktOTGeiaP6gSq8fytIHFzwMFU6DIwxJsrF/pNBgPV1i1t2Y8yTC/hhVz6/GNabO87vT4tmCaR1aOk3mmjs4FSnuenHuXDeg5DS0+WLjDEmesV+MDjzrqqzhXHyaGfmlVHaLIdXrj2Fkf06H3rPdU7AgWz4aDKkpsPQ6xqo4MYY03BivploVtlwJpdcV2UFsRdLz6VHXDYft3+IkT3iaz/Ix3+CwlwY8wTEedjfGGOiTMw/GTw0Zz2ZxT9hBj+psn1dcjoP7/07vHIB/M8saN3V/QAb58LKN+G0idDl2AYosTHGNLyYfzLYHmC46My8Y+HnMyBnK7x8HuT49ytQfMDpNO7QF079Q5hLaowxkRPzweDQsFC37X1Ohf+Z7fQJvHweZP9YdafP/wY5W2D0Y5CY1AClNcaYyIj5YDBxVL/Aw0UBeg6Fa96DkoPw8vmw4FF4ZABMSYGFT0LaqZA2PAIlN8aYhhPzwWDs4FTuv2QgqSnJCJCaksz9lwysOmKo2wlwzQdOQPj0bt9QVF8yoowlzrBSY4yJYTHfgQweU0h37g/NWkLR/qrbS6vOVjbGmFgU808GQclzX6cg4CxmY4yJERYMKmsbYIn6QNuNMSZGWDCo7My7ILHa6KPEZGe7McbEMAsGlR1/GYx+HNr2BMT5e/Tj1l9gjIl5TaIDOSjHX2YXf2NMkyOqGukyBEVEsoAtdfx4R8B/8eLoFmt1irX6QOzVKdbqA7FXJ7f69FbVToE+EHXBoD5EZImqpke6HKEUa3WKtfpA7NUp1uoDsVenutTH+gyMMcZYMDDGGNP0gsHzkS5AGMRanWKtPhB7dYq1+kDs1Sno+jSpPgNjjDHumtqTgTHGGBdNJhiIyLkisl5ENorI5EiXJxREZLOIrBaRFSKyJNLlCZaIvCQiu0VkTaVt7UXkExHZ4Pu7XSTLGKwAdZoiIpm+87RCRM6PZBmDISI9ReRzEVknIt+JyM2+7VF5nmqoTzSfoyQR+VZEVvrqdI9vex8RWeQ7R/8WkWY1HqcpNBOJSDzwA3A2kAEsBq5Q1bURLVg9ichmIF1Vo3J8tIicBuQDr6nqAN+2B4G9qvqAL2i3U9VJkSxnMALUaQqQr6p/j2TZ6kJEugHdVHWZiLQGlgJjgWuIwvNUQ30uI3rPkQAtVTVfRBKBBcDNwG3ATFWdJiLPAitV9ZlAx2kqTwYnARtVdZOqFgPTgIsiXKYmT1XnA3urbb4IeNX386s4v6hRI0Cdopaq7lDVZb6f84B1QCpRep5qqE/UUke+72Wi748CZwAzfNtrPUdNJRikApUXOc4gyv8D+CjwsYgsFZEJkS5MiHRR1R3g/OICnSNcnlD5nYis8jUjRUWTSnUikgYMBhYRA+epWn0gis+RiMSLyApgN/AJ8COQo6qlvl1qveY1lWAgLttioX1suKqeCJwH3OBrojCNzzPAkcAgYAfwcGSLEzwRaQW8Ddyiqvtr27+xc6lPVJ8jVS1T1UFAD5yWkGPcdqvpGE0lGGQAPSu97gFsj1BZQkZVt/v+3g28g/OfINrt8rXrVrTv7o5weepNVXf5flnLgX8SZefJ1w79NjBVVWf6NkfteXKrT7SfowqqmgN8AQwDUkSkIhlprde8phIMFgN9fb3rzYDxwLsRLlO9iEhLXwcYItISOAdYU/OnosK7wNW+n68GZkewLCFRcdH0uZgoOk++zskXgXWq+o9Kb0XleQpUnyg/R51EJMX3czJwFk5fyOfAON9utZ6jJjGaCMA3VOxRIB54SVX/GuEi1YuIHIHzNABOKvI3oq1OIvImMBInw+Iu4G5gFjAd6AVsBS5V1ajpkA1Qp5E4zQ8KbAaur2hvb+xEZATwJbAaKPdt/iNOO3vUnaca6nMF0XuOjsfpII7HucGfrqr3+q4R04D2wHLgKlUtCnicphIMjDHGBNZUmomMMcbUwIKBMcYYCwbGGGMsGBhjjMGCgTHGGCwYGNOgRGSkiLwf6XIYU50FA2OMMRYMjHEjIlf5csSvEJHnfInA8kXkYRFZJiJzRaSTb99BIvKNL8nZOxVJzkTkKBH51JdnfpmIHOk7fCsRmSEi34vIVN+sWGMiyoKBMdWIyDHA5TiJAAcBZcDPgZbAMl9ywHk4s4sBXgMmqerxODNbK7ZPBZ5S1ROAn+AkQAMnU+YtwLHAEcDwsFfKmFok1L6LMU3OmcAQYLHvpj0ZJxFbOfBv3z7/AmaKSFsgRVXn+ba/CrzlyxuVqqrvAKhqIYDveN+qaobv9QogDWdBEmMixoKBMf4EeFVV76iyUeTOavvVlMulpqafyvlhyrDfQ9MIWDORMf7mAuNEpDMcWu+3N87vS0UWyCuBBaqaC+wTkVN9238BzPPlyM8QkbG+YzQXkRYNWgtjgmB3JMZUo6prReTPOKvIxQElwA3AAeA4EVkK5OL0K4CTHvhZ38V+E3Ctb/svgOdE5F7fMS5twGoYExTLWmqMRyKSr6qtIl0OY8LBmomMMcbYk4Exxhh7MjDGGIMFA2OMMVgwMMYYgwUDY4wxWDAwxhiDBQNjjDHA/wO8bY+IbUaYoAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"\"\"\n",
    "Graphs\n",
    "1. loss vs. iterations\n",
    "2. Train/Validation accuracy along epoch\n",
    "\"\"\"\n",
    "plt.subplot(2,1,1)\n",
    "plt.plot(loss_history, 'o')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('loss')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(train_acc_history, '-o')\n",
    "plt.plot(val_acc_history, '-o')\n",
    "plt.legend(['train', 'val'], loc='upper left')\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('accuracy')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}