grapevine-leaves-livedemo-step2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "lyric-upper",
   "metadata": {
    "papermill": {
     "duration": 0.011307,
     "end_time": "2021-05-08T11:34:11.883685",
     "exception": false,
     "start_time": "2021-05-08T11:34:11.872378",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Pytorch Grapevine Leaf Classifier\n",
    "\n",
    "Taken with very little modification (just path and use pyplot) from Maxim Vlah\n",
    "\n",
    "Dataset and code (c) 2021 Maxim Vlah, Licensed CC-BY-SA-NC 4.0 and used with permission.\n",
    "\n",
    "Dataset: https://www.kaggle.com/maximvlah/grapevine-leaves\n",
    "\n",
    "Original notebook: https://www.kaggle.com/maximvlah/pytorch-resnet18-94-f1\n",
    "\n",
    "**Important notice from Thomas**: I picked this notebook because I liked the dataset and the code was reasonable and protypical of a lot of code I see in my work. Creating, curating and releasing a useful dataset (from your own images, too) is a great accomplishment, and a demonstration of the dataset with simple demo code is cool, too. I do not think of this code as primarily _unoptimized_ but rather as primarily _worth optimizing_. Thank you Maxim!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "theoretical-carbon",
   "metadata": {
    "_kg_hide-input": false,
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:11.916438Z",
     "iopub.status.busy": "2021-05-08T11:34:11.915919Z",
     "iopub.status.idle": "2021-05-08T11:34:16.622168Z",
     "shell.execute_reply": "2021-05-08T11:34:16.621538Z"
    },
    "papermill": {
     "duration": 4.728604,
     "end_time": "2021-05-08T11:34:16.622318",
     "exception": false,
     "start_time": "2021-05-08T11:34:11.893714",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "'''Load librarires'''\n",
    "import pickle\n",
    "import time\n",
    "import random\n",
    "import ntpath\n",
    "import glob\n",
    "import os\n",
    "from copy import deepcopy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "from tqdm import tqdm\n",
    "from pathlib import Path\n",
    "from collections import defaultdict\n",
    "\n",
    "from matplotlib import pyplot\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "import torchvision\n",
    "from torchvision import models, transforms, utils\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "composite-bracelet",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:16.655371Z",
     "iopub.status.busy": "2021-05-08T11:34:16.654741Z",
     "iopub.status.idle": "2021-05-08T11:34:19.158662Z",
     "shell.execute_reply": "2021-05-08T11:34:19.159326Z"
    },
    "papermill": {
     "duration": 2.526336,
     "end_time": "2021-05-08T11:34:19.159477",
     "exception": false,
     "start_time": "2021-05-08T11:34:16.633141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are using -> cuda:0\n"
     ]
    }
   ],
   "source": [
    "class CFG:\n",
    "\n",
    "    '''Store all hyperparameters here.''' \n",
    "\n",
    "    SEED = 420\n",
    "    TEST_SIZE = 0.2\n",
    "    VAL_SIZE = 0.25\n",
    "    CLASSES = None #Need to update manually\n",
    "    OUTPUT_FEATURES = None #Need to update manually\n",
    "    \n",
    "    #transforms\n",
    "    TRAIN_TRANSFORMS = transforms.Compose([\n",
    "        #Rotate the image by given angle.\n",
    "        transforms.RandomRotation(5),\n",
    "        #Crop the given PIL Image to random size and aspect ratio.\n",
    "        transforms.Resize((224,224)),\n",
    "        #Horizontally flip the given PIL Image randomly with a given probability.\n",
    "        transforms.RandomHorizontalFlip(p = 0.2),\n",
    "        #Convert a PIL Image or numpy.ndarray to tensor.\n",
    "        transforms.ToTensor(),\n",
    "        #Normalize a tensor image with mean and standard deviation\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                             std=[0.229, 0.224, 0.225])\n",
    "    ])\n",
    "\n",
    "    VAL_TRANSFORMS = transforms.Compose([\n",
    "        #Rotate the image by given angle.\n",
    "        transforms.RandomRotation(5),\n",
    "        #Crop the given PIL Image to random size and aspect ratio.\n",
    "        transforms.Resize((224,224)),\n",
    "        #Horizontally flip the given PIL Image randomly with a given probability.\n",
    "        transforms.RandomHorizontalFlip(p = 0.2),\n",
    "        #Convert a PIL Image or numpy.ndarray to tensor.\n",
    "        transforms.ToTensor(),\n",
    "        #Normalize a tensor image with mean and standard deviation\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                             std=[0.229, 0.224, 0.225])\n",
    "    ])\n",
    "    \n",
    "    TEST_TRANSFORMS = transforms.Compose([\n",
    "        #Crop the given PIL Image to random size and aspect ratio.\n",
    "        transforms.Resize((224,224)),\n",
    "        #Convert a PIL Image or numpy.ndarray to tensor.\n",
    "        transforms.ToTensor(),\n",
    "        #Normalize a tensor image with mean and standard deviation\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                             std=[0.229, 0.224, 0.225])\n",
    "    ])\n",
    "    \n",
    "    #models\n",
    "    \n",
    "    # MODEL1 = {\n",
    "    #     'name': 'custom_cnn',\n",
    "    #     'transfer': False,\n",
    "    #     'architecture': nn.Sequential(\n",
    "    #       nn.Conv2d(1,20,5),\n",
    "    #       nn.ReLU(),\n",
    "    #       nn.Conv2d(20,64,5),\n",
    "    #       nn.ReLU()\n",
    "    #     ),\n",
    "    #     'criterion': nn.CrossEntropyLoss(),\n",
    "    #     'optimizer': optim.SGD,\n",
    "    #     'momentum': 0.9,\n",
    "    #     'lr': 0.003,\n",
    "    #     'history': None\n",
    "    # }\n",
    "\n",
    "    MODEL2 = {\n",
    "        'name': 'resnet18',\n",
    "        'transfer': True,\n",
    "        'architecture': models.resnet18(pretrained=True), # ResNet18\n",
    "        'criterion': nn.CrossEntropyLoss(),\n",
    "        'optimizer': optim.SGD,\n",
    "        'momentum': 0.9,\n",
    "        'lr': 0.002,\n",
    "        'history': None\n",
    "    }\n",
    "\n",
    "    # MODELS =[MODEL1,MODEL2]\n",
    "\n",
    "    BATCH_SIZE = 64\n",
    "    EPOCHS = 120\n",
    "    \n",
    "    DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print('You are using ->', DEVICE)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "swedish-girlfriend",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.187740Z",
     "iopub.status.busy": "2021-05-08T11:34:19.187112Z",
     "iopub.status.idle": "2021-05-08T11:34:19.192431Z",
     "shell.execute_reply": "2021-05-08T11:34:19.191996Z"
    },
    "papermill": {
     "duration": 0.020998,
     "end_time": "2021-05-08T11:34:19.192538",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.171540",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed):\n",
    "    '''Make the results reproducible'''\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = True \n",
    "\n",
    "seed_everything(CFG.SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "engaging-ranch",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.219882Z",
     "iopub.status.busy": "2021-05-08T11:34:19.219376Z",
     "iopub.status.idle": "2021-05-08T11:34:19.347056Z",
     "shell.execute_reply": "2021-05-08T11:34:19.347469Z"
    },
    "papermill": {
     "duration": 0.143883,
     "end_time": "2021-05-08T11:34:19.347610",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.203727",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>./wine-data/Muller Thurgau/IMG_2087.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>./wine-data/Muller Thurgau/IMG_2006.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>./wine-data/Muller Thurgau/IMG_2046.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>./wine-data/Muller Thurgau/IMG_2111.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>./wine-data/Muller Thurgau/IMG_2036.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       path  label\n",
       "0  ./wine-data/Muller Thurgau/IMG_2087.JPEG      5\n",
       "1  ./wine-data/Muller Thurgau/IMG_2006.JPEG      5\n",
       "2  ./wine-data/Muller Thurgau/IMG_2046.JPEG      5\n",
       "3  ./wine-data/Muller Thurgau/IMG_2111.JPEG      5\n",
       "4  ./wine-data/Muller Thurgau/IMG_2036.JPEG      5"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''Store image paths and their labels in pandas dataframe. Will be used to create pytorch datasets. '''\n",
    "\n",
    "paths = glob.glob('./wine-data//*/*' )\n",
    "meta = pd.DataFrame([(path, ntpath.basename(ntpath.dirname(path))) for path in paths], columns = ['path','label'])\n",
    "#get class mappings\n",
    "classes = dict(enumerate(meta.label.astype('category').cat.categories))\n",
    "CFG.CLASSES = classes\n",
    "CFG.OUTPUT_FEATURES = len(CFG.CLASSES)\n",
    "\n",
    "meta.label = meta.label.astype('category').cat.codes\n",
    "meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "valid-scanning",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.375694Z",
     "iopub.status.busy": "2021-05-08T11:34:19.374979Z",
     "iopub.status.idle": "2021-05-08T11:34:19.378018Z",
     "shell.execute_reply": "2021-05-08T11:34:19.378390Z"
    },
    "papermill": {
     "duration": 0.019042,
     "end_time": "2021-05-08T11:34:19.378513",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.359471",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'Auxerrois',\n",
       " 1: 'Cabernet Franc',\n",
       " 2: 'Cabernet Sauvignon',\n",
       " 3: 'Chardonnay',\n",
       " 4: 'Merlot',\n",
       " 5: 'Muller Thurgau',\n",
       " 6: 'Pinot Noir',\n",
       " 7: 'Riesling',\n",
       " 8: 'Sauvignon Blanc',\n",
       " 9: 'Syrah',\n",
       " 10: 'Tempranillo'}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "analyzed-student",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.409007Z",
     "iopub.status.busy": "2021-05-08T11:34:19.408493Z",
     "iopub.status.idle": "2021-05-08T11:34:19.418535Z",
     "shell.execute_reply": "2021-05-08T11:34:19.418993Z"
    },
    "papermill": {
     "duration": 0.028516,
     "end_time": "2021-05-08T11:34:19.419140",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.390624",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train length -> 605\n",
      "Val length -> 202\n",
      "Test length -> 202\n"
     ]
    }
   ],
   "source": [
    "'''Split data into train, validation and test sets'''\n",
    "\n",
    "X = list(meta.path)\n",
    "y = list(meta.label)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                                                    y, \n",
    "                                                    test_size=CFG.TEST_SIZE, \n",
    "                                                    random_state=CFG.SEED, \n",
    "                                                    stratify=y) #stratified split\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train,\n",
    "                                                  y_train,\n",
    "                                                  test_size=CFG.VAL_SIZE,\n",
    "                                                  random_state=CFG.SEED,\n",
    "                                                  stratify=y_train) #stratified split\n",
    "\n",
    "print(f'Train length -> {len(X_train)}')\n",
    "print(f'Val length -> {len(X_val)}')\n",
    "print(f'Test length -> {len(X_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "elder-turkish",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.453050Z",
     "iopub.status.busy": "2021-05-08T11:34:19.452507Z",
     "iopub.status.idle": "2021-05-08T11:34:19.455373Z",
     "shell.execute_reply": "2021-05-08T11:34:19.454956Z"
    },
    "papermill": {
     "duration": 0.023129,
     "end_time": "2021-05-08T11:34:19.455469",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.432340",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "'''Custom pytorch dataset implementation.'''\n",
    "class LeafDataset(Dataset):\n",
    "    def __init__(self,X,y, transform=None):\n",
    "        self.X = X\n",
    "        self.y = torch.tensor(y, dtype=torch.long)\n",
    "        self.transform = transform\n",
    "\n",
    "        assert len(self.X) == len(self.y), f'X and y have different lengths -> {len(self.X)} != {len(self.y)} '\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.X)\n",
    "\n",
    "    def __getitem__(self,idx):\n",
    "        img_path = self.X[idx]\n",
    "        img = Image.open(img_path)\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "        label = self.y[idx]\n",
    "        return (img,label) \n",
    "\n",
    "    def show_img(self,idx):\n",
    "        '''Plot image'''\n",
    "        img,label = self.__getitem__(idx)\n",
    "        img = img.numpy().transpose((1, 2, 0))\n",
    "        pyplot.figure(figsize=(16, 8))\n",
    "        pyplot.axis('off')\n",
    "        pyplot.imshow(img)\n",
    "        pyplot.title(CFG.CLASSES[int(label)]) #using CFG.CLASSES dict\n",
    "        pyplot.pause(0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "personalized-inside",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.488750Z",
     "iopub.status.busy": "2021-05-08T11:34:19.488073Z",
     "iopub.status.idle": "2021-05-08T11:34:19.490869Z",
     "shell.execute_reply": "2021-05-08T11:34:19.490439Z"
    },
    "papermill": {
     "duration": 0.021416,
     "end_time": "2021-05-08T11:34:19.490986",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.469570",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "'''Instantiate pytorch train, validation and test sets'''\n",
    "TRAIN = LeafDataset(X_train,y_train, CFG.TRAIN_TRANSFORMS)\n",
    "VAL = LeafDataset(X_val,y_val, CFG.VAL_TRANSFORMS)\n",
    "TEST = LeafDataset(X_test,y_test, CFG.TEST_TRANSFORMS)\n",
    "\n",
    "'''Instantiate Dataloaders'''\n",
    "TRAIN_LOADER = DataLoader(TRAIN,CFG.BATCH_SIZE)\n",
    "VAL_LOADER = DataLoader(VAL,CFG.BATCH_SIZE)\n",
    "TEST_LOADER = DataLoader(TEST,CFG.BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "sorted-teens",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.525117Z",
     "iopub.status.busy": "2021-05-08T11:34:19.524065Z",
     "iopub.status.idle": "2021-05-08T11:34:19.562116Z",
     "shell.execute_reply": "2021-05-08T11:34:19.561647Z"
    },
    "papermill": {
     "duration": 0.057173,
     "end_time": "2021-05-08T11:34:19.562226",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.505053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    '''\n",
    "    ========================\n",
    "          NEURAL NET\n",
    "    ========================\n",
    "    \n",
    "    Args:\n",
    "        model_dict(dict): configuration dict containing the model architecture\n",
    "        output_features(int): length of output tensor; for classification equals to number of classes\n",
    "    '''\n",
    "    def __init__(self, model_dict, output_features):\n",
    "        super().__init__()\n",
    "        self.__dict__.update(model_dict) #unpack model dict from CFG into this class\n",
    "        \n",
    "        if self.transfer:\n",
    "            model = self.architecture\n",
    "            num_ftrs = model.fc.in_features\n",
    "            model.fc = nn.Linear(num_ftrs, output_features)\n",
    "            self.model = model\n",
    "        else:\n",
    "            self.model = self.architecture\n",
    "        \n",
    "        self.output_features = output_features\n",
    "        #optimizer\n",
    "        self.optimizer = self.optimizer(self.model.parameters(),self.lr)\n",
    "        #path where to save model\n",
    "        self.save_path = 'models'\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "\n",
    "    def fit(self,\n",
    "            train_loader,\n",
    "            val_loader,\n",
    "            epochs = 5,\n",
    "            batch_size = 32,\n",
    "            device = 'cpu'):\n",
    "        '''\n",
    "        =============================\n",
    "            OPTIMIZATION LOOP\n",
    "        =============================\n",
    "\n",
    "        Args:\n",
    "            train_loader(torch dataloader)\n",
    "            val_loader(torch dataloader)\n",
    "            epochs(int)\n",
    "            batch_size(int)\n",
    "            device(str)\n",
    "\n",
    "            \n",
    "        Output style inspired by the skorch fit() method\n",
    "\n",
    "        '''\n",
    "        #may be changed if lrscheduler is used???\n",
    "        lr = deepcopy(self.lr)\n",
    "\n",
    "        #get model training history\n",
    "        history = self.history\n",
    "        if history == None:\n",
    "            history = defaultdict(list)\n",
    "        else:\n",
    "            pass\n",
    "        #get train and val sizes\n",
    "        train_size = len(train_loader.dataset)\n",
    "        val_size = len(val_loader.dataset)\n",
    "        #stuff for printing epoch metrics as a beautiful table\n",
    "        headers = ['epoch','train_loss','val_loss','val_acc','cp','lr','dur']\n",
    "        template = '{:<10} {:<10} {:<10} {:<10} {:<10} {:<10} {:<10}'\n",
    "        print(template.format(*headers))\n",
    "        print(template.replace(':', ':-').format('','','','','','',''))\n",
    "        cyan = \"\\033[96m{:<10}\\033[00m\" #cyan\n",
    "        purple = \"\\033[95m{:<10}\\033[00m\" #purple\n",
    "        green = \"\\033[92m{:<10}\\033[00m\" #green\n",
    "        white = \"\\033[0m{:<10}\\033[0m\" #white\n",
    "        #set model into train mode\n",
    "        self.model.train()\n",
    "        #send model to device\n",
    "        self.model.to(device)\n",
    "        #training loop\n",
    "        for epoch in range(epochs):\n",
    "            start_time = time.time()\n",
    "            train_loss = 0\n",
    "            val_loss, val_acc = 0, 0\n",
    "            #optimization  loop\n",
    "            for (X,y) in train_loader:\n",
    "                #Send training data to device\n",
    "                X,y = X.to(device), y.to(device)\n",
    "                #Forward propagation\n",
    "                pred = self.model(X)\n",
    "                loss = self.criterion(pred,y)\n",
    "                #Backpropagation\n",
    "                self.optimizer.zero_grad()\n",
    "                loss.backward()\n",
    "                self.optimizer.step()\n",
    "                #update loss\n",
    "                train_loss = loss.item()\n",
    "            #validation loop\n",
    "            with torch.no_grad():\n",
    "                for X, y in val_loader:\n",
    "                    X,y = X.to(device),y.to(device)\n",
    "                    pred = self.model(X)\n",
    "                    val_loss = self.criterion(pred,y).item()\n",
    "                    val_acc += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
    "            #calculate validation accuracy after the epoch\n",
    "            val_acc /= val_size\n",
    "            #append epoch results\n",
    "            history['epoch'].append(epoch+1)\n",
    "            history['train_loss'].append(train_loss)\n",
    "            history['val_loss'].append(val_loss)\n",
    "            history['val_acc'].append(val_acc)\n",
    "            \n",
    "            #colorize epoch's output if it improves\n",
    "            colortemp = template.split(' ')\n",
    "            # colorize train loss if it decreases\n",
    "            if history['train_loss'][-1] == min(history['train_loss']):\n",
    "                colortemp[1] = cyan\n",
    "            else:\n",
    "                colortemp[1] = white\n",
    "            #colorize validation loss if it decreases\n",
    "            if history['val_loss'][-1] == min(history['val_loss']):\n",
    "                colortemp[2] = purple\n",
    "            else:\n",
    "                colortemp[2] = white\n",
    "            # colorize validation accuracy & save best weights if it increases\n",
    "            if history['val_acc'][-1] == max(history['val_acc']):\n",
    "                #colorize       \n",
    "                colortemp[3] = green\n",
    "                #checkpoint\n",
    "                cp = '+'\n",
    "                if not os.path.exists(self.save_path):\n",
    "                    os.mkdir(self.save_path)\n",
    "                torch.save(self.model.state_dict(), Path(self.save_path,f'best_{self.name}.pth'))\n",
    "            else:\n",
    "                colortemp[3] = white\n",
    "                cp = '-'\n",
    "            colortemp = ' '.join(colortemp)\n",
    "\n",
    "            #calculate epoch duration (in seconds)\n",
    "            end_time = time.time()\n",
    "            dur = end_time - start_time\n",
    "            #append the rest of epoch results\n",
    "            history['cp'].append(cp)\n",
    "            history['lr'].append(lr)\n",
    "            history['dur'].append(dur)\n",
    "            #display the epoch results\n",
    "            print(colortemp.format(*f'{epoch+1}/{epochs} {train_loss:.4f} {val_loss:.4f} {val_acc:.2f} {cp} {lr} {dur:.2f}'.split(' ')))\n",
    "        #update epoch number of the entire training history\n",
    "        history['epoch'] = [e+1 for e in range(len(history['epoch']))]\n",
    "        #update model's training history\n",
    "        self.history = history\n",
    "        #save training history as csv\n",
    "        self.save_history()\n",
    "\n",
    "    def eval_model(self,dataloader,avg=None,device ='cpu'):\n",
    "        '''\n",
    "        ==================================\n",
    "           ACCURACY PRECISION RECALL F1\n",
    "        ==================================\n",
    "        '''\n",
    "        labels = [l for l in range(self.output_features)]\n",
    "        loader_size = len(dataloader)\n",
    "        dataset_size = len(dataloader.dataset)\n",
    "\n",
    "        acc =0\n",
    "        precision = 0\n",
    "        recall = 0\n",
    "        f1 = 0\n",
    "\n",
    "        #set model to evaluation mode\n",
    "        self.model.eval()\n",
    "        #model to device, default cpu\n",
    "        self.model.to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            for X, y in dataloader:\n",
    "                pred = self.model(X)\n",
    "                #accuracy\n",
    "                acc += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
    "\n",
    "                pred = pred.argmax(1)\n",
    "                pred,y = list(pred), list(y)\n",
    "                #precision\n",
    "                p = precision_score(y, pred, labels = labels, zero_division = 1, average = avg)\n",
    "                precision+=p\n",
    "                #recall\n",
    "                r = recall_score(y, pred, labels = labels, zero_division = 1,  average = avg)\n",
    "                recall+=r\n",
    "                #f1 score\n",
    "                f = f1_score(y, pred, labels = labels, zero_division = 1,  average = avg)\n",
    "                f1 += f\n",
    "\n",
    "        acc /= dataset_size\n",
    "        precision /= loader_size\n",
    "        recall /= loader_size\n",
    "        f1 /= loader_size\n",
    "\n",
    "        print(f\" Accuracy: {(100*acc):>0.1f}%\")\n",
    "        print(f\"Precision: {(100*np.mean(precision)):>0.1f}%\")\n",
    "        print(f\"   Recall: {(100*np.mean(recall)):>0.1f}%\")\n",
    "        print(f\" F1 Score: {(100*np.mean(f1)):>0.1f}%\")\n",
    "\n",
    "        \n",
    "    def plot_loss_history(self):\n",
    "        '''\n",
    "        Plot loss history\n",
    "        '''\n",
    "        assert self.history != None, 'No history to plot -> the model has not been trained yet!'\n",
    "        \n",
    "        df = pd.DataFrame(self.history)\n",
    "        fig = pyplot.figure()\n",
    "        pyplot.plot(df.epoch, df.train_loss, label='train loss')\n",
    "        pyplot.plot(df.epoch, df.val_loss, label='val loss')\n",
    "        pyplot.title('Loss History')\n",
    "        pyplot.legend()\n",
    "        pyplot.show()\n",
    "\n",
    "    def save_history(self):\n",
    "        '''Save model's training history'''\n",
    "        assert self.history != None, 'No history to save -> the model has not been trained yet!'\n",
    "        #save as csv\n",
    "        pd.DataFrame(self.history).to_csv(Path(f'models/{self.name}_history.csv')) \n",
    "        #save as pickle file\n",
    "        with open(Path(f'models/{self.name}_history.pkl'), 'wb') as f:\n",
    "            pickle.dump(self.history, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "    def save_model(self):\n",
    "        torch.save(self.model.state_dict(), Path(self.save_path,f'latest_{self.name}.pth'))\n",
    "\n",
    "    def load_model(self,path = 'models'):\n",
    "        '''Load model'''\n",
    "        try:\n",
    "            #load model weights\n",
    "            p = Path(path,f'best_{self.name}.pth')\n",
    "            self.model.load_state_dict(torch.load(p))\n",
    "            #load model training history\n",
    "            with open(Path(path,f'{self.name}_history.pkl'), 'rb') as h: \n",
    "                self.history = pickle.load(h)\n",
    "        except:\n",
    "            print('No model to load!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "norman-national",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.600420Z",
     "iopub.status.busy": "2021-05-08T11:34:19.599609Z",
     "iopub.status.idle": "2021-05-08T11:34:19.602614Z",
     "shell.execute_reply": "2021-05-08T11:34:19.602163Z"
    },
    "papermill": {
     "duration": 0.023493,
     "end_time": "2021-05-08T11:34:19.602721",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.579228",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#instantiate model and send to device\n",
    "Resnet18 = Net(CFG.MODEL2,CFG.OUTPUT_FEATURES)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "wooden-baptist",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.640153Z",
     "iopub.status.busy": "2021-05-08T11:34:19.639534Z",
     "iopub.status.idle": "2021-05-08T13:39:36.915991Z",
     "shell.execute_reply": "2021-05-08T13:39:36.916582Z"
    },
    "papermill": {
     "duration": 7517.298847,
     "end_time": "2021-05-08T13:39:36.916788",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.617941",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      train_loss val_loss   val_acc    cp         lr         dur       \n",
      "---------- ---------- ---------- ---------- ---------- ---------- ----------\n",
      "1/3        \u001b[96m2.4533    \u001b[00m \u001b[95m2.4262    \u001b[00m \u001b[92m0.16      \u001b[00m +          0.002      19.82     \n",
      "2/3        \u001b[96m2.2960    \u001b[00m \u001b[95m2.2723    \u001b[00m \u001b[92m0.24      \u001b[00m +          0.002      19.51     \n",
      "3/3        \u001b[96m2.1078    \u001b[00m \u001b[95m2.2093    \u001b[00m \u001b[92m0.32      \u001b[00m +          0.002      19.66     \n",
      "65.15046834293753\n"
     ]
    }
   ],
   "source": [
    "t0 = time.perf_counter()\n",
    "with torch.profiler.profile() as prof:\n",
    "    Resnet18.fit(TRAIN_LOADER,\n",
    "             VAL_LOADER,\n",
    "             3, # CFG.EPOCHS,\n",
    "             CFG.BATCH_SIZE,\n",
    "             CFG.DEVICE)\n",
    "t1 = time.perf_counter()\n",
    "print(t1 - t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ed612db3",
   "metadata": {},
   "outputs": [],
   "source": [
    "ka = prof.key_averages()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "632c3830",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
      "                                                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg     Self CUDA   Self CUDA %    CUDA total  CUDA time avg    # of Calls  \n",
      "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
      "enumerate(DataLoader)#_SingleProcessDataLoaderIter._...        96.27%       59.201s        97.59%       60.014s        1.250s       0.000us         0.00%       0.000us       0.000us            48  \n",
      "                                        cudaMemcpyAsync         1.78%        1.092s         1.78%        1.092s       2.167ms       0.000us         0.00%       0.000us       0.000us           504  \n",
      "                                            aten::copy_         0.42%     260.550ms         0.61%     372.561ms      25.479us     107.241ms         7.19%     107.241ms       7.334us         14622  \n",
      "                                             aten::_cat         0.35%     218.161ms         0.36%     219.181ms       2.609ms       0.000us         0.00%       0.000us       0.000us            84  \n",
      "                                       cudaLaunchKernel         0.13%      82.500ms         0.13%      82.500ms       4.887us       0.000us         0.00%       0.000us       0.000us         16881  \n",
      "-------------------------------------------------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
      "Self CPU time total: 61.492s\n",
      "Self CUDA time total: 1.492s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(ka.table(sort_by=\"self_cpu_time_total\", row_limit=5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "powerful-nevada",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T13:39:37.009864Z",
     "iopub.status.busy": "2021-05-08T13:39:37.009314Z",
     "iopub.status.idle": "2021-05-08T13:39:37.901738Z",
     "shell.execute_reply": "2021-05-08T13:39:37.901249Z"
    },
    "papermill": {
     "duration": 0.940682,
     "end_time": "2021-05-08T13:39:37.901891",
     "exception": false,
     "start_time": "2021-05-08T13:39:36.961209",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "        <script type=\"text/javascript\">\n",
       "        window.PlotlyConfig = {MathJaxConfig: 'local'};\n",
       "        if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}\n",
       "        if (typeof require !== 'undefined') {\n",
       "        require.undef(\"plotly\");\n",
       "        requirejs.config({\n",
       "            paths: {\n",
       "                'plotly': ['https://cdn.plot.ly/plotly-latest.min']\n",
       "            }\n",
       "        });\n",
       "        require(['plotly'], function(Plotly) {\n",
       "            window._Plotly = Plotly;\n",
       "        });\n",
       "        }\n",
       "        </script>\n",
       "        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>                            <div id=\"86e43d7d-dd78-4b8c-91ec-8b3e46a33c84\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"86e43d7d-dd78-4b8c-91ec-8b3e46a33c84\")) {                    Plotly.newPlot(                        \"86e43d7d-dd78-4b8c-91ec-8b3e46a33c84\",                        [{\"hovertemplate\": \"loss=%{y}<br>epoch=%{x}<extra></extra>\", \"legendgroup\": \"wide_variable_0\", \"line\": {\"color\": \"#636efa\", \"dash\": \"solid\"}, \"mode\": \"lines\", \"name\": \"train\", \"orientation\": \"v\", \"showlegend\": true, \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120], \"xaxis\": \"x\", \"y\": [2.3397841453552246, 2.2869551181793213, 2.1062192916870117, 2.0536227226257324, 1.929480791091919, 1.8463377952575684, 1.77859628200531, 1.6834133863449097, 1.5879956483840942, 1.502600908279419, 1.397341251373291, 1.388824224472046, 1.2698394060134888, 1.200344204902649, 1.1182438135147095, 1.1341758966445923, 1.0094773769378662, 1.0103679895401, 0.9074334502220154, 0.8246611952781677, 0.8806682229042053, 0.756818950176239, 0.727847695350647, 0.6533275842666626, 0.655692994594574, 0.6169918179512024, 0.6303624510765076, 0.5981965065002441, 0.5214539766311646, 0.47746890783309937, 0.49786049127578735, 0.4526429772377014, 0.447428435087204, 0.43717440962791443, 0.38721540570259094, 0.3731028437614441, 0.3421037197113037, 0.3043235242366791, 0.29838502407073975, 0.279396116733551, 0.3117947280406952, 0.27198097109794617, 0.26122382283210754, 0.26715919375419617, 0.26881223917007446, 0.22717423737049103, 0.23760689795017242, 0.24545583128929138, 0.20559312403202057, 0.19668446481227875, 0.19001829624176025, 0.1994677037000656, 0.1832388937473297, 0.14938896894454956, 0.17190244793891907, 0.16122356057167053, 0.1533065140247345, 0.1616211235523224, 0.14814502000808716, 0.15319599211215973, 0.14417965710163116, 0.11951011419296265, 0.1116272434592247, 0.118045374751091, 0.11880014836788177, 0.1018756702542305, 0.12351907044649124, 0.11401176452636719, 0.11822757869958878, 0.11544691771268845, 0.10216271132230759, 0.08447348326444626, 0.07892297953367233, 0.08158859610557556, 0.08753424882888794, 0.0707024410367012, 0.07595250010490417, 0.0714048221707344, 0.07632199674844742, 0.07507117092609406, 0.0713590607047081, 0.06981068849563599, 0.07958103716373444, 0.07119794189929962, 0.055602800101041794, 0.07490576058626175, 0.061841439455747604, 0.06117800250649452, 0.05634859949350357, 0.05755968391895294, 0.056161440908908844, 0.05375870317220688, 0.04980390891432762, 0.05449731647968292, 0.0736890658736229, 0.05897125229239464, 0.06869636476039886, 0.05405477061867714, 0.052569903433322906, 0.04778803512454033, 0.05346853658556938, 0.04272237792611122, 0.04509485512971878, 0.04619200527667999, 0.04901678115129471, 0.03854575753211975, 0.057431258261203766, 0.05172340199351311, 0.033937159925699234, 0.03469143062829971, 0.04323940351605415, 0.03495444357395172, 0.04444718733429909, 0.03702530264854431, 0.03509972244501114, 0.03818496689200401, 0.03243508189916611, 0.03227720037102699, 0.04805851727724075, 0.028319763019680977], \"yaxis\": \"y\"}, {\"hovertemplate\": \"loss=%{y}<br>epoch=%{x}<extra></extra>\", \"legendgroup\": \"wide_variable_1\", \"line\": {\"color\": \"#EF553B\", \"dash\": \"solid\"}, \"mode\": \"lines\", \"name\": \"val\", \"orientation\": \"v\", \"showlegend\": true, \"type\": \"scatter\", \"x\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120], \"xaxis\": \"x\", \"y\": [2.370363712310791, 2.227571964263916, 2.304915189743042, 2.221888542175293, 2.11378812789917, 2.110182285308838, 1.8710724115371704, 1.8948055505752563, 1.8183386325836182, 1.8511817455291748, 1.7555885314941406, 1.6600501537322998, 1.5622813701629639, 1.539713740348816, 1.526663899421692, 1.5036046504974365, 1.4426190853118896, 1.400747537612915, 1.3247736692428589, 1.3992221355438232, 1.30129873752594, 1.17190420627594, 1.1763265132904053, 1.1527644395828247, 1.0885413885116577, 1.0943971872329712, 0.9850966334342957, 0.9778620004653931, 0.9425562620162964, 0.9263128042221069, 0.9078471064567566, 0.8558145761489868, 0.9835025072097778, 0.7931811809539795, 0.7777730226516724, 0.824743390083313, 0.7781370282173157, 0.7554003596305847, 0.7990448474884033, 0.7696633338928223, 0.715813398361206, 0.6992273926734924, 0.6923214197158813, 0.6661033630371094, 0.6378315091133118, 0.7017577886581421, 0.629080593585968, 0.6029632687568665, 0.6285074353218079, 0.5705438256263733, 0.612603485584259, 0.6199272871017456, 0.5968309640884399, 0.6075388193130493, 0.6024346947669983, 0.5302067995071411, 0.5035846829414368, 0.5173574090003967, 0.584482729434967, 0.43351975083351135, 0.5880114436149597, 0.48490896821022034, 0.5069690942764282, 0.5543821454048157, 0.5113590955734253, 0.4580844044685364, 0.46430912613868713, 0.4273717999458313, 0.4441758096218109, 0.4410203993320465, 0.508985161781311, 0.44438546895980835, 0.4131603240966797, 0.37029093503952026, 0.4896911084651947, 0.4254383146762848, 0.3819901645183563, 0.4021776616573334, 0.4010077118873596, 0.4268563389778137, 0.2926212251186371, 0.35109037160873413, 0.32751908898353577, 0.3280673325061798, 0.38221636414527893, 0.34016498923301697, 0.35405343770980835, 0.3760550022125244, 0.28937166929244995, 0.3813011646270752, 0.30648520588874817, 0.354338675737381, 0.34456369280815125, 0.319694459438324, 0.3286390006542206, 0.3774917721748352, 0.32882004976272583, 0.33098262548446655, 0.30609485507011414, 0.2738361954689026, 0.2542259991168976, 0.275430291891098, 0.2976157069206238, 0.3780011534690857, 0.24104347825050354, 0.3064801096916199, 0.25790324807167053, 0.3159465193748474, 0.333347886800766, 0.36736494302749634, 0.2741539180278778, 0.30936142802238464, 0.2964180111885071, 0.2897038757801056, 0.2509382367134094, 0.2654016315937042, 0.21382467448711395, 0.24879951775074005, 0.25193989276885986, 0.20688459277153015], \"yaxis\": \"y\"}],                        {\"legend\": {\"title\": {\"text\": \"loss\"}, \"tracegroupgap\": 0}, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"autotypenumbers\": \"strict\", \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Loss History\"}, \"xaxis\": {\"anchor\": \"y\", \"domain\": [0.0, 1.0], \"title\": {\"text\": \"epoch\"}}, \"yaxis\": {\"anchor\": \"x\", \"domain\": [0.0, 1.0], \"title\": {\"text\": \"loss\"}}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('86e43d7d-dd78-4b8c-91ec-8b3e46a33c84');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "renewable-secretariat",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T13:39:37.996367Z",
     "iopub.status.busy": "2021-05-08T13:39:37.995804Z",
     "iopub.status.idle": "2021-05-08T13:40:34.091138Z",
     "shell.execute_reply": "2021-05-08T13:40:34.091944Z"
    },
    "papermill": {
     "duration": 56.145498,
     "end_time": "2021-05-08T13:40:34.092196",
     "exception": false,
     "start_time": "2021-05-08T13:39:37.946698",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Model:\n",
      "--------------------\n",
      " Accuracy: 92.6%\n",
      "Precision: 94.3%\n",
      "   Recall: 94.5%\n",
      " F1 Score: 93.8%\n",
      "Current Model:\n",
      "--------------------\n",
      " Accuracy: 92.6%\n",
      "Precision: 94.3%\n",
      "   Recall: 94.5%\n",
      " F1 Score: 93.8%\n"
     ]
    }
   ],
   "source": [
    "#evaluate best model's performance on the test set\n",
    "print('Best Model:')\n",
    "print('-'*20)\n",
    "best_model = deepcopy(Resnet18)\n",
    "best_model.load_model()\n",
    "best_model.eval_model(TEST_LOADER)\n",
    "#evaluate current model's performance on the test set\n",
    "print('Current Model:')\n",
    "print('-'*20)\n",
    "Resnet18.eval_model(TEST_LOADER)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eleven-iceland",
   "metadata": {
    "papermill": {
     "duration": 0.09136,
     "end_time": "2021-05-08T13:40:34.273030",
     "exception": false,
     "start_time": "2021-05-08T13:40:34.181670",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# What's next?\n",
    "\n",
    "- Train function as model class method\n",
    "- LR Scheduler\n",
    "- Train Custom Model\n",
    "- Confusion Matrix\n",
    "- Random & Grid Search\n",
    "- Cross-Validation\n",
    "- Ensemble Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "postal-lindsay",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T13:40:34.408154Z",
     "iopub.status.busy": "2021-05-08T13:40:34.407271Z",
     "iopub.status.idle": "2021-05-08T13:40:34.409472Z",
     "shell.execute_reply": "2021-05-08T13:40:34.409832Z"
    },
    "papermill": {
     "duration": 0.05594,
     "end_time": "2021-05-08T13:40:34.409985",
     "exception": false,
     "start_time": "2021-05-08T13:40:34.354045",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def cross_val():\n",
    "    '''\n",
    "    ======================\n",
    "       CROSS VALIDATION\n",
    "    ======================\n",
    "    '''\n",
    "    pass"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 7591.635951,
   "end_time": "2021-05-08T13:40:37.120459",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-05-08T11:34:05.484508",
   "version": "2.3.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "16fccae819c14f5a9f2abe761fe94b11": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "465c3752ef80471888972aa4ddc64f18": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6c82535f3c844dbc8f4226ec61de6da3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "6ef956557a4640398a518d156c53d449": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_16fccae819c14f5a9f2abe761fe94b11",
       "placeholder": "​",
       "style": "IPY_MODEL_da26b7aad3c94c2bb7bfc018fe0806b9",
       "value": " 44.7M/44.7M [00:01&lt;00:00, 51.3MB/s]"
      }
     },
     "93b1f22c1e11400291e4b743e807a33a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ae20fbe41b234a4487c246bd621d6705",
       "placeholder": "​",
       "style": "IPY_MODEL_d3b1752519fa4bf2bae6d77524177170",
       "value": "100%"
      }
     },
     "971520e5e9d14d9580fd40b15687ea83": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ffe343c87abd4354adf2e215ba8ff5c7",
       "max": 46827520,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_6c82535f3c844dbc8f4226ec61de6da3",
       "value": 46827520
      }
     },
     "9e145c8cd29b4639bcd97058c7d8ba14": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_93b1f22c1e11400291e4b743e807a33a",
        "IPY_MODEL_971520e5e9d14d9580fd40b15687ea83",
        "IPY_MODEL_6ef956557a4640398a518d156c53d449"
       ],
       "layout": "IPY_MODEL_465c3752ef80471888972aa4ddc64f18"
      }
     },
     "ae20fbe41b234a4487c246bd621d6705": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d3b1752519fa4bf2bae6d77524177170": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "da26b7aad3c94c2bb7bfc018fe0806b9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ffe343c87abd4354adf2e215ba8ff5c7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}