step03_EmbeddingModel-02_CV0_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PyTorch Modules\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import torchvision\n",
    "from torchvision import models\n",
    "from torchvision import transforms\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as dsets\n",
    "\n",
    "# Other non-PyTorch Modules\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from matplotlib.pyplot import imshow\n",
    "import matplotlib.pylab as plt\n",
    "from PIL import Image\n",
    "import time\n",
    "from datetime import datetime\n",
    "import pickle\n",
    "import json\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20201020_1319\n"
     ]
    }
   ],
   "source": [
    "now = datetime.now()\n",
    "DATESTRING = now.strftime(\"%Y%m%d_%H%M\")\n",
    "print(DATESTRING)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataDF = pd.read_csv('data_fold.csv')\n",
    "dataDF = dataDF.set_index('SOPInstanceUID')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['StudyInstanceUID', 'SeriesInstanceUID', 'pe_present_on_image',\n",
       "       'negative_exam_for_pe', 'qa_motion', 'qa_contrast', 'flow_artifact',\n",
       "       'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1', 'leftsided_pe', 'chronic_pe',\n",
       "       'true_filling_defect_not_pe', 'rightsided_pe', 'acute_and_chronic_pe',\n",
       "       'central_pe', 'indeterminate', 'window_center', 'window_width',\n",
       "       'intercept', 'slope', 'slice_thickness', 'kvp', 'ma', 'exposure',\n",
       "       'img_pos', 'conv_kernel', 'patient_position', 'pixel_spacing',\n",
       "       'bits_stored', 'high_bit', 'img_count', 'fold'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataDF.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataDF = dataDF.reindex(columns=['StudyInstanceUID', 'SeriesInstanceUID', 'pe_present_on_image', 'negative_exam_for_pe',\n",
    "                       'indeterminate', 'chronic_pe', 'acute_and_chronic_pe', 'central_pe', 'leftsided_pe',\n",
    "                       'rightsided_pe', 'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1','fold','img_pos','patient_position',\n",
    "                       'intercept', 'slope'])                  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StudyInstanceUID</th>\n",
       "      <th>SeriesInstanceUID</th>\n",
       "      <th>pe_present_on_image</th>\n",
       "      <th>negative_exam_for_pe</th>\n",
       "      <th>indeterminate</th>\n",
       "      <th>chronic_pe</th>\n",
       "      <th>acute_and_chronic_pe</th>\n",
       "      <th>central_pe</th>\n",
       "      <th>leftsided_pe</th>\n",
       "      <th>rightsided_pe</th>\n",
       "      <th>rv_lv_ratio_gte_1</th>\n",
       "      <th>rv_lv_ratio_lt_1</th>\n",
       "      <th>fold</th>\n",
       "      <th>img_pos</th>\n",
       "      <th>patient_position</th>\n",
       "      <th>intercept</th>\n",
       "      <th>slope</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SOPInstanceUID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>c0f3cb036d06</th>\n",
       "      <td>6897fa9de148</td>\n",
       "      <td>2bfbb7fd2e8b</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-234.5</td>\n",
       "      <td>HFS</td>\n",
       "      <td>-1024</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f57ffd3883b6</th>\n",
       "      <td>6897fa9de148</td>\n",
       "      <td>2bfbb7fd2e8b</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-252.5</td>\n",
       "      <td>HFS</td>\n",
       "      <td>-1024</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41220fda34a3</th>\n",
       "      <td>6897fa9de148</td>\n",
       "      <td>2bfbb7fd2e8b</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-432.5</td>\n",
       "      <td>HFS</td>\n",
       "      <td>-1024</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13b685b4b14f</th>\n",
       "      <td>6897fa9de148</td>\n",
       "      <td>2bfbb7fd2e8b</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-434.5</td>\n",
       "      <td>HFS</td>\n",
       "      <td>-1024</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>be0b7524ffb4</th>\n",
       "      <td>6897fa9de148</td>\n",
       "      <td>2bfbb7fd2e8b</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-436.5</td>\n",
       "      <td>HFS</td>\n",
       "      <td>-1024</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               StudyInstanceUID SeriesInstanceUID  pe_present_on_image  \\\n",
       "SOPInstanceUID                                                           \n",
       "c0f3cb036d06       6897fa9de148      2bfbb7fd2e8b                    0   \n",
       "f57ffd3883b6       6897fa9de148      2bfbb7fd2e8b                    0   \n",
       "41220fda34a3       6897fa9de148      2bfbb7fd2e8b                    0   \n",
       "13b685b4b14f       6897fa9de148      2bfbb7fd2e8b                    0   \n",
       "be0b7524ffb4       6897fa9de148      2bfbb7fd2e8b                    0   \n",
       "\n",
       "                negative_exam_for_pe  indeterminate  chronic_pe  \\\n",
       "SOPInstanceUID                                                    \n",
       "c0f3cb036d06                       0              0           0   \n",
       "f57ffd3883b6                       0              0           0   \n",
       "41220fda34a3                       0              0           0   \n",
       "13b685b4b14f                       0              0           0   \n",
       "be0b7524ffb4                       0              0           0   \n",
       "\n",
       "                acute_and_chronic_pe  central_pe  leftsided_pe  rightsided_pe  \\\n",
       "SOPInstanceUID                                                                  \n",
       "c0f3cb036d06                       0           0             1              1   \n",
       "f57ffd3883b6                       0           0             1              1   \n",
       "41220fda34a3                       0           0             1              1   \n",
       "13b685b4b14f                       0           0             1              1   \n",
       "be0b7524ffb4                       0           0             1              1   \n",
       "\n",
       "                rv_lv_ratio_gte_1  rv_lv_ratio_lt_1  fold  img_pos  \\\n",
       "SOPInstanceUID                                                       \n",
       "c0f3cb036d06                    0                 1     3   -234.5   \n",
       "f57ffd3883b6                    0                 1     3   -252.5   \n",
       "41220fda34a3                    0                 1     3   -432.5   \n",
       "13b685b4b14f                    0                 1     3   -434.5   \n",
       "be0b7524ffb4                    0                 1     3   -436.5   \n",
       "\n",
       "               patient_position  intercept  slope  \n",
       "SOPInstanceUID                                     \n",
       "c0f3cb036d06                HFS      -1024      1  \n",
       "f57ffd3883b6                HFS      -1024      1  \n",
       "41220fda34a3                HFS      -1024      1  \n",
       "13b685b4b14f                HFS      -1024      1  \n",
       "be0b7524ffb4                HFS      -1024      1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainDF = dataDF[dataDF['fold']!=0]\n",
    "valDF = dataDF[dataDF['fold']==0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class embeddingsDataset(Dataset):\n",
    "    \"\"\"create sample dataset to work with\"\"\"\n",
    "\n",
    "    def __init__(self, dataDF = None, listOfStudies = None, embeddingDirPath = None):\n",
    "        self.dataDF = dataDF\n",
    "        self.listOfStudies = listOfStudies\n",
    "        self.embeddingDirPath = embeddingDirPath\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.listOfStudies)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        embedDict = pickle.load(open(self.embeddingDirPath+self.listOfStudies[idx]+'.p', 'rb'))\n",
    "        embeddingVolume = np.array(embedDict['embeddings'])\n",
    "        listOfImages = embedDict['ids']\n",
    "        imageLevelLabels = [self.dataDF.loc[eachImageID, 'pe_present_on_image']for eachImageID in listOfImages]\n",
    "        imageLevelLabels = np.array(imageLevelLabels).astype(np.float32)\n",
    "        studyLevelLabels = self.dataDF.loc[listOfImages[0]][3:12].values\n",
    "        studyLevelLabels = np.array(studyLevelLabels).astype(np.float32)\n",
    "        return embeddingVolume, (imageLevelLabels, studyLevelLabels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_SIZE = 64\n",
    "HIDDEN_SIZE = 32\n",
    "NUM_LAYERS = 1\n",
    "NUM_CLASSES = 1\n",
    "\n",
    "class BiGRU(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, num_layers, num_classes):\n",
    "        super(BiGRU, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.GRU = nn.GRU(\n",
    "            input_size, hidden_size, num_layers, batch_first=True, bidirectional=True\n",
    "        )\n",
    "        self.linear1 = nn.Linear(hidden_size*2, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, num_classes)\n",
    "        self.linear3 = nn.Linear(hidden_size*2, hidden_size)\n",
    "        self.linear4 = nn.Linear(hidden_size, 9)\n",
    "\n",
    "    def forward(self, x):\n",
    "        imageLevelOutputs = []\n",
    "        h0 = torch.zeros(self.num_layers * 2, x.size(0), self.hidden_size).cuda()\n",
    "        #c0 = torch.zeros(self.num_layers * 2, x.size(0), self.hidden_size).cuda()\n",
    "\n",
    "        out, h_n = self.GRU(x, h0)\n",
    "        \n",
    "        for i, out_t in enumerate(out.chunk(out.size(1), dim=1)):\n",
    "            out_t = out_t.squeeze(1)\n",
    "            out_t = F.relu(self.linear1(out_t))\n",
    "            out_t = self.linear2(out_t)\n",
    "            imageLevelOutputs += [out_t]\n",
    "        imageLevelOutputs = torch.stack(imageLevelOutputs, 1).squeeze(2)\n",
    "        \n",
    "        h_n = h_n.view(1,-1)\n",
    "        studyLevelOutputs = F.relu(self.linear3(h_n))\n",
    "        studyLevelOutputs = self.linear4(studyLevelOutputs)\n",
    "        \n",
    "        return (imageLevelOutputs, studyLevelOutputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq = BiGRU(INPUT_SIZE, HIDDEN_SIZE, NUM_LAYERS, NUM_CLASSES).cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# batch,seqNum,features\n",
    "sampleInput = torch.rand((1,120,64)).cuda()\n",
    "sampleImgOutput, sampleStdOutput = seq(sampleInput)\n",
    "print(sampleImgOutput.size())\n",
    "print(sampleStdOutput.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def customLoss(imageLevelOutputLogits, imageLevelLabels, studyLeveloutputLogits, studyLevelLabels, rho=0.5):\n",
    "    imageLevelLoss = F.binary_cross_entropy_with_logits(imageLevelOutputLogits,imageLevelLabels,pos_weight = torch.tensor([3.0]).cuda())\n",
    "    \n",
    "    studyLevelLoss = 0\n",
    "    weightList = [0.0736196319, 0.09202453988, 0.1042944785, 0.1042944785, 0.1877300613, 0.06257668712, 0.06257668712, 0.2346625767, 0.0782208589]\n",
    "    for eachInd in range(9):\n",
    "        studyLevelLoss += weightList[eachInd]*F.binary_cross_entropy_with_logits(studyLeveloutputLogits[:,eachInd],studyLevelLabels[:,eachInd])\n",
    "    \n",
    "    return rho*imageLevelLoss+(1-rho)*studyLevelLoss, imageLevelLoss, studyLevelLoss\n",
    "\n",
    "def competitionMetric(imageLevelOutputLogits, imageLevelLabels, studyLeveloutputLogits, studyLevelLabels):\n",
    "    imageLevelLoss = 0.07361963*(torch.sum(imageLevelLabels)/imageLevelLabels.size(1))*F.binary_cross_entropy_with_logits(imageLevelOutputLogits,imageLevelLabels,reduction='sum')\n",
    "    \n",
    "    studyLevelLoss = 0\n",
    "    weightList = [0.0736196319, 0.09202453988, 0.1042944785, 0.1042944785, 0.1877300613, 0.06257668712, 0.06257668712, 0.2346625767, 0.0782208589]\n",
    "    for eachInd in range(9):\n",
    "        studyLevelLoss += weightList[eachInd]*F.binary_cross_entropy_with_logits(studyLeveloutputLogits[:,eachInd],studyLevelLabels[:,eachInd], reduction='sum')\n",
    "    \n",
    "    return imageLevelLoss+studyLevelLoss, imageLevelLabels.size(1)+9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(seq.parameters(), lr=5e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "valEmbeddingDirPath = 'data/embeddings/CNNmodel_01_cv0_epoch1_20201014_0012/val/'\n",
    "augEmbeddingDirPathBase = 'data/embeddings/CNNmodel_01_cv0_epoch1_20201014_0012/aug'\n",
    "\n",
    "valEmbeddingsDataset = embeddingsDataset(dataDF=dataDF, listOfStudies=valDF['StudyInstanceUID'].unique(), embeddingDirPath=valEmbeddingDirPath)\n",
    "valEmbeddingsDataloader = DataLoader(valEmbeddingsDataset, batch_size=1, shuffle=False, num_workers=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_loop(model, epoch):\n",
    "    train_total = train_correct = train_cost_tot = train_cost1 = train_cost2 = 0\n",
    "    seq.train()\n",
    "    thisEmbeddingDirPath = augEmbeddingDirPathBase + str(epoch).zfill(2) + '/'\n",
    "    thisEmbeddingsDataset = embeddingsDataset(dataDF=dataDF, listOfStudies=trainDF['StudyInstanceUID'].unique(), embeddingDirPath=thisEmbeddingDirPath)\n",
    "    train_loader = DataLoader(thisEmbeddingsDataset, batch_size=1, shuffle=True, num_workers=1)\n",
    "    for x, (y_img, y_std) in tqdm(train_loader):\n",
    "        x = x.cuda()\n",
    "        y_img = y_img.cuda()\n",
    "        y_std = y_std.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        (o_img, o_std) = seq(x)\n",
    "        train_total += y_img.size(1)\n",
    "        train_correct += ((torch.sigmoid(o_img[0,:])>0.5) == (y_img[0,:]>0.5)).sum().item()\n",
    "        loss, loss1, loss2 = customLoss(o_img, y_img, o_std, y_std)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        train_cost_tot += loss.item()\n",
    "        train_cost1 += loss1.item()\n",
    "        train_cost2 += loss2.item()\n",
    "    return 100*train_cost_tot/train_total, 100*train_cost1/train_total, 100*train_cost2/train_total, train_correct/train_total \n",
    "\n",
    "def valid_loop(model, valid_loader):\n",
    "    # Evaluate on validation  data \n",
    "    val_total = val_correct = val_cost_tot = val_cost1 = val_cost2 = val_metric_tot = rows_tot = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for x_val, (y_val_img, y_val_std) in tqdm(valid_loader):\n",
    "            x_val = x_val.cuda()\n",
    "            y_val_img = y_val_img.cuda()\n",
    "            y_val_std = y_val_std.cuda()\n",
    "            (o_val_img, o_val_std) = seq(x_val)\n",
    "            # Calculate loss and metric\n",
    "            val_total += y_val_img.size(1)\n",
    "            val_correct += ((torch.sigmoid(o_val_img[0,:])>0.5) == (y_val_img[0,:]>0.5)).sum().item()\n",
    "            loss, loss1, loss2 = customLoss(o_val_img, y_val_img, o_val_std, y_val_std)\n",
    "            metric, numRows = competitionMetric(o_val_img, y_val_img, o_val_std, y_val_std)\n",
    "            val_cost_tot += loss.item()\n",
    "            val_cost1 += loss1.item()\n",
    "            val_cost2 += loss2.item()\n",
    "            val_metric_tot += metric.item()\n",
    "            rows_tot += numRows\n",
    "    return 100*val_cost_tot/val_total, 100*val_cost1/val_total, 100*val_cost2/val_total, 100*val_metric_tot/rows_tot, val_correct/val_total\n",
    "\n",
    "def main_loop(n_epochs):\n",
    "    val_avgCostList = []\n",
    "    val_Cost1List = []\n",
    "    val_Cost2List = []\n",
    "    for epoch in range(n_epochs):\n",
    "        print('epoch ' + str(epoch) + ':')\n",
    "        train_avgCost, train_Cost1, train_Cost2, train_acc = train_loop(seq, epoch)\n",
    "        val_avgCost, val_Cost1, val_Cost2, val_compMetric, val_acc = valid_loop(seq, valEmbeddingsDataloader)\n",
    "        val_avgCostList.append(val_avgCost)\n",
    "        val_Cost1List.append(val_Cost1)\n",
    "        val_Cost2List.append(val_Cost2)\n",
    "        print('train_cost: %.4f, train_cost_img: %.4f, train_cost_std: %.4f,train_acc: %.4f'\\\n",
    "              % (train_avgCost, train_Cost1, train_Cost2, train_acc))\n",
    "        print('val_cost: %.4f, val_cost_img: %.4f, val_cost_std: %.4f, val_compMetric: %.4f, val_acc: %.4f'\\\n",
    "              % (val_avgCost, val_Cost1, train_Cost2, val_compMetric, val_acc))\n",
    "        modelPath = 'models/embedderModel/cv0/CNNmodel_01_cv0_epoch1_20201014_0012_sequence' + str(epoch) + '_' + DATESTRING +'.pth'\n",
    "        print('saving: ',modelPath)\n",
    "        torch.save(seq, modelPath)\n",
    "    return val_avgCostList, val_Cost1List, val_Cost2List"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Note mistake in the model name!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 0:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [05:59<00:00, 16.21it/s]\n",
      "100%|██████████| 1456/1456 [00:39<00:00, 37.15it/s]\n",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.1028, train_cost_img: 0.0667, train_cost_std: 0.1390,train_acc: 0.9679\n",
      "val_cost: 0.0931, val_cost_img: 0.0726, val_cost_std: 0.1390, val_compMetric: 0.2135, val_acc: 0.9675\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence0_20201020_1319.pth\n",
      "epoch 1:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [05:59<00:00, 16.18it/s]\n",
      "100%|██████████| 1456/1456 [00:38<00:00, 38.03it/s]\n",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.0717, train_cost_img: 0.0502, train_cost_std: 0.0932,train_acc: 0.9732\n",
      "val_cost: 0.0813, val_cost_img: 0.0696, val_cost_std: 0.0932, val_compMetric: 0.1917, val_acc: 0.9635\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence1_20201020_1319.pth\n",
      "epoch 2:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [05:57<00:00, 16.31it/s]\n",
      "100%|██████████| 1456/1456 [00:38<00:00, 38.08it/s]\n",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.0659, train_cost_img: 0.0491, train_cost_std: 0.0827,train_acc: 0.9735\n",
      "val_cost: 0.0809, val_cost_img: 0.0720, val_cost_std: 0.0827, val_compMetric: 0.1905, val_acc: 0.9703\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence2_20201020_1319.pth\n",
      "epoch 3:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [05:56<00:00, 16.34it/s]\n",
      "100%|██████████| 1456/1456 [00:38<00:00, 38.01it/s]\n",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.0647, train_cost_img: 0.0484, train_cost_std: 0.0810,train_acc: 0.9741\n",
      "val_cost: 0.0801, val_cost_img: 0.0708, val_cost_std: 0.0810, val_compMetric: 0.1888, val_acc: 0.9695\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence3_20201020_1319.pth\n",
      "epoch 4:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [05:59<00:00, 16.21it/s]\n",
      "100%|██████████| 1456/1456 [00:38<00:00, 37.69it/s]\n",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.0639, train_cost_img: 0.0475, train_cost_std: 0.0802,train_acc: 0.9751\n",
      "val_cost: 0.0805, val_cost_img: 0.0719, val_cost_std: 0.0802, val_compMetric: 0.1901, val_acc: 0.9702\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence4_20201020_1319.pth\n",
      "epoch 5:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [05:58<00:00, 16.22it/s]\n",
      "100%|██████████| 1456/1456 [00:38<00:00, 37.90it/s]\n",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.0634, train_cost_img: 0.0477, train_cost_std: 0.0790,train_acc: 0.9749\n",
      "val_cost: 0.0789, val_cost_img: 0.0682, val_cost_std: 0.0790, val_compMetric: 0.1878, val_acc: 0.9672\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence5_20201020_1319.pth\n",
      "epoch 6:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [05:56<00:00, 16.32it/s]\n",
      "100%|██████████| 1456/1456 [00:38<00:00, 37.99it/s]\n",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.0627, train_cost_img: 0.0472, train_cost_std: 0.0782,train_acc: 0.9750\n",
      "val_cost: 0.0780, val_cost_img: 0.0677, val_cost_std: 0.0782, val_compMetric: 0.1852, val_acc: 0.9689\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence6_20201020_1319.pth\n",
      "epoch 7:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [05:59<00:00, 16.18it/s]\n",
      "100%|██████████| 1456/1456 [00:38<00:00, 37.85it/s]\n",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.0627, train_cost_img: 0.0473, train_cost_std: 0.0780,train_acc: 0.9751\n",
      "val_cost: 0.0780, val_cost_img: 0.0686, val_cost_std: 0.0780, val_compMetric: 0.1847, val_acc: 0.9695\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence7_20201020_1319.pth\n",
      "epoch 8:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [06:00<00:00, 16.17it/s]\n",
      "100%|██████████| 1456/1456 [00:38<00:00, 38.12it/s]\n",
      "  0%|          | 0/5823 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.0622, train_cost_img: 0.0472, train_cost_std: 0.0771,train_acc: 0.9749\n",
      "val_cost: 0.0797, val_cost_img: 0.0715, val_cost_std: 0.0771, val_compMetric: 0.1886, val_acc: 0.9703\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence8_20201020_1319.pth\n",
      "epoch 9:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5823/5823 [05:59<00:00, 16.21it/s]\n",
      "100%|██████████| 1456/1456 [00:39<00:00, 37.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_cost: 0.0619, train_cost_img: 0.0465, train_cost_std: 0.0772,train_acc: 0.9757\n",
      "val_cost: 0.0772, val_cost_img: 0.0682, val_cost_std: 0.0772, val_compMetric: 0.1843, val_acc: 0.9692\n",
      "saving:  models/embedderModel/cv0/CNNmodel_01_cv0_epoch2_20201014_0901_sequence9_20201020_1319.pth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Note mistakes in the model name!!!\n",
    "list1, list2, list3 = main_loop(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f5143246750>]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAscElEQVR4nO3deXgc1Znv8e9Rd0sttTZLshZLlmVZ3lfASyBgwmpjQyyDSSDLTJhJuMwNSS6ZGTIzzzx37r3PkwyTSWZCgIQQAiTDBGIgBhIWs2+2MZaxLW/YlhdZshZLtna1pF7O/eO05JYs2S2r5WqV3s/z+Omu7uruV23pd06dOlWltNYIIYSwrzirCxBCCDG6JOiFEMLmJOiFEMLmJOiFEMLmJOiFEMLmnFYXMJisrCxdVFRkdRlCCDFmbN++vVFrPXGw52Iy6IuKiigrK7O6DCGEGDOUUpVDPSdDN0IIYXMS9EIIYXMS9EIIYXMS9EIIYXMS9EIIYXMS9EIIYXMS9EIIYXO2CfoufxdP7XmKT2o/sboUIYSIKbYJemeck9/u+y3/tf+/rC5FCCFiiq2C/pZpt/Bh9Yc0ehutLkcIIWKGbYIeoLSklIAO8OfDf7a6FCGEiBm2CvritGIWTlzIixUvIpdIFEIIw1ZBD6ZXf7jlMLsbd1tdihBCxATbBf3KopW4HW5erHjR6lKEECIm2C7ok+OTuWHKDbx29DW8fq/V5QghhOVsF/QAa6evpd3XztvH37a6FCGEsJwtg/6ynMvIT87nxUMvWl2KEEJYzpZBH6fiKC0pZWvdVk60n7C6HCGEsJQtgx5gzbQ1KBQvVbxkdSlCCGEp2wZ9XnIen8v7HC9VvERQB60uRwghLBNR0CulViqlDiilKpRS/zDI87OUUluUUt1Kqb8b8NwTSqmTSqk90So6UqUlpdR01LCtbtvF/mghhIgZ5w16pZQDeAS4CZgD3KmUmjNgtdPAd4GfDPIWTwErR1bmhbm28FpS4lPYULHBio8XQoiYEEmPfilQobU+orXuAZ4F1oSvoLU+qbXeBvgGvlhr/QGmIbjo3E43q6au4q3Kt2jrabOiBCGEsFwkQZ8PVIUtV4ceiyql1N1KqTKlVFlDQ0PU3re0pJTuQDevHX0tau8phBBjSSRBrwZ5LOpnDNNaP6a1Xqy1Xjxx4sSove/czLmUpJfI7BshxLgVSdBXA5PDlguAmtEpJ/qUUpSWlFLeWM7h5sNWlyOEEBddJEG/DZiulJqqlIoH7gBeHt2youvm4ptxKqec6EwIMS6dN+i11n7gXmAjsB9Yr7Xeq5S6Ryl1D4BSKlcpVQ18H/hnpVS1Uio19NwzwBZgZujxvx6tH2YomYmZLC9YzsuHX8YXPGt/sRBC2JozkpW01q8Crw547NGw+3WYIZ3BXnvnSAqMlrXT1/JO1Tt8VP0R1xReY3U5Qghx0dj2yNiBrsy/kkx3psypF0KMO+Mm6J1xTr447Yty8XAhxLgzboIezJx6v/bzypFXrC5FCCEumnEV9MXpxSyYuEAuHi6EGFfGVdCD6dVXNFew99Req0sRQoiLYtwFfe/Fwzcckp2yQojxYdwFfUp8CtdPuZ7Xjr5Gl7/L6nKEEGLUjbugBzN80+Zrk4uHCyHGhXEZ9Etyl5iLh8spEYQQ48C4DPo4FceaaWvYWruVmvYxc342IYS4IOMy6AHWlJhrp7x0WE5fLISwt3Eb9JOSJ7E0b6lcPFwIYXvjNugB1pas5UT7CcrqyqwuRQghRs24DvrrCq8jxSUXDxdC2Nu4Dnq3081NU2+Si4cLIWxtXAc9mDn1XYEuNh7baHUpQggxKsZ90M/LmkdJeokM3wghbGvcB33fxcMbyjnSfMTqcoQQIurGfdADrC5eLRcPF0LYlgQ9kJWYxVUFV8nFw4UQtiRBH1JaUsqprlNsOrHJ6lKEECKqJOhDriq4igx3hgzfCCFsR4I+xBXn4pbiW3i/6n1OeU9ZXY4QQkSNBH0YuXi4EMKOJOjDlEwoYX7WfDZUbJCLhwshbEOCfoDei4fvO7XP6lKEECIqJOgHuGnqTSQ4EuRIWSGEbUjQD9B78fBXj75Kd6Db6nKEEGLEJOgHUVpSSltPG+8cf8fqUoQQYsQk6AexNHcpkzyT2HBIhm+EEGOfBP0g4lQca0rW8HHtx9S211pdjhBCjIgE/RC+OO2LaLRcPFwIMeZJ0A+hIKWAZbnL5OLhQogxT4L+HNaUrKG6vZrt9dutLkUIIS5YREGvlFqplDqglKpQSv3DIM/PUkptUUp1K6X+bjivjWXXT7meZFeynOhMCDGmnTfolVIO4BHgJmAOcKdSas6A1U4D3wV+cgGvjVmJzkRWTl3JG8feoL2n3epyhBDigkTSo18KVGitj2ite4BngTXhK2itT2qttwEDr9px3tfGurUla+Xi4UKIMS2SoM8HqsKWq0OPRWIkr40J87PmU5xWLKdEEEKMWZEEvRrksUhP7Rjxa5VSdyulypRSZQ0NDRG+/ehTSrG2ZC27GnZxpEUuHi6EGHsiCfpqYHLYcgFQE+H7R/xarfVjWuvFWuvFEydOjPDtL46bp92MQzlkp6wQYkyKJOi3AdOVUlOVUvHAHcDLEb7/SF4bM3ovHv6nw3/CH/RbXY4QQgzLeYNea+0H7gU2AvuB9VrrvUqpe5RS9wAopXKVUtXA94F/VkpVK6VSh3rtaP0wo6m0pJRGbyObazZbXYoQQgyLM5KVtNavAq8OeOzRsPt1mGGZiF47Fi0vWE6GO4MNhzawvGC51eUIIUTE5MjYCLniXNxcfDPvVb/H6a7TVpcjhBARk6AfhtKSUvxBuXi4EGJskaAfhukTpjMvc55cPFwIMaZI0A9TaUkph5oOse+0XDxcCDE2SNAP003F5uLhLx560epShBAiIhL0w5Qan8q1hdfyytFX5OLhQogxQYL+AqwtWUtbTxvvHn/X6lKEEOK8JOgvwLK8ZeR58uREZ0KIMUGC/gL0Xjx8S80W6jrqrC5HCCHOSYL+Aq2ZtgaN5uXDY+7UPUKIcUaC/gIVpBSwNHcpL1a8KHPqhRAxTYJ+BEpLSqlqq5KLhwshYpoE/QhcP+V6PC6P7JQVQsQ0CfoRSHQmsrJoJW9WvkmHr8PqcoQQYlAS9CNUWlKK1++Vi4cLIWKWBP0ILZy4kKlpU+Uyg0KImCVBP0JKKUpLStlxcgdHW45aXY4QQpxFgj4Kbim+BYdy8FLFS1aXIoQQZ5Ggj4KJSRO5Mv9KXj78slw8XAgRcyToo2RtyVoavA1y8XAhRMyRoI+S5QXLmZAwQXbKCiFijgR9lLgcLm6edjPvVr1LU1eT1eUIIUQfCfoo6r14+KtHX7W6FCGE6GOroN9b00Jju3VXfZoxYQZzM+ey4ZCcEkEIETtsE/QtnT5uf3QL//LSXkvrKC0p5UDTAZ757Bm8fq+ltQghBNgo6NOSXHz7mhJe2V3LK+W1ltWxung1szNm86OtP+K6567jgU8e4HDzYcvqEUIIFYvnUl+8eLEuKysb9uv8gSBrf7GZmmYvb9y3nMzkhFGo7vy01myv385zB5/jzco38QV9XJp9KetmrOPGohtJcFhTlxDCvpRS27XWiwd9zk5BD3Cgro2bH/qQFXNzefgrl0a5suFr6mripYqXeP7Q81S2VpKWkMaaaWtYN2MdU9OmWl2eEMImxlXQAzz8ziF+8sZBfvnVS7lpfl4UK7twQR1kW9021h9YzzvH38Gv/SzNXcrtM27nusLrcDlcVpcohBjDxl3Q9w7h1LZ4eeO+q8nwxEexupFr9DbyYsWLPH/weU60nyDDnUFpSSnrZqxjcspkq8sTQoxB4y7oAT6ra+WWhz5i5bw8HrrzkihVFl1BHWRLzRbWH1jP+9XvE9ABLs+7nC/N/BJXT74aV5z08oUQkRmXQQ/w0NuH+OmbB3n0a5eycl5sDOEMpb6jng0VG3jh0AvUddSRlZjF2pK1rJuxjknJk6wuTwgR48Zt0PsCQUof2UR9axdv3nc1E2JsCGcwgWCAj058xPqD6/mw+kMArsy/kttn3M5VBVfhjHNaXKEQIhaN26AH2F9rhnBWL8jjwTticwhnKLXttbxw6AX+eOiPNHgbyEnK4bbpt3Hr9FvJ8eRYXZ4QIoaM66AHePCtQ/znWwf51dcvY8Xc3Ki978XiC/r4oOoDnjv4HJtrNhOn4lhesJzbZ9zOFZOuwBHnsLpEIYTFRhz0SqmVwIOAA3hca/3AgOdV6PlVQCfwDa31p6Hnvgd8C1DAr7XWPzvf50U76H2BIGse3sTJtm7e+v5y0pNifwhnKFVtVbxw8AU2VGzgdNdpJnkmsW7GOtZOX0tWYpbV5QkhLDKioFdKOYCDwA1ANbANuFNrvS9snVXAdzBBvwx4UGu9TCk1D3gWWAr0AK8Df6O1PnSuz4x20IM54dmahzdxy8JJ/OeXF0X1va3gC/h4p+odnjvwHFvrtuJUTq4pvIbbZ9zOsrxlxCnbnN1CCBGBcwV9JGmwFKjQWh/RWvdggnvNgHXWAL/TxsdAulIqD5gNfKy17tRa+4H3gbUX/JOMwNxJaXz7mhI27DjBm/vqrSghqlwOFyuKVvD4isf5U+mf+Orsr7Ktbht3v3k3N2+4mSf2PMEp7ymryxRCxIBIevTrgJVa62+Glr8OLNNa3xu2zp+BB7TWH4WW3wZ+AHQALwGXA17gbaBMa/2dQT7nbuBugMLCwssqKytH/tMN0OMPsuaRTTS2d/PmfWN7CGcw3YFu3qp8i/UH1vPpyU8BiI+LJzk+mWRXMh6Xh+T40G1oOSU+pd9ysiu5b/1kVzKeePNYvMNe35UQdnOuHn0kc/XUII8NbB0GXUdrvV8p9W/Am0A7sAsY9OrZWuvHgMfADN1EUNewxTvj+MntC1jz8Cb+35/28R82GMIJl+BIYHXxalYXr+Zw82Heq3qP1p5W2nvaafe10+HroN3XTl1HHe09ZrnN1xbRBc1dca7BG4f4czcanngPHqeHeEc88Y54XHEucz8ufkztRNZa4/V76fB10OHroNPfaW595rbDb+6HL3f4OvD6vH33e59PTUhlSuoUilKLzG1aEUWpRaQnpGN2dwkRXZEEfTUQflx+AVAT6Tpa698AvwFQSv0otK5l5k5K439eU8LP3z7Eqvl5XD/HntMUp6VPY1r6tPOup7WmJ9jTF/y9DUJbT1u/5d7Got3XTkePeby+s572ljPr+4K+YdXoUI6zwj/eEY/L4eq7Hx83YDm0foIjof9rw18/4LHe12v0eYO509d5doj7OvD6veiz+jeDS3QmkuRMwuPy4HF5SHQmkunOpDClkCRXEk1dTRxtOcr71e/3a2RT41P7Qj+8IZiSOgW30z2s71aIcJEE/TZgulJqKnACuAP4yoB1XgbuVUo9i9kZ26K1rgVQSmVrrU8qpQqBWzHDOJa695oS3thbxz9t2M2SogzSksbvqQaUUiQ4EkhITCAzMXNE79UT6OnXEPQ2Eh2+DnoCPfiCPnoCPfQEe8xt2GPdge4zz4fW8QV8phHytZ9ZDnt97/oBHRhR3QODOcmV1BfMvcsDn/c4Q7dhy72hHumWij/op7a9lqOtR6lsraSytZJjLcfYWruVlw+/3G/dPE9eX/iHNwZ5nrwxtWUkrBHp9MpVwM8w0yuf0Fr/UCl1D4DW+tHQ9MqHgZWY6ZV3aa3LQq/9EMgEfMD3tdZvn+/zRmPWzUB7TrSw5pFNlC7K56dfWjiqnyVGVyAY6Nd4DNaQ9AR7UKgRBfPF1Onr5HjbcY61HuNYy7G+RuBY6zHafe1968XHxVOYWihDQUIOmBrKT984wEPvVPDENxZz7Sx7DuEIe9Fac6rrVL8tgGOt5l9VW5UMBY1jEvRD6PYH+OJDm2j29vDGfVeTljh+h3DE2OcP+qlpr+FY65ktgMrWSo62HuVk58l+6+Z58sjz5JHjySEnKfQvdD87KZusxCw5r9IYI0F/DuXVzaz9xWZuvSSff79dhnCEPfUNBYW2ACpbK6nrqKO+s576jnp6gj391o9TcWQlZpGblEt2Una/BqF3OTspe8xfFjMQDNDh70BrTVpCmtXljMhIp1fa2oKCdO65uphH3j3MqgV5XDMz2+qShIi6JFcSszJmMStj1lnPaa1p7m7mZOdJ6jvr+xqAk50nqe+o50jLET6u/bjfvoFeExImDNoIhG8heFyeqP88Wmu6A91mp3/YjLD2nva+GWNtvra+iQHhj/Wt39NOp7+z7z1zPbnMz5rPgqwFzJ84nzmZc0h0Jka9diuM+x49mCGcWx76iFavnze+v5xUtwzhCDFQe087JztPUtdZ19cI1HfW92sUmrqbznpdsiu5L/izk7L7NQKZiZl0+88f2H3HfYQ9FsnxH4nORFJcKXjiPeY2dNBg7zEfKa4UkuOTCQQD7D21l92NuznRfgIw039nTJjB/Kz5zJ9oGoCitKKYPb2IDN1EoHcI57ZL8/nxOhnCEeJCdAe6h2wEepcbvY0EdfCc7+NQjn5HaPfdH/BYeIgPXMfj8lzQfoZGbyN7GvdQ3lDO7sbd7Gnc07c1k+JKYW7WXNPzn7iA+VnzRzwtOVok6CP049c/4xfvHeapu5bwBRnCEWJU+IN+Gr2N1HfWc8p7CrfTfVZgux3umJkaGtRBjrUco7yxvC/8DzUd6jt+Iz853/T6Q+E/O3O2JfsuJOgj1O0PcPPPP6K928/G+2QIRwgxOK/fy75T+9jdsJvyRhP+dR11ADiVkxkZM/r1+qekThn1IR8J+mHYWdXMrb/YxJcWT+aB2xZYUoMQYuxp6Gwwod+wu2/Ip3dnb0p8St9O3t7e/wT3hKh+vgT9MD3w2mc8+v5hfvdXS1k+Y6JldQghxq5AMMCRliPsbtzdN+RT0VzRt39icsrkfr3+WRmzRnSWWAn6YeryBVj98w/x9gTYeN9yUmQIRwgRBZ2+zr7ZPb3DPr0Hs7niXMzPms+TK5+8oGEemUc/TG6Xg3+/fSHrfrmZH736Gf9663yrSxJC2ECSK4kluUtYkruk77H6jnrT628sp62nbVTG8iXoh3Bp4QS+dVUxv/rgCKvn53HldLkeqxAi+nI85riC66dcP2qfEZsz/2PEfTfMYNpEDz94oZz27vMfnCGEELFIgv4ceodwalu8/OjV/VaXI4QQF0SC/jwuLZzAN68q5vdbj/PRoUaryxFCiGGToI/A92+YQXGWDOEIIcYmCfoImCGcBdS0eHngNRnCEUKMLRL0EbpsSgZ//fmpPP3xcTZXyBCOEGLskKAfhr+9cSZTszzc/0I5HTKEI4QYIyTohyEx3sGP1y3gRLOXf3v9M6vLEUKIiEjQD9OSogzuumIqv9tSyZbDp6wuRwghzkuC/gL8/YqZFGUmcf8Lu+jskSEcIURsk6C/AGYIZyHVTV5+/PoBq8sRQohzkqC/QEunZvCNK4p4avMxPj4iQzhCiNglQT8Cf79iJlMyk7j/+XIZwhFCxCwJ+hFIinfy49sWcPx0pwzhCCFilgT9CC0rzuwbwtkqQzhCiBgkQR8F96+cSWFGEve/UI63J2B1OUII0Y8EfRQkxTv5t9sWUHmqk3/fKEM4QojYIkEfJZdPy+QvLp/Ck5uPsu3YaavLEUKIPhL0UfSDlbMomJDI3z+3iw8ONvDp8SYO1rdxotlLi9dHIBh7F2IXQtifXDM2ijwJZgjn67/5hL944pNB13G74khOcJGc4MCT4MST4CQldOtJcJLiduKJd+JJcJj7oceTw/71Lsc7pZ0WQpyfBH2UXTEtiw/uv4aaZi/t3X7au/x0dPvN/e7e+4F+j9W1doU956fLF4zos+IdcXgSHCSHGoe+RsDtJDneNBrZqQnkpLrJTnGTm+YmJzWBpHj5bxdiPJG/+FGQn55IfnriBb/eHwjS0R2gvceEf1uosRiswWjv9pl1Q481dfZQ1dRJR7efFq9v0EYjJcFJTij0c1LcZKe6ye1tEFLN49kpbtliEMImIgp6pdRK4EHAATyutX5gwPMq9PwqoBP4htb609Bz9wHfBDSwG7hLa90VtZ/AhpyOONKS4khLco3ofbTWtHf7qW/tor61e8Ct+bf16GlOtnXhC5y9/yDTE98X/LlhjUBOaOsgOzWBTE8Cjjg1ojoj+Tm6/UHauvx9W0lt3T7au840fOHPnVn29XssKd7J/II0FuSnsaAgnXn5qaS4R/YdCzEWnDfolVIO4BHgBqAa2KaUellrvS9stZuA6aF/y4BfAsuUUvnAd4E5WmuvUmo9cAfwVFR/CjEopRQpbhcpbhcl2SlDrhcMapo6ewY0At3Ut3VR39JFfVsXe2taaWzvRg9oDxxxionJCWYLIcVsFeSmuckO3c9JdZOa6OzbMjkT1P4hgtrXfzm0nj+CHdnOOEWKOzR0leAiJcFJdoqb4izzWHNnD7uqmnmlvDb0/UBxlocFBeksKEhjQUEac/LSSIx3jOh7FyLWRNKjXwpUaK2PACilngXWAOFBvwb4ndZaAx8rpdKVUnlhn5GolPIBSUBN1KoXUREXp8hMTiAzOYE5k1KHXM8fCNLQ3n3WVkHv8rFTHWw9epoWr29Yn9+7gzrFfWaH8+SMJFJ69zeEbs8su0gO7bhODlsnwRmH2bg8t9MdPZRXN7O7uoVd1S1sPtzIhh0nANNwTc9OZmFBOvML0lhYkM7M3BQZxhJjWiRBnw9UhS1XY3rt51snX2tdppT6CXAc8AJvaK3fGOxDlFJ3A3cDFBYWRla9uKicjjjy0hLJSzv3/ocuX4CTrd3UhRqCti7/gKB29gW1J8GJy3FxQzTDE88XZmbzhZnZfY/Vt3ZRXt1CeXUz5dUtvLGvjj+UmV/peEccs/NSzLBPqPdfMjEZ50WuW0RXIKipONnOrqpm6lu7uHJ6FgsL0okb5aFIK0QS9IP91AO3owddRyk1AdPbnwo0A88ppb6mtX76rJW1fgx4DGDx4sUy4XwMc7scFGYmUZiZZHUpEctJdXPDHDc3zMkBzH6B6iavCf8TzZRXtfDSjhqe/vg4AIkuB3Mnpfb1+ucXpDE102PLkLCL+tYudhxvZmdVM7uqmimvbqYj7JQlP33zILmpbm6cm8OKubksnZpx0TshoyWSoK8GJoctF3D28MtQ61wPHNVaNwAopf4IXAGcFfRCxBKlFJMzkpickcTqBWYUMhjUHD3VERryMUM/z3xynCc3HQPMbKZ5+WksmJzGgnzT8y+YkBjRcJKIro5uP7tPtLCzqpmdx5vZVd1MbYuZA+JyKGbnpXLbZQUsmpzOwsnpZHrieeezk2zcW8f6sip+t6WStEQX183OZuXcXJbPmIjbNXb33Sg9cO/awBWUcgIHgeuAE8A24Cta671h66wG7sXMulkG/FxrvVQptQx4AliCGbp5CijTWj90rs9cvHixLisru9CfSYiLxh8IUtHQTnlVqOdf3cL+2ta+WUwZnnjm56eFdvaa8M9JdVtctb0EgpqD9W3sqjK99Z1VzRysb6N3/31hRhKLJqebf4XpzMlLPWdoe3sCfHCogY176nhrfz2tXX4SXQ6unjGRFfNyuHZWDmmJsTdbSym1XWu9eNDnzhf0oTdYBfwMM73yCa31D5VS9wBorR8NTa98GFiJmV55l9a6LPTa/wt8GfADO4Bvaq27z/V5EvRiLOv2BzhQ19ZvzP/Qyfa+U2AkJzjJTXOTl+YmNzRL6cxyInlpbtKTXLIlMITaFi87jzezs9r01nefaKEzNASTluhiYSjULwn11jM88Rf8Wb5AkK1HTrNxbx0b99Zxsq0bZ5zi8mmZrJiby41zcsiOkYZ7xEF/sUnQC7vx9gTYV9tCeXULlac6qWvpoq61i7qWLk62dTFw9miCM840AKmmAchJc5OX6iY3zTQEuWluspJH/xgGq7V3+ymvPjOuvrOqmfpW0090ORRzJqWxqCCNRYXpLJo8gaLMpFFrIINBzc7qZhP6e+o4dqoTpeDSwgmsCI3rT8n0jMpnR0KCXogY1jtttbbFHLdQG2oE+pZbvdS3dNMT6H+UsyNOkZOSENoaSCQn1Cj0bh30HscwVqaG+gNBDta3h4ZfmthV1cLBk219x24UZSb1jakvmpzOnEmpJDitGTfXWnOwvr2vp7+3phWAWbkprJiby4q5uczOS7moW2US9EKMccGg5nRnj9kSaOmitrWLuhavaQxCjUJtcxde39kXvslKTugL/t6GIDfVTYJr8AZADTKJbqi8GirGBl//7Ae7/QH21rT2DcH01p+e5OobV184OZ1FBelMGMEQzGirOt3Jxr11vLG3nm2Vp9Ha7Bvo7elfWjhh1GdkSdALMQ5orWnt8vcF/8CGoC60tTDcA9pGW7wjjjmTUs24emE6CwvSmTKKQzCjraGtm7f217Nxbx2bKhrxBTRZyQncMCeHlfNyubw4c1S2siTohRB9vD0B6lu78AXOPuHdYGkwVEToQdcefP2h3sMRpyjKSrJsCGa0tXX5ePeAmcHz7oGTdPYESHE7uXZWNivm5nL1jIl4EqJzbkkJeiGEsFiXL8CmikY27q3jzX31NHX6SHDGcdX0iayYm8P1s3NGNDx1rqCX0xQLIcRF4HY5uG52DtfNzsEfCLLtWFNoXN/M13fEKS4vzuTJu5ZE/YhcCXohhLjInI44Lp+WyeXTMvmXW+aw+0QLG/fWUdfSPSqnXZCgF0IICymlQkdNp4/aZ4yNCbZCCCEumAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnAS9EELYnL2C3t9tdQVCCBFz7BP0wSA8sgx+fwd89goEfFZXJIQQMcE+lxL0e2HOGtj1DBx8DTwTYeEdsOhrkD3L6uqEEMIySmttdQ1nWbx4sS4rK7uwFwf8UPEW7PgvOPg6BP1QsAQu+RrMvRXcqdEtNpZ1tUJCCihldSVCiFGmlNqutV486HO2C/pw7Q1Q/izseBoaPgNnIswtNaE/5fP2C8CAH45vMQ3cwdfhVAUkpEJmyZl/WWH34z1WVzx6ejqg6RicPgqnj0DTUUjKhMV/Dal5VlcnRNSN36DvpTWc2G56+btfgJ42mDDVBP7COyEtP3qfdbF5m6DibTjwGlS8CV0t4IiHoqtgyuXQVm8C/1QFtFT1f23KJMicBlnT+zcG6VPAEeOjelpD5ykT5E1Hz75tr++/vjsdultBOWDhl+GK78LEmZaULsRokKAP19MJ+182vfxjH4KKg2nXmdCfeRM4E0bnc6OpscLshzjwuunB6wAkZcGMFTBjJUy7xgzZDOTzmt5t46FQ+B+GU4fMclfzmfXiXJAxNRT80yAz1BBkTTf7Pi7WllAwAK0nBgnzI9BUaYI7XGq+acAzikK3U8/cJk4wr9vyCOz4b7NPZ+Yq+Pz3oPBzF+fnEbGlp9NM3Nj1e6jfB/NuhSXfNL/zY5AE/VBOH4Gdvzf/Wk9AYgYs+LIJ/dx5o//5kQr44PjHZjjmwGtw+rB5PHsuzFwJM26C/MsgbgSTqDpOnen5nwo1BI0V5jsKhE1bTUjtH/69WwQZ0yAhefif6+uC5krzOQMDvfk4BHrOrBvngglTzg7xjGKzFeJyR/izNsInv4ZPHgPvaZi8zPTwZ64a2XcoYl8waDpHu34Pe18yW/dpkyFnntkiDvpNx2/p3TD9BohzWF1xxCTozycYgCPvml7+Z6+YcMlbZAJ//u2QmH7xaunlbYJDb5mee8Vb/YdkZt5keu/phaNfRzAALdWh8D/cf2ugpQoI+/1JyQvbFxA2HJQ4wYyX9+uVh5Zba/q/R3xKKLynnh3oqfnR/cPr6TC9+y0Pm8Ymczpc8R3T2EfaaIix4fQR2PUHMyuvuRJcHrO/buGdZn9dXBy01cH230LZE9BeZ/6+lnwTLvk6JGVY/ROclwT9cHSehvL1Zjy/fg843TD7FhP6RctHt8fXeMj02A9uPDMk45kI01eYYB9qSMYqkQ4FDZScc3aI994mZV78neQBP+x/CTY9CLW7TH3L/gcs/ivTSImxqasF9r5owv34FkDB1OWw6Cvmb3qoyQgBH3z2Z/jkcaj8yGTAvNtg6bdg0iUX8ycYlhEHvVJqJfAg4AAe11o/MOB5FXp+FdAJfENr/alSaibwh7BVi4H/rbX+2bk+z9Kg76W1+aPf8TTsXm9+adIK4ZKvml+UaPSmA77QLJmN/YdkcuaFxtujMCRjlc7TZxoAbxNMKAoFelHszvbRGo6+bwL/8DsQnwyXfQM+9zeQVmB1dSISvVvnO58xYe3vMltqi+40W2rD/X+s3wvbHjdbA74OyF9shnXmlsbc/rwRBb1SygEcBG4AqoFtwJ1a631h66wCvoMJ+mXAg1rrZYO8zwlgmda68lyfGRNBH87XZX5pdjwNR94zjxVfbTbpZt08vM38WBmSEedWWw6bH4I9L5gtjHnr4PPfhZy5VlcmBnNyv9nXVr7eDLu4000vfNFXTGdppFuJXS2m8dj2a9N5ScqCy/7SbPXFSCdgpEF/OfB/tNYrQsv/CKC1/tewdX4FvKe1fia0fAD4gta6NmydG4F/0Vp//nwFx1zQh2s+bv7Ddz5t7rvTYP6XzNBO3sLBf6H6hmReNztVw4dkZq6E4i/E1pCMOKP5OGz5BXz6W/B1QskNZqZO0ZX2Ow5jrOk4BXueNwFfu9NMnZ1+ozkifrRm0AWDcPQ9szP/4OvmsZmrzLDO1Kst/Z0YadCvA1Zqrb8ZWv46pld+b9g6fwYe0Fp/FFp+G/iB1rosbJ0ngE+11g8P8Tl3A3cDFBYWXlZZec5Ov/WCQTj2genl73vZzEzJmR86AnctNB4w0x8Pvm6vIZnxqvM0lP0Gtv4KOhpg0qWmhz/7i2NqZsaY5++BQxth17NmyDPog9z5sPArZuJE8sSLV0tTpdlx++nvzOytrJlm5+3COyw5An+kQX87sGJA0C/VWn8nbJ1XgH8dEPT3a623h5bjgRpgrta6fuBnDBTTPfrBeJvMJv6Op6Fmx5nHZUjGfnxes3Nv80NmR/SEqXDFvbDoq+BKtLo6e9La/F3tegZ2P29C1ZMNC75kZs1YPRXa1wV7/2h6+TWfmn07C++AJd+6qOfZsnzoRim1Bvi21vrGSAoec0Efrm6PGabJngXF11zY3HIR+4IBMxV308/MUddJWWYn3dJvjYmpeGNCa40Zc9/1jDmFiSMBZq0yvfdp18bm0dvV2804/p4/mq38oqvM78XMVaNe70iD3onZGXsdZmfqNuArWuu9YeusBu7lzM7Yn2utl4Y9/yywUWv9ZCQFj+mgF+OL1lC52czUObQRXElmJ/3l3zYHd4nhCT9a9ch7oINQsNTMmpm7duxMd+1oNEM6ZU+Y401S82HxXXDpX0Jy9qh8ZDSmV64CfoaZXvmE1vqHSql7ALTWj4amVz4MrMRMr7yrd3xeKZUEVAHFWuuWSAqWoBdj0sn9ZkinfL0JqLlrzTh+3kKrK4ttWptpxjt/b+a99x6tuvAOWHCHORHfWBUMmP10n/zaTPuMc5nfi6XfMmfVjeLOWzlgSoiLqeUEbP0llD1lQqv4C2amTvE1MlPH1wVtNdBaa4ZmGg/A7ufMkdMuj7mmxKI7YcqV9pus0HjIzMnf+Xtznqa8hWYcf/66qOzfkaAXwgpdLVD2JHz8SzO3O3c+XPE9cw6VhFR7BZnW5mjo3gBvqzG3rTXQVnvmvvf0gBeGjlZdeKc5WnU87NPqbofyP5jQP7nPDEdd8jVzCu2MqRf8thL0QljJ322Gczb/HBoPmsdUnDkGw51uzqXU73bCII+F3SakXtwtg2AA2k+eO8Bba8wZQQdKyoLUSWf+pUwy1wPouz9pfF0MKJzWULnJDOvs/5MZ7puxAr70uws6BkCCXohYEAyaUys0fGZ6v94m8DaH7g+4DfqHfh/lMI3EYI1A4oShGwh3+tlXHPN5zw7ttlpzNtfe3nl7vTnIL1ycy5zEbmBop+aduZ+SG3OnCYhZrTWw/Slz1O26Jy7oLSTohRhLtDZn1vQ2Dd4I9N0O0VAMDOVwynFmq6D3PQaKTzk7tPvdn2R66nYaerKBcwV9DE5EFWKcU8qMVSckA5OH91qtoaf9/I1BV4vZKkjNM1P/Unp75nnjdyjFxiTohbATpczwTEIKw24khG3JtpcQQticBL0QQticBL0QQticBL0QQticBL0QQticBL0QQticBL0QQticBL0QQthcTJ4CQSnVAFzoRWOzgMYoljOWyXfRn3wf/cn3cYYdvospWutBL5obk0E/EkqpsqHO9zDeyHfRn3wf/cn3cYbdvwsZuhFCCJuToBdCCJuzY9A/ZnUBMUS+i/7k++hPvo8zbP1d2G6MXgghRH927NELIYQII0EvhBA2Z5ugV0qtVEodUEpVKKX+wep6rKSUmqyUelcptV8ptVcp9T2ra7KaUsqhlNqhlPqz1bVYTSmVrpR6Xin1Weh35HKra7KSUuq+0N/JHqXUM0opt9U1RZstgl4p5QAeAW4C5gB3KqXmWFuVpfzA32qtZwOfA749zr8PgO8B+60uIkY8CLyutZ4FLGQcfy9KqXzgu8BirfU8wAHcYW1V0WeLoAeWAhVa6yNa6x7gWWCNxTVZRmtdq7X+NHS/DfOHnG9tVdZRShUAq4HHra7FakqpVGA58BsArXWP1rrZ0qKs5wQSlVJOIAmosbieqLNL0OcDVWHL1YzjYAunlCoCLgG2WlyKlX4G3A8ELa4jFhQDDcCToaGsx5VSHquLsorW+gTwE+A4UAu0aK3fsLaq6LNL0KtBHhv380aVUsnAC8D/0lq3Wl2PFZRSNwMntdbbra4lRjiBS4Ffaq0vATqAcbtPSyk1AbP1PxWYBHiUUl+ztqros0vQV9P/kvcF2HDzaziUUi5MyP+31vqPVtdjoc8DX1RKHcMM6V2rlHra2pIsVQ1Ua617t/CexwT/eHU9cFRr3aC19gF/BK6wuKaos0vQbwOmK6WmKqXiMTtTXra4JssopRRmDHa/1vo/rK7HSlrrf9RaF2itizC/F+9orW3XY4uU1roOqFJKzQw9dB2wz8KSrHYc+JxSKin0d3MdNtw57bS6gGjQWvuVUvcCGzF7zZ/QWu+1uCwrfR74OrBbKbUz9Ng/aa1fta4kEUO+A/x3qFN0BLjL4noso7XeqpR6HvgUM1ttBzY8HYKcAkEIIWzOLkM3QgghhiBBL4QQNidBL4QQNidBL4QQNidBL4QQNidBL4QQNidBL4QQNvf/AWVWv0VCRToTAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(list1)\n",
    "plt.plot(list2)\n",
    "plt.plot(list3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "iterVal = iter(valEmbeddingsDataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0009033124661073089 0.0\n",
      "0.0005709259421564639 0.0\n",
      "0.00043395336251705885 0.0\n",
      "0.00039422052213922143 0.0\n",
      "0.0003740342508535832 0.0\n",
      "0.00035327201476320624 0.0\n",
      "0.00032831935095600784 0.0\n",
      "0.0003413487283978611 0.0\n",
      "0.0003516829165164381 0.0\n",
      "0.0003477193822618574 0.0\n",
      "0.0003532306000124663 0.0\n",
      "0.00034345712629146874 0.0\n",
      "0.0003345227742101997 0.0\n",
      "0.00032930081943050027 0.0\n",
      "0.0003195177996531129 0.0\n",
      "0.0003292421461082995 0.0\n",
      "0.0003074228297919035 0.0\n",
      "0.0003179946797899902 0.0\n",
      "0.00033059492125175893 0.0\n",
      "0.00033379168598912656 0.0\n",
      "0.00033630579127930105 0.0\n",
      "0.00037190201692283154 0.0\n",
      "0.00039421714609488845 0.0\n",
      "0.0005195172852836549 0.0\n",
      "0.000855035032145679 0.0\n",
      "0.0007881468627601862 0.0\n",
      "0.000917330093216151 0.0\n",
      "0.0011043609119951725 0.0\n",
      "0.0009291339083574712 0.0\n",
      "0.0011502375127747655 0.0\n",
      "0.0016806048806756735 0.0\n",
      "0.001940552145242691 0.0\n",
      "0.002063982654362917 0.0\n",
      "0.0028843225445598364 0.0\n",
      "0.004875782877206802 0.0\n",
      "0.006577412132173777 0.0\n",
      "0.009231621399521828 0.0\n",
      "0.006140832789242268 0.0\n",
      "0.006455888040363789 0.0\n",
      "0.007283882703632116 0.0\n",
      "0.00839944463223219 0.0\n",
      "0.008548823185265064 0.0\n",
      "0.007587732747197151 0.0\n",
      "0.009255665354430676 0.0\n",
      "0.010260037146508694 0.0\n",
      "0.013188386335968971 0.0\n",
      "0.01702032797038555 0.0\n",
      "0.02339482493698597 0.0\n",
      "0.01615784503519535 0.0\n",
      "0.015456213615834713 0.0\n",
      "0.016537178307771683 0.0\n",
      "0.021005291491746902 0.0\n",
      "0.029688844457268715 0.0\n",
      "0.06098166108131409 0.0\n",
      "0.04509077966213226 0.0\n",
      "0.03859073296189308 0.0\n",
      "0.032421715557575226 0.0\n",
      "0.04255778715014458 0.0\n",
      "0.07078781723976135 0.0\n",
      "0.043972332030534744 0.0\n",
      "0.03140109032392502 0.0\n",
      "0.02468072436749935 0.0\n",
      "0.02249072678387165 0.0\n",
      "0.05973684415221214 0.0\n",
      "0.0391494482755661 0.0\n",
      "0.033542219549417496 0.0\n",
      "0.039564792066812515 0.0\n",
      "0.05121266841888428 0.0\n",
      "0.03126237541437149 0.0\n",
      "0.024577206000685692 0.0\n",
      "0.024800026789307594 0.0\n",
      "0.038769420236349106 0.0\n",
      "0.01981630176305771 0.0\n",
      "0.015688562765717506 0.0\n",
      "0.018842145800590515 0.0\n",
      "0.020315634086728096 0.0\n",
      "0.0224233977496624 0.0\n",
      "0.025706909596920013 0.0\n",
      "0.08572673052549362 0.0\n",
      "0.03069893829524517 0.0\n",
      "0.029102671891450882 0.0\n",
      "0.03219882398843765 0.0\n",
      "0.0509735569357872 0.0\n",
      "0.0367431640625 0.0\n",
      "0.03640215843915939 0.0\n",
      "0.06221875920891762 0.0\n",
      "0.07692049443721771 0.0\n",
      "0.08174224942922592 0.0\n",
      "0.07626646012067795 0.0\n",
      "0.07154452055692673 0.0\n",
      "0.09620270133018494 0.0\n",
      "0.10362234711647034 0.0\n",
      "0.1379374861717224 0.0\n",
      "0.12878763675689697 0.0\n",
      "0.137245774269104 0.0\n",
      "0.12804730236530304 0.0\n",
      "0.10879380255937576 0.0\n",
      "0.12633654475212097 0.0\n",
      "0.07428155094385147 0.0\n",
      "0.03270046040415764 0.0\n",
      "0.015386530198156834 0.0\n",
      "0.010956703685224056 0.0\n",
      "0.009733769111335278 0.0\n",
      "0.012911579571664333 0.0\n",
      "0.005429494194686413 0.0\n",
      "0.005230841226875782 0.0\n",
      "0.0035851607099175453 0.0\n",
      "0.003112291218712926 0.0\n",
      "0.0028071000706404448 0.0\n",
      "0.0028172319289296865 0.0\n",
      "0.0030857976526021957 0.0\n",
      "0.0032841297797858715 0.0\n",
      "0.0038067626301199198 0.0\n",
      "0.0038881348446011543 0.0\n",
      "0.0042562843300402164 0.0\n",
      "0.003931737970560789 0.0\n",
      "0.003646420082077384 0.0\n",
      "0.003504635766148567 0.0\n",
      "0.003853803500533104 0.0\n",
      "0.0045891692861914635 0.0\n",
      "0.004701153840869665 0.0\n",
      "0.005311404354870319 0.0\n",
      "0.004503576084971428 0.0\n",
      "0.0037008863873779774 0.0\n",
      "0.0036607575602829456 0.0\n",
      "0.0028085261583328247 0.0\n",
      "0.001874201581813395 0.0\n",
      "0.0012088957009837031 0.0\n",
      "0.0009272833121940494 0.0\n",
      "0.0008430382586084306 0.0\n",
      "0.0009798010578379035 0.0\n",
      "0.0009372404892928898 0.0\n",
      "0.0009542314801365137 0.0\n",
      "0.0012305170530453324 0.0\n",
      "0.0011802359949797392 0.0\n",
      "0.0015554819256067276 0.0\n",
      "0.0012088560033589602 0.0\n",
      "0.0011546645546332002 0.0\n",
      "0.0016521582147106528 0.0\n",
      "0.0014436609344556928 0.0\n",
      "0.00256713037379086 0.0\n",
      "0.0020670334342867136 0.0\n",
      "0.001347277546301484 0.0\n",
      "0.0013269261689856648 0.0\n",
      "0.0022404780611395836 0.0\n"
     ]
    }
   ],
   "source": [
    "# Sanity Check\n",
    "seq.eval()\n",
    "with torch.no_grad():\n",
    "    x,(y_img, _) = next(iterVal)\n",
    "    x=x.cuda()\n",
    "    o_img, _ = seq(x)\n",
    "    pred = torch.sigmoid(o_img)\n",
    "    for eachIndex in range(pred.size(1)):\n",
    "        print((pred[0,eachIndex]).type(torch.float).item(), y_img[0, eachIndex].item())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pyTorchEnv37",
   "language": "python",
   "name": "pytorchenv37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}