vnir_resnet_DC_planet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from pprint import pprint\n",
    "from time import time\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "import os\n",
    "import torch\n",
    "import pandas as pd\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms, utils\n",
    "\n",
    "from torchsat.models.classification import resnet34, resnet18\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "import rasterio as rio\n",
    "from rasterio.mask import mask\n",
    "from rasterio import windows #.from_bounds\n",
    "from fiona.crs import from_epsg\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Ignore warnings\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "plt.ion()   # interactive mode\n",
    "\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RoofImageDataset_Planet(Dataset):\n",
    "    \"\"\"PlanetScope Roof Image Dataset class!\"\"\"\n",
    "\n",
    "    def __init__(self, geo_df, image_dir, imdim=10, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            geo_df (string): GeoPandas GeoDataFrame containing a 'geometry' column and 'class_code' column. \n",
    "                             everything must be in linear dimensions.\n",
    "            image_dir (string): full path to directory containing multiple Planet images\n",
    "            imdim (int): image dimension for CNN\n",
    "            transform (callable, optional): Optional transform to be applied\n",
    "                on a sample.\n",
    "        \"\"\"\n",
    "        \n",
    "        if not os.path.exists(image_dir):\n",
    "            raise ValueError(f'{image_dir} does not exist')\n",
    "        \n",
    "            \n",
    "        self.geometries = [p.centroid for p in geo_df.geometry.values]\n",
    "        self.imdir = image_dir\n",
    "        self.image_dim = imdim\n",
    "        self.Y = geo_df.class_code.values\n",
    "        self.transform = transform\n",
    "        self.planet_images = glob(f'{self.imdir}/*/*/*_SR.tif')\n",
    "        \n",
    "        if len(self.planet_images) < 1:\n",
    "            raise ValueError(f'{self.imdir} does not contain any images')\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.geometries)\n",
    "\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        if torch.is_tensor(idx):\n",
    "            idx = idx.tolist()\n",
    "\n",
    "        # sample the list of planet images with the specified geometry\n",
    "        left, bottom, right, top = self.geometries[idx].bounds\n",
    "        geom = self.geometries[idx]\n",
    "        \n",
    "        try:\n",
    "            sample = self.sample_many_planet_images(geom, self.planet_images)\n",
    "            cc = self.Y.codes.astype('uint8')[idx]\n",
    "\n",
    "            assert sample.shape == (4, self.image_dim, self.image_dim), f'array shape is not as specified {sample.shape}'\n",
    "\n",
    "            if self.transform:\n",
    "                sample = self.transform(sample)\n",
    "        \n",
    "        except Exception as e:\n",
    "            raise ValueError(e)\n",
    "            sample = torch.from_numpy(np.zeros((4, int(self.image_dim), int(self.image_dim))))\n",
    "                \n",
    "            cc = 255 # highest int8 number\n",
    "            \n",
    "        \n",
    "        # normalize to SR\n",
    "        sample /= 10000\n",
    "        \n",
    "        # convert to tensor\n",
    "        sample = torch.from_numpy(sample)\n",
    "        \n",
    "        return {'image': sample.type(torch.FloatTensor),\n",
    "                'class_code': torch.tensor(cc).type(torch.LongTensor)}\n",
    "    \n",
    "    def sample_many_planet_images(self, geom, planet_files):\n",
    "    \n",
    "        # sample the geometry in each image\n",
    "        samples = []\n",
    "        left, bottom, right, top = geom.bounds\n",
    "        for f in planet_files:\n",
    "            \n",
    "            # use the windows.from_bounds() method to return the window\n",
    "            with rio.open(f) as src:\n",
    "                                \n",
    "                # Get pixel coordinates from map coordinates\n",
    "                py, px = src.index(geom.x, geom.y)\n",
    "\n",
    "                # Build an NxN window\n",
    "                N = self.image_dim\n",
    "                window = rio.windows.Window(px - N//2, py - N//2, N, N)\n",
    "                #print(window)\n",
    "\n",
    "                # Read the data in the window\n",
    "                # clip is a nbands * N * N numpy array\n",
    "                sample = src.read(window=window)\n",
    "        \n",
    "                #print(sample.shape)\n",
    "            \n",
    "            if sample.shape != (4, self.image_dim, self.image_dim):\n",
    "                continue\n",
    "            else:\n",
    "                samples.append(sample)\n",
    "            \n",
    "        # convert to numpy array\n",
    "        samples_arr = np.array(samples)\n",
    "        \n",
    "        #print(samples_arr.shape)\n",
    "\n",
    "        # check to see if geometry was in none of the images. \n",
    "        # If it was, then the sum should be > 0.0, and take the average\n",
    "        # Else, average should be all zero anyway\n",
    "        if samples_arr.sum() > 0:\n",
    "            ans = np.ma.masked_equal(samples_arr, 0).mean(axis=0)\n",
    "        else:\n",
    "            ans = samples_arr.mean(axis=0)\n",
    "\n",
    "        return ans\n",
    "    \n",
    "# function to return subset of batch if 255 detected in class labels\n",
    "def make_good_batch(batch):\n",
    "    \n",
    "    '''removes bad sample if 255 detected in class labels\n",
    "    \n",
    "    batch: dictionary containing 'image' tensor and 'class_code' tensor\n",
    "    \n",
    "    returns: dictionary same as input with class_code==255 removed from class_code \n",
    "             and corresponding image tensor removed\n",
    "    '''\n",
    "    \n",
    "    _idx = torch.where(batch['class_code'] != 255)[0]\n",
    "    \n",
    "    new_batch={}\n",
    "    new_batch['image']= batch['image'][_idx]\n",
    "    new_batch['class_code']= batch['class_code'][_idx]\n",
    "    \n",
    "    return new_batch\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dataframe processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_file = '../DC/training-data/ocm_w_ztrax_11001_matched.geojson' \n",
    "df = gpd.read_file(df_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.to_crs(32618)\n",
    "df['areaUTM'] = [geom.area for geom in df.geometry]\n",
    "df['areaUTMsqft'] = [geom.area*10.7639 for geom in df.geometry]\n",
    "\n",
    "# filter the footprints by distance metric and area mismatch\n",
    "df = df.loc[(df._distance <= 10) & (df.areaUTMsqft <= df.LotSizeSquareFeet)]\n",
    "\n",
    "# add the class code categorical variable\n",
    "df['class_code']=df.RoofCoverStndCode.astype('category') # category type is required for encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "skipping BU class\n",
      "skipping  class\n",
      "skipping OT class\n",
      "class CN has shape (4, 86)...skipping...\n"
     ]
    }
   ],
   "source": [
    "# first split into train/test for each class\n",
    "train_df, test_df, val_df = [], [], []\n",
    "\n",
    "ts = 0.4\n",
    "vs = 0.2\n",
    "bad_classes = ['', 'BU', 'OT']\n",
    "for cl in df.RoofCoverStndCode.unique():\n",
    "    \n",
    "    # don't use the '' class\n",
    "    if cl in bad_classes:\n",
    "        print(f'skipping {cl} class')\n",
    "        continue\n",
    "        \n",
    "           \n",
    "    #subset to class\n",
    "    _df = df.loc[df.RoofCoverStndCode == cl]\n",
    "    \n",
    "    # if sample size is small, skip... that sucks\n",
    "    if _df.shape[0] < 5:\n",
    "        print(f'class {cl} has shape {_df.shape}...skipping...')\n",
    "        continue\n",
    "    \n",
    "    # get train and test validation arrays. test array is validation array split in half.\n",
    "    _train, _valtest = train_test_split(_df, random_state=27, test_size=ts)\n",
    "    train_df.append(_train)\n",
    "    \n",
    "    _val, _test = train_test_split(_valtest, random_state=27, test_size=vs)\n",
    "    test_df.append(_test)\n",
    "    val_df.append(_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatenate training, validaton, and test dataframes\n",
    "all_train_df = pd.concat(train_df)\n",
    "all_train_df = gpd.GeoDataFrame(all_train_df, crs=from_epsg(4326))\n",
    "\n",
    "all_val_df = pd.concat(val_df)\n",
    "all_val_df = gpd.GeoDataFrame(all_val_df, crs=from_epsg(4326))\n",
    "\n",
    "all_test_df = pd.concat(test_df)\n",
    "all_test_df = gpd.GeoDataFrame(all_test_df, crs=from_epsg(4326))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([CS, ME, SL, UR, TL, WS, SH, WD]\n",
       " Categories (8, object): [CS, ME, SL, UR, TL, WS, SH, WD],\n",
       " [CS, ME, SL, UR, TL, WS, SH, WD]\n",
       " Categories (8, object): [CS, ME, SL, UR, TL, WS, SH, WD],\n",
       " [CS, ME, SL, UR, TL, WS, SH, WD]\n",
       " Categories (8, object): [CS, ME, SL, UR, TL, WS, SH, WD])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_train_df.class_code.unique(), all_val_df.class_code.unique(), all_test_df.class_code.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# image file\n",
    "imfile = '../DC/dc_02222016_10400100185A4600_output_ms_vnir.tif'\n",
    "imdir = '../DC/planet-data/PSScene4Band-PSSD'\n",
    "    \n",
    "# dataset and dataloader\n",
    "bs = 128\n",
    "train_ds = RoofImageDataset_Planet(all_train_df[['geometry', 'class_code']], imdir, imdim=224)\n",
    "train_loader = DataLoader(train_ds, batch_size=bs, shuffle=True, num_workers=0)\n",
    "\n",
    "val_ds = RoofImageDataset_Planet(all_val_df[['geometry', 'class_code']], imdir, imdim=224)\n",
    "val_loader = DataLoader(val_ds, batch_size=bs, shuffle=True, num_workers=0)\n",
    "\n",
    "test_ds = RoofImageDataset_Planet(all_test_df[['geometry', 'class_code']], imdir, imdim=224)\n",
    "test_loader = DataLoader(test_ds, batch_size=bs, shuffle=True, num_workers=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Let's use 4 GPUs!\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") # get device for gpu or cpu\n",
    "# device=torch.device('cpu')\n",
    "\n",
    "# model\n",
    "## seems the errors are thrown when number of classes are == number of channels. Odd!\n",
    "n_classes = df.RoofCoverStndCode.unique().shape[0] # edit to reflect weird classes?? or keep all?\n",
    "# n_classes = all_train_df.class_code.unique().shape[0] # this is 8, instead of 12. 1 removed due to small size, other 3 due to being too general\n",
    "# n_classes = 8\n",
    "model = resnet18(n_classes, in_channels=4, pretrained=False)\n",
    "model.to(device)\n",
    "\n",
    "# make model parallel and on GPU\n",
    "if torch.cuda.device_count() >= 1:\n",
    "    print(\"Let's use\", torch.cuda.device_count(), \"GPUs!\")\n",
    "    model = nn.DataParallel(model)\n",
    "    model.to(device)\n",
    "else:\n",
    "    #ps_model = nn.DataParallel(ps_model)\n",
    "    model = nn.DataParallel(model)\n",
    "    print('made cpu parallel')\n",
    "\n",
    "# if args.resume:\n",
    "#     model.load_state_dict(torch.load(args.resume, map_location=device))\n",
    "\n",
    "# loss\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# optim and lr scheduler\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "# lr_scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1, eta_min=1e-8) # for resuming training?\n",
    "lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)\n",
    "\n",
    "# writer = SummaryWriter(args.ckp_dir) # need tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "learning rate: 0.001, epoch 0\n",
      "Train Epoch: 0 [0/47676 (0%)]\tLoss: 2.655574\n",
      "train/loss 2.655573606491089 0\n",
      "Train Epoch: 0 [12800/47676 (27%)]\tLoss: 0.850665\n",
      "train/loss 0.8506652116775513 100\n",
      "Train Epoch: 0 [25600/47676 (54%)]\tLoss: 0.750864\n",
      "train/loss 0.7508643865585327 200\n",
      "Train Epoch: 0 [38400/47676 (80%)]\tLoss: 0.740783\n",
      "train/loss 0.7407829165458679 300\n",
      "\n",
      "Test set: Average loss: 1.8760, Accuracy: 11465/25428 (45%)\n",
      "\n",
      "learning rate: 0.001, epoch 1\n",
      "Train Epoch: 1 [0/47676 (0%)]\tLoss: 0.773639\n",
      "train/loss 0.7736390829086304 373\n",
      "Train Epoch: 1 [12800/47676 (27%)]\tLoss: 0.640580\n",
      "train/loss 0.6405799388885498 473\n",
      "Train Epoch: 1 [25600/47676 (54%)]\tLoss: 0.515301\n",
      "train/loss 0.5153011083602905 573\n",
      "Train Epoch: 1 [38400/47676 (80%)]\tLoss: 0.583963\n",
      "train/loss 0.5839627385139465 673\n",
      "\n",
      "Test set: Average loss: 0.7514, Accuracy: 19147/25428 (75%)\n",
      "\n",
      "learning rate: 0.001, epoch 2\n",
      "Train Epoch: 2 [0/47676 (0%)]\tLoss: 0.730551\n",
      "train/loss 0.7305508255958557 746\n",
      "Train Epoch: 2 [12800/47676 (27%)]\tLoss: 0.732026\n",
      "train/loss 0.7320258021354675 846\n",
      "Train Epoch: 2 [25600/47676 (54%)]\tLoss: 0.653401\n",
      "train/loss 0.6534008383750916 946\n",
      "Train Epoch: 2 [38400/47676 (80%)]\tLoss: 0.669041\n",
      "train/loss 0.6690406799316406 1046\n",
      "\n",
      "Test set: Average loss: 1.4359, Accuracy: 17149/25428 (67%)\n",
      "\n",
      "learning rate: 0.001, epoch 3\n",
      "Train Epoch: 3 [0/47676 (0%)]\tLoss: 0.562576\n",
      "train/loss 0.5625759363174438 1119\n",
      "Train Epoch: 3 [12800/47676 (27%)]\tLoss: 0.615991\n",
      "train/loss 0.6159906983375549 1219\n",
      "Train Epoch: 3 [25600/47676 (54%)]\tLoss: 0.590728\n",
      "train/loss 0.5907279253005981 1319\n",
      "Train Epoch: 3 [38400/47676 (80%)]\tLoss: 0.719885\n",
      "train/loss 0.7198852896690369 1419\n",
      "\n",
      "Test set: Average loss: 0.7621, Accuracy: 18627/25428 (73%)\n",
      "\n",
      "learning rate: 0.001, epoch 4\n",
      "Train Epoch: 4 [0/47676 (0%)]\tLoss: 0.621257\n",
      "train/loss 0.6212573647499084 1492\n",
      "Train Epoch: 4 [12800/47676 (27%)]\tLoss: 0.461762\n",
      "train/loss 0.4617622196674347 1592\n",
      "Train Epoch: 4 [25600/47676 (54%)]\tLoss: 0.606997\n",
      "train/loss 0.6069973111152649 1692\n",
      "Train Epoch: 4 [38400/47676 (80%)]\tLoss: 0.815571\n",
      "train/loss 0.8155708312988281 1792\n",
      "\n",
      "Test set: Average loss: 0.7027, Accuracy: 18945/25428 (75%)\n",
      "\n",
      "learning rate: 0.001, epoch 5\n",
      "Train Epoch: 5 [0/47676 (0%)]\tLoss: 0.595598\n",
      "train/loss 0.5955979228019714 1865\n",
      "Train Epoch: 5 [12800/47676 (27%)]\tLoss: 0.613667\n",
      "train/loss 0.6136671900749207 1965\n",
      "Train Epoch: 5 [25600/47676 (54%)]\tLoss: 0.473278\n",
      "train/loss 0.4732777774333954 2065\n",
      "Train Epoch: 5 [38400/47676 (80%)]\tLoss: 0.617654\n",
      "train/loss 0.6176538467407227 2165\n",
      "\n",
      "Test set: Average loss: 0.8710, Accuracy: 18085/25428 (71%)\n",
      "\n",
      "learning rate: 0.001, epoch 6\n",
      "Train Epoch: 6 [0/47676 (0%)]\tLoss: 0.535646\n",
      "train/loss 0.5356459617614746 2238\n",
      "Train Epoch: 6 [12800/47676 (27%)]\tLoss: 0.499171\n",
      "train/loss 0.49917125701904297 2338\n",
      "Train Epoch: 6 [25600/47676 (54%)]\tLoss: 0.506880\n",
      "train/loss 0.5068795680999756 2438\n",
      "Train Epoch: 6 [38400/47676 (80%)]\tLoss: 0.521882\n",
      "train/loss 0.5218824744224548 2538\n",
      "\n",
      "Test set: Average loss: 0.9107, Accuracy: 19034/25428 (75%)\n",
      "\n",
      "learning rate: 0.001, epoch 7\n",
      "Train Epoch: 7 [0/47676 (0%)]\tLoss: 0.572914\n",
      "train/loss 0.5729140639305115 2611\n",
      "Train Epoch: 7 [12800/47676 (27%)]\tLoss: 0.528383\n",
      "train/loss 0.5283828377723694 2711\n",
      "Train Epoch: 7 [25600/47676 (54%)]\tLoss: 0.603432\n",
      "train/loss 0.6034315824508667 2811\n",
      "Train Epoch: 7 [38400/47676 (80%)]\tLoss: 0.430792\n",
      "train/loss 0.4307916760444641 2911\n",
      "\n",
      "Test set: Average loss: 0.6665, Accuracy: 19976/25428 (79%)\n",
      "\n",
      "learning rate: 0.001, epoch 8\n",
      "Train Epoch: 8 [0/47676 (0%)]\tLoss: 0.282543\n",
      "train/loss 0.282542884349823 2984\n",
      "Train Epoch: 8 [12800/47676 (27%)]\tLoss: 0.425038\n",
      "train/loss 0.4250379204750061 3084\n",
      "Train Epoch: 8 [25600/47676 (54%)]\tLoss: 0.320117\n",
      "train/loss 0.32011717557907104 3184\n",
      "Train Epoch: 8 [38400/47676 (80%)]\tLoss: 0.375315\n",
      "train/loss 0.3753148317337036 3284\n",
      "\n",
      "Test set: Average loss: 0.7580, Accuracy: 18815/25428 (74%)\n",
      "\n",
      "learning rate: 0.001, epoch 9\n",
      "Train Epoch: 9 [0/47676 (0%)]\tLoss: 0.393982\n",
      "train/loss 0.39398193359375 3357\n",
      "Train Epoch: 9 [12800/47676 (27%)]\tLoss: 0.291868\n",
      "train/loss 0.29186785221099854 3457\n",
      "Train Epoch: 9 [25600/47676 (54%)]\tLoss: 0.414067\n",
      "train/loss 0.41406700015068054 3557\n",
      "Train Epoch: 9 [38400/47676 (80%)]\tLoss: 0.419049\n",
      "train/loss 0.4190487563610077 3657\n",
      "\n",
      "Test set: Average loss: 0.9184, Accuracy: 17485/25428 (69%)\n",
      "\n",
      "learning rate: 1e-05, epoch 10\n",
      "Train Epoch: 10 [0/47676 (0%)]\tLoss: 0.310546\n",
      "train/loss 0.31054559350013733 3730\n",
      "Train Epoch: 10 [12800/47676 (27%)]\tLoss: 0.254297\n",
      "train/loss 0.2542973756790161 3830\n",
      "Train Epoch: 10 [25600/47676 (54%)]\tLoss: 0.268855\n",
      "train/loss 0.2688547968864441 3930\n",
      "Train Epoch: 10 [38400/47676 (80%)]\tLoss: 0.182476\n",
      "train/loss 0.18247616291046143 4030\n",
      "\n",
      "Test set: Average loss: 0.6616, Accuracy: 20832/25428 (82%)\n",
      "\n",
      "learning rate: 0.0001, epoch 11\n",
      "Train Epoch: 11 [0/47676 (0%)]\tLoss: 0.142954\n",
      "train/loss 0.14295423030853271 4103\n",
      "Train Epoch: 11 [12800/47676 (27%)]\tLoss: 0.180955\n",
      "train/loss 0.180955171585083 4203\n",
      "Train Epoch: 11 [25600/47676 (54%)]\tLoss: 0.122527\n",
      "train/loss 0.12252733111381531 4303\n",
      "Train Epoch: 11 [38400/47676 (80%)]\tLoss: 0.126125\n",
      "train/loss 0.12612484395503998 4403\n",
      "\n",
      "Test set: Average loss: 0.7187, Accuracy: 20722/25428 (81%)\n",
      "\n",
      "learning rate: 0.0001, epoch 12\n",
      "Train Epoch: 12 [0/47676 (0%)]\tLoss: 0.231354\n",
      "train/loss 0.2313539683818817 4476\n",
      "Train Epoch: 12 [12800/47676 (27%)]\tLoss: 0.123392\n",
      "train/loss 0.12339239567518234 4576\n",
      "Train Epoch: 12 [25600/47676 (54%)]\tLoss: 0.123295\n",
      "train/loss 0.12329502403736115 4676\n",
      "Train Epoch: 12 [38400/47676 (80%)]\tLoss: 0.119284\n",
      "train/loss 0.11928372085094452 4776\n",
      "\n",
      "Test set: Average loss: 0.7809, Accuracy: 20842/25428 (82%)\n",
      "\n",
      "learning rate: 0.0001, epoch 13\n",
      "Train Epoch: 13 [0/47676 (0%)]\tLoss: 0.108544\n",
      "train/loss 0.10854409635066986 4849\n",
      "Train Epoch: 13 [12800/47676 (27%)]\tLoss: 0.125123\n",
      "train/loss 0.12512265145778656 4949\n",
      "Train Epoch: 13 [25600/47676 (54%)]\tLoss: 0.121376\n",
      "train/loss 0.1213756799697876 5049\n",
      "Train Epoch: 13 [38400/47676 (80%)]\tLoss: 0.154268\n",
      "train/loss 0.15426842868328094 5149\n",
      "\n",
      "Test set: Average loss: 0.8098, Accuracy: 20702/25428 (81%)\n",
      "\n",
      "learning rate: 0.0001, epoch 14\n",
      "Train Epoch: 14 [0/47676 (0%)]\tLoss: 0.068140\n",
      "train/loss 0.06814026832580566 5222\n",
      "Train Epoch: 14 [12800/47676 (27%)]\tLoss: 0.082768\n",
      "train/loss 0.08276833593845367 5322\n",
      "Train Epoch: 14 [25600/47676 (54%)]\tLoss: 0.117538\n",
      "train/loss 0.11753781139850616 5422\n",
      "Train Epoch: 14 [38400/47676 (80%)]\tLoss: 0.119158\n",
      "train/loss 0.11915821582078934 5522\n",
      "\n",
      "Test set: Average loss: 0.8647, Accuracy: 20670/25428 (81%)\n",
      "\n",
      "learning rate: 0.0001, epoch 15\n",
      "Train Epoch: 15 [0/47676 (0%)]\tLoss: 0.062556\n",
      "train/loss 0.06255607306957245 5595\n",
      "Train Epoch: 15 [12800/47676 (27%)]\tLoss: 0.045715\n",
      "train/loss 0.04571504518389702 5695\n",
      "Train Epoch: 15 [25600/47676 (54%)]\tLoss: 0.103046\n",
      "train/loss 0.10304597020149231 5795\n",
      "Train Epoch: 15 [38400/47676 (80%)]\tLoss: 0.103890\n",
      "train/loss 0.10388960689306259 5895\n",
      "\n",
      "Test set: Average loss: 0.9330, Accuracy: 20755/25428 (82%)\n",
      "\n",
      "learning rate: 0.0001, epoch 16\n",
      "Train Epoch: 16 [0/47676 (0%)]\tLoss: 0.033561\n",
      "train/loss 0.03356088325381279 5968\n",
      "Train Epoch: 16 [12800/47676 (27%)]\tLoss: 0.065610\n",
      "train/loss 0.06560984253883362 6068\n",
      "Train Epoch: 16 [25600/47676 (54%)]\tLoss: 0.043033\n",
      "train/loss 0.043033208698034286 6168\n",
      "Train Epoch: 16 [38400/47676 (80%)]\tLoss: 0.068179\n",
      "train/loss 0.06817885488271713 6268\n",
      "\n",
      "Test set: Average loss: 0.9660, Accuracy: 20602/25428 (81%)\n",
      "\n",
      "learning rate: 0.0001, epoch 17\n",
      "Train Epoch: 17 [0/47676 (0%)]\tLoss: 0.028791\n",
      "train/loss 0.02879060059785843 6341\n",
      "Train Epoch: 17 [12800/47676 (27%)]\tLoss: 0.038482\n",
      "train/loss 0.03848184272646904 6441\n",
      "Train Epoch: 17 [25600/47676 (54%)]\tLoss: 0.069930\n",
      "train/loss 0.06993000209331512 6541\n",
      "Train Epoch: 17 [38400/47676 (80%)]\tLoss: 0.033458\n",
      "train/loss 0.03345772996544838 6641\n",
      "\n",
      "Test set: Average loss: 1.0334, Accuracy: 20721/25428 (81%)\n",
      "\n",
      "learning rate: 0.0001, epoch 18\n",
      "Train Epoch: 18 [0/47676 (0%)]\tLoss: 0.033578\n",
      "train/loss 0.033578116446733475 6714\n",
      "Train Epoch: 18 [12800/47676 (27%)]\tLoss: 0.043704\n",
      "train/loss 0.043703507632017136 6814\n",
      "Train Epoch: 18 [25600/47676 (54%)]\tLoss: 0.050516\n",
      "train/loss 0.05051606893539429 6914\n",
      "Train Epoch: 18 [38400/47676 (80%)]\tLoss: 0.045432\n",
      "train/loss 0.045431602746248245 7014\n",
      "\n",
      "Test set: Average loss: 1.0625, Accuracy: 20579/25428 (81%)\n",
      "\n",
      "learning rate: 0.0001, epoch 19\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 19 [0/47676 (0%)]\tLoss: 0.044677\n",
      "train/loss 0.044677067548036575 7087\n",
      "Train Epoch: 19 [12800/47676 (27%)]\tLoss: 0.013516\n",
      "train/loss 0.013516200706362724 7187\n",
      "Train Epoch: 19 [25600/47676 (54%)]\tLoss: 0.025469\n",
      "train/loss 0.025468530133366585 7287\n",
      "Train Epoch: 19 [38400/47676 (80%)]\tLoss: 0.037730\n",
      "train/loss 0.037730395793914795 7387\n",
      "\n",
      "Test set: Average loss: 1.1309, Accuracy: 20597/25428 (81%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_freq=100\n",
    "losses = []\n",
    "epoch_loss = []\n",
    "val_losses = []\n",
    "\n",
    "num_epochs=20\n",
    "for epoch in range(num_epochs):\n",
    "    #writer.add_scalar('train/learning_rate', lr_scheduler.get_lr()[0], epoch)\n",
    "    print(f'learning rate: {lr_scheduler.get_lr()[0]}, epoch {epoch}')\n",
    "    \n",
    "    model.train()\n",
    "    for idx, batch in enumerate(train_loader):\n",
    "        \n",
    "        # account for bad samples\n",
    "        batch = make_good_batch(batch)\n",
    "        \n",
    "        # extract samples\n",
    "        image,target = batch['image'], batch['class_code']\n",
    "        image, target = image.to(device), target.to(device)\n",
    "        \n",
    "        output = model(image.float()) \n",
    "        loss = criterion(output, target.long())\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if idx % print_freq == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, idx * len(image), len(train_loader.dataset), 100. * idx / len(train_loader), loss.item()))\n",
    "            #writer.add_scalar('train/loss', loss.item(), len(data_loader) * epoch + idx)\n",
    "            print(f'train/loss {loss.item()} {len(train_loader) * epoch + idx}')\n",
    "            losses.append((idx, loss.item()))\n",
    "    \n",
    "    #train_one_epoch(model, criterion, optimizer, train_loader, device, epoch, args.print_freq, writer)\n",
    "    \n",
    "    lr_scheduler.step()\n",
    "    \n",
    "    # average loss for the epoch\n",
    "    epoch_loss.append(np.array(losses)[:,1].mean())\n",
    "    \n",
    "    ## need validation dataset\n",
    "    #evaluate(epoch, model, criterion, val_loader, device, writer)\n",
    "    model.eval()\n",
    "    val_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for idx, batch in enumerate(val_loader):\n",
    "\n",
    "            # account for bad samples\n",
    "            batch = make_good_batch(batch)\n",
    "\n",
    "            # extract samples\n",
    "            image,target = batch['image'], batch['class_code']\n",
    "\n",
    "            image = image.to(device, non_blocking=True)\n",
    "            target = target.to(device, non_blocking=True)\n",
    "            output = model(image)\n",
    "            val_loss += criterion(output, target).item()\n",
    "\n",
    "            pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability\n",
    "            correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "\n",
    "        val_loss /= len(val_loader.dataset)/val_loader.batch_size\n",
    "\n",
    "        val_losses.append(val_loss)\n",
    "        \n",
    "        print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "            val_loss, correct, len(val_loader.dataset),\n",
    "            100. * correct / len(val_loader.dataset)))\n",
    "    \n",
    "    #torch.save(model.state_dict(), os.path.join(args.ckp_dir, \"cls_epoch_{}.pth\".format(epoch)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fe1e5da2070>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXzU1b3/8ddnZpJAQgiEhC0Bwir7DuKOiAu44Fr1Vmu9Vq3XWm3t7a3Xbra2t7e/2l6XVvC2XmvrrrhTlIriBsgiYNgRCIQtIUAWsifn98dMQlYSyODMN7yfj0cemeU73/kkkHdOzjnfc8w5h4iIeJ8v0gWIiEh4KNBFRNoJBbqISDuhQBcRaScU6CIi7UQgUm+ckpLiMjIyIvX2IiKetGLFiv3OudSmnotYoGdkZLB8+fJIvb2IiCeZWVZzz6nLRUSknVCgi4i0Ewp0EZF2QoEuItJOKNBFRNoJBbqISDuhQBcRaSc8F+gb9xby0LsbySsqi3QpIiJRxXOB/mVuEY8u3EKuAl1EpB7PBXpcIFhyWUV1hCsREYkungv02FCgl1cp0EVE6vJeoPtDgV6pQBcRqctzgR4X4wegrLIqwpWIiEQXzwW6WugiIk3zXqDXDIoq0EVE6vFcoNfMclELXUSkPs8GulroIiL1tRjoZtbHzN43s/VmttbM7m7imKlmlm9mq0IfPz0x5daZtqhAFxGppzVb0FUC9zrnVppZIrDCzBY459Y1OO4j59wl4S+xPs1DFxFpWostdOfcHufcytDtQmA9kHaiC2tOzSwXXSkqIlLfMfWhm1kGMA5Y2sTTp5nZajP7h5mNaOb1t5nZcjNbnpube8zFAgT8Pvw+o7xK89BFROpqdaCbWSfgFeAe51xBg6dXAv2cc2OAR4HXmjqHc+4J59xE59zE1NTU462ZWL9PfegiIg20KtDNLIZgmD/jnJvb8HnnXIFzrih0ex4QY2YpYa20jrgYn2a5iIg00JpZLgb8BVjvnPt9M8f0DB2HmU0OnTcvnIXWpRa6iEhjrZnlcgZwI/CFma0KPfafQF8A59xs4GrgDjOrBEqA65xz7gTUCwRnuijQRUTqazHQnXMfA9bCMY8Bj4WrqJbEBnyUadqiiEg9nrtSFCAu4Ne0RRGRBjwZ6LEBny4sEhFpwJOBHuf3Ua710EVE6vFmoGvaoohII54MdE1bFBFpzJuBrmmLIiKNeDbQ1eUiIlKfJwM9Ti10EZFGPBnomrYoItKYNwPd71cLXUSkAU8GenDaouahi4jU5clAj/X7qKhyVFefsPW/REQ8x5uBrn1FRUQa8WSgx4UCXVMXRUSO8HSga2BUROQITwa6ulxERBrzdKCXVWimi4hIDU8GelzAD6iFLiJSlycDPdavPnQRkYa8GegaFBURacSTga5piyIijXky0NVCFxFpzNOBrha6iMgRngz0mlkuWqBLROQIjwa6ulxERBryZKDrSlERkca8Gej+mitFFegiIjU8GehxMWqhi4g01GKgm1kfM3vfzNab2Vozu7uJY8zMHjGzLWa2xszGn5hyg3SlqIhIY4FWHFMJ3OucW2lmicAKM1vgnFtX55gZwODQx6nA46HPJ0TA78NnCnQRkbpabKE75/Y451aGbhcC64G0BofNAp52QUuALmbWK+zV1hEX8GvaoohIHcfUh25mGcA4YGmDp9KAnXXuZ9M49DGz28xsuZktz83NPbZKG4gN+NRCFxGpo9WBbmadgFeAe5xzBQ2fbuIljXZwds494Zyb6JybmJqaemyVNhAb8GlQVESkjlYFupnFEAzzZ5xzc5s4JBvoU+d+OrC77eU1Ly7g07RFEZE6WjPLxYC/AOudc79v5rA3gG+EZrtMAfKdc3vCWGcjsQEfZWqhi4jUas0slzOAG4EvzGxV6LH/BPoCOOdmA/OAmcAWoBi4Ofyl1hfrVx+6iEhdLQa6c+5jmu4jr3uMA+4MV1GtERfwabVFEZE6PHmlKASnLZZr2qKISC3PBrqmLYqI1OftQNegqIhILc8GuqYtiojU59lAVwtdRKQ+7wa6pi2KiNTj2UCPi9G0RRGRujwb6LF+v1roIiJ1eDfQNW1RRKQebwd6VTXV1Y0WdRQROSl5NtDjAtpXVESkLgW6iEg74dlAjw1oo2gRkbo8G+g1LXRNXRQRCfJsoKuFLiJSn3cD3e8HFOgiIjW8G+i1XS5aE11EBDwc6HHqchERqcezga4+dBGR+jwf6JrlIiIS5NlA17RFEZH6PB/oulJURCTIs4GuaYsiIvV5NtDjYjRtUUSkLs8Geqxfs1xEROrybqBr2qKISD2eD3TNchERCfJsoAd8hs/UQhcRqeHZQDez2m3oRESkFYFuZk+aWY6ZZTbz/FQzyzezVaGPn4a/zKbF+n2UVWiWi4gIQKAVxzwFPAY8fZRjPnLOXRKWio5BXIxfLXQRkZAWW+jOuQ+BA19BLccs1u/ToKiISEi4+tBPM7PVZvYPMxvR3EFmdpuZLTez5bm5uW1+07iAT4OiIiIh4Qj0lUA/59wY4FHgteYOdM494Zyb6JybmJqa2uY3jg2ohS4iUqPNge6cK3DOFYVuzwNizCylzZW1glroIiJHtDnQzaynmVno9uTQOfPaet7WiFWgi4jUanGWi5k9B0wFUswsG/gZEAPgnJsNXA3cYWaVQAlwnXPOnbCK64gN+Cgp17RFERFoRaA7565v4fnHCE5r/MrFBfzkl1RE4q1FRKKOZ68UheC0RXW5iIgEeTvQNctFRKSWpwNds1xERI7wdKBrlouIyBEKdBGRdsLTgR4X8KsPXUQkxNOBXrMe+lc07V1EJKp5OtDjavYV1RK6IiLeDvRYv/YVFRGp4elAj4sJtdAV6CIi3g70mha6Al1ExOuBHlCXi4hIDU8HelzAD6iFLiICHg/0mha6Al1EpL0EepXWRBcR8XSg18xDL6tQC11ExNOBXjsoqguLREQ8HuiatigiUsvTgR6naYsiIrU8HuiatigiUsPTga5piyIiR7SLQC+r1LRFERFPB3qcWugiIrU8HejqchEROcLTgR7wGWba4EJEBDwe6GZGXMCnaYsiIng80CF4cZG6XERE2kOgB/xqoYuI0IpAN7MnzSzHzDKbed7M7BEz22Jma8xsfPjLbF6wy0XTFkVEWtNCfwq46CjPzwAGhz5uAx5ve1mtFxdQl4uICLQi0J1zHwIHjnLILOBpF7QE6GJmvcJVYEtiFegiIkB4+tDTgJ117meHHmvEzG4zs+Vmtjw3NzcMbx0MdPWhi4iEJ9CticdcUwc6555wzk10zk1MTU0Nw1ury0VEpEY4Aj0b6FPnfjqwOwznbZXYgE8XFomIEJ5AfwP4Rmi2yxQg3zm3JwznbRXNQxcRCQq0dICZPQdMBVLMLBv4GRAD4JybDcwDZgJbgGLg5hNVbFNiNW1RRARoRaA7565v4XkH3Bm2io5RXMCvFrqICO3iSlF1uYiIQDsJdE1bFBFpB4GuaYsiIkGeD/TYgI8yTVsUEfF+oMeFpi0Gx2ZFRE5eng/0HkkdAFiRdTDClYiIRJbnA/2KcWmkdIrjoXc3RboUEZGI8nygx8cGuPPcgSzemsenW/ZHupyocLiskpue/Iz1ewoiXYqIfIU8H+gA10/uS6+kDvzu3Y3qSwc+/TKPRZtyeWP1V7akjohEgXYR6B1i/Nw1bTArdxzig43hWZbXy5ZuzQNg+fajLWMvIu1Nuwh0gGsmptM3OZ7fvbuR6uqmW+l5RWU8+NY6PmnnXTNLtwWDfPXOfEortM6NyMmi3QR6jN/H3ecNZu3uAt5Zu7fec8455q7MZvrvF/Hnj7fxq7fXR6jKE6+gtIK1u/MZ2jOR8qpqMnflR7okEfmKtJtAB7h8XBoDUxP4/ouruXbOYh58ax2vrMjmG09+xvdfXE1GSgK3nNmfdXsK2m3Qrdh+kGoHd547CIBl2zWdU+Rk0eJqi17i9xmzb5jAXxdvJ3NXAX9bkkVZZTUJsX4euGwEN0zpR1FZJX9fksULy3YyMi3pmM6ffbCYV1fuorSyivLKasoqq5ncP5lLRvc+MV/QcViyNY8YvzF9WA8GpCaE+tEHRrosEfkKtKtABxjcI5EHLx8FQGVVNVv3H6ZbQizdOsUBkNQxhhkje/Laql3cf/EwOsT4W33u372zkddW7cbvM2L9Pszgb0uy6N2lI+P7dg3r17Fs+wE+2ryf700fjFlTu/w1bcm2A4xJ70LHWD+T+iUzf+1eqqsdPl/rzyEi3tSuulwaCvh9DOmRWBvmNb42qQ+FpZXMz9zbzCsbKymv4t11+7huUh++/PVM1v/yIj67fzq9kzryw5fXhH3w8b/mreeR9zazaFPrZ+0UlVWSuSufUwckAzAxoyv5JRV8mVsU1tpEJDq160BvzpT+3eibHM8Ly3a2+jX/XL+P4vIqZo1Nq32sU1yAX185ii05RTy6cHPY6tuwt4CVOw5hxjHNrV+RdZCqasep/bsBMCkjGOzqRxc5OZyUge7zGV+bmM7irXlk5R1u1WteX7Wbnp07MLl/cr3HzxmSytUT0pm9aGvYBlqf/2wnsX4f988cRuauglb/JbF0ax4BnzGhX7D7p1+3eFI6xWk+ushJ4qQMdICrJ/TBZ/Di8pZb6YeKy1m0KYdLx/TC30Rf9E8uHk5yQiw/eGk1ZZVVrNxxkN/O38Clj37Myyuyj6mukvIq5q7MZsaontx8Rn8Gde/E797dSFUzc+vrWrI1j1HpSSTEBYdGzIxJGV1ZlqVAFzkZnLSB3jOpA+cMSeXlFdlUtrCe+j8y91JR5ep1t9SVFB/Dry4fyYa9hYz7xQKu/NOnzPlwKzsPFvPf8zccU//6vC/2UFBayfWT++L3GfeeP4Qvcw/z6ue7jvq64vJK1mTn13a31JiYkczOAyXszS9tdQ0i4k0nbaADXDupD/sKyloceHx91S4GpCYwonfnZo+5YERP7pg6kPOG9eDh68ay8sfn8/jXJ5BbWMZLrfgroMZzn+1gQEoCp4a6di4a2ZORaZ35n39uOurOTCuzDlFZ7WoHRGtMygh2vyxXK12k3TupA33a0B6kdIrj2aU7mj1mT34JS7cdYNaYtBanD/7HRUN59PpxzBqbRlJ8DFMGJDOxX1dmL9raqm3yNu0rZHnWQa6f3Lf2vcyMH1xwCtkHS3hhWfN1Lt2Wh99nTOxXf/rksF6d6RjjZ/kxDoxqkTMR7zmpAz024OO6SX1YuDGH7IPFTR7z1uo9OAeXjT32i4fMjO9MG8SuQyW81qDLZPGXeVzwh0X8/t2NHCouB4Kt81i/j6smpNc79pwhqUzs15X//Whbs0G7ZGseI3t3JrFDTL3HY/w+xvXtwrLQwOi2/Yd5+J+b+dEra6hopqvp3hdXc8Nflja7Jo6IRKeTOtABrj+1L0YwTJvy+updjElPon9KwnGd/5whqYxKS+JPH2yp7atfkXWAW/66jP1F5TyycAtn/GYhv/nHBuau3MWFI3uSnBBb7xxmxtUT0tlxoJh1TaxxXlBaweqd+UwZ0K3RcxDsR1+/p4DLHvuYc3/3AX/45yaeX7aTD5voajp4uJw3Vu/iky15PNvM9yRcnHPc+exK7n/1ixP6PiIni5M+0NO6dGTa0O68sGxno26RLTlFZO4q4LJmBkNbw8y489xBbM8r5u0v9rAm+xDffHIZPTp3YP49Z/HOPWczbVgP5nz4JfklFfzL5L5Nnuf84T3wGU1OYZz/xV7Kq6qZMapXk6+dekoq1Q6qneP+mcP46IfnkpwQy9yVjQda3/5iDxVVjoGpCfz3/A3kFJy4wdRnlu7g7TV7eHlFNiXlWhVSpK1O+kAH+PqUfuwvKmd+nVUaq6odP3ktkw4xPi4d3XRQttYFw3swpEcnHnp3Ezf+5TOS4mN45lun0j2xA6f0TOTR68ex4Hvn8PB1Y5nSYFCzRrdOcUzun8w/mgj011btIqNbPGPSm16bZnzfrqz/xUW8dddZ3Hr2APokx3PZmN4sWL+P/OKKese+vmoXg7p34s83TaKsspoH3lrXpq992fYD/PyNteSX1H+f7fsP86u315PWpSNlldUs3tq+lzQW+Soo0IFzBqfSJ7kjf1+SVfvYI+9tZvHWPH45ayTdO3do0/l9vmArfceBYjrG+Hnu1in07tKx3jGDundi1tijD7zOGNmLLTlFbMkprH1sX0Epi7fmcVkLr+0YW3/NmqvGp1NeWc3bX+ypfWzngWKWbT/IFePS6J+SwF3nDuLtNXt4f0POsX7JAHy6ZT83/mUpT326nWtmf1o7TlFV7bj3pdUE/Mazt55KQqyff64/vvcQkSMU6AQD9+un9uOzbQfYtK+QT7bs55GFm7lqfDrXTOwTlve4ZHRvfnLJcF64fQp9kuOP6xwXjugJ1O92eXP1bpyDy49x0HZkWmcGd+/E3JVHLnyq2bLusjHBc91+zkAGd+/Ej1/LpLi88pjO//Hm/dz81DL6JSfw2L+MY09+KVf86VPWZB/iiQ+3siLrIL+cNZJ+3RI4a3AqC9fnaGaNSBu1KtDN7CIz22hmW8zsR008P9XM8s1sVejjp+Ev9cS6ZkI6sX4fD7+3mbufX8XA1E788vIRYTu/32fccmZ/+nU7vsFVCF4MNa5vl3rdLq+t2sXo9CQGpHY6pnOZGVeOT2d51kGy8g7jnOO1z3cxKaNr7S+c2ICPX185il2HSrj3xdUUlla0cNagDzflcstfl9E/JYFnbz2VS0b3Zu4dpxPr93HtnCX8fsFGZozsyazQL6Fpw7qzt6C0yQFfEWm9FgPdzPzAH4EZwHDgejMb3sShHznnxoY+fhHmOk+4bp3imDmqJ2+v2UNRWQV/+vp44mOjb3XhGSN7snZ3ATvyimsHbZu7grUll4/rjRnMXbmLdXsK2JxT1OhckzKSuW/GUN5Zu5eZj3zU7Low+SUVvLl6N99/cRXfeno5A1I78eytU2pXuhzcI5HX7jyDIT0TSU6I5cHLR9Z2EZ17SnfM4D11u4i0SWsSazKwxTm3FcDMngdmAW0bLYtCN5/Rn3fX7eMXs0YypEdipMtp0kUjevHreRt4Z+1eCksr8BnHPWjbK6kjZwxMYe7n2RwuqyTgMy5uYqbM7ecMZGJGV+55YRVfm7OYO88dxNg+Xdi2/zBb9x9m095CPt95iKpqR5f4GC4Z3YufXDycrg2mX6YmxvHqHadTWllV75dlamIcY9K78N6GHL573uDj+lpEpHWBngbUvXY9Gzi1ieNOM7PVwG7gB865tWGo7ys1pk8XPv/p+cQFWr/pxVetb7d4hvfqzLzMPeQVlXP6wJQ2DdpeOT6N77+4mr8tyWLqKd0bhXCNCf2Smffds/j5G+t4dOGW2se7xMcwMLUTd5wzkHOHpjK2T9cmFzCr4fNZk3/5nDe0Ow8t2EROYSndE9s2CC1ysmpNoDf109lw9Gol0M85V2RmM4HXgEZNLTO7DbgNoG/fpudbR1o0h3mNGSN78tCCTQDcNW1Qm8514YiexMdmUlxexeXjjj6wmtghhoe+NoYbpvSl2jkGpHRq9hfAsTpvWA8eWrCJDzbk8rVJ4RmIFjnZtGZQNBuo+xOWTrAVXss5V+CcKwrdngfEmFlKwxM5555wzk10zk1MTU1tQ9kntxmjgrNdYgM+LhrZs03nSogLcPGoXiR1jGH6sB6tes24vl2Z0C85bGEOMKxXIr2SOvDehn1hO6fIyaY1gb4MGGxm/c0sFrgOeKPuAWbW00IjXGY2OXTevHAXK0GDuicyKi2JS0b3arR2y/H4+WUjePu7Zx7T/qrhZmZMG9qdjzbvD/t2fiInixa7XJxzlWb2HeAdwA886Zxba2bfDj0/G7gauMPMKoES4DqnScUn1EvfPu2ofdXHIiEuULspRiRNH9aDZ5buYOm2A5wzRH/BiRyrVv0Uh7pR5jV4bHad248Bj4W3NDmaSLamT5TTBnajQ4yPBev2NhnopRVVPLpwM/96Rv9GG3+3hXOO/JIKDhVX0Dc5Hl+YflGKfNUi3ywTCekQ42fa0O7Mz9zLzy8dQcBfv0fwjdW7+eP7X1JUWskDs0a26b2cc/z09bUs3JBDblFZ7cJs/zlzKLedPbBN5xaJFF36L1HlsjG92V9UzqdfNh6CqVlT/rllO9nXilUgdx4oZkde0+vcr8g6yN+WZDEgNYGbT8/gxxcPY0K/rjzx4Vb14YtnKdAlqkw9pTuJcYHadWVq7MkvYfHWPK4cn0ZVtWPOoq1Nvv5QcTnPLM3i6sc/5azfvs9lf/y4ySULnl6cRWKHALNvmMB9M4fxrbMG8O8XnsL+ovJWbRwuEo0U6BJVOsT4uWBET97J3Fuvpfz6quAiZN+dNpjLx6bx7GdZ5BaW1XvtU59sY/Kv3uP+VzM5VFLBt87sz6HiCp76ZHu943IKSpn3xR6umdCn3mDwqf2TmdCvK3MWbW12NyeRaKZAl6hz2djeFJZV8sHG4I5KzjleXbmL8X27kJGSwJ3nDqS8spo/f3Sklf7isp38/M11nDk4hbfuOpMF3zubH18ynOnDuvO/H22loE4r/bnPdlJZ7bjxtH713tfM+LepA9l1qIQ3VtX/C0HECxToEnXOGNiNbgmxvLkmGKrr9xSycV8hV4wLLhw2ILUTl43pzdOLs8grKuMfX+zhR3PXcNbgFB6/YTwj05JqF/66Z/oQCkor+b+PtwNQUVXNM0uzOGdIapPbCk4b2p2hPRN5fNGX2lNVPEeBLlEn4Pcxc1Qv3lu/j8Nllbz6eTYBn3HJ6CNLE3xn2iBKK6u496XV3P38Ksb17cqcGyc0WrphZFoS5w/vwZ8/3kp+SQXvrN1LTmEZN53er+HbAsFW+h1TB7Ilp4h31+mqVfEWBbpEpcvG9qa0opr5mXt5fdXuRguHDeqeyMxRvfhgYy4DUhN48qZJzS53fM/0wRSWVvLkx9t4+tMs+iR35Jwh3Zt974tH9aJvcjyPf7BFm26IpyjQJSpN6NuV3kkdghtVF5Zx5fjGa77/6KKhXD+5D0/fMpmk+OaXQBjRO4kLRwQ34v5s+wFunNLvqFfZBvw+vn3OQFZn52vGi3iKAl2iks9nXDqmNzmFZSR2CDBtaOMWdZ/keP7rytGtWm73nulDKK2oJi7g42ut2FbwmonpnDkohf98NZN/qutFPEKBLlHr0tDepheP6tXmpQ6G9erMt88ZyN3TB9MlvuVVImP8PmbfOIGRvTtz57MrWdbMTk0i0cQi1Uc4ceJEt3z58oi8t3iDc44Xl+/k7CGp9ErqGJEa8orKuGbOYnILy3jhttOoqnYs2pTDBxtzqax2PHfrFDrGtr91dSR6mdkK59zEJp9ToIscXfbBYq5+fDF76yw3MKxXZ9bvKeCuaYO494JTIlidnGyOFuhanEukBeld4/n7tybz5CfbmdivK2cNTiU1MY7vvbCKOYu2ctX4dDKamNN+vJxzHC6volMULGks3qI+dJFWGNQ9kV9fMYorx6eTmhhcuve+GUOJDfh44M21YZ3e+MzSHUx8cAFZeYfDdk45OSjQRY5T984duGf6YN7fmMt763PCcs6aNd9LK6qZ3cwCZCLNUaCLtMFNp2cwuHsnHnhrbViW3X1pRTb7CsoYnZ7EKyuy2Zvf8jLBIjUU6CJtEOP38cCsEew8UMJjC7c0e9yKrAMcPFx+1HOVV1Yz+4MvGd+3C3/8l/FUOVdvATKRlijQRdro9IEpXDEujcfe38JPXsukrPJIS720oor75n7BVY8v5tonFpNf0nht9hpzV2az61AJ3z1vMH2S45k1pjfPLN3R4i8CkRoKdJEw+O3Vo7n97AH8bUkW18xezM4DxWzeV8isxz7huc92cOW4NLbtP8ztf1teL/BrVFZV86cPvmR0elLtfqp3TB1ISUUV//fJthNW96Hicpbroql2Q4EuEgYxfh/3zRzGnBsnsC33MJc8+jGXPvYx+4vKeOrmSfz+2rH8v6vHsGTrAX748ppGS/O+vmo3Ow4Uc9e0wbVL/w7ukciFI3rw1Kfbm9x1qS2cc7y+ahfnPbSIq2cv5uPN+8N6fokMBbpIGF04oidv3nUmGSkJTO7fjXl3n8XUU4Lr0Fw+Lo1/v/AUXl+1m9++s5GiskqyDxaTuSufP76/hWG9OjN9WP01a/5t6iAKSiv5ayjUC0srKCqrbPWOSs45tu8/zM4DxRSVVeKcY9ehEv71qWXc/fwq0rt2pE9yR372RmbtRtniXbpSVOQr5Jzj/tcyeXbpjkbPzblxAheO6Nno8Rv/spSPmmhBJ3YI0DU+lq4JsfTvFs/o9C6M6ZMUuoq1kPmZe5i/di87D5TUvibGb6HPPu694BS+eXoGH2zM4Za/Lue+GUO5/ZyBzdZeVlnFkx9vZ+PeAsoqqymrrKaq2nHrWQM4c3DK8Xw75Djo0n+RKFJZVc3zy3ZSUl5FUnwMXTrGkN41nuG9Ozd5fPbBYuZn7gWg5se1uLyKg8XlHCwu58DhcjbvK6q3NAEEw/uMQSlMH9aDWL+PQyXlHCyuoKraceOUfvRJjq899panlrF4ax4L751Kz6TGq1dm7srnBy+tZsPeQvokd6RDwE9cjI/9heUcKinnmW+dyoR+yUf9unfkFfP8sh21vwgqq6sZnd6lVatfyhEKdJGTwL6CUlbvPMS6PQVkdEtg2rDudO7Q/Drxde3IK2b6HxZx4YiePHr9uNrHK6qq+eP7W3hs4RaSE2L5zVWjmDa0R+3z+4vKuGb2Yg4cLuelb5/GkB6JTZ5/b34pVz3+KXsLSukY48fvM5xzFJRW8vjXxzNjVK+2ffEnEQW6iLToDws28fB7m5lz4wQqq46sKplTWMYV49L42aXDm1x6eOeBYq58/FMCPuOVO06nd5f6K2Pml1Rw7ZzgzJ8Xbj+NkWlJQHDe/TWzP2Xb/sPMu/ss0rvGNzp3uDjnqHYcdWMTr1Cgi0iLSiuqOP8Pi2r73Dt3CHDW4FSumpBWr1XelPV7CvjanMV0T4zj11eMYmJGMn6fUVpRxU1PfsbKHQd58puTOGtwar3X7cgrZuYjH3FKz0ReuG0KAX9wnkZOYSl/ev9LOncIcGz6qaEAAAioSURBVMOUfnTvfKQbKL+4gic/2cYbq3dz+9kDuHZSn9qZQU35fMdBfvJ6JgcPV/C7a8Zw2sBux/stigoKdBFpldU7D/HBxlzOGNSNsX261AZsayzdmsctf11OUVklKZ3iuHBED/YVlPLP9Tk8fN1YZo1tvI0gwBurd/Pd5z7nO+cO4vvnD+GZz3bw2/kbKK2oorLa1W4Qfu2kPny4KZenF2dRVFZJv27xZOUVc9X4dB68fGSjdekPFZfz3/M38vyyHXRPjKNjjJ+sA8XcfvZAvn/+EGIDzX9th4rL2Z5XzI4Dxew8UMy+glKSOsbQPTGO1MQOpHSKJcbvw+8zAn4jpVMcKZ3iWv29agsFuoh8JQ6XVfL+xhz+8cVeFm7IoaSiih9fPIxvnTXgqK/74cureWlFNkN7BteZP31gNx68fCR+n/HUp9t5cdlODpdXYQYzR/XiO+cOYkiPRB55bzOPLNzMkO6JPHz9WCoqHev25LNudwFvrtlDfkkF3zw9g3umD8ZnxoNvr+O5z3YyMq0zt509kJoemGoH2/cfJnNXPmt3F7DrUEm9+jp3CFBUVkl1M3EZ4zduPWsA35k2qMnNyp1zZB8sYdn2AyzbfoAzB6Vy8ejjGzdoc6Cb2UXAw4Af+LNz7jcNnrfQ8zOBYuCbzrmVRzunAl2kfSspr2J73mGG9Wp69k5dxeWVzHrsEw4cLufHlwzj8rFp9bpRCkoreG/9PkalJTGoe/2B10Wbcrnn+c85WHzk4quEWD8TM5L5j4uGNpo99M7avfzolTX1jq8xICWBEWlJjOzdmYGpnejbLZ70rh2Jjw1QVe3IKyojp7CMvMPlVFVXU1nlqKp2LFi3j7mf7yKtS0d+eulwpg/rwYa9BazIOsiy7QdZvv0Ae0ILrXXuEODOcwcddYro0bQp0M3MD2wCzgeygWXA9c65dXWOmQncRTDQTwUeds6derTzKtBFpK7i8koMO64t/XYfKuHN1bvpkxzP8F6d6Zscj+8oA6CFpRXsPhQMWDMwoGdSBxJbOSuoKUu35vHT19eycV8hHWJ8lFYEL9Tq0TmOiRnJnNo/mUkZyZzSI/GotbWkrYF+GvBz59yFofv3ATjn/qvOMXOAD5xzz4XubwSmOuf2NHdeBbqItDcVVdU8sySLrfsPM75vVyb060p6145HHbQ9Vm3dgi4N2FnnfjbBVnhLx6QB9QLdzG4DbgPo27dvK95aRMQ7Yvw+vnlG/4i9f2uGsJv61dKwWd+aY3DOPeGcm+icm5iamtrES0RE5Hi1JtCzgbrX5qYDu4/jGBEROYFaE+jLgMFm1t/MYoHrgDcaHPMG8A0LmgLkH63/XEREwq/FPnTnXKWZfQd4h+C0xSedc2vN7Nuh52cD8wjOcNlCcNrizSeuZBERaUprBkVxzs0jGNp1H5td57YD7gxvaSIiciy0wYWISDuhQBcRaScU6CIi7UTEFucys1wg6zhfngJE66620VpbtNYFqu14RGtdEL21RWtdcGy19XPONXkhT8QCvS3MbHlzl75GWrTWFq11gWo7HtFaF0RvbdFaF4SvNnW5iIi0Ewp0EZF2wquB/kSkCziKaK0tWusC1XY8orUuiN7aorUuCFNtnuxDFxGRxrzaQhcRkQYU6CIi7YTnAt3MLjKzjWa2xcx+FOFanjSzHDPLrPNYspktMLPNoc9dI1BXHzN738zWm9laM7s7Gmozsw5m9pmZrQ7V9UA01NWgRr+ZfW5mb0VLbWa23cy+MLNVZrY8WuoK1dHFzF42sw2h/2+nRUNtZnZK6PtV81FgZvdESW3fC/3/zzSz50I/F2Gpy1OBHtrf9I/ADGA4cL2ZDY9gSU8BFzV47EfAe865wcB7oftftUrgXufcMGAKcGfo+xTp2sqAac65McBY4KLQcsuRrquuu4H1de5HS23nOufG1pmrHC11PQzMd84NBcYQ/N5FvDbn3MbQ92ssMIHgKrCvRro2M0sDvgtMdM6NJLiC7XVhq8s555kP4DTgnTr37wPui3BNGUBmnfsbgV6h272AjVHwfXud4CbfUVMbEA+sJLidYVTURXBjlveAacBb0fLvCWwHUho8Fg11dQa2EZpcEU21NajnAuCTaKiNI9t1JhNc7fatUH1hqctTLXSa37s0mvRwoc09Qp+7R7IYM8sAxgFLiYLaQl0aq4AcYIFzLirqCvkf4IdAdZ3HoqE2B7xrZitC+/JGS10DgFzg/0LdVH82s4Qoqa2u64DnQrcjWptzbhfwO2AHwT2X851z74arLq8Feqv2LpUgM+sEvALc45wriHQ9AM65Khf8MzgdmGxmIyNdE4CZXQLkOOdWRLqWJpzhnBtPsKvxTjM7O9IFhQSA8cDjzrlxwGEi213WSGiXtcuAlyJdC0Cob3wW0B/oDSSY2Q3hOr/XAt0Le5fuM7NeAKHPOZEowsxiCIb5M865udFUG4Bz7hDwAcExiGio6wzgMjPbDjwPTDOzv0dDbc653aHPOQT7gSdHQ10Efx6zQ39lAbxMMOCjobYaM4CVzrl9ofuRrm06sM05l+ucqwDmAqeHqy6vBXpr9jeNtDeAm0K3byLYf/2VMjMD/gKsd879PlpqM7NUM+sSut2R4H/uDZGuC8A5d59zLt05l0Hw/9VC59wNka7NzBLMLLHmNsH+1sxI1wXgnNsL7DSzU0IPnQesi4ba6rieI90tEPnadgBTzCw+9HN6HsGB5PDUFcnBiuMcVJgJbAK+BO6PcC3PEewHqyDYWrkF6EZwYG1z6HNyBOo6k2BX1BpgVehjZqRrA0YDn4fqygR+Gno84t+zBnVO5cigaKS/ZwOA1aGPtTX/5yNdV536xgLLQ/+mrwFdo6i2eCAPSKrzWMRrAx4g2JDJBP4GxIWrLl36LyLSTnity0VERJqhQBcRaScU6CIi7YQCXUSknVCgi4i0Ewp0EZF2QoEuItJO/H/OWHhi+F4X9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(np.array(losses)[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd3hUVfrA8e9J74UkpJBC6CVAgNC7YFcQu2JfF1lXXdfy022KusWua2Wxoaur2FZRUFcUBASkt1ACBAghkJBAQgoh7fz+OJMQQsokmclMJu/nefJkMvfMvSeX4c2ZU96jtNYIIYRo/9wcXQEhhBC2IQFdCCFchAR0IYRwERLQhRDCRUhAF0IIF+HhqAuHh4frrl27OuryQgjRLq1fvz5Xax1R3zGHBfSuXbuybt06R11eCCHaJaXUgYaOSZeLEEK4CAnoQgjhIiSgCyGEi3BYH7oQHU15eTmZmZmUlpY6uiqiHfDx8SE2NhZPT0+rXyMBXYg2kpmZSWBgIF27dkUp5ejqCCemtSYvL4/MzEwSExOtfp10uQjRRkpLSwkLC5NgLpqklCIsLKzZn+YkoAvRhiSYC2u15L3S/gJ6diosfgxOHnd0TYQQwqm0v4B+bB+seB6O73d0TYQQtSxdupRLLrmkyXLz5s0jKyur2eefM2cO7733XqNl1q1bxz333NPsc9dn9uzZPPvsszY5V1tpf4OiQTHm+4ksiBns2LoIIZpt3rx5JCUlERMTc9axyspK3N3d633drFmzmjx3SkoKKSkpra5je9X+WuhBXcz3E83/Cy9ER/f+++8zfPhwkpOTueOOO6isrAQgICCA+++/nyFDhjB58mSOHj0KwKZNmxg5ciQDBw5k+vTpHD9uujr37NnDlClTGDRoEEOGDGHv3r0AFBUVceWVV9KnTx9mzJhB3R3RPv30U9atW8eMGTNITk7m5MmTdO3alccff5yxY8fyySef8MYbbzBs2DAGDRrEFVdcQUlJCXBmi3nixIk89NBDDB8+nF69erF8+XLgzE8Js2fP5rbbbmPixIl069aNl156qaYeTzzxBH369OHcc8/luuuua7Il3tB9eOmll+jXrx8DBw7k2muvBeCnn34iOTmZ5ORkBg8eTGFhYQv/tZqv/bXQ/SPAzQNOHHJ0TYRosce+SmV71gmbnrNfTBCPXtq/weM7duxg/vz5/Pzzz3h6enLnnXfywQcfcNNNN1FcXMyQIUN47rnnePzxx3nsscd45ZVXuOmmm3j55ZeZMGECjzzyCI899hgvvvgiM2bM4OGHH2b69OmUlpZSVVXFwYMH2bhxI6mpqcTExDBmzBh+/vlnxo4dW1OHK6+8kldeeYVnn332jJa0j48PK1asACAvL49f//rXAPz5z3/mrbfe4u677z7r96moqGDNmjUsWrSIxx57jMWLF59VZufOnSxZsoTCwkJ69+7Nb37zGzZv3sxnn33Gxo0bqaioYMiQIQwdOrTRe9vQfXjyySfZt28f3t7e5OfnA/Dss8/y6quvMmbMGIqKivDx8Wn03LbU/lrobm4QGCMtdCGa6YcffmD9+vUMGzaM5ORkfvjhB9LT0wFwc3PjmmuuAeCGG25gxYoVFBQUkJ+fz4QJEwC4+eabWbZsGYWFhRw6dIjp06cDJhj7+fkBMHz4cGJjY3FzcyM5OZn9+/dbVbfqawNs27aNcePGMWDAAD744ANSU1Prfc3ll18OwNChQxu8zsUXX4y3tzfh4eF07tyZ7OxsVqxYwbRp0/D19SUwMJBLL7200bo1dB8ABg4cyIwZM3j//ffx8DDt4zFjxnDffffx0ksvkZ+fX/N8W2h/LXQw/egS0EU71lhL2l601tx888384x//aLJsY1PmGttY3tvbu+axu7s7FRUVVtXN39+/5vEtt9zCF198waBBg5g3bx5Lly5t9FqNXae++jRW/+ZauHAhy5YtY8GCBTzxxBOkpqby8MMPc/HFF7No0SJGjhzJ4sWL6dOnj82u2Zj210IHS0CXLhchmmPy5Ml8+umn5OTkAHDs2DEOHDCZWKuqqvj0008B+M9//sPYsWMJDg4mNDS0pn/63//+NxMmTCAoKIjY2Fi++OILAE6dOlXTz22NwMDARvuVCwsLiY6Opry8nA8++KBFv2tjxo4dy1dffUVpaSlFRUUsXLiw0fIN3YfqbqZJkybx9NNPk5+fT1FREXv37mXAgAE89NBDpKSksHPnTpv/Dg1psoWulHobuATI0Von1XM8GHgfiLec71mt9Tu2rugZgmJg1yLQGmShhhBW6devH3/9618577zzqKqqwtPTk1dffZWEhAT8/f1JTU1l6NChBAcHM3/+fADeffddZs2aRUlJCd26deOdd8x/7X//+9/ccccdPPLII3h6evLJJ59YXY9bbrmFWbNm4evry6pVq846/sQTTzBixAgSEhIYMGCAzQcVhw0bxtSpUxk0aBAJCQmkpKQQHBzc6Gvquw+VlZXccMMNFBQUoLXm97//PSEhIfzlL39hyZIluLu7069fPy688EKb1r8xqqmPH0qp8UAR8F4DAf2PQLDW+iGlVASwC4jSWpc1dt6UlBTd4g0uVr0G3/0B/m8f+HVq2TmEaGM7duygb9++jq5GvQICAigqKnJ0NdpMUVERAQEBlJSUMH78eObOncuQIUMcXa2z1PeeUUqt11rXOzezyRa61nqZUqprY0WAQGU63QKAY4B1HWctVTMX/ZAEdCFEs82cOZPt27dTWlrKzTff7JTBvCVsMSj6CrAAyAICgWu01lX1FVRKzQRmAsTHx7f8ijVz0Q9D1ICWn0cIAdChWudgxglckS0GRc8HNgExQDLwilIqqL6CWuu5WusUrXVKRES9e5xap3YLXQghBGCbgH4r8Lk29gD7APvO0QmIBOUmUxeFEKIWWwT0DGAygFIqEugNpNvgvA1z94CAKAnoQghRizXTFj8EJgLhSqlM4FHAE0BrPQd4ApinlNoKKOAhrXWu3WpcTeaiCyHEGZpsoWutr9NaR2utPbXWsVrrt7TWcyzBHK11ltb6PK31AK11ktb6fftXG1ktKoSTsTZ9bnPdcsstNYuebr/9drZv335WmXnz5nHXXXc1Wb+VK1fW/GxNOl5r7N+/n6Sks2Z0O0T7XPoPZqbL3iWOroUQog29+eabLX7t0qVLCQgIYPTo0YB16Xjbm/a59B9MC72sEEptm7FOCFfm6PS5O3bsYPjw4TU/79+/n4EDBwLw+OOPM2zYMJKSkpg5c2a9OVcmTpxI9YLEd955h169ejFhwgR+/vnnmjJfffUVI0aMYPDgwUyZMoXs7Gz279/PnDlzeOGFF0hOTmb58uVnpONt6PdsKE1vQ0pLS7n11lsZMGAAgwcPZskS0+hMTU2tue8DBw5k9+7dFBcXc/HFFzNo0CCSkpJqVue2Rjtuodfa6MKn3lmSQjivbx6GI1tte86oAXDhkw0edob0uX379qWsrIz09HS6devG/PnzufrqqwG46667eOSRRwC48cYb+frrrxvMhHj48GEeffRR1q9fT3BwMJMmTWLwYLPhzdixY1m9ejVKKd58802efvppnnvuOWbNmkVAQAAPPPAAYLJPVmvo9wTr0vRWe/XVVwHYunUrO3fu5LzzziMtLY05c+bwu9/9jhkzZlBWVkZlZSWLFi0iJiamJpdMQUFB4/++VmjHLfTqxUUyMCqENZwlfe7VV1/Nxx9/DMD8+fNrrrtkyRJGjBjBgAED+PHHHxtMmwvwyy+/MHHiRCIiIvDy8joj/W5mZibnn38+AwYM4Jlnnmn0PNB4elywLk1vtRUrVnDjjTcC0KdPHxISEkhLS2PUqFH8/e9/56mnnuLAgQP4+voyYMAAFi9ezEMPPcTy5cubzCdjDddooQvR3jTSkrYXZ0mfe80113DVVVdx+eWXo5SiZ8+elJaWcuedd7Ju3Tri4uKYPXs2paWlLarj3XffzX333cfUqVNZunQps2fPbvQ8TbEmTW+1hu7N9ddfz4gRI1i4cCHnn38+b775Jueccw7r169n0aJF/OEPf+C8886r+YTSUu23hR4Ybb5LQBfCKs6SPrd79+64u7vzxBNP1LSsq4N3eHg4RUVFNXVpyIgRI1i6dCl5eXmUl5efke2xoKCALl3MJ/h333235vmG0vY29Hu2xPjx42tS/qalpZGRkUHv3r1rupjuuecepk6dypYtW8jKysLPz48bbriBBx54gA0bNrTomrW13xa6hxf4d5YuFyGs5Czpc8G00h988EH27dsHQEhICL/+9a8ZMGAAXbt2ZdiwYY2+Pjo6mtmzZzNq1Ciio6MZMmRIzQDv7Nmzueqqq+jSpQsjR46sucall17KlVdeyZdffsnLL798xvka+j2b684772TWrFkMGDAADw8P5s2bh7e3N/Pnz+f999/H09OTqKgoHnnkEdauXcuDDz6Im5sbnp6evP766y26Zm1Nps+1l1alz632rwlmj9EbGv9rLoQzkPS5ormamz63/Xa5gBkYlS4XIYQA2n1Al+X/QtiCtM5dQ/sP6KX5UFbs6JoIYRVHdXGK9qcl75X2H9DBbHQhhJPz8fEhLy9PgrpoktaavLw8fHx8mvW69jvLBc7c6CK8h2PrIkQTYmNjyczMrFlWL0RjfHx8iI2NbdZr2nlAr14tKgOjwvl5enqSmJjo6GoIF9a+u1xqFhfJwKgQQrTvgO7lB76h0kIXQgjae0AHmYsuhBAWTQZ0pdTbSqkcpdS2RspMVEptUkqlKqV+sm0VmyBz0YUQArCuhT4PuKChg0qpEOA1YKrWuj9wlW2qZiXZik4IIQDr9hRdBhxrpMj1wOda6wxL+Rwb1c06QV2gJBfKG0+1KYQQrs4Wfei9gFCl1FKl1Hql1E0NFVRKzVRKrVNKrbPZXNzqueiFsrhICNGx2SKgewBDgYuB84G/KKV61VdQaz1Xa52itU6JiIiwwaWRjS6EEMLCFguLMoFcrXUxUKyUWgYMAtJscO6myeIiIYQAbNNC/xIYp5TyUEr5ASOAHTY4r3VqL/8XQogOrMkWulLqQ2AiEK6UygQeBTwBtNZztNY7lFLfAluAKuBNrXWDUxxtzjsQvIOkhS6E6PCaDOha6+usKPMM8IxNatQSMhddCCFcYKUomIAus1yEEB2c6wR06XIRQnRwLhLQu0DhEagsd3RNhBDCYVwkoMcAGoqyHV0TIYRwGBcJ6DIXXQghXCSgy1x0IYRwsYAuLXQhRMflGgHdJwQ8/SSgCyE6NNcI6ErJ4iIhRIfnGgEdZC66EKLDc6GALnuLCiE6NhcK6Jbl/1WVjq6JEEI4hGsF9KoKKLbRTkhCCNHOuFBAr15cJAOjQoiOyYUCehvORS8rsf81hBCimVwooLfR8v+MX+DJOMjdbd/rCCFEM7lOQPcLA3cv+3e5ZKw0ffUHf7HvdYQQopmaDOhKqbeVUjlKqUa3lVNKDVNKVSqlrrRd9ZqhZnGRnVvo2anm+5G222VPCCGsYU0LfR5wQWMFlFLuwFPAdzaoU8u1xVz06oCeLQFdCOFcmgzoWutlwLEmit0NfAbk2KJSLRYYbd8ul4pTkJtmHmdvA63tdy0hhGimVvehK6W6ANOBOVaUnamUWqeUWnf0qB3mi1d3udgr0Oammf7z+NFw8risTBVCOBVbDIq+CDyktW5yiabWeq7WOkVrnRIREWGDS9cR1AUqy6Akz/bnhtPdLQOvtvws3S5CCOdhi4CeAnyklNoPXAm8ppS6zAbnbT57b3SRvQ3cvaHftNM/CyGEk/Bo7Qm01onVj5VS84CvtdZftPa8LVJ7Lnr0INufPzsVOvcBv04QEi8zXYQQTqXJgK6U+hCYCIQrpTKBRwFPAK11k/3mbcruLfRU6DHFPI5Mkha6EMKpNBnQtdbXWXsyrfUtrapNawV0BuVun8HKoqNQlA2R/c3PkUmQ9i2UnwRPX9tfTwghmsl1VooCuLlbpi7aIaDnWAZEqwN6VBLoKsjZYftrCSFEC7hWQAf7bUVXPcMlMunM79LtIoRwEi4a0O3QQs9OhYBI8A83P4cmgqe/DIwKIZyGCwb0LvZZXJS97XR3C4CbG0T2O91yF0IIB3PBgB4D5SVQmm+7c1ZWQM7OMwM6mJ+zt0oKACGEU3DNgA627XY5thcqT53uN68WmQSlBVCQabtrCSFEC7lgQLfDRhfZdWa4VIsacOZxIYRwIBcM6HZooWengpsHhPc68/nO/SzHt9ruWkII0UKuF9ADowBl+4Ae3gs8vM983icIQhJkposQwim4XkB39zTTC205Fz079ezulmpRA6TLRQjhFFwvoINt56KXFkBBxunulboi+5tB07IS21xPCCFaSAJ6U7K3m+91Z7hUi5QUAEII59AuA7puat63LfcWrV7a32CXi6QAEEI4h3YX0JfszGHsU0vIKzrVcKGgGDhVAKcKW3/B7FTwCTk9e6aukK7gFSABXQjhcO0uoMeG+nIo/ySfbWhkMU/NXPTDrb9gdqrpVlGq/uNubqZ/XWa6CCEcrN0F9J6RgQxNCOWjtQcb7nqx1UYXVVWQs73h7pZqUUkm8EsKACGEA7W7gA5w3fB40o8Ws2bfsfoL2GpxUf4BKCtqOqBHJpkunoKDrbueEEK0QpMBXSn1tlIqRylVb5+CUmqGUmqL5WulUsoOm3me6eIB0QT6ePDR2gYCaGC0+d7agF43B3pDqo9Lt4sQwoGsaaHPAy5o5Pg+YILWeiDwBDDXBvVqlK+XO5cld2Hh1sPkl5SdXcDTB/zCW9/lkp0KKLMxdGMiq1MAyAIjIYTjNBnQtdbLgAb6NkBrvVJrfdzy42og1kZ1a9S1w+Moq6jivxsbCNq2mIuevQ06dQMv/8bLeQeaDS8kp4sQwoFs3Yf+K+Cbhg4qpWYqpdYppdYdPXq0VRfqHxPMoNhgPlrTwOCoLeaiN7bkv67I/tLlIoRwKJsFdKXUJExAf6ihMlrruVrrFK11SkRERKuvee3weHZlF7Iho57NLFq7t2hZMRxLb7r/vFrUAFO+rLjl1xRCiFawSUBXSg0E3gSmaa3zbHFOa1w6KAY/L3c+WpNx9sGgGDh5DMpPtuzkOTsB3YwWepIpLykAhBAO0uqArpSKBz4HbtRap7W+StYL8PZg6qAYvt5ymMLS8jMPtnaji6aW/NdVXe6I9KMLIRzDmmmLHwKrgN5KqUyl1K+UUrOUUrMsRR4BwoDXlFKblFLr7Fjfs1w3PJ6T5ZV8ualO4G7tXPTsVLOkPyTBuvIhCeAVKDNdhBAO49FUAa31dU0cvx243WY1aqaBscH0jQ7iwzUZ3DCyVvBtdQs91Szpd7PyQ4ybm2XTaBkYFUI4RrtcKVqbUorrhseRmnWCrZkFpw8EVS8uasHAqNYmMFvb3VItsr+kABBCOEy7D+gA05K74OPpxodraw2OevmbLIktaaGfyILS/OYH9KgkOHUC8usZpBVCCDtziYAe7OvJxQNiWLApi+JTFacPtHRxUc2S/+a20AdYXi/dLkKIemgN27+Eo7vscnqXCOgA1w2Po+hUBV9vqRXAWzoXPccS0Bvadq4hnfsCShYYCSHOdnAtvH0+fHwTrHnDLpdwmYA+NCGUHp0D+HBNrYRdrWmhB8eBb0jzXucdAJ0SpYUuhDjt+H745FZ4awoc2weX/hMueNIul2pylkt7YQZH43ni6+3sOHyCvtFBZqZLcQ5UlIGHl/Una86S/7oikySgCyHgZD4sfxZ++Rcodxj/fzDmHpP7yU5cpoUOcPngLni5u51eOVo9F72wGTsXVZyC3LTWBfRj++BUUcteL4Ro3yrKYPUceCkZVr4CA66Cu9fDOX+yazAHFwvoof5eXJAUxX83HqK0vLJli4ty06CqouUBPUpSAAjRIWkNO76C10bCtw+Z/E53/ASXvQbBXdqkCi4V0MGk1T1RWsGirYdrLS5qxsCotZtaNKT6dZJKV4iO49B6mHcxzL8B3Dzg+o/hpgUQbff9fs7gMn3o1UZ1C6NrmB8frTnI5f0ss1Sa00LP3gbu3tCpe8sqEBIP3kEy00WIjiA/A354HLZ+YjbVufh5GHIzuDsmtLpcQFdKce3weJ78Zid7ChQ9vAKaGdBTzQ5FLf0HUer0ilEhhGsqLYDlz8Pq183/+XH3w5h7wSfIodVyuS4XgCuGxOLhpvhobWbz56Jnp7a8u6VaZJI5T1VV684jhHAeVVXm//XPL8FLg+HnF6H/dLhrHUx+xOHBHFywhQ4QEejNuf0i+WxDJn9MiMbN2hZ60VEoym75gGi1qCRYWwj5B8y8dCFE+1NxCrI2QsYqOLAKDq42LXOAhLFw/l8hZrBj61iHSwZ0MGl1v9l2hMzKTsQX7rHuRTktXPJfV83AaKoEdCHai9ITcHANZKyEjNVmoLOi1BwL6wn9pkH8KPMV2tV0tTgZlw3oY3uEExvqy/rjvsQXHYHKiqb7xVs7w6VadQqA7G3Q95LWnUsIYR+FR+CAJXhnrLRkSq0yi4CiB0HKryDBEsD9wx1dW6u4bEB3c1NckxLH2h99me5ZaVaMVs9Lb0h2KgREtv4fz8sfwrrL7kVCOJv8DFjxAuxdAsf3mec8/SA2xazkjB8JscNMGo92yGUDOsBVKXH86cdO5ocTWVYE9BbkQG9IZH84vMU25xJCtM7JfFj+nFmGD9BjCgz7FcSPhuiB4O7p2PrZSJMBXSn1NnAJkKO1PqsvQimlgH8CFwElwC1a6w22rmhLRAX7EJfQE7KgIj8Tj9iUhgtXVpiNoUfMtM3FIweYNJmnCu2+3FcI0YCKU7D2LVj2tAnqg66FSX+CkDhH18wurJm2OA+4oJHjFwI9LV8zgddbXy3bmTTcrNTavbuJ/MPH9kLlqdb3n1eLqh4Y3W6b8wkhrKc1bPsMXhkG3/3B9Inf8RNMn+OywRysCOha62XAsUaKTAPe08ZqIEQpFW2rCrbWmKSelOLFvvTdjReszpBoyy6X2ucVQrSN/T/DG+fAp7eZjd5v+Axu/KLNl+E7gi360LsAtZKQk2l57qwUh0qpmZhWPPHx8Ta4dNM8PNwp9ImkIj+Tg8dKiOvkV3/B7FQzuh3eyzYXDo4Dn2AJ6EK0laO7YPFs2LUIAmNg2mumi8XN3dE1azO2WCla32TMendJ1lrP1VqnaK1TIiIibHBp6/iGxxGtjvHJuoMNF8pONcHcw9s2F1XKdN9IThch7KswG766F14bBfuWm1Wbd6+HwTM6VDAH2wT0TKB2p1Qs0IJtguzHp1McXT0L+HhdJhWVDSzHb82mFg2J7A852yUFgBD2cKoIlj5pluFv/DcMux1+t8nkVfFq4JO4i7NFQF8A3KSMkUCB1roZO0q0gaAYwqryyD5Rwk9pR88+fjIfCg7aIaAnQVkR5O+37XmF6MgqK2DdO/DyEFj6D+g5BX67Bi56ut0sALIXa6YtfghMBMKVUpnAo4AngNZ6DrAIM2VxD2ba4q32qmyLBXXBTZfT0/8UH67JYHLfyDOP51hmothqhku16pkuR7ZBp262PbcQHcmpQrOqc+8SSPvG7NMZNxKu+QDihjm6dk6jyYCutb6uieMa+K3NamQPlgVF1/d15/H1ORwpKCUq2Of08Wwb5XCpK6IvKDdz/n5TbXtuIVxZZQVkbYD0pSaIZ64xO4l5+Jil+Of9Ffpc4pT5VBzJpVeK1rAE9IsSqpi9Dj5Zd5C7J/c8fTw7FXxCml5J2lxefmajDJnpIkTjtIa8vZC+xATxfcvhVAGgzHTD0XdDt4mmVe7p0/i5OrAOEtDNVnSd9TFGdx/AR2sPcuekHri7Wf66V+dAt8df+6gkOOQUC2eFcC7FubDvJ9MCT19qxrHA7PrV/zLoPgm6jgf/MIdWsz3pGAHdLxzcPOHEIW4adQmz3t/AVXNW8vSVg+gR7mf60JNn2OfakUmQ+l+TmtMJEuAL4VAlx8wuP2nfwhFLriOfYEgcD2N/b1rhnbpJV0oLdYyA7uYGQdFwIovzJ0fx4jXJzP4qlYteWs4jY3y5oazI9v3n1aoHWnO2m0xuQnREFadgzVxY9owZ4IwfDef8GbqdAzHJHW6+uL10jIAOptvlRBZKKS4b3IXRPcJ45ItUli3/ihu8YL9HV7ra47o1M122SkB3BqUnYMXzMHym7cdMxNm0Np9QF882O3j1OBfOfRwi+zm6Zi7JJfcUrVedvUU7B/ow58ah3D+ogioUUz/O45+Ld1NWYeNFQEFdzICrKwyMnsyHzfNh4QOm/7O90Rq+/K3Jh/3j3xxdG9eX8Qu8dS58eqvJOHrjf+GGTyWY21EHaqHHwM6F5j91rf653hygMjSRiZ0TeWFxGt9sO8yzVw0iqUuwba5bnQKgempke1Oca+7bjgWQ/hNUlZvnlYKLnnFs3Zpr9evm9+jUDbZ8BBMfMgNwwraOpZsW+fYvISAKpr4CyddLt0ob6EAt9C5mf8CTx898PjsV96gkXrpuMHNvHMqx4jKmvfozz363i1MVlba5dlSSSaPbXlIAnMiCX+bCvEvg2Z7w1T2QuxtGzoJffQ+Db4T186DgUJOnchoZv8D3fzFzl29aYJ5b+bJj6+RqSo7Bt3+EV4bD7u9h4h/gng0w5EYJ5m2kY7XQwXS7+Fl2MSorNq2JgdcAcF7/KEYkhvHEwu28smQP36Ue4ekrBzI4PrR1145MgvJis+VVWPfWncteju+H7Qtgx1dmEQdAeG+TF6PvpRA18PQnm4BI2Pyh6Yu++DmHVdlqxbnwyS0mA+a0V8E3xGTh2/AejH8QAjo7uobtW8UpWPsm/PQ0lBbA4BvMgGdglKNr1uF0oIBu5qJzIguiBpjHOTsBfcYMl2A/T569ahCXDIzmD59v5YrXV3L7uG7cd24vfDxb2MqonRvdmQL60V2WIL7g9BSyqIHmP2PfqRDRu/7XhSaY/7Qb3jNTzYJj267OzVVVCZ/dDiV5cPtiE8wBxt4Hm/4Dq16Fcx9zbB3bK61Nt8riR02DoPs5cO4TpycCiDbXcQJ6oGXPjVoDo41tajGxd2f+9/vx/OObncxdls7327N5+sqBDOvaqfnX7mxJAXBkG/Sb1jWagRoAAB50SURBVILK21B5qRkUTP0cctPMc7HDzX/EvpdCp0TrzjPuftj4ASx/Hi553n71ba2fnjarD6e+bPaOrBbWHfpdZrYnG3sv+LbyU1hHc3ANfPcn82mucz+ziUSPKY6uVYfXcfrQAyJNUD1RK7NvdqrZ0SQkod6XBPp48vfpA/jg9hGUV1Zx9b9WMXtBKiVlFc27tqcvhPV0jpku3/0RfnrS3I8Ln4H7dsDt38OYe6wP5mAGE6tb6fmN5Jl3pD2L4aenzKKxwTeefXzc/VBWCGveaPu6tVdZG+Hjm8zslfwDcOlLMGuFBHMn0XECuruHGXGvG9A79zULjxoxpkc43907nptHdWXeyv2c98IyPl2fSXlDudXrE9nf8QF9+wJY9xaMugtu+dpsiN2audjj7jfflzthP3r+Qfjs16b1eNGz9a88jEqCXhfA6tdMbm1Rv6oq2LkI3rkY5k6EPT/ChIfg7g0w9GYZ8HQiHSegw5lz0bU2AdbKFaL+3h7Mntqfj+8YRaCPJw98spmJzyzl36v2U1puxWyYqCTIzzCDRo6QfxAW3AXRyTD5UducMyQOhtwEG983v5uzqCgzg6CV5XD1e41vdjDuATPzaf28tqpd+1FWYgY7X0mBj64z/eTn/RXuS4VJfwTvAEfXUNTRAQO6pYV+IgtK85udA314YicW3TOWt29JITLIm798mcq4p5cwd9leik810hUTaRmIdcR89MoKMzBYVQlXvg0eXrY797j7TevXmVrp3/8FDq2Daa9AeI/Gy8YNg67jzBTGilNtUz9nV3gEfngCXugHC+83uVaufBt+t9lkPfSx0RoNYXMdLKB3MXOntW5VDnSlFOf0ieSz34zmP78eQe/IQP6+aCdjnvqRfy7eTX5J2dkvqpnp4oCA/tNTcHA1XPy87WfZBHeBITebVvrxA7Y9d0ts+xx+mQMj7zQZ+6wx7n4oOgKbPrBv3ZzdkW3w39/AC0nmD3TCGLj1W/j1j5B0hem2FE6tY/0LBcWY+eCnTpzuz+7c8mXISilGdw9ndPdwNmYc59Ule3lhcRpzl+3lhlEJ3D62GxGB3qev7Rtqcrq0pX3LTUKkQdfBoGvsc42xv4cN78LyZ81sEkfJ3Q0L7jazdqY0Yypit4nQZSiseBEG39SxApfWZvB41Ssmha2nH6TcCiNmOdcUW2EVq1roSqkLlFK7lFJ7lFIP13M8WCn1lVJqs1IqVSnlfNvQQa3FRVmmpRwcd3pecisNjg/lzZtT+PbecZzTN5I3lqUz9qkfefTLbRzKP1krBUAbDowW58HnvzZL3S961n7XCe4CQ28x87qP77ffdRpTVgzzbwQPb7jqneZ1KyllWun5B8x0zo6gvBTWvwuvjYQPrjRrEqbMhvu2m5QOEszbpSYDulLKHXgVuBDoB1ynlKrbrP0tsF1rPQiz/+hzSikbdtTaSM3iokOWTS1snzK3T1QQL183mB/un8hlyV34z5oMJjy9hAc/2UxBUC/I2WH6su2tOhFVca7p/7T3ANbY+0C5m08DbU1r+Po+OLoTLn+jZQudel1oPq0tf679pGhorvKTsH+FpX+8v0np4O4J0+fC77aYT1oyH79ds+az5XBgj9Y6HUAp9REwDdheq4wGApVSCggAjgHNnKzdBqpb6Mf2mUU1fS6y26USw/156sqB/G5KT+YuS+fDNRko7c7TniXs2bmVHv2S7XZtwOSeTvsGzv+HyTdtb0HR5qP6mjfMzJHmzGlvrQ3vWpJt/QF6TG7ZOdzczB+lz2+HXYug7yW2raMjlByDjNWQscp8z9poSa6moNf5MOq3ZkBYNpNwGdYE9C5A7ZUjmcCIOmVeARYAWUAgcI3W2vmaOdWrRdOXgq6036YWtcSE+DJ7an/uOqcHX393CrbO5fn3P2Nnp3zG94pgQu8IRiaG4etlw7m8h7fA//4MPc+Hkb+x3XmbMvb3Zvrfsmfhslfb5ppZm2DR/5ll5+MfbN25+k+HJX8zYwF9Lm5fgU5r02VUHcAPrILcXeaYuxfEDDEBPGE0xA2XlriLsiag1/eu1nV+Ph/YBJwDdAe+V0ot11qfOONESs0EZgLExzsgbamHF/h3NmlgodlTFlsjPMCbW6ZegN7mzh29T/JClR8frslg3sr9eHm4MSKxExN6RTCxdwTdIwJQLQ0mZcXw6W3g2wkue61tg1JgFAy91Xw6GHef/fthTx43qxb9w+HyN1u/wMXdw6QB+Op3Jl1A93NsU097qKo03Ya1W+CFlim53sEQP8IMgsePMsFcNlbuEKwJ6JlAXK2fYzEt8dpuBZ7UWmtgj1JqH9AHWFO7kNZ6LjAXICUlpe4fhbYRFAOHN4G7N3Rq44EfTx9UeE8GeWYy7/rhlJZXsmbfMX5KO8pPaUf568Id/HXhDrqE+JrWe68IxvQII9DH0/prLPo/yNsDNy8wga6tjb0X1r9jWunTX7ffdaqqzBS7E4fM1DpbbSQ86DpY+hQse845A/r+n82c+QM/m9laYMaGEkabHbHiR1lWP8vqzY7ImoC+FuiplEoEDgHXAtfXKZMBTAaWK6Uigd5Aui0rajNBXUxA79zHMdPTIpPg4C8A+Hi6M75XBON7RfAXIPN4CcvSclmWdpSvN2fx4ZoMPNwUQxJCmWAJ8P2ig3Bza6DVvfVT2PS+6cNOHN92v1NtgVGQ8iv45XUY/4D9WukrXzJjBBc8ZRYH2YqHt1k8890fTKvXWbYNzFgNS/4O+34ynzKTrjgdxIPj2lf3kLCbJiOa1rpCKXUX8B3gDryttU5VSs2yHJ8DPAHMU0ptxXTRPKS1ds49yqoHRtuwu+UMUUmw7VPTXVCnHzM21I/rR8Rz/Yh4yiur2JiRz09pOfyUdpRnvtvFM9/tIjzAm/E9wxnTI5xR3cOICfE1Lz6WDl/dC3EjzOCgI429F9a9bWa8TJ9j+/PvXwE/PG6yJY64w/bnH3qz6Udf/jzM+Nj252+Og2th6d9h74/gHwHn/Q1Sbms8nYHosKxqomqtFwGL6jw3p9bjLOA821bNTmoCuv0HROsVZxlPfuMcM4g48Np650x7ursxPLETwxM78eD5fThaeIrlu03XzNK0o3y+0eSk6Rrmx9jEYO4/dA/Byg23K950/MKYgM4w7Fcm6dW4B5peft8cR7aZMYJOiWYRkz1apl7+ZjD5x7+aAebaaXfbyqH1sOQfsOd78AszGysPu93UTYgGKNPt3fZSUlL0unXr2v7Cm+fDf2fCjV9A90ltf32AHV/Dsqfh8GYIijUt2sE3mDS7Vqiq0uzKLmTV3jxW7s1jdPo/uU0tYFbZvaRHnMPo7qb1PjIxjGC/ZvS/21JRDrw4EPpNhcvntv58WpsZNN8+bHKJ3PiFfTcbPpkPLw4w0yCvmme/69SVtRGWPglp35pPcKPvgeEzJRGWqKGUWq+1Tqn3WIcL6KUF8Mu/TOvY3UHBDk4vuV72jOlT9+9s+m5Tbmvef949i+H9K8jtM4NPou5n5d5c1u0/zsnySpSCpJhgRnUPY1T3MIZ37YS/dxu23v/3Z7Mj0G/XQHjPlp+n9AR8fS9s+wy6TTJ/INpi27jFj5nNQO5a27r6W+PwFhPIdy0EnxAYfRcMvwN8gux7XdHuSEB3ZlqbPuFlz5gBL99Qk1hq+Mym0xIUZsOcMeAXDjOX1LTwyyqq2JyZz8o9eazcm8vGjHzKKqvwcFMMigthVDcT4IcmhLZ8Wz1rFB2Ffw40c7qveLNl5zi82aTCPb4fJv3JLP5pIn+9zRQdNa30pCvsN6/+yDZY+g/Y+bWZbjj6LjMuIBkNRQMkoLcXB9eawbi0b8E7yPSZjvpt/dMPq6rg/cvNHOSZS81UtQacLKtkQ8ZxVu7NZeXePLZkFlBZpfFydyM5LoSR3cMY1S2MwfEhtg/w3z8CP78Ev/2l4T1K66O1ycX93R/NH6wr3zKzOtraNw+Zetyz0ezSZCvZ283OUdu/NP/WI+80/fY2yi0kXJcE9Pbm8BaTU2T7l6bVPfRW0x0TFH26zIoXzea8l7xgummaobC0nHX7j7M6PY9V6XlsO1RAlQYvDzeGxIcwspsJ8MnxIXh7tDLAF+eZVm7vC01QtsbJfJNnZPuX0ONcmP4v280zb66CTPhnsklrcJEN8tQc2Wpmz6T+9/Tg68g7wa8Fe9WKDkkCent1dJfpw93ysVkoMvgGGHMvFB+Ft8+H3heZHXlaOdPjRGk5a/cdY9XePFbvyyM16wRag7eHG0MTQhnVLYyR3cMYFBuCl0cLujsWzzZ/gO5cbeb/N+bQevjkVhNIpzwKo+5uuy6Whnx5F2z9BO7d2rK+e61h/3JzD/b+YPaxHT7T/JGWQC6aSQJ6e3dsH/z8Imz8AHSVGSjzCoBZy+2Sk6OgpJw1+02AX5Wex47DZkWir6c7KV1DGZHYif4xwfSKCiQm2KfpNAXFeaYvved5JrVtfbSG1a+bLprAKJMhMm64jX+zFsrba7ZhG30PnNuMPOtVlbBzofmjnLXBzCMfMctM6ZRcKqKFJKC7ioJDZtn3zoVmkDG+bo40+zheXMYv+46xOj2P1el57DxSWHMswNuDnpEB9I4MpFdkIL2jAukZGUBEgPeZgb56xsidq87u7y85ZlrBuxaaTx3TXnW+luunt0Ha/+D3W5sOxhWnYPNHZjVr3h4I7Wpa48kzrJ6aKkRDJKALmyooKSctp5BdRwrZnV3Irmzz+HhJeU2ZUD/PmgDfKzKQfsEVDP7veFSPKXD1u6dPdnAtfHqr2cfy3MdNn7IzLmM/ss3MKJr0J5jwf/WXKT1hVsiuft1saRc10EyP7TdNcqsIm5GALuxOa01uURlp2YU1X7uOFJKWXUSRZfPs+z0+5m6PL/hD1FwS+w7hitIvCPvlSbN696p5Zhs4Z/afa83erPduO3OtQGG2yV2z9i2TMCtxglks1m2Sc/5xEu1aYwG9A22eKOxJKUVEoDcRgd6M6XF6mqXWmqyCUtKyCzlwMJrSlf/jsty5FC2uJMx9I8vcR7G662OMPZnAsMoqPN2deN/ycffDW1PMitXRd5m+9ZUvwaYPzcYRfaeaQB4z2NE1FR2UtNBF2/rxr7DsGbS7F2t63c/rxZNYmX6MsooqAr09GN87gsl9OjOpd2dC/Z1vF0PevRSOppnxi+0LzOYRydebPnLZh1O0AWmhC+cx+m4oPYFKvp4RMcmMAErKKlixO5cfd+bww84cFm45jJuCIfGhTO4byeS+nenZuRWbftjSuAfgvamwd6npHx8xCwIjHV0rIQBpoQsnU1Wl2ZZVwOIdOfy4M5tth8yUybhOvkzuE8k5fTozolun1i94ao2Da82qV8mzIhxABkVFu3WkoJQfd5rgvmJPLqXlVfh5uZMcF3L6Kz6EzoGyxZroGCSgC5dwsqySVem5LN11lI0Z+ew4fIKKKvP+7RLiS3J8CIMtQT6pS7B9E48J4SDShy5cgq+XO+f0ieScPqbPurS8km2HCth0MJ+NB/PZlJHPwi2HAfBwU/SJDrS04kNJjguhW7h/w9v3CeECrGqhK6UuAP6J2YLuTa31k/WUmQi8CHgCuVrrCY2dU1rowh5yCkvZlJHPpoPma0tmQc08+CAfDwbFmVb8SEv6YIf2xQvRAq3qclFKuQNpwLlAJmbT6Ou01ttrlQkBVgIXaK0zlFKdtdY5jZ1XArpoC5VVmr1Hi9iUYVrxGzOOk5ZdSJU2uWmGJ3ZibI9wxvYMp09UoHPMpBGiEa3tchkO7NFap1tO9hEwDdheq8z1wOda6wyApoK5EG3F3U3Ry5Jn5uphcYBJH7w6/Rgrdh9l+Z5c/rZoBwDhAd6M7RHG2J4RjOsZTmSQDLSK9sWagN4FOFjr50ygblaoXoCnUmopEAj8U2v9nk1qKISNBfp4cm6/SM7tZ/ris/JPsmJPLit257J8dy5fbMoCoGfnAMb2DGdcz3BGJIa17fZ9QrSANe/Q+j6D1u2n8QCGApMBX2CVUmq11jrtjBMpNROYCRAfb8PdX4RohZgQX65OiePqlDiqqjQ7jpxgxe5cVuzJ5T+/ZPDOz/vxdFcMjg9lbA8T4AfGhuAuA6zCyVgT0DOBuFo/xwJZ9ZTJ1VoXA8VKqWXAIEzfew2t9VxgLpg+9JZWWgh7cXNT9I8Jpn9MMHdM6E5peSXrDxxn+e5cVuw5yguL03j++zQ6+XsxsXcEU/pGMq5nOIE+DtxwXAgLawL6WqCnUioROARci+kzr+1L4BWllAfghemSecGWFRXCEXw83RnTI9yScKwPx4rLWLEnlx93ZPPDjhw+33AIT3fFyG5hnNOnM1P6RhLXyc/R1RYdlLXTFi/CTEl0B97WWv9NKTULQGs9x1LmQeBWoAoztfHFxs4ps1xEe1dRWcWGjHx+2JHN4h3Z7D1aDJi+98l9I5nStzOD40Ola0bYlKwUFaIN7M8t5oedOfywI5s1+45RUaUJ9fNkUu/OTO4byfhe0jUjWk8CuhBt7ERpOcvSjvLDjhyW7Mohv6QcT3fF8MROTO5j+t17OEsGSdGuSEAXwoEqKqvYeDCfxZZ+9z05RQCE+XsxolsnRnYLY2S3MOdJESycmgR0IZzIwWMlrLJsuL16bx5ZBaWABHhhHUnOJYQTievkR1wnP65OiUNrTebxk6xKz+OX9GOsTs9j0dYjwOkAPyLxdICX5GKiMRLQhXAgpdQZAR5MC351eh6r6wT4Tv5ejEg0LfiUrqH0jgzEw5n3YBVtTgK6EE6mOsBf1UCA/2abCfD+Xu4kx4cwND6UIQmhDI4PJdhXZtF0ZBLQhXBydQN85vES1h84zoYDx1mfcZxXluyhSoNSZg780IRQhsSHMjQhlMRwf+mH70BkUFSIdq74VAWbD+az3hLgNxw4zolSkwO+k78XQ+JDGJIQytD4UAbFhchOTu2cDIoK4cL8vT0Y3SOc0T3CAbPR9t6jRSbAW4L84h0mo7WHm6J/TBCD40MZFBdMclwoXcP8pBXvIqSFLkQHcKy4jI0Zx2uC/NZDBZSUVQIQ7OvJoLgQkmODSY4PYVBsCGEB3g6usWiItNCF6OA6+XsxuW8kk/uaHPAVlVXszili88HT2/W9suQolj23ievky6DYEMuerLLpdnshLXQhBGD64qs33d6caTbdrl705O6m6BNlNt0eFBfCwNhgekQEyLRJB5CVokKIFsk5UXo6wB/MZ8vBAgotm257e7jRJzqIpJggkroEkxQTTK+oANl4284koAshbKKqSpOeW8S2QyfYdqiAbVkFpB46URPkPd3NHq5JMcEkdQmif5dg+kYF4eslQd5WJKALIeymqkpz8HiJCfJZBSbQHyrgeEk5AG4KenQOICkmmP5dgkmKCaJvTBBBkkq4RWRQVAhhN25uioQwfxLC/Ll4YDQAWmsOF5RaWvEnSD1UwM97c/l846Ga18V18qVvVBB9o4PoFxNEv+ggYkN9ZQplK0hAF0LYnFKKmBBfYkJ8Oa9/VM3zOYWlpB46wfbD5mvH4RN8vyOb6o6CQG8P+kYH0Tc6sCbQ94oMlBk2VpKALoRoM50Dfejcx4dJfTrXPFdSVsGuI4XsOFzI9sMF7DhcyKfrMym2zJN3U9AtIoB+0UE1wb5PVBCRQd7Smq/DqoCulLoA+CdmT9E3tdZPNlBuGLAauEZr/anNaimEcFl+Xh4MjjfJxapVVWkyjpWwo1ZLft3+YyzYnFVTJsjHg16RgfSMDKRXZAC9LY/DA7w6bKBvclBUKeUOpAHnApnAWuA6rfX2esp9D5RiNpJuNKDLoKgQornyS8rYcbiQtOzaX0UUnCyvKRPq50nPyEB6WwJ99eNQfy8H1tx2WjsoOhzYo7VOt5zsI2AasL1OubuBz4BhrairEEI0KMTPi1HdwxjVPazmOa01RwtPkZZdxK7sQnZbAv0XGw/VTKcECA/wpldkAL0iA+neOYBu4f50i/AnKsjHZVr01gT0LsDBWj9nAiNqF1BKdQGmA+fQSEBXSs0EZgLEx8c3t65CCHEWpRSdg3zoHOTD2J7hNc9rrTlyopRdRwrZnV1kWvM5RXy87mBNHhsAPy93uoaZ4N4tIoDuEf4khpuvwHY2tdKagF7fn666/TQvAg9prSsb+0untZ4LzAXT5WJtJYUQormUUkQH+xId7MvE3qcHYasD/b6jxezNLSb9aBHpR4vZklnAoq2Ha/LZAHQO9CYx/MxA3y0igNhQXzydMO2BNQE9E4ir9XMskFWnTArwkSWYhwMXKaUqtNZf2KSWQghhI7UDfXXK4WqnKio5kFdC+tFi0nNNoN+XW8y32w7XLJQCk9smNtSXhDB/uob50TXMn67hfiSE+RMX6oeXh2OCvTUBfS3QUymVCBwCrgWur11Aa51Y/VgpNQ/4WoK5EKK98fZwp1dkIL0iA886dry4jHRLi/5AXgn784rZn1fMxgPHz+ird1MQE+JL1zB/EizBPiHMj8Rwf+I6+dl1Tn2TAV1rXaGUugv4DjNt8W2tdapSapbl+By71U4IIZxEqL8XQ/29GJoQesbzWmuOFZexP6+EA3nFp7/nFvPV5qya3aPAbBMYHeTDbWMTuX1cN5vX0ap56FrrRcCiOs/VG8i11re0vlpCCNE+KKUIC/AmLMD7rGAPZqrl6SBvvkcE2mcDEVkpKoQQdhTi50WynxfJcSF2v5bzDdMKIYRoEQnoQgjhIiSgCyGEi5CALoQQLkICuhBCuAgJ6EII4SIkoAshhIuQgC6EEC6iyQ0u7HZhpY4CB1r48nAg14bVsTVnrx84fx2lfq0j9WsdZ65fgtY6or4DDgvoraGUWtfQjh3OwNnrB85fR6lf60j9WsfZ69cQ6XIRQggXIQFdCCFcRHsN6HMdXYEmOHv9wPnrKPVrHalf6zh7/erVLvvQhRBCnK29ttCFEELUIQFdCCFchFMHdKXUBUqpXUqpPUqph+s5rpRSL1mOb1FKDWnDusUppZYopXYopVKVUr+rp8xEpVSBUmqT5euRtqqf5fr7lVJbLddeV89xR96/3rXuyyal1Aml1L11yrT5/VNKva2UylFKbav1XCel1PdKqd2W72dvS0PT71c71u8ZpdROy7/hf5VS9e6k0NT7wY71m62UOlTr3/GiBl7rqPs3v1bd9iulNjXwWrvfv1bTWjvlF2b/0r1AN8AL2Az0q1PmIuAbQAEjgV/asH7RwBDL40AgrZ76TcRsmO2oe7gfCG/kuMPuXz3/1kcwCyYcev+A8cAQYFut554GHrY8fhh4qoHfodH3qx3rdx7gYXn8VH31s+b9YMf6zQYesOI94JD7V+f4c8Ajjrp/rf1y5hb6cGCP1jpda10GfARMq1NmGvCeNlYDIUqp6LaonNb6sNZ6g+VxIbAD6NIW17Yhh92/OiYDe7XWLV05bDNa62XAsTpPTwPetTx+F7isnpda8361S/201v/TWlfvRLwaiLX1da3VwP2zhsPuXzWllAKuBj609XXbijMH9C7AwVo/Z3J2wLSmjN0ppboCg4Ff6jk8Sim1WSn1jVKqf5tWDDTwP6XUeqXUzHqOO8X9A66l4f9Ejrx/1SK11ofB/CEHOtdTxlnu5W2YT131aer9YE93WbqE3m6gy8oZ7t84IFtrvbuB4468f1Zx5oCu6nmu7hxLa8rYlVIqAPgMuFdrfaLO4Q2YboRBwMvAF21ZN2CM1noIcCHwW6XU+DrHneH+eQFTgU/qOezo+9ccznAv/wRUAB80UKSp94O9vA50B5KBw5hujbocfv+A62i8de6o+2c1Zw7omUBcrZ9jgawWlLEbpZQnJph/oLX+vO5xrfUJrXWR5fEiwFMpFd5W9dNaZ1m+5wD/xXysrc2h98/iQmCD1jq77gFH379asqu7oizfc+op4+j34s3AJcAMbenwrcuK94NdaK2ztdaVWusq4I0Gruvo++cBXA7Mb6iMo+5fczhzQF8L9FRKJVpacdcCC+qUWQDcZJmtMRIoqP5obG+W/ra3gB1a6+cbKBNlKYdSajjmfue1Uf38lVKB1Y8xA2fb6hRz2P2rpcFWkSPvXx0LgJstj28GvqynjDXvV7tQSl0APARM1VqXNFDGmveDvepXe1xmegPXddj9s5gC7NRaZ9Z30JH3r1kcPSrb2BdmFkYaZvT7T5bnZgGzLI8V8Krl+FYgpQ3rNhbzkXALsMnydVGd+t0FpGJG7FcDo9uwft0s191sqYNT3T/L9f0wATq41nMOvX+YPy6HgXJMq/FXQBjwA7Db8r2TpWwMsKix92sb1W8Ppv+5+n04p279Gno/tFH9/m15f23BBOloZ7p/lufnVb/vapVt8/vX2i9Z+i+EEC7CmbtchBBCNIMEdCGEcBES0IUQwkVIQBdCCBchAV0IIVyEBHQhhHAREtCFEMJF/D+3GDsLe8CGCAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(epoch_loss, label='epoch training loss')\n",
    "plt.plot(val_losses, label='epoch validation loss')\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "save_res = True\n",
    "\n",
    "if save_res:\n",
    "    # directory\n",
    "    save_dir = 'files_planet'\n",
    "    if not os.path.exists(save_dir):\n",
    "        os.makedirs(save_dir)\n",
    "\n",
    "    # save the model\n",
    "    step = len(losses)\n",
    "    model_path = 'dc_planet_ms_ep{}_step{}_b{}.pt'.format(num_epochs, step, bs)\n",
    "    model_path = os.path.join(save_dir, model_path)\n",
    "    save = lambda ep: torch.save({\n",
    "            'model': model.state_dict(),\n",
    "            'epoch': epoch,\n",
    "            'step': step,\n",
    "        }, str(model_path))\n",
    "\n",
    "    save(model_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(os.path.join(save_dir, 'losses_ep{}_step{}_b{}.txt'.format(num_epochs, step, bs)), np.array(losses))\n",
    "np.savetxt(os.path.join(save_dir, 'epoch_loss_ep{}_step{}_b{}.txt'.format(num_epochs, step, bs)), np.array(epoch_loss))\n",
    "np.savetxt(os.path.join(save_dir, 'val_losses_ep{}_step{}_b{}.txt'.format(num_epochs, step, bs)), np.array(val_losses))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_gbdx017)",
   "language": "python",
   "name": "conda_gbdx017"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}