main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**File name**: main.ipynb<br>\n",
    "**Authors**: Anton Karazeev &lt;[anton.karazeev@gmail.com](mailto:anton.karazeev@gmail.com)&gt;, Anton Lukashchuk &lt;[academik116@gmail.com](mailto:academik116@gmail.com)&gt;<br>\n",
    "This file is part of REDE project (https://github.com/akarazeev/REDE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from utils.dataset import REDE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://github.com/akarazeev/REDE/raw/master/data/rede/raw/1848-5-parameters.pkl\n",
      "Downloading https://github.com/akarazeev/REDE/raw/master/data/rede/raw/1848-62-111-images.pkl\n",
      "Downloading https://github.com/akarazeev/REDE/raw/master/data/rede/raw/1848-frequencies_modes.pkl\n",
      "Processing...\n",
      "Done!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset REDE\n",
       "    Number of datapoints: 1478\n",
       "    Split: train (test_size: 0.2)\n",
       "    Root Location: data/rede\n",
       "    Transforms (if any): <torchvision.transforms.ToTensor object at 0x1099c5e10>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset = REDE('data/rede', train=True, transform=torchvision.transforms.ToTensor(),\n",
    "                     download=True, test_size=0.2)\n",
    "train_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset REDE\n",
       "    Number of datapoints: 370\n",
       "    Split: test (test_size: 0.2)\n",
       "    Root Location: data/rede\n",
       "    Transforms (if any): <torchvision.transforms.ToTensor object at 0x10b680a58>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_dataset = REDE('data/rede', train=False, transform=torchvision.transforms.ToTensor(),\n",
    "                   download=True, test_size=0.2, test_indices=train_dataset.test_indices)\n",
    "test_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH = 10\n",
    "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH)\n",
    "test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=BATCH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAABTCAYAAACRdMzaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAACOVJREFUeJzt3VGoHFcdx/Hvz8SkWKFJrISYBJti\nUIKgjUES9KGoxTSI8aFIQ6FBAnlRrCJIgg/im4JYW5Bg0GoVadVaNAQx1FjwyZgGJaZp09xa29yQ\nNK3WiPqgwb8PezaZu7nbnd2d2Zk9+/vAcnfOzp353/+e+c/M2dk7igjMzCxfb2g6ADMzq5cLvZlZ\n5lzozcwy50JvZpY5F3ozs8y50JuZZa6WQi9pu6QzkuYk7atjHWZmVo6qvo5e0hLgOeAOYB44DuyK\niNOVrsjMzEqp44j+/cBcRPw5Iv4DPArsrGE9ZmZWQh2Ffi1wrjA9n9rMzKwBS5tasaS9wN40+b6m\n4jAzm2KvRsRbB81UR6E/D6wvTK9LbQtExEHgIIAk/8MdM7PhvVhmpjqGbo4DGyVtkLQMuBs4VMN6\nzMyshMqP6CPiiqTPAEeAJcBDEfF01esxM7NyKr+8cqQgPHRjZjaKExGxZdBM/masmVnmXOjNzDLn\nQm9mlrnGrqMvY7HPDyQ1EImZ2fRqdaGHa4W9W/S7P13wzcrpPWDytjN7WlvoI2JBhyw+j4gFnbdt\nHXcaNqx+V1u1MVYb3WIHRj5Tnj2tLPS9Rb5X72tVXSI6bmdfbOfTu1MaR1UbY7+zoipjXWz5w6rr\n0t86i1oN/w125N/ttx3V9b5XmddJXvZd9XY1jrr6ZisL/bCqSE6xs4+yvH6/W1Vsr7eOYZfV7/er\n7vCDdthlllHHRljX8F+Vy+19z4dd7jBFp6odch15rftMY9ztvruMonHqR10jFa0r9OMUh3EsdgRe\nNo66PzfodwYz7Pomldt+wwTD5LPqOBdbXlXv2yQK3Cj9clLbUb9h1XEK56Rj78Y9yjZVXM64cRSX\nW2W/al2hb9qob/wkd069H1CXWXdT34AeNp9NbeTF6XGWVZdxDkQmqarCOWlVnsVXYbGDpXHW4+vo\n++gtAv00dQYC5WNsgzbHOk5sk37/27xTLxonp23cgfUzyTPkcd7X1h3Rt+lNLhNL0/FKKnX01HSc\nZWNow05zmBiaHmYs82FrU0bJaVtiL6Ops/hR1usj+gxM08bRdt0d5zSYlvd9WuKcBqP2Txf6THhj\nqs40FXubPT6iN6uId5yWExd6M7PMudCbmWXOhd7MLHMu9GZmmXOhNzPLnAu9mVnmXOjNzDLnQm9m\nljkXejOzzLnQm5llzoXezCxzLvRmZplzoTczy9zAQi/pIUmXJJ0qtK2S9ISks+nnytQuSQ9KmpN0\nUtLmOoM3M7PByhzRfx/Y3tO2DzgaERuBo2ka4E5gY3rsBQ6ME1z3zkmDHm0wDTGa2WwaWOgj4rfA\n33qadwIPp+cPA58otP8gOn4HrJC0ZpwAJS366Ilx5EcVijfv7RfnODF6R2Fm4xj1nrGrI+JCen4R\nWJ2erwXOFeabT20XGNKk7oHaW0RHvWt9v99rS5xmNrvGvjl4RISkoQ85Je2lM7yz2DLHDWuYOBas\nd1DhHrSMuix2FjOpdZvZdBv1qpuXu0My6eel1H4eWF+Yb11qu05EHIyILRGxZbHXmyhgvUV/kCaH\nVIp3hTczez2jFvpDwO70fDfwi0L7venqm63A5cIQTylNF67i2HqZWJo8onaxN7MyBg7dSHoEuB24\nWdI88GXgq8BPJO0BXgQ+mWb/JbADmAP+DXxq2IDaMhRRLKJ1j7+Po0ycZjbb1IajwVHG+CfJRdTM\nWupEv+HvIn8ztgQXeTObZi70ZmaZc6E3M8ucC72ZWeZc6M3MMudCb2aWubH/BUJF/gmcaTqIKXAz\n8GrTQUwB56kc56mcNufp7WVmakuhP1PmWtBZJ+kp52kw56kc56mcHPLkoRszs8y50JuZZa4thf5g\n0wFMCeepHOepHOepnKnPUyv+142ZmdWnLUf0ZmZWk8YLvaTtks5ImpO0b/Bv5EnSeklPSjot6WlJ\n96X2VZKekHQ2/VyZ2iXpwZS3k5I2N/sXTJakJZL+IOlwmt4g6VjKx48lLUvty9P0XHr9libjniRJ\nKyQ9JulZSc9I2ub+dD1Jn0/b3ClJj0i6Ibf+1Gihl7QE+BZwJ7AJ2CVpU5MxNegK8IWI2ARsBT6d\ncrEPOBoRG4GjaRo6OduYHnuBA5MPuVH3Ac8Upr8G3B8R7wBeA/ak9j3Aa6n9/jTfrHgA+FVEvAt4\nD518uT8VSFoLfBbYEhHvBpYAd5Nbf+reJ7WJB7ANOFKY3g/sbzKmtjzo3LXrDjpfJFuT2tbQ+c4B\nwLeBXYX5r86X+4POLSqPAh8CDgOi84WWpen1q/0KOAJsS8+XpvnU9N8wgRzdBLzQ+7e6P12Xp7XA\nOWBV6h+HgY/m1p+aHrrpJrlrPrXNtHQ6eBtwDFgd127HeBFYnZ7Pcu6+CXwR+F+afgvw94i4kqaL\nubiap/T65TR/7jYArwDfS0Nc35F0I+5PC0TEeeDrwEvABTr94wSZ9aemC731kPRm4GfA5yLiH8XX\nonMYMdOXSUn6GHApIk40HUvLLQU2Awci4jbgX1wbpgHcnwDSZxQ76ewY3wbcCGxvNKgaNF3ozwPr\nC9PrUttMkvRGOkX+RxHxeGp+WdKa9Poa4FJqn9XcfQD4uKS/AI/SGb55AFghqfsvPYq5uJqn9PpN\nwF8nGXBD5oH5iDiWph+jU/jdnxb6CPBCRLwSEf8FHqfTx7LqT00X+uPAxvQJ9zI6H4IcajimRqhz\nv8LvAs9ExDcKLx0Cdqfnu+mM3Xfb701XS2wFLhdOybMVEfsjYl1E3EKnv/wmIu4BngTuSrP15qmb\nv7vS/NkfxUbEReCcpHempg8Dp3F/6vUSsFXSm9I22M1TXv2p6Q8JgB3Ac8DzwJeajqfBPHyQzmn0\nSeCP6bGDzvjfUeAs8GtgVZpfdK5Yeh74E52rBhr/Oyacs9uBw+n5rcDvgTngp8Dy1H5Dmp5Lr9/a\ndNwTzM97gadSn/o5sNL9adE8fQV4FjgF/BBYnlt/8jdjzcwy1/TQjZmZ1cyF3swscy70ZmaZc6E3\nM8ucC72ZWeZc6M3MMudCb2aWORd6M7PM/R8JMWo/kOw65AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x109801080>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "# Function to show an image.\n",
    "def imshow(img):\n",
    "    npimg = img.numpy()\n",
    "    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n",
    "\n",
    "dataiter = iter(train_loader)\n",
    "images, parameters = dataiter.next()\n",
    "\n",
    "# Show images.\n",
    "imshow(torchvision.utils.make_grid(images))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "1.00000e-05 *\n",
       "  0.0400  0.1550  0.0800  2.4000  0.0960\n",
       "  0.0300  0.1550  0.0750  2.2000  0.0960\n",
       "  0.0300  0.1450  0.0800  2.3000  0.0920\n",
       "  0.0350  0.1450  0.0700  1.8000  0.0960\n",
       "  0.0250  0.1500  0.0700  2.4000  0.0800\n",
       "  0.0300  0.1550  0.0850  2.0000  0.0960\n",
       "  0.0400  0.1550  0.0750  2.2000  0.0800\n",
       "  0.0250  0.1550  0.0700  2.4000  0.0920\n",
       "  0.0400  0.1450  0.0750  2.0000  0.0960\n",
       "  0.0400  0.1550  0.0750  1.8000  0.0920\n",
       "[torch.FloatTensor of size 10x5]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Corresponding parameters of simulated models -\n",
    "# (gap, width1, height, radius1, width2).\n",
    "parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.encoder = nn.Sequential(\n",
    "            nn.Conv2d(1, 3, 5, stride=2),\n",
    "            nn.MaxPool2d(5, 1),\n",
    "            nn.Conv2d(3, 5, 3, stride=1),\n",
    "            nn.MaxPool2d(3, 1),\n",
    "            nn.ReLU(True),\n",
    "            nn.Conv2d(5, 2, 3, stride=1),\n",
    "            nn.MaxPool2d(2, 2)\n",
    "        )\n",
    "        self.head = nn.Linear(396, 5)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.encoder(x)\n",
    "        return self.head(x.view(x.size(0), -1))\n",
    "\n",
    "criterion = nn.MSELoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_net(net):\n",
    "    \"\"\"Function helps to calculate loss on test dataset.\"\"\"\n",
    "    test_loss = 0\n",
    "    for data in test_loader:\n",
    "        \n",
    "        inputs, labels = data\n",
    "        labels = labels * 1e7\n",
    "\n",
    "        inputs, labels = Variable(inputs), Variable(labels)\n",
    "\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        test_loss += loss.data[0]\n",
    "    return test_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,    30] loss: 892.943\n",
      "[1,    60] loss: 907.639\n",
      "[1,    90] loss: 898.684\n",
      "[1,   120] loss: 900.220\n",
      "[2,    30] loss: 852.693\n",
      "[2,    60] loss: 840.837\n",
      "[2,    90] loss: 794.376\n",
      "[2,   120] loss: 743.243\n",
      "[2] Test loss: 596.033\n",
      "LR: 0.0001\n",
      "[3,    30] loss: 555.026\n",
      "[3,    60] loss: 445.326\n",
      "[3,    90] loss: 307.447\n",
      "[3,   120] loss: 182.556\n",
      "[4,    30] loss: 36.213\n",
      "[4,    60] loss: 17.143\n",
      "[4,    90] loss: 11.459\n",
      "[4,   120] loss: 9.654\n",
      "[4] Test loss: 10.788\n",
      "LR: 0.0001\n",
      "[5,    30] loss: 10.943\n",
      "[5,    60] loss: 10.057\n",
      "[5,    90] loss: 10.493\n",
      "[5,   120] loss: 9.491\n",
      "[6,    30] loss: 10.899\n",
      "[6,    60] loss: 9.982\n",
      "[6,    90] loss: 10.443\n",
      "[6,   120] loss: 9.433\n",
      "[6] Test loss: 10.744\n",
      "LR: 5e-05\n",
      "[7,    30] loss: 10.839\n",
      "[7,    60] loss: 9.927\n",
      "[7,    90] loss: 10.397\n",
      "[7,   120] loss: 9.375\n",
      "[8,    30] loss: 10.777\n",
      "[8,    60] loss: 9.871\n",
      "[8,    90] loss: 10.351\n",
      "[8,   120] loss: 9.316\n",
      "[8] Test loss: 10.635\n",
      "LR: 5e-05\n",
      "[9,    30] loss: 10.710\n",
      "[9,    60] loss: 9.818\n",
      "[9,    90] loss: 10.323\n",
      "[9,   120] loss: 9.279\n",
      "[10,    30] loss: 10.685\n",
      "[10,    60] loss: 9.774\n",
      "[10,    90] loss: 10.295\n",
      "[10,   120] loss: 9.245\n",
      "[10] Test loss: 10.617\n",
      "LR: 2.5e-05\n",
      "Finished Training\n",
      "CPU times: user 19.3 s, sys: 3.23 s, total: 22.5 s\n",
      "Wall time: 21 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "EPOCH = 10  # Total number of epochs to train.\n",
    "LR = 0.0001  # Initial learning rate.\n",
    "\n",
    "net = Net()\n",
    "optimizer = optim.Adam(net.parameters(), LR)\n",
    "\n",
    "print_each = 30  # Print training loss every `print_each` iteration.\n",
    "test_each = 2    # Print testing loss every `test_each` epoch.\n",
    "reduce_lr = 4    # Reduce `LR` by 2 every `reduce_lr` epoch.\n",
    "\n",
    "curtimestep = 0\n",
    "test_steps = []\n",
    "test_losses = []\n",
    "train_steps = []\n",
    "train_losses = []\n",
    "\n",
    "for epoch in range(EPOCH):  # Loop over the dataset multiple times.\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(train_loader, 0):\n",
    "        for param_group in optimizer.param_groups:\n",
    "            param_group['lr'] = LR\n",
    "        \n",
    "        # Get the inputs.\n",
    "        inputs, parameters = data\n",
    "        parameters = parameters * 1e7\n",
    "\n",
    "        # Wrap them in Variable.\n",
    "        inputs, parameters = Variable(inputs), Variable(parameters)\n",
    "\n",
    "        # Zero the parameter gradients.\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # Forward -> backward -> optimize.\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, parameters)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # Add loss per element.\n",
    "        running_loss += loss.data[0] / len(parameters)\n",
    "        \n",
    "        curtimestep += 1\n",
    "        \n",
    "        # Print statistics.\n",
    "        if i % print_each == print_each - 1:\n",
    "            loss_per_element = running_loss / print_each\n",
    "            print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, loss_per_element))\n",
    "            \n",
    "            train_losses.append(loss_per_element)\n",
    "            train_steps.append(curtimestep)\n",
    "            running_loss = 0.0\n",
    "\n",
    "    # Calculate loss on test dataset.\n",
    "    if epoch % test_each == test_each - 1:\n",
    "        test_loss = test_net(net)\n",
    "        test_loss /= len(test_dataset)\n",
    "        print('[%d] Test loss: %.3f' % (epoch + 1, test_loss))\n",
    "        print('LR: {}'.format(LR))\n",
    "        \n",
    "        test_losses.append(test_loss)\n",
    "        test_steps.append(curtimestep)\n",
    "    \n",
    "    # Reduce LR by 2 every `reduce_lr` epoch.\n",
    "    if epoch % reduce_lr == reduce_lr - 1:\n",
    "        LR /= 2.\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl4FfW9x/H395yTEJaQBRLABAEB\n2Rchsgi4AG31Wpf2Yt1Q61Jq3bW2xbb3drG3j1br1lqta92qrbhWbbFFBQFFArKDssgS1hDZIZDk\n/O4fMwkBA5wkZ0vyeT3PPJmZM+c335nk5Htm5reYcw4REZHDBRIdgIiIJCclCBERqZEShIiI1EgJ\nQkREaqQEISIiNVKCEBGRGilBiIhIjZQgRESkRkoQIiJSo1CiA6iPtm3bus6dOyc6DBGRBmXOnDlb\nnXM5x9quQSeIzp07U1hYmOgwREQaFDNbE8l2usUkIiI1UoIQEZEaKUGIiEiNGvQzCBFpHMrKyigq\nKqK0tDTRoTQqaWlp5Ofnk5KSUqf3K0GISMIVFRWRnp5O586dMbNEh9MoOOcoKSmhqKiILl261KkM\n3WISkYQrLS2lTZs2Sg5RZGa0adOmXldlShAikhSUHKKvvudUCaKarbv385cZXzBzxVbKKsKJDkdE\nJKH0DALYtucAj324ir/MWM2+sgoAMpqnMKZnLl/v045TT8yhRapOlUhjVVJSwpgxYwDYtGkTwWCQ\nnByvofEnn3xCamrqMcu48sormThxIj169Ihon0888QSLFi3igQceqHvgMdak/+vtLC3jiQ+/4Knp\nX7DnQDnnDjiOa0/rytov9/Lu4s1MWbaZVz9dT7NQgFHd25KX2Zzd+yvYvb+M3fvLvfnSMvKyWnDP\nuP60a52W6EMSkTpo06YN8+bNA+CXv/wlrVq14vbbbz9kG+cczjkCgZpvvDz99NMxjzPemuQtpj37\ny3n4/RWMuvt9HpqynFHd2zL5llN58KKT6NWhNd/o057ff2cAhT8by4vfG8YlQ4/ns827eO3T9Xy8\nqoTVW/dSWhYms3kKPdqnM2f1l3zr4Rks3bgz0YcmIlG0YsUKevfuzaWXXkqfPn3YuHEjEyZMoKCg\ngD59+vDrX/+6atuRI0cyb948ysvLyczMZOLEiQwYMIDhw4ezZcuWiPf5/PPP069fP/r27ctPf/pT\nAMrLy7nsssuq1j/00EMA3H///fTu3Zv+/fszfvz46B48TfQK4tGpK/nDeysY0zOXW792In3zMmrc\nLhQMMLxrG4Z3bcMvzulzxPIWb9jB1X8p5IJHP+KPl5zE6T1yYxW6SKP3q38sZsmG6H7Z6n1c66N+\nho9m2bJlPPvssxQUFABw1113kZ2dTXl5OWeccQbjxo2jd+/eh7xnx44dnHbaadx1113cdtttPPXU\nU0ycOPGY+yoqKuLnP/85hYWFZGRkMHbsWN566y1ycnLYunUrCxcuBGD79u0A/O53v2PNmjWkpqZW\nrYumJnkFceWILrx63Sk8+d2Tj5gcaqPPcRm8fv0Ijs9uwdXPFPL8xxH1gyUiDUDXrl2rkgPAiy++\nyKBBgxg0aBBLly5lyZIlX3lP8+bNOeusswAYPHgwq1evjmhfs2bNYvTo0bRt25aUlBQuueQSpk2b\nRrdu3fjss8+46aabmDx5MhkZ3v+tPn36MH78eF544YU6N4Y7miZ5BZHdMpXslsd+6FQb7TPSePna\n4dz44qf8/PVFrCnZwx1n9SIQUNU9kdqo6zf9WGnZsmXV/PLly3nwwQf55JNPyMzMZPz48TW2M6j+\nUDsYDFJeXl6vGNq0acOCBQv45z//ycMPP8wrr7zCY489xuTJk5k6dSpvvvkmv/3tb1mwYAHBYLBe\n+6quSV5BxErLZiEev7yA757Smcc//IIfvDCHfQcqEh2WiETJzp07SU9Pp3Xr1mzcuJHJkydHtfyh\nQ4fy/vvvU1JSQnl5OS+99BKnnXYaxcXFOOe44IIL+PWvf83cuXOpqKigqKiI0aNH87vf/Y6tW7ey\nd+/eqMbTJK8gYikYMH55bh86tWnBnW8t4aLHPuLxKwrITVcNJ5GGbtCgQfTu3ZuePXvSqVMnRowY\nUa/ynnzySSZNmlS1XFhYyJ133snpp5+Oc45zzjmHs88+m7lz53L11VfjnMPMuPvuuykvL+eSSy5h\n165dhMNhbr/9dtLT0+t7iIcw51xUC4yngoICl8wDBv1nyWZufPFTslum8tR3T6ZH++j+8kQai6VL\nl9KrV69Eh9Eo1XRuzWyOc67gCG+poltMMTS2dztevnY4ZRVhxj0ykw+XFyc6JBGRiClBxFjfPK+G\nU15Wc7779Gxe/GRtokMSEYmIEkQcHJfZnEk/OIVR3dtyx6sL+c1bS9hZWpbosEREjkoJIk5aNQvx\nxOUFXDasE09M/4Jhv53CT19bqNbXIpK0VIspjkLBAHee35dxg/N5/uM1vDKniL/OWktBpywuG96J\nM/u2p1koenWYRUTqQwkiAQZ0zGRAx0x+dnYvJs0p4vmP13DzS/No2yqVP106mCFdshMdooiIbjEl\nUmaLVK4ZdQLv/fB0nr1qCM1Tg/x40nxKy9S4TiSeSkpKGDhwIAMHDqR9+/bk5eVVLR84cCDicp56\n6ik2bdpU42vjx4/n9ddfj1bIcaEEkQQCAePUE3P47bf6sbpkL498sDLRIYk0KZXdfc+bN49rr72W\nW2+9tWo5krEgKh0tQTREShBJZFT3HM4ZcByPfLCSL7buSXQ4IgI888wzDBkyhIEDB3LdddcRDodr\n7H77b3/7G/PmzePCCy+M+MojHA5z22230bdvX/r161fVqnr9+vWMHDmSgQMH0rdvX2bOnHnELr9j\nSc8gksz/nN2LD5Zt4X9eX8RzVw/ROL3S9PxzImxaGN0y2/eDs+6q9dsWLVrEa6+9xsyZMwmFQkyY\nMIGXXnqJrl27fqX77czMTP7whz/wxz/+kYEDB0ZU/ssvv8zSpUuZP38+xcXFnHzyyZx66qk8//zz\nnHPOOfzkJz+hoqKCffv2MWfOnBq7/I4lXUEkmdzWafzozB5MX7GVN+dvSHQ4Ik3af/7zH2bPnk1B\nQQEDBw5k6tSprFy58ojdb9fW9OnTufjiiwkGg7Rv356RI0dSWFjIySefzBNPPMGvfvUrFi1aRKtW\nraK2z9qI6RWEmd0KXAM4YCFwJdABeAloA8wBLnPOHTCzZsCzwGCgBLjQObc6lvElq0uHdmLSnCJ+\n8/ZSTu+RS0bz6PfzLpK06vBNP1acc1x11VXceeedX3mtpu63o2X06NF88MEHvP3221x++eX8+Mc/\n5tJLL43pPmsSsysIM8sDbgIKnHN9gSBwEXA3cL9zrhuwDbjaf8vVwDZ//f3+dk1SMGD83/n9KNm9\nn9+/+1miwxFpssaOHcvf//53tm7dCni1ndauXVtj99sA6enp7Nq1K+LyR40axUsvvUQ4HGbz5s3M\nmDGDgoIC1qxZQ/v27ZkwYQJXXnkln3766RH3GUuxfgYRApqbWRnQAtgIjAYu8V9/Bvgl8Ahwnj8P\nMAn4o5mZa8jdzdZDv/wMLh/emWc+Ws24wfn0z89MdEgiTU6/fv34xS9+wdixYwmHw6SkpPDoo48S\nDAa/0v02wJVXXsk111xD8+bN+eSTT75SA+qaa67hhhtuAKBLly5MnTqVjz/+mP79+2Nm3HfffeTm\n5vLUU09x3333kZKSQnp6Os899xzr1q2rcZ+xFNPuvs3sZuD/gH3Au8DNwMf+VQJm1hH4p3Our5kt\nAs50zhX5r60Ehjrnth6p/GTv7ru+dpaWMfb3U8lt3Yw3rh9JUKPTSSOl7r5jJym7+zazLLyrgi7A\ncUBL4MwolDvBzArNrLC4uHF3n906LYX/Pac3i9bv5LmPVic6HBFpYmJZi2ks8IVzrtg5Vwa8CowA\nMs2s8tZWPrDen18PdATwX8/Ae1h9COfcY865AudcQU5OTgzDTw5n9+vAqO5tuf8/y9m+N/IWnSIi\n9RXLBLEWGGZmLcyrzD8GWAK8D4zzt7kCeMOff9Nfxn/9vab6/KE6M+PnZ/dmV2kZD05ZnuhwRGJG\nH/foq+85jVmCcM7NwnvYPBevimsAeAz4CXCbma3Aq+r6pP+WJ4E2/vrbgImxiq2h6dE+nQtP7shz\nH61RC2tplNLS0igpKVGSiCLnHCUlJaSlpdW5DI1J3UBs2VXKGfd8wMjubfnzZcd8tiTSoJSVlVFU\nVERpaWmiQ2lU0tLSyM/PJyXl0LZUkT6kVlcbDURueho/OL0r9777ObNWlTD0hDaJDkkkalJSUujS\npUuiw5DDqKuNBuTqkSfQISON37y9lHC44V75iUjDoATRgDRPDfKjb/Rg4fodvDF//bHfICJSD0oQ\nDcz5A/Pol5fBPf/6TAMLiUhMKUE0MIGA8fOze7FhRylPTv8i0eGISCOmBNEADT2hDV/v3Y4/vb+C\nLbtU60NEYkMJooGaeFZP9peHuf/fajwnIrGhBNFAnZDTikuHHs/fC9epCw4RiQkliAbs/JPyqAg7\npn7euDstFJHEUIJowAbkZ9KmZSrvLduS6FBEpBFSgmjAAgHjtB45TP28mAo1nBORKFOCaODG9GzH\n9r1lfLp2W6JDEZFGRgmigRt1YltCAWOKbjOJSJQpQTRwrdNSKOicxftKECISZUoQjcDonrks27SL\n9dv3JToUEWlElCAagdE92wHoKkJEokoJohHomtOS47NbqLqriESVEkQjYGaM7pnLzJVb1cOriESN\nEkQjcUbPXErLwny0siTRoYhII6EE0UgM7ZJNi9SgbjOJSNQoQTQSaSlBRnRry3vLtuCcWlWLSP0p\nQTQio3vmsn77Pj7fvDvRoYhII6AE0Yic0SMXQLeZRCQqlCAakfYZafQ5rrXaQ4hIVChBNDKje+ZS\nuOZLDSIkIvWmBNHInNEzl7BDgwiJSL0pQTQylYMI6TaTiNSXEkQjE/QHEfpAgwiJSD0dM0GY2d2R\nrJPkMbpnrgYREpF6i+QK4ms1rDsr2oFI9IzqnkMwYKruKiL1csQEYWY/MLOFQA8zW1Bt+gJYEL8Q\npbYymqdwcucsJQgRqZejXUH8FTgHeNP/WTkNds6Nj0NsUg8aREhE6uuICcI5t8M5t9o5dzFQBJQB\nDmhlZsfHK0Cpm9E9vVbVqs0kInUVyUPqG4DNwL+Bt/3prRjHJfXUNacVHbObK0GISJ2FItjmFqCH\nc04DDTQgZsaYnu14afZaSssqSEsJJjokEWlgIqnFtA7YUZfCzSzTzCaZ2TIzW2pmw80s28z+bWbL\n/Z9Z/rZmZg+Z2Qr/YfiguuxTDtIgQiJSH5EkiFXAB2Z2h5ndVjlFWP6DwL+ccz2BAcBSYCIwxTnX\nHZjiL4NXdba7P00AHqnFcUgNhnbJpnmKBhESkbqJJEGsxXv+kAqkV5uOyswygFOBJwGccwecc9uB\n84Bn/M2eAc73588DnnWej4FMM+tQi2ORw6SlBBnZXYMIiUjdHPMZhHPuVwBm1sI5t7cWZXcBioGn\nzWwAMAe4GWjnnNvob7MJaOfP5+HdzqpU5K/biNTe/l3QLJ3RPXP595LNLN+ymxPbHTOvi4hUiaQW\n03AzWwIs85cHmNmfIig7BAwCHnHOnQTs4eDtJACc97W2Vl9tzWyCmRWaWWFxsXosrdE/boGnzwLn\nqgYRmrJUt5lEpHYiucX0APANoATAOTcf79bRsRQBRc65Wf7yJLyEsbny1pH/s/I/13qgY7X35/vr\nDuGce8w5V+CcK8jJyYkgjCYo/2TYtBCWv0v7jDR6d9AgQiJSexH15uqcW3fYqooI3rMJWGdmPfxV\nY4AleC2zr/DXXQG84c+/CVzu12YaBuyoditKaqP/dyDjeJh2DzjHmF65zFm7TYMIiUitRFTN1cxO\nAZyZpZjZ7Xi1kSJxI/CCmS0ABgK/Be4CvmZmy4Gx/jLAO3g1plYAjwPXRX4YcohgCoy8GYpmwxfT\nOKNnLhVhp0GERKRWImkody1eddU8vFs+7wLXR1K4c24eUFDDS2Nq2NZFWq5EYOB4mHoPTLuHAZf/\ng2x/EKHzBuYlOjIRaSAiqcW0Fbg0DrFINKWkwSk3wrs/I1j0CaefmMP7n22hIuwIBizR0YlIAxBJ\nLaYuZnafmb1qZm9WTvEITuqp4Epong0f3svoXrls21vGvHUaREhEIhPJLabX8Rq7/QMIxzYciarU\nljD8OnjvN5w2YiPBgDFl6RYGd8pOdGQi0gBE8pC61Dn3kHPufefc1Mop5pFJdAyZAM0ySJ/1AAWd\nNIiQiEQukgTxoJn9wm8wN6hyinlkEh1pGTDke7D0H3y7426WbdrFBg0iJCIRiCRB9AO+h1cd9ff+\ndG8sg5IoG3YdpDTnzG1/BWDGiq0JDkhEGoJInkFcAJzgnFMrq4aqZRsouIrWHz/CgBajmbmyhAsK\nOh77fSLSpEVyBbEIyIx1IBJjw2/AAiF+nP5PZqzYqt5dReSYIkkQmcAyM5usaq4NWOsOcNJ4hu2c\nTGDXBlYW70l0RCKS5CK5xfSLmEch8THiZgJzn+H7obeYufJUuuW2SnREIpLEjnkF4VdpXQ2k+POz\ngbkxjktiIasT1v9CLg69z4Jlnyc6GhFJcpG0pP4eXlfdf/ZX5eE1npOGaORtpFJGrzXPUxHWcwgR\nObJInkFcD4wAdgI455YDubEMSmKobTc25J3FhW4yy1atSXQ0IpLEIkkQ+6tXcTWzELUcBU6SS/Mx\nP6KVlbJ3eiQDA4pIUxVJgphqZj8FmpvZ14CX8fplkgaqzQmDmBkaSq+1f/XGrhYRqUEkCWIiUAws\nBL6PN7DPz2MZlMTeoq7fo1V4F+Wznkh0KCKSpCKpxRR2zj3unLvAOTfOn9ctpgauc/9RTKvoR3jm\nH6BMfTOJyFcdMUGY2UIzW3CkKZ5BSvQNPaEND1ecT2ppCcx9NtHhiEgSOlpDuW/GLQqJu4zmKZTm\nDWfJ9r70nvEgDL4SQqmJDktEksgRryCcc2sqJ39Vd39+C/BlXKKTmBrRtQ337P0m7FwP819MdDgi\nkmTq0lAuHzWUaxRO6dqW9yv6sTO7H0y/DyrKEx2SiCQRNZRrwgo6Z5EaCvKvrEth22pY/GqiQxKR\nJKKGck1YWkqQwcdn8UxJb8jtDR/+HsIadlxEPGoo18SN6NaGxZt2s3vIzVC8DJa9leiQRCRJqKFc\nEze8a1sApqaMhOyuMO0eUDMXEUEN5Zq8AfkZtGoWYsaqbTDyVti0AFb8J9FhiUgSiOQKQhqxUDDA\n0C7ZzFyxFfpfCBkddRUhIoAShACndGvL6pK9rN9dASNuhnWzYPX0RIclIgl21ARhZkEzuzdewUhi\njOjWBsC7ijhpPLRq511FiEiTdtQE4ZyrAEbGKRZJkB7t0mmdFmLu2u2Q0hyG3wBfTIV1sxMdmogk\nUCS3mD41szfN7DIz+3blFPPIJG7MjP75mSwo2u6tKLgKmmfBh7p4FGnKIkkQaUAJMBo4x5/UkV8j\nM6BjBss27aK0rAKatYJh18Pn/4KN6rhXpKk6Wm+uADjnroxHIJJY/fMzqQg7Fm/YyeBOWTDkezDz\nIa919XeeSXR4IpIAkXTWd6KZTTGzRf5yfzNTQ7lGZkB+JsDB20zNM70kseQNKP48gZGJSKJEcovp\nceAOoAzAObcAuCiWQUn8tc9IIze9GQuKdhxcOew676H19PsSF5iIJEwkCaKFc+6Tw9ZF3C+0X1X2\nUzN7y1/uYmazzGyFmf3NzFL99c385RX+650j3YdER//8TOZXXkEAtGzrDSS04O9eb68i0qREkiC2\nmllX/B5czWwcsLEW+7gZWFpt+W7gfudcN2AbcLW//mpgm7/+fn87iaMB+RmsKt7DztKygytPuREC\nQZj+QOICE5GEiHQ8iD8DPc1sPXALcG0khZtZPnA28IS/bHi1oSb5mzwDnO/Pn+cv478+xt9e4qR/\nR+85xMLqt5lad/Aaz817AXZuSFBkIpIIkXTWt8o5NxbIAXo650ZWG4b0WB4AfgxUDjLQBtjunKu8\nRVUE5PnzecA6f5/lwA5/e4mT/nkZAIfeZgKv+41wBcz8QwKiEpFEiaQWUxszewj4EPjAzB40s2P+\n4zazbwJbnHNzohBn9XInmFmhmRUWFxdHs+gmL6tlKp3atGDBuh2HvdDZ68iv8GnYrXMu0lREcovp\nJbzxIP4bGOfP/y2C940AzjWz1X4Zo4EHgUx/VDrwxrde78+vBzpC1ah1GXgN9A7hnHvMOVfgnCvI\nycmJIAypjUNaVFc36jYoL4WP/xT/oEQkISJJEB2cc3c6577wp98A7Y71JufcHc65fOdcZ7xqse85\n5y4F3sdLNABXAG/482/6y/ivv6dxJ+JvQH4GG3aUUrxr/6EvtO0Ofc6HTx6HfdsSE5yIxFUkCeJd\nM7vIzAL+9B1gcj32+RPgNjNbgfeM4Ul//ZNAG3/9bXgj2Umc9T+8wVx1o34IB3Z5SUJEGr1IEsT3\ngL8C+/3pJeD7ZrbLzHZGshPn3AfOuW/686ucc0Occ938Uer2++tL/eVu/uur6nZIUh9981oTMJhf\ntOOrL7bvByee5d1m2r87/sGJSFxFUosp3TkXcM6l+FPAX5funGsdjyAlflqkhuiem878dTVcQQCc\nert3i6nwqfgGJiJxpxHl5Cv652ewoGg7NT4Cyi+AE073qryW7Yt3aCISR0oQ8hX9O2aybW8ZRduO\nkABG3Q57tsCnz8c3MBGJKyUI+YoB+UdoMFep80joOMzrfqP8QBwjE5F4iqShXFcza+bPn25mN5lZ\nZuxDk0Tp2b41qcHAoT27VmcGp/4IdhbBgkiaxIhIQxTJFcQrQIWZdQMew2vM9teYRiUJlRoK0Ou4\n1kd+UA3QbQx0GOh1BV4Rcee+ItKARJIgwn7fSN8C/uCc+xHQIbZhSaINyM9g0fodVISP0FbRzKvR\n9OUqWPJ6fIMTkbiIJEGUmdnFeK2c3/LXpcQuJEkG/fMz2XOggpXFR2nv0ONsyOkF0+6FcPjI24lI\ngxRJgrgSGA78n3PuCzPrAjwX27Ak0aoeVB/tNlMg4LWuLl4Kn70Tp8hEJF4iaSi3xDl3k3PuRTPL\nAtKdcxrMp5E7IacVLVODR35QXanPtyCrC0y7B9R1lkijEkktpg/MrLWZZQNzgcfNTIMUN3LBgNE3\nL6PmPpkO2TDk9fS6cR6snBKf4EQkLiK5xZThnNsJfBt41jk3FBgb27AkGQzomMnSjbs4UH6M5wv9\nL4LW+d6zCBFpNCJJECEz6wB8h4MPqaUJGJCfyYGKMMs2HaNPxlCqN+rc2o9g9Yz4BCciMRdJgvg1\nXvfeK51zs83sBGB5bMOSZNA/kgfVlQZdBi1zvWcRItIoRPKQ+mXnXH/n3A/85VXOuf+OfWiSaPlZ\nzclumVpz19+HS2kOp9wAq96HoqiOMisiCRLJQ+p8M3vNzLb40ytmlh+P4CSxzIwB+RnMi+QKAqDg\nKkjLhA/1LEKkMYjkFtPTeMOBHudP//DXSRMwuFMWK7bsZvveCDrla5YOw67z2kRsWhT74EQkpiJJ\nEDnOuaedc+X+9BcgJ8ZxSZIY3CkbgLlrIxyHeugESE2HD38fw6hEJB4iSRAlZjbezIL+NB4oiXVg\nkhwGdMwgGDAKV0eYIJpnwZBrYPFrsFV1GUQaskgSxFV4VVw3ARuBccB3YxiTJJEWqSH6HNeaOWsi\nTBAAw66HUBpMvz92gYlIzEVSi2mNc+5c51yOcy7XOXc+oFpMTcjgTlnML9pOWUWEHfK1yoHB34X5\nL8G2NTGNTURip64jyt0W1SgkqQ3ulEVpWZjFG47RYK66U24EC8CMB2MXmIjEVF0ThEU1CklqBf6D\n6lrdZsrIg5MuhU+fg50bYxSZiMRSXROEuu1sQtpnpJGX2Zw5a76s3RtH3ALhCvjoj7EJTERi6ogJ\nwsx2mdnOGqZdeO0hpAkZ3CmLOWu24WrTpXd2F+h3ARQ+BXtU8U2koTlignDOpTvnWtcwpTvnQvEM\nUhKvoHMWm3fup2jbvtq9cdRtULYPPv5TbAITkZip6y0maWIGd8oCavkcAiCnB/Q+Fz55DPZF2GWH\niCQFJQiJSI926bRMDdY+QQCMuh3274TZj0c/MBGJGSUIiUgoGOCk47MorEuC6NAfun8DPvoT7N8d\n/eBEJCaUICRigztl8dmmnewqLav9m0+9HfZ9CXP+EvW4RCQ2lCAkYgWdswg7Iu/+u7qOQ6DLqTDz\nISgrjX5wIhJ1ShASsYEdMwkYkXfcd7hTfwS7N8O856MbmIjEhBKERCw9LYUe7WvZcV91nUdB/hCY\n/iBU1OE2lYjElRKE1EpBpyw+XbuNinAdGtObeVcRO9bCgr9HPzgRiSolCKmVwZ2y2HOggmWbatFx\nX3Xdvwbt+3sDCoUrohuciERVzBKEmXU0s/fNbImZLTazm/312Wb2bzNb7v/M8tebmT1kZivMbIGZ\nDYpVbFJ3lQ3m5tb1NpOZV6Ppy5Ww5PUoRiYi0RbLK4hy4IfOud7AMOB6M+sNTASmOOe6A1P8ZYCz\ngO7+NAF4JIaxSR3lZzWnXetmdWsPUannOdC2B0z7PYQjHGNCROIuZgnCObfROTfXn98FLAXygPOA\nZ/zNngHO9+fPA551no+BTDPrEKv4pG7MjMGdsupekwkgEIBRP4Qti+Hzf0UvOBGJqrg8gzCzzsBJ\nwCygnXOucoCATUA7fz4PWFftbUX+Okkygztls377PjbtqEd7hr7/DVmdYdo9UJseYkUkbmKeIMys\nFfAKcItz7pAnm87rO7pW/x3MbIKZFZpZYXFxcRQjlUgV1LXjvuqCIRh5K2yYC6vej1JkIhJNMU0Q\nZpaClxxecM696q/eXHnryP+5xV+/HuhY7e35/rpDOOcec84VOOcKcnJyYhe8HFHv41qTlhKgsLYD\nCB1uwMWQfhxMuzc6gYlIVMWyFpMBTwJLnXP3VXvpTeAKf/4K4I1q6y/3azMNA3ZUuxUlSSQlGGBA\nfmbdazJVCjWDETfDmhmwZmZ0ghORqInlFcQI4DJgtJnN86f/Au4CvmZmy4Gx/jLAO8AqYAXwOHBd\nDGOTeironMXiDXXsuK+6QZcON6mDAAAQbUlEQVRDyxxdRYgkoVjWYprunDPnXH/n3EB/esc5V+Kc\nG+Oc6+6cG+uc+9Lf3jnnrnfOdXXO9XPOFcYqNqm/sb3aUR52vLOwnhd5qS1g+PWwcgqsnxud4EQk\nKtSSWupkYMdMuuW24uXCovoXVnA1pGV6ratFJGkoQUidmBkXDM6ncM02VhXXcxCgtNYw9FpY9hZs\nXhydAEWk3pQgpM6+dVIewYAxaU4UriKGfh9SW8GH9x17WxGJCyUIqbPc1mmcdmIOr85dX7feXatr\nkQ0nXw2LX4WSldEJUETqRQlC6uWCwfls2lnK9BVb61/Y8BsgmArTdRUhkgyUIKRexvRqR1aLFF4u\nXHfsjY+lVS4MugLmvwTb19a/PBGpFyUIqZfUUIDzBubx7pLN7NgbhVHiRtwEGMx4qP5liUi9KEFI\nvY0bnM+B8jBvzv9Kzyi1l5EPAy+Guc/Crk31L09E6kwJQuqtb14GvTq05uVo1GYCrxO/cBl89Mfo\nlCcidaIEIVFxweB8FhTt4LNNu+pfWPYJ0HcczH4K9tazQ0ARqTMlCImK8wYeRyhgTJoThYfV4A0o\nVLYHPtbAgiKJogQhUdGmVTPG9MrltU/XU1YRhWFEc3tCr3Ng1p+hdEf9yxORWlOCkKi5YHBHtu4+\nwAefRWkgp1G3w/4dMPuJ6JQnIrWiBCFRc3qPHNq2ahadNhEAxw2Ebl+Djx6GA3uiU6aIREwJQqIm\nFAzw7UF5vLdsC1t3749Ooaf+CPaWwJxnolOeiERMCUKiatzgfMrDjjfmbYhOgccPhc6jYOZDUB6l\npCMiEVGCkKg6sV06A/IzeLlwHc7VswO/SqfeDrs2wrwXolOeiERECUKiblxBR5Zt2sXiDTujU2CX\n0yD/ZJh+P1REoTsPEYmIEoRE3bn9jyM1FIjew2ozr0bT9rWwcFJ0yhSRY1KCkKjLaJHC13u34435\nG9hfXhGdQk/8BrTr5w1LGo5SmSJyVEoQEhMXFHRk+94ypizdEp0CzeDUH0LJclj6ZnTKFJGjUoKQ\nmBjZrS0dMtKid5sJoNe50PZEmPZ7iNYDcBE5IiUIiYlgwPj2oDymfl7M5p2l0Sk0EISRt8HmhfD5\n5OiUKSJHpAQhMTNucEfCDl6dG4VxIir1GweZnWDaPbqKEIkxJQiJmS5tW1LQKYuX50SxTUQwBUbe\nAusL4Yup0SlTRGqkBCExdUFBPquK9zB37fboFTrwUkjvANPujV6ZIvIVShASU2f3P47mKUEmRWu0\nOYBQMzjlJlj9Iaz9OHrlisghlCAkplo1C3FWv/a8NX8D+w5Esf3C4CugRVtdRYjEkBKExNy4wfns\n2l/O5MWboldoaksYfh2s+Dds+DR65YpIFSUIiblhXdqQn9Wcl6M1HGmlk78HaRle62oRiTolCIm5\nQMAYNzifmStLmL58a/QKTmsNQ74PS/8BW5ZGr1wRAZQgJE4uGXo8ndu0ZPyTs/if1xexZ395dAoe\n9gNIaQkf3hed8kSkihKExEVuehrv3DSKq0Z04flZa/jGA9OYuTIKVxMtsuHkq2DRJChZWf/yRKSK\nEoTETfPUIP97Tm/+/v3hpAQDXPJ4lK4mht8IgRSY8UB0AhURQAlCEuDkztm8c9Morh558Gri8Wmr\n+GhlCTtL6zAgUHo7r9rrvBdhRxTbW4g0cRa1LhCiwMzOBB4EgsATzrm7jrZ9QUGBKywsjEtsEhuF\nq7/kZ68t4rPNu6rWdWnbkr55GfQ9rjVdc1qR0SKF1mkptG4eonVaCi1Sg5hZ1fbOOcq+XEvKw4M4\n0Oc77B40gfIKR0XYUV4RpqzCUR52pASNlGCA1FCAVP9nStAwjIqwo8J576mcB2gWDJIaMlJDAQz7\nSvwADq/8yv2UlYcp83+W+zGEgpX7NFJDwaoYDL6y3/IKf9+hyhiP/j0uHHaUhcOUhzm476rjDhMK\nWNXxpgaDpKYESAkEMIOwgwoX9vcPFWGHA1KDRrNQkGDg6Pt2eHGXhR3l5Y4D4TDlFQf3HwxAM/94\nU4JGaihUdc4r31sR9s5bOOwIu4PHHTjGvit/9wfC3jmr/F17+w4TDPjnOmhV5zEUqPZ3A5RVePst\nc2EI452jUOAIv+nDj907X5X7K6twHKjwjj9glcdt3jkPBai266r3Vh27c1W/o0j2DUDLtl4tvjow\nsznOuYJjbpcsCcLMgsDnwNeAImA2cLFzbsmR3qME0Xhs3b2fRet3sGj9Dhau38Gi9TtZv31fjdsG\nA0aL1GC1D6f3N3xX6DEuCn0Qx6hFEujs++Dkq+v01kgTRKhOpcfGEGCFc24VgJm9BJwHHDFBSOPR\ntlUzTu+Ry+k9cqvWfbnnAOu+3Muu0nJ27CtjZ2kZO/eVsWNfGXsPVBAKGCn+N8PUoLHX/ZopO2YT\nCjhCASMYMEKBAMGAETSjwvnfqqt9yywPe8klYEbAvCq5QTMC5n3DrfpG6n8zP1AepjwcPqTsFH9f\nKUF/n/431YPbQEUYysPhg/utcBzwE1sw4O0/aEbAnwfv221ZeZgDYX//ld9OA0YoYISCXvmhoBEy\nCAUD/v799X4Mld9SyyrClJeHKQuHOVAe9vdt/r69Y6/cd3nFwVgPVPtmHjSv/OrHWbnv6sccCkIw\nECDsHGXl3pVMedWxhwm7aue68vgDlft2lJVXVMVZue9AwAhV7f/w46bq91EZh3OV59y/uvCv5iqc\nI2iBg+fdjwOzqhirH/OB8jCB6ue36vxX39/BeIKBgH91cvBqqjKOcPjg7++Q48eqfkeVx3zAj6Vv\nXmsGH5/91Q9N3uBYfRyrJFOCyAOqt6QqAoYmKBZJAtktU8lumVrLdw2ISSwiTVGDe0htZhPMrNDM\nCouLixMdjohIo5VMCWI90LHacr6/7hDOuceccwXOuYKcnJy4BSci0tQkU4KYDXQ3sy5mlgpcBGh0\nehGRBEmaZxDOuXIzuwGYjFfN9Snn3OIEhyUi0mQlTYIAcM69A7yT6DhERCS5bjGJiEgSUYIQEZEa\nKUGIiEiNkqarjbows2JgzTE2awtEcZSamFGc0aU4o0txRlei4+zknDtmO4EGnSAiYWaFkfQ5kmiK\nM7oUZ3QpzuhqKHHqFpOIiNRICUJERGrUFBLEY4kOIEKKM7oUZ3QpzuhqEHE2+mcQIiJSN03hCkJE\nROqg0SYIMzvTzD4zsxVmNjHBsXQ0s/fNbImZLTazm/312Wb2bzNb7v/M8tebmT3kx77AzAbFOd6g\nmX1qZm/5y13MbJYfz9/8zhQxs2b+8gr/9c5xjDHTzCaZ2TIzW2pmw5PxfJrZrf7vfJGZvWhmaclw\nPs3sKTPbYmaLqq2r9fkzsyv87Zeb2RVxivMe//e+wMxeM7PMaq/d4cf5mZl9o9r6mP4/qCnOaq/9\n0MycmbX1lxN2PmvNOdfoJrzO/lYCJwCpwHygdwLj6QAM8ufT8YZW7Q38Dpjor58I3O3P/xfwT8CA\nYcCsOMd7G/BX4C1/+e/ARf78o8AP/PnrgEf9+YuAv8UxxmeAa/z5VCAz2c4n3iBYXwDNq53H7ybD\n+QROBQYBi6qtq9X5A7KBVf7PLH8+Kw5xfh0I+fN3V4uzt/9ZbwZ08f8HBOPx/6CmOP31HfE6IF0D\ntE30+az1cSVy5zE7KBgOTK62fAdwR6LjqhbPG3hjb38GdPDXdQA+8+f/jDced+X2VdvFIbZ8YAow\nGnjL/yPeWu0DWXVu/T/84f58yN/O4hBjhv+P1w5bn1Tnk4OjJGb75+ct4BvJcj6Bzof9463V+QMu\nBv5cbf0h28UqzsNe+xbwgj9/yOe88nzG6/9BTXECk/CGOVzNwQSR0PNZm6mx3mKqafjSvATFcgj/\ntsFJwCygnXNuo//SJqCdP5/I+B8AfgyE/eU2wHbnXHkNsVTF6b++w98+1roAxcDT/q2wJ8ysJUl2\nPp1z64F7gbXARrzzM4fkO5+Vanv+kuFzdhXet3GOEk9C4jSz84D1zrn5h72UVHEeTWNNEEnJzFoB\nrwC3OOd2Vn/NeV8ZElqlzMy+CWxxzs1JZBwRCOFdzj/inDsJ2IN3S6RKkpzPLOA8vIR2HNASODOR\nMUUqGc7fsZjZz4By4IVEx3I4M2sB/BT430THUh+NNUFENHxpPJlZCl5yeME596q/erOZdfBf7wBs\n8dcnKv4RwLlmthp4Ce8204NApplVjh1SPZaqOP3XM4CSOMRZBBQ552b5y5PwEkaync+xwBfOuWLn\nXBnwKt45TrbzWam25y9hnzMz+y7wTeBSP5lxlHgSEWdXvC8G8/3PUz4w18zaJ1mcR9VYE0RSDV9q\nZgY8CSx1zt1X7aU3gcqaClfgPZuoXH+5X9thGLCj2qV/zDjn7nDO5TvnOuOds/ecc5cC7wPjjhBn\nZfzj/O1j/q3TObcJWGdmPfxVY4AlJNn5xLu1NMzMWvh/A5VxJtX5rKa2528y8HUzy/Kvlr7ur4sp\nMzsT7zbouc65vYfFf5FfG6wL0B34hAT8P3DOLXTO5TrnOvufpyK8iiqbSLLzeVSJfAASywmvpsDn\neLUXfpbgWEbiXa4vAOb503/h3V+eAiwH/gNk+9sb8LAf+0KgIAExn87BWkwn4H3QVgAvA8389Wn+\n8gr/9RPiGN9AoNA/p6/j1fpIuvMJ/ApYBiwCnsOrYZPw8wm8iPdcpAzvn9fVdTl/eM8AVvjTlXGK\ncwXevfrKz9Kj1bb/mR/nZ8BZ1dbH9P9BTXEe9vpqDj6kTtj5rO2kltQiIlKjxnqLSURE6kkJQkRE\naqQEISIiNVKCEBGRGilBiIhIjULH3kSkcTGzyuqcAO2BCryuOwD2OudOidF+OwOnOOf+GovyRaJN\n1VylSTOzXwK7nXP3xmFfpwO3O+e+Get9iUSDbjGJVGNmu/2fp5vZVDN7w8xWmdldZnapmX1iZgvN\nrKu/XY6ZvWJms/1phL/+NDOb50+fmlk6cBcwyl93q3njbtzjv2+BmX2/2r6nmdnb/hgGj5pZwN/+\nL+aNLbHQzG5N1HmSpkG3mESObADQC/gSr2/+J5xzQ8wb8OlG4Ba8vqrud85NN7Pj8bpG6AXcDlzv\nnJvhd9JYitehYNUVhJlNwOtm4WQzawbMMLN3/X0PwRvfYA3wL+DbeF2c5znn+vrvrxooRyQWlCBE\njmy28/tsMrOVQOU/74XAGf78WKC319USAK39hDADuM/MXgBedc4VVdum0teB/mZW2S9TBl7/QQeA\nT5xzq/x9v4jXXcsU4AQz+wPwdrV4RGJCCULkyPZXmw9XWw5z8LMTAIY550oPe+9dZvY2Xh9AM6za\n8JfVGHCjc+6QDtn8ZxWHPxx0zrltZjYAb9Cha4Hv4PXdIxITegYhUj/v4t1uAsDMBvo/uzqvR8+7\n8XoT7QnswhtyttJk4Ad+V/CY2Yn+wEcAQ/zeRwPAhcB088Y0DjjnXgF+jtfFuUjM6ApCpH5uAh42\nswV4n6dpeN/ubzGzM/CuNhbjjXoWBirMbD7wF7znF53xxgkwvKq25/vlzgb+CHTD6x78NaAf3ih6\nlV/s7oj1wUnTpmquIklG1WElWegWk4iI1EhXECIiUiNdQYiISI2UIEREpEZKECIiUiMlCBERqZES\nhIiI1EgJQkREavT/tCqaORh+/rQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e4ee160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_steps, train_losses, label='Train Loss')\n",
    "plt.plot(test_steps, test_losses, label='Test Loss')\n",
    "plt.xlabel('Timesteps')\n",
    "plt.ylabel('Loss per element')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAABTCAYAAACRdMzaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAACJ5JREFUeJzt3VGIXFcdx/Hvz8SkWKFJrISYBJvi\nogRBG4Mk6ENRi2kQ40ORhkKDBPKiWEWQBB/ENwWxtiDBoNUq0qq1aAhiqGvBJ2MalJgmTbO1ttmQ\nNK3WiPqgwb8Pcya52Wx27s7eO+fO2d8Hhp175uzc//zvmf/cOXNnriICMzMr1xtyB2BmZu1yoTcz\nK5wLvZlZ4VzozcwK50JvZlY4F3ozs8K1UuglbZN0WtKUpL1trMPMzOpR08fRS1oCPA/cBUwDR4Gd\nEXGy0RWZmVktbezRvx+Yiog/R8R/gMeBHS2sx8zMamij0K8FzlaWp1ObmZllsDTXiiXtAfakxffl\nisPMbIy9FhFvHdSpjUJ/DlhfWV6X2q4REQeAAwCS/IM7Zmbz91KdTm1M3RwFJiRtkLQMuBc42MJ6\nzMyshsb36CPisqTPAIeBJcAjEfFs0+sxM7N6Gj+8cqggPHVjZjaMYxGxeVAnfzPWzKxwLvRmZoVz\noTczK5wLvZlZ4VzozcwKl+2bsfNxoyODJI04EhuVuY4G83a3LhinutT5Qt9P5mzJa+LQ0FFslCYP\nYW0y3lEeWjufuCNizv7jsN2bym1bcba17ZuOt4Vf123kftquS1VNxNzpQj/oCb+QBPQ3RnWjtPGk\nmmtADHM/TcXbVFx11zNoW1b7D+rX5HZvc5sv5P7bHJ9tPPam89rk467eVxPxDbqPpvPaRMydLfRt\n723OTFo/oU1vpKbu70bxznZbnbiG+b9hVNcxaL2jeIcxWx7nimk+mryvufYUF1qk2tjuTea16fHZ\nRi5HNT0zn+fPXDpb6GG0c139dTW5R9Jm/NV4h4k5xzxinXWOOq6mtvsoXjwXGmvbY7Jq2FhHtROy\nkPhyzcFLGnqHtJNH3eT8WYaZA6DrJDU6/51L7nw3sd1zFdE6cuV3mFhz7uB13bC56WShn2/xamv9\n4/Ckn+/6uljkIf8278cAwz3pc78TqdM/555ol8fnfJ7vucfosDF0stB3xUKLvY2fJl7kR6ULL44l\nKTmXLvQDjMuT3szsRlzoayj5ld5m521uJXGhNzMrnAu9mVnhXOjNzArnQm9mVjgXejOzwrnQm5kV\nzoXezKxwLvRmZoVzoTczK5wLvZlZ4VzozcwK50JvZlY4F3ozs8INLPSSHpF0UdKJStsqSU9JOpP+\nrkztkvSwpClJxyVtajN4MzMbrM4e/feBbTPa9gKTETEBTKZlgLuBiXTZA+xvJsyr+udMrJ4r1czM\nbmxgoY+I3wJ/m9G8A3g0XX8U+ESl/QfR8ztghaQ1TQU7W2GfWfgXemlSl2Mzs8Vj6ZD/tzoizqfr\nF4DV6fpa4Gyl33RqO88Cjeqk1tWCupD1tXE2+5nF3ifHMLM6hi30V0RESJr37qakPfSmd+qsY95x\nDataPIcp1m0U+L6Z99nmusysHMMedfNKf0om/b2Y2s8B6yv91qW260TEgYjYHBGb66ww19nhYf4v\nNKOKddj4zGxxGbbQHwR2peu7gF9U2u9PR99sAS5VpniGkruISapduEc1vVTVjy93nsysuwZO3Uh6\nDLgTuFXSNPBl4KvATyTtBl4CPpm6/xLYDkwB/wY+tdAAx2laImes45QnMxstdWFPcJg5fjMz41id\n6W9/M9bMrHAu9GZmhXOhNzMrnAu9mVnhXOjNzArnQm9mVrgF/wRCQ/4JnM4dxBi4FXgtdxBjwHmq\nx3mqp8t5enudTl0p9Kfr/hTCYibpGedpMOepHuepnhLy5KkbM7PCudCbmRWuK4X+QO4AxoTzVI/z\nVI/zVM/Y56kTv3VjZmbt6coevZmZtSR7oZe0TdJpSVOS9g7+jzJJWi/paUknJT0r6YHUvkrSU5LO\npL8rU7skPZzydlzSpryPYLQkLZH0B0mH0vIGSUdSPn4saVlqX56Wp9Ltt+WMe5QkrZD0hKTnJJ2S\ntNXj6XqSPp+ecyckPSbpptLGU9ZCL2kJ8C3gbmAjsFPSxpwxZXQZ+EJEbAS2AJ9OudgLTEbEBDCZ\nlqGXs4l02QPsH33IWT0AnKosfw14MCLeAbwO7E7tu4HXU/uDqd9i8RDwq4h4F/AeevnyeKqQtBb4\nLLA5It4NLAHupbTxFBHZLsBW4HBleR+wL2dMXbnQO2vXXfS+SLYmta2h950DgG8DOyv9r/Qr/ULv\nFJWTwIeAQ4DofaFlabr9yrgCDgNb0/WlqZ9yP4YR5OgW4MWZj9Xj6bo8rQXOAqvS+DgEfLS08ZR7\n6qaf5L7p1LaopbeDdwBHgNVx9XSMF4DV6fpizt03gS8C/0vLbwH+HhGX03I1F1fylG6/lPqXbgPw\nKvC9NMX1HUk34/F0jYg4B3wdeBk4T298HKOw8ZS70NsMkt4M/Az4XET8o3pb9HYjFvVhUpI+BlyM\niGO5Y+m4pcAmYH9E3AH8i6vTNIDHE0D6jGIHvRfGtwE3A9uyBtWC3IX+HLC+srwutS1Kkt5Ir8j/\nKCKeTM2vSFqTbl8DXEztizV3HwA+LukvwOP0pm8eAlZI6v+kRzUXV/KUbr8F+OsoA85kGpiOiCNp\n+Ql6hd/j6VofAV6MiFcj4r/Ak/TGWFHjKXehPwpMpE+4l9H7EORg5piyUO/s3t8FTkXENyo3HQR2\npeu76M3d99vvT0dLbAEuVd6SFysi9kXEuoi4jd54+U1E3Ac8DdyTus3MUz9/96T+xe/FRsQF4Kyk\nd6amDwMn8Xia6WVgi6Q3pedgP09ljafcHxIA24HngReAL+WOJ2MePkjvbfRx4I/psp3e/N8kcAb4\nNbAq9Re9I5ZeAP5E76iB7I9jxDm7EziUrt8O/B6YAn4KLE/tN6XlqXT77bnjHmF+3gs8k8bUz4GV\nHk+z5ukrwHPACeCHwPLSxpO/GWtmVrjcUzdmZtYyF3ozs8K50JuZFc6F3syscC70ZmaFc6E3Myuc\nC72ZWeFc6M3MCvd/rpcfP6vb4cMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10b8c1b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Get some random testing images.\n",
    "dataiter = iter(test_loader)\n",
    "images, parameters = dataiter.next()\n",
    "\n",
    "# Print images.\n",
    "imshow(torchvision.utils.make_grid(images))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "   2.5000   15.5000    8.5000  180.0000    9.6000\n",
       "   2.5000   14.5000    7.5000  240.0000    8.0000\n",
       "   4.0000   15.0000    7.5000  200.0000    8.0000\n",
       "   3.5000   15.0000    7.5000  180.0000    9.6000\n",
       "   2.5000   15.5000    7.5000  200.0000    8.4000\n",
       "   3.5000   14.5000    7.0000  210.0000    8.4000\n",
       "   3.0000   14.5000    7.0000  240.0000    8.0000\n",
       "   3.5000   15.0000    8.0000  180.0000    8.4000\n",
       "   3.5000   15.0000    8.0000  180.0000    8.0000\n",
       "   3.0000   14.5000    7.5000  200.0000    8.4000\n",
       "[torch.FloatTensor of size 10x5]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters * 1e7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "   3.1878   14.8130    7.7425  213.8460    9.6695\n",
       "   2.8913   14.3568    7.6488  198.5177    8.3187\n",
       "   3.2177   14.6830    7.4555  204.6153    8.1965\n",
       "   3.3047   14.8176    7.6961  212.8840    9.5835\n",
       "   2.8923   14.7750    7.5932  205.4570    8.8629\n",
       "   3.1564   14.5487    7.4780  206.6520    8.5491\n",
       "   3.0166   15.0243    7.3524  210.3667    8.9454\n",
       "   3.2426   14.5079    7.6490  207.6126    8.5079\n",
       "   3.3398   14.5313    7.7125  208.2065    8.3498\n",
       "   3.1131   14.6619    7.4911  207.7990    8.6078\n",
       "[torch.FloatTensor of size 10x5]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = Variable(images)\n",
    "outputs = net(inputs)\n",
    "outputs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}