weight_initialization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Weight Initialization\n",
    "======"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# install pytorch (http://pytorch.org/) if run from Google Colaboratory\n",
    "import sys\n",
    "if 'google.colab' in sys.modules and 'torch' not in sys.modules:\n",
    "    from os.path import exists\n",
    "    from wheel.pep425tags import get_abbr_impl, get_impl_ver, get_abi_tag\n",
    "    platform = '{}{}-{}'.format(get_abbr_impl(), get_impl_ver(), get_abi_tag())\n",
    "    cuda_output = !ldconfig -p|grep cudart.so|sed -e 's/.*\\.\\([0-9]*\\)\\.\\([0-9]*\\)$/cu\\1\\2/'\n",
    "    accelerator = cuda_output[0] if exists('/dev/nvidia0') else 'cpu'\n",
    "\n",
    "    !pip3 install https://download.pytorch.org/whl/{accelerator}/torch-1.1.0-{platform}-linux_x86_64.whl\n",
    "    !pip3 install https://download.pytorch.org/whl/{accelerator}/torchvision-0.3.0-{platform}-linux_x86_64.whl\n",
    "\n",
    "%matplotlib inline\n",
    "import os\n",
    "import math\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parameter Settings\n",
    "-------------------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 20\n",
    "batch_size = 128\n",
    "learning_rate = 1e-3\n",
    "use_gpu = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MNIST Data Loading\n",
    "-------------------\n",
    "\n",
    "MNIST images show digits from 0-9 in 28x28 grayscale images. We normalize and center them around 0, which gives a slight performance boost during training.\n",
    "We create both a training set and a test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision.datasets import MNIST\n",
    "\n",
    "img_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.5,), (0.5,))\n",
    "])\n",
    "\n",
    "train_dataset = MNIST(root='./data/MNIST', download=True, train=True, transform=img_transform)\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "test_dataset = MNIST(root='./data/MNIST', download=True, train=False, transform=img_transform)\n",
    "test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ConvNet Definition\n",
    "-----------------------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of parameters for each network: 25384\n"
     ]
    }
   ],
   "source": [
    "class ConvNet(nn.Module):\n",
    "    def __init__(self, init_type):\n",
    "        super(ConvNet, self).__init__()\n",
    "        \n",
    "        self.conv1 = nn.Conv2d(1, 8, kernel_size=4, stride=2, padding=1) # out: 8 x 14 x 14\n",
    "        self.conv2 = nn.Conv2d(8, 16, kernel_size=4, stride=2, padding=1) # out: 16 x 7 x 7\n",
    "        self.conv3 = nn.Conv2d(16, 32, kernel_size=4, stride=2, padding=1) # out: 32 x 3 x 3\n",
    "        self.fc1 = nn.Linear(288, 50)\n",
    "        self.fc2 = nn.Linear(50, 10) # 10 outputs: probability for each digit class\n",
    "        \n",
    "        self.init_weights(init_type)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # convolutional part\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = x.view(x.size(0), -1) # flatten batch of multi-channel feature maps to a batch of feature vectors\n",
    "        \n",
    "        # fully connected part\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.log_softmax(self.fc2(x), dim=1) # last activation is log softmax to get log class probabilities\n",
    "        \n",
    "        return x\n",
    "    \n",
    "    def init_weights(self, init_type):\n",
    "\n",
    "        with torch.no_grad():\n",
    "        \n",
    "            # iterate over all parameter tensors\n",
    "            for m in self.modules():\n",
    "\n",
    "                if isinstance(m, nn.Conv2d):\n",
    "                    if init_type == 'zero':\n",
    "\n",
    "                        # initialize to zero\n",
    "                        m.weight.zero_()\n",
    "\n",
    "                    elif init_type == 'normal':\n",
    "\n",
    "                        # initialization with a normal distribution of small variance\n",
    "                        m.weight.normal_(0, 0.01)\n",
    "\n",
    "                    elif init_type == 'kaiming':\n",
    "\n",
    "                        # kaiming initialization for convolutional layers\n",
    "                        # this is equivalent to nn.init.kaiming_normal(m.weight)\n",
    "                        fan_in = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "                        m.weight.normal_(0, math.sqrt(2. / fan_in))\n",
    "\n",
    "                    else:\n",
    "                        raise ValueError('bad initialization type')\n",
    "\n",
    "                    # bias can be initialized to 0\n",
    "                    m.bias.zero_()\n",
    "            \n",
    "                elif isinstance(m, nn.Linear):\n",
    "                    if init_type == 'zero':\n",
    "\n",
    "                        # initialize to zero\n",
    "                        m.weight.zero_()\n",
    "\n",
    "                    elif init_type == 'normal':\n",
    "\n",
    "                        # initialization with a normal distribution of small variance\n",
    "                        m.weight.normal_(0, 0.01)\n",
    "\n",
    "                    elif init_type == 'kaiming':\n",
    "\n",
    "                        # kaiming initialization for convolutional layers\n",
    "                        # this is equivalent to nn.init.kaiming_normal(m.weight)\n",
    "                        fan_in = m.in_features\n",
    "                        m.weight.normal_(0, math.sqrt(2. / fan_in))\n",
    "\n",
    "                    else:\n",
    "                        raise ValueError('bad initialization type')\n",
    "\n",
    "                    # bias can be initialized to 0\n",
    "                    m.bias.zero_()\n",
    "\n",
    "convnet_zero = ConvNet('zero')\n",
    "convnet_normal = ConvNet('normal')\n",
    "convnet_kaiming = ConvNet('kaiming')\n",
    "\n",
    "device = torch.device(\"cuda:0\" if use_gpu and torch.cuda.is_available() else \"cpu\")\n",
    "convnet_zero = convnet_zero.to(device)\n",
    "convnet_normal = convnet_normal.to(device)\n",
    "convnet_kaiming = convnet_kaiming.to(device)\n",
    "\n",
    "num_params = sum(p.numel() for p in convnet_zero.parameters() if p.requires_grad)\n",
    "print('Number of parameters for each network: %d' % num_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train ConvNets\n",
    "--------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training convnet with zero initialization ...\n",
      "Epoch [1 / 20] average loss: 2.301702\n",
      "Epoch [2 / 20] average loss: 2.301292\n",
      "Epoch [3 / 20] average loss: 2.301244\n",
      "Epoch [4 / 20] average loss: 2.301257\n",
      "Epoch [5 / 20] average loss: 2.301271\n",
      "Epoch [6 / 20] average loss: 2.301267\n",
      "Epoch [7 / 20] average loss: 2.301265\n",
      "Epoch [8 / 20] average loss: 2.301281\n",
      "Epoch [9 / 20] average loss: 2.301273\n",
      "Epoch [10 / 20] average loss: 2.301266\n",
      "Epoch [11 / 20] average loss: 2.301249\n",
      "Epoch [12 / 20] average loss: 2.301257\n",
      "Epoch [13 / 20] average loss: 2.301264\n",
      "Epoch [14 / 20] average loss: 2.301273\n",
      "Epoch [15 / 20] average loss: 2.301256\n",
      "Epoch [16 / 20] average loss: 2.301269\n",
      "Epoch [17 / 20] average loss: 2.301272\n",
      "Epoch [18 / 20] average loss: 2.301254\n",
      "Epoch [19 / 20] average loss: 2.301261\n",
      "Epoch [20 / 20] average loss: 2.301265\n",
      "Training convnet with normal initialization ...\n",
      "Epoch [1 / 20] average loss: 0.764194\n",
      "Epoch [2 / 20] average loss: 0.195844\n",
      "Epoch [3 / 20] average loss: 0.127389\n",
      "Epoch [4 / 20] average loss: 0.101366\n",
      "Epoch [5 / 20] average loss: 0.087230\n",
      "Epoch [6 / 20] average loss: 0.075683\n",
      "Epoch [7 / 20] average loss: 0.067001\n",
      "Epoch [8 / 20] average loss: 0.062195\n",
      "Epoch [9 / 20] average loss: 0.057652\n",
      "Epoch [10 / 20] average loss: 0.050892\n",
      "Epoch [11 / 20] average loss: 0.048085\n",
      "Epoch [12 / 20] average loss: 0.044245\n",
      "Epoch [13 / 20] average loss: 0.042099\n",
      "Epoch [14 / 20] average loss: 0.040629\n",
      "Epoch [15 / 20] average loss: 0.037394\n",
      "Epoch [16 / 20] average loss: 0.036119\n",
      "Epoch [17 / 20] average loss: 0.033004\n",
      "Epoch [18 / 20] average loss: 0.031087\n",
      "Epoch [19 / 20] average loss: 0.030324\n",
      "Epoch [20 / 20] average loss: 0.026770\n",
      "Training convnet with kaiming initialization ...\n",
      "Epoch [1 / 20] average loss: 0.406676\n",
      "Epoch [2 / 20] average loss: 0.110997\n",
      "Epoch [3 / 20] average loss: 0.074704\n",
      "Epoch [4 / 20] average loss: 0.058704\n",
      "Epoch [5 / 20] average loss: 0.047906\n",
      "Epoch [6 / 20] average loss: 0.041313\n",
      "Epoch [7 / 20] average loss: 0.036983\n",
      "Epoch [8 / 20] average loss: 0.033277\n",
      "Epoch [9 / 20] average loss: 0.028744\n",
      "Epoch [10 / 20] average loss: 0.024786\n",
      "Epoch [11 / 20] average loss: 0.023274\n",
      "Epoch [12 / 20] average loss: 0.018980\n",
      "Epoch [13 / 20] average loss: 0.016776\n",
      "Epoch [14 / 20] average loss: 0.015717\n",
      "Epoch [15 / 20] average loss: 0.014709\n",
      "Epoch [16 / 20] average loss: 0.013657\n",
      "Epoch [17 / 20] average loss: 0.010374\n",
      "Epoch [18 / 20] average loss: 0.011438\n",
      "Epoch [19 / 20] average loss: 0.009466\n",
      "Epoch [20 / 20] average loss: 0.009854\n"
     ]
    }
   ],
   "source": [
    "def train_convnet(convnet):\n",
    "    optimizer = torch.optim.Adam(params=convnet.parameters(), lr=learning_rate)\n",
    "\n",
    "    # set to training mode\n",
    "    convnet.train()\n",
    "\n",
    "    train_loss_avg = []\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        train_loss_avg.append(0)\n",
    "        num_batches = 0\n",
    "\n",
    "        for image_batch, label_batch in train_dataloader:\n",
    "\n",
    "            image_batch = image_batch.to(device)\n",
    "            label_batch = label_batch.to(device)\n",
    "\n",
    "            # class predictions\n",
    "            prediction_batch = convnet(image_batch)\n",
    "\n",
    "            # The cross-entropy loss.\n",
    "            # The first input are the predicted log class probabilities.\n",
    "            # The ground truth probabilites for each image are expected to be\n",
    "            # 1 for a single class and 0 for all other classes.\n",
    "            # This function expects as second input the index of the class with probability 1.\n",
    "            # (this function is not called cross-entropy, since this function assumes\n",
    "            # that the inputs are log probabilities, not probabilities).\n",
    "            loss = F.nll_loss(prediction_batch, label_batch)\n",
    "\n",
    "            # backpropagation\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "\n",
    "            # one step of the optmizer (using the gradients from backpropagation)\n",
    "            optimizer.step()\n",
    "\n",
    "            train_loss_avg[-1] += loss.item()\n",
    "            num_batches += 1\n",
    "\n",
    "        train_loss_avg[-1] /= num_batches\n",
    "        print('Epoch [%d / %d] average loss: %f' % (epoch+1, num_epochs, train_loss_avg[-1]))\n",
    "    \n",
    "    return train_loss_avg\n",
    "        \n",
    "print('Training convnet with zero initialization ...')\n",
    "train_loss_zero = train_convnet(convnet_zero)\n",
    "print('Training convnet with normal initialization ...')\n",
    "train_loss_normal = train_convnet(convnet_normal)\n",
    "print('Training convnet with kaiming initialization ...')\n",
    "train_loss_kaiming = train_convnet(convnet_kaiming)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternatively: Load Pre-Trained Model and Loss Histories\n",
    "-----------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "convnet_zero.load_state_dict(torch.load('./pretrained/convnet_init_zero.pth'))\n",
    "convnet_normal.load_state_dict(torch.load('./pretrained/convnet_init_normal.pth'))\n",
    "convnet_kaiming.load_state_dict(torch.load('./pretrained/convnet_init_kaiming.pth'))\n",
    "train_loss_zero = torch.load('./pretrained/convnet_init_zero_losshist.pth')\n",
    "train_loss_normal = torch.load('./pretrained/convnet_init_normal_losshist.pth')\n",
    "train_loss_kaiming = torch.load('./pretrained/convnet_init_kaiming_losshist.pth')\n",
    "\n",
    "import urllib\n",
    "if not os.path.isdir('./pretrained'):\n",
    "    os.makedirs('./pretrained')\n",
    "print('downloading ...')\n",
    "urllib.request.urlretrieve (\"http://geometry.cs.ucl.ac.uk/creativeai/pretrained/convnet_init_zero.pth\", \"./pretrained/convnet_init_zero.pth\")\n",
    "urllib.request.urlretrieve (\"http://geometry.cs.ucl.ac.uk/creativeai/pretrained/convnet_init_normal.pth\", \"./pretrained/convnet_init_normal.pth\")\n",
    "urllib.request.urlretrieve (\"http://geometry.cs.ucl.ac.uk/creativeai/pretrained/convnet_init_kaiming.pth\", \"./pretrained/convnet_init_kaiming.pth\")\n",
    "urllib.request.urlretrieve (\"http://geometry.cs.ucl.ac.uk/creativeai/pretrained/convnet_init_zero_losshist.pth\", \"./pretrained/convnet_init_zero_losshist.pth\")\n",
    "urllib.request.urlretrieve (\"http://geometry.cs.ucl.ac.uk/creativeai/pretrained/convnet_init_normal_losshist.pth\", \"./pretrained/convnet_init_normal_losshist.pth\")\n",
    "urllib.request.urlretrieve (\"http://geometry.cs.ucl.ac.uk/creativeai/pretrained/convnet_init_kaiming_losshist.pth\", \"./pretrained/convnet_init_kaiming_losshist.pth\")\n",
    "convnet_zero.load_state_dict(torch.load('./pretrained/convnet_init_zero.pth'))\n",
    "convnet_normal.load_state_dict(torch.load('./pretrained/convnet_init_normal.pth'))\n",
    "convnet_kaiming.load_state_dict(torch.load('./pretrained/convnet_init_kaiming.pth'))\n",
    "train_loss_zero = torch.load('./pretrained/convnet_init_zero_losshist.pth')\n",
    "train_loss_normal = torch.load('./pretrained/convnet_init_normal_losshist.pth')\n",
    "train_loss_kaiming = torch.load('./pretrained/convnet_init_kaiming_losshist.pth')\n",
    "print('done')\n",
    "\n",
    "# this is how the model parameters and loss histories can be saved:\n",
    "# torch.save(convnet_zero.state_dict(), './pretrained/my_convnet_init_zero.pth')\n",
    "# torch.save(convnet_normal.state_dict(), './pretrained/my_convnet_init_normal.pth')\n",
    "# torch.save(convnet_kaiming.state_dict(), './pretrained/my_convnet_init_kaiming.pth')\n",
    "# torch.save(train_loss_zero, './pretrained/my_convnet_init_zero_losshist.pth')\n",
    "# torch.save(train_loss_normal, './pretrained/my_convnet_init_normal_losshist.pth')\n",
    "# torch.save(train_loss_kaiming, './pretrained/my_convnet_init_kaiming_losshist.pth')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot Training Curve\n",
    "--------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3sAAAFACAYAAAAF72WkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcXGWd7/HPr7o76WyEhES2AAmIiiHIEtnVgBsoy6ggYbxDRB0uDAiM4gyOCsLAfQnjgAJeuaiETRY39sXgMiAIQhIhJOxg0EiALCQs2Xp57h9V3amuVHVXJ1Vd3dWfN5xXneV5zvlVnVR3f+ssFSklJEmSJEn1JVPrAiRJkiRJlWfYkyRJkqQ6ZNiTJEmSpDpk2JMkSZKkOmTYkyRJkqQ6ZNiTJEmSpDpk2JMkSZKkOmTYkyRJkqQ6ZNiTJEmSpDrUWOsCemvcuHFp4sSJtS5DkiRJkmpizpw5S1NK43tqN+DC3sSJE5k9e3aty5AkSZKkmoiIl8pp52mckiRJklSHDHuSJEmSVIcMe5IkSZJUhwx7kiRJklSHDHuSJEmSVIcMe5IkSZJUhwx7kiRJklSHDHuSJEmSVIcMe5IkSZJUhxprXUA9eGLRSua/vLLLvMgfDwqWbTCjm77RXdMu6+5pO4XLJUmSJJU2tLGBQ3bdqtZlbDTDXgXc+9SrXPLb52pdhiRJkqQKGjdyqGFvsPvnD0ziH/fevnM6kdaPp65tCyZJBQ0K23e3bFO209/07+o0GHjge+P5/pUk1auGAX5qnGGvAkY1NzGquanWZUiSJElSJ2/QIkmSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdahqYS8itouI30fEUxGxICJOK9ImIuKSiHg+IuZFxJ7VqkeSJEmSBpPGKq67FfhqSmluRIwC5kTEvSmlJ/PaHArsnBv2AX6Ye5QkSZIkbYKqHdlLKS1OKc3Njb8JPAVsW9DsSOCalPUwsHlEbF2tmiRJkiRpsOiTa/YiYiKwB/CngkXbAn/Lm17EhoGQiDghImZHxOwlS5ZUq0xJkiRJqhtVD3sRMRL4JXB6SumNwsVFuqQNZqR0RUppakpp6vjx46tRpiRJkiTVlaqGvYhoIhv0fppS+lWRJouA7fKmJwAvV7MmSZIkSRoMqnk3zgB+AjyVUrqoRLPbgONyd+XcF1iZUlpcrZokSZIkabCo5t04DwD+CXgiIh7LzfsPYHuAlNLlwF3AJ4DngVXA8VWsR5IkSZIGjaqFvZTSAxS/Ji+/TQJOrlYNkiRJkjRY9cndOCVJkiRJfcuwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0mSJEl1qMewFxFHR8So3Pg3I+JXEbFn9UuTJEmSJG2sco7sfSul9GZEHAh8HLga+GF1y5IkSZIkbYpywl5b7vGTwA9TSrcCQ6pXkiRJkiRpU5UT9v4eEf8P+CxwV0QMLadfRFwZEa9FxPwSy6dFxMqIeCw3nNW70iVJkiRJpZQT9j4L/Bo4JKW0AhgLfK2MflcBh/TQ5g8ppd1zw7llrFOSJEmSVIbGMtpsDdyZUlobEdOA3YBreuqUUro/IiZuUnWSJEmSpI1SzpG9XwJtEfFO4CfAJOD6Cm1/v4h4PCLujojJFVqnJEmSJA165YS99pRSK/Bp4HsppX8le7RvU80FdkgpvQ+4FLilVMOIOCEiZkfE7CVLllRg05IkSZJU38oJey0RcSxwHHBHbl7Tpm44pfRGSumt3PhdQFNEjCvR9oqU0tSU0tTx48dv6qYlSZIkqe6VE/aOB/YDzk8p/SUiJgHXbeqGI2KriIjc+N65WpZt6nolSZIkSWXcoCWl9GREnAG8KyJ2BZ5JKX2np34RcQMwDRgXEYuAs8kdEUwpXQ4cBZwUEa3AamB6Silt9DORJEmSJHXqMezl7sB5NbAQCGC7iJiRUrq/u34ppWN7WH4ZcFnZlUqSJEmSylbOVy/8N/CxlNIzABHxLuAGYK9qFiZJkiRJ2njlXLPX1BH0AFJKz1KBG7RIkiRJkqqnnCN7syPiJ8C1uenPAXOqV5IkSZIkaVOVE/ZOAk4GTiV7zd79wP+tZlGSJEmSpE1Tzt041wIX5QZJkiRJ0gBQMuxFxBNAya9CSCntVpWKJEmSJEmbrLsje4f1WRWSJEmSpIoqGfZSSi/1ZSGSJEmSpMop5wYtkiRJklQRLS0tLFq0iDVr1tS6lH6vubmZCRMm0NS0cd98Z9iTJEmS1GcWLVrEqFGjmDhxIhFR63L6rZQSy5YtY9GiRUyaNGmj1tHjl6pHxGERUc6Xr0uSJElSt9asWcMWW2xh0OtBRLDFFlts0hHQckLcdOC5iLgwInbZ6C1JkiRJEhj0yrSpr1OPYS+l9L+APYAXgJkR8VBEnBARozZpy5IkSZKkqinr9MyU0hvAL4Ebga2BTwFzI+LLVaxNkiRJkrSRyrlm7/CIuBn4HdAE7J1SOhR4H3BGleuTJEmSpH6hra2t1iX0Sjl34zwauDildH/+zJTSqoj4QnXKkiRJklTvzrl9AU++/EZF1/nebTbj7MMnd9vm8ssv5/LLLwdg5cqVTJw4ka9//eucffbZrF27lp122omZM2cycuRIJk6cyBe+8AVmzZrFKaecwnve8x5OPPFEVq1axU477cSVV17JmDFjKvocKqWca/aOA56NiCNyR/m2ylv226pWJ0mSJEkVduKJJ/LYY4/x6KOPMmHCBL7whS9w3nnn8Zvf/Ia5c+cydepULrroos72zc3NPPDAA0yfPp3jjjuOCy64gHnz5jFlyhTOOeecGj6T7vV4ZC8ivgicTfY0zgAujYhzU0pXVrs4SZIkSfWrpyNw1Xbaaadx8MEHM2bMGJ588kkOOOAAANatW8d+++3X2e6YY44BskcBV6xYwYc+9CEAZsyYwdFHH933hZepnNM4/w3YI6W0DCAitgD+CBj2JEmSJA1IV111FS+99BKXXXYZd955Jx/96Ee54YYbirYdMWJEH1dXGeXcjXMR8Gbe9JvA36pTjiRJkiRV15w5c/jud7/LddddRyaTYd999+XBBx/k+eefB2DVqlU8++yzG/QbPXo0Y8aM4Q9/+AMA1157bedRvv6onCN7fwf+FBG3Agk4EngkIr4CkFK6qLvOkiRJktSfXHbZZSxfvpyDDjoIgKlTp3LVVVdx7LHHsnbtWgDOO+883vWud23Q9+qrr+68QcuOO+7IzJkz+7T23ign7L2QGzrcmnv0S9UlSZIkDTilAtqjjz66wbyFCxd2md599915+OGHq1FWxfUY9lJK5wBExKjsZHqr6lVJkiRJkjZJOV+qvmtE/BmYDyyIiDkRUdvb5kiSJEmSulXODVquAL6SUtohpbQD8FXgR9UtS5IkSZK0KcoJeyNSSr/vmEgp/Q8wMO89KkmSJEmDRDk3aHkxIr4FXJub/l/AX6pXkiRJkiRpU5VzZO8LwHjgV7lhHHB8NYuSJEmSJG2absNeRDQA/5FSOjWltGduOD2l9Hof1SdJkiRJdWXatGnMnj276tvpNuyllNqAvapehSRJkiQNAK2trbUuoWzlXLP354i4Dfg58HbHzJTSr6pWlSRJkqT6d/eZ8MoTlV3nVlPg0O9022ThwoUceuihHHjggfzxj39k22235dZbb+WZZ57hxBNPZNWqVey0005ceeWVjBkzhmnTprH//vvz4IMPcsQRR/DEE08wbNgwnn76aV566SVmzpzJ1VdfzUMPPcQ+++zDVVddBcBJJ53Eo48+yurVqznqqKM455xzKvtce1DONXtjgWXAwcDhueGwahYlSZIkSdX03HPPcfLJJ7NgwQI233xzfvnLX3LcccdxwQUXMG/ePKZMmdIlnK1YsYL77ruPr371qwC8/vrr/O53v+Piiy/m8MMP51//9V9ZsGABTzzxBI899hgA559/PrNnz2bevHncd999zJs3r0+fYzlH9n6cUnowf0ZEHFCleiRJkiQNFj0cgaumSZMmsfvuuwOw11578cILL7BixQo+9KEPATBjxgyOPvrozvbHHHNMl/6HH344EcGUKVPYcsstmTJlCgCTJ09m4cKF7L777vzsZz/jiiuuoLW1lcWLF/Pkk0+y22679dEzLO/I3qVlzpMkSZKkAWHo0KGd4w0NDaxYsaLb9iNGdP2q8Y7+mUymy7oymQytra385S9/4bvf/S6//e1vmTdvHp/85CdZs2ZNBZ9Bz0oe2YuI/YD9gfER8ZW8RZsBDdUuTJIkSZL6yujRoxkzZgx/+MMf+MAHPsC1117beZRvY7zxxhuMGDGC0aNH8+qrr3L33Xczbdq0yhVchu5O4xwCjMy1GZU3/w3gqGoWJUmSJEl97eqrr+68QcuOO+7IzJkzN3pd73vf+9hjjz2YPHkyO+64Iwcc0PdXwkVKqfsGETuklF7qo3p6NHXq1NQX30khSZIkqfKeeuopdtlll1qXMWAUe70iYk5KaWpPfcu5QcvQiLgCmJjfPqV0cC/rlCRJkiT1kXLC3s+By4EfA23VLUeSJEmSVAnlhL3WlNIPq16JJEmSJKliyvnqhdsj4l8iYuuIGNsxVL0ySZIkSdJGK+fI3ozc49fy5iVgx8qXI0mSJEmqhB7DXkppUl8UIkmSJEmqnB5P44yI4RHxzdwdOYmInSPisOqXJkmSJEmVt3DhQnbdddey2r788sscdVTvvmb8rLPO4je/+c3GlFZR5ZzGOROYA+yfm15E9g6dd1SrKEmSJEnqD7bZZht+8Ytf9KrPueeeW6VqeqecsLdTSumYiDgWIKW0OiKip04RcSVwGPBaSmmD2Jxbx/eBTwCrgM+nlOb2qnpJkiRJA9YFj1zA08ufrug63zP2Pfz73v9edvsXX3yRz3zmM1xxxRV85Stf4e233wbgsssuY//992fhwoUcdthhzJ8/n6uuuopbbrmFtrY25s+fz1e/+lXWrVvHtddey9ChQ7nrrrsYO3Ysn//85znssMM46qijmDhxIjNmzOD222+npaWFn//857znPe9hyZIl/OM//iPLli3j/e9/P/fccw9z5sxh3LhxFXstyrkb57qIGEb2pixExE7A2jL6XQUc0s3yQ4Gdc8MJgF/vIEmSJKnPPPPMM3zmM59h5syZTJ48mXvvvZe5c+dy0003ceqppxbtM3/+fK6//noeeeQRvvGNbzB8+HD+/Oc/s99++3HNNdcU7TNu3Djmzp3LSSedxHe/+10AzjnnHA4++GDmzp3Lpz71Kf76179W/PmVc2TvbOAeYLuI+ClwAPD5njqllO6PiIndNDkSuCallICHI2LziNg6pbS4jJokSZIkDXC9OQJXaUuWLOHII4/kl7/8JZMnT2blypWccsopPPbYYzQ0NPDss88W7XfQQQcxatQoRo0axejRozn88MMBmDJlCvPmzSva59Of/jQAe+21F7/61a8AeOCBB7j55psBOOSQQxgzZkyln2JZd+O8NyLmAvsCAZyWUlpagW1vC/wtb3pRbp5hT5IkSVJVjR49mu22244HH3yQyZMnc/HFF7Plllvy+OOP097eTnNzc9F+Q4cO7RzPZDKd05lMhtbW1m77NDQ0dLbJHvOqrnJO4ySltCyldCcwtUJBD7LBcYNNFW0YcUJEzI6I2UuWLKnQ5iVJkiQNVkOGDOGWW27hmmuu4frrr2flypVsvfXWZDIZrr32Wtra2qq6/QMPPJCf/exnAMyaNYvXX3+94tsoK+zlOaKC214EbJc3PQF4uVjDlNIVKaWpKaWp48ePr2AJkiRJkgarESNGcMcdd3DxxRczceJErr76avbdd1+effZZRowYUdVtn3322cyaNYs999yTu+++m6233ppRo0ZVdBvRm8OHEfHnlNIevWg/EbijxN04PwmcQvZunPsAl6SU9u5pnVOnTk2zZ88uu2ZJkiRJ/cdTTz3FLrvsUusyam7t2rU0NDTQ2NjIQw89xEknncRjjz22Qbtir1dEzEkpTe1pG+XcoCXfXuU2jIgbgGnAuIhYRPZGL00AKaXLgbvIBr3nyX71wvG9rEWSJEmSBqS//vWvfPazn6W9vZ0hQ4bwox/9qOLb6DHsRcSFwHnAauCeiHgfcHpK6bru+qWUju1heQJO7kWtkiRJklQXdt55Z/785z9XdRvlXLP3sZTSG2S/IH0R8C7ga1WtSpIkSVLd6os7UdaDTX2dygl7TbnHTwA3pJSWb9IWJUmSJA1azc3NLFu2zMDXg5QSy5YtK/kVEOUo55q92yPiabKncf5LRIwH1mz0FiVJkiQNWhMmTGDRokX4lWo9a25uZsKECRvdv5wvVT8zIi4A3kgptUXE28CRG71FSZIkSYNWU1MTkyZNqnUZg0KPp3FGxNFAay7ofRO4Dtim6pVJkiRJkjZaOdfsfSul9GZEHAh8HLga+GF1y5IkSZIkbYpywl5b7vGTwA9TSrcCQ6pXkiRJkiRpU5UT9v4eEf8P+CxwV0QMLbOfJEmSJKlGygltnwV+DRySUloBjMXv2ZMkSZKkfq3HsJdSWgW8AHw8Ik4B3pFSmlX1yiRJkiRJG62cu3GeBvwUeEduuC4ivlztwiRJkiRJG6+cL1X/IrBPSultgNx37j0EXFrNwiRJkiRJG6+ca/aC9XfkJDce1SlHkiRJklQJ5RzZmwn8KSJuzk3/A/CT6pUkSZIkSdpUPYa9lNJFEfE/wIFkj+gdn1L6c7ULkyRJkiRtvG7DXkRkgHkppV2BuX1TkiRJkiRpU3V7zV5KqR14PCK276N6JEmSJEkVUM41e1sDCyLiEeDtjpkppSOqVpUkSZIkaZOUE/bOqXoVkiRJkqSKKhn2IuKdwJYppfsK5n8Q+Hu1CxtQnr4LnvgZfOZKyJTzbRaSJEmSVF3dJZPvAW8Wmb8qt0wdVi2FBTfDnJm1rkSSJEmSgO7D3sSU0rzCmSml2cDEqlU0EO3xT7DjNLj3LFjxt1pXI0mSJEndhr3mbpYNq3QhA1oEHH4JpAR3nJ59lCRJkqQa6i7sPRoR/1w4MyK+CMypXkkD1Jgd4CNnw/O/gcdvrHU1kiRJkga57u7GeTpwc0R8jvXhbiowBPhUtQsbkN7/zzD/V3DPmbDTwTBqy1pXJEmSJGmQKnlkL6X0akppf7JfvbAwN5yTUtovpfRK35Q3wGQycORl0LIa7vpqrauRJEmSNIj1+D17KaXfA7/vg1rqw7id4aCvw2++DQtugcn/UOuKJEmSJA1CfilcNez3Zdh6d7jrDFi1vNbVSJIkSRqEDHvV0NAIR/4AVr+evX5PkiRJkvqYYa9attoVPvBVmHcTPPvrWlcjSZIkaZAx7FXTB86Ad7wXbj8d1qysdTWSJEmSBhHDXjU1DsnenfOtV+Des2pdjSRJkqRBxLBXbdvuBfudDHOughfvq3U1kiRJkgYJw15fOOgbMHYnuO3LsO7tWlcjSZIkaRAw7PWFpmFwxKWw4iX43Xm1rkaSJEnSIGDY6ysTD4D3fwke/iH89U+1rkaSJElSnTPs9aWPfBtGT4DbToGWNbWuRpIkSVIdM+z1paGj4PDvw9Jn4f4La12NJEmSpDpm2Otr7/ww7P45eOB78PJjta5GkiRJUp0y7NXCx8+HEeOyp3O2tdS6GkmSJEl1yLBXC8PGwCcvgleegAe/V+tqJEmSJNUhw16t7HIYTP4U3HchvPZ0rauRJEmSVGcMe7V06H/BkJFw68nQ3lbraiRJkiTVEcNeLY0cD4deCH+fDX+6vNbVSJIkSaojhr1am3IUvOsQ+O1/wrIXal2NJEmSpDpR1bAXEYdExDMR8XxEnFlk+ecjYklEPJYbvlTNevqlCDjsYmhogttPg/b2WlckSZIkqQ5ULexFRAPwA+BQ4L3AsRHx3iJNb0op7Z4bflytevq1zbaBj50HC/8Ac6+qdTWSJEmS6kA1j+ztDTyfUnoxpbQOuBE4sorbG9j2PA4mfQhmnQUrF9W6GkmSJEkDXDXD3rbA3/KmF+XmFfpMRMyLiF9ExHZVrKd/i4AjLoHUBrefDinVuiJJkiRJA1g1w14UmVeYYG4HJqaUdgN+A1xddEURJ0TE7IiYvWTJkgqX2Y+MmQgfPhuevxcev7HW1UiSJEkawKoZ9hYB+UfqJgAv5zdIKS1LKa3NTf4I2KvYilJKV6SUpqaUpo4fP74qxfYbe58A2+0D95wJb75a62okSZIkDVDVDHuPAjtHxKSIGAJMB27LbxARW+dNHgE8VcV6BoZMBo64DFpWw11n1LoaSZIkSQNU1cJeSqkVOAX4NdkQ97OU0oKIODcijsg1OzUiFkTE48CpwOerVc+AMv5dMO1MeOo2WHBLrauRJEmSNABFGmA3Apk6dWqaPXt2rcuovrZW+PHB8MbLcPIjMHxsrSuSJEmS1A9ExJyU0tSe2lX1S9W1CRoa4cgfwOrX4Z6v17oaSZIkSQOMYa8/22oKHPgVmHcjPDur1tVIkiRJGkAMe/3dB8+A8e+BO06HNW/UuhpJkiRJA4RhrwLWtq3lxRUvVmfljUOzp3O+uRjuPas625AkSZJUdwx7FXDp3EuZfud07ll4T3U2MGEq7PsvMGcm/OX+6mxDkiRJUl0x7FXAjMkzePeYd/O1+77GRXMuorW9tfIbOegbMHZHuO3LsO7tyq9fkiRJUl0x7FXA+OHjufLjV3LMu49h5vyZnPSbk1ixZkVlNzJkOBxxKby+EH53fmXXLUmSJKnuGPYqpKmhiW/u+03O2f8c5rw6h+l3Tufp5U9XdiMTD4SpX4SH/y/87ZHKrluSJElSXTHsVdind/40Vx9yNS3tLfzTXf/EHS/eUdkNfOTbsNm2cOsp0Lq2suuWJEmSVDcMe1UwZfwUbjrsJiaPm8zX//B1Lnjkgspdx9e8GRz+fVj6DNx3YWXWKUmSJKnuGPaqZNywcfzoYz/ic7t8juueuo4T7j2BZauXVWblO38E3veP8MDFsPjxyqxTkiRJUl0x7FVRU6aJM/c+k/9z4P9h3pJ5TL9zOguWLqjMyj9+PgzfAm49GdpaKrNOSZIkSXXDsNcHDt/pcK459BqC4Li7j+OW52/Z9JUOHwuHXQSvPAEPfn/T1ydJkiSprhj2+sh7t3gvNx12E3u8Yw++9eC3OP/h82nZ1CNyuxwO7/0HuO8CeK3Cd/6UJEmSNKAZ9vrQmOYxXP7Ry5nx3hnc+MyNfGnWl1i6eummrfQT/wVDRsBtp0B7W2UKlSRJkjTgGfb6WGOmkTPefwYXfvBCnlz2JMfcfgyPL9mEm6yMfAcceiEsehT+dHnlCpUkSZI0oBn2auTQSYdy3SeuY0jDEI6/53h+8ewvNn5lU46GnT8Ov/1PWP5i5YqUJEmSNGAZ9mro3WPfzY2H3cjeW+3NOQ+dwzkPncO6tnW9X1EEHHYxNDTBbadCe3vli5UkSZI0oBj2amz00NH84MM/4EtTvsQvnv0Fx//6eF5b9dpGrGhb+Nh/wsI/wNyrKl6nJEmSpIHFsNcPNGQaOG3P07ho2kU89/pzfPb2zzL31bm9X9GeM2DSB2HWWbByUeULlSRJkjRgGPb6kY/u8FGu/8T1jBwyki/++ovc+PSNpJTKX0EEHH4JpDa441+hN30lSZIk1RXDXj/zzjHv5PpPXs/+2+7P+X86n7P+eBZr29aWv4Kxk+DDZ8Fzs2DeTdUrVJIkSVK/ZtjrhzYbshmXHnwpJ77vRG55/hZm3D2DV95+pfwV7H0CTNgb7jkT3tqI6/8kSZIkDXiGvX4qExlO3v1kLjnoEha+sZBj7jiGR195tMzODXDkZbDubbjrjOoWKkmSJKlfMuz1cwdtfxA3fPIGRg8dzT/P+meue/K68q7jG/9umHYmPHlrdpAkSZI0qBj2BoBJoydx/Seu50MTPsQFj17A1x/4OqtbV/fccf9TYavd4M4zYNXy6hcqSZIkqd8w7A0QI4eM5OKDLubLe3yZu168i+PuPo6/v/X37js1NMGRP4DVy+H2U2HxPGjtxc1eJEmSJA1Y0atb+/cDU6dOTbNnz651GTV1/6L7OfP+M2nINHDhBy9kv232677DfRfC78/PjmcaYdy7YatdYctdc49TYOT46hcuSZIkaZNFxJyU0tQe2xn2Bqa/vvFXTvv9aby48kVO3/N0Pj/580RE6Q5Ln4NX5sEr8+GVJ+DV+fDm4vXLR26VFwCnZIexO0FDY/WfjCRJkqSyGfYGgVUtq/jWg99i1kuz+PjEj3Pu/ucyvGl4+St4exm8+kTXALjkGWhvyS5vbIZ37LI+AHYcCWweXZ0nJEmSJKlHhr1BIqXEVQuu4ntzv8dOm+/E96d9n+02227jV9i6DpY+kw2Ar85ffzRwdd4NXjbfPnvqZ/6poJtPhIyXgEqSJEnVZtgbZP748h/5t/v/jfbUzoUfvJADtz2wcitPKXvK5yvzux4JXP4CpPZsmyGjYMvJXU8Ffcd7YUgvjjRKkiRJ6pEpDudOAAAUWElEQVRhbxBa9OYiTv/96Tz7+rN8eY8v86UpX+r+Or5NtW4VvPbU+gD46vzs47o3cw0Cttgp7xTQ3ONm20A165IkSZLqmGFvkFrduppzHjqHO1+8k49s/xHOO/A8RjSN6LsC2tthxUvrg9+ruaOAK15a32bYmFz42239kcDx74bGoX1XpyRJkjRAGfYGsZQS1z11Hf89+7/ZYbMd+P5B32fi6Im1LWrNSnh1QddTQV97ElrXrG8zdDQMG50Ng+UOzZtDU3PtnpckSZLUxwx74pHFj3DGfWfQ0t7CN/f9JruN241xw8cxrHFYrUvLam+DZS9kbwKz7AVY/XrpIbWVXk/T8Gzo6wyBm5cXFIeM8HRSSZIkDTiGPQGw+K3FnP4/p/Pksic7541sGsm4YeMYP3x89nHYeMYPG8+44V3HRzWNqu41f+VKCda+2X0YXL0C1qzoOm/VcmhbW3q9maYyg+HmuSOIw6FpWN4wHBqa+u51kCRJkjDsKc+6tnXMeXUOr616jSWrl7B09VKWrMo95qZXt67eoN/QhqHrw2BeMOwIih3jY5rHkIl++rULLat7CIlFQuPq12HdW+WtP9MIjQUBsKl5fTBszBsvXNY0LK9vkSCZ379xqEchJUmSBJQf9hr7ohjV1pCGIey3zX4ll6eUeLvl7S5BsHN89RKWrlrKCyte4OHFD/Nm550212uMRsYOG7vBEcLCoLjFsC1oyvTxkbCO8LTZNr3r17oue51hRwhcswJaVmXDY5dhVfa6w2LLVi0vvqzjS+t7JXoIi7lg2NicDYYlH7tbVuSxYajfnyhJkjRAGfZERDByyEhGDhnJpNGTum27pnUNS1cv7QyC+UcIl6xewuK3FzNv6TxeX/M6ia5HjYNgTPOYDY4Qjhs2ji2at2B403CGNQ4rOgxtGNq3p5Q2DoGR47NDpbW1rA9+rXmhsWVN3njhsm5C5rq34O0l0Lo2N6xZ/9jdaazlahjSu4C4QWAcsv6xoSn3WM54N8s9yilJktQjw556pbmxmQmjJjBh1IRu27W0t7B89fIuQXDpqq7jz694nmWrl9GaWnvcbiYyNDc0rw+ATRsGwuGN68Nid8GxcGhubO7b01AbmrJD82bV31Z7O7St6xoASz4WmVdO37Vv5oXNIm2qIVNGIOx8LLNtNGRPy81k8sYbcuMNeeMd8zPdtMm163Z+pqBNY26dBX072kqSJPWSYU9V0ZRpYssRW7LliC27bdee2lmxdgXLVy9ndevqDYZVrauyjy2rii5f3bqa19e8vsG89tTeq3rLCYVDGoZkh8wQhjYMpamhqXO8c1lDblmmqXN+/nhhm6qHzEwGMs21+3qKlNYHxraW7HjbuoLx1hLzezPezfKW1cXbtresH29dC/Tn65eja2AsFhY75kVhuCzsV9i2sTL9M43ZoaGpyHgTNDTmjTfl+jSVaN9D38h4dFeSpDIY9lRTmcgwtnksY5vHVmydKSXWta9jdUvx4Fh0aCk+/7VVr3X2bWlrYW3bWta1r6O1veejkeVoyjR1BshSYbFweUebLvMyQ2jMNNKYaaQh00BjNK6fjobO8cbILc+bzm/TkGmgKdPUZboxGrPzMg29D6cR668V7O9Syn4dSHtr9qs+Osfbi8xvK2jTlj2KukHf3PyUm+7Sr61gfntBm9bSfTtq2qB/fpsiteb3b11XRv9i228vsr5uvhqlWnobFCN3NLYjKHaOF5suNS9vIHpu0+vtdITp3HjkhfyOo76d45ky5+evNzbcRibvdSk1v1i9RJHpMIRLUj9j2FPdiQiGNgxlaMNQNmfzqmyjPbWzrm0d69rXZR/b1mWDYG58XXt2uqWtpct4Z5uCfi3tLSX7r2pZxdr2gv55bfpSJjLrw2PkhctcYGzKNG0QHjvadfTt8pgLkEWXFbQpd/lGrSMayGQyZMh0rqNwXZ1DJkOmoYFMNHVpE8T650Nmg/U1RAMR0flYVzqDcks2GLblHouOt2Tbdo63Zo/uVqRvN+tJ7bkhV2vKn5cbSOvbFC7rMqQS493NK9hOPSsMw0WDYQaCHpbnT5fRtmObHe27TG/MIxvRL1O87wahuYEuz6NkuM4bNmgTdP0Qo7BNFGyrsE1BXZT5mtHxUMbrV7H19fBBStHXr+CDlKLL6+xnsVSEYU/aCJnI0NzYTDM1Oj0yJ6VEa3tr59HGjqEttWXHU266vet0fpu29jZaUktnm475Le25eWnD/h3zi22zLbWt71swvz21d31sby8+P7X32Gag2yBEFgTMILoExFKhtSOodheGe70sb32N0dhtqO5YR37Y3aDuwufakKGhsZFMDNmgbamQ3d10sWX9PlDnB8KOI6sp/whr6noktst4e3nz84/Wdobccue3547i5q2ro+YuoTgVTBcs32AZ3ayrh3WXbNuxnLx2G/OY17+9fSPXQ15thcvy91F+7Xn7r7NN4YcEeW1UQd2F7cKQWSxsF0xv8OFDJi/Idtem2IcFvW1feJQ9b3yDcJ0XxotO08PywjBfZtuS05E7K6HgdS7cL13OfMjknbFQ4oyGzrMjiq03SvTpaJNXT8MQGP+ukv+K+ruqhr2IOAT4PtAA/Dil9J2C5UOBa4C9gGXAMSmlhdWsSaonEUFTQxNNg/DL3buEv/aNC4yFbfP7pJSKLuvShkRbe+4xt512SrTNra/Ueouur2B5sefRmlqz283NKwzWbe1tG74eeetrTa0bvmZ5r2fhXXUHkvyw2RH+giAi6Pgv+3/XeR1tM5Epq12X+blHoPt1FrTrOBrc7dHnvNDdXeAtFdxLBvvGjukhRUN6x+tXWHvH9oo9r6LTea9Rhtx0Ge3LXXeHDfZN3nT+fincd0WX5/9byXboMp3/nPpMR1BsLwyJBaG8ZJuCI8/tbfQYYPMDcTmht6POSq2v26Pphc+xyPPrbvkm9897XfNrzd9XXeaXGs+1z98fPfal6/xSH44Uvt55D133V2+nN7Zv4bZLfFjVX4zcEs54ttZVbLSqhb2IaAB+AHwUWAQ8GhG3pZSezGv2ReD1lNI7I2I6cAFwTLVqklQ/8v/QpKG2tdSzckJ1W3tbNuR2hNH88dTeZboj0BYL1cXCcNHQuxHTKSU6/8v94dExnv8IbNC2N+2y/5du257ai7breI1KhfbuAn/RIF9nwX0gKRY212fFDYNix3i3/bMTxfsWhtkSbTa2f34t+bUWPpfulhUL00WXdxOmO06Rzz8dvvO0+dzvg+6WZTJ5yzs/rMieYdClb94HGRsMue0Xm9fl/V7w/geK/tzJX1bYt2O8WN/O8c7w3PXnSWHb/Ney8MOSjufTsd+KLd+gPes/YCm1vo592GV53r7vbJP37zL/OecmcuF3fdBOnR9Y5MZJuSPhiUTH2QrZAJy6fLiRSKk9t678UF2w3ty8lBfuhzY282EGrmoe2dsbeD6l9CJARNwIHAnkh70jgW/nxn8BXBYRkfL/dUqSasZQXT8Kg3WxANldcMz/IzV/uuiyvHDcTm66SN+S02Usy193x/Pr/AM4bzr/+ff4h3QZf2jn/1HeEdqLLi+ynsK++f3ya8jvX7RNwbqLrb8S/dP6Ist6bUs9z5KvUZHXoVjfjn3flto6PxTK/xCj8EyJovM6zq7IW1f+v9+OD63y5w1UhWF5g/2pXhk3bBwf3u9rtS5jo1Uz7G0L/C1vehGwT6k2KaXWiFgJbAEszW8UEScAJwBsv/321apXkqS6FRE0hpfqS+XoCH35p+Z3CZkdR+Rz84sdAS129LPUst6eStyxrVJHiLt7XvkflnQE6sIPVvI/YOl8PWjfYHmx9oXzy9lO4fPPl7+sUE9tN5guchQ7b0bRvg0xsD/prOZP/WJ7pvAjhXLakFK6ArgCYOrUqX4sIUmSpKrpOK2zoc5OaegMh0HdPTcVl+m5yUZbBGyXNz0BeLlUm4hoBEYDy6tYkyRJkiQNCtUMe48CO0fEpIgYAkwHbitocxswIzd+FPA7r9eTJEmSpE1XtdM4c9fgnQL8muxl/VemlBZExLnA7JTSbcBPgGsj4nmyR/SmV6seSZIkSRpMqnqldkrpLuCugnln5Y2vAY6uZg2SJEmSNBhV8zROSZIkSVKNGPYkSZIkqQ4Z9iRJkiSpDhn2JEmSJKkOGfYkSZIkqQ4Z9iRJkiSpDsVA+w7ziFgCvFTrOooYByytdRHq5P7oX9wf/Y/7pH9xf/Qv7o/+xf3Rv7g/+ocdUkrje2o04MJefxURs1NKU2tdh7LcH/2L+6P/cZ/0L+6P/sX90b+4P/oX98fA4mmckiRJklSHDHuSJEmSVIcMe5VzRa0LUBfuj/7F/dH/uE/6F/dH/+L+6F/cH/2L+2MA8Zo9SZIkSapDHtmTJEmSpDpk2JMkSZKkOmTY66WIOCQinomI5yPizCLLh0bETbnlf4qIiX1f5eAQEdtFxO8j4qmIWBARpxVpMy0iVkbEY7nhrFrUOlhExMKIeCL3Ws8usjwi4pLc+2NeROxZizoHg4h4d96/+8ci4o2IOL2gje+PKouIKyPitYiYnzdvbETcGxHP5R7HlOg7I9fmuYiY0XdV168S++O/IuLp3M+kmyNi8xJ9u/35pt4rsT++HRF/z/u59IkSfbv9e0y9V2J/3JS3LxZGxGMl+vr+6Ke8Zq8XIqIBeBb4KLAIeBQ4NqX0ZF6bfwF2SymdGBHTgU+llI6pScF1LiK2BrZOKc2NiFHAHOAfCvbHNOCMlNJhNSpzUImIhcDUlFLRL1vN/dL+MvAJYB/g+ymlffquwsEp97Pr78A+KaWX8uZPw/dHVUXEB4G3gGtSSrvm5l0ILE8pfSf3R+qYlNK/F/QbC8wGpgKJ7M+3vVJKr/fpE6gzJfbHx4DfpZRaI+ICgML9kWu3kG5+vqn3SuyPbwNvpZS+202/Hv8eU+8V2x8Fy/8bWJlSOrfIsoX4/uiXPLLXO3sDz6eUXkwprQNuBI4saHMkcHVu/BfAhyMi+rDGQSOltDilNDc3/ibwFLBtbatSD44k+0skpZQeBjbPhXZV14eBF/KDnvpGSul+YHnB7PzfE1cD/1Ck68eBe1NKy3MB717gkKoVOkgU2x8ppVkppdbc5MPAhD4vbJAq8f4oRzl/j6mXutsfub9lPwvc0KdFaZMZ9npnW+BvedOL2DBcdLbJ/fJYCWzRJ9UNYrnTZfcA/lRk8X4R8XhE3B0Rk/u0sMEnAbMiYk5EnFBkeTnvIVXedEr/gvb90fe2TCkthuyHVsA7irTxvVIbXwDuLrGsp59vqpxTcqfVXlniNGffH33vA8CrKaXnSiz3/dFPGfZ6p9gRusLzYMtpowqKiJHAL4HTU0pvFCyeC+yQUnofcClwS1/XN8gckFLaEzgUODl3Skg+3x99LCKGAEcAPy+y2PdH/+V7pY9FxDeAVuCnJZr09PNNlfFDYCdgd2Ax8N9F2vj+6HvH0v1RPd8f/ZRhr3cWAdvlTU8AXi7VJiIagdFs3CkKKkNENJENej9NKf2qcHlK6Y2U0lu58buApogY18dlDhoppZdzj68BN5M91SZfOe8hVdahwNyU0quFC3x/1MyrHacv5x5fK9LG90ofyt0A5zDgc6nEzQzK+PmmCkgpvZpSaksptQM/ovjr7PujD+X+nv00cFOpNr4/+i/DXu88CuwcEZNyn5ZPB24raHMb0HHXtKPIXvTtp01VkDt//CfAUymli0q02arjmsmI2Jvsv/llfVfl4BERI3I3yiEiRgAfA+YXNLsNOC6y9iV7offiPi51sCn5aazvj5rJ/z0xA7i1SJtfAx+LiDG509g+lpunCouIQ4B/B45IKa0q0aacn2+qgILruD9F8de5nL/HVDkfAZ5OKS0qttD3R//WWOsCBpLcnbpOIfsLtwG4MqW0ICLOBWanlG4jGz6ujYjnyR7Rm167iuveAcA/AU/k3Qr4P4DtAVJKl5MN3CdFRCuwGphu+K6aLYGbc9mhEbg+pXRPRJwInfvjLrJ34nweWAUcX6NaB4WIGE72bnX/O29e/v7w/VFlEXEDMA0YFxGLgLOB7wA/i4gvAn8Fjs61nQqcmFL6UkppeUT8J9k/agHOTSl5lsgmKrE/vg4MBe7N/fx6OHdH7W2AH6eUPkGJn281eAp1pcT+mBYRu5M9LXMhuZ9f+fuj1N9jNXgKdaXY/kgp/YQi1337/hg4/OoFSZIkSapDnsYpSZIkSXXIsCdJkiRJdciwJ0mSJEl1yLAnSZIkSXXIsCdJkiRJdciwJ0kaVCKiLSIeyxvOrOC6J0aE3y8lSeoX/J49SdJgszqltHuti5Akqdo8sidJEhARCyPigoh4JDe8Mzd/h4j4bUTMyz1un5u/ZUTcHBGP54b9c6tqiIgfRcSCiJgVEcNy7U+NiCdz67mxRk9TkjSIGPYkSYPNsILTOI/JW/ZGSmlv4DLge7l5lwHXpJR2A34KXJKbfwlwX0rpfcCewILc/J2BH6SUJgMrgM/k5p8J7JFbz4nVenKSJHWIlFKta5Akqc9ExFsppZFF5i8EDk4pvRgRTcArKaUtImIpsHVKqSU3f3FKaVxELAEmpJTW5q1jInBvSmnn3PS/A00ppfMi4h7gLeAW4JaU0ltVfqqSpEHOI3uSJK2XSoyXalPM2rzxNtZfH/9J4AfAXsCciPC6eUlSVRn2JEla75i8x4dy438EpufGPwc8kBv/LXASQEQ0RMRmpVYaERlgu5TS74F/AzYHNji6KElSJfmpoiRpsBkWEY/lTd+TUur4+oWhEfEnsh+GHpubdypwZUR8DVgCHJ+bfxpwRUR8kewRvJOAxSW22QBcFxGjgQAuTimtqNgzkiSpCK/ZkySJzmv2pqaUlta6FkmSKsHTOCVJkiSpDnlkT5IkSZLqkEf2JEmSJKkOGfYkSZIkqQ4Z9iRJkiSpDhn2JEmSJKkOGfYkSZIkqQ79f8TSJ8HI45FyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.ion()\n",
    "\n",
    "fig = plt.figure(figsize=(15, 5))\n",
    "plt.plot(train_loss_zero, label='zero')\n",
    "plt.plot(train_loss_normal, label='normal')\n",
    "plt.plot(train_loss_kaiming, label='kaiming')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Cross-entropy loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}