simple_neural_network_fashion_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import required packages and functions and set the session seed\n",
    "import numpy as np\n",
    "np.random.seed(1234)\n",
    "from tensorflow import set_random_seed\n",
    "set_random_seed(1234)\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation, Flatten, Conv2D, MaxPooling2D\n",
    "from keras.layers import Dropout, SpatialDropout2D\n",
    "from keras.applications import VGG19\n",
    "from keras.applications.vgg19 import preprocess_input\n",
    "from keras.models import Model\n",
    "from keras.datasets import fashion_mnist\n",
    "from keras.utils import to_categorical\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import optimizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the Fashion MNIST data from Keras\n",
    "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize the image data by dividing through the maximum pixel value (=255)\n",
    "train_images = train_images / train_images.max()\n",
    "test_images = test_images / test_images.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "# Build a simple three-layer (1 hidden layer) model\n",
    "# The input size is 28 x 28 pixels and is flattened to a vector of length 784\n",
    "# The activation function is RELU (rectified linear unit) and performs the \n",
    "# multiplication of input and weights (plus bias)\n",
    "# The output (softmax) layer returns probabilities for all ten classes\n",
    "three_layer_model = Sequential()\n",
    "three_layer_model.add(Flatten(input_shape = (28, 28)))\n",
    "three_layer_model.add(Dense(128, activation = 'relu'))\n",
    "three_layer_model.add(Dense(10, activation = 'softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Train on 42000 samples, validate on 18000 samples\n",
      "Epoch 1/10\n",
      " - 2s - loss: 0.5272 - acc: 0.8147 - val_loss: 0.4240 - val_acc: 0.8543\n",
      "Epoch 2/10\n",
      " - 3s - loss: 0.3953 - acc: 0.8574 - val_loss: 0.3958 - val_acc: 0.8587\n",
      "Epoch 3/10\n",
      " - 2s - loss: 0.3549 - acc: 0.8712 - val_loss: 0.3641 - val_acc: 0.8684\n",
      "Epoch 4/10\n",
      " - 2s - loss: 0.3300 - acc: 0.8783 - val_loss: 0.3533 - val_acc: 0.8698\n",
      "Epoch 5/10\n",
      " - 2s - loss: 0.3076 - acc: 0.8864 - val_loss: 0.3546 - val_acc: 0.8732\n",
      "Epoch 6/10\n",
      " - 2s - loss: 0.2942 - acc: 0.8912 - val_loss: 0.3352 - val_acc: 0.8768\n",
      "Epoch 7/10\n",
      " - 2s - loss: 0.2782 - acc: 0.8973 - val_loss: 0.3289 - val_acc: 0.8811\n",
      "Epoch 8/10\n",
      " - 2s - loss: 0.2657 - acc: 0.9015 - val_loss: 0.3672 - val_acc: 0.8673\n",
      "Epoch 9/10\n",
      " - 2s - loss: 0.2579 - acc: 0.9045 - val_loss: 0.3380 - val_acc: 0.8794\n",
      "Epoch 10/10\n",
      " - 2s - loss: 0.2465 - acc: 0.9090 - val_loss: 0.3414 - val_acc: 0.8770\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x15258dd30>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compile the model with accuracy metric and adam optimizer\n",
    "# Sparse categorical cross-entropy is the loss function for integer labels\n",
    "# Fit the model using 70 percent of the data and 10 epochs\n",
    "three_layer_model.compile(loss = 'sparse_categorical_crossentropy', \n",
    "                          optimizer = 'adam', metrics = ['accuracy'])\n",
    "three_layer_model.fit(train_images, train_labels, epochs = 10, \n",
    "                      validation_split = 0.3, verbose = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 14us/step\n",
      "Model with three layers and ten epochs -- Test loss: 36.64220314621925\n",
      "Model with three layers and ten epochs -- Test accuracy: 87.09\n"
     ]
    }
   ],
   "source": [
    "# Compute and print the test loss and accuracy\n",
    "test_loss, test_acc = three_layer_model.evaluate(test_images, test_labels)\n",
    "print(\"Model with three layers and ten epochs -- Test loss:\", test_loss * 100)\n",
    "print(\"Model with three layers and ten epochs -- Test accuracy:\", test_acc * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Similarly as before, build a five-layer (3 hidden layers) model\n",
    "five_layer_model = Sequential()\n",
    "five_layer_model.add(Flatten(input_shape = (28, 28)))\n",
    "five_layer_model.add(Dense(128, activation = 'relu'))\n",
    "five_layer_model.add(Dense(128, activation = 'relu'))\n",
    "five_layer_model.add(Dense(128, activation = 'relu'))\n",
    "five_layer_model.add(Dense(10, activation = 'softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 42000 samples, validate on 18000 samples\n",
      "Epoch 1/10\n",
      " - 3s - loss: 0.5278 - acc: 0.8103 - val_loss: 0.4236 - val_acc: 0.8461\n",
      "Epoch 2/10\n",
      " - 4s - loss: 0.3843 - acc: 0.8592 - val_loss: 0.3684 - val_acc: 0.8664\n",
      "Epoch 3/10\n",
      " - 3s - loss: 0.3430 - acc: 0.8735 - val_loss: 0.3724 - val_acc: 0.8654\n",
      "Epoch 4/10\n",
      " - 3s - loss: 0.3200 - acc: 0.8819 - val_loss: 0.3828 - val_acc: 0.8661\n",
      "Epoch 5/10\n",
      " - 3s - loss: 0.3012 - acc: 0.8873 - val_loss: 0.3639 - val_acc: 0.8648\n",
      "Epoch 6/10\n",
      " - 2s - loss: 0.2842 - acc: 0.8940 - val_loss: 0.3294 - val_acc: 0.8802\n",
      "Epoch 7/10\n",
      " - 2s - loss: 0.2747 - acc: 0.8968 - val_loss: 0.3431 - val_acc: 0.8776\n",
      "Epoch 8/10\n",
      " - 3s - loss: 0.2596 - acc: 0.9026 - val_loss: 0.3171 - val_acc: 0.8886\n",
      "Epoch 9/10\n",
      " - 3s - loss: 0.2493 - acc: 0.9052 - val_loss: 0.3398 - val_acc: 0.8817\n",
      "Epoch 10/10\n",
      " - 3s - loss: 0.2434 - acc: 0.9076 - val_loss: 0.3577 - val_acc: 0.8742\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1588dc710>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compile the model with accuracy metric and adam optimizer\n",
    "# Fit the model using 70 percent of the data and 10 epochs\n",
    "five_layer_model.compile(loss = 'sparse_categorical_crossentropy', \n",
    "                         optimizer = 'adam', metrics = ['accuracy'])\n",
    "five_layer_model.fit(train_images, train_labels, epochs = 10, \n",
    "                     validation_split = 0.3, verbose = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 34us/step\n",
      "Model with five layers and ten epochs -- Test loss: 38.488838860988615\n",
      "Model with five layers and ten epochs -- Test accuracy: 86.03\n"
     ]
    }
   ],
   "source": [
    "# Compute and print the test loss and accuracy\n",
    "test_loss, test_acc = five_layer_model.evaluate(test_images, test_labels)\n",
    "print(\"Model with five layers and ten epochs -- Test loss:\", test_loss * 100)\n",
    "print(\"Model with five layers and ten epochs -- Test accuracy:\", test_acc * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Similarly as before, build a ten-layer (8 hidden layers) model\n",
    "ten_layer_model = Sequential()\n",
    "ten_layer_model.add(Flatten(input_shape = (28, 28)))\n",
    "ten_layer_model.add(Dense(128, activation = 'relu'))\n",
    "ten_layer_model.add(Dense(128, activation = 'relu'))\n",
    "ten_layer_model.add(Dense(128, activation = 'relu'))\n",
    "ten_layer_model.add(Dense(128, activation = 'relu'))\n",
    "ten_layer_model.add(Dense(128, activation = 'relu'))\n",
    "ten_layer_model.add(Dense(128, activation = 'relu'))\n",
    "ten_layer_model.add(Dense(128, activation = 'relu'))\n",
    "ten_layer_model.add(Dense(128, activation = 'relu'))\n",
    "ten_layer_model.add(Dense(10, activation = 'softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 42000 samples, validate on 18000 samples\n",
      "Epoch 1/10\n",
      " - 5s - loss: 0.5818 - acc: 0.7879 - val_loss: 0.5485 - val_acc: 0.8081\n",
      "Epoch 2/10\n",
      " - 5s - loss: 0.4234 - acc: 0.8480 - val_loss: 0.4156 - val_acc: 0.8493\n",
      "Epoch 3/10\n",
      " - 5s - loss: 0.3863 - acc: 0.8603 - val_loss: 0.4471 - val_acc: 0.8338\n",
      "Epoch 4/10\n",
      " - 5s - loss: 0.3597 - acc: 0.8697 - val_loss: 0.3600 - val_acc: 0.8714\n",
      "Epoch 5/10\n",
      " - 5s - loss: 0.3423 - acc: 0.8761 - val_loss: 0.3530 - val_acc: 0.8770\n",
      "Epoch 6/10\n",
      " - 6s - loss: 0.3266 - acc: 0.8809 - val_loss: 0.3722 - val_acc: 0.8708\n",
      "Epoch 7/10\n",
      " - 5s - loss: 0.3139 - acc: 0.8873 - val_loss: 0.3683 - val_acc: 0.8713\n",
      "Epoch 8/10\n",
      " - 6s - loss: 0.3015 - acc: 0.8897 - val_loss: 0.3334 - val_acc: 0.8824\n",
      "Epoch 9/10\n",
      " - 5s - loss: 0.2917 - acc: 0.8921 - val_loss: 0.3505 - val_acc: 0.8762\n",
      "Epoch 10/10\n",
      " - 5s - loss: 0.2829 - acc: 0.8964 - val_loss: 0.3382 - val_acc: 0.8821\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1583fd860>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compile the model with accuracy metric and adam optimizer\n",
    "# Fit the model using 70 percent of the data and 10 epochs\n",
    "ten_layer_model.compile(loss = 'sparse_categorical_crossentropy', \n",
    "                        optimizer = 'adam', metrics = ['accuracy'])\n",
    "ten_layer_model.fit(train_images, train_labels, epochs = 10, \n",
    "                    validation_split = 0.3, verbose = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 38us/step\n",
      "Model with ten layers and ten epochs -- Test loss: 36.13102122664451\n",
      "Model with ten layers and ten epochs -- Test accuracy: 87.58\n"
     ]
    }
   ],
   "source": [
    "# Compute and print the test loss and accuracy\n",
    "test_loss, test_acc = ten_layer_model.evaluate(test_images, test_labels)\n",
    "print(\"Model with ten layers and ten epochs -- Test loss:\", test_loss * 100)\n",
    "print(\"Model with ten layers and ten epochs -- Test accuracy:\", test_acc * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 42000 samples, validate on 18000 samples\n",
      "Epoch 1/50\n",
      " - 2s - loss: 0.2380 - acc: 0.9109 - val_loss: 0.3264 - val_acc: 0.8854\n",
      "Epoch 2/50\n",
      " - 2s - loss: 0.2303 - acc: 0.9142 - val_loss: 0.3300 - val_acc: 0.8842\n",
      "Epoch 3/50\n",
      " - 2s - loss: 0.2201 - acc: 0.9187 - val_loss: 0.3325 - val_acc: 0.8842\n",
      "Epoch 4/50\n",
      " - 2s - loss: 0.2149 - acc: 0.9203 - val_loss: 0.3373 - val_acc: 0.8854\n",
      "Epoch 5/50\n",
      " - 2s - loss: 0.2098 - acc: 0.9213 - val_loss: 0.3133 - val_acc: 0.8929\n",
      "Epoch 6/50\n",
      " - 2s - loss: 0.2013 - acc: 0.9237 - val_loss: 0.3382 - val_acc: 0.8887\n",
      "Epoch 7/50\n",
      " - 2s - loss: 0.1959 - acc: 0.9265 - val_loss: 0.3345 - val_acc: 0.8881\n",
      "Epoch 8/50\n",
      " - 2s - loss: 0.1877 - acc: 0.9301 - val_loss: 0.3384 - val_acc: 0.8874\n",
      "Epoch 9/50\n",
      " - 2s - loss: 0.1822 - acc: 0.9316 - val_loss: 0.3352 - val_acc: 0.8887\n",
      "Epoch 10/50\n",
      " - 2s - loss: 0.1785 - acc: 0.9329 - val_loss: 0.3505 - val_acc: 0.8887\n",
      "Epoch 11/50\n",
      " - 2s - loss: 0.1710 - acc: 0.9375 - val_loss: 0.3702 - val_acc: 0.8837\n",
      "Epoch 12/50\n",
      " - 2s - loss: 0.1712 - acc: 0.9357 - val_loss: 0.3502 - val_acc: 0.8907\n",
      "Epoch 13/50\n",
      " - 2s - loss: 0.1631 - acc: 0.9377 - val_loss: 0.3427 - val_acc: 0.8935\n",
      "Epoch 14/50\n",
      " - 2s - loss: 0.1587 - acc: 0.9392 - val_loss: 0.3539 - val_acc: 0.8892\n",
      "Epoch 15/50\n",
      " - 2s - loss: 0.1565 - acc: 0.9410 - val_loss: 0.3835 - val_acc: 0.8864\n",
      "Epoch 16/50\n",
      " - 2s - loss: 0.1502 - acc: 0.9449 - val_loss: 0.3559 - val_acc: 0.8894\n",
      "Epoch 17/50\n",
      " - 2s - loss: 0.1456 - acc: 0.9451 - val_loss: 0.3684 - val_acc: 0.8912\n",
      "Epoch 18/50\n",
      " - 2s - loss: 0.1421 - acc: 0.9467 - val_loss: 0.3763 - val_acc: 0.8868\n",
      "Epoch 19/50\n",
      " - 2s - loss: 0.1409 - acc: 0.9478 - val_loss: 0.3775 - val_acc: 0.8902\n",
      "Epoch 20/50\n",
      " - 2s - loss: 0.1372 - acc: 0.9493 - val_loss: 0.3974 - val_acc: 0.8774\n",
      "Epoch 21/50\n",
      " - 2s - loss: 0.1341 - acc: 0.9495 - val_loss: 0.3957 - val_acc: 0.8906\n",
      "Epoch 22/50\n",
      " - 2s - loss: 0.1298 - acc: 0.9519 - val_loss: 0.3842 - val_acc: 0.8939\n",
      "Epoch 23/50\n",
      " - 2s - loss: 0.1292 - acc: 0.9514 - val_loss: 0.3803 - val_acc: 0.8913\n",
      "Epoch 24/50\n",
      " - 2s - loss: 0.1250 - acc: 0.9531 - val_loss: 0.3939 - val_acc: 0.8889\n",
      "Epoch 25/50\n",
      " - 2s - loss: 0.1219 - acc: 0.9546 - val_loss: 0.3997 - val_acc: 0.8917\n",
      "Epoch 26/50\n",
      " - 2s - loss: 0.1189 - acc: 0.9553 - val_loss: 0.4365 - val_acc: 0.8803\n",
      "Epoch 27/50\n",
      " - 2s - loss: 0.1149 - acc: 0.9564 - val_loss: 0.4095 - val_acc: 0.8882\n",
      "Epoch 28/50\n",
      " - 2s - loss: 0.1150 - acc: 0.9569 - val_loss: 0.4414 - val_acc: 0.8858\n",
      "Epoch 29/50\n",
      " - 2s - loss: 0.1090 - acc: 0.9593 - val_loss: 0.4334 - val_acc: 0.8901\n",
      "Epoch 30/50\n",
      " - 2s - loss: 0.1073 - acc: 0.9595 - val_loss: 0.4292 - val_acc: 0.8899\n",
      "Epoch 31/50\n",
      " - 2s - loss: 0.1049 - acc: 0.9613 - val_loss: 0.4384 - val_acc: 0.8849\n",
      "Epoch 32/50\n",
      " - 2s - loss: 0.1017 - acc: 0.9620 - val_loss: 0.4383 - val_acc: 0.8883\n",
      "Epoch 33/50\n",
      " - 2s - loss: 0.1027 - acc: 0.9618 - val_loss: 0.4370 - val_acc: 0.8897\n",
      "Epoch 34/50\n",
      " - 2s - loss: 0.0980 - acc: 0.9650 - val_loss: 0.4702 - val_acc: 0.8821\n",
      "Epoch 35/50\n",
      " - 2s - loss: 0.0955 - acc: 0.9641 - val_loss: 0.4760 - val_acc: 0.8834\n",
      "Epoch 36/50\n",
      " - 2s - loss: 0.0973 - acc: 0.9638 - val_loss: 0.4568 - val_acc: 0.8903\n",
      "Epoch 37/50\n",
      " - 2s - loss: 0.0906 - acc: 0.9665 - val_loss: 0.4347 - val_acc: 0.8924\n",
      "Epoch 38/50\n",
      " - 2s - loss: 0.0944 - acc: 0.9636 - val_loss: 0.4709 - val_acc: 0.8911\n",
      "Epoch 39/50\n",
      " - 2s - loss: 0.0885 - acc: 0.9672 - val_loss: 0.4761 - val_acc: 0.8857\n",
      "Epoch 40/50\n",
      " - 2s - loss: 0.0858 - acc: 0.9679 - val_loss: 0.4872 - val_acc: 0.8885\n",
      "Epoch 41/50\n",
      " - 2s - loss: 0.0868 - acc: 0.9671 - val_loss: 0.4738 - val_acc: 0.8901\n",
      "Epoch 42/50\n",
      " - 2s - loss: 0.0882 - acc: 0.9671 - val_loss: 0.4740 - val_acc: 0.8907\n",
      "Epoch 43/50\n",
      " - 2s - loss: 0.0829 - acc: 0.9697 - val_loss: 0.4880 - val_acc: 0.8878\n",
      "Epoch 44/50\n",
      " - 2s - loss: 0.0817 - acc: 0.9696 - val_loss: 0.4859 - val_acc: 0.8887\n",
      "Epoch 45/50\n",
      " - 2s - loss: 0.0812 - acc: 0.9693 - val_loss: 0.4888 - val_acc: 0.8912\n",
      "Epoch 46/50\n",
      " - 2s - loss: 0.0773 - acc: 0.9705 - val_loss: 0.5004 - val_acc: 0.8903\n",
      "Epoch 47/50\n",
      " - 2s - loss: 0.0770 - acc: 0.9717 - val_loss: 0.5121 - val_acc: 0.8898\n",
      "Epoch 48/50\n",
      " - 2s - loss: 0.0745 - acc: 0.9726 - val_loss: 0.5078 - val_acc: 0.8856\n",
      "Epoch 49/50\n",
      " - 3s - loss: 0.0734 - acc: 0.9727 - val_loss: 0.5066 - val_acc: 0.8908\n",
      "Epoch 50/50\n",
      " - 2s - loss: 0.0731 - acc: 0.9722 - val_loss: 0.4995 - val_acc: 0.8932\n"
     ]
    }
   ],
   "source": [
    "# Compile the model with accuracy metric and adam optimizer\n",
    "# Fit the model using 70 percent of the data and 50 epochs\n",
    "three_layer_model_50_epochs = three_layer_model.fit(train_images, train_labels, \n",
    "                                                  epochs = 50, validation_split = 0.3,\n",
    "                                                  verbose = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 19us/step\n",
      "Model with three layers and fifty epochs -- Test loss: 54.732778462767605\n",
      "Model with three layers and fifty epochs -- Test accuracy: 88.68\n"
     ]
    }
   ],
   "source": [
    "# Compute and print the test loss and accuracy\n",
    "test_loss, test_acc = three_layer_model.evaluate(test_images, test_labels)\n",
    "print(\"Model with three layers and fifty epochs -- Test loss:\", test_loss * 100)\n",
    "print(\"Model with three layers and fifty epochs -- Test accuracy:\", test_acc * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAEjCAYAAAC1lZ+DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOydZ5gVVdKA3yLnjAmQpChJEEYwA8oqignQXRBEWBATiouimBEThsWcMIKuKGJCFmUVUEHlExBQCUpQJCkDkpPMTH0/qi9zZ+ZOYJiZG6be57nP7T7ndJ/q7tNdfepU1xFVxXEcx3ESgRLRFsBxHMdxCgpXao7jOE7C4ErNcRzHSRhcqTmO4zgJgys1x3EcJ2FwpeY4juMkDFFRaiLSQERURErloWw/EZl1sPuJNURkhIi8EW05nKJDRD4XkYEFsJ9jRGSBiGwXkesLQraDlCfbe7SI6n9NRO6LVv2xiIh0FJE10ZbjYMjv8z1XpSYiv4rIXyJSK1P6/KDCBgcmquM4B8nNwAxVrayqTxZlxfH8IukUDMH1PyracmRHXntqvwC9Qisi0hKoUCgSOVEn0R5YiXY8QH1gUX42jPa5ECPuhz2ieR4T5RzmxMGc37yemNeBvmHrlwPjMglRVUTGiUiyiKwSkTtCJ15ESorIoyKyUURWAl0jbPuyiKwXkbUicp+IlDzQgxGRI0Rkkoj8KSLLReSKsLx2IjJXRLaJyB8iMjpILycib4jIJhHZIiJzROTQbPY/XERWBGafxSLSLSyvn4jMCo5zs4j8IiLnhOU3FJEvgm0/BWpFqiMoW11EJgfncnOwXDcsv4aIvCoi64L8D8LyLgxMU9sCWbsE6b+KSOewcvvNn2Fv3wNE5DdgepD+joj8LiJbReRLEWketn15Efl3cK23BsdeXkT+KyLXZTqe78PPVaa8C0RkUXDuPxeRpkH6LSIyMVPZJ0TkyWA52zYTXIuvROQxEdkEjIhQb4mw67lJRCaISI1M52NQcI7Xi8hNYduWFZHHg7x1wXLZ3K5BQP1Atu0i8j8JLCB5bYciMh3oBDwtIjtEpInkfO/l5Vy0E5FvgnrXi8jTIlIm0vUCvgz+twT1nxS2n+za/ucicr+IfAXsAhrldP2Cbf4pIkuC/U0VkfrZyJP5WLK9d0TkEhGZl6n8UBH5MFguGxzDb2LPiOdFpHyQ11FE1gTt8nfg1Qh15/YMyKnNZhiOkEw94mzOYf/gHG0XkZUicmVezlGwPxWRq0RkWXDdnxERye38i0jo+i8Mrv8/xJ5rPYL8U4J9dw3WzxSRBcFyiaBtrhKRDUGbrZrpeDM8gzLJ3EPsOdYix4NT1Rx/wK9AZ+AnoClQEliDvS0q0CAoNw74EKgMNAB+BgYEeVcBS4F6QA1gRrBtqSD/feAFoCJwCPAtcGWQ1w+YlY1sDTLt50vgWaAc0BpIBs4I8r4BLguWKwEnBstXAh9hPc+SQFugSjb1XQIcgb0M/APYCRweJuc+4IpgP1cD6wAJq380UBY4HdgOvJFNPTWBHoFMlYF3gA/C8v8LvA1UB0oDHYL0dsBW4G+BjHWAY8OvY9g+RoTqDzuP44JrUD5I/2dQf1ngcWBB2PbPAJ8HdZQETg7K/R34v7ByrYBNQJkIx9kkOId/C47jZmA5UAZrX7uAykHZksD6sOuWW5tJAa4DSoWOJ1PdQ4DZQN1A7heA8ZnOx/hg/y2xttQ5yB8ZbHsIUBv4Grg3D9fgc2BFcNzlg/VR+WiHnwMDw9Zzuvfyci7aAicG+Q2AJcANebnn8tj2Pwd+A5oHdZTO5fpdGLSDpkH5O4Cvc3hGvQbcl9u9E1znP4GmYdvOB3oEy48Bk7BnVOXgejwY5HUMzuNDwX4incfczkNOxzyCsOdB5vOczTnsCjQGBOiA3S9twuRdk8M5U2AyUA04EmvfXfJy/oNtjwpbHwk8FSzfhrXxh8Lyngh7niwHGmHP4PeA17N7BoWfA6B/sO1R2R3TfnkOQKndATwIdAE+DSrSoOKSwF9As7DtrgQ+D5anA1eF5Z0VJuyhwN7wRoKZOmeENZRclRqmMFMJHoJB/oPAa8Hyl8A9QK1M+/gn9lA6LrdzEaH+BcCFYXIuD8urEMh2WNBoUoCKYflvko1Si1BPa2BzsHw4kAZUj1DuBeCxnK5j2PoIsiq1RjnIUC0oUxV7WO8GWkUoVw7YDBwdrD8KPJvNPu8EJoStlwDWAh2D9VlA32D5b8CKYDkvbea3XM7pEuDMsPXDsQdS6MGuBMooyH8YeDlYXgGcG5Z3NvBrHq7B58AdYevXAJ8caDskTKmR+72X67mIsP8bgPdzu+fC0vqRTdsPk3dkWH5u1+9jAqUc1i52AfWzkek1AqWW070TrD8H3B8sNw/aallMMewEGoeVPQn4JVjuGJzncjmct2zPQx6OeQS5K7WR2dUdlPkAGBImb25K7dSw9QnA8Lycf7IqtTOB74PlT4CBwOxg/Quge7A8DbgmbLtjyHrPNYpwDm4CFgN189J+D8Qu+zpwaXDhxmXKq4W9OawKS1uFvaWC9W5WZ8oLUT/Ydn3QDd6CPRgOOQDZQnX8qarbs5FhAPaGvDQw7ZwXdlxTgbfETEkPi0jpSBWISF8xs1JIzhZkNCP+HlpQ1V3BYqVAts2qujOTbBERkQoi8kLQTd+GKeRqgamiXnCcmyNsWg974OaX/ddIzGQ8Ssx8tg1TimDHWwtTXlnqUtU9WC+yj5gJrBd2jiNxBGHnQVXTAhlC1+xN0sdyLw3WIW9tJry9RaI+8H7Y9kuwl6Jwk1/mNntEJLkz5eV2DX4PW96FtQ84gHaYidzuvczHkQUxE+ZkMVPzNuABcjCPZ0N2bT+SDLldv/rAE2F5f2JKp46I3BaYvXaIyPMRjiWnewdgLHBpYGq7DHup2ov1uCsA88Lq/SRID5EctO/8nIeCeM5luI4ico6IzBYbbtkCnMuBXbfs2mK25z+b/XwDNBEzl7fG9EM9MdN6O9JN1pHum1DHJuIxBgwDnlHVPHlz5lmpqeoqzGHkXKzbGM5GTOPWD0s7EnvrBjMb1cuUF2I19gZTS1WrBb8qqtqcA2MdUENEKkeSQVWXqWovrBE9BEwUkYqquk9V71HVZpgJ7Twyjh8CENiUXwQGAzVVtRrwI3axc2M9UF1EKmaSLTtuxN5i2qtqFcxcSVDX6uA4q0XYbjVmjojETjI69xwWoYyGLV+KmSE6Y72zBmEybAT25FDXWKA39ga3S1W/yabcOsLaTPCgqUd6u3kH6Cg2JtKNdKWWlzYTfiyRWA2cE7Z9NVUtp6prw8pkbrPrIsmdKS+na5AteW2HEcjt3oPcz8Vz2PDA0UF7u43s23Vu+8qO8O1yu36rMbNc+LUpr6pfq+oDqlop+F0VoZ6c7h1UdTbW4zoNa+OhF66NmPWheVidVVU1XDHn99jzcswHdH+KjeG+i1lCDg2eR1PI2/MoL7JGPP+RCgfKex5m0v9RVf/CrA5DMevKxqBopPsmBfgj0jGGcRZwR2jcLjcO1INmADZGFd7jQFVTse7r/SJSOVAAQ4HQwOcE4HoRqSsi1YHhYduuB/4H/FtEqgSDiY1FpMOBCKaqq7ET+aDYoPtxgbwhZ4g+IlI76A1sCTZLE5FOItIyeJPbhj0g0iJUURE74cnB/vpjPbW8yLYKmAvcIyJlRORU4PwcNqmM3WBbxJwX7g7b13rMPPCs2KB4aREJ3bgvA/2DwdkSIlJHRI4N8hYAPYPyScDFuYhdGbsJN2E32wNhMqQBrwCjxZxzSorIScGNRqDE0oB/k30vDaxddA3kLY09kPZi1xFVTcbMLq9iZqAlYefgYNvM81h7rQ8gIrVF5MJMZe4M3vybYzb9t4P08dhNVjt4G72L9Lae0zXIlgNohxnIw72XFyoHde4IZL06h7LJgVyNDmD/GcjD9XseuDU47yEHi0vyuPts750wxgFPA/tUdVYgUxr20vqYiBwS1FtHRM7O52FmIA/HvAA4XUSOFHOeuDWXXZbBzKbJQIqYQ8pZBSEruZ//P8h6/b/AXvi/CNY/z7QOdt/8S8xprhL2THlbVVNykWcRNuz1jIhckJvwB6TUVHWFqs7NJvs67G1jJTYW8ib24ANrLFOBhcB3ZO3p9cUu0mLMxj0RG+M4UHphPYp12KDs3ar6WZDXBVgkIjuAJ4CeqrobeyOaiN3US7CLkOVBrKqLsYf0N9hFbQl8dQCyXQq0x7ryd5PVhBvO49hA6UbMIeGTTPmXYQ+9pcAGbAwEVf0We/g+hjkrfEH6m9GdWA9iMza2+CY5Mw4zD6zFrsvsTPk3AT8Ac4JjeoiM7Wkcdo6yfbiq6k9AH+Cp4FjPB84P3vRCvIn1FjPLe7Bt5gnMKeB/IrIdO772mcp8gQ1OTwMeVdX/Ben3YS8p32Pn4LsgLbdrkBN5aofZkNO9lxduwtrnduxefTu7gsFb+f3AV4F56sQDqCecbK+fqr6Ptae3AhPij8A52ewnM7ndO2DntQVZ2+Yt2PWeHdT7GdbrKyhyOuZPsfP+PdbrmZzTjoJhluuxF5rN2PWbVBBC5uH8jwDGBtf/70HaF9gLxZfZrIO1ydeDtF8wa08GT+kcZFqIWS9elDCP0kiEvHIcp0ARkb7AIFU9NdqyHChiAQV+AUrn4S3SiTPE3PQ3YJ6Cy6Itj1OwJPQHfE50EJEKmGffmGjL4jgRuBqY4wotMUm0SAtOlAnGIN7DTDe5mTgdp0gRkV8xZ4qLoiyKU0i4+dFxHMdJGNz86DiO4yQMrtQcx3GchMGVmuM4jpMwuFJzHMdxEgZXao7jOE7C4ErNcRzHSRhcqTmO4zgJgys1x3EcJ2FwpeY4juMkDK7UHMdxnITBlZrjOI6TMLhScxzHcRIGV2qO4zhOwuBKzXEcx0kYisV8arVq1dIGDRpEWwwnzpg3b95GVa0dbTkOFm//Tn6I1/ZfLJRagwYNmDt3brTFcOIMEVkVbRkKAm//Tn6I1/bv5kfHcRwnYXCl5jiO4yQMrtQcx3GchKFYjKlFYt++faxZs4Y9e/ZEWxTnIClXrhx169aldOnS0RbFcZwoU2yV2po1a6hcuTINGjRARKItjpNPVJVNmzaxZs0aGjZsGG1xHMeJMsXW/Lhnzx5q1qzpCi3OERFq1qzpPW7HcYBirNQAV2gJgl9Hx3FCFFvzo5P4pKXBuHHQqRPUrx9taRwnyixeDD/9BKtXQ4kS0KsX1KwZbakKnGLdU4s2v//+Oz179qRx48a0bduWc889l59//jnaYmWhQYMGbNy4EYCTTz45Ypl+/foxceLEHPfz2muvsW7duv3rAwcOZPHixQUnaCbefRf694dWrWzZcYolW7bAP/8JzZtD9+4wZAhcdx3UrQsDB8L330dbwgLFlVqUUFW6detGx44dWbFiBfPmzePBBx/kjz/+2F8mJSUlihJG5uuvv873tpmV2ksvvUSzZs0KQqwspKXBPfdAkyZwzDFw8cVw2WXw9tuwYgWoFkq1jhNbTJsGLVvC2LEwfDjMmwcbNsAPP8Dll8Obb8L990dbygIlppSaiHQRkZ9EZLmIDI+Q309EkkVkQfAbGA05C4IZM2ZQunRprrrqqv1prVq1IjU1ldNOO40LLrhg/wN/9OjRtGjRghYtWvD4448DsHPnTrp27UqrVq1o0aIFb7/9NgDDhw+nWbNmHHfccdx0001Z6n3++ecZNmzY/vXXXnuNwYMHA3DRRRfRtm1bmjdvzpgxYyLKXalSJcCU8uDBgznmmGPo3LkzGzZs2F9m5MiRnHDCCbRo0YJBgwahqkycOJG5c+fSu3dvWrduze7du+nYseP+8E3jx4+nZcuWtGjRgltuuSVDfbfffjutWrXixBNPzKD0c2LiRFi0yBTbrFlwyy3wzjvQsyccdRR06wapqXnalePEJ6+/DmefDZUqwezZ8OCD0KYN1K4NLVrA88/D2rXwyCPRlrRgUdWY+AElgRVAI6AMsBBolqlMP+DpA91327ZtNTOLFy/evzxkiGqHDgX7GzIkS5UZeOKJJ/SGG27Ikj5jxgytUKGCrly5UlVV586dqy1atNAdO3bo9u3btVmzZvrdd9/pxIkTdeDAgfu327Jli27cuFGbNGmiaWlpqqq6efPmLPvfsGGDNm7ceP96ly5ddObMmaqqumnTJlVV3bVrlzZv3lw3btyoqqr169fX5ORkVVWtWLGiqqq+++672rlzZ01JSdG1a9dq1apV9Z133smwH1XVPn366KRJk1RVtUOHDjpnzpz9eaH1tWvXar169XTDhg26b98+7dSpk77//vuqqgrs337YsGF67733Rjyf4dczNVW1WTPVpk1VU1LSy/z1l+p336nefrsqqN50U8Rd7QeYqzFwbxzsL1L7dxKc0aOtkZ9xhuq2bfnaRby2/1jqqbUDlqvqSlX9C3gLuDDKMkWFdu3a7f/matasWXTr1o2KFStSqVIlunfvzsyZM2nZsiWffvopt9xyCzNnzqRq1apUrVqVcuXKMWDAAN577z0qVKiQZd+1a9emUaNGzJ49m02bNrF06VJOOeUUAJ588sn9PaLVq1ezbNmybGX88ssv6dWrFyVLluSII47gjDPO2J83Y8YM2rdvT8uWLZk+fTqLFi3K8XjnzJlDx44dqV27NqVKlaJ37958+eWXAJQpU4bzzjsPgLZt2/Lrr7/mev4mTrQx8bvvhpIl09NLl4bjj4f77oNrroFHHzVHEsdJKO65B4YOhR49YMoUqFw52hIVKbHk/VgHWB22vgZoH6FcDxE5HfgZ+Jeqro5Q5oAILHpFSvPmzbN1rKhYsWKu2zdp0oTvvvuOKVOmcMcdd3DmmWdy11138e233zJt2jQmTpzI008/zaeffkrbtm0BuOCCCxg5ciQ9e/ZkwoQJHHvssXTr1g0R4fPPP+ezzz7jm2++oUKFCnTs2DFf337t2bOHa665hrlz51KvXj1GjBhxUN+QlS5der/LfsmSJXMdZ0xNtXu6aVMbR8uOxx+HJUvgiits3O3EE/MtouMULarw889QtixUqQLVqpk3I8DIkTBihI2Xvfxyxre6YkIs9dTywkdAA1U9DvgUGJtdQREZJCJzRWRucnJykQmYV8444wz27t2bYezq+++/Z+bMmRnKnXbaaXzwwQfs2rWLnTt38v7773Paaaexbt06KlSoQJ8+fRg2bBjfffcdO3bsYOvWrZx77rk89thjLFy4kJIlS7JgwQIWLFjAyJEjAejWrRsffvgh48ePp2fPngBs3bqV6tWrU6FCBZYuXcrs2bNzlP/000/n7bffJjU1lfXr1zNjxgyA/QqsVq1a7NixI4Pirly5Mtu3b8+yr3bt2vHFF1+wceNGUlNTGT9+PB06dMjHWYUJE6yXNmJEzvdz6dI2xnbaaZCHdwjHiR3GjIFjj4WGDc0lv2ZNGyQeONDME337FluFBrHVU1sL1Atbrxuk7UdVN4WtvgQ8nN3OVHUMMAYgKSkp5nzdRIT333+fG264gYceeohy5crRoEEDLrroogzl2rRpQ79+/WjXrh1gbvDHH388U6dOZdiwYZQoUYLSpUvz3HPPsX37di688EL27NmDqjJ69OiIdVevXp2mTZuyePHi/fvt0qULzz//PE2bNuWYY47hxFy6Lt26dWP69Ok0a9aMI488kpNOOgmAatWqccUVV9CiRQsOO+wwTjjhhP3b9OvXj6uuuory5cvzzTff7E8//PDDGTVqFJ06dUJV6dq1KxdeeOCW51AvrUWLnHtpIWrWhM8+O+BqHCd67NkD994LSUlmQ9+61TwZp02DVatMob3ySrFVaEBMOYqUAlYCDUl3FGmeqczhYcvdgNl52XdujiJO/LN48WJ9/XUbG584sWD2SZwOlGf+uaNIAvHUU9bIp03LmJ6WpvrnnwVaVby2/5gxP6pqCjAYmAosASao6iIRGSkiFwTFrheRRSKyELge84Z0HFStl9aqlVliHCdu+eknMyOuXZsxffdueOABOP10C5MTjghUr150MsYwsWR+RFWnAFMypd0VtnwrcGtRy+UULSkp9itXLnL+li2wYwfUqAHly9u9npwMy5fD+++nj5k7TlyxY4e55o4eDfv2wZNP2q9PH1NaY8bA+vX2wbTHO82WmFJqjqNqET927YLjjss6NJCWZkMH+/bB779DmTLw11821HD//ZCPoTjHiT5paXDKKRay6vLLYdAguPlmGyMbMSK9wXfsaD8nW1ypOTHF9u32A9i0CQ45JGP+li12fzdsaI4hW7dCrVrWYzv77KKX13EOmL/+gpdeMuUVcr2dNcsU2gsvmEID+OILeOYZ+69SxcyL118fPbnjBFdqTsygCmvWWO+rVCkLUVe7dkZLy4YN9nlOjRqWHlJ6W7dGR2bHOWDGjLGAwps3w+23W9obb5iC6907vVzJkqbEXJEdED764MQMmzeb2fGII+DQQ82kuG1bev6uXTbskFnROU7csG9feqzFp5+GvXutoU+YYBH0/aPJg8aVWhTYtGkTrVu3pnXr1hx22GHUqVNn//pff/2Vp33079+fn376KccyzzzzDP/5z38KQuR8k5KSt8DBqrBunTmH1KxplpbSpSE8fnFysimzWrUKT17HOWj27YOHHrKQNZkZPx5++816X7//bk4fU6aYqaFPn6KXNQER+xwhsUlKStJQNPgQS5YsoWnTplGSKJ0RI0ZQqVKlLBH1939zEceufDt3WjSfKlWgcePsy6WlwS+/WE+tceN0z+R16+zXqJF5NK5caXlBWMwMFMb1FJF5qppUoDuNApHav1OIvP22TQdRsSK8+KJNxgnW0Fu0MNv6ggXQurW9zR11lEXRX73a8mKEeG3/8fvETECWL19Os2bN6N27N82bN2f9+vUMGjSIpKQkmjdvvj/MFcCpp57KggULSElJoVq1agwfPpxWrVpx0kkn7Z8G5o477tg/Vc2pp57K8OHDadeuHcccc8z+edF27txJjx49aNasGRdffDFJSUksWLDgoI9lxw5TaKmp5tyRXW8tJQWWLTOFVreuhbELUbt2ujJbvtyeCZkdRxwn5hgzBurVM6V16aXmEPLf/9q42ZIlNq9ZiRJw443w44/wwQem+GJIocUzfhYBbrjB3pwKktat8xUpeenSpYwbN46kJHtBGjVqFDVq1CAlJYVOnTpx8cUXZ5lYc+vWrXTo0IFRo0YxdOhQXnnlFYYPzzIdHarKt99+y6RJkxg5ciSffPIJTz31FIcddhjvvvsuCxcupE2bNvk73jB27TKFVrq03du//moejeEKK8SKFaYAQ2HswildGpo1M2tOiRJ2z5cte9DiOU7hsWIFTJ9uoaxuuQVuuw2eeCJ9OohGjeDvf7flXr3g1lvt2zM3PRYY3lOLMRo3brxfoYFNntmmTRvatGnDkiVLWLx4cZZtypcvzznnnAPkPD1L9+7ds5SZNWvW/qDGrVq1onnz5gd9DJs2mVXlmGPMS7FkSeutZSYtzZTdoYdmVWghypWzmTMqVnSF5sQBL71kb2D9+9tb2SOP2HjZjBn2YfW4cek9sjJlbOLOnj1tTiSnQPCeGkRn7plsCJ92ZtmyZTzxxBN8++23VKtWjT59+kScxqVMmTL7l3OanqVsoBXyMoXLwbB7t303FhKralVTaqoZvRZDhxJh2jfHiT/27YNXX4XzzoM6ddLTy5fP/qPpyy+3n1NgeE8thtm2bRuVK1emSpUqrF+/nqlTpxZ4HaeccgoTJkwA4IcffojYEzxQdu/OqKiqVbOxs507s5aD7MNhOU7Mk5pqg8KbNsGkSeaue8UV0ZaqWOM9tRimTZs2NGvWjGOPPZb69evvn6G6ILnuuuvo27cvzZo12/+rWrVqvve3b5/9ypdPT6tSxXpoW7ZApUrp6aGemis1J+5QtUCjd9yR0XW/bl3o0iV6cjnu0l/cSUlJISUlhXLlyrFs2TLOOussli1bRql8emJt22ZOIk2amDIL8fPPFh2oRYv0tBUrrLcWnpZf4smlX0S6AE8AJYGXVHVUpvz6wCtAbeBPoI+qrgnyjsTmEqwHKHCuqv6aU33u0l/AbNtmiuubb2yyzuuuS4/NeMYZ8Le/RVvCAiFeXfq9p1bM2bFjB2eeeSYpKSmoKi+88EK+FRqkmxTDe2pgJsjffrPeWahntnt38euliUhJ4Bngb8AaYI6ITFLVcLvvo8A4VR0rImcADwKXBXnjgPtV9VMRqQSkFaH4DsDgwfB//2ffoPXvX7wn5IxBXKkVc6pVq8a8efMKbH+7d5tzV+nSGdNDFs2tW02RpaVZhKBIbv4JTjtguaquBBCRt4ALgXCl1gwYGizPAD4IyjYDSqnqpwCquqOohHYCxo+H11+3yPkDB0ZbGicCxdpRpDiYXouaXbuy9tLA3PHLlk2P5bh3rw1LRCp7oMTZdawDrA5bXxOkhbMQ6B4sdwMqi0hNoAmwRUTeE5H5IvJI0PNzioJff4WrroKTT04PROzEHMVWqZUrV45NmzbF2wMxplE182J2LvpVqth3aWlpBeckoqps2rSJcollx7wJ6CAi84EOwFogFbOsnBbknwA0IpvZ30VkkIjMFZG5ycnJRSJ0QrJyJZx/vn003aiRpb3xhkf/iGGK7ZWpW7cua9aswW/4gmPfPpsaJjXVooRkZtcuC0r8/fem1LZssZffgw1vWa5cOerWrXtwOyk61mJOHiHqBmn7UdV1BD21YNysh6puEZE1wIIw0+UHwInAy5krUdUxwBgwR5FCOI7iwZAh8PnnptguvxwuuCBy8FEnZii2Sq106dI09MZZoLz7Llx8McyZA5EcEf/8E044wYYjfvoJvvrKlFoxYw5wtIg0xJRZT+DS8AIiUgv4U1XTgFsxT8jQttVEpLaqJgNnAO7WWFjMmgWTJ1vUjwhh55zYpNiaH52C54cfrNeVKTTlfmrUgDZt4LPP7NOe7MolMqqaAgwGpgJLgAmqukhERorIBUGxjsBPIvIzcChwf7BtKmZ6nCYiPwACvFjEh1A8ULXYjUcc4ZN0xhnFtqfmZM/69fDww3DPPRm/NcuNH36wWTRyCnvVuTP8+7sUsE8AACAASURBVN82JNGp08HLGo+o6hRgSqa0u8KWJwITs9n2U+C4QhXQgY8+gq+/hhde8DhucYYrNScLEydaOMw//oD//CfrLNMpKRaf9a+/rGdWpw40b25jZa1a5bzvzp1t/sSUlOLZU3NinLQ0mDoVhg6Fo4+279CcuMLNj8WcX3+1+QnDWbjQ/sePh7Fjs25zww1w1lkWt/Xcc02RVa9uc561bJlzfaeckh5t3wO6ODHFxIkWIeTccy1Q6fPPZ/3g0ol5XKkVc2691e7htLC4FAsXpgcVHzzYnDpCfPUVPPusxWz99luLFPTGG3DZZXD66XDRRTnXV748nHqqLbtSc2KCtDT77uySS2yOozffhFWrLOSVE3e4+bGYs2iRzTr988/2kpqSYpPxXnONWWCOO86U3vjx1iO74gqb+HP06PTgxCeeCL17573Oq6+Gww6z3p3jRJXkZGvUH35o/08/nT5nkhOXeE+tGJOaasoM0k2Qy5bZN2StWtlY2eTJpuhOPtlMjkuWmFUmPNr+gdKjh/XuHCdq7Nxpk3Y2bmyN/KmnzCnEFVrc40qtGPPLLxauCiw+K8CCBfYfcvg46SQzR/bsCV9+CZdeCsEk244TnyQnWwO/807zXPrxR7OzZ/aIcuISNz8WY0LTQNWokd5TW7jQxsbDx7uqVbOe1Q03FMw0MY4TNfbts7GzNWvsg8kzz4y2RE4B4z21YkxIqfXube74O3eaUmvaNLIVJimp+E0V4yQYQ4fCF1/ASy+5QktQXKkVY5YsMYeNLl3MAWzePFNquX1r5jhxySuvmCPIjTdCnz7RlsYpJFypFWOWLrVeWbt2tv7RRxZNxJWak3DMm2cuvZ07w6hRuZd34hZXasUUVeupNW0KtWpZeKvXXrO81q2jKprjHDwLFsD8+bb8558WafuQQ+zbFJ82JqHxq1tM+f13m4X62GNtvX17C4kF3lNz4pyNG+G002z+o+OPtxA2a9da1P1ataItnVPIeE+tmBJyEgl5OZ54ov0fcYTf906cM2qUeT2NHGkmidmz4ckn0+3sTkLjPbViSnZKzXtpTlyzbh0884w5gtx5J9xxh32Xdsgh0ZbMKSK8p1ZMWbIEKle2nhlYOKyaNS3gsOPELffdZyFwRoywdRFXaMWMmFJqItJFRH4SkeUiku1UsyLSQ0RURJKKUr5EIuQkEgqiUKaMBS6++eboyuU4+WLvXhsze+klGDAAGjWKtkROlIgZpSYiJYFngHOAZkAvEcky45aIVAaGAP9XtBImFiF3/nBq1vSZNpw4Y8UKiwpQsaI5h5QpYyZHp9gSM0oNaAcsV9WVqvoX8BZwYYRy9wIPAXuKUrhEYutWG3rwqV+cuCY52SIH/PILDB9uU8YsWQJ160ZbMieKxJKjSB1gddj6GqB9eAERaQPUU9X/isiwohQukZgxw/5dqTlxy86d0LWrxXCcPt0ibzsOsdVTyxERKQGMBm7MY/lBIjJXROYmJycXrnBR5NNPoUMHs7h89ZVNJ5MT06ZZpP1mzaBTp6KR0XEKlG3bbDbaefPg7bddoTkZiCWlthaoF7ZeN0gLURloAXwuIr8CJwKTsnMWUdUxqpqkqkm1a9cuJJGjzzPP2LQxo0bZjNL9+2dfdupUOO88ix4yY4Z5PzpOXLF6tTX0GTMsluMFF0RbIifGiCWlNgc4WkQaikgZoCcwKZSpqltVtZaqNlDVBsBs4AJVnRsdcaPPnj3WU/vnP2144dpr4fXX0+dEC2fHDvjHP+CYY8xa417OTtyxaJGFvlm1Cj7+GC6/PNoSOTFIzCg1VU0BBgNTgSXABFVdJCIjRcRfxyLwxRewa5f1vqpXt090qlWDu+7KWnbsWHMQef55jxjixCHbtkG3bjadxFdfwd/+Fm2JnBgllhxFUNUpwJRMaREe0aCqHYtCplhm8mQoXz59bKxaNbjpJhtf+/bb9KhAaWk2W327dumRQxwnblC1b89WrjQzg89U6+RAzPTUnANDFf77X5vnsHz59PTrr7fvzcJ7a59+ah9WX3990cvpOAdFaiqMHg0TJ8KDD8Lpp0dbIifGcaUWQ0yfDu+/n7eyS5bY5zldu2ZMr1wZbrnFnEIefdR6aU8+aZOBXnJJwcvsOIXCs8+ai26FCmZ+uPBC+3ecXIgp82Nx57bbzLmrW7fcy/73v/afWakBDB5sEYOGDYN33jFT5IgRFmzBcWKelBS4+26oXRv+9S+bH+nvf0+P6eY4OeBKLUZISYGFC82jcc2a3IMiTJ5sEfXr1cuaV748fPABjBsHQ4aYMrvyysKR23EKnC+/tDnRXngBunePtjROnOFKLUZYssQUGljPKpJS+/lnePdd++Z01iyLDJQdIubxfNZZ8McfZn50nLjgnXfM7NilS7QlceIQV2oxwrx56cvffpv1BXXtWjj5ZNi0CRo3NmtMXnpfhx9uP8eJC1JT4b337DuVChWiLY0Th7hSixHmzYNKlezj6P/LNP9Aair07m09uR9/hObNoyOj4xQ6M2fChg1w8cXRlsSJU9z7sZDZscM+it69O+dy330HrVvbd2Rz52aM4Xjvvfah9bPPukJzEpx33rFB4XPPjbYkTpziSq2Q+c9/bFb5iROzL5OaaqGt2ra1KEA7dth8Z2DBE0aOtPGxvn2LRmbHiQqpqTZo3LWrzY/mOPnAlVoh89FH9j95cvZlli61cFdt26ZHAfn2W/u/+25z8nj66cKV0yk6cpvhXUTqi8g0EfleRD4XkbqZ8quIyBoRSaxWMWOGeTW56dE5CFypFSK7dtlULyVK2MfQ+/ZFLvfdd/bfpg0cfTRUrWrjanPm2PZDh9p4mxP/5HGG90eBcap6HDASeDBT/r3Al4Uta5Gyezdcdx3UqWNOIo6TT1ypFSLTpplzx5VXWjDhr7+OXG7ePHP0OvZYU4AnnGA9tYcesniOgwYVrdxOoZKXGd6bAdOD5Rnh+SLSFjgU+F8RyFp03HabmSxefdVNj85B4UqtEJk82XpY994LpUunRwHJzLx55iRSsqStt2tnH2K/955NJ1OlStHJ7BQ6kWZ4r5OpzEIg9FFHN6CyiNQMJsr9N5BrvKi4miR3xgx4/HFr7B593zlIXKkVEqqm1M4+2wIMn3565HG1tDSYP99MjyHat7f0smU9CHEx5Sagg4jMBzpgk+WmAtcAU1R1TW47iJtJcv/4A/r1s5lrH3oo2tI4CYB/p1ZIzJ8P69bB+efb+nnnWRi7X36Bhg3Ty/38M+zcaU4iIdq1MzPkgAE+mWcCktsM76jqOoKemohUAnqo6hYROQk4TUSuASoBZURkh6rmEFsmhtmxwzwdN260b1bc7OgUAN5TKyQmT7ZQVeecY+uhwMOZTZBTgtnjTjghPe2wwywM1sMPF76cTpGT4wzvACJSKzA1AtwKvAKgqr1V9chg5vebMGeS+FRo+/bZtBELFsCECZCUFG2JnATBlVoh8dFHZkYM9bSOPtp+k8IeX9u3w6hRcMYZWT+qPukkjxKUiORxhveOwE8i8jPmFHJ/VIQtKBYssPhu4QwdCp98YlOxR5pqwnHyiSu1QmDpUosK0qNHxvTLLrMJO99809afeAKSk+GBB4peRid6qOoUVW2iqo1V9f4g7S5VnRQsT1TVo4MyA1V1b4R9vKaqg4ta9gNmzx449VQbXN4bHMbMmfbh5ZAhMHBgdOVzEg5XaoXAK6+YJ+Nll2VMHz7c7u8rrrDZNR55xOY+bN8+OnI6TqEzc6YNGs+bZzfA3r32jUqDBnB/fHdAndjEHUUKmH37bB6z886DQw/NmFe6tA0ftGljJse0NHP3d5yEZepUc+O97DJz21+yxEwZn3zijiFOoeA9tQLm44/NS/mf/4ycf/jhptjAIu+3bFl0sjlOkfPJJ3DaaWZubNPGlFzv3maOdJxCwJXaAZCSYgorHFV7+QyFwHr5ZfNezCnI+Gmn2cvqSy8VnqyOE3XWrIFFi0yBlS1rUb2vucZ6bI5TSLhSi8DevXDrrTBmTMb0xx+3ntYNN9gnNqtW2f3arJl5L774orns9+0LpXIx7B51lN3njpOwTJ1q/6EZrBs2hGeegVq1oieTk/C4UsvE77/beNeoUTYPmmp63tSpNgzw5JPQtCm0aGHxHG+/3cbLBg2y2TOyMz06iYWIXCci1aMtR8wydaoFKPZJAJ0ixJVawL598Pbb9hH0/Pnmlbh6tfXGwEyP33xjEX1mzbLQV6ecYjNR33cffP89vPYajB5ts1c7xYJDgTkiMiGYTkaiLVDMkJICn30GZ51lUQgcp4go9t6Pu3bBY4+ZVWT9emjSxD6cLlkSPvzQovc0aGDfj+7caS75J59s6+GULGkTeTrFB1W9Q0TuBM4C+gNPi8gE4GVVXRFd6aLMnDmweXO66dFxiohi3VP75BMzId5xB7RqZeNhS5ZYxPzmzaFGDfueDKx3BtY7c5wQqqrA78EvBagOTBSR4hvkTBXGjrUApp07R1sap5hRbHtqAwbYR9LHHguffw4dOmTML1HCvBS/+MLWZ82yHlvdupn35BRXRGQI0BfYCLwEDFPVfUHcxmXAzdGULyqompfVCy+Yp2ONGtGWyClmFFul1rq1ffg8bFj2Xoinn24myLVrTan5VE9OJmoA3VV1VXiiqqaJSPGbvlnVpqJ44gm4+mp46qloS+QUQ4qtUrvuutzLhHpvr7xi36edemrhyuTEHR8Df4ZWRKQK0FRV/09Vl0RPrCjxwAOm0IYMsYFqdxBxokCxHlPLjVatoHJlu0/BlZqTheeAHWHrO4K04seHH9rgdO/ertCcqOJKLQdKlTJFtmkTVK9u36Y5ThgSOIoAZnakOFo/fvjBlNkJJ1gEAldoThRxpZYLp59u/6ecYs4jjhPGShG5XkRKB78hwMpoC1WkqEKvXlC1KnzwAZQvH22JnGKOP6ZzITSu5qZHJwJXAScDa4E1QHtgUFQlKmrmzrX4jiNHwhFHRFsaxymGppIDpH17+zC7V69oS+LEGqq6AegZbTmiyhtvmPtw5hlxHSdKFJpSE5HGwBpV3SsiHYHjgHGquqWw6iwMSpSwz20cJzMiUg4YADQHyoXSVbV4RP9MSYG33rLJA6tVi7Y0jgMUrvnxXSBVRI4CxgD1gDdz2iCIn/eTiCwXkeER8q8SkR9EZIGIzBKRZoUjuuPkideBw4CzgS+AusD2qEpUlEybBhs2mJOI48QIhanU0lQ1BegGPKWqw4DDsyssIiWBZ4BzgGZArwhK601VbamqrYGHgdGFI7rj5ImjVPVOYKeqjgW6YuNqxYM33rAeWk6TBzpOEVOYSm2fiPQCLgcmB2mlcyjfDliuqitV9S/gLeDC8AKqui1stSKgOE70CKaGZYuItACqAodEUZ6iY+dOeP99uOQSnxjQiSkK01GkP+Yddr+q/iIiDTFzTXbUAVaHrYe8yTIgItcCQ4EywBkFJ67jHDBjgvnU7gAmAZWAO6MrUhHx4Yem2Nz06MQYhabUVHUxcD1AcONXVtWHCmC/zwDPiMil2MMk4oQvIjKIwL36yCOPPNhqHScDQdDibaq6GfgSaBRlkYoOVYvr2KiRRf12nBii0MyPIvK5iFQRkRrAd8CLIpLTGNhazJkkRN0gLTveAi7KLlNVx6hqkqom1a5d+0BEd5xcCaKHFL8o/ABffQWzZ8PQoR6RwIk5CrNFVg3GwLpjrvztgZwmV5oDHC0iDUWkDPb9z6TwAiJydNhqV2x6D8eJFp+JyE0iUk9EaoR+0Raq0HnkEZv6vX//aEviOFkozDG1UiJyOPB34PbcCqtqiogMBqYCJYFXVHWRiIwE5qrqJGCwiHTGBug3k43p0XGKiH8E/9eGpSmJbIpcuhQmTYK77oIKFaItjeNkoTCV2khMQX2lqnNEpBG59KxUdQowJVPaXWHLQwpDUMfJD6raMNoyFDn//jeUKweDB0dbEseJSGE6irwDvBO2vhLwWDpOwiAifSOlq+q4opalSFi/HsaNg3/+E3yc2olRCjNMVl3gKeCUIGkmMERV1xRWnY5TxJwQtlwOOBNzikpMpXb//ZCWZtPFO06MUpjmx1exsFiXBOt9grS/FWKdjlNkqGqG+dNFpBrmlZt4/PorjBkDAwaYK7/jxCiF6f1YW1VfVdWU4Pca4DYLJ5HZCSTmONvIkea+f8cd0ZbEcXKkMHtqm0SkDzA+WO8FbCrE+hynSBGRj0gP1VYCi1k6IXoSFRJLl8LYsTBkCNStG21pHCdHClOp/RMbU3sMu/G/BvoVYn2OU9Q8GracAqzKy5ixiHQBnsA+XXlJVUdlyq8PvIJZNv4E+qjqGhFpDTwHVAFSsRB0bxfIkWTHrl1w/fU2o/XwLBNnOE7MUZjej6uAC8LTROQG4PHCqtNxipjfgPWqugdARMqLSANV/TW7DcJmo/gbFt90johMCsLKhXgUC1gwVkTOAB4ELgN2AX1VdZmIHAHME5GphTZH4apVcNFFsHAhPPssHFI8YjU78U1Rx7gZWsT1OU5h8g6QFraeSthnLNmQ62wUmBlzerA8I5Svqj+r6rJgeR2wgcIap54zB5KS4JdfYPJkuOqqQqnGcQqaolZqUsT1OU5hUipQTAAEy2Vy2SbSbBR1MpVZiIWXA5uPsLKI1AwvICLtgrpW5EPunFmzBs4/HypVgm+/9fnSnLiiqJWaz3/mJBLJIrLfxC4iFwIbC2C/NwEdRGQ+0AEL7J0aVs/h2DRO/YPAylkQkUEiMldE5iYnJ+e95l274MIL7X/yZGjS5GCOw3GKnAIfUxOR7URWXgKUL+j6HCeKXAX8R0SeDtbXABGjjISR62wUgWmxO4CIVAJ6hMbNRKQK8F/gdlWdnV0lqjoGGAOQlJSUt5dJVfsObf58i+/YvHmeNnOcWKLAlZqqVi7ofTpOLKKqK4ATA8WDqu7Iw2b7Z6PAlFlP4NLwAiJSC/gz6IXdinlCEsxe8T7mRDKxwA4kxKJF8NZbcPfdcN55Bb57xykKfDIkx8knIvKAiFRT1R2qukNEqovIfTlto6opQGg2iiXAhNBsFGGmzI7ATyLyM3AocH+Q/nfgdKCfiCwIfq0L7IA+/tj+r7iiwHbpOEWNqCb+MFdSUpLOnTs32mI4cYaIzFPVpBzy56vq8ZnSvlPVNoUvXd7Jc/s/4wzYtMlc+J1iT27tP1bxnprj5J+SIlI2tCIi5YGyOZSPXbZvh1mzoEuXaEviOAdFYUYUcZxE5z/ANBF5FXOE6geMjapE+WXaNNi3D845J9qSOM5B4UrNcfKJqj4kIguBzpjH71SgfnSlyicffwyVK8Mpp+Re1nFiGDc/Os7B8Qem0C4BzsCcP+ILVVNqnTtD6dLRlsZxDgrvqTnOASIiTbBZJ3phH1u/jTlddYqqYPll8WJYvRruvDPakjjOQeNKzXEOnKXYTO7nqepyABH5V3RFOghCrvw+nuYkAG5+dJwDpzuwHpghIi+KyJnEc1zTSZOgRQufK81JCIqvUtu920wujnOAqOoHqtoTOBaLon8DcIiIPCciZ0VXugNkxQqYORN69Yq2JI5TIBRfpXbBBdC1qwVudZx8oKo7VfVNVT0fi+E4H7glymIdGK+9BiVKQN/cQlY6TnxQfJXajTfCjz/CNdeY95fjHASqullVx6jqmdGWJc+kpsLYsXDWWW56dBKG4qvUunQxb6+xY+GVV6ItjeMUPdOnmwm+f/9oS+I4BUbxVWoAd91l3+Zce63Hu3OKH6++CtWrmynecRKE4q3USpaEN9+EqlVNsbkZ0ikubN4M770Hl14K5cpFWxrHKTCKt1IDqF0b7r8fvvoKJkyItjSOUzS8/Tbs3eumRyfhcKUGdmO3bg0332yu/o6T6Fx6qU0I2iamZslxnIPGlRqYGfLxx+G33+DRR6MtjeMUPlWqwD/+ARK/34w7TiRcqYXo0AF69IAHHzSvMMdxHCfucKUWzlNPQaNG5u7/5pvRlsZxHMc5QFyphXP44Tb77ymnQO/eMGqUe0Q6juPEEa7UMlOtGnzyicXCu/VWuPJKmxHYcRzHiXl86plIlC0Lb7xhpsj774dVq2DiRJsZ2HEcx4lZvKeWHSVKwH33wUsvwWefwXXXRVsix3EcJxdiSqmJSBcR+UlElovI8Aj5Q0VksYh8LyLTRKR+oQs1YADcdpvFiPzgg0KvznEcx8k/MaPURKQk8AxwDtAM6CUizTIVmw8kqepxwETg4SIR7s474fjjYdAg2LChSKp0HMdxDpyYUWpAO2C5qq5U1b+At4ALwwuo6gxVDU2ANhubw6rwKVMGxo2DrVtNsaWlFUm1juM4zoERS0qtDhA+FfWaIC07BgAfF6pE4bRoYS7+H35oIYb27i2yqh3HcZy8EZfejyLSB0gCOuRQZhAwCODII48smIr/9S+bWHHYMNi0yaKcu0ek4zhOzBBLPbW1QL2w9bpBWgZEpDNwO3CBqmbbXQpmIU5S1aTatWsXnJQ33WROIzNmQPfubop0HMeJIWJJqc0BjhaRhiJSBugJTAovICLHAy9gCi16Hht9+8Kzz5qr/+jRURPDcRzHyUjMKDVVTQEGA1OBJcAEVV0kIiNFJDQ17yNAJeAdEVkgIpOy2V3hc8UV0K2buft/913UxHAcx3HSiakxNVWdAkzJlHZX2HLnIhcqO0TgxRehVSsLqTV3ro+vOY7jRJmY6anFJTVrmqv/8uVw1lmweXO0JXIcxynWuFI7WM44A955x0yQHTvC779HWyLHcZxiiyu1gqB7d/jvf63HduKJ8Omn0ZbIiWHyEA6ufhAG7nsR+VxE6oblXS4iy4Lf5UUruePEPq7UCorOnc3Nv2xZM0X27WvfsjlOGHkMB/coMC4IBzcSeDDYtgZwN9Aei8Bzt4hULyrZHScecKVWkLRrBwsXwh13wPjxcM45sGdPtKVyYotcw8Fhym56sDwjLP9s4FNV/VNVNwOfAl2KQGbHiRtcqRU05crBvffaONucOXD99dGWyIkt8hIObiHQPVjuBlQWkZp53BawiDoiMldE5iYnJxeI4I4TD7hSKywuusi+YXvxRZuTDUDVfo6TMzcBHURkPhYKbi2QeiA7KLSIOo4T47hSK0xGjrTxtSuvhIoVoVQpOOUU2LUr922dRCXXcHCquk5Vu6vq8VhIOFR1S162dZzijiu1wqRkSRtbu/VWuPpqmz37m2/gxhujLZkTPfISDq6WiITuzVuBV4LlqcBZIlI9cBA5K0hzHCcgpiKKJCQ1asB996WvlykDjzwCZ59tJkqnWKGqKSISCgdXEnglFA4OmKuqk4COwIMiosCXwLXBtn+KyL2YYgQYqap/FvlBOE4MI1oMxniSkpJ07ty50RbD+OsvOOkk+PVXWLAA6tXLdRMnOojIPFVNirYcB0tMtX8nbojX9u/mx6KmTBkzSe7da8pt1qxoS+Q4jpMwuFKLBk2awMyZ5v7fsSM88ADs3BltqRzHceIeV2rR4vjjLV5k9+5w++1w6KEWhcTNRI7jOPnGlVo0qVIF3n4bvvwSLr0UJk2C006zXpzjOI5zwLhSizYipsjGjIFly6B+fTjvPJg/P9qSOY7jxB2u1GKJ2rUtwn/Vquby/+OP0ZbIcRwnrnClFmvUqweffWYfbp94opknHSfB+Ouvwg2sc9FFcNddhbd/J3ZxpRaLNGkC8+ZB69bQsycMGeKhtZyEol8/OPnkwtn3/Pnw4Yfw0EOwZk3h1JEo3Htv4r03u1KLVY44wuZnGzIEnnwSmjeHyZOjLZVTDOnXD/71r4Lb34oV8NZbNkvTzz8X3H5DvPyyTWuYlgajRhX8/hOFZcvgnnsSzy/NlVosU7o0PP44fP45VKgA558PbdrAVVdZ9P9t26ItoZPgqFqv55VXzGRYEDz2mFnXAT76qGD2GWL3bnjjDejRA/r3t9sk1nprP/wAzZrBBx9EV44777RPZe+8M7pyFDSu1OKBDh3MpjJ6NFSvbvaCQYNi485wEpr162HLFnt/Kog3+j//hFdfhT59oGXLgjc+vPsubN0KAwfazE9paWaGzMy0afDbbwVbd15IS7PY5kuWwD/+AVOjFI563jx7jPzrX/aJbCLhSi1eKFPGWuC0afZk+OorqFkTunWzUfGffoq2hE4CEu6AO2lS9uXyyvPP2/Dw0KH25crMmaY0s0PVFN/WrXnb/8svQ6NG9h7YoIGZTseMydhb++03cy5+8MGDOZL88frrduv++9/2TnrRRfDFFwWz7x9/hB078lb2ttvs8XHTTQVTdyzhSi0eEbFR9rlz7TV02jQbc7viCli3LtrSOQnEokX2366dmQoPJv753r3w1FM2xWDLlqbUUlNz7q385z9mdR89Ovf9L19ulvoBA6BE8GS77TaT+f7708s98YTVu2pV/o8lP2zZAsOGWcjXG26A//0PGja0HltaWs7bqtq12LMncv68eXDccXZeP/ss533NmGF133abfT2UcKhqwv/atm2rCc0ff6gOGaJaurRqjRqqkyZFW6KEAJsKJurt92B/B9P+BwxQrVVL9YUXbNr2H3/M96500iTbx5Qptp6SYvvu3Tty+VWrVKtUsW2SkjLmrVihunp1xrQHHrCya9ZkTL/6atVSpWybLVtUK1e2ci1b5i5zWprq5s32f7Bcd51qiRKq8+enp732Ws7nde9e1TfesOMH1cGDI5fr0sVu/SZNrNzll6v+8EPksgMHqlarprp7d87yxmv7j7oARfFLeKUW4qefVI8/3i7rddepTp+uumSJ6q5d0ZYsLonXmzrz72Da/4knqnbooLp2rTWrBx7I9670nntURVR37EhP69vXHsb79mUsm5qq2rGjaqVKplhBdf369LyGDVXPOy/jNoMGqdaunbXetWtVy5Wzuh591PZ14on2YM+N99+38tWqqZ56quorr2RfdsuWjMcWTlqa7ObIlAAAE4hJREFUavXqqn36ZExfvtz2//zzWbfZt0/1rLMs/9hjVU85RbV8edWNGzOW+/JLK/Pww3ar33KLapkylta+veoXX2Qs36SJ6vnn537s8dr+oy5AUfyKjVJTVd2zR/X66+3Shn7Vq6u++260JYs74vWmzvzLb/tPS7Oe0rXX2nrbtqonnZSvXamqavfuqkcfnTFtwgRrojNnZkx/7DFLf/ll1e++s+VXX7W86dNtvUWLjNucf77qccdFrvumm6yXVLu2KekHH7R9bN+es8y9eqnWrGm9vQYNVA85JHKvbeVK1Tp17BgjsWqV1ffssxnT09JUDz00q7JTNeUEqk89ZYr8xx9t/d57M25/2mmqhx2munNnenpysuro0SZT06bp6evX2z4eeSTn41aN3/YfdQGK4leslFqIlStVp03LaLu48sqMLd/JkXi9qTP/8tv+f/tNMzyIR4ywntYff2S/zcaN9gBftSprr6VxY9UePTKmbd1qvYqrr05P273bzJJnn20P7bQ01cMPV73kEsu/7DKTq2bNjPtKSrJtIpGcnG52/Ogjuy3ADBnZsW+fvQ9efrmtv/yyRjQVrl5tCg9U69WLvK+Q6fWrr7Lm9ehh24fzzjtW/qqrMqZ36WJKMGQ6/OQTK/f005Hrffxxy1++3NZDLxGzZ2d72PuJ1/YfdQGK4lcslVo4e/eq3nyzXe46dczWsXdvtKWKeeL1ps78y2/7//hjazIh89XChbbeoIHqc89lHZN59lnrDYUMBA0apPdqtm+3tJEjs9bTv7+Z1ZKTbT2kPKZPTy8zYID1GjdtsrKlS1uZPXvSy9Stm66AIvHkk6YUUlPtmED1f/9Lz9+1y8bPQsycaWUmTLD1lSt1f88pxIYNqsccYwrz73+3/K1bs9Y9cqS9EGzbljUv1CsNjRGuWKFasaKZSMOPT1X10091fw924kQz3TZokP3tHDJvPvGErV93nWqFCqp//ZX9eQoRr+0/6gIUxa/YK7UQX36pevLJdtkbNlQdO9ZG652IxOtNnfmX3/YfGn8KH8OZMsXGaULvR2PHmpJ47jlLO+cce+BedZVm6NV8/bWtf/hh1noWLbK8ESNMCbZqZU4c4Wa+d9+1Mn372n9onO3XXy0/NdWcQYYPz9uxrVhh24ePkV1xhWr9+unK+pZbbJ9btqSXqV8/o4nxxhutzMyZdmzZ9YJ69FA96qjIssyZY9u99ZatX3WVatmy1lPOTFqamVgrVrRtTjjBhtJzomlT1c6dbblVq/Tl3IjX9h91AYri50otjLQ0ezK1aaP7R6Dfey/aUsUk8XpTZ/7lt/3362djNZlJS1P97DN7oIaaEKh27Zres8jcq3n+eVv/5ZfIdZ13npkcQ73DF1/MmL91a3rv7JhjVCdPtuWvv7b8DRs0Q48kN/bssfL33JOe1rq1pY0ebestWqh26pRxu/79rXeUmmrmyUMPVb3wQstbtkz396Iyc9RRWU2vIfbtMyU1eLAdR7ly5qGYHRMmmCK966689biGDbNzt2qV9RbDjzkn4rX9+3dqxQ0ROOcc+8btvfcsXlH37nDJJZCcHG3pnBhi0SL7/DEzInDmmTB7tn1MvHMnXHihRfMoW9bKNGxoUwPOmGHrCxfaN1H160eu6+abYeNGmyu3Zk3o3TtjfpUqNu0g2AfVRxxhy+vXZ/w//PC8HVvZshZJY/VqW09NhaVLbfmBB+zYf/wRunbNuF2nThb74PvvbZaoP/6Ayy9PP+Zy5dK/7QuxfbvFu2zdOrIspUrZhByzZsFzz9m3aEOHZi/7JZfYR9b33GOR9HLj/PNh3z64+24zDJ9+eu7bxDOu1IorIhaNZMECC60waZI9wS6+2NL794cNG6ItpRMl0tJg8eLISi1EiRIW7mrVKnj//XSFFqJTJ/sYOi3NlNpxx1mzi8Spp0L79rB5s0WAK18+a5mLLzalcdll6UotFGvg99/t/7DD8n6MdeumRxpZtcqUyaBBplwvvtjSIyk1MGU9dizUqAHnnmtpJUtC06ZZldoPP5gyadUqe1lOPdXO0VNP2f6aNs1Z9sznOidOOsmi640da0qwffu8bxuPuFIr7pQqBcOHp091s2RJehj1zp1h06ZoS+hEgd9+sx5YTkothEhkZRXq1SxcaA/2447LeR/33ANHHgnXXhu5zJVXmlx16th8uiVL5r+nBjZ1YaintmSJ/V9+ufU6ly6Fxo3hmGMyblO3Lhx1lCnxDz6AXr0yKpjmzbMqtYUL7T83paZqCvXGG/N+DHmhVCkzzqhaZJhILwyJhCs1x2jRwmLnLFpktpWPPrJ5Qc46K+fgfE5csns3jBuXfdir0IM5L0otO0K9mldfNRNcTg91sHiMq1aZ0opEiRKmzELLhx2W3lMrKKXWtKnNMSZiYbyyU9YzZ1rYr5DpMUTz5tb7C49VuXCh9ZTq1ctelvbtTUkff3z6eStIzjvP/kMm3ETGlZoTmc6dbczthx9M4V15pa37ZKUJwUsv2QN5+PCsii05GUaMsB5Iixb5r6NePevVvPqqrefUU8sPhx+esadWuTJUrJj37evWtdkHtm0zU+thh5nyadnSgg7ffXfk7c44w/6bNoWkpIx5oZeAxYvT0xYsMIWenekVTPbnn4cXXsi5XH7p2tV6a5nHKhORmFJqItJFRH4SkeUiMjxC/uki8p2IpIjIxdGQsVhx7rnwySd2544fb5NU1a1r9pHly6MtnXMQXHstXHMNPPwwXH99ekDdVavMFPbjjzBx4sEHvO3UyZwaRA5OQUbiiCMyjqkdSC8N0ntOa9ZYTy18HCs0DhWJTp1M4Q8cmFUBNWtm/yGllppq74W59VLB9nfCCQd2DHmlShWYMqXgr0EsEjNK7f/bu/cYucoyjuPfX2+ClUulUrYstaS0SCuVS1Wg/EFWGlruiYpUIF6aaFBICwSBaMQYJCgIFIqJRaQEMFBQsZGbQIkhstEKllYgBCRIwa0FvFUj1pbHP55z6FCK3dmd3Zk58/skk5lzZnb6nu579jnv+z7veySNBK4D5gHTgfmSpm/zsReBzwI/Gt7SdbCenhw8eO21XP57zpy8E/e0aZlMUt6U6o03ckDA2sKIEbBkSV6fLFmSf9D33z//6G3YkJl9ZZfVYJRdaVOn1teK6o/aoNbXV1+SCOT1GWQX5FNP7Tg5ozRhQl7TLVr09vf23TfHrMru2z/8ITs33inz0RqvZYIa8BHguYh4PiI2AbcBJ9V+ICJeiIg1wA5u1GANN3p05nHffnsGsnPPzdbbtGl5GTp2bA54zJ+fGQbW8iS4/PK8ufq0aXlT9dNOy9TyI49szL9x1FH53OiuR8iW2Wuv5dhWX9/AW2qrVmUX5PRtL6H/j+7urbe3qTVixFszIFevzuf+tNSsMUY1uwA19gbW1Wy/BFQ8+bRNdXXBFVdkv9Wll2aQO/roHJy5+uq87L3rrrxstZYmwcKF+RgKXV05bleOQzVSmda/fv3AgtrEiXn85f3c+ttS25EZM2DlymyhLV4M7353fQHTBqeVglpDSfoC8AWASZMmNbk0FTVpUo5u15ozJ2fQzpyZs2S/9KUcgFixAnp7MwOhUX89rC0M1R2myyD27LPZOVBvUBszJrsSe3tzu5FB7eabc2pAby8sX17fvDIbnFbqfnwZqE167S72DUhELI2IWREx631lHrANvXK1kpNPhqVL8xJ1yhQ455xsvc2duzVlLQLuvTcHHszqVLbUHn88n+sdU4PsgtyyJRNiBvLz21O2yh58MFcI+YRT2oZVKwW1VcBUSftKGgOcCqxocplsIKZMyUvVdeuym/Kqq3Jkvbc3B0GOOy5nt55wQmZYHn74W3OgzfqhbJk99thbt+tRjqtNn964VPpZs3Llk0suyZkwNrxaJqhFxGbgLOB+4GlgeUQ8Kembkk4EkPRhSS8BnwS+L+nJd/5Ga7o998z0ukWLMtAdcgjccUdO7j7ggBx4uPjinHXa07N18b0O0I/pK5MkPSzpd5LWSDq22D9a0k2S1kp6WtJFw1/61lCuKlK21AYS1MoMyEb2iHd15XoFX/1q477T+q9lghpARNwTEdMiYkpEfKvY9/WIWFG8XhUR3RExNiL2iIhBrHdgTTFvHixblquyrl2bY2wrV2ZXZE8P3HILbNq09fObN+el+OLFcNFFlVi2q5/TV75GXtgdTPZafK/Y/0ngXRFxIHAo8EVJk4ej3K1m5Mit6fUwuJZao4d5PYbWPJVNFLEWdvrp+SiVrbZTTsnVai+8MJcSf+aZ7JZ8/fX8nJTjcvffn0kq7evN6SsAksrpK7V9sAHsWrzeDfhTzf6xkkYBOwObgH8MR6FbUTlXbfToXFy4XrXdj1YNLdVSsw42Y0a23O6+Gz7wgVxcb/x4OPPMnA+3bl0u+d7XB0ccAY8+miP87Wl701e2XfHwG8DpRXf7PcDZxf47gX8BfeRiBFdExF+GtLQtrGyd7bXXwMbEjjkmc5jK+XTW/txSs9YxYkQmjpT38thWd3cGu2OOgdmzc8L3QQfB5Mk5wLLffrBgQY7St7/5wLKI+K6kw4GbJX2QbOVtASYC44BHJD1YtvpKnTKlpcyAHEjXI8Duu8OVVzauPNZ8bqlZeznwwFymYdky+Pzn8/K8tzdX6D3rLDj00K3pcK2rP9NXFgDLASKiF9gJGA98GrgvIv4bERuAXwHbLKvbOVNaymA20KBm1eOWmrWfPffMJea3ve/HffflqrAf/WguXDhuXE5AOv74XOJrKJY/H5g3p6+QwexUMljVehH4GLBM0gFkUHul2N9DttzGAocBVw9XwVtN2VJr1Bwza38OalYdc+fmuNwFF+S9QzZuzGzJxYtznG7BgrxL4syZ2e/UJBGxWVI5fWUk8MNy+grw2yLb9zzgeknnkMkhn42IkHQdcGMxnUXAjcV6qB3JLTXbloOaVcu4cbmSSen113Nu3LXXwvnnb92/6655E6tddsm7DsyZM6zFjIh7yASQ2n1fr3n9FDB7Oz/3TzKt3xj8mJpVj4OaVdtOO+U0gTPOyNzvJ57Iyd99fdmS27ixqa02G5yZM7NhftJJO/6sdQYHNescEyfmY968ZpfEGmTUKLjssmaXwlqJsx/NzKwyHNTMzKwyHNTMzKwyHNTMzKwyHNTMzKwyHNTMzKwyHNTMzKwyHNTMzKwyFBHNLsOQk/QK8MftvDUeeHWYizPUfEyN8/6IaPsl7l3/257rfx06Iqi9E0m/jYi33bajnfmYrL+q+P/qYzJ3P5qZWWU4qJmZWWV0elBbuuOPtB0fk/VXFf9ffUwdrqPH1MzMrFo6vaVmZmYV0rFBTdJcSc9Iek7Shc0uT70k7SPpYUlPSXpS0sJi/3slPSDp2eJ5XLPLWi9JIyX9TtLPi+19Jf26+F3dLmlMs8vY7tq9/kN1zwHX/8HpyKAmaSRwHTAPmA7MlzS9uaWq22bgvIiYDhwGfLk4hguBhyJiKvBQsd1uFgJP12x/G7gqIvYD/gosaEqpKqIi9R+qew64/g9CRwY14CPAcxHxfERsAm4D2uqG8BHRFxGPF683kifB3uRx3FR87Cbg5OaUcGAkdQPHAT8otgX0AHcWH2m7Y2pBbV//oZrngOv/4HVqUNsbWFez/VKxry1JmgwcDPwamBARfcVb64EJTSrWQF0NfAV4o9jeA/hbRGwuttv6d9UiKlX/oVLngOv/IHVqUKsMSe8Bfgwsioh/1L4XmdraNumtko4HNkTEY80ui7WPqpwDrv+NMarZBWiSl4F9ara7i31tRdJo8mS+NSJ+Uuz+s6SuiOiT1AVsaF4J6zYbOFHSscBOwK7AYmB3SaOKq9W2/F21mErUf6jcOeD63wCd2lJbBUwtsorGAKcCK5pcproUfe03AE9HxJU1b60APlO8/gzws+Eu20BFxEUR0R0Rk8nfycqIOA14GPhE8bG2OqYW1fb1H6p3Drj+N0ZHBrXiiucs4H5ycHl5RDzZ3FLVbTZwBtAjaXXxOBa4DJgj6Vng6GK73V0AnCvpOXKM4YYml6etVaT+Q+ecA67/dfCKImZmVhkd2VIzM7NqclAzM7PKcFAzM7PKcFAzM7PKcFAzM7PKcFCrCElbatKaVzdy5XVJkyX9vlHfZ9Zorv9W6tQVRaro3xFxULMLYdYkrv8GuKVWeZJekPQdSWsl/UbSfsX+yZJWSloj6SFJk4r9EyT9VNITxeOI4qtGSrq+uG/VLyTt3LSDMusn1//O46BWHTtv0/3yqZr3/h4RBwJLyFXAAa4FboqImcCtwDXF/muAX0bEh4BDgHKlianAdRExA/gb8PEhPh6zerj+G+AVRSpD0j8j4j3b2f8C0BMRzxeLv66PiD0kvQp0RcR/i/19ETFe0itAd0T8p+Y7JgMPFDddRNIFwOiIuGToj8xsx1z/reSWWmeId3hdj//UvN6Cx2Otfbj+dxAHtc7wqZrn3uL1o+RK4ACnAY8Urx8CzgSQNFLSbsNVSLMh4vrfQXy1UR07S1pds31fRJRpzeMkrSGvNucX+84GbpR0PvAK8Lli/0JgqaQF5BXpmUAfZq3N9d8Aj6lVXjGmMCsiXm12WcyGm+t/53H3o5mZVYZbamZmVhluqZmZWWU4qJmZWWU4qJmZWWU4qJmZWWU4qJmZWWU4qJmZWWX8D1fN3qQrBAPHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot loss as function of epochs\n",
    "plt.subplot(1, 2, 1)\n",
    "plt.plot(three_layer_model_50_epochs.history['val_loss'], 'blue')\n",
    "plt.plot(three_layer_model_50_epochs.history['loss'], 'red')\n",
    "plt.legend(['Cross-validation', 'Training'], loc = 'upper left')\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('Epoch')\n",
    "\n",
    "# Plot accuracy as function of epochs\n",
    "plt.subplot(1, 2, 2)\n",
    "plt.plot(three_layer_model_50_epochs.history['val_acc'], 'blue')\n",
    "plt.plot(three_layer_model_50_epochs.history['acc'], 'red')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.subplots_adjust(wspace = .35)\n",
    "\n",
    "# Include plot title and show the plot\n",
    "plt.suptitle('Model loss and accuracy over epochs for a three-layer neural network')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction 0: 9\n",
      "Actual 0: 9\n",
      "Prediction 1: 2\n",
      "Actual 1: 2\n",
      "Prediction 2: 1\n",
      "Actual 2: 1\n",
      "Prediction 3: 1\n",
      "Actual 3: 1\n",
      "Prediction 4: 0\n",
      "Actual 4: 6\n",
      "Prediction 5: 1\n",
      "Actual 5: 1\n",
      "Prediction 6: 4\n",
      "Actual 6: 4\n",
      "Prediction 7: 6\n",
      "Actual 7: 6\n",
      "Prediction 8: 5\n",
      "Actual 8: 5\n",
      "Prediction 9: 7\n",
      "Actual 9: 7\n"
     ]
    }
   ],
   "source": [
    "# Calculate and print predictions versus actual labels\n",
    "predictions = three_layer_model.predict(test_images)\n",
    "for i in range(10):\n",
    "  print(\"Prediction \" + str(i) + \": \" + str(np.argmax(np.round(predictions[i]))))\n",
    "  print(\"Actual \" + str(i) + \": \" + str(test_labels[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reload the data for a convolutional neural network\n",
    "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape the data to the correct format (the last 1 stands for greyscale)\n",
    "train_images = train_images.reshape(60000, 28, 28, 1)\n",
    "test_images = test_images.reshape(10000, 28, 28, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert the image data to numeric data and normalize them\n",
    "train_images = train_images.astype('float32')\n",
    "test_images = test_images.astype('float32')\n",
    "train_images = train_images / train_images.max()\n",
    "test_images = test_images / test_images.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One-hot encode the label data\n",
    "# Convert every number to a vector of the length of the number of categories\n",
    "# The vector has zero everywhere except a one on the position of the number it \n",
    "# represents. Example: 3 = [0 0 0 1 0 0 0 0 0 0]\n",
    "train_labels_bin = to_categorical(train_labels)\n",
    "test_labels_bin = to_categorical(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a convolutional neural network with two convolutional layers\n",
    "conv_model = Sequential()\n",
    "conv_model.add(Conv2D(128, (3, 3), input_shape = (28, 28, 1)))\n",
    "conv_model.add(Activation('relu'))\n",
    "conv_model.add(MaxPooling2D(pool_size = (2, 2)))\n",
    "conv_model.add(Conv2D(128, (3, 3)))\n",
    "conv_model.add(Activation('relu'))\n",
    "conv_model.add(MaxPooling2D(pool_size = (2, 2)))\n",
    "conv_model.add(Flatten())\n",
    "conv_model.add(Dense(128))\n",
    "conv_model.add(Dense(10))\n",
    "conv_model.add(Activation('softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      " - 127s - loss: 0.4629 - acc: 0.8333\n",
      "Epoch 2/10\n",
      " - 126s - loss: 0.3047 - acc: 0.8893\n",
      "Epoch 3/10\n",
      " - 120s - loss: 0.2619 - acc: 0.9052\n",
      "Epoch 4/10\n",
      " - 121s - loss: 0.2324 - acc: 0.9142\n",
      "Epoch 5/10\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-e91eb8fb38e3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m                    optimizer = 'adam', metrics = ['accuracy'])\n\u001b[1;32m      6\u001b[0m conv_model.fit(train_images, train_labels_bin, batch_size = 128, \n\u001b[0;32m----> 7\u001b[0;31m                epochs = 10, verbose = 2)\n\u001b[0m",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1037\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1040\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2713\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2714\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2715\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2716\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2717\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2673\u001b[0m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_metadata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2674\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2675\u001b[0;31m             \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2676\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1437\u001b[0m           ret = tf_session.TF_SessionRunCallable(\n\u001b[1;32m   1438\u001b[0m               \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1439\u001b[0;31m               run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1440\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1441\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Compile and fit the model with adam optimizer and accuracy metric\n",
    "# Categorical cross-entropy is the loss function for one-hot encoded labels and\n",
    "# batch size equal to the number of neurons in the convolutional layers and 10 epochs\n",
    "conv_model.compile(loss = \"categorical_crossentropy\", \n",
    "                   optimizer = 'adam', metrics = ['accuracy'])\n",
    "conv_model.fit(train_images, train_labels_bin, batch_size = 128, \n",
    "               epochs = 10, verbose = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute and print the test loss and accuracy\n",
    "test_loss, test_acc = conv_model.evaluate(test_images, test_labels_bin)\n",
    "print(\"Convolutional model ten epochs -- Test loss:\", test_loss * 100)\n",
    "print(\"Convolutional model ten epochs -- Test accuracy:\", test_acc * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a convolutional neural network with two convolutional layers\n",
    "# Decrease number of neurons and add dropout to reduce overfitting\n",
    "conv_model_reduce_overfit = Sequential()\n",
    "conv_model_reduce_overfit.add(Conv2D(64, (3, 3), input_shape = (28, 28, 1)))\n",
    "conv_model_reduce_overfit.add(Activation('relu'))\n",
    "conv_model_reduce_overfit.add(MaxPooling2D(pool_size = (2, 2)))\n",
    "conv_model_reduce_overfit.add(Dropout(0.5))\n",
    "conv_model_reduce_overfit.add(Conv2D(64, (3, 3)))\n",
    "conv_model_reduce_overfit.add(SpatialDropout2D(0.5))\n",
    "conv_model_reduce_overfit.add(Activation('relu'))\n",
    "conv_model_reduce_overfit.add(MaxPooling2D(pool_size = (2, 2)))\n",
    "conv_model_reduce_overfit.add(Flatten())\n",
    "conv_model_reduce_overfit.add(Dense(64))\n",
    "conv_model_reduce_overfit.add(Dropout(0.5))\n",
    "conv_model_reduce_overfit.add(Dense(10))\n",
    "conv_model_reduce_overfit.add(Activation('softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 42000 samples, validate on 18000 samples\n",
      "Epoch 1/10\n",
      " - 31s - loss: 0.7606 - acc: 0.7243 - val_loss: 0.4695 - val_acc: 0.8369\n",
      "Epoch 2/10\n",
      " - 34s - loss: 0.5165 - acc: 0.8166 - val_loss: 0.3896 - val_acc: 0.8603\n",
      "Epoch 3/10\n",
      " - 32s - loss: 0.4597 - acc: 0.8383 - val_loss: 0.3502 - val_acc: 0.8748\n",
      "Epoch 4/10\n",
      " - 33s - loss: 0.4294 - acc: 0.8476 - val_loss: 0.3333 - val_acc: 0.8774\n",
      "Epoch 5/10\n",
      " - 33s - loss: 0.4075 - acc: 0.8543 - val_loss: 0.3229 - val_acc: 0.8831\n",
      "Epoch 6/10\n",
      " - 34s - loss: 0.3951 - acc: 0.8602 - val_loss: 0.3167 - val_acc: 0.8839\n",
      "Epoch 7/10\n",
      " - 33s - loss: 0.3825 - acc: 0.8651 - val_loss: 0.3151 - val_acc: 0.8839\n",
      "Epoch 8/10\n",
      " - 35s - loss: 0.3751 - acc: 0.8678 - val_loss: 0.2962 - val_acc: 0.8916\n",
      "Epoch 9/10\n",
      " - 35s - loss: 0.3686 - acc: 0.8698 - val_loss: 0.3034 - val_acc: 0.8879\n",
      "Epoch 10/10\n",
      " - 33s - loss: 0.3642 - acc: 0.8699 - val_loss: 0.2917 - val_acc: 0.8936\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x135a84208>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compile and fit the model with adam optimizer and accuracy metric\n",
    "# Categorical cross-entropy is the loss function for one-hot encoded labels and\n",
    "# batch size equal to the number of neurons in the convolutional layers and 10 epochs\n",
    "# Add early stopping to avoid overfitting\n",
    "conv_model_reduce_overfit.compile(loss = \"categorical_crossentropy\", \n",
    "                   optimizer = 'adam', metrics = ['accuracy'])\n",
    "conv_callback = keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 3)\n",
    "conv_model_reduce_overfit.fit(train_images, train_labels_bin, validation_split = 0.3,\n",
    "               epochs = 10, verbose = 2, callbacks = [conv_callback], batch_size = 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 2s 208us/step\n",
      "Convolutional model ten epochs reduced overfit -- Test loss: 30.780641422271728\n",
      "Convolutional model ten epochs reduced overfit -- Test accuracy: 88.83\n"
     ]
    }
   ],
   "source": [
    "# Compute and print the test loss and accuracy\n",
    "test_loss, test_acc = conv_model_reduce_overfit.evaluate(test_images, test_labels_bin)\n",
    "print(\"Convolutional model ten epochs reduced overfit -- Test loss:\", test_loss * 100)\n",
    "print(\"Convolutional model ten epochs reduced overfit -- Test accuracy:\", test_acc * 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction 0: 9\n",
      "Actual 0: 9\n",
      "Prediction 1: 2\n",
      "Actual 1: 2\n",
      "Prediction 2: 1\n",
      "Actual 2: 1\n",
      "Prediction 3: 1\n",
      "Actual 3: 1\n",
      "Prediction 4: 6\n",
      "Actual 4: 6\n",
      "Prediction 5: 1\n",
      "Actual 5: 1\n",
      "Prediction 6: 4\n",
      "Actual 6: 4\n",
      "Prediction 7: 6\n",
      "Actual 7: 6\n",
      "Prediction 8: 5\n",
      "Actual 8: 5\n",
      "Prediction 9: 7\n",
      "Actual 9: 7\n"
     ]
    }
   ],
   "source": [
    "# Calculate and print predictions versus actual labels\n",
    "predictions = conv_model_reduce_overfit.predict(test_images)\n",
    "for i in range(10):\n",
    "  print(\"Prediction \" + str(i) + \": \" + str(np.argmax(np.round(predictions[i]))))\n",
    "  print(\"Actual \" + str(i) + \": \" + str(test_labels[i]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}