09. Convolutional Neural Networks (CIFAR-10).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "rO8YZk9MvQXd"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "\n",
    "from time import time\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "cellView": "code",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 208,
     "output_extras": [
      {
       "item_id": 21
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9011,
     "status": "ok",
     "timestamp": 1521001289514,
     "user": {
      "displayName": "Mayur Kulkarni",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "116301151646646358552"
     },
     "user_tz": -330
    },
    "id": "mYhJ2wOLvQYO",
    "outputId": "522ac1d8-2ceb-47fb-979b-df68a5cd837e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2018-03-14 04:21:21--  https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\r\n",
      "Resolving www.cs.toronto.edu (www.cs.toronto.edu)... 128.100.3.30\n",
      "Connecting to www.cs.toronto.edu (www.cs.toronto.edu)|128.100.3.30|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 170498071 (163M) [application/x-gzip]\n",
      "Saving to: ‘cifar-10-python.tar.gz’\n",
      "\n",
      "cifar-10-python.tar 100%[===================>] 162.60M  17.1MB/s    in 7.8s    \n",
      "\n",
      "2018-03-14 04:21:28 (20.9 MB/s) - ‘cifar-10-python.tar.gz’ saved [170498071/170498071]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# download CIFAR-10 dataset from: https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
    "!wget https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1040,
     "status": "ok",
     "timestamp": 1521001354995,
     "user": {
      "displayName": "Mayur Kulkarni",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "116301151646646358552"
     },
     "user_tz": -330
    },
    "id": "UlPzuH-EvlzO",
    "outputId": "1ba69957-de71-44c6-92fa-9517c87bfde2"
   },
   "outputs": [],
   "source": [
    "!mv cifar-10-python.tar.gz /datasets/cifar-10-python.tar.gz\n",
    "# move wherever your want and set the CIFAR_DIR variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "hPSeFrxfvQYi"
   },
   "outputs": [],
   "source": [
    "def unpickle(file):\n",
    "    CIFAR_DIR = \"datasets/cifar-10-batches-py/\"\n",
    "    with open(CIFAR_DIR + file, 'rb') as fo:\n",
    "        dict = pickle.load(fo, encoding='bytes')\n",
    "    return dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "ZgOXFYPbvQY6"
   },
   "outputs": [],
   "source": [
    "dirs = ['batches.meta', 'data_batch_1', 'data_batch_2', \n",
    "        'data_batch_3', 'data_batch_4', 'data_batch_5',\n",
    "        'test_batch']\n",
    "\n",
    "batch_meta = unpickle(dirs[0])\n",
    "data_batch1 = unpickle(dirs[1])\n",
    "data_batch2 = unpickle(dirs[2])\n",
    "data_batch3 = unpickle(dirs[3])\n",
    "data_batch4 = unpickle(dirs[4])\n",
    "data_batch5 = unpickle(dirs[5])\n",
    "test_batch = unpickle(dirs[6])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 225,
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 846,
     "status": "ok",
     "timestamp": 1521001369662,
     "user": {
      "displayName": "Mayur Kulkarni",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "116301151646646358552"
     },
     "user_tz": -330
    },
    "id": "PqPF81F3vQZH",
    "outputId": "7cd33046-192a-4691-9e31-f7f68f4a9c54"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{b'label_names': [b'airplane',\n",
       "  b'automobile',\n",
       "  b'bird',\n",
       "  b'cat',\n",
       "  b'deer',\n",
       "  b'dog',\n",
       "  b'frog',\n",
       "  b'horse',\n",
       "  b'ship',\n",
       "  b'truck'],\n",
       " b'num_cases_per_batch': 10000,\n",
       " b'num_vis': 3072}"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "qVSwi862vQZe"
   },
   "outputs": [],
   "source": [
    "# the shape of the images is stored in row major\n",
    "# that's why we need to transpose it according\n",
    "# to our channels\n",
    "\n",
    "X = data_batch1[b\"data\"].reshape(10000, 3, 32, 32).transpose(0, 2, 3, 1).astype(\"uint8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 283,
     "output_extras": [
      {
       "item_id": 1
      },
      {
       "item_id": 2
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 885,
     "status": "ok",
     "timestamp": 1521001376749,
     "user": {
      "displayName": "Mayur Kulkarni",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "116301151646646358552"
     },
     "user_tz": -330
    },
    "id": "LxIlXkJkvQZv",
    "outputId": "3d616d9d-4573-4509-f4f4-7b83d09e594e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32, 32, 3)"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAD5CAYAAAAOeCiTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztnXmYXFW16H/V1fM8pZPupJPOuA2E\nIQS4BAgERQKIcL0gDnzKVeCBAtcrouLwfMq9V7zwIT7BCfXKJIrecCHMswwiMsiUADskZE46Q6fn\n7uqurqr3x6kTUzl7nbRNUs3zrN/35UuftWufs2vXWbVPrbXXWrFMJoOiKH/fFIz3ABRF2f+ooitK\nBFBFV5QIoIquKBFAFV1RIoAquqJEgMKxdjTGXAccBWSAL1hrX5Be+/Nlj+b48M5cfBRL//AcABvf\nekm8xvY1bzrlqZQ87IlT3ye2TZ05N+f4n045jjsfeAqAuklTxX6lZe7rrVzxrNhn3arXxLZkb1/O\n8Te+fiX/8d1vARAPeW/VdTViW2FpuVN+5DHHiX1mzcmdq2mTJrCufTsAie6dYr8Vy18W29LpYad8\nOJkQ+7yx4vWc4y9d9r+59vv/BkBP1w6x39DwkNiWHI475Ts7BsQ+fQO5Y7z55ps499x/BmAkJV9r\nwoR6sa2uvlJsS2V6nfKRZO7xNd/7KV++4iIAEoOyO/yuOx+KSW1jWtGNMccDs621C4HzgB/+Lf3r\nq+U3n0/qa6vGewgAtLRMHu8hAFBSVDTeQwBg0qT3xnzMmDFjvIcAQGtr27s+x1gf3T8A3AVgrX0T\nqDPGVL/r0SiKsl8Yq6JPArbvdrw9K1MU5T1IbCxbYI0xNwL3WWvvzh4/A3zWWrvS9fqdPX2Z98rj\nuqL8HSP+Rh+rMW4zuSt4C7BFerFvePO54PQT+fmyR4HxNcad/4kP8Yvf3AeMrzHuRzf8kosvOQ8Y\nX2PcnNYWVm7YDIyvMe6aq3/Kl7/iGZ/G0xj39NNPsWiRN3/jaYz77a8f5OPnnAzs1Rgnto310f1h\n4CwAY8xhwGZrrXvUiqKMO2Na0a21zxpjXjLGPAukgYvDXt/TGVwdfFlDrfxtmJkw0S0vlO1+zVNl\nS2kqnRRlBWn5mz49MOKUJzo7xD6ZQXkFm9zYJMqmts4S+7XOmia2tUye4pQ3NbnnEKCoqCQgm9JY\nC8BIrfsJAaB1imyOGRlxr+iJxKDYp6uzLyBraZkOwI4d8pNFYXGp2EbMvaLXNQTfs09pRXCMjRO8\n+eju6RT7lZTKapTOuO8dgKJC91h6ursCsoEBb2zDQ2OLNh2zH91ae8VY+yqKkl90Z5yiRABVdEWJ\nAKroihIBVNEVJQKooitKBBiz1f1vIhl0a/my4SFHW5aBAberpm2OHPTQ198vtrk2bezo8Hby1jeG\nbEYpcn8fzp49R+xz9FGHi22TJwZdYWef8xkAamomiP2ShSmxrbzU7aopDPHGxEaCrh9fNtgfdHn5\nDLk+T38cZW63XF1t0KXoM3PGAaLszTet2I+YPI6hIbe7tKa6TuxTVByUVVeXAdDds1Xsl8F9nwKk\n0/IH0NnpvlcHB4Kbc3zZWHO56oquKBFAFV1RIoAquqJEAFV0RYkAquiKEgHyYnUfcQQ0+LLYiGxJ\nLikuc8q7d8ihiw2T3MEdAFMPDAaMzD90HgBNrS1ivyKXORaC8YS7kRyRg1re2pIbDDNrNrzV7skG\n3tnu6uKds0C27trXX3XKj5gbtGj7HHfkETnHZcDwsPd5hOUp6OnpFtvWr9vslBcXyQEoxcXBICVf\n1jhB9rCs3/C2fE4hbLdvUPbK9PQE76vObk9WWCSGelNdLQcADQ7KwVIpId5lZCQtykpKhHtxL+iK\nrigRQBVdUSKAKrqiRABVdEWJAKroihIBVNEVJQLkxb02NBB0afiyyjLZ7VJd7w7wOOyQQ8U+rTNm\ni229jiCOCRObAbDvbBD79Qy4XSR9XcHcXj4dXXI+uS3tufnHTlu0kN8/+AQA1SFBLRTImUjvvWOp\nU150tvxdfvzCYwOyWNwLjikqkl2HkybJrkgybtdnV6ecO/QvL+dmzD355ON3yQodee18Kqrk3IEj\nKbd7cLhP/szijqnyZWGZXlMp2e3ZsVN2BRfgdssVFgbV0pfV1srBV2Hoiq4oEUAVXVEigCq6okQA\nVXRFiQCq6IoSAVTRFSUCjMm9ZoxZDPweWJEVvW6tvVR6fUlJkShLxqvE6wyWuQvUremRy/u88szz\nYtvOjtw8aEeYNu687ykANm2Wc4IVxd2RS0UFwSgjnyGhNBFAIhFsS/R40W7NE+SPZFv7OrGtWohq\n6u3qEfusXLMm5/jw+QfukjU3N4r9iorkMTa3uss1tQhygPXtQddm26ys2/N12e3Z1Cy7IteuF9xa\nSfkzSw8H23xZKiRfX2mx7AIsKQze+z6DCfc5q6uDbsOqrCuxUCjjtDfejR/9SWvtWe+iv6IoeUIf\n3RUlArybFf0AY8wyoB74jrX2kX00JkVR9jGxsEwiEsaYycCxwO+AGcATwCxrrfOH6fYdHZkJjQ3v\nZpyKouwdMQ3OWOujbwLuyB6uNsa0A5OBNa7X/9ev78g5/uoXPs9//t8fewdFteJ1BoUvoZZJchL+\n7j45hdOexrirLvs0X/v+LUB+jXEDexjj7rj+Sj526bcAmD5zuthvW/tqse2lZ/7klC85/gSxz9ln\nfSTn+PD5B/Liy559dazGuHihe67kREzw4MOP5xx/8qNncfvv/xsA+7psXE07jLw+kjFupFveez7Q\nn1uL/e57nuCMD3vzV1Yup3CqqJINZNu2bRPbBhPuX857GuPu/O8H+aezTgagvFxOW3XbLXeKbWP6\njW6MOccYc3n270nARGDTWM6lKMr+Z6y/0ZcBtxtjzgCKgc9Jj+0A5eUTRdm2LiFDHrBqg9u18saK\n5WKfgpDVJuUo/7Rq5RsADPbKSQPjwso9OCS7rrp65bZeR7mj1179IwBrN74p9qsok12RZqZxN4Q8\nWfzx6T/kHB8+/8BdsmnT5SeLOUYuRdXQ4I6uKimVP5ea6uCK6MsKRuRElP1D8jrlKmsEMNglR9Gl\nUsGnwaGEJystk58e+nrkc1aHRNiVlMad8uHh4H06ko28HBAiKffGWB/de4EPj+mKiqLkHXWvKUoE\nUEVXlAigiq4oEUAVXVEigCq6okSAvCSHrK0Pbr7wZas2rBT7bVnr3H9DeZGcJLG7v1Ns6+sJbl5Y\n+Zrn1oql5c0vXb1BdxhA16C8OacwZDNH48SmgCxT4G0OKquSk/9NbjtEbGsVXDVrXnVvpAGIx4Ku\nt/bNnkszmZKjtbbvkBNfHnTQXKd81uwZYp9WRxSaL6s8ar7Y77W31ottQwl30tGhopDoNYKusCmt\nXu23dEZ2A7e3u+vNARSXyJtpauqC94FH0NVbXOyp6uCgHLkZhq7oihIBVNEVJQKooitKBFBFV5QI\noIquKBEgL1b31av3DDU8fpfsrdWrxH6bt7jDMlMhAShVNRVim5ndFpTN8GTz5s4T+23Z7rZ0rtsu\nj2PCpGAgj880Ryjq2WdfDEBVg2SJha2d8vUyO9weivXrZMv0dkfZqBeypZDmHiB244Nz3JZ1gP4+\n91ylZSM+meGg9d+XrXhO9hrMNnJpromT3eHPzz3/lNinfWswEGlgwJMlk7LVPTEoBw51hpSiKqt0\njzGdceSuy8r6HeXNRoOu6IoSAVTRFSUCqKIrSgRQRVeUCKCKrigRQBVdUSJAXtxrzz21R8r3r3x5\nl6xwopDrDJg59yCnvMxROsdn7gGzxTYzZ0pAdtppZwKQSriDQgAyBW6XUT9yRtHCIndQBUA8HnSr\n+LLkiBwE0d+7U2yrGXa7f0ZScjrv9duCAUC+rLRSzvVZUy1n4Z0xs80pz4SsKYNdwTxovuytP78i\n9ssMyvfBvCUnO+UHHSwH1wy+GHSv1dR5efpWr1or9isvd5cOA6ipDUtz7vY59vQEPxdfNjQ0tpxx\nuqIrSgRQRVeUCKCKrigRQBVdUSKAKrqiRABVdEWJAKNyrxlj5gF3A9dZa28wxrQCtwJxYAvwKWut\nmMht24agG8qXzT/kQ+J1S0qCucQA6mVPGM0tcgmcnY5yPDu7PNfZhlWy62o47XZ5FcTkkKx4oez6\nSWWCU7VLNhJWUkrOF5ZJua9XWSMXS+zoC0ZCpUu8+SsolqMA06EVeIU2eTqoLA1+Zr6sraVV7Fca\nl8dRgDvP30Hz5FJTtbVBt+dJJy4GYNngw2K/9i1ynsLJTS1iWyrmzjnoKmLZ0tIMQE+PXOorjL2u\n6MaYCuB64LHdxFcCP7LWLgJWAZ8d09UVRckLo3l0HwJOBXZPdbkYr9AiwD3Aift2WIqi7Ev2+uhu\nrR0BRozJ2cFWsduj+jageT+MTVGUfUQsE/p7668YY74N7Mj+Rt9mrW3KymcBt1hrj5b6rlq1KjNr\n1qx9MV5FUWRiUsNY97r3GWPKrLWDwGRyH+sDnH3m2TnHf3n1Lxx2yGEAzD/tXLHfvjbGJYdyjXH/\netEn+MFPfwOMzRjXm5GNY8UVZWLbpCm5BporLzqVb/30fgDiZbIRbNOGLWJb3eBWp/zFPz8p9lm3\nhzFu1XMPMeuoJQAcEFIf/YufO09smzWrzSkvLpYLWmx/642c43nHHs3yZ54F4OGffVfsVzPRnYoJ\nYM6Ji5zysjp5fjdszDWqfeTMS/mfpdcDsOyesRnjpk6T51Eyxg0P5xprb/nVMj79mdOBcGPcXUv/\nILaN1b32KHBm9u8zgQfHeB5FUfLAXld0Y8wC4FqgDUgaY84CzgFuMsZcCKwDbg47R3llvSgrCvnl\n0NUVLKEEUFIvf5MPjMh+nITjC7Q3KyvLRik5r5cWnogSsnstEzKziWQwAsmXlZbJHQscJZR80gXu\nfpUNsnunOBN8iimu8KKt4mVyhFqmWH6kSsfc0VWxlLySFsSDY/dlRRXFYr+ySrltZMidlLFjk/vJ\nB6ChIvgE2VDhjeOMU5eI/V58da3Y1heSODIxtN0pH3KUXfLdp7VV8r0fxmiMcS/hWdn35INjuqKi\nKHlHd8YpSgRQRVeUCKCKrigRQBVdUSKAKrqiRIC8JIdsnhrcNODLYgXyd00i4d4csLVHHnZxrRyt\nlRwJumM6s7JYkbyhY7DPHQmVzMhjLyyUkzyOxINtvqy8Wt7w09TQJbZldro37wyH1AyLpYPj92Vl\nZfKGn4KQDUvpjPt6qZTsiiwoCp7Ql2Xi8hz39ct1zWJpt5u1JOR+69kedL35srLyoIvY57iFB4tt\ndvU6sW35G+1OeV9PMKrQlxWHJB0NQ1d0RYkAquiKEgFU0RUlAqiiK0oEUEVXlAigiq4oESAv7rVM\nLOg+8WXJEPfPQK/bfVIS4vrp7QmJK08EkzL2ZF8/0CO7aoqE4LWqCtmFNqFOdsdU1wcjuVqzsgm1\n8ntLFdaIbYMl7nncOU2OXhtKBePbJ1Zno/gcEXa7xjESEkUnRPqlCuSowpjDvebLauvlKLp0KmSM\nwn1VUyPPb3EsGEpZX+3VVevqDXFtJt3uV4BD504S22qr3PfPvfcGY9+Lsz7N7Vvlen9h6IquKBFA\nFV1RIoAquqJEAFV0RYkAquiKEgHyYnXHZaXNygrTsgW3Rti/31ojZrXlfTPknFqVpUGL60mHtQEQ\nj8nfef09botrYqBb7FNWkRTbzOygRf6YrKx12hSxX0HRNLGtr8s9xtZmOeW+WRPMyfePJy8GoLpe\nDp6or5MDbwoL3Xnc0iG5ATOOIBlfVlpRLvYbScgemwLhekVhQVQEvTJpvGs0NFaK/foGZOt/f5c7\ncAVg8gR3luN//PBJouyu+x4VzxeGruiKEgFU0RUlAqiiK0oEUEVXlAigiq4oEUAVXVEiwKjca8aY\necDdwHXZaqo3AQuAjuxLrrHW3if1P37hAlE244BDxOtu3rTJKZ/cIgeMzJk9U2ybNKEpIDv1/QsB\niGdkl12vENAwFBL4ESuQz1dZEQxqmTNtotdWKbu14sVyQEaR4KYc7HeX/QE4bF7QXefL2ua0if2S\nadl1mBHWjpG07ArLxINz5cviRfItmkzIPru0ENRSUCivbbHS4Dh2yUL6DSXl+SiMy7kIU8Pu+2qC\nw5U3odFzMx676AjxfGGMpvZaBXA98NgeTV+z1t47pqsqipJXRvPoPgScyl5KIyuK8t5lNEUWR4AR\nY8yeTZcYYy4DtgGXWGvHFiirKMp+J5bJhOxN3A1jzLeBHdnf6B8AOqy1rxhjrgCmWGsvkfp27OjI\nNDQ27JMBK4oiIhqGxrTX3Vq7++/1ZcBPwl5/+62/zTm+9IsXc/11PwLG1xhXWV1JX4+XHWQ8jXG1\njc107fCyvVRWyvvIw4xxnd3uB6rHH/+D2GdS09Sc42MXHcUzTz8HjN0YF3NkE4LwAg7De2QFmjv3\nEN5881UA3rj/FrFfordDbJs0a4ZT3jRZztLTM5zIOT5myVf540P/CYQX5OjY0Sm2hRnjYjG3+sWK\nc41xJ374Eh695wYA3nwnmBXI59Iv/IfYNib3mjFmqTHGn8nFwPKxnEdRlPwwGqv7AuBaoA1IGmPO\nwrPC32GMGQD6gM+EnWPBwe8TZQfOl1f0wXnu1bmiRl715MxkkIm53Djed11ByDdvfYU771dIRabQ\nb9C0o1xQRbaU1EhIDj1C3DhDQ+6STDNnTXXKAcqKg26+xkZPNtgvR+ZlCkJuG2GVyjjysfmkHT8f\nfVnK8Zntek1ISNzwoHs+Uunge/YpKAxey5cVhHyivR3yk926NRvEtmOOne+UDySD+QszWVm5wwU4\nGkZjjHsJb9Xek6VjuqKiKHlHd8YpSgRQRVeUCKCKrigRQBVdUSKAKrqiRIC8JIcsc0Rr+bLKUnkj\nQkW5MLxC96YMCE9CGHO4akpLPLdaQZgbJ+N22qWTsjPP5TLaNQ5HgsJMduAjIQ7CkD04ZITklpW1\n8uaikVTwWpnsfKTS8hwjlF0CyODeGFMQNviUoy0rSxXKbs8MIR+2UDYqlpY37pQ43nPJiCcrSslr\nYkVCnqvMVrebD2D7O1ud8ikmmCC0LqsnOwrk8k9h6IquKBFAFV1RIoAquqJEAFV0RYkAquiKEgFU\n0RUlAuTFvVZVE3Tx+LJMSNTYwJDbRZIZCtbI8hkS+gD09/XnHE+fMZ2N6zcCMJyU+w0NuaPGRkZk\nV1gyJNIsuce1jj12Ic8//xIAAyF1vAb6g1FNu8biiIgDqKqX46+raoJ16jq7vDmqrWoU+5UWu+ur\nAaSkWnqxkDppBNt8WVWVnCyzY5v8mSUG3W6odLpO7BMj+L5iWXdhOiXfc9VVsot42tSJYtvgQL9T\nnnEk0vRlNVVy9F0YuqIrSgRQRVeUCKCKrigRQBVdUSKAKrqiRIC8WN3vWvZAzvHlc+fskqWKnhb7\ndXa6N/33CRlPAQpC4hz2tMj/7MYb+N73rgVg61b3tQBSQqRMvaPEk09dSHrrknjutB977ELuue9h\nAPp3ujPOAqx8+02xrafPbWVunR4su+QTL8r1eBx920388PvXA1BdJY9/+nQ5D92UVnd+vekzJot9\n6kscufxSXkbWqlLZK5MOyR1I3B1okkzJ1v+4o+xSOustiDvG6DOxLcRDUS1b5JMZd4BN3OHU8GX1\n9SHvOQRd0RUlAqiiK0oEUEVXlAigiq4oEUAVXVEigCq6okSAUbnXjDFXA4uyr78KeAG4FYgDW4BP\nWWvFXf+PPPFszvHlX/3CLlntlEA55l1kUm6X0cvPPiH2mTYlmG/Lp7Eh6DLq7vKusWlju9hvRMgz\nVl4fDArxGS6QA162bgyW6VmflX3gyIViv0MPPlBsGxhKOOUFRfJHvGb9uoBscnMzACvfXi32e335\ny2JbbU2lU37mWR8R+xxz4JyALJaNCSoOqXs1pblVbBsW3GthxS/dpaG8/5NCLjyAgsKQPHS1clBO\nmSN3IEA6HgzW8T2ysrMxnL2u6MaYE4B51tqFwMnAD4ArgR9ZaxcBq4DPjvH6iqLkgdE8uj8FfDT7\ndxdQgVeLbVlWdg9w4j4fmaIo+4zRFFlMAX7g7HnA/cCS3R7VtwHN+2d4iqLsC2KZkPzju2OMOQP4\nOnAS8La1tikrnwXcYq09Wuq7du2GTFub/HtKUZR9gmiAGK0xbgnwDeBka223MabPGFNmrR0EJgOb\nw/pfeNHlOccPPXgHS07+GDC+xrjf/u4mPn72PwOwerVsfJKMcXMOPkDs09AsZxbp3JS7r/43t/6C\nT3zqfCDcGBe2kX9fGOOuveYqvvTlrwHhxrgdHXKswb4wxs094hjefOGPAHS9LcdClKTlLD6SMS5e\nF1JIYo8a7kcs+TovPPRdILw+ekmRbHBLhRT5KBilMe6Q93+FVx+/GoARysXzLXj/JfK1xJYsxpga\n4BrgNGvtzqz4UeDM7N9nAg/u7TyKoowfo1nRPwY0Ar8zZtfqey7wC2PMhcA64OawE3z0E58WZSVN\ns8V+A71ul9fbr78q9mmeJP9EcH2DlpR4K1BZqRwVNJx2l9WZM08ee12zHNk20BjMW3bgIe8D4LRT\nZLtmeVWZ2NYvrOgh1ZMYcZSa+pdLLwQgMeI+H8C2bTvFtnVr3A935eXy/LZv7Mg5nnvEX2VrV7wt\n9itIyGN8p32bU37kSYeLfaa1tQRkZaVVQHjUW0GpnEOPItn1FnPkhvMagn18F15xTH5CCGM0xrgb\ngRsdTR8c0xUVRck7ujNOUSKAKrqiRABVdEWJAKroihIBVNEVJQLkJTlkSbHDrZWVrXxrudivp9vt\nXgvbzZcclsv09PUFS+B07PDcOLGY7IcqLXHHDCUH5BJJ3dvlMW5dH4xe27h+DQAPPPRAoM2nszfk\nen3dTnlVtezWqqnLLZV1wQXn8/BDjwJQEZLUcONGeX9UU6M7CWRptexufPq+3Pd8wkdO5+lHvE1R\nO99+TeyXGpY3zKxqdyf73BhS1mr23Fx36bzj4d4HnwegplreqFJTJ5e9KiuXN9PUVLjvq6LS4Gaf\nrl7vfiovlz+XMHRFV5QIoIquKBFAFV1RIoAquqJEAFV0RYkAquiKEgHy4l7r7Qi6yXzZ43ffJ/bb\n0L7RKS9IuqPJAF57rUceiMOF9s6aVQCMjMjRSQgRQ4/c+7jYpbhIdoMcOv+w4CVSnhtmuLhK7Ncz\nNCC2vbPeHa3V0SHXaxtO5L6vCy44nzt+630em9vXiv3WrJXPefj8BU75v1x8mdjn+ef+JMpGujsC\nbT49Q2I+UgZxuzffeTHo2vR5+qUtOcdXXAX/ddtTAFQUyq68omJ37DtAvES+D6oE99qUaW05x4s+\nBL+61YvLP+PMj4vnc8+8h67oihIBVNEVJQKooitKBFBFV5QIoIquKBEgL1b35onBtO++bHbbdLFf\nBre1uzCk3FE8JDilIB78XmueOtW7VloOQikurXA3hGT/bGlxB3cALF6yxCE7C4Cq8pDgidJgrjmf\nN5a78+itXCVnc500uS0gS2QDhhIhpZDiZfIYl698yz2+lSvFPuVtc0XZ5s3ye66rlduait153Mor\n5bx7O9uDJapmzPbKYHVsWiX2277DHUADkEiFBGAJCf22dAXV8oVXPQ/U0R8ISQIYgq7oihIBVNEV\nJQKooitKBFBFV5QIoIquKBFAFV1RIsBoiyxeDSzKvv4q4HS8PfR+xME11loxOmXn9mAJH1921D+I\nRVg5+vjjnfKSEjmIoNDhQvNxlWQ6/4KLAEg7yhP5xHFfLzksl9sZHJYDUDo2rtlDsnCXbGdCDp7Y\nuUMuhfSO4EbbvM2ddw+gsilYgqgvmc25VyK7DmPFsntteMQdaPLIk8+IfabNPCggK2rwilS21stu\nytIC+fYtF4KKhhJyzrh3elYEZF09XqHPyio5914qIwdEtXe6C4UCNDa2OeUDjsKMA0nv/Tz+5PPi\n+c6/IFj6zGevim6MOQGYZ61daIxpAF4GHge+Zq29d2/9FUUZf0azoj8F+F8jXUAFCEucoijvSUZT\nZDEF+HmSzwPuB1LAJcaYy4BtwCXWWrlotqIo40osLEf67hhjzgC+DpwEHA50WGtfMcZcAUyx1opV\n2Ds7ujJ1DbX7YryKosiI+2NHa4xbAnwDONla2w08tlvzMuAnYf3vvOP+nOPzPv9Jfvnj2wFIxuS9\nxwWl7gwc+8oY94mPfpDf/P4RIL/GuJFEboacz3/+k/w4Ox+xMRrj/ueBpU75G2vlPdpz5uVmunnl\n6fs5dNGpAPQIBSEAtm8N7gn3SQvGuPnzjhT77GmM+/XPv8c5F1zhnS8j36L72hi3/NVcg+FzLz7F\nUYcfB0AZ8ufZ3SN/LmHGuGrBGJfcwxi3etWLzJzl1XX/h6OOEs93+203iG17da8ZY2qAa4DTrLU7\ns7KlxpgZ2ZcsBuRyK4qijDujWdE/BjQCvzPG+LJfAXcYYwaAPuAzYSeocJSR8WUdPQmx38uvveSU\nNzXJUUsTmxrFtmQyuFq2b/JWp87OLrEfCfcYC9Py6jt5etB15dNaF8wLNz37ljat3BJo8+nvk3Ok\nNU2c5JSXh/xkipcGXUZ1Nd7rBwblz6W5earY1r7ZnedvR4f8hNDcEiyVNTzoyWIhPy37huT5p9C9\noifT8lNYSVkwStGXlYRERQ53bJfHUeB+KgWY6IgeBBgeCpYVa2qeAsAof2kHGI0x7kbgRkfTzWO7\npKIo+UZ3xilKBFBFV5QIoIquKBFAFV1RIoAquqJEgLwkhywpCm5G8WVDCdmt9eyzjznlmaTs+qku\nlzfgJJO5UUZf/Nfzuf1XnkMhMSiXeSoUvg+ntbWKfeYddYDYNnNq0PU2c5Z3rq4NbvcUQHunvMu4\nuMztTprZ4Ha7AWzfHtzMURH3kioeZOaJ/Q48yIhtv73tFqe8EHeyRoBkf/Dz9GXDw/JnnRmRXWWU\nuiPKwkoktU2fIcq2bbDytQrkDVxlFfL15s6d45QnBoKfi5nTBkBrc5M8jhB0RVeUCKCKrigRQBVd\nUSKAKrqiRABVdEWJAKroihIB8uJeGxgMxvLukjkSNvosOeU0pzw9HIx28okn5UR96VTQzXfcMV68\ncSYuu0jihW7XUGmFnCSxvUshDQSfAAAHnUlEQVR21/V25dYhm3PEIv70mifbOSiPP1YqJ2y0r7zj\nlHf8SY6smjE96Cbbsc17/RGzZov9hkMi28qK3e6kjCNy0McVKefLCuLyLSqULgNgMC3U7UvJ8ztt\nStC9NmmK5/ZM9HUE2nwOqBZq8wHPv/Sy2LZ5ndtlN9gfvL83rfXuj8xAp3i+MHRFV5QIoIquKBFA\nFV1RIoAquqJEAFV0RYkAquiKEgHy4l6rqAy6p3xZTUiyu6oJ7uieoSE5SWJpyHdXcSw4jkPme+lz\nM2Vy1FtJudu9lk7IqXx7e3vEtnh5MCljvNiTNc2UkznOLJej195e4669Rkx2GxY5knb6sk1b1ov9\nGhrl5JxSm5/s0cXQUDBxpC/rd0S27XqNI8rLJznkTs9cWCq7RCe2TAjIunq99NDrtmwV+21dL8w9\nkAhJm716xStOeUNDcBxDvV6UZ6auXjxfGLqiK0oEUEVXlAigiq4oEUAVXVEigCq6okSAvVrdjTHl\nwE3ARKAU+DfgVeBWvDrpW4BPWWtFU/hA70pZlpa/a4pilU751q2yJfPtN9aKbaWFuZb1T19wFn98\nystLV1wjW7sbhRJQLY01Yp/CkGCdhpoGh8wbmyPuZheJQTmgoakpaMkHmNwiW2m3tLcHZPG4Vw5o\n5co3xX5tw9PFNskj0tsrf2YDA0GL9qZNawDo6Za9F2FW99SwO6goXiIHoKxYHizntWL564C7TJJP\nU9NEsW3ywXLuvaYJ7n6NE4J5/o5bdDwApSHjD2M0K/qHgRettccDZwPfB64EfmStXQSsAj47pqsr\nipIXRlN77Y7dDluBjXgVVC/Kyu4BLmcvpZMVRRk/Rr1hxhjzLDAFOA14dLdH9W1A834Ym6Io+4hY\n5m+ow2qMORS4BWi21k7IymYBt1hrj5b6de3clqmtH1s+akVRRo2YimM0xrgFwDZr7QZr7SvGmEKg\n1xhTZq0dBCYDm8POcf/S3Kf6T17wf7j9598BYDDEGBcvcxvjNq7bN8a4n936Ey781OeAfW+MKwgx\nxrW05D4AnXL6aTyw7F4g3Bj3/GvLxbY33nrLKS8qlD/iPY1xf3j4Xhaf5GX12bFTNoK1tcnGuM7t\n7q2ivd1yhpaBgdxtrqtWvs6sOQcB+TXGHbRgYc7xM08s49gTTvfGGDL+ooxsqJs8Kbid1We0xrh/\nv/YqvvmlrwHhxrhvfvebYttojHHHAV8CMMZMBCqBR4Ezs+1nAg+O4jyKoowTo/mN/lPgl8aYp4Ey\n4GLgReAWY8yFwDrg5rATpB1ldXxZQch3TWHSHZBR7Sjx5PPSc0+Kbe1bc4NCfnbrT1i69DYAYkVy\n6Zwjj1zglB+78HCxT3e3/NTx2l/+nHN8yumn8eC9dwHQn5CDOFau3yC2vbN2rVM+OOAO7gDIZIJP\neq+9/DwApdXyStTT0yu29Qplo/p7ZNeg63lzwzrPvVYYlxPD1VTJASot091PHXUNsjmpqSXo1pqW\nlbXMP0jsVx+SM644LBeh1OYIRGpszP70zYxt68torO6DwCcdTR8c0xUVRck7ujNOUSKAKrqiRABV\ndEWJAKroihIBVNEVJQL8TTvjFEX5/xNd0RUlAqiiK0oEUEVXlAigiq4oEUAVXVEigCq6okSAvJRk\n8jHGXAccBWSAL1hrX8jn9bNjWAz8HliRFb1urb00z2OYB9wNXGetvcEY08rfkGxzP47jJmAB4Adf\nX2OtvS8P47gaWIR3P14FvMD4zMee4zidPM7HvkjEKpG3Fd0Yczww21q7EDgP+GG+ru3gSWvt4uy/\nfCt5BXA98Nhu4rwn2xTGAfC13eYmH0p+AjAve1+cDPyA8ZkP1zggv/Ox3xKx5vPR/QPAXQDW2jeB\nOmOMO0fx3zdDwKnkZuVZDCzL/n0PcOI4jWM8eAr4aPbvLqCC8ZkP1zjkYPL9gLX2Dmvt1dnD3ROx\nvuu5yOej+yTgpd2Ot2dlcq6g/ccBxphlQD3wHWvtI/m6sLV2BBgxxuwursh3sk1hHACXGGMuy47j\nEmutXMJ134wjBfilVs8D7geWjMN8uMaRIs/zAfsnEet4GuPk1CH7l7eB7wBnAOfiZc9x10UeH8Zr\nXsD7LXiFtfb9wCvAt/N1YWPMGXgKdskeTXmdjz3GMS7zkU20ejpwG7nvf8xzkU9F34y3gvu04BkX\n8oq1dlP2ESljrV0NtOMluBxP+owxfubKvSbb3F9Yax+z1vpFu5cBcv6kfYgxZgnwDeAUa2034zQf\ne44j3/NhjFmQNcySve6uRKzZl4x5LvKp6A8DZwEYYw4DNltr5eRj+wljzDnGmMuzf0/Cs3Buyvc4\n9uA9kWzTGLPUGDMje7gYkNPO7rtr1gDXAKdZa3dmxXmfD9c4xmE+9lsi1rxGrxljvof3ZtLAxdba\nV/N28b+OoQq4HagFivF+o9+fx+svAK4F2oAk3pfMOXhulVK8ZJufsdYmx2Ec1wNXAANAX3Yc2/bz\nOP4X3iPx7gX6zgV+QX7nwzWOX+E9wudlPrIr9y/xDHFleD8xX8SrpfCu5kLDVBUlAujOOEWJAKro\nihIBVNEVJQKooitKBFBFV5QIoIquKBFAFV1RIoAquqJEgP8HWlydiEs7TMAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9b4c7732e8>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Show random image\n",
    "\n",
    "plt.imshow(X[1])\n",
    "X[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "DJrZ0e4uvQaK"
   },
   "outputs": [],
   "source": [
    "class CIFARSetup():\n",
    "    def __init__(self):\n",
    "        self.training_images = None\n",
    "        self.training_labels = None\n",
    "        self.testing_images = None\n",
    "        self.testing_labels = None\n",
    "        self.train_batches = [data_batch1, data_batch2, data_batch3,\n",
    "                        data_batch4, data_batch5]\n",
    "        self.test_batch = test_batch\n",
    "        self.training_size = 0\n",
    "        self.yield_counter = 0\n",
    "        self.consolidate()\n",
    "        \n",
    "    def one_hot_encode(self, inp, columns=10):\n",
    "        rows = len(inp)\n",
    "        one_hot = np.zeros((rows, columns))\n",
    "        one_hot[range(rows), inp] = 1\n",
    "        return one_hot\n",
    "    \n",
    "    def consolidate(self):\n",
    "        # consolidate all the images in rows\n",
    "        self.training_images = np.vstack((d[b\"data\"] \n",
    "                                          for d in self.train_batches))\n",
    "        train_length = len(self.training_images)\n",
    "        self.training_size = train_length\n",
    "        # Reshape and normalize\n",
    "        self.training_images = self.training_images \\\n",
    "                                   .reshape(train_length, 3, 32, 32) \\\n",
    "                                   .transpose(0, 2, 3, 1) / 255\n",
    "        \n",
    "        # consolidate all image labels in column (horizontally)\n",
    "        labels_in_column = np.hstack((d[b\"labels\"] \n",
    "                                        for d in self.train_batches))\n",
    "        \n",
    "        # one hot encode them\n",
    "        self.training_labels = self.one_hot_encode(labels_in_column)\n",
    "        \n",
    "        # Testing\n",
    "        # consolidate all the images in rows\n",
    "        self.testing_images = self.test_batch[b\"data\"]\n",
    "        test_length = len(self.testing_images)\n",
    "        \n",
    "        # Reshape and normalize\n",
    "        self.testing_images = self.testing_images \\\n",
    "                                   .reshape(test_length, 3, 32, 32) \\\n",
    "                                   .transpose(0, 2, 3, 1) / 255\n",
    "        self.testing_images = self.testing_images.reshape(test_length,\n",
    "                                                         32, 32, 3)\n",
    "        # consolidate all image labels in column (horizontally)\n",
    "        labels_in_column = self.test_batch[b\"labels\"]\n",
    "        \n",
    "        # one hot encode them\n",
    "        self.testing_labels = self.one_hot_encode(labels_in_column)\n",
    "    \n",
    "    def next_batch(self, batch_size):\n",
    "        # we need to reshape it to (32, 32, 3) since that'll\n",
    "        # be the format for conv2d\n",
    "        batch_x = self.training_images[self.yield_counter: \n",
    "                                       self.yield_counter + batch_size] \\\n",
    "                      .reshape(batch_size, 32, 32, 3)\n",
    "        batch_y = self.training_labels[self.yield_counter: \n",
    "                                       self.yield_counter + batch_size]\n",
    "        self.yield_counter = (self.yield_counter + batch_size) % self.training_size\n",
    "        return batch_x, batch_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "Rx6U8cDIvQah"
   },
   "outputs": [],
   "source": [
    "class ConvolutionalNeuralNetwork:\n",
    "    def __init__(self):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        X : Input tensor with dimensions: \n",
    "            [batch, height, width, channels]\n",
    "        \"\"\"\n",
    "        self.X = tf.placeholder(tf.float32, \n",
    "                                shape=[None, 32, 32, 3])\n",
    "        self.y = tf.placeholder(tf.float32, shape=[None, 10])\n",
    "        self.dropout_prob = tf.placeholder(tf.float32)\n",
    "        self.cifar_setup = CIFARSetup()\n",
    "    \n",
    "    def init_weights_and_bias(self, weights_shape, bias_shape):\n",
    "        \"\"\" Randomly initializes biases and weights\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        weights_shape : shape of the weights Tensor.\n",
    "        \n",
    "        bias_shape : shape of the bias Tensor.\n",
    "        \"\"\"\n",
    "        rand_dist = tf.truncated_normal(weights_shape, stddev=0.1)\n",
    "        W = tf.Variable(rand_dist)\n",
    "        consts = tf.constant(0.1, shape=bias_shape)\n",
    "        b = tf.Variable(consts)\n",
    "        return W, b\n",
    "        \n",
    "    def conv2d(self, X, kernel):\n",
    "        \"\"\"Performs 2D convolution\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        X : Input tensor with dimensions: \n",
    "            [batch, height, width, channels]\n",
    "            \n",
    "        kernel : Kernel tensor with dimensions:\n",
    "            [filter height, filter width, channels in, \n",
    "                channels out]\n",
    "        \"\"\"\n",
    "        # strides indicate the step of CNN to move in \n",
    "        # either direction or the number of steps to \n",
    "        # take in every dimension\n",
    "        strides = [1, 1, 1, 1]\n",
    "        return tf.nn.conv2d(X, kernel, strides, padding=\"SAME\")\n",
    "    \n",
    "    def max_pooling(self, X):\n",
    "        \"\"\"Performs 2x2 Max Pooling\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        X : Input tensor with dimensions: \n",
    "            [batch, height, width, channels]\n",
    "        \"\"\"\n",
    "        # ksize is the size of the window for \n",
    "        # each dimension of the inpuy since we are \n",
    "        # performing 2x2 pooling and our dimensions \n",
    "        # are [batch, height, width, channels] we\n",
    "        # need 2 for height and width\n",
    "        ksize = [1, 2, 2, 1]\n",
    "        \n",
    "        # same holds for strides, the only \n",
    "        # difference is strides is the step. Since we are\n",
    "        # taking 2x2 pooling we need to increment our\n",
    "        # step by 2 after every max pooling step.\n",
    "        strides = [1, 2, 2, 1]\n",
    "        return tf.nn.max_pool(X, ksize=ksize, strides=strides,\n",
    "                              padding=\"SAME\")\n",
    "        \n",
    "    def convolution_layer(self, X, shape):\n",
    "        \"\"\"Creates a Convolutional Layer\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        X : Input tensor with dimensions: \n",
    "            [batch, height, width, channels]\n",
    "        \n",
    "        shape : Shape of the Kernel with dimensions:\n",
    "            [filter height, filter width, channels in, \n",
    "                channels out]\n",
    "        \"\"\"\n",
    "        W, b = self.init_weights_and_bias(weights_shape=shape,\n",
    "                                          bias_shape=[shape[3]])\n",
    "        return tf.nn.relu(self.conv2d(X, W) + b)\n",
    "    \n",
    "    def dense_layer(self, input_layer, size):\n",
    "        \"\"\" Creates a Fully Connected Layer\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        input_layer : Tensor of the previous layer.\n",
    "        \n",
    "        size : output size of the fully connected layer. \n",
    "        \n",
    "        Returns\n",
    "        -------\n",
    "        Tensor output of the fully connected layer. \n",
    "        \"\"\"\n",
    "        input_layer_size = int(input_layer.get_shape()[1])\n",
    "        W, b = self.init_weights_and_bias(weights_shape=\n",
    "                                          [input_layer_size, size],\n",
    "                                          bias_shape=[size])\n",
    "        return tf.matmul(input_layer, W) + b\n",
    "        \n",
    "    def forward_pass(self):\n",
    "        \"\"\" Performs a forward pass and outputs probabilities \"\"\"\n",
    "        # Reshape the image (None, 784) back to (-1, 28, 28, 1)\n",
    "        X_image = self.X\n",
    "        \n",
    "        ################### Layer 1 (CNN) ###########################\n",
    "        \n",
    "        cnn_layer_1 = self.convolution_layer(X_image, \n",
    "                                             shape=[4, 4, 3, 32])\n",
    "        cnn_layer_1_pooled = self.max_pooling(cnn_layer_1)\n",
    "        \n",
    "        ################### Layer 2 (CNN) ###########################\n",
    "        \n",
    "        cnn_layer_2 = self.convolution_layer(cnn_layer_1_pooled, \n",
    "                                             shape=[4, 4, 32, 64])\n",
    "        cnn_layer_2_pooled = self.max_pooling(cnn_layer_2)\n",
    "        \n",
    "        ################### Layer 3 (Dense) ##########################\n",
    "        \n",
    "        cnn_layer_2_flat = tf.reshape(cnn_layer_2_pooled, (-1, 8*8*64))\n",
    "        dense_layer_1 = tf.nn.relu(self.dense_layer(cnn_layer_2_flat, 1024))\n",
    "        \n",
    "        ######################## Dropout #############################\n",
    "        \n",
    "        dense_layer_1_drop = tf.nn.dropout(dense_layer_1, \n",
    "                                           keep_prob=self.dropout_prob)\n",
    "        \n",
    "        ######################## Predict #############################\n",
    "        \n",
    "        return self.dense_layer(dense_layer_1_drop, 10)\n",
    "    \n",
    "    def train(self):\n",
    "        \"\"\" Trains the CNN using AdamOptimizer \"\"\"\n",
    "        y_pred = self.forward_pass()\n",
    "        cross_entropy_loss = tf.reduce_mean(\n",
    "            tf.nn.softmax_cross_entropy_with_logits(labels=self.y,\n",
    "                                                    logits=y_pred))\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=0.001)\n",
    "        train_cnn = optimizer.minimize(cross_entropy_loss)\n",
    "        init = tf.global_variables_initializer()\n",
    "        steps = 2001\n",
    "        \n",
    "        with tf.Session() as sess:\n",
    "            sess.run(init)\n",
    "            for step in range(steps):\n",
    "                start_time = time()\n",
    "                X_batch, y_batch = self.cifar_setup.next_batch(50)\n",
    "                sess.run(train_cnn, feed_dict={self.X: X_batch, self.y: y_batch,\n",
    "                                               self.dropout_prob: 0.5})\n",
    "                matches = tf.equal(tf.argmax(y_pred, 1), tf.argmax(self.y, 1))\n",
    "                accuracy = tf.reduce_mean(tf.cast(matches, tf.float32))\n",
    "                result = sess.run(accuracy, \n",
    "                                  feed_dict={self.X: self.cifar_setup.testing_images, \n",
    "                                             self.y: self.cifar_setup.testing_labels,\n",
    "                                             self.dropout_prob: 1.0})\n",
    "                if step % 100 == 0:\n",
    "                    print(\"Step: {0} Accuracy: {1:.2f}% Time: {2:.2f} secs/step\"\n",
    "                          .format(step, result * 100, time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 382,
     "output_extras": [
      {
       "item_id": 21
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1424860,
     "status": "ok",
     "timestamp": 1521002836357,
     "user": {
      "displayName": "Mayur Kulkarni",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "116301151646646358552"
     },
     "user_tz": -330
    },
    "id": "LQWnZ-DsvQbF",
    "outputId": "b9dd32a0-3fff-4cb7-a9a9-82493f9c275a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 0 Accuracy: 10.05% Time: 0.69 secs/step\n",
      "Step: 100 Accuracy: 33.49% Time: 0.54 secs/step\n",
      "Step: 200 Accuracy: 40.57% Time: 0.55 secs/step\n",
      "Step: 300 Accuracy: 44.67% Time: 0.64 secs/step\n",
      "Step: 400 Accuracy: 45.77% Time: 0.58 secs/step\n",
      "Step: 500 Accuracy: 49.31% Time: 0.61 secs/step\n",
      "Step: 600 Accuracy: 51.97% Time: 0.63 secs/step\n",
      "Step: 700 Accuracy: 52.69% Time: 0.65 secs/step\n",
      "Step: 800 Accuracy: 53.75% Time: 0.67 secs/step\n",
      "Step: 900 Accuracy: 54.67% Time: 0.68 secs/step\n",
      "Step: 1000 Accuracy: 57.66% Time: 0.71 secs/step\n",
      "Step: 1100 Accuracy: 55.69% Time: 0.72 secs/step\n",
      "Step: 1200 Accuracy: 56.11% Time: 0.74 secs/step\n",
      "Step: 1300 Accuracy: 58.17% Time: 0.77 secs/step\n",
      "Step: 1400 Accuracy: 59.64% Time: 0.78 secs/step\n",
      "Step: 1500 Accuracy: 59.46% Time: 0.81 secs/step\n",
      "Step: 1600 Accuracy: 60.85% Time: 0.83 secs/step\n",
      "Step: 1700 Accuracy: 61.44% Time: 0.87 secs/step\n",
      "Step: 1800 Accuracy: 59.50% Time: 0.93 secs/step\n",
      "Step: 1900 Accuracy: 62.27% Time: 0.92 secs/step\n",
      "Step: 2000 Accuracy: 64.08% Time: 0.93 secs/step\n"
     ]
    }
   ],
   "source": [
    "ConvolutionalNeuralNetwork().train()\n",
    "# Warning: This will take an entire day to \n",
    "# train if not on a GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "xlraO91NvQbq"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "9. Convolutional Neural Networks (CIFAR-10).ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}