5_Minimal_Gated_Unit.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Minimal Gated Unit\n",
    "\n",
    "This is jupyter notebook for Minimal Gated Unit for RNN.\n",
    "The paper of this algorithms is [Minimal Gated Unit for Recurrent Neural Networks](https://arxiv.org/pdf/1603.09420.pdf)\n",
    "This `Minial Gated Unit` will be tested by `MNIST dataset`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# Import modules\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-086d30c7fab6>:4: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import MNIST data\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', validation_size=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters \n",
    "\n",
    "img_size = 28\n",
    "img_flat_size = img_size * img_size\n",
    "\n",
    "# labels: 0 - 9\n",
    "num_label = 10\n",
    "\n",
    "num_epoch = 10\n",
    "\n",
    "# Parameters for optimizer\n",
    "learning_rate = 5e-4\n",
    "epsilon = 1e-8\n",
    "\n",
    "batch_size = 512\n",
    "\n",
    "# Parameter for Minimal RNN\n",
    "rnn_size = 512\n",
    "step_size = img_size\n",
    "flatten_size = img_size\n",
    "\n",
    "validation_ratio = 0.1\n",
    "gpu_fraction = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADgpJREFUeJzt3X+MVfWZx/HPs1j+kKI4aQRCYSnEYJW4082IjSWrxkzVDQZHrekkJjQapn8wiU02ZA3/VNNgyCrslmiamaZYSFpKE3VB0iw0otLGZuKIWC0srTFsO3IDNTjywx9kmGf/mEMzxbnfe+fec++5zPN+JeT+eM6558kNnznn3O+592vuLgDx/EPRDQAoBuEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxDUZc3cmJlxOSHQYO5u1SxX157fzO40syNm9q6ZPVrPawFoLqv12n4zmybpj5I6JQ1Jel1St7sfSqzDnh9osGbs+ZdJetfd33P3c5J+IWllHa8HoInqCf88SX8Z93goe+7vmFmPmQ2a2WAd2wKQs3o+8Jvo0OJzh/Xu3i+pX+KwH2gl9ez5hyTNH/f4y5KO1dcOgGapJ/yvS7rGzL5iZtMlfVvSrnzaAtBoNR/2u/uImfVK2iNpmqQt7v6H3DoD0FA1D/XVtDHO+YGGa8pFPgAuXYQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EVfMU3ZJkZkclnZZ0XtKIu3fk0RTyM23atGT9yiuvbOj2e3t7y9Yuv/zy5LpLlixJ1tesWZOsP/XUU2Vr3d3dyXU//fTTZH3Dhg3J+uOPP56st4K6wp+5zd0/yOF1ADQRh/1AUPWG3yXtNbM3zKwnj4YANEe9h/3fcPdjZna1pF+b2f+6+/7xC2R/FPjDALSYuvb87n4suz0h6QVJyyZYpt/dO/gwEGgtNYffzGaY2cwL9yV9U9I7eTUGoLHqOeyfLekFM7vwOj939//JpSsADVdz+N39PUn/lGMvU9aCBQuS9enTpyfrN998c7K+fPnysrVZs2Yl173vvvuS9SINDQ0l65s3b07Wu7q6ytZOnz6dXPett95K1l999dVk/VLAUB8QFOEHgiL8QFCEHwiK8ANBEX4gKHP35m3MrHkba6L29vZkfd++fcl6o79W26pGR0eT9YceeihZP3PmTM3bLpVKyfqHH36YrB85cqTmbTeau1s1y7HnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgGOfPQVtbW7I+MDCQrC9atCjPdnJVqffh4eFk/bbbbitbO3fuXHLdqNc/1ItxfgBJhB8IivADQRF+ICjCDwRF+IGgCD8QVB6z9IZ38uTJZH3t2rXJ+ooVK5L1N998M1mv9BPWKQcPHkzWOzs7k/WzZ88m69dff33Z2iOPPJJcF43Fnh8IivADQRF+ICjCDwRF+IGgCD8QFOEHgqr4fX4z2yJphaQT7r40e65N0g5JCyUdlfSAu6d/6FxT9/v89briiiuS9UrTSff19ZWtPfzww8l1H3zwwWR9+/btyTpaT57f5/+ppDsveu5RSS+5+zWSXsoeA7iEVAy/u++XdPElbCslbc3ub5V0T859AWiwWs/5Z7t7SZKy26vzawlAMzT82n4z65HU0+jtAJicWvf8x81sriRltyfKLeju/e7e4e4dNW4LQAPUGv5dklZl91dJ2plPOwCapWL4zWy7pN9JWmJmQ2b2sKQNkjrN7E+SOrPHAC4hFc/53b27TOn2nHsJ69SpU3Wt/9FHH9W87urVq5P1HTt2JOujo6M1bxvF4go/ICjCDwRF+IGgCD8QFOEHgiL8QFBM0T0FzJgxo2ztxRdfTK57yy23JOt33XVXsr53795kHc3HFN0Akgg/EBThB4Ii/EBQhB8IivADQRF+ICjG+ae4xYsXJ+sHDhxI1oeHh5P1l19+OVkfHBwsW3vmmWeS6zbz/+ZUwjg/gCTCDwRF+IGgCD8QFOEHgiL8QFCEHwiKcf7gurq6kvVnn302WZ85c2bN2163bl2yvm3btmS9VCrVvO2pjHF+AEmEHwiK8ANBEX4gKMIPBEX4gaAIPxBUxXF+M9siaYWkE+6+NHvuMUmrJf01W2ydu/+q4sYY57/kLF26NFnftGlTsn777bXP5N7X15esr1+/Pll///33a972pSzPcf6fSrpzguf/093bs38Vgw+gtVQMv7vvl3SyCb0AaKJ6zvl7zez3ZrbFzK7KrSMATVFr+H8kabGkdkklSRvLLWhmPWY2aGblf8wNQNPVFH53P+7u5919VNKPJS1LLNvv7h3u3lFrkwDyV1P4zWzuuIddkt7Jpx0AzXJZpQXMbLukWyV9ycyGJH1f0q1m1i7JJR2V9N0G9gigAfg+P+oya9asZP3uu+8uW6v0WwFm6eHqffv2JeudnZ3J+lTF9/kBJBF+ICjCDwRF+IGgCD8QFOEHgmKoD4X57LPPkvXLLktfhjIyMpKs33HHHWVrr7zySnLdSxlDfQCSCD8QFOEHgiL8QFCEHwiK8ANBEX4gqIrf50dsN9xwQ7J+//33J+s33nhj2VqlcfxKDh06lKzv37+/rtef6tjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQjPNPcUuWLEnWe3t7k/V77703WZ8zZ86ke6rW+fPnk/VSqZSsj46O5tnOlMOeHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCqjjOb2bzJW2TNEfSqKR+d/+hmbVJ2iFpoaSjkh5w9w8b12pclcbSu7u7y9YqjeMvXLiwlpZyMTg4mKyvX78+Wd+1a1ee7YRTzZ5/RNK/uftXJX1d0hozu07So5JecvdrJL2UPQZwiagYfncvufuB7P5pSYclzZO0UtLWbLGtku5pVJMA8jepc34zWyjpa5IGJM1295I09gdC0tV5Nwegcaq+tt/MvijpOUnfc/dTZlVNByYz65HUU1t7ABqlqj2/mX1BY8H/mbs/nz193MzmZvW5kk5MtK6797t7h7t35NEwgHxUDL+N7eJ/Iumwu28aV9olaVV2f5Wknfm3B6BRKk7RbWbLJf1G0tsaG+qTpHUaO+//paQFkv4s6VvufrLCa4Wconv27NnJ+nXXXZesP/3008n6tddeO+me8jIwMJCsP/nkk2VrO3em9xd8Jbc21U7RXfGc391/K6nci90+maYAtA6u8AOCIvxAUIQfCIrwA0ERfiAowg8ExU93V6mtra1sra+vL7lue3t7sr5o0aKaesrDa6+9lqxv3LgxWd+zZ0+y/sknn0y6JzQHe34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCCrMOP9NN92UrK9duzZZX7ZsWdnavHnzauopLx9//HHZ2ubNm5PrPvHEE8n62bNna+oJrY89PxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EFWacv6urq656PQ4dOpSs7969O1kfGRlJ1lPfuR8eHk6ui7jY8wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUObu6QXM5kvaJmmOpFFJ/e7+QzN7TNJqSX/NFl3n7r+q8FrpjQGom7tbNctVE/65kua6+wEzmynpDUn3SHpA0hl3f6rapgg/0HjVhr/iFX7uXpJUyu6fNrPDkor96RoAdZvUOb+ZLZT0NUkD2VO9ZvZ7M9tiZleVWafHzAbNbLCuTgHkquJh/98WNPuipFclrXf3581stqQPJLmkH2js1OChCq/BYT/QYLmd80uSmX1B0m5Je9x90wT1hZJ2u/vSCq9D+IEGqzb8FQ/7zcwk/UTS4fHBzz4IvKBL0juTbRJAcar5tH+5pN9IeltjQ32StE5St6R2jR32H5X03ezDwdRrsecHGizXw/68EH6g8XI77AcwNRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCavYU3R9I+r9xj7+UPdeKWrW3Vu1Lorda5dnbP1a7YFO/z/+5jZsNuntHYQ0ktGpvrdqXRG+1Kqo3DvuBoAg/EFTR4e8vePsprdpbq/Yl0VutCumt0HN+AMUpes8PoCCFhN/M7jSzI2b2rpk9WkQP5ZjZUTN728wOFj3FWDYN2gkze2fcc21m9msz+1N2O+E0aQX19piZvZ+9dwfN7F8L6m2+mb1sZofN7A9m9kj2fKHvXaKvQt63ph/2m9k0SX+U1ClpSNLrkrrd/VBTGynDzI5K6nD3wseEzexfJJ2RtO3CbEhm9h+STrr7huwP51Xu/u8t0ttjmuTMzQ3qrdzM0t9Rge9dnjNe56GIPf8ySe+6+3vufk7SLyStLKCPlufu+yWdvOjplZK2Zve3auw/T9OV6a0luHvJ3Q9k909LujCzdKHvXaKvQhQR/nmS/jLu8ZBaa8pvl7TXzN4ws56im5nA7AszI2W3Vxfcz8UqztzcTBfNLN0y710tM17nrYjwTzSbSCsNOXzD3f9Z0l2S1mSHt6jOjyQt1tg0biVJG4tsJptZ+jlJ33P3U0X2Mt4EfRXyvhUR/iFJ88c9/rKkYwX0MSF3P5bdnpD0gsZOU1rJ8QuTpGa3Jwru52/c/bi7n3f3UUk/VoHvXTaz9HOSfubuz2dPF/7eTdRXUe9bEeF/XdI1ZvYVM5su6duSdhXQx+eY2YzsgxiZ2QxJ31TrzT68S9Kq7P4qSTsL7OXvtMrMzeVmllbB712rzXhdyEU+2VDGf0maJmmLu69vehMTMLNFGtvbS2PfePx5kb2Z2XZJt2rsW1/HJX1f0n9L+qWkBZL+LOlb7t70D97K9HarJjlzc4N6Kzez9IAKfO/ynPE6l364wg+IiSv8gKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8E9f/Ex0YKZYOZcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting example image\n",
    "img = mnist.train.images[0]\n",
    "img_resize = img.reshape((img_size, img_size))\n",
    "plt.imshow(img_resize, cmap = 'gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize weights and bias \n",
    "# Convolution and pooling\n",
    "def conv2d(x,w, stride):\n",
    "\treturn tf.nn.conv2d(x,w,strides=[1, stride, stride, 1], padding='SAME')\n",
    "\n",
    "# Get Variables\n",
    "def weight_variable(name, shape):\n",
    "    return tf.get_variable(name,shape=shape, initializer=tf.contrib.layers.xavier_initializer())\n",
    "\n",
    "def bias_variable(name, shape):\n",
    "    return tf.get_variable(name,shape=shape, initializer=tf.contrib.layers.xavier_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Minimal RNN Function!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset the graph\n",
    "tf.reset_default_graph()\n",
    "\n",
    "#Minimal Gated Unit Variables\n",
    "Wf = weight_variable('Wf', [flatten_size + rnn_size, rnn_size])\n",
    "Wh = weight_variable('Uh', [flatten_size + rnn_size, rnn_size])\n",
    "\n",
    "bf = bias_variable('bf', [rnn_size])\n",
    "bh = bias_variable('bh', [rnn_size])\n",
    "\n",
    "#Minimal Gated Unit function\n",
    "def MGU_cell(h_prev, x_rnn):\n",
    "    # h_prev: output from cell of previous time step (shape: [batch_size, rnn_size])\n",
    "    # x_rnn: input of rnn (shape: [batch_size, data_flatten_size])\n",
    "\n",
    "    input_concat = tf.concat([x_rnn, h_prev], 1)\n",
    "    f = tf.sigmoid(tf.matmul(input_concat, Wf) + bf)\n",
    "    h = tf.tanh(tf.matmul(tf.concat([x_rnn, tf.multiply(f, h_prev)],1), Wh) + bh)\n",
    "\n",
    "    h_t = tf.multiply((1-f), h_prev) + tf.multiply(f, h)\n",
    "    \n",
    "    return h_t # Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN Network with MGU function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Network\n",
    "\n",
    "# Input \n",
    "x_image  = tf.placeholder(tf.float32, shape = [None, img_flat_size])\n",
    "y_target = tf.placeholder(tf.float32, shape=[None, num_label])\n",
    "\n",
    "input_flat = tf.reshape(x_image,[-1, step_size , flatten_size])\n",
    "input_unstack = tf.unstack(input_flat, axis = 1)\n",
    "################################### Minimal RNN ###################################\n",
    "rnn_batch_size = tf.shape(input_flat)[0]\n",
    "rnn_step_size = tf.shape(input_flat)[1]\n",
    "\n",
    "rnn_out = tf.zeros([rnn_batch_size, rnn_size], tf.float32)\n",
    "\n",
    "for i in range(len(input_unstack)):\n",
    "    rnn_out = MGU_cell(rnn_out, input_unstack[i])\n",
    "###################################################################################\n",
    "\n",
    "rnn_out = tf.reshape(rnn_out ,shape = [-1, rnn_size])\n",
    "\n",
    "# Densely connect layer variables \n",
    "w_fc1 = weight_variable('W_fc1', [rnn_size, 256])\n",
    "b_fc1 = bias_variable('b_fc1', [256])\n",
    "\n",
    "w_fc2 = weight_variable('W_fc2',[256, num_label])\n",
    "b_fc2 = bias_variable('b_fc2',[num_label])\n",
    "\n",
    "# Fully Connected Layer\n",
    "h_fc1 = tf.nn.relu(tf.matmul(rnn_out, w_fc1)+b_fc1)\n",
    "output = tf.matmul(h_fc1, w_fc2)+b_fc2\n",
    "\n",
    "# Training \n",
    "Loss = tf.nn.sigmoid_cross_entropy_with_logits(labels = y_target, logits = output)\n",
    "Cost = tf.reduce_mean(Loss)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate, epsilon = epsilon).minimize(Cost)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y_target,1), tf.argmax(output,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Making Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set: (60000, 784)\n",
      "Testing set: (9000, 784)\n",
      "Validation set: (1000, 784)\n"
     ]
    }
   ],
   "source": [
    "# Dataset for train, test, validation\n",
    "\n",
    "test_len = mnist.test.images.shape[0]\n",
    "validation_len = int(test_len * validation_ratio)\n",
    "\n",
    "train_x = mnist.train.images\n",
    "test_x = mnist.test.images[validation_len : test_len, :]\n",
    "validation_x = mnist.test.images[ : validation_len, :]\n",
    "\n",
    "train_y = mnist.train.labels\n",
    "test_y = mnist.test.labels[validation_len : test_len]\n",
    "validation_y = mnist.test.labels[ : validation_len]\n",
    "\n",
    "print(\"Training set: \" + str(train_x.shape))\n",
    "print(\"Testing set: \" + str(test_x.shape))\n",
    "print(\"Validation set: \" + str(validation_x.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = gpu_fraction\n",
    "\n",
    "sess = tf.InteractiveSession(config=config)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "data = np.insert(train_x, img_flat_size, train_y, axis = 1)\n",
    "len_data = data.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 / Batch: 0/60000 / Cost: 0.6824816 / Training Accuracy: 0.111328125 / Validation Accuracy: 0.122\n",
      "Epoch: 1 / Batch: 512/60000 / Cost: 0.6464821 / Training Accuracy: 0.1171875 / Validation Accuracy: 0.107\n",
      "Epoch: 1 / Batch: 1024/60000 / Cost: 0.60917443 / Training Accuracy: 0.115234375 / Validation Accuracy: 0.107\n",
      "Epoch: 1 / Batch: 1536/60000 / Cost: 0.5643174 / Training Accuracy: 0.07421875 / Validation Accuracy: 0.107\n",
      "Epoch: 1 / Batch: 2048/60000 / Cost: 0.50071156 / Training Accuracy: 0.109375 / Validation Accuracy: 0.107\n",
      "Epoch: 1 / Batch: 2560/60000 / Cost: 0.41766176 / Training Accuracy: 0.109375 / Validation Accuracy: 0.107\n",
      "Epoch: 1 / Batch: 3072/60000 / Cost: 0.3452627 / Training Accuracy: 0.09765625 / Validation Accuracy: 0.069\n",
      "Epoch: 1 / Batch: 3584/60000 / Cost: 0.3655276 / Training Accuracy: 0.1015625 / Validation Accuracy: 0.096\n",
      "Epoch: 1 / Batch: 4096/60000 / Cost: 0.3955947 / Training Accuracy: 0.078125 / Validation Accuracy: 0.097\n",
      "Epoch: 1 / Batch: 4608/60000 / Cost: 0.38176432 / Training Accuracy: 0.13671875 / Validation Accuracy: 0.15\n",
      "Epoch: 1 / Batch: 5120/60000 / Cost: 0.35289627 / Training Accuracy: 0.10546875 / Validation Accuracy: 0.128\n",
      "Epoch: 1 / Batch: 5632/60000 / Cost: 0.32980925 / Training Accuracy: 0.12695312 / Validation Accuracy: 0.127\n",
      "Epoch: 1 / Batch: 6144/60000 / Cost: 0.3257381 / Training Accuracy: 0.12890625 / Validation Accuracy: 0.124\n",
      "Epoch: 1 / Batch: 6656/60000 / Cost: 0.33588856 / Training Accuracy: 0.13671875 / Validation Accuracy: 0.144\n",
      "Epoch: 1 / Batch: 7168/60000 / Cost: 0.34466118 / Training Accuracy: 0.15625 / Validation Accuracy: 0.174\n",
      "Epoch: 1 / Batch: 7680/60000 / Cost: 0.34630257 / Training Accuracy: 0.13671875 / Validation Accuracy: 0.178\n",
      "Epoch: 1 / Batch: 8192/60000 / Cost: 0.33962446 / Training Accuracy: 0.17773438 / Validation Accuracy: 0.161\n",
      "Epoch: 1 / Batch: 8704/60000 / Cost: 0.33353367 / Training Accuracy: 0.12695312 / Validation Accuracy: 0.107\n",
      "Epoch: 1 / Batch: 9216/60000 / Cost: 0.32706207 / Training Accuracy: 0.095703125 / Validation Accuracy: 0.094\n",
      "Epoch: 1 / Batch: 9728/60000 / Cost: 0.32458624 / Training Accuracy: 0.080078125 / Validation Accuracy: 0.092\n",
      "Epoch: 1 / Batch: 10240/60000 / Cost: 0.32349315 / Training Accuracy: 0.1015625 / Validation Accuracy: 0.09\n",
      "Epoch: 1 / Batch: 10752/60000 / Cost: 0.32626432 / Training Accuracy: 0.08203125 / Validation Accuracy: 0.092\n",
      "Epoch: 1 / Batch: 11264/60000 / Cost: 0.32979062 / Training Accuracy: 0.18554688 / Validation Accuracy: 0.185\n",
      "Epoch: 1 / Batch: 11776/60000 / Cost: 0.33028167 / Training Accuracy: 0.15820312 / Validation Accuracy: 0.177\n",
      "Epoch: 1 / Batch: 12288/60000 / Cost: 0.3271815 / Training Accuracy: 0.15234375 / Validation Accuracy: 0.124\n",
      "Epoch: 1 / Batch: 12800/60000 / Cost: 0.32508397 / Training Accuracy: 0.115234375 / Validation Accuracy: 0.102\n",
      "Epoch: 1 / Batch: 13312/60000 / Cost: 0.32233673 / Training Accuracy: 0.15429688 / Validation Accuracy: 0.134\n",
      "Epoch: 1 / Batch: 13824/60000 / Cost: 0.32044274 / Training Accuracy: 0.23828125 / Validation Accuracy: 0.201\n",
      "Epoch: 1 / Batch: 14336/60000 / Cost: 0.32087794 / Training Accuracy: 0.16015625 / Validation Accuracy: 0.155\n",
      "Epoch: 1 / Batch: 14848/60000 / Cost: 0.3204593 / Training Accuracy: 0.20507812 / Validation Accuracy: 0.185\n",
      "Epoch: 1 / Batch: 15360/60000 / Cost: 0.32172495 / Training Accuracy: 0.25390625 / Validation Accuracy: 0.253\n",
      "Epoch: 1 / Batch: 15872/60000 / Cost: 0.32158622 / Training Accuracy: 0.25976562 / Validation Accuracy: 0.246\n",
      "Epoch: 1 / Batch: 16384/60000 / Cost: 0.32036453 / Training Accuracy: 0.30273438 / Validation Accuracy: 0.248\n",
      "Epoch: 1 / Batch: 16896/60000 / Cost: 0.32078654 / Training Accuracy: 0.23242188 / Validation Accuracy: 0.22\n",
      "Epoch: 1 / Batch: 17408/60000 / Cost: 0.32004085 / Training Accuracy: 0.21875 / Validation Accuracy: 0.227\n",
      "Epoch: 1 / Batch: 17920/60000 / Cost: 0.31793618 / Training Accuracy: 0.21289062 / Validation Accuracy: 0.234\n",
      "Epoch: 1 / Batch: 18432/60000 / Cost: 0.31609806 / Training Accuracy: 0.2890625 / Validation Accuracy: 0.257\n",
      "Epoch: 1 / Batch: 18944/60000 / Cost: 0.3152175 / Training Accuracy: 0.33398438 / Validation Accuracy: 0.293\n",
      "Epoch: 1 / Batch: 19456/60000 / Cost: 0.31596345 / Training Accuracy: 0.30078125 / Validation Accuracy: 0.307\n",
      "Epoch: 1 / Batch: 19968/60000 / Cost: 0.31576803 / Training Accuracy: 0.29101562 / Validation Accuracy: 0.331\n",
      "Epoch: 1 / Batch: 20480/60000 / Cost: 0.31492528 / Training Accuracy: 0.31445312 / Validation Accuracy: 0.343\n",
      "Epoch: 1 / Batch: 20992/60000 / Cost: 0.3126216 / Training Accuracy: 0.31835938 / Validation Accuracy: 0.344\n",
      "Epoch: 1 / Batch: 21504/60000 / Cost: 0.30992955 / Training Accuracy: 0.35742188 / Validation Accuracy: 0.304\n",
      "Epoch: 1 / Batch: 22016/60000 / Cost: 0.31095082 / Training Accuracy: 0.31640625 / Validation Accuracy: 0.314\n",
      "Epoch: 1 / Batch: 22528/60000 / Cost: 0.30733162 / Training Accuracy: 0.33398438 / Validation Accuracy: 0.31\n",
      "Epoch: 1 / Batch: 23040/60000 / Cost: 0.3069899 / Training Accuracy: 0.34375 / Validation Accuracy: 0.354\n",
      "Epoch: 1 / Batch: 23552/60000 / Cost: 0.30483755 / Training Accuracy: 0.38867188 / Validation Accuracy: 0.403\n",
      "Epoch: 1 / Batch: 24064/60000 / Cost: 0.3000638 / Training Accuracy: 0.421875 / Validation Accuracy: 0.385\n",
      "Epoch: 1 / Batch: 24576/60000 / Cost: 0.29673016 / Training Accuracy: 0.40429688 / Validation Accuracy: 0.34\n",
      "Epoch: 1 / Batch: 25088/60000 / Cost: 0.2940571 / Training Accuracy: 0.33203125 / Validation Accuracy: 0.314\n",
      "Epoch: 1 / Batch: 25600/60000 / Cost: 0.289855 / Training Accuracy: 0.32226562 / Validation Accuracy: 0.293\n",
      "Epoch: 1 / Batch: 26112/60000 / Cost: 0.28139627 / Training Accuracy: 0.3359375 / Validation Accuracy: 0.292\n",
      "Epoch: 1 / Batch: 26624/60000 / Cost: 0.28207836 / Training Accuracy: 0.29882812 / Validation Accuracy: 0.297\n",
      "Epoch: 1 / Batch: 27136/60000 / Cost: 0.2770665 / Training Accuracy: 0.3046875 / Validation Accuracy: 0.3\n",
      "Epoch: 1 / Batch: 27648/60000 / Cost: 0.2701974 / Training Accuracy: 0.33398438 / Validation Accuracy: 0.352\n",
      "Epoch: 1 / Batch: 28160/60000 / Cost: 0.26425704 / Training Accuracy: 0.37695312 / Validation Accuracy: 0.399\n",
      "Epoch: 1 / Batch: 28672/60000 / Cost: 0.25553462 / Training Accuracy: 0.44726562 / Validation Accuracy: 0.445\n",
      "Epoch: 1 / Batch: 29184/60000 / Cost: 0.24950843 / Training Accuracy: 0.43945312 / Validation Accuracy: 0.462\n",
      "Epoch: 1 / Batch: 29696/60000 / Cost: 0.24328904 / Training Accuracy: 0.47265625 / Validation Accuracy: 0.461\n",
      "Epoch: 1 / Batch: 30208/60000 / Cost: 0.24223718 / Training Accuracy: 0.4453125 / Validation Accuracy: 0.467\n",
      "Epoch: 1 / Batch: 30720/60000 / Cost: 0.23207144 / Training Accuracy: 0.49414062 / Validation Accuracy: 0.466\n",
      "Epoch: 1 / Batch: 31232/60000 / Cost: 0.22824422 / Training Accuracy: 0.5 / Validation Accuracy: 0.505\n",
      "Epoch: 1 / Batch: 31744/60000 / Cost: 0.21840706 / Training Accuracy: 0.5136719 / Validation Accuracy: 0.479\n",
      "Epoch: 1 / Batch: 32256/60000 / Cost: 0.21444623 / Training Accuracy: 0.53515625 / Validation Accuracy: 0.498\n",
      "Epoch: 1 / Batch: 32768/60000 / Cost: 0.21921174 / Training Accuracy: 0.5019531 / Validation Accuracy: 0.542\n",
      "Epoch: 1 / Batch: 33280/60000 / Cost: 0.20977965 / Training Accuracy: 0.53515625 / Validation Accuracy: 0.537\n",
      "Epoch: 1 / Batch: 33792/60000 / Cost: 0.21071963 / Training Accuracy: 0.5546875 / Validation Accuracy: 0.521\n",
      "Epoch: 1 / Batch: 34304/60000 / Cost: 0.20306304 / Training Accuracy: 0.5546875 / Validation Accuracy: 0.539\n",
      "Epoch: 1 / Batch: 34816/60000 / Cost: 0.2092376 / Training Accuracy: 0.5175781 / Validation Accuracy: 0.511\n",
      "Epoch: 1 / Batch: 35328/60000 / Cost: 0.19719371 / Training Accuracy: 0.5800781 / Validation Accuracy: 0.563\n",
      "Epoch: 1 / Batch: 35840/60000 / Cost: 0.21181993 / Training Accuracy: 0.54296875 / Validation Accuracy: 0.567\n",
      "Epoch: 1 / Batch: 36352/60000 / Cost: 0.19990623 / Training Accuracy: 0.56640625 / Validation Accuracy: 0.569\n",
      "Epoch: 1 / Batch: 36864/60000 / Cost: 0.18986392 / Training Accuracy: 0.6191406 / Validation Accuracy: 0.546\n",
      "Epoch: 1 / Batch: 37376/60000 / Cost: 0.18774678 / Training Accuracy: 0.61328125 / Validation Accuracy: 0.608\n",
      "Epoch: 1 / Batch: 37888/60000 / Cost: 0.19953723 / Training Accuracy: 0.5625 / Validation Accuracy: 0.595\n",
      "Epoch: 1 / Batch: 38400/60000 / Cost: 0.19018309 / Training Accuracy: 0.5859375 / Validation Accuracy: 0.614\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 / Batch: 38912/60000 / Cost: 0.19166215 / Training Accuracy: 0.609375 / Validation Accuracy: 0.603\n",
      "Epoch: 1 / Batch: 39424/60000 / Cost: 0.187124 / Training Accuracy: 0.6269531 / Validation Accuracy: 0.586\n",
      "Epoch: 1 / Batch: 39936/60000 / Cost: 0.18008558 / Training Accuracy: 0.6191406 / Validation Accuracy: 0.602\n",
      "Epoch: 1 / Batch: 40448/60000 / Cost: 0.17715274 / Training Accuracy: 0.625 / Validation Accuracy: 0.593\n",
      "Epoch: 1 / Batch: 40960/60000 / Cost: 0.1830419 / Training Accuracy: 0.6035156 / Validation Accuracy: 0.612\n",
      "Epoch: 1 / Batch: 41472/60000 / Cost: 0.18101199 / Training Accuracy: 0.61328125 / Validation Accuracy: 0.62\n",
      "Epoch: 1 / Batch: 41984/60000 / Cost: 0.18172047 / Training Accuracy: 0.62109375 / Validation Accuracy: 0.619\n",
      "Epoch: 1 / Batch: 42496/60000 / Cost: 0.17865749 / Training Accuracy: 0.62109375 / Validation Accuracy: 0.622\n",
      "Epoch: 1 / Batch: 43008/60000 / Cost: 0.19152094 / Training Accuracy: 0.57421875 / Validation Accuracy: 0.641\n",
      "Epoch: 1 / Batch: 43520/60000 / Cost: 0.18468711 / Training Accuracy: 0.625 / Validation Accuracy: 0.646\n",
      "Epoch: 1 / Batch: 44032/60000 / Cost: 0.16736425 / Training Accuracy: 0.65234375 / Validation Accuracy: 0.6\n",
      "Epoch: 1 / Batch: 44544/60000 / Cost: 0.16757815 / Training Accuracy: 0.66015625 / Validation Accuracy: 0.625\n",
      "Epoch: 1 / Batch: 45056/60000 / Cost: 0.16511655 / Training Accuracy: 0.65234375 / Validation Accuracy: 0.617\n",
      "Epoch: 1 / Batch: 45568/60000 / Cost: 0.16620953 / Training Accuracy: 0.6386719 / Validation Accuracy: 0.621\n",
      "Epoch: 1 / Batch: 46080/60000 / Cost: 0.17227371 / Training Accuracy: 0.640625 / Validation Accuracy: 0.611\n",
      "Epoch: 1 / Batch: 46592/60000 / Cost: 0.1798658 / Training Accuracy: 0.5996094 / Validation Accuracy: 0.604\n",
      "Epoch: 1 / Batch: 47104/60000 / Cost: 0.15720421 / Training Accuracy: 0.6640625 / Validation Accuracy: 0.643\n",
      "Epoch: 1 / Batch: 47616/60000 / Cost: 0.17320079 / Training Accuracy: 0.625 / Validation Accuracy: 0.633\n",
      "Epoch: 1 / Batch: 48128/60000 / Cost: 0.15749972 / Training Accuracy: 0.6855469 / Validation Accuracy: 0.651\n",
      "Epoch: 1 / Batch: 48640/60000 / Cost: 0.16153777 / Training Accuracy: 0.65625 / Validation Accuracy: 0.648\n",
      "Epoch: 1 / Batch: 49152/60000 / Cost: 0.15983431 / Training Accuracy: 0.66796875 / Validation Accuracy: 0.675\n",
      "Epoch: 1 / Batch: 49664/60000 / Cost: 0.16181193 / Training Accuracy: 0.6738281 / Validation Accuracy: 0.683\n",
      "Epoch: 1 / Batch: 50176/60000 / Cost: 0.16060758 / Training Accuracy: 0.67578125 / Validation Accuracy: 0.674\n",
      "Epoch: 1 / Batch: 50688/60000 / Cost: 0.15801978 / Training Accuracy: 0.68359375 / Validation Accuracy: 0.669\n",
      "Epoch: 1 / Batch: 51200/60000 / Cost: 0.15012269 / Training Accuracy: 0.6953125 / Validation Accuracy: 0.671\n",
      "Epoch: 1 / Batch: 51712/60000 / Cost: 0.15052144 / Training Accuracy: 0.68359375 / Validation Accuracy: 0.668\n",
      "Epoch: 1 / Batch: 52224/60000 / Cost: 0.15062973 / Training Accuracy: 0.6875 / Validation Accuracy: 0.649\n",
      "Epoch: 1 / Batch: 52736/60000 / Cost: 0.1576402 / Training Accuracy: 0.6640625 / Validation Accuracy: 0.678\n",
      "Epoch: 1 / Batch: 53248/60000 / Cost: 0.16017565 / Training Accuracy: 0.6582031 / Validation Accuracy: 0.665\n",
      "Epoch: 1 / Batch: 53760/60000 / Cost: 0.1463201 / Training Accuracy: 0.703125 / Validation Accuracy: 0.673\n",
      "Epoch: 1 / Batch: 54272/60000 / Cost: 0.14648911 / Training Accuracy: 0.7265625 / Validation Accuracy: 0.68\n",
      "Epoch: 1 / Batch: 54784/60000 / Cost: 0.14359185 / Training Accuracy: 0.7109375 / Validation Accuracy: 0.694\n",
      "Epoch: 1 / Batch: 55296/60000 / Cost: 0.15117794 / Training Accuracy: 0.7089844 / Validation Accuracy: 0.695\n",
      "Epoch: 1 / Batch: 55808/60000 / Cost: 0.15076046 / Training Accuracy: 0.68359375 / Validation Accuracy: 0.683\n",
      "Epoch: 1 / Batch: 56320/60000 / Cost: 0.14688082 / Training Accuracy: 0.6855469 / Validation Accuracy: 0.683\n",
      "Epoch: 1 / Batch: 56832/60000 / Cost: 0.14426799 / Training Accuracy: 0.7246094 / Validation Accuracy: 0.691\n",
      "Epoch: 1 / Batch: 57344/60000 / Cost: 0.14268416 / Training Accuracy: 0.7050781 / Validation Accuracy: 0.677\n",
      "Epoch: 1 / Batch: 57856/60000 / Cost: 0.14698127 / Training Accuracy: 0.69921875 / Validation Accuracy: 0.696\n",
      "Epoch: 1 / Batch: 58368/60000 / Cost: 0.14196947 / Training Accuracy: 0.7402344 / Validation Accuracy: 0.692\n",
      "Epoch: 1 / Batch: 58880/60000 / Cost: 0.14429715 / Training Accuracy: 0.7089844 / Validation Accuracy: 0.693\n",
      "Epoch: 1 / Batch: 59392/60000 / Cost: 0.13641325 / Training Accuracy: 0.7421875 / Validation Accuracy: 0.703\n",
      "Epoch: 1 / Batch: 59904/60000 / Cost: 0.13541165 / Training Accuracy: 0.7395833 / Validation Accuracy: 0.723\n",
      "Epoch: 2 / Batch: 0/60000 / Cost: 0.13661993 / Training Accuracy: 0.72265625 / Validation Accuracy: 0.721\n",
      "Epoch: 2 / Batch: 512/60000 / Cost: 0.13856176 / Training Accuracy: 0.7421875 / Validation Accuracy: 0.718\n",
      "Epoch: 2 / Batch: 1024/60000 / Cost: 0.13944733 / Training Accuracy: 0.75 / Validation Accuracy: 0.728\n",
      "Epoch: 2 / Batch: 1536/60000 / Cost: 0.1238446 / Training Accuracy: 0.7578125 / Validation Accuracy: 0.715\n",
      "Epoch: 2 / Batch: 2048/60000 / Cost: 0.14528091 / Training Accuracy: 0.7246094 / Validation Accuracy: 0.683\n",
      "Epoch: 2 / Batch: 2560/60000 / Cost: 0.14069963 / Training Accuracy: 0.71484375 / Validation Accuracy: 0.713\n",
      "Epoch: 2 / Batch: 3072/60000 / Cost: 0.13873024 / Training Accuracy: 0.7109375 / Validation Accuracy: 0.697\n",
      "Epoch: 2 / Batch: 3584/60000 / Cost: 0.13243262 / Training Accuracy: 0.7636719 / Validation Accuracy: 0.728\n",
      "Epoch: 2 / Batch: 4096/60000 / Cost: 0.14143066 / Training Accuracy: 0.6875 / Validation Accuracy: 0.685\n",
      "Epoch: 2 / Batch: 4608/60000 / Cost: 0.12921561 / Training Accuracy: 0.7363281 / Validation Accuracy: 0.708\n",
      "Epoch: 2 / Batch: 5120/60000 / Cost: 0.13330008 / Training Accuracy: 0.73046875 / Validation Accuracy: 0.716\n",
      "Epoch: 2 / Batch: 5632/60000 / Cost: 0.13375835 / Training Accuracy: 0.75390625 / Validation Accuracy: 0.724\n",
      "Epoch: 2 / Batch: 6144/60000 / Cost: 0.12827446 / Training Accuracy: 0.77734375 / Validation Accuracy: 0.747\n",
      "Epoch: 2 / Batch: 6656/60000 / Cost: 0.12944351 / Training Accuracy: 0.7402344 / Validation Accuracy: 0.7\n",
      "Epoch: 2 / Batch: 7168/60000 / Cost: 0.1277571 / Training Accuracy: 0.7675781 / Validation Accuracy: 0.728\n",
      "Epoch: 2 / Batch: 7680/60000 / Cost: 0.1282473 / Training Accuracy: 0.73828125 / Validation Accuracy: 0.743\n",
      "Epoch: 2 / Batch: 8192/60000 / Cost: 0.121523224 / Training Accuracy: 0.76953125 / Validation Accuracy: 0.745\n",
      "Epoch: 2 / Batch: 8704/60000 / Cost: 0.11986091 / Training Accuracy: 0.7578125 / Validation Accuracy: 0.755\n",
      "Epoch: 2 / Batch: 9216/60000 / Cost: 0.12480112 / Training Accuracy: 0.7441406 / Validation Accuracy: 0.751\n",
      "Epoch: 2 / Batch: 9728/60000 / Cost: 0.12250908 / Training Accuracy: 0.7636719 / Validation Accuracy: 0.721\n",
      "Epoch: 2 / Batch: 10240/60000 / Cost: 0.11938536 / Training Accuracy: 0.78515625 / Validation Accuracy: 0.738\n",
      "Epoch: 2 / Batch: 10752/60000 / Cost: 0.12550403 / Training Accuracy: 0.73828125 / Validation Accuracy: 0.755\n",
      "Epoch: 2 / Batch: 11264/60000 / Cost: 0.13010308 / Training Accuracy: 0.7675781 / Validation Accuracy: 0.763\n",
      "Epoch: 2 / Batch: 11776/60000 / Cost: 0.11674325 / Training Accuracy: 0.7910156 / Validation Accuracy: 0.763\n",
      "Epoch: 2 / Batch: 12288/60000 / Cost: 0.12172942 / Training Accuracy: 0.7636719 / Validation Accuracy: 0.756\n",
      "Epoch: 2 / Batch: 12800/60000 / Cost: 0.1206551 / Training Accuracy: 0.7441406 / Validation Accuracy: 0.76\n",
      "Epoch: 2 / Batch: 13312/60000 / Cost: 0.120742895 / Training Accuracy: 0.7675781 / Validation Accuracy: 0.76\n",
      "Epoch: 2 / Batch: 13824/60000 / Cost: 0.11425425 / Training Accuracy: 0.7910156 / Validation Accuracy: 0.76\n",
      "Epoch: 2 / Batch: 14336/60000 / Cost: 0.110240385 / Training Accuracy: 0.7910156 / Validation Accuracy: 0.759\n",
      "Epoch: 2 / Batch: 14848/60000 / Cost: 0.11664057 / Training Accuracy: 0.7871094 / Validation Accuracy: 0.764\n",
      "Epoch: 2 / Batch: 15360/60000 / Cost: 0.121945366 / Training Accuracy: 0.76171875 / Validation Accuracy: 0.774\n",
      "Epoch: 2 / Batch: 15872/60000 / Cost: 0.11285772 / Training Accuracy: 0.7910156 / Validation Accuracy: 0.772\n",
      "Epoch: 2 / Batch: 16384/60000 / Cost: 0.124431774 / Training Accuracy: 0.7636719 / Validation Accuracy: 0.762\n",
      "Epoch: 2 / Batch: 16896/60000 / Cost: 0.112202406 / Training Accuracy: 0.7714844 / Validation Accuracy: 0.779\n",
      "Epoch: 2 / Batch: 17408/60000 / Cost: 0.11755607 / Training Accuracy: 0.7792969 / Validation Accuracy: 0.785\n",
      "Epoch: 2 / Batch: 17920/60000 / Cost: 0.10633121 / Training Accuracy: 0.8066406 / Validation Accuracy: 0.789\n",
      "Epoch: 2 / Batch: 18432/60000 / Cost: 0.113973975 / Training Accuracy: 0.7871094 / Validation Accuracy: 0.776\n",
      "Epoch: 2 / Batch: 18944/60000 / Cost: 0.104181625 / Training Accuracy: 0.8027344 / Validation Accuracy: 0.785\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 / Batch: 19456/60000 / Cost: 0.107715465 / Training Accuracy: 0.81640625 / Validation Accuracy: 0.795\n",
      "Epoch: 2 / Batch: 19968/60000 / Cost: 0.10663052 / Training Accuracy: 0.8046875 / Validation Accuracy: 0.796\n",
      "Epoch: 2 / Batch: 20480/60000 / Cost: 0.10855796 / Training Accuracy: 0.8183594 / Validation Accuracy: 0.796\n",
      "Epoch: 2 / Batch: 20992/60000 / Cost: 0.11341127 / Training Accuracy: 0.78125 / Validation Accuracy: 0.789\n",
      "Epoch: 2 / Batch: 21504/60000 / Cost: 0.11002942 / Training Accuracy: 0.796875 / Validation Accuracy: 0.794\n",
      "Epoch: 2 / Batch: 22016/60000 / Cost: 0.09394243 / Training Accuracy: 0.84765625 / Validation Accuracy: 0.789\n",
      "Epoch: 2 / Batch: 22528/60000 / Cost: 0.116454676 / Training Accuracy: 0.7753906 / Validation Accuracy: 0.797\n",
      "Epoch: 2 / Batch: 23040/60000 / Cost: 0.09217049 / Training Accuracy: 0.8378906 / Validation Accuracy: 0.787\n",
      "Epoch: 2 / Batch: 23552/60000 / Cost: 0.10536609 / Training Accuracy: 0.8066406 / Validation Accuracy: 0.777\n",
      "Epoch: 2 / Batch: 24064/60000 / Cost: 0.10729289 / Training Accuracy: 0.8105469 / Validation Accuracy: 0.783\n",
      "Epoch: 2 / Batch: 24576/60000 / Cost: 0.1120827 / Training Accuracy: 0.76953125 / Validation Accuracy: 0.786\n",
      "Epoch: 2 / Batch: 25088/60000 / Cost: 0.1043008 / Training Accuracy: 0.8144531 / Validation Accuracy: 0.796\n",
      "Epoch: 2 / Batch: 25600/60000 / Cost: 0.10885702 / Training Accuracy: 0.77734375 / Validation Accuracy: 0.761\n",
      "Epoch: 2 / Batch: 26112/60000 / Cost: 0.10360553 / Training Accuracy: 0.8105469 / Validation Accuracy: 0.792\n",
      "Epoch: 2 / Batch: 26624/60000 / Cost: 0.10042844 / Training Accuracy: 0.8066406 / Validation Accuracy: 0.792\n",
      "Epoch: 2 / Batch: 27136/60000 / Cost: 0.107973956 / Training Accuracy: 0.8046875 / Validation Accuracy: 0.803\n",
      "Epoch: 2 / Batch: 27648/60000 / Cost: 0.10265692 / Training Accuracy: 0.8105469 / Validation Accuracy: 0.791\n",
      "Epoch: 2 / Batch: 28160/60000 / Cost: 0.10516387 / Training Accuracy: 0.7988281 / Validation Accuracy: 0.807\n",
      "Epoch: 2 / Batch: 28672/60000 / Cost: 0.107844256 / Training Accuracy: 0.8027344 / Validation Accuracy: 0.778\n",
      "Epoch: 2 / Batch: 29184/60000 / Cost: 0.09327906 / Training Accuracy: 0.83203125 / Validation Accuracy: 0.793\n",
      "Epoch: 2 / Batch: 29696/60000 / Cost: 0.10097094 / Training Accuracy: 0.8222656 / Validation Accuracy: 0.809\n",
      "Epoch: 2 / Batch: 30208/60000 / Cost: 0.1065789 / Training Accuracy: 0.7890625 / Validation Accuracy: 0.801\n",
      "Epoch: 2 / Batch: 30720/60000 / Cost: 0.08950456 / Training Accuracy: 0.8496094 / Validation Accuracy: 0.812\n",
      "Epoch: 2 / Batch: 31232/60000 / Cost: 0.092655875 / Training Accuracy: 0.8300781 / Validation Accuracy: 0.81\n",
      "Epoch: 2 / Batch: 31744/60000 / Cost: 0.09040226 / Training Accuracy: 0.83984375 / Validation Accuracy: 0.793\n",
      "Epoch: 2 / Batch: 32256/60000 / Cost: 0.096534885 / Training Accuracy: 0.8261719 / Validation Accuracy: 0.815\n",
      "Epoch: 2 / Batch: 32768/60000 / Cost: 0.0889359 / Training Accuracy: 0.84765625 / Validation Accuracy: 0.82\n",
      "Epoch: 2 / Batch: 33280/60000 / Cost: 0.09292866 / Training Accuracy: 0.83203125 / Validation Accuracy: 0.804\n",
      "Epoch: 2 / Batch: 33792/60000 / Cost: 0.09482552 / Training Accuracy: 0.8222656 / Validation Accuracy: 0.792\n",
      "Epoch: 2 / Batch: 34304/60000 / Cost: 0.10045451 / Training Accuracy: 0.8203125 / Validation Accuracy: 0.805\n",
      "Epoch: 2 / Batch: 34816/60000 / Cost: 0.092375375 / Training Accuracy: 0.82421875 / Validation Accuracy: 0.815\n",
      "Epoch: 2 / Batch: 35328/60000 / Cost: 0.09177425 / Training Accuracy: 0.8378906 / Validation Accuracy: 0.813\n",
      "Epoch: 2 / Batch: 35840/60000 / Cost: 0.09532726 / Training Accuracy: 0.8222656 / Validation Accuracy: 0.821\n",
      "Epoch: 2 / Batch: 36352/60000 / Cost: 0.084030434 / Training Accuracy: 0.8300781 / Validation Accuracy: 0.826\n",
      "Epoch: 2 / Batch: 36864/60000 / Cost: 0.0888641 / Training Accuracy: 0.84375 / Validation Accuracy: 0.821\n",
      "Epoch: 2 / Batch: 37376/60000 / Cost: 0.08103697 / Training Accuracy: 0.86328125 / Validation Accuracy: 0.814\n",
      "Epoch: 2 / Batch: 37888/60000 / Cost: 0.079314664 / Training Accuracy: 0.859375 / Validation Accuracy: 0.814\n",
      "Epoch: 2 / Batch: 38400/60000 / Cost: 0.09439771 / Training Accuracy: 0.83984375 / Validation Accuracy: 0.814\n",
      "Epoch: 2 / Batch: 38912/60000 / Cost: 0.079801716 / Training Accuracy: 0.8457031 / Validation Accuracy: 0.814\n",
      "Epoch: 2 / Batch: 39424/60000 / Cost: 0.078661576 / Training Accuracy: 0.8613281 / Validation Accuracy: 0.825\n",
      "Epoch: 2 / Batch: 39936/60000 / Cost: 0.09165543 / Training Accuracy: 0.8378906 / Validation Accuracy: 0.832\n",
      "Epoch: 2 / Batch: 40448/60000 / Cost: 0.0852303 / Training Accuracy: 0.8535156 / Validation Accuracy: 0.828\n",
      "Epoch: 2 / Batch: 40960/60000 / Cost: 0.09104619 / Training Accuracy: 0.84765625 / Validation Accuracy: 0.825\n",
      "Epoch: 2 / Batch: 41472/60000 / Cost: 0.07635597 / Training Accuracy: 0.86328125 / Validation Accuracy: 0.839\n",
      "Epoch: 2 / Batch: 41984/60000 / Cost: 0.08143251 / Training Accuracy: 0.83984375 / Validation Accuracy: 0.825\n",
      "Epoch: 2 / Batch: 42496/60000 / Cost: 0.089761086 / Training Accuracy: 0.8417969 / Validation Accuracy: 0.828\n",
      "Epoch: 2 / Batch: 43008/60000 / Cost: 0.082872584 / Training Accuracy: 0.8535156 / Validation Accuracy: 0.83\n",
      "Epoch: 2 / Batch: 43520/60000 / Cost: 0.076009415 / Training Accuracy: 0.8652344 / Validation Accuracy: 0.838\n",
      "Epoch: 2 / Batch: 44032/60000 / Cost: 0.08114382 / Training Accuracy: 0.8535156 / Validation Accuracy: 0.829\n",
      "Epoch: 2 / Batch: 44544/60000 / Cost: 0.08169785 / Training Accuracy: 0.8417969 / Validation Accuracy: 0.829\n",
      "Epoch: 2 / Batch: 45056/60000 / Cost: 0.08623497 / Training Accuracy: 0.83203125 / Validation Accuracy: 0.826\n",
      "Epoch: 2 / Batch: 45568/60000 / Cost: 0.08651899 / Training Accuracy: 0.8652344 / Validation Accuracy: 0.825\n",
      "Epoch: 2 / Batch: 46080/60000 / Cost: 0.0725004 / Training Accuracy: 0.8535156 / Validation Accuracy: 0.826\n",
      "Epoch: 2 / Batch: 46592/60000 / Cost: 0.08328852 / Training Accuracy: 0.859375 / Validation Accuracy: 0.837\n",
      "Epoch: 2 / Batch: 47104/60000 / Cost: 0.092683606 / Training Accuracy: 0.83203125 / Validation Accuracy: 0.828\n",
      "Epoch: 2 / Batch: 47616/60000 / Cost: 0.07711024 / Training Accuracy: 0.85546875 / Validation Accuracy: 0.849\n",
      "Epoch: 2 / Batch: 48128/60000 / Cost: 0.09647987 / Training Accuracy: 0.80859375 / Validation Accuracy: 0.81\n",
      "Epoch: 2 / Batch: 48640/60000 / Cost: 0.06982149 / Training Accuracy: 0.8730469 / Validation Accuracy: 0.835\n",
      "Epoch: 2 / Batch: 49152/60000 / Cost: 0.08766042 / Training Accuracy: 0.83203125 / Validation Accuracy: 0.817\n",
      "Epoch: 2 / Batch: 49664/60000 / Cost: 0.08055268 / Training Accuracy: 0.859375 / Validation Accuracy: 0.839\n",
      "Epoch: 2 / Batch: 50176/60000 / Cost: 0.07087083 / Training Accuracy: 0.87890625 / Validation Accuracy: 0.845\n",
      "Epoch: 2 / Batch: 50688/60000 / Cost: 0.07440233 / Training Accuracy: 0.86328125 / Validation Accuracy: 0.833\n",
      "Epoch: 2 / Batch: 51200/60000 / Cost: 0.084037125 / Training Accuracy: 0.8496094 / Validation Accuracy: 0.836\n",
      "Epoch: 2 / Batch: 51712/60000 / Cost: 0.0812436 / Training Accuracy: 0.85546875 / Validation Accuracy: 0.847\n",
      "Epoch: 2 / Batch: 52224/60000 / Cost: 0.08902898 / Training Accuracy: 0.81640625 / Validation Accuracy: 0.821\n",
      "Epoch: 2 / Batch: 52736/60000 / Cost: 0.073105484 / Training Accuracy: 0.8769531 / Validation Accuracy: 0.831\n",
      "Epoch: 2 / Batch: 53248/60000 / Cost: 0.08069788 / Training Accuracy: 0.8496094 / Validation Accuracy: 0.838\n",
      "Epoch: 2 / Batch: 53760/60000 / Cost: 0.07321832 / Training Accuracy: 0.8808594 / Validation Accuracy: 0.833\n",
      "Epoch: 2 / Batch: 54272/60000 / Cost: 0.08048372 / Training Accuracy: 0.8574219 / Validation Accuracy: 0.847\n",
      "Epoch: 2 / Batch: 54784/60000 / Cost: 0.070460096 / Training Accuracy: 0.8808594 / Validation Accuracy: 0.861\n",
      "Epoch: 2 / Batch: 55296/60000 / Cost: 0.07652768 / Training Accuracy: 0.8496094 / Validation Accuracy: 0.853\n",
      "Epoch: 2 / Batch: 55808/60000 / Cost: 0.067462936 / Training Accuracy: 0.87109375 / Validation Accuracy: 0.858\n",
      "Epoch: 2 / Batch: 56320/60000 / Cost: 0.06627988 / Training Accuracy: 0.8769531 / Validation Accuracy: 0.866\n",
      "Epoch: 2 / Batch: 56832/60000 / Cost: 0.06645292 / Training Accuracy: 0.8964844 / Validation Accuracy: 0.865\n",
      "Epoch: 2 / Batch: 57344/60000 / Cost: 0.06717546 / Training Accuracy: 0.8828125 / Validation Accuracy: 0.868\n",
      "Epoch: 2 / Batch: 57856/60000 / Cost: 0.07376073 / Training Accuracy: 0.8691406 / Validation Accuracy: 0.866\n",
      "Epoch: 2 / Batch: 58368/60000 / Cost: 0.07418555 / Training Accuracy: 0.8691406 / Validation Accuracy: 0.86\n",
      "Epoch: 2 / Batch: 58880/60000 / Cost: 0.075792074 / Training Accuracy: 0.89453125 / Validation Accuracy: 0.866\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 / Batch: 59392/60000 / Cost: 0.07051817 / Training Accuracy: 0.8847656 / Validation Accuracy: 0.873\n",
      "Epoch: 2 / Batch: 59904/60000 / Cost: 0.060489815 / Training Accuracy: 0.90625 / Validation Accuracy: 0.869\n",
      "Epoch: 3 / Batch: 0/60000 / Cost: 0.071078934 / Training Accuracy: 0.8671875 / Validation Accuracy: 0.866\n",
      "Epoch: 3 / Batch: 512/60000 / Cost: 0.075263515 / Training Accuracy: 0.8613281 / Validation Accuracy: 0.881\n",
      "Epoch: 3 / Batch: 1024/60000 / Cost: 0.06106295 / Training Accuracy: 0.8925781 / Validation Accuracy: 0.878\n",
      "Epoch: 3 / Batch: 1536/60000 / Cost: 0.058805652 / Training Accuracy: 0.9003906 / Validation Accuracy: 0.871\n",
      "Epoch: 3 / Batch: 2048/60000 / Cost: 0.07241098 / Training Accuracy: 0.875 / Validation Accuracy: 0.879\n",
      "Epoch: 3 / Batch: 2560/60000 / Cost: 0.05919212 / Training Accuracy: 0.9003906 / Validation Accuracy: 0.866\n",
      "Epoch: 3 / Batch: 3072/60000 / Cost: 0.065432444 / Training Accuracy: 0.890625 / Validation Accuracy: 0.871\n",
      "Epoch: 3 / Batch: 3584/60000 / Cost: 0.066425376 / Training Accuracy: 0.8808594 / Validation Accuracy: 0.871\n",
      "Epoch: 3 / Batch: 4096/60000 / Cost: 0.07261576 / Training Accuracy: 0.859375 / Validation Accuracy: 0.883\n",
      "Epoch: 3 / Batch: 4608/60000 / Cost: 0.05887078 / Training Accuracy: 0.90234375 / Validation Accuracy: 0.886\n",
      "Epoch: 3 / Batch: 5120/60000 / Cost: 0.064972416 / Training Accuracy: 0.8769531 / Validation Accuracy: 0.89\n",
      "Epoch: 3 / Batch: 5632/60000 / Cost: 0.057810854 / Training Accuracy: 0.9003906 / Validation Accuracy: 0.882\n",
      "Epoch: 3 / Batch: 6144/60000 / Cost: 0.064597376 / Training Accuracy: 0.8886719 / Validation Accuracy: 0.878\n",
      "Epoch: 3 / Batch: 6656/60000 / Cost: 0.06702149 / Training Accuracy: 0.8847656 / Validation Accuracy: 0.875\n",
      "Epoch: 3 / Batch: 7168/60000 / Cost: 0.062476598 / Training Accuracy: 0.8984375 / Validation Accuracy: 0.876\n",
      "Epoch: 3 / Batch: 7680/60000 / Cost: 0.06048273 / Training Accuracy: 0.91015625 / Validation Accuracy: 0.872\n",
      "Epoch: 3 / Batch: 8192/60000 / Cost: 0.063503906 / Training Accuracy: 0.890625 / Validation Accuracy: 0.878\n",
      "Epoch: 3 / Batch: 8704/60000 / Cost: 0.06695156 / Training Accuracy: 0.88671875 / Validation Accuracy: 0.889\n",
      "Epoch: 3 / Batch: 9216/60000 / Cost: 0.06415489 / Training Accuracy: 0.8828125 / Validation Accuracy: 0.876\n",
      "Epoch: 3 / Batch: 9728/60000 / Cost: 0.0589417 / Training Accuracy: 0.9160156 / Validation Accuracy: 0.883\n",
      "Epoch: 3 / Batch: 10240/60000 / Cost: 0.055289306 / Training Accuracy: 0.90625 / Validation Accuracy: 0.882\n",
      "Epoch: 3 / Batch: 10752/60000 / Cost: 0.05527038 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.872\n",
      "Epoch: 3 / Batch: 11264/60000 / Cost: 0.057659578 / Training Accuracy: 0.90234375 / Validation Accuracy: 0.883\n",
      "Epoch: 3 / Batch: 11776/60000 / Cost: 0.0638458 / Training Accuracy: 0.89453125 / Validation Accuracy: 0.872\n",
      "Epoch: 3 / Batch: 12288/60000 / Cost: 0.062396277 / Training Accuracy: 0.88671875 / Validation Accuracy: 0.885\n",
      "Epoch: 3 / Batch: 12800/60000 / Cost: 0.06422261 / Training Accuracy: 0.8847656 / Validation Accuracy: 0.879\n",
      "Epoch: 3 / Batch: 13312/60000 / Cost: 0.068486616 / Training Accuracy: 0.8886719 / Validation Accuracy: 0.875\n",
      "Epoch: 3 / Batch: 13824/60000 / Cost: 0.06474839 / Training Accuracy: 0.88671875 / Validation Accuracy: 0.89\n",
      "Epoch: 3 / Batch: 14336/60000 / Cost: 0.064454265 / Training Accuracy: 0.8886719 / Validation Accuracy: 0.898\n",
      "Epoch: 3 / Batch: 14848/60000 / Cost: 0.061328262 / Training Accuracy: 0.8925781 / Validation Accuracy: 0.885\n",
      "Epoch: 3 / Batch: 15360/60000 / Cost: 0.053314555 / Training Accuracy: 0.91015625 / Validation Accuracy: 0.885\n",
      "Epoch: 3 / Batch: 15872/60000 / Cost: 0.056708913 / Training Accuracy: 0.9003906 / Validation Accuracy: 0.886\n",
      "Epoch: 3 / Batch: 16384/60000 / Cost: 0.068517104 / Training Accuracy: 0.8730469 / Validation Accuracy: 0.876\n",
      "Epoch: 3 / Batch: 16896/60000 / Cost: 0.05272665 / Training Accuracy: 0.90234375 / Validation Accuracy: 0.887\n",
      "Epoch: 3 / Batch: 17408/60000 / Cost: 0.06401827 / Training Accuracy: 0.88671875 / Validation Accuracy: 0.879\n",
      "Epoch: 3 / Batch: 17920/60000 / Cost: 0.053426303 / Training Accuracy: 0.90625 / Validation Accuracy: 0.883\n",
      "Epoch: 3 / Batch: 18432/60000 / Cost: 0.056678183 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.895\n",
      "Epoch: 3 / Batch: 18944/60000 / Cost: 0.053677388 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.901\n",
      "Epoch: 3 / Batch: 19456/60000 / Cost: 0.058039952 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.903\n",
      "Epoch: 3 / Batch: 19968/60000 / Cost: 0.05815385 / Training Accuracy: 0.8925781 / Validation Accuracy: 0.898\n",
      "Epoch: 3 / Batch: 20480/60000 / Cost: 0.055091165 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.888\n",
      "Epoch: 3 / Batch: 20992/60000 / Cost: 0.054517336 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.89\n",
      "Epoch: 3 / Batch: 21504/60000 / Cost: 0.06021403 / Training Accuracy: 0.8964844 / Validation Accuracy: 0.888\n",
      "Epoch: 3 / Batch: 22016/60000 / Cost: 0.05033102 / Training Accuracy: 0.9199219 / Validation Accuracy: 0.894\n",
      "Epoch: 3 / Batch: 22528/60000 / Cost: 0.057512175 / Training Accuracy: 0.8984375 / Validation Accuracy: 0.9\n",
      "Epoch: 3 / Batch: 23040/60000 / Cost: 0.044751 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.885\n",
      "Epoch: 3 / Batch: 23552/60000 / Cost: 0.052987598 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.882\n",
      "Epoch: 3 / Batch: 24064/60000 / Cost: 0.05902003 / Training Accuracy: 0.90625 / Validation Accuracy: 0.9\n",
      "Epoch: 3 / Batch: 24576/60000 / Cost: 0.058017887 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.903\n",
      "Epoch: 3 / Batch: 25088/60000 / Cost: 0.053207 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.905\n",
      "Epoch: 3 / Batch: 25600/60000 / Cost: 0.055520736 / Training Accuracy: 0.9082031 / Validation Accuracy: 0.891\n",
      "Epoch: 3 / Batch: 26112/60000 / Cost: 0.055902857 / Training Accuracy: 0.91015625 / Validation Accuracy: 0.894\n",
      "Epoch: 3 / Batch: 26624/60000 / Cost: 0.047497444 / Training Accuracy: 0.9160156 / Validation Accuracy: 0.903\n",
      "Epoch: 3 / Batch: 27136/60000 / Cost: 0.0560788 / Training Accuracy: 0.8964844 / Validation Accuracy: 0.907\n",
      "Epoch: 3 / Batch: 27648/60000 / Cost: 0.044934385 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.909\n",
      "Epoch: 3 / Batch: 28160/60000 / Cost: 0.055852167 / Training Accuracy: 0.90625 / Validation Accuracy: 0.906\n",
      "Epoch: 3 / Batch: 28672/60000 / Cost: 0.050260387 / Training Accuracy: 0.9082031 / Validation Accuracy: 0.903\n",
      "Epoch: 3 / Batch: 29184/60000 / Cost: 0.056095958 / Training Accuracy: 0.90625 / Validation Accuracy: 0.9\n",
      "Epoch: 3 / Batch: 29696/60000 / Cost: 0.04827387 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.897\n",
      "Epoch: 3 / Batch: 30208/60000 / Cost: 0.0520326 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.905\n",
      "Epoch: 3 / Batch: 30720/60000 / Cost: 0.04840027 / Training Accuracy: 0.921875 / Validation Accuracy: 0.905\n",
      "Epoch: 3 / Batch: 31232/60000 / Cost: 0.043614205 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.893\n",
      "Epoch: 3 / Batch: 31744/60000 / Cost: 0.046178855 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.9\n",
      "Epoch: 3 / Batch: 32256/60000 / Cost: 0.05973959 / Training Accuracy: 0.89453125 / Validation Accuracy: 0.909\n",
      "Epoch: 3 / Batch: 32768/60000 / Cost: 0.05516064 / Training Accuracy: 0.8984375 / Validation Accuracy: 0.899\n",
      "Epoch: 3 / Batch: 33280/60000 / Cost: 0.045338757 / Training Accuracy: 0.921875 / Validation Accuracy: 0.912\n",
      "Epoch: 3 / Batch: 33792/60000 / Cost: 0.039938867 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.911\n",
      "Epoch: 3 / Batch: 34304/60000 / Cost: 0.04618623 / Training Accuracy: 0.9160156 / Validation Accuracy: 0.907\n",
      "Epoch: 3 / Batch: 34816/60000 / Cost: 0.045462202 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.908\n",
      "Epoch: 3 / Batch: 35328/60000 / Cost: 0.050237127 / Training Accuracy: 0.9160156 / Validation Accuracy: 0.909\n",
      "Epoch: 3 / Batch: 35840/60000 / Cost: 0.04947217 / Training Accuracy: 0.90625 / Validation Accuracy: 0.912\n",
      "Epoch: 3 / Batch: 36352/60000 / Cost: 0.047112565 / Training Accuracy: 0.9199219 / Validation Accuracy: 0.909\n",
      "Epoch: 3 / Batch: 36864/60000 / Cost: 0.04627843 / Training Accuracy: 0.921875 / Validation Accuracy: 0.908\n",
      "Epoch: 3 / Batch: 37376/60000 / Cost: 0.04773605 / Training Accuracy: 0.921875 / Validation Accuracy: 0.903\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 / Batch: 37888/60000 / Cost: 0.048063174 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.9\n",
      "Epoch: 3 / Batch: 38400/60000 / Cost: 0.044573374 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.909\n",
      "Epoch: 3 / Batch: 38912/60000 / Cost: 0.054434158 / Training Accuracy: 0.9160156 / Validation Accuracy: 0.918\n",
      "Epoch: 3 / Batch: 39424/60000 / Cost: 0.051912226 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.914\n",
      "Epoch: 3 / Batch: 39936/60000 / Cost: 0.039240416 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.915\n",
      "Epoch: 3 / Batch: 40448/60000 / Cost: 0.042182 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.921\n",
      "Epoch: 3 / Batch: 40960/60000 / Cost: 0.045862865 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.915\n",
      "Epoch: 3 / Batch: 41472/60000 / Cost: 0.0541723 / Training Accuracy: 0.90234375 / Validation Accuracy: 0.916\n",
      "Epoch: 3 / Batch: 41984/60000 / Cost: 0.05456557 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.916\n",
      "Epoch: 3 / Batch: 42496/60000 / Cost: 0.046617743 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.901\n",
      "Epoch: 3 / Batch: 43008/60000 / Cost: 0.05240766 / Training Accuracy: 0.91015625 / Validation Accuracy: 0.913\n",
      "Epoch: 3 / Batch: 43520/60000 / Cost: 0.044254094 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.915\n",
      "Epoch: 3 / Batch: 44032/60000 / Cost: 0.044742066 / Training Accuracy: 0.921875 / Validation Accuracy: 0.924\n",
      "Epoch: 3 / Batch: 44544/60000 / Cost: 0.048295118 / Training Accuracy: 0.9082031 / Validation Accuracy: 0.911\n",
      "Epoch: 3 / Batch: 45056/60000 / Cost: 0.04858855 / Training Accuracy: 0.9199219 / Validation Accuracy: 0.907\n",
      "Epoch: 3 / Batch: 45568/60000 / Cost: 0.04353672 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.914\n",
      "Epoch: 3 / Batch: 46080/60000 / Cost: 0.04217019 / Training Accuracy: 0.921875 / Validation Accuracy: 0.91\n",
      "Epoch: 3 / Batch: 46592/60000 / Cost: 0.052910782 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.883\n",
      "Epoch: 3 / Batch: 47104/60000 / Cost: 0.04719931 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.912\n",
      "Epoch: 3 / Batch: 47616/60000 / Cost: 0.051274814 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.91\n",
      "Epoch: 3 / Batch: 48128/60000 / Cost: 0.040868014 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.903\n",
      "Epoch: 3 / Batch: 48640/60000 / Cost: 0.04774911 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.911\n",
      "Epoch: 3 / Batch: 49152/60000 / Cost: 0.05509302 / Training Accuracy: 0.8984375 / Validation Accuracy: 0.892\n",
      "Epoch: 3 / Batch: 49664/60000 / Cost: 0.041308604 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.914\n",
      "Epoch: 3 / Batch: 50176/60000 / Cost: 0.04484456 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.91\n",
      "Epoch: 3 / Batch: 50688/60000 / Cost: 0.04687513 / Training Accuracy: 0.921875 / Validation Accuracy: 0.909\n",
      "Epoch: 3 / Batch: 51200/60000 / Cost: 0.047313698 / Training Accuracy: 0.90625 / Validation Accuracy: 0.911\n",
      "Epoch: 3 / Batch: 51712/60000 / Cost: 0.042686887 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.919\n",
      "Epoch: 3 / Batch: 52224/60000 / Cost: 0.035725273 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.919\n",
      "Epoch: 3 / Batch: 52736/60000 / Cost: 0.04264303 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.916\n",
      "Epoch: 3 / Batch: 53248/60000 / Cost: 0.039161433 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.922\n",
      "Epoch: 3 / Batch: 53760/60000 / Cost: 0.03829787 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.926\n",
      "Epoch: 3 / Batch: 54272/60000 / Cost: 0.042358637 / Training Accuracy: 0.9375 / Validation Accuracy: 0.917\n",
      "Epoch: 3 / Batch: 54784/60000 / Cost: 0.054878242 / Training Accuracy: 0.91015625 / Validation Accuracy: 0.913\n",
      "Epoch: 3 / Batch: 55296/60000 / Cost: 0.043396432 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.914\n",
      "Epoch: 3 / Batch: 55808/60000 / Cost: 0.041095905 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.919\n",
      "Epoch: 3 / Batch: 56320/60000 / Cost: 0.039299138 / Training Accuracy: 0.9375 / Validation Accuracy: 0.922\n",
      "Epoch: 3 / Batch: 56832/60000 / Cost: 0.051440895 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.915\n",
      "Epoch: 3 / Batch: 57344/60000 / Cost: 0.044114854 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.909\n",
      "Epoch: 3 / Batch: 57856/60000 / Cost: 0.037985798 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.92\n",
      "Epoch: 3 / Batch: 58368/60000 / Cost: 0.045496147 / Training Accuracy: 0.9375 / Validation Accuracy: 0.925\n",
      "Epoch: 3 / Batch: 58880/60000 / Cost: 0.04296895 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.918\n",
      "Epoch: 3 / Batch: 59392/60000 / Cost: 0.04771054 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.905\n",
      "Epoch: 3 / Batch: 59904/60000 / Cost: 0.017645253 / Training Accuracy: 0.9895833 / Validation Accuracy: 0.902\n",
      "Epoch: 4 / Batch: 0/60000 / Cost: 0.04080639 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.91\n",
      "Epoch: 4 / Batch: 512/60000 / Cost: 0.050388765 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.922\n",
      "Epoch: 4 / Batch: 1024/60000 / Cost: 0.046811838 / Training Accuracy: 0.9199219 / Validation Accuracy: 0.923\n",
      "Epoch: 4 / Batch: 1536/60000 / Cost: 0.039909273 / Training Accuracy: 0.9375 / Validation Accuracy: 0.921\n",
      "Epoch: 4 / Batch: 2048/60000 / Cost: 0.043028235 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.916\n",
      "Epoch: 4 / Batch: 2560/60000 / Cost: 0.043601476 / Training Accuracy: 0.921875 / Validation Accuracy: 0.913\n",
      "Epoch: 4 / Batch: 3072/60000 / Cost: 0.047036067 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.915\n",
      "Epoch: 4 / Batch: 3584/60000 / Cost: 0.040248193 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.906\n",
      "Epoch: 4 / Batch: 4096/60000 / Cost: 0.034312356 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.922\n",
      "Epoch: 4 / Batch: 4608/60000 / Cost: 0.03574414 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.924\n",
      "Epoch: 4 / Batch: 5120/60000 / Cost: 0.038981352 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.917\n",
      "Epoch: 4 / Batch: 5632/60000 / Cost: 0.0421971 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.92\n",
      "Epoch: 4 / Batch: 6144/60000 / Cost: 0.035357736 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.921\n",
      "Epoch: 4 / Batch: 6656/60000 / Cost: 0.036800332 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.919\n",
      "Epoch: 4 / Batch: 7168/60000 / Cost: 0.042853523 / Training Accuracy: 0.9375 / Validation Accuracy: 0.925\n",
      "Epoch: 4 / Batch: 7680/60000 / Cost: 0.037713766 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.931\n",
      "Epoch: 4 / Batch: 8192/60000 / Cost: 0.04114578 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.926\n",
      "Epoch: 4 / Batch: 8704/60000 / Cost: 0.043837424 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.915\n",
      "Epoch: 4 / Batch: 9216/60000 / Cost: 0.0429718 / Training Accuracy: 0.921875 / Validation Accuracy: 0.926\n",
      "Epoch: 4 / Batch: 9728/60000 / Cost: 0.038786612 / Training Accuracy: 0.9375 / Validation Accuracy: 0.923\n",
      "Epoch: 4 / Batch: 10240/60000 / Cost: 0.049997956 / Training Accuracy: 0.9082031 / Validation Accuracy: 0.925\n",
      "Epoch: 4 / Batch: 10752/60000 / Cost: 0.033401147 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.929\n",
      "Epoch: 4 / Batch: 11264/60000 / Cost: 0.04268914 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.934\n",
      "Epoch: 4 / Batch: 11776/60000 / Cost: 0.041589033 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.923\n",
      "Epoch: 4 / Batch: 12288/60000 / Cost: 0.03722601 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.927\n",
      "Epoch: 4 / Batch: 12800/60000 / Cost: 0.040943574 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.937\n",
      "Epoch: 4 / Batch: 13312/60000 / Cost: 0.0472999 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.928\n",
      "Epoch: 4 / Batch: 13824/60000 / Cost: 0.038173728 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.928\n",
      "Epoch: 4 / Batch: 14336/60000 / Cost: 0.03440534 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.928\n",
      "Epoch: 4 / Batch: 14848/60000 / Cost: 0.03756154 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.92\n",
      "Epoch: 4 / Batch: 15360/60000 / Cost: 0.043759823 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.917\n",
      "Epoch: 4 / Batch: 15872/60000 / Cost: 0.03665629 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.923\n",
      "Epoch: 4 / Batch: 16384/60000 / Cost: 0.044904742 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.918\n",
      "Epoch: 4 / Batch: 16896/60000 / Cost: 0.04595322 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.908\n",
      "Epoch: 4 / Batch: 17408/60000 / Cost: 0.042942323 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.914\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 / Batch: 17920/60000 / Cost: 0.039842147 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.925\n",
      "Epoch: 4 / Batch: 18432/60000 / Cost: 0.041365787 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.92\n",
      "Epoch: 4 / Batch: 18944/60000 / Cost: 0.050929397 / Training Accuracy: 0.9160156 / Validation Accuracy: 0.916\n",
      "Epoch: 4 / Batch: 19456/60000 / Cost: 0.05054315 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.924\n",
      "Epoch: 4 / Batch: 19968/60000 / Cost: 0.037848514 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.918\n",
      "Epoch: 4 / Batch: 20480/60000 / Cost: 0.041178036 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.919\n",
      "Epoch: 4 / Batch: 20992/60000 / Cost: 0.034022715 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.929\n",
      "Epoch: 4 / Batch: 21504/60000 / Cost: 0.035697274 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.926\n",
      "Epoch: 4 / Batch: 22016/60000 / Cost: 0.053170502 / Training Accuracy: 0.90625 / Validation Accuracy: 0.921\n",
      "Epoch: 4 / Batch: 22528/60000 / Cost: 0.04912261 / Training Accuracy: 0.9082031 / Validation Accuracy: 0.932\n",
      "Epoch: 4 / Batch: 23040/60000 / Cost: 0.036919333 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.934\n",
      "Epoch: 4 / Batch: 23552/60000 / Cost: 0.04261712 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.928\n",
      "Epoch: 4 / Batch: 24064/60000 / Cost: 0.031329677 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.933\n",
      "Epoch: 4 / Batch: 24576/60000 / Cost: 0.040779095 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.932\n",
      "Epoch: 4 / Batch: 25088/60000 / Cost: 0.042546064 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.934\n",
      "Epoch: 4 / Batch: 25600/60000 / Cost: 0.038280856 / Training Accuracy: 0.9375 / Validation Accuracy: 0.929\n",
      "Epoch: 4 / Batch: 26112/60000 / Cost: 0.03899603 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.933\n",
      "Epoch: 4 / Batch: 26624/60000 / Cost: 0.039165527 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.928\n",
      "Epoch: 4 / Batch: 27136/60000 / Cost: 0.043285407 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.918\n",
      "Epoch: 4 / Batch: 27648/60000 / Cost: 0.031797312 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.925\n",
      "Epoch: 4 / Batch: 28160/60000 / Cost: 0.042036228 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.927\n",
      "Epoch: 4 / Batch: 28672/60000 / Cost: 0.032278214 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.926\n",
      "Epoch: 4 / Batch: 29184/60000 / Cost: 0.042592358 / Training Accuracy: 0.9199219 / Validation Accuracy: 0.932\n",
      "Epoch: 4 / Batch: 29696/60000 / Cost: 0.033722114 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.93\n",
      "Epoch: 4 / Batch: 30208/60000 / Cost: 0.037440844 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.914\n",
      "Epoch: 4 / Batch: 30720/60000 / Cost: 0.03646613 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.921\n",
      "Epoch: 4 / Batch: 31232/60000 / Cost: 0.04032162 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.931\n",
      "Epoch: 4 / Batch: 31744/60000 / Cost: 0.04008896 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.927\n",
      "Epoch: 4 / Batch: 32256/60000 / Cost: 0.036583632 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.929\n",
      "Epoch: 4 / Batch: 32768/60000 / Cost: 0.029356325 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.928\n",
      "Epoch: 4 / Batch: 33280/60000 / Cost: 0.03268247 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.927\n",
      "Epoch: 4 / Batch: 33792/60000 / Cost: 0.032373596 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.921\n",
      "Epoch: 4 / Batch: 34304/60000 / Cost: 0.038177125 / Training Accuracy: 0.9375 / Validation Accuracy: 0.93\n",
      "Epoch: 4 / Batch: 34816/60000 / Cost: 0.032570668 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.937\n",
      "Epoch: 4 / Batch: 35328/60000 / Cost: 0.032647006 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.938\n",
      "Epoch: 4 / Batch: 35840/60000 / Cost: 0.031369288 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.942\n",
      "Epoch: 4 / Batch: 36352/60000 / Cost: 0.042848703 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.932\n",
      "Epoch: 4 / Batch: 36864/60000 / Cost: 0.03825534 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.933\n",
      "Epoch: 4 / Batch: 37376/60000 / Cost: 0.030704748 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.937\n",
      "Epoch: 4 / Batch: 37888/60000 / Cost: 0.03334911 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.937\n",
      "Epoch: 4 / Batch: 38400/60000 / Cost: 0.036627956 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.932\n",
      "Epoch: 4 / Batch: 38912/60000 / Cost: 0.035305694 / Training Accuracy: 0.9375 / Validation Accuracy: 0.934\n",
      "Epoch: 4 / Batch: 39424/60000 / Cost: 0.03763681 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.946\n",
      "Epoch: 4 / Batch: 39936/60000 / Cost: 0.029945249 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.941\n",
      "Epoch: 4 / Batch: 40448/60000 / Cost: 0.03602711 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.929\n",
      "Epoch: 4 / Batch: 40960/60000 / Cost: 0.03878566 / Training Accuracy: 0.9375 / Validation Accuracy: 0.933\n",
      "Epoch: 4 / Batch: 41472/60000 / Cost: 0.027405951 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.932\n",
      "Epoch: 4 / Batch: 41984/60000 / Cost: 0.030992359 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.931\n",
      "Epoch: 4 / Batch: 42496/60000 / Cost: 0.03186553 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.936\n",
      "Epoch: 4 / Batch: 43008/60000 / Cost: 0.027200803 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.938\n",
      "Epoch: 4 / Batch: 43520/60000 / Cost: 0.027927842 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.941\n",
      "Epoch: 4 / Batch: 44032/60000 / Cost: 0.031628117 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.94\n",
      "Epoch: 4 / Batch: 44544/60000 / Cost: 0.027391478 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.937\n",
      "Epoch: 4 / Batch: 45056/60000 / Cost: 0.03107206 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.941\n",
      "Epoch: 4 / Batch: 45568/60000 / Cost: 0.03060997 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.943\n",
      "Epoch: 4 / Batch: 46080/60000 / Cost: 0.034302518 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.944\n",
      "Epoch: 4 / Batch: 46592/60000 / Cost: 0.03614505 / Training Accuracy: 0.9375 / Validation Accuracy: 0.941\n",
      "Epoch: 4 / Batch: 47104/60000 / Cost: 0.023899445 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.936\n",
      "Epoch: 4 / Batch: 47616/60000 / Cost: 0.03171418 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.945\n",
      "Epoch: 4 / Batch: 48128/60000 / Cost: 0.031710707 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.94\n",
      "Epoch: 4 / Batch: 48640/60000 / Cost: 0.03902935 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.937\n",
      "Epoch: 4 / Batch: 49152/60000 / Cost: 0.030976236 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.939\n",
      "Epoch: 4 / Batch: 49664/60000 / Cost: 0.034398817 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.944\n",
      "Epoch: 4 / Batch: 50176/60000 / Cost: 0.030426964 / Training Accuracy: 0.953125 / Validation Accuracy: 0.947\n",
      "Epoch: 4 / Batch: 50688/60000 / Cost: 0.025894184 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.95\n",
      "Epoch: 4 / Batch: 51200/60000 / Cost: 0.031371914 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.95\n",
      "Epoch: 4 / Batch: 51712/60000 / Cost: 0.034133065 / Training Accuracy: 0.9375 / Validation Accuracy: 0.947\n",
      "Epoch: 4 / Batch: 52224/60000 / Cost: 0.029613197 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.941\n",
      "Epoch: 4 / Batch: 52736/60000 / Cost: 0.034915965 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.939\n",
      "Epoch: 4 / Batch: 53248/60000 / Cost: 0.034658134 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.941\n",
      "Epoch: 4 / Batch: 53760/60000 / Cost: 0.030936211 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.944\n",
      "Epoch: 4 / Batch: 54272/60000 / Cost: 0.02971062 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.944\n",
      "Epoch: 4 / Batch: 54784/60000 / Cost: 0.033252474 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.939\n",
      "Epoch: 4 / Batch: 55296/60000 / Cost: 0.023009207 / Training Accuracy: 0.96875 / Validation Accuracy: 0.943\n",
      "Epoch: 4 / Batch: 55808/60000 / Cost: 0.026065638 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.947\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 / Batch: 56320/60000 / Cost: 0.029699009 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.947\n",
      "Epoch: 4 / Batch: 56832/60000 / Cost: 0.031439453 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.945\n",
      "Epoch: 4 / Batch: 57344/60000 / Cost: 0.028401121 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.941\n",
      "Epoch: 4 / Batch: 57856/60000 / Cost: 0.03469876 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.941\n",
      "Epoch: 4 / Batch: 58368/60000 / Cost: 0.0394081 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.942\n",
      "Epoch: 4 / Batch: 58880/60000 / Cost: 0.030926596 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.943\n",
      "Epoch: 4 / Batch: 59392/60000 / Cost: 0.024551228 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.943\n",
      "Epoch: 4 / Batch: 59904/60000 / Cost: 0.016982399 / Training Accuracy: 0.9895833 / Validation Accuracy: 0.943\n",
      "Epoch: 5 / Batch: 0/60000 / Cost: 0.03438762 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.945\n",
      "Epoch: 5 / Batch: 512/60000 / Cost: 0.030791085 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.942\n",
      "Epoch: 5 / Batch: 1024/60000 / Cost: 0.03807261 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.941\n",
      "Epoch: 5 / Batch: 1536/60000 / Cost: 0.02831108 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.94\n",
      "Epoch: 5 / Batch: 2048/60000 / Cost: 0.035895385 / Training Accuracy: 0.9375 / Validation Accuracy: 0.947\n",
      "Epoch: 5 / Batch: 2560/60000 / Cost: 0.026073998 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.947\n",
      "Epoch: 5 / Batch: 3072/60000 / Cost: 0.034878947 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.945\n",
      "Epoch: 5 / Batch: 3584/60000 / Cost: 0.027740002 / Training Accuracy: 0.96875 / Validation Accuracy: 0.947\n",
      "Epoch: 5 / Batch: 4096/60000 / Cost: 0.033206027 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.95\n",
      "Epoch: 5 / Batch: 4608/60000 / Cost: 0.028927356 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.936\n",
      "Epoch: 5 / Batch: 5120/60000 / Cost: 0.031776514 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.941\n",
      "Epoch: 5 / Batch: 5632/60000 / Cost: 0.028117891 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.944\n",
      "Epoch: 5 / Batch: 6144/60000 / Cost: 0.027143484 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.929\n",
      "Epoch: 5 / Batch: 6656/60000 / Cost: 0.042166043 / Training Accuracy: 0.921875 / Validation Accuracy: 0.931\n",
      "Epoch: 5 / Batch: 7168/60000 / Cost: 0.03892375 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.935\n",
      "Epoch: 5 / Batch: 7680/60000 / Cost: 0.021352679 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.938\n",
      "Epoch: 5 / Batch: 8192/60000 / Cost: 0.031678505 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.939\n",
      "Epoch: 5 / Batch: 8704/60000 / Cost: 0.035120346 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.943\n",
      "Epoch: 5 / Batch: 9216/60000 / Cost: 0.041892607 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.946\n",
      "Epoch: 5 / Batch: 9728/60000 / Cost: 0.03840073 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.939\n",
      "Epoch: 5 / Batch: 10240/60000 / Cost: 0.029590834 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.936\n",
      "Epoch: 5 / Batch: 10752/60000 / Cost: 0.032376487 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.933\n",
      "Epoch: 5 / Batch: 11264/60000 / Cost: 0.027751615 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.938\n",
      "Epoch: 5 / Batch: 11776/60000 / Cost: 0.025414104 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.949\n",
      "Epoch: 5 / Batch: 12288/60000 / Cost: 0.030042559 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.94\n",
      "Epoch: 5 / Batch: 12800/60000 / Cost: 0.036923017 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.94\n",
      "Epoch: 5 / Batch: 13312/60000 / Cost: 0.027297527 / Training Accuracy: 0.953125 / Validation Accuracy: 0.942\n",
      "Epoch: 5 / Batch: 13824/60000 / Cost: 0.027951155 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.941\n",
      "Epoch: 5 / Batch: 14336/60000 / Cost: 0.028518867 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.936\n",
      "Epoch: 5 / Batch: 14848/60000 / Cost: 0.026828315 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.941\n",
      "Epoch: 5 / Batch: 15360/60000 / Cost: 0.02596634 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.942\n",
      "Epoch: 5 / Batch: 15872/60000 / Cost: 0.03829069 / Training Accuracy: 0.9375 / Validation Accuracy: 0.943\n",
      "Epoch: 5 / Batch: 16384/60000 / Cost: 0.036444634 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.944\n",
      "Epoch: 5 / Batch: 16896/60000 / Cost: 0.028498221 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.943\n",
      "Epoch: 5 / Batch: 17408/60000 / Cost: 0.030991074 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.944\n",
      "Epoch: 5 / Batch: 17920/60000 / Cost: 0.025726464 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.946\n",
      "Epoch: 5 / Batch: 18432/60000 / Cost: 0.023027986 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.942\n",
      "Epoch: 5 / Batch: 18944/60000 / Cost: 0.02979743 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.94\n",
      "Epoch: 5 / Batch: 19456/60000 / Cost: 0.032031644 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.938\n",
      "Epoch: 5 / Batch: 19968/60000 / Cost: 0.028109204 / Training Accuracy: 0.953125 / Validation Accuracy: 0.938\n",
      "Epoch: 5 / Batch: 20480/60000 / Cost: 0.032342378 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.941\n",
      "Epoch: 5 / Batch: 20992/60000 / Cost: 0.031599175 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.94\n",
      "Epoch: 5 / Batch: 21504/60000 / Cost: 0.028013643 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.948\n",
      "Epoch: 5 / Batch: 22016/60000 / Cost: 0.029175196 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.952\n",
      "Epoch: 5 / Batch: 22528/60000 / Cost: 0.024914924 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.944\n",
      "Epoch: 5 / Batch: 23040/60000 / Cost: 0.0324544 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.939\n",
      "Epoch: 5 / Batch: 23552/60000 / Cost: 0.027066808 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.937\n",
      "Epoch: 5 / Batch: 24064/60000 / Cost: 0.022622712 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.941\n",
      "Epoch: 5 / Batch: 24576/60000 / Cost: 0.0279454 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.942\n",
      "Epoch: 5 / Batch: 25088/60000 / Cost: 0.022440564 / Training Accuracy: 0.96875 / Validation Accuracy: 0.948\n",
      "Epoch: 5 / Batch: 25600/60000 / Cost: 0.021263067 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.953\n",
      "Epoch: 5 / Batch: 26112/60000 / Cost: 0.023893807 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.953\n",
      "Epoch: 5 / Batch: 26624/60000 / Cost: 0.025317442 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.951\n",
      "Epoch: 5 / Batch: 27136/60000 / Cost: 0.02939645 / Training Accuracy: 0.953125 / Validation Accuracy: 0.951\n",
      "Epoch: 5 / Batch: 27648/60000 / Cost: 0.023349665 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.948\n",
      "Epoch: 5 / Batch: 28160/60000 / Cost: 0.026990866 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.953\n",
      "Epoch: 5 / Batch: 28672/60000 / Cost: 0.024626156 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 29184/60000 / Cost: 0.023745691 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.946\n",
      "Epoch: 5 / Batch: 29696/60000 / Cost: 0.027581805 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.943\n",
      "Epoch: 5 / Batch: 30208/60000 / Cost: 0.031961564 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.944\n",
      "Epoch: 5 / Batch: 30720/60000 / Cost: 0.026096847 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.948\n",
      "Epoch: 5 / Batch: 31232/60000 / Cost: 0.0350667 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.949\n",
      "Epoch: 5 / Batch: 31744/60000 / Cost: 0.024956536 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.946\n",
      "Epoch: 5 / Batch: 32256/60000 / Cost: 0.024243224 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.95\n",
      "Epoch: 5 / Batch: 32768/60000 / Cost: 0.0265531 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 33280/60000 / Cost: 0.023579588 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 33792/60000 / Cost: 0.024062019 / Training Accuracy: 0.953125 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 34304/60000 / Cost: 0.02359434 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.953\n",
      "Epoch: 5 / Batch: 34816/60000 / Cost: 0.028296003 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.953\n",
      "Epoch: 5 / Batch: 35328/60000 / Cost: 0.023816507 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.95\n",
      "Epoch: 5 / Batch: 35840/60000 / Cost: 0.030951474 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.948\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 / Batch: 36352/60000 / Cost: 0.028418541 / Training Accuracy: 0.9375 / Validation Accuracy: 0.94\n",
      "Epoch: 5 / Batch: 36864/60000 / Cost: 0.026095804 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.946\n",
      "Epoch: 5 / Batch: 37376/60000 / Cost: 0.027218917 / Training Accuracy: 0.953125 / Validation Accuracy: 0.948\n",
      "Epoch: 5 / Batch: 37888/60000 / Cost: 0.023431817 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 38400/60000 / Cost: 0.030531455 / Training Accuracy: 0.953125 / Validation Accuracy: 0.954\n",
      "Epoch: 5 / Batch: 38912/60000 / Cost: 0.027720397 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.945\n",
      "Epoch: 5 / Batch: 39424/60000 / Cost: 0.02872786 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.941\n",
      "Epoch: 5 / Batch: 39936/60000 / Cost: 0.020449009 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.949\n",
      "Epoch: 5 / Batch: 40448/60000 / Cost: 0.02540898 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.945\n",
      "Epoch: 5 / Batch: 40960/60000 / Cost: 0.0337331 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.951\n",
      "Epoch: 5 / Batch: 41472/60000 / Cost: 0.031759687 / Training Accuracy: 0.953125 / Validation Accuracy: 0.953\n",
      "Epoch: 5 / Batch: 41984/60000 / Cost: 0.02734809 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 42496/60000 / Cost: 0.025143078 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.954\n",
      "Epoch: 5 / Batch: 43008/60000 / Cost: 0.027663622 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.946\n",
      "Epoch: 5 / Batch: 43520/60000 / Cost: 0.018868674 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.949\n",
      "Epoch: 5 / Batch: 44032/60000 / Cost: 0.020228276 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 44544/60000 / Cost: 0.019797023 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 5 / Batch: 45056/60000 / Cost: 0.030534435 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 45568/60000 / Cost: 0.027899856 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.952\n",
      "Epoch: 5 / Batch: 46080/60000 / Cost: 0.026320677 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.952\n",
      "Epoch: 5 / Batch: 46592/60000 / Cost: 0.022224056 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.946\n",
      "Epoch: 5 / Batch: 47104/60000 / Cost: 0.01954143 / Training Accuracy: 0.96875 / Validation Accuracy: 0.951\n",
      "Epoch: 5 / Batch: 47616/60000 / Cost: 0.02941234 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 48128/60000 / Cost: 0.023354193 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.951\n",
      "Epoch: 5 / Batch: 48640/60000 / Cost: 0.026960518 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 49152/60000 / Cost: 0.02287821 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 49664/60000 / Cost: 0.02316885 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 50176/60000 / Cost: 0.04048735 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 50688/60000 / Cost: 0.03242577 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.954\n",
      "Epoch: 5 / Batch: 51200/60000 / Cost: 0.024234679 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 51712/60000 / Cost: 0.022535812 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 52224/60000 / Cost: 0.01956391 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 52736/60000 / Cost: 0.023876797 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 53248/60000 / Cost: 0.031391736 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 53760/60000 / Cost: 0.021074615 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 54272/60000 / Cost: 0.023943773 / Training Accuracy: 0.953125 / Validation Accuracy: 0.963\n",
      "Epoch: 5 / Batch: 54784/60000 / Cost: 0.022978056 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 55296/60000 / Cost: 0.022140574 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 55808/60000 / Cost: 0.030738497 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 56320/60000 / Cost: 0.023818087 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.961\n",
      "Epoch: 5 / Batch: 56832/60000 / Cost: 0.023013588 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 57344/60000 / Cost: 0.029052328 / Training Accuracy: 0.953125 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 57856/60000 / Cost: 0.02477049 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 58368/60000 / Cost: 0.023381138 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 58880/60000 / Cost: 0.02573936 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 59392/60000 / Cost: 0.025842756 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 59904/60000 / Cost: 0.022531986 / Training Accuracy: 0.9791667 / Validation Accuracy: 0.956\n",
      "Epoch: 6 / Batch: 0/60000 / Cost: 0.022788305 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.956\n",
      "Epoch: 6 / Batch: 512/60000 / Cost: 0.033562873 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 1024/60000 / Cost: 0.022515843 / Training Accuracy: 0.96875 / Validation Accuracy: 0.959\n",
      "Epoch: 6 / Batch: 1536/60000 / Cost: 0.017612685 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.959\n",
      "Epoch: 6 / Batch: 2048/60000 / Cost: 0.024288066 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 2560/60000 / Cost: 0.020491559 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.956\n",
      "Epoch: 6 / Batch: 3072/60000 / Cost: 0.019306218 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.949\n",
      "Epoch: 6 / Batch: 3584/60000 / Cost: 0.019807326 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.955\n",
      "Epoch: 6 / Batch: 4096/60000 / Cost: 0.025017072 / Training Accuracy: 0.953125 / Validation Accuracy: 0.949\n",
      "Epoch: 6 / Batch: 4608/60000 / Cost: 0.025026042 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.944\n",
      "Epoch: 6 / Batch: 5120/60000 / Cost: 0.031209135 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.95\n",
      "Epoch: 6 / Batch: 5632/60000 / Cost: 0.020728346 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 6144/60000 / Cost: 0.017797066 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 6656/60000 / Cost: 0.022715487 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 7168/60000 / Cost: 0.026088098 / Training Accuracy: 0.953125 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 7680/60000 / Cost: 0.019591961 / Training Accuracy: 0.96875 / Validation Accuracy: 0.955\n",
      "Epoch: 6 / Batch: 8192/60000 / Cost: 0.019082813 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.952\n",
      "Epoch: 6 / Batch: 8704/60000 / Cost: 0.021537628 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.951\n",
      "Epoch: 6 / Batch: 9216/60000 / Cost: 0.029017663 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.952\n",
      "Epoch: 6 / Batch: 9728/60000 / Cost: 0.013800254 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.948\n",
      "Epoch: 6 / Batch: 10240/60000 / Cost: 0.029245045 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.95\n",
      "Epoch: 6 / Batch: 10752/60000 / Cost: 0.028192911 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.956\n",
      "Epoch: 6 / Batch: 11264/60000 / Cost: 0.01917861 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.951\n",
      "Epoch: 6 / Batch: 11776/60000 / Cost: 0.01907593 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.951\n",
      "Epoch: 6 / Batch: 12288/60000 / Cost: 0.023411447 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.95\n",
      "Epoch: 6 / Batch: 12800/60000 / Cost: 0.018586366 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.949\n",
      "Epoch: 6 / Batch: 13312/60000 / Cost: 0.018991765 / Training Accuracy: 0.96875 / Validation Accuracy: 0.953\n",
      "Epoch: 6 / Batch: 13824/60000 / Cost: 0.018614637 / Training Accuracy: 0.96875 / Validation Accuracy: 0.957\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 / Batch: 14336/60000 / Cost: 0.021105196 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 14848/60000 / Cost: 0.024839357 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 15360/60000 / Cost: 0.023386788 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 15872/60000 / Cost: 0.022079391 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 16384/60000 / Cost: 0.024497043 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 16896/60000 / Cost: 0.021097267 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 17408/60000 / Cost: 0.019825142 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 17920/60000 / Cost: 0.024055272 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 18432/60000 / Cost: 0.020482667 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 18944/60000 / Cost: 0.02001602 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 19456/60000 / Cost: 0.023249997 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 19968/60000 / Cost: 0.016842345 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 20480/60000 / Cost: 0.02610378 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 20992/60000 / Cost: 0.025639048 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 21504/60000 / Cost: 0.016480427 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 22016/60000 / Cost: 0.023669533 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 22528/60000 / Cost: 0.025934612 / Training Accuracy: 0.953125 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 23040/60000 / Cost: 0.017120238 / Training Accuracy: 0.96875 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 23552/60000 / Cost: 0.019361144 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 24064/60000 / Cost: 0.021487098 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 24576/60000 / Cost: 0.021376668 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 25088/60000 / Cost: 0.018782254 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 6 / Batch: 25600/60000 / Cost: 0.014040573 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 26112/60000 / Cost: 0.022243883 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 26624/60000 / Cost: 0.0249555 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 27136/60000 / Cost: 0.017820593 / Training Accuracy: 0.96875 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 27648/60000 / Cost: 0.018437637 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 28160/60000 / Cost: 0.016462257 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 28672/60000 / Cost: 0.02636381 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.959\n",
      "Epoch: 6 / Batch: 29184/60000 / Cost: 0.021367308 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 29696/60000 / Cost: 0.016647302 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 30208/60000 / Cost: 0.01521601 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.959\n",
      "Epoch: 6 / Batch: 30720/60000 / Cost: 0.020347461 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 31232/60000 / Cost: 0.0203698 / Training Accuracy: 0.96875 / Validation Accuracy: 0.956\n",
      "Epoch: 6 / Batch: 31744/60000 / Cost: 0.020266822 / Training Accuracy: 0.96875 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 32256/60000 / Cost: 0.02629667 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 32768/60000 / Cost: 0.021831404 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 33280/60000 / Cost: 0.015925068 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 33792/60000 / Cost: 0.020851357 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 34304/60000 / Cost: 0.013386907 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.955\n",
      "Epoch: 6 / Batch: 34816/60000 / Cost: 0.018232288 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.954\n",
      "Epoch: 6 / Batch: 35328/60000 / Cost: 0.02332269 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 35840/60000 / Cost: 0.024077643 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 36352/60000 / Cost: 0.018556828 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 36864/60000 / Cost: 0.021005772 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 37376/60000 / Cost: 0.022346964 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.955\n",
      "Epoch: 6 / Batch: 37888/60000 / Cost: 0.027593195 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.954\n",
      "Epoch: 6 / Batch: 38400/60000 / Cost: 0.02262524 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 38912/60000 / Cost: 0.01965416 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 39424/60000 / Cost: 0.02079273 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 39936/60000 / Cost: 0.018984545 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 40448/60000 / Cost: 0.016937198 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 40960/60000 / Cost: 0.026173523 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 41472/60000 / Cost: 0.014763695 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 41984/60000 / Cost: 0.01920062 / Training Accuracy: 0.96875 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 42496/60000 / Cost: 0.014486191 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 43008/60000 / Cost: 0.02205879 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 43520/60000 / Cost: 0.019560244 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 44032/60000 / Cost: 0.024704322 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.954\n",
      "Epoch: 6 / Batch: 44544/60000 / Cost: 0.020010104 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.949\n",
      "Epoch: 6 / Batch: 45056/60000 / Cost: 0.019588519 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.956\n",
      "Epoch: 6 / Batch: 45568/60000 / Cost: 0.012080347 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 46080/60000 / Cost: 0.01405676 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 46592/60000 / Cost: 0.019657608 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 47104/60000 / Cost: 0.019458309 / Training Accuracy: 0.96875 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 47616/60000 / Cost: 0.026579618 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 48128/60000 / Cost: 0.0113703385 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.954\n",
      "Epoch: 6 / Batch: 48640/60000 / Cost: 0.027540335 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.948\n",
      "Epoch: 6 / Batch: 49152/60000 / Cost: 0.024050098 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.952\n",
      "Epoch: 6 / Batch: 49664/60000 / Cost: 0.016969595 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 50176/60000 / Cost: 0.02069496 / Training Accuracy: 0.96875 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 50688/60000 / Cost: 0.020245275 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 51200/60000 / Cost: 0.023052549 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 51712/60000 / Cost: 0.025731713 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 52224/60000 / Cost: 0.017321233 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.967\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 / Batch: 52736/60000 / Cost: 0.017014498 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 53248/60000 / Cost: 0.018717634 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 53760/60000 / Cost: 0.02434526 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 54272/60000 / Cost: 0.015754938 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 54784/60000 / Cost: 0.026408672 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 55296/60000 / Cost: 0.021847084 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 55808/60000 / Cost: 0.02095272 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 56320/60000 / Cost: 0.020858172 / Training Accuracy: 0.96875 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 56832/60000 / Cost: 0.013921306 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 57344/60000 / Cost: 0.017695453 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 57856/60000 / Cost: 0.023323081 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 58368/60000 / Cost: 0.02763177 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 58880/60000 / Cost: 0.01737368 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 59392/60000 / Cost: 0.01360484 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 59904/60000 / Cost: 0.026090885 / Training Accuracy: 0.9583333 / Validation Accuracy: 0.955\n",
      "Epoch: 7 / Batch: 0/60000 / Cost: 0.017954858 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.957\n",
      "Epoch: 7 / Batch: 512/60000 / Cost: 0.021877494 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.955\n",
      "Epoch: 7 / Batch: 1024/60000 / Cost: 0.019199647 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.956\n",
      "Epoch: 7 / Batch: 1536/60000 / Cost: 0.017702784 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.96\n",
      "Epoch: 7 / Batch: 2048/60000 / Cost: 0.026411697 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.957\n",
      "Epoch: 7 / Batch: 2560/60000 / Cost: 0.014190167 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.956\n",
      "Epoch: 7 / Batch: 3072/60000 / Cost: 0.020320915 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.953\n",
      "Epoch: 7 / Batch: 3584/60000 / Cost: 0.013749519 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.955\n",
      "Epoch: 7 / Batch: 4096/60000 / Cost: 0.019702058 / Training Accuracy: 0.96875 / Validation Accuracy: 0.958\n",
      "Epoch: 7 / Batch: 4608/60000 / Cost: 0.013757992 / Training Accuracy: 0.984375 / Validation Accuracy: 0.96\n",
      "Epoch: 7 / Batch: 5120/60000 / Cost: 0.017856717 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 5632/60000 / Cost: 0.017787268 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 6144/60000 / Cost: 0.033530913 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 6656/60000 / Cost: 0.017417973 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 7168/60000 / Cost: 0.015983537 / Training Accuracy: 0.96875 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 7680/60000 / Cost: 0.023721898 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.963\n",
      "Epoch: 7 / Batch: 8192/60000 / Cost: 0.02328887 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.959\n",
      "Epoch: 7 / Batch: 8704/60000 / Cost: 0.022323985 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.961\n",
      "Epoch: 7 / Batch: 9216/60000 / Cost: 0.01852397 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 9728/60000 / Cost: 0.017543916 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.964\n",
      "Epoch: 7 / Batch: 10240/60000 / Cost: 0.018743172 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 10752/60000 / Cost: 0.016375015 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.958\n",
      "Epoch: 7 / Batch: 11264/60000 / Cost: 0.019241005 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.956\n",
      "Epoch: 7 / Batch: 11776/60000 / Cost: 0.012493985 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.96\n",
      "Epoch: 7 / Batch: 12288/60000 / Cost: 0.0130999265 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 12800/60000 / Cost: 0.013150561 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.964\n",
      "Epoch: 7 / Batch: 13312/60000 / Cost: 0.021381672 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 13824/60000 / Cost: 0.02135444 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 14336/60000 / Cost: 0.019873459 / Training Accuracy: 0.96875 / Validation Accuracy: 0.96\n",
      "Epoch: 7 / Batch: 14848/60000 / Cost: 0.011704628 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.96\n",
      "Epoch: 7 / Batch: 15360/60000 / Cost: 0.016970988 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.958\n",
      "Epoch: 7 / Batch: 15872/60000 / Cost: 0.020085339 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.956\n",
      "Epoch: 7 / Batch: 16384/60000 / Cost: 0.015108499 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.959\n",
      "Epoch: 7 / Batch: 16896/60000 / Cost: 0.014322323 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 17408/60000 / Cost: 0.013214691 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 17920/60000 / Cost: 0.02032592 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 18432/60000 / Cost: 0.019361962 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 18944/60000 / Cost: 0.013647121 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 19456/60000 / Cost: 0.012446586 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.964\n",
      "Epoch: 7 / Batch: 19968/60000 / Cost: 0.013234654 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 7 / Batch: 20480/60000 / Cost: 0.017415162 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.964\n",
      "Epoch: 7 / Batch: 20992/60000 / Cost: 0.019987123 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 21504/60000 / Cost: 0.020885035 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.96\n",
      "Epoch: 7 / Batch: 22016/60000 / Cost: 0.017367346 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.961\n",
      "Epoch: 7 / Batch: 22528/60000 / Cost: 0.019147173 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.96\n",
      "Epoch: 7 / Batch: 23040/60000 / Cost: 0.016365875 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 23552/60000 / Cost: 0.020604335 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.963\n",
      "Epoch: 7 / Batch: 24064/60000 / Cost: 0.010388956 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 7 / Batch: 24576/60000 / Cost: 0.012649226 / Training Accuracy: 0.984375 / Validation Accuracy: 0.964\n",
      "Epoch: 7 / Batch: 25088/60000 / Cost: 0.016165081 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 25600/60000 / Cost: 0.019346828 / Training Accuracy: 0.96875 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 26112/60000 / Cost: 0.022185339 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 26624/60000 / Cost: 0.017593516 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 27136/60000 / Cost: 0.02297249 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 27648/60000 / Cost: 0.013341928 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 28160/60000 / Cost: 0.024663009 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 28672/60000 / Cost: 0.016756196 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.944\n",
      "Epoch: 7 / Batch: 29184/60000 / Cost: 0.023051335 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.957\n",
      "Epoch: 7 / Batch: 29696/60000 / Cost: 0.01940755 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 30208/60000 / Cost: 0.020036329 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.964\n",
      "Epoch: 7 / Batch: 30720/60000 / Cost: 0.019113934 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 7 / Batch: 31232/60000 / Cost: 0.02326541 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.959\n",
      "Epoch: 7 / Batch: 31744/60000 / Cost: 0.016228702 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 32256/60000 / Cost: 0.01681747 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.964\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 / Batch: 32768/60000 / Cost: 0.021656225 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 33280/60000 / Cost: 0.014538787 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.953\n",
      "Epoch: 7 / Batch: 33792/60000 / Cost: 0.020629812 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.954\n",
      "Epoch: 7 / Batch: 34304/60000 / Cost: 0.02145233 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 34816/60000 / Cost: 0.017084485 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.961\n",
      "Epoch: 7 / Batch: 35328/60000 / Cost: 0.018742625 / Training Accuracy: 0.96875 / Validation Accuracy: 0.958\n",
      "Epoch: 7 / Batch: 35840/60000 / Cost: 0.018570494 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.958\n",
      "Epoch: 7 / Batch: 36352/60000 / Cost: 0.018204382 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 36864/60000 / Cost: 0.012693512 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 37376/60000 / Cost: 0.019498592 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 37888/60000 / Cost: 0.01762015 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 38400/60000 / Cost: 0.019259218 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.963\n",
      "Epoch: 7 / Batch: 38912/60000 / Cost: 0.024409648 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.96\n",
      "Epoch: 7 / Batch: 39424/60000 / Cost: 0.020797702 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 39936/60000 / Cost: 0.019911254 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 40448/60000 / Cost: 0.011231452 / Training Accuracy: 0.984375 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 40960/60000 / Cost: 0.016950462 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 41472/60000 / Cost: 0.016329462 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 41984/60000 / Cost: 0.025270432 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 42496/60000 / Cost: 0.01908938 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 43008/60000 / Cost: 0.017335195 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 43520/60000 / Cost: 0.01857317 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 44032/60000 / Cost: 0.0137813855 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 44544/60000 / Cost: 0.013333214 / Training Accuracy: 0.984375 / Validation Accuracy: 0.961\n",
      "Epoch: 7 / Batch: 45056/60000 / Cost: 0.015739668 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 45568/60000 / Cost: 0.01728684 / Training Accuracy: 0.96875 / Validation Accuracy: 0.963\n",
      "Epoch: 7 / Batch: 46080/60000 / Cost: 0.020318035 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 46592/60000 / Cost: 0.014223183 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 47104/60000 / Cost: 0.015735622 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 47616/60000 / Cost: 0.015944278 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 48128/60000 / Cost: 0.01775944 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 48640/60000 / Cost: 0.01676496 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 49152/60000 / Cost: 0.022331309 / Training Accuracy: 0.96875 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 49664/60000 / Cost: 0.016414445 / Training Accuracy: 0.96875 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 50176/60000 / Cost: 0.017197615 / Training Accuracy: 0.96875 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 50688/60000 / Cost: 0.012823196 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 51200/60000 / Cost: 0.016974641 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 51712/60000 / Cost: 0.013460996 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 52224/60000 / Cost: 0.01142801 / Training Accuracy: 0.984375 / Validation Accuracy: 0.958\n",
      "Epoch: 7 / Batch: 52736/60000 / Cost: 0.020015413 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.956\n",
      "Epoch: 7 / Batch: 53248/60000 / Cost: 0.023991916 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.959\n",
      "Epoch: 7 / Batch: 53760/60000 / Cost: 0.025141459 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.961\n",
      "Epoch: 7 / Batch: 54272/60000 / Cost: 0.015697405 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.963\n",
      "Epoch: 7 / Batch: 54784/60000 / Cost: 0.018566718 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.963\n",
      "Epoch: 7 / Batch: 55296/60000 / Cost: 0.024077833 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 55808/60000 / Cost: 0.01650556 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 56320/60000 / Cost: 0.017765736 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 56832/60000 / Cost: 0.017528089 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 57344/60000 / Cost: 0.01503029 / Training Accuracy: 0.984375 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 57856/60000 / Cost: 0.014851657 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 58368/60000 / Cost: 0.015624707 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 58880/60000 / Cost: 0.016012913 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 59392/60000 / Cost: 0.015031802 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 59904/60000 / Cost: 0.008955559 / Training Accuracy: 0.9895833 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 0/60000 / Cost: 0.020501802 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 512/60000 / Cost: 0.020434901 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 1024/60000 / Cost: 0.018803427 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 1536/60000 / Cost: 0.0178367 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.961\n",
      "Epoch: 8 / Batch: 2048/60000 / Cost: 0.015669588 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 2560/60000 / Cost: 0.015615085 / Training Accuracy: 0.96875 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 3072/60000 / Cost: 0.006761943 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 3584/60000 / Cost: 0.019215304 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 4096/60000 / Cost: 0.00893362 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 4608/60000 / Cost: 0.015464383 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 5120/60000 / Cost: 0.018774522 / Training Accuracy: 0.96875 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 5632/60000 / Cost: 0.013971636 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 6144/60000 / Cost: 0.012560104 / Training Accuracy: 0.984375 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 6656/60000 / Cost: 0.014235839 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.96\n",
      "Epoch: 8 / Batch: 7168/60000 / Cost: 0.020318005 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 7680/60000 / Cost: 0.013419923 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 8192/60000 / Cost: 0.01574076 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 8 / Batch: 8704/60000 / Cost: 0.011122091 / Training Accuracy: 0.984375 / Validation Accuracy: 0.967\n",
      "Epoch: 8 / Batch: 9216/60000 / Cost: 0.010057864 / Training Accuracy: 0.984375 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 9728/60000 / Cost: 0.016408306 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 10240/60000 / Cost: 0.016393032 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 10752/60000 / Cost: 0.01642923 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.964\n",
      "Epoch: 8 / Batch: 11264/60000 / Cost: 0.023784252 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.968\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 / Batch: 11776/60000 / Cost: 0.016836634 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 12288/60000 / Cost: 0.019773256 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 12800/60000 / Cost: 0.01814114 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 13312/60000 / Cost: 0.013422972 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 13824/60000 / Cost: 0.012327274 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 14336/60000 / Cost: 0.014465275 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 14848/60000 / Cost: 0.014145638 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.96\n",
      "Epoch: 8 / Batch: 15360/60000 / Cost: 0.016467487 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.962\n",
      "Epoch: 8 / Batch: 15872/60000 / Cost: 0.019740282 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.967\n",
      "Epoch: 8 / Batch: 16384/60000 / Cost: 0.010845346 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 16896/60000 / Cost: 0.020679435 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 17408/60000 / Cost: 0.010712287 / Training Accuracy: 0.984375 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 17920/60000 / Cost: 0.013698315 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 18432/60000 / Cost: 0.010757188 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.96\n",
      "Epoch: 8 / Batch: 18944/60000 / Cost: 0.014988613 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 19456/60000 / Cost: 0.017323187 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 19968/60000 / Cost: 0.013215685 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 20480/60000 / Cost: 0.019466626 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 20992/60000 / Cost: 0.017100416 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 21504/60000 / Cost: 0.01285578 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 22016/60000 / Cost: 0.0143033 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 22528/60000 / Cost: 0.01209351 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 23040/60000 / Cost: 0.0151525345 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 23552/60000 / Cost: 0.014390146 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 24064/60000 / Cost: 0.013038188 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 24576/60000 / Cost: 0.012395576 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.962\n",
      "Epoch: 8 / Batch: 25088/60000 / Cost: 0.02007241 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.962\n",
      "Epoch: 8 / Batch: 25600/60000 / Cost: 0.013533814 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 26112/60000 / Cost: 0.01573235 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 26624/60000 / Cost: 0.012812214 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 27136/60000 / Cost: 0.017041262 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 27648/60000 / Cost: 0.013228518 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 28160/60000 / Cost: 0.020647764 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 28672/60000 / Cost: 0.021566296 / Training Accuracy: 0.96875 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 29184/60000 / Cost: 0.013192353 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 29696/60000 / Cost: 0.017159019 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 30208/60000 / Cost: 0.013591302 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 30720/60000 / Cost: 0.018810758 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 31232/60000 / Cost: 0.016633902 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 31744/60000 / Cost: 0.0119227 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 32256/60000 / Cost: 0.022037527 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 32768/60000 / Cost: 0.015590702 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 33280/60000 / Cost: 0.01592895 / Training Accuracy: 0.96875 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 33792/60000 / Cost: 0.015322244 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 34304/60000 / Cost: 0.016197767 / Training Accuracy: 0.96875 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 34816/60000 / Cost: 0.014960001 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 35328/60000 / Cost: 0.0126230465 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 35840/60000 / Cost: 0.015371097 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 36352/60000 / Cost: 0.014108783 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 36864/60000 / Cost: 0.011900891 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 37376/60000 / Cost: 0.025428409 / Training Accuracy: 0.953125 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 37888/60000 / Cost: 0.015576658 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 38400/60000 / Cost: 0.017833885 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.967\n",
      "Epoch: 8 / Batch: 38912/60000 / Cost: 0.016022474 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.959\n",
      "Epoch: 8 / Batch: 39424/60000 / Cost: 0.014455582 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 8 / Batch: 39936/60000 / Cost: 0.010039622 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.964\n",
      "Epoch: 8 / Batch: 40448/60000 / Cost: 0.010750046 / Training Accuracy: 0.984375 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 40960/60000 / Cost: 0.012480283 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 41472/60000 / Cost: 0.014518996 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 41984/60000 / Cost: 0.011921932 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 42496/60000 / Cost: 0.0153140975 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 43008/60000 / Cost: 0.018949447 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 8 / Batch: 43520/60000 / Cost: 0.01168002 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 44032/60000 / Cost: 0.015957242 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 44544/60000 / Cost: 0.013060875 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.967\n",
      "Epoch: 8 / Batch: 45056/60000 / Cost: 0.018851558 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 45568/60000 / Cost: 0.008829118 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 46080/60000 / Cost: 0.01566496 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 46592/60000 / Cost: 0.013514211 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 47104/60000 / Cost: 0.014041074 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 47616/60000 / Cost: 0.009453593 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 48128/60000 / Cost: 0.013105027 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 48640/60000 / Cost: 0.014734601 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 49152/60000 / Cost: 0.018220216 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 49664/60000 / Cost: 0.00996038 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 50176/60000 / Cost: 0.017405335 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.971\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 / Batch: 50688/60000 / Cost: 0.0124592595 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 51200/60000 / Cost: 0.013628012 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 51712/60000 / Cost: 0.011598485 / Training Accuracy: 0.984375 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 52224/60000 / Cost: 0.01002559 / Training Accuracy: 0.984375 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 52736/60000 / Cost: 0.013570619 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 53248/60000 / Cost: 0.011554889 / Training Accuracy: 0.984375 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 53760/60000 / Cost: 0.019104617 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 54272/60000 / Cost: 0.010892708 / Training Accuracy: 0.984375 / Validation Accuracy: 0.967\n",
      "Epoch: 8 / Batch: 54784/60000 / Cost: 0.010190029 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 55296/60000 / Cost: 0.017258028 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 55808/60000 / Cost: 0.015357295 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 56320/60000 / Cost: 0.0156186 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 56832/60000 / Cost: 0.015156249 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 57344/60000 / Cost: 0.017190987 / Training Accuracy: 0.96875 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 57856/60000 / Cost: 0.015509586 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 58368/60000 / Cost: 0.0091750305 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 58880/60000 / Cost: 0.013783345 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 8 / Batch: 59392/60000 / Cost: 0.009269092 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 59904/60000 / Cost: 0.0035389024 / Training Accuracy: 1.0 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 0/60000 / Cost: 0.015272138 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 512/60000 / Cost: 0.013552681 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 1024/60000 / Cost: 0.012894663 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 1536/60000 / Cost: 0.010761977 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 2048/60000 / Cost: 0.016091147 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 2560/60000 / Cost: 0.013447722 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 3072/60000 / Cost: 0.01564603 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 3584/60000 / Cost: 0.0111318845 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 4096/60000 / Cost: 0.013351356 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 4608/60000 / Cost: 0.013043717 / Training Accuracy: 0.984375 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 5120/60000 / Cost: 0.010033308 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 5632/60000 / Cost: 0.011173501 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.966\n",
      "Epoch: 9 / Batch: 6144/60000 / Cost: 0.0103102755 / Training Accuracy: 0.984375 / Validation Accuracy: 0.961\n",
      "Epoch: 9 / Batch: 6656/60000 / Cost: 0.011032951 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.967\n",
      "Epoch: 9 / Batch: 7168/60000 / Cost: 0.016753275 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 7680/60000 / Cost: 0.011301218 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 8192/60000 / Cost: 0.013357731 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 8704/60000 / Cost: 0.014136599 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 9216/60000 / Cost: 0.014583704 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 9 / Batch: 9728/60000 / Cost: 0.01913522 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 10240/60000 / Cost: 0.013147997 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 10752/60000 / Cost: 0.016545247 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 11264/60000 / Cost: 0.012453629 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 11776/60000 / Cost: 0.014199597 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 12288/60000 / Cost: 0.00871129 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 12800/60000 / Cost: 0.013454584 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.96\n",
      "Epoch: 9 / Batch: 13312/60000 / Cost: 0.013357602 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.961\n",
      "Epoch: 9 / Batch: 13824/60000 / Cost: 0.013158733 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 14336/60000 / Cost: 0.0084262695 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 14848/60000 / Cost: 0.012945135 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 15360/60000 / Cost: 0.011719666 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 15872/60000 / Cost: 0.0116119115 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 16384/60000 / Cost: 0.013527522 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 16896/60000 / Cost: 0.009369426 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 17408/60000 / Cost: 0.015492132 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 17920/60000 / Cost: 0.009258388 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.967\n",
      "Epoch: 9 / Batch: 18432/60000 / Cost: 0.008365571 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.966\n",
      "Epoch: 9 / Batch: 18944/60000 / Cost: 0.0117097525 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.965\n",
      "Epoch: 9 / Batch: 19456/60000 / Cost: 0.0122660175 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.965\n",
      "Epoch: 9 / Batch: 19968/60000 / Cost: 0.014130717 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 20480/60000 / Cost: 0.014609243 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 20992/60000 / Cost: 0.007838274 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 21504/60000 / Cost: 0.00972713 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 22016/60000 / Cost: 0.010848207 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 22528/60000 / Cost: 0.013441684 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 23040/60000 / Cost: 0.009584397 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 23552/60000 / Cost: 0.018617392 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.967\n",
      "Epoch: 9 / Batch: 24064/60000 / Cost: 0.011361559 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 24576/60000 / Cost: 0.009402657 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 25088/60000 / Cost: 0.016405668 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 25600/60000 / Cost: 0.0113721425 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 26112/60000 / Cost: 0.018300751 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.967\n",
      "Epoch: 9 / Batch: 26624/60000 / Cost: 0.011426933 / Training Accuracy: 0.984375 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 27136/60000 / Cost: 0.014251825 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 27648/60000 / Cost: 0.014239048 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.967\n",
      "Epoch: 9 / Batch: 28160/60000 / Cost: 0.006873938 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 28672/60000 / Cost: 0.013293207 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 29184/60000 / Cost: 0.011311746 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 29696/60000 / Cost: 0.012622769 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 / Batch: 30208/60000 / Cost: 0.01503537 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.965\n",
      "Epoch: 9 / Batch: 30720/60000 / Cost: 0.011117637 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 31232/60000 / Cost: 0.010948342 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 31744/60000 / Cost: 0.010097972 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 32256/60000 / Cost: 0.010525924 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 32768/60000 / Cost: 0.015182371 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 33280/60000 / Cost: 0.018504804 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 33792/60000 / Cost: 0.010956594 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 34304/60000 / Cost: 0.012411907 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 34816/60000 / Cost: 0.012851417 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 35328/60000 / Cost: 0.011210693 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 35840/60000 / Cost: 0.009925613 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 36352/60000 / Cost: 0.010196559 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.982\n",
      "Epoch: 9 / Batch: 36864/60000 / Cost: 0.014659531 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.981\n",
      "Epoch: 9 / Batch: 37376/60000 / Cost: 0.013560876 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 37888/60000 / Cost: 0.010153599 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 38400/60000 / Cost: 0.012297177 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 38912/60000 / Cost: 0.011460384 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 39424/60000 / Cost: 0.011829926 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.981\n",
      "Epoch: 9 / Batch: 39936/60000 / Cost: 0.009914232 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 40448/60000 / Cost: 0.0110801365 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 40960/60000 / Cost: 0.012451691 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 41472/60000 / Cost: 0.0082622655 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 41984/60000 / Cost: 0.009607365 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 42496/60000 / Cost: 0.0131112635 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 43008/60000 / Cost: 0.011240508 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 43520/60000 / Cost: 0.0069915922 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 44032/60000 / Cost: 0.015227655 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 44544/60000 / Cost: 0.014109661 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 45056/60000 / Cost: 0.021738648 / Training Accuracy: 0.96875 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 45568/60000 / Cost: 0.0124313785 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 46080/60000 / Cost: 0.011351709 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 46592/60000 / Cost: 0.012631431 / Training Accuracy: 0.984375 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 47104/60000 / Cost: 0.01318725 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.966\n",
      "Epoch: 9 / Batch: 47616/60000 / Cost: 0.010140997 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.959\n",
      "Epoch: 9 / Batch: 48128/60000 / Cost: 0.016276294 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 48640/60000 / Cost: 0.010867185 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 49152/60000 / Cost: 0.0097033195 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 49664/60000 / Cost: 0.013503963 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 50176/60000 / Cost: 0.015307685 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 50688/60000 / Cost: 0.019516975 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 51200/60000 / Cost: 0.019216636 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 51712/60000 / Cost: 0.01202435 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 52224/60000 / Cost: 0.015709974 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 52736/60000 / Cost: 0.01175191 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.981\n",
      "Epoch: 9 / Batch: 53248/60000 / Cost: 0.007746329 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 53760/60000 / Cost: 0.01577383 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 54272/60000 / Cost: 0.009950408 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 54784/60000 / Cost: 0.0094928695 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 55296/60000 / Cost: 0.018248942 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 55808/60000 / Cost: 0.012951066 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 56320/60000 / Cost: 0.00895972 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 56832/60000 / Cost: 0.010403827 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 57344/60000 / Cost: 0.011417988 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 57856/60000 / Cost: 0.012557307 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 58368/60000 / Cost: 0.009263688 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 58880/60000 / Cost: 0.010005409 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 59392/60000 / Cost: 0.012651277 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 59904/60000 / Cost: 0.0035400328 / Training Accuracy: 1.0 / Validation Accuracy: 0.97\n",
      "Epoch: 10 / Batch: 0/60000 / Cost: 0.016929913 / Training Accuracy: 0.96875 / Validation Accuracy: 0.964\n",
      "Epoch: 10 / Batch: 512/60000 / Cost: 0.012625212 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 10 / Batch: 1024/60000 / Cost: 0.0082952455 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 1536/60000 / Cost: 0.0099185025 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 10 / Batch: 2048/60000 / Cost: 0.016867932 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.967\n",
      "Epoch: 10 / Batch: 2560/60000 / Cost: 0.009641876 / Training Accuracy: 0.984375 / Validation Accuracy: 0.966\n",
      "Epoch: 10 / Batch: 3072/60000 / Cost: 0.010908363 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.967\n",
      "Epoch: 10 / Batch: 3584/60000 / Cost: 0.006576503 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 4096/60000 / Cost: 0.021264907 / Training Accuracy: 0.96875 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 4608/60000 / Cost: 0.020183396 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.97\n",
      "Epoch: 10 / Batch: 5120/60000 / Cost: 0.0130605055 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 5632/60000 / Cost: 0.01624855 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 6144/60000 / Cost: 0.012132378 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 6656/60000 / Cost: 0.009277387 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 7168/60000 / Cost: 0.00897962 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 7680/60000 / Cost: 0.008619852 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 8192/60000 / Cost: 0.011604032 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.969\n",
      "Epoch: 10 / Batch: 8704/60000 / Cost: 0.013752872 / Training Accuracy: 0.96875 / Validation Accuracy: 0.97\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 / Batch: 9216/60000 / Cost: 0.010291176 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 9728/60000 / Cost: 0.012819348 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 10240/60000 / Cost: 0.0100797955 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 10752/60000 / Cost: 0.008160415 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 11264/60000 / Cost: 0.008643527 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 11776/60000 / Cost: 0.01633487 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 12288/60000 / Cost: 0.015170371 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 12800/60000 / Cost: 0.0070499303 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 13312/60000 / Cost: 0.007791878 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 13824/60000 / Cost: 0.0091569405 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 14336/60000 / Cost: 0.009031882 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 14848/60000 / Cost: 0.00999818 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 15360/60000 / Cost: 0.010831112 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 15872/60000 / Cost: 0.012584962 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 16384/60000 / Cost: 0.009118189 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 16896/60000 / Cost: 0.015015239 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 17408/60000 / Cost: 0.006033479 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 17920/60000 / Cost: 0.012447817 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 18432/60000 / Cost: 0.011435027 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 18944/60000 / Cost: 0.012130486 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 19456/60000 / Cost: 0.010674724 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 19968/60000 / Cost: 0.010982005 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 20480/60000 / Cost: 0.00945086 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 20992/60000 / Cost: 0.011679196 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 21504/60000 / Cost: 0.011769795 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 22016/60000 / Cost: 0.010852032 / Training Accuracy: 0.984375 / Validation Accuracy: 0.97\n",
      "Epoch: 10 / Batch: 22528/60000 / Cost: 0.009797408 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 23040/60000 / Cost: 0.009180695 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 23552/60000 / Cost: 0.011461845 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 24064/60000 / Cost: 0.008477327 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 24576/60000 / Cost: 0.014614811 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 25088/60000 / Cost: 0.015503789 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 25600/60000 / Cost: 0.008625258 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 26112/60000 / Cost: 0.015664317 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 26624/60000 / Cost: 0.010097364 / Training Accuracy: 0.984375 / Validation Accuracy: 0.969\n",
      "Epoch: 10 / Batch: 27136/60000 / Cost: 0.012511319 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 27648/60000 / Cost: 0.012849094 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 28160/60000 / Cost: 0.012969638 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 28672/60000 / Cost: 0.009249312 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 29184/60000 / Cost: 0.006414061 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 29696/60000 / Cost: 0.009179751 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 30208/60000 / Cost: 0.0071090013 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 30720/60000 / Cost: 0.017652273 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 31232/60000 / Cost: 0.010123847 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 31744/60000 / Cost: 0.016027812 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 32256/60000 / Cost: 0.0077669187 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 32768/60000 / Cost: 0.008112343 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 33280/60000 / Cost: 0.008005815 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 33792/60000 / Cost: 0.009830387 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 34304/60000 / Cost: 0.01143237 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 34816/60000 / Cost: 0.0084869005 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 35328/60000 / Cost: 0.011509257 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 35840/60000 / Cost: 0.010017468 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 36352/60000 / Cost: 0.0057512694 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.969\n",
      "Epoch: 10 / Batch: 36864/60000 / Cost: 0.01447852 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 37376/60000 / Cost: 0.008407759 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 37888/60000 / Cost: 0.013973409 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 38400/60000 / Cost: 0.011404891 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 38912/60000 / Cost: 0.0069413306 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 39424/60000 / Cost: 0.00795048 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 39936/60000 / Cost: 0.007884893 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 40448/60000 / Cost: 0.011487764 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 40960/60000 / Cost: 0.008665366 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 41472/60000 / Cost: 0.0107090995 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 41984/60000 / Cost: 0.009708126 / Training Accuracy: 0.984375 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 42496/60000 / Cost: 0.011142036 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 43008/60000 / Cost: 0.008167586 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 43520/60000 / Cost: 0.007099077 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 44032/60000 / Cost: 0.012032695 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 44544/60000 / Cost: 0.013202215 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 45056/60000 / Cost: 0.011580912 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 45568/60000 / Cost: 0.009621164 / Training Accuracy: 0.984375 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 46080/60000 / Cost: 0.01185039 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 46592/60000 / Cost: 0.010584083 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 47104/60000 / Cost: 0.012182291 / Training Accuracy: 0.984375 / Validation Accuracy: 0.964\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 / Batch: 47616/60000 / Cost: 0.019396532 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.962\n",
      "Epoch: 10 / Batch: 48128/60000 / Cost: 0.016055847 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.966\n",
      "Epoch: 10 / Batch: 48640/60000 / Cost: 0.007951865 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 49152/60000 / Cost: 0.014207098 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 49664/60000 / Cost: 0.013327899 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 50176/60000 / Cost: 0.012478639 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 50688/60000 / Cost: 0.015116775 / Training Accuracy: 0.96875 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 51200/60000 / Cost: 0.0108445315 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 51712/60000 / Cost: 0.00788709 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 52224/60000 / Cost: 0.015066549 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 10 / Batch: 52736/60000 / Cost: 0.0177331 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.966\n",
      "Epoch: 10 / Batch: 53248/60000 / Cost: 0.012805986 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 53760/60000 / Cost: 0.012563671 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 54272/60000 / Cost: 0.011188364 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 54784/60000 / Cost: 0.01293357 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 55296/60000 / Cost: 0.013256502 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 55808/60000 / Cost: 0.00612748 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.97\n",
      "Epoch: 10 / Batch: 56320/60000 / Cost: 0.0067997775 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.97\n",
      "Epoch: 10 / Batch: 56832/60000 / Cost: 0.010116093 / Training Accuracy: 0.984375 / Validation Accuracy: 0.97\n",
      "Epoch: 10 / Batch: 57344/60000 / Cost: 0.009273466 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.969\n",
      "Epoch: 10 / Batch: 57856/60000 / Cost: 0.014273889 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 58368/60000 / Cost: 0.0077723926 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 58880/60000 / Cost: 0.007851108 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 59392/60000 / Cost: 0.011580262 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 59904/60000 / Cost: 0.008166422 / Training Accuracy: 0.9895833 / Validation Accuracy: 0.973\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "for i in range(num_epoch):\n",
    "    np.random.shuffle(data)\n",
    "\n",
    "    data_x = data[:, :img_flat_size]\n",
    "    data_y = data[:, img_flat_size]\n",
    "\n",
    "    data_y_onehot = np.zeros([data_y.shape[0], num_label])\n",
    "    for j in range(data_y.shape[0]):\n",
    "        data_y_onehot[j, int(data_y[j])] = 1\n",
    "    \n",
    "    data_y_onehot_val = np.zeros([validation_y.shape[0], num_label])\n",
    "    for j in range(validation_y.shape[0]):\n",
    "        data_y_onehot_val[j, int(validation_y[j])] = 1\n",
    "        \n",
    "    batch_count = 1\n",
    "    for j in range(0, len_data, batch_size):\n",
    "        if j + batch_size < len_data:\n",
    "            data_x_in = data_x[j : j + batch_size, :]\n",
    "            data_y_in = data_y_onehot[j : j + batch_size, :]\n",
    "        else:\n",
    "            data_x_in = data_x[j : len_data, :]\n",
    "            data_y_in = data_y_onehot[j : len_data, :]\n",
    "\n",
    "        optimizer.run(feed_dict = {x_image: data_x_in, y_target: data_y_in})\n",
    "        cost = sess.run(Cost, feed_dict = {x_image: data_x_in, y_target: data_y_in})\n",
    "        acc = sess.run(accuracy, feed_dict = {x_image: data_x_in, y_target: data_y_in})\n",
    "        val_acc = sess.run(accuracy, feed_dict = {x_image: validation_x, y_target: data_y_onehot_val})\n",
    "        \n",
    "        print(\"Epoch: \" + str(i+1) + ' / ' + \"Batch: \" + str(j) + '/' + str(len_data) + ' / ' + \"Cost: \" + str(cost) + ' / ' + \\\n",
    "              \"Training Accuracy: \" + str(acc) + ' / ' + \"Validation Accuracy: \" + str(val_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.9757777777777777\n"
     ]
    }
   ],
   "source": [
    "# Testing\n",
    "\n",
    "test_y_onehot = np.zeros([test_y.shape[0], num_label])\n",
    "for i in range(test_y.shape[0]):\n",
    "    test_y_onehot[i, int(test_y[i])] = 1\n",
    "\n",
    "test_result = sess.run(output, feed_dict = {x_image: test_x, rnn_batch_size: test_x.shape[0], rnn_step_size: step_size})\n",
    "\n",
    "count_correct = 0\n",
    "for i in range(test_result.shape[0]):\n",
    "    prediction_y = np.argmax(test_result[i,:])\n",
    "    \n",
    "    if prediction_y == test_y[i]:\n",
    "        count_correct += 1\n",
    "\n",
    "test_acc = count_correct / test_result.shape[0]\n",
    "\n",
    "print(\"Test Accuracy: \" + str(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}