1_CNN_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Convolutional Neural Network Code with MNIST\n",
    "\n",
    "This is jupyter notebook for basic CNN and dense layer using `Tensorflow`. \n",
    "<br> Dataset is MNIST dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# Import libraries\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-4d0e456e9717>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import MNIST data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', validation_size=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters \n",
    "img_size = 28\n",
    "img_flat_size = img_size * img_size\n",
    "\n",
    "# labels: 0 - 9\n",
    "num_label = 10\n",
    "\n",
    "# Parameters for optimizer\n",
    "num_epoch = 10\n",
    "\n",
    "learning_rate = 5e-4\n",
    "epsilon = 1e-8\n",
    "\n",
    "batch_size = 512\n",
    "\n",
    "validation_ratio = 0.1\n",
    "gpu_fraction = 0.4\n",
    "\n",
    "# Convolution parameters [kernel size, kernel size, # of channels(input), # of channels (output)]\n",
    "first_conv  = [3,3, 1,32]\n",
    "second_conv = [3,3,32,64]\n",
    "third_conv  = [3,3,64,128]\n",
    "\n",
    "# Fully connected parameters [# of input data, # of output data]\n",
    "first_fc = [4 * 4 * 128, 512]\n",
    "second_fc = [512, num_label]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADgpJREFUeJzt3X+MVfWZx/HPs1j+kKI4aQRCYSnEYJW4082IjSWrxkzVDQZHrekkJjQapn8wiU02ZA3/VNNgyCrslmiamaZYSFpKE3VB0iw0otLGZuKIWC0srTFsO3IDNTjywx9kmGf/mEMzxbnfe+fec++5zPN+JeT+eM6558kNnznn3O+592vuLgDx/EPRDQAoBuEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxDUZc3cmJlxOSHQYO5u1SxX157fzO40syNm9q6ZPVrPawFoLqv12n4zmybpj5I6JQ1Jel1St7sfSqzDnh9osGbs+ZdJetfd33P3c5J+IWllHa8HoInqCf88SX8Z93goe+7vmFmPmQ2a2WAd2wKQs3o+8Jvo0OJzh/Xu3i+pX+KwH2gl9ez5hyTNH/f4y5KO1dcOgGapJ/yvS7rGzL5iZtMlfVvSrnzaAtBoNR/2u/uImfVK2iNpmqQt7v6H3DoD0FA1D/XVtDHO+YGGa8pFPgAuXYQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EVfMU3ZJkZkclnZZ0XtKIu3fk0RTyM23atGT9yiuvbOj2e3t7y9Yuv/zy5LpLlixJ1tesWZOsP/XUU2Vr3d3dyXU//fTTZH3Dhg3J+uOPP56st4K6wp+5zd0/yOF1ADQRh/1AUPWG3yXtNbM3zKwnj4YANEe9h/3fcPdjZna1pF+b2f+6+/7xC2R/FPjDALSYuvb87n4suz0h6QVJyyZYpt/dO/gwEGgtNYffzGaY2cwL9yV9U9I7eTUGoLHqOeyfLekFM7vwOj939//JpSsADVdz+N39PUn/lGMvU9aCBQuS9enTpyfrN998c7K+fPnysrVZs2Yl173vvvuS9SINDQ0l65s3b07Wu7q6ytZOnz6dXPett95K1l999dVk/VLAUB8QFOEHgiL8QFCEHwiK8ANBEX4gKHP35m3MrHkba6L29vZkfd++fcl6o79W26pGR0eT9YceeihZP3PmTM3bLpVKyfqHH36YrB85cqTmbTeau1s1y7HnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgGOfPQVtbW7I+MDCQrC9atCjPdnJVqffh4eFk/bbbbitbO3fuXHLdqNc/1ItxfgBJhB8IivADQRF+ICjCDwRF+IGgCD8QVB6z9IZ38uTJZH3t2rXJ+ooVK5L1N998M1mv9BPWKQcPHkzWOzs7k/WzZ88m69dff33Z2iOPPJJcF43Fnh8IivADQRF+ICjCDwRF+IGgCD8QFOEHgqr4fX4z2yJphaQT7r40e65N0g5JCyUdlfSAu6d/6FxT9/v89briiiuS9UrTSff19ZWtPfzww8l1H3zwwWR9+/btyTpaT57f5/+ppDsveu5RSS+5+zWSXsoeA7iEVAy/u++XdPElbCslbc3ub5V0T859AWiwWs/5Z7t7SZKy26vzawlAMzT82n4z65HU0+jtAJicWvf8x81sriRltyfKLeju/e7e4e4dNW4LQAPUGv5dklZl91dJ2plPOwCapWL4zWy7pN9JWmJmQ2b2sKQNkjrN7E+SOrPHAC4hFc/53b27TOn2nHsJ69SpU3Wt/9FHH9W87urVq5P1HTt2JOujo6M1bxvF4go/ICjCDwRF+IGgCD8QFOEHgiL8QFBM0T0FzJgxo2ztxRdfTK57yy23JOt33XVXsr53795kHc3HFN0Akgg/EBThB4Ii/EBQhB8IivADQRF+ICjG+ae4xYsXJ+sHDhxI1oeHh5P1l19+OVkfHBwsW3vmmWeS6zbz/+ZUwjg/gCTCDwRF+IGgCD8QFOEHgiL8QFCEHwiKcf7gurq6kvVnn302WZ85c2bN2163bl2yvm3btmS9VCrVvO2pjHF+AEmEHwiK8ANBEX4gKMIPBEX4gaAIPxBUxXF+M9siaYWkE+6+NHvuMUmrJf01W2ydu/+q4sYY57/kLF26NFnftGlTsn777bXP5N7X15esr1+/Pll///33a972pSzPcf6fSrpzguf/093bs38Vgw+gtVQMv7vvl3SyCb0AaKJ6zvl7zez3ZrbFzK7KrSMATVFr+H8kabGkdkklSRvLLWhmPWY2aGblf8wNQNPVFH53P+7u5919VNKPJS1LLNvv7h3u3lFrkwDyV1P4zWzuuIddkt7Jpx0AzXJZpQXMbLukWyV9ycyGJH1f0q1m1i7JJR2V9N0G9gigAfg+P+oya9asZP3uu+8uW6v0WwFm6eHqffv2JeudnZ3J+lTF9/kBJBF+ICjCDwRF+IGgCD8QFOEHgmKoD4X57LPPkvXLLktfhjIyMpKs33HHHWVrr7zySnLdSxlDfQCSCD8QFOEHgiL8QFCEHwiK8ANBEX4gqIrf50dsN9xwQ7J+//33J+s33nhj2VqlcfxKDh06lKzv37+/rtef6tjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQjPNPcUuWLEnWe3t7k/V77703WZ8zZ86ke6rW+fPnk/VSqZSsj46O5tnOlMOeHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCqjjOb2bzJW2TNEfSqKR+d/+hmbVJ2iFpoaSjkh5w9w8b12pclcbSu7u7y9YqjeMvXLiwlpZyMTg4mKyvX78+Wd+1a1ee7YRTzZ5/RNK/uftXJX1d0hozu07So5JecvdrJL2UPQZwiagYfncvufuB7P5pSYclzZO0UtLWbLGtku5pVJMA8jepc34zWyjpa5IGJM1295I09gdC0tV5Nwegcaq+tt/MvijpOUnfc/dTZlVNByYz65HUU1t7ABqlqj2/mX1BY8H/mbs/nz193MzmZvW5kk5MtK6797t7h7t35NEwgHxUDL+N7eJ/Iumwu28aV9olaVV2f5Wknfm3B6BRKk7RbWbLJf1G0tsaG+qTpHUaO+//paQFkv4s6VvufrLCa4Wconv27NnJ+nXXXZesP/3008n6tddeO+me8jIwMJCsP/nkk2VrO3em9xd8Jbc21U7RXfGc391/K6nci90+maYAtA6u8AOCIvxAUIQfCIrwA0ERfiAowg8ExU93V6mtra1sra+vL7lue3t7sr5o0aKaesrDa6+9lqxv3LgxWd+zZ0+y/sknn0y6JzQHe34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCCrMOP9NN92UrK9duzZZX7ZsWdnavHnzauopLx9//HHZ2ubNm5PrPvHEE8n62bNna+oJrY89PxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EFWacv6urq656PQ4dOpSs7969O1kfGRlJ1lPfuR8eHk6ui7jY8wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUObu6QXM5kvaJmmOpFFJ/e7+QzN7TNJqSX/NFl3n7r+q8FrpjQGom7tbNctVE/65kua6+wEzmynpDUn3SHpA0hl3f6rapgg/0HjVhr/iFX7uXpJUyu6fNrPDkor96RoAdZvUOb+ZLZT0NUkD2VO9ZvZ7M9tiZleVWafHzAbNbLCuTgHkquJh/98WNPuipFclrXf3581stqQPJLmkH2js1OChCq/BYT/QYLmd80uSmX1B0m5Je9x90wT1hZJ2u/vSCq9D+IEGqzb8FQ/7zcwk/UTS4fHBzz4IvKBL0juTbRJAcar5tH+5pN9IeltjQ32StE5St6R2jR32H5X03ezDwdRrsecHGizXw/68EH6g8XI77AcwNRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCavYU3R9I+r9xj7+UPdeKWrW3Vu1Lorda5dnbP1a7YFO/z/+5jZsNuntHYQ0ktGpvrdqXRG+1Kqo3DvuBoAg/EFTR4e8vePsprdpbq/Yl0VutCumt0HN+AMUpes8PoCCFhN/M7jSzI2b2rpk9WkQP5ZjZUTN728wOFj3FWDYN2gkze2fcc21m9msz+1N2O+E0aQX19piZvZ+9dwfN7F8L6m2+mb1sZofN7A9m9kj2fKHvXaKvQt63ph/2m9k0SX+U1ClpSNLrkrrd/VBTGynDzI5K6nD3wseEzexfJJ2RtO3CbEhm9h+STrr7huwP51Xu/u8t0ttjmuTMzQ3qrdzM0t9Rge9dnjNe56GIPf8ySe+6+3vufk7SLyStLKCPlufu+yWdvOjplZK2Zve3auw/T9OV6a0luHvJ3Q9k909LujCzdKHvXaKvQhQR/nmS/jLu8ZBaa8pvl7TXzN4ws56im5nA7AszI2W3Vxfcz8UqztzcTBfNLN0y710tM17nrYjwTzSbSCsNOXzD3f9Z0l2S1mSHt6jOjyQt1tg0biVJG4tsJptZ+jlJ33P3U0X2Mt4EfRXyvhUR/iFJ88c9/rKkYwX0MSF3P5bdnpD0gsZOU1rJ8QuTpGa3Jwru52/c/bi7n3f3UUk/VoHvXTaz9HOSfubuz2dPF/7eTdRXUe9bEeF/XdI1ZvYVM5su6duSdhXQx+eY2YzsgxiZ2QxJ31TrzT68S9Kq7P4qSTsL7OXvtMrMzeVmllbB712rzXhdyEU+2VDGf0maJmmLu69vehMTMLNFGtvbS2PfePx5kb2Z2XZJt2rsW1/HJX1f0n9L+qWkBZL+LOlb7t70D97K9HarJjlzc4N6Kzez9IAKfO/ynPE6l364wg+IiSv8gKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8E9f/Ex0YKZYOZcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting example image\n",
    "img = mnist.train.images[0]\n",
    "img_resize = img.reshape((img_size, img_size))\n",
    "plt.imshow(img_resize, cmap = 'gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize weights and bias \n",
    "def conv_weight_variable(name, shape):\n",
    "    return tf.get_variable(name, shape = shape, initializer = tf.contrib.layers.xavier_initializer_conv2d())\n",
    "\n",
    "def weight_variable(name, shape):\n",
    "    return tf.get_variable(name, shape=shape, initializer=tf.contrib.layers.xavier_initializer())\n",
    "\n",
    "def bias_variable(name, shape):\n",
    "    return tf.get_variable(name, shape=shape, initializer=tf.contrib.layers.xavier_initializer())\n",
    "\n",
    "# Convolution and pooling\n",
    "def conv2d(x,w, stride):\n",
    "    return tf.nn.conv2d(x,w,strides=[1, stride, stride, 1], padding='SAME')\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "    return tf.nn.max_pool(x, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Network\n",
    "\n",
    "# Reset graph\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# Input \n",
    "x_image  = tf.placeholder(tf.float32, shape = [None, img_size, img_size, 1])\n",
    "y_target = tf.placeholder(tf.float32, shape=[None, num_label])\n",
    "\n",
    "# Keep Prob\n",
    "keep_prob = tf.placeholder(\"float\")\n",
    "\n",
    "# Convolution variables\n",
    "w_conv1 = conv_weight_variable('w_conv1', first_conv)\n",
    "b_conv1 = bias_variable('b_conv1', [first_conv[3]])\n",
    "\n",
    "w_conv2 = conv_weight_variable('w_conv2', second_conv)\n",
    "b_conv2 = bias_variable('b_conv2', [second_conv[3]])\n",
    "\n",
    "w_conv3 = conv_weight_variable('w_conv3', third_conv)\n",
    "b_conv3 = bias_variable('b_conv3', [third_conv[3]])\n",
    "    \n",
    "# Densely connect layer variables \n",
    "w_fc1 = weight_variable('w_fc1', first_fc)\n",
    "b_fc1 = bias_variable('b_fc1', first_fc[1])\n",
    "\n",
    "w_fc2 = weight_variable('w_fc2', second_fc)\n",
    "b_fc2 = bias_variable('b_fc2', second_fc[1])\n",
    "\n",
    "# Conv layers\n",
    "h_conv1 = tf.nn.relu(conv2d(x_image, w_conv1, 1) + b_conv1)\n",
    "h_pool1 = max_pool_2x2(h_conv1)\n",
    "\n",
    "h_conv2 = tf.nn.relu(conv2d(h_pool1, w_conv2, 1) + b_conv2)\n",
    "h_pool2 = max_pool_2x2(h_conv2)\n",
    "\n",
    "h_conv3 = tf.nn.relu(conv2d(h_pool2, w_conv3, 1) + b_conv3)\n",
    "h_pool3 = max_pool_2x2(h_conv3)\n",
    "\n",
    "# Flatten\n",
    "h_flatten = tf.reshape(h_pool3, [-1, 4 * 4 * 128])\n",
    "\n",
    "# Fully connected Layer\n",
    "h_fc1 = tf.nn.relu(tf.matmul(h_flatten, w_fc1)+b_fc1)\n",
    "\n",
    "# Dropout\n",
    "keep_prob = tf.placeholder(\"float\")\n",
    "h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "output = tf.nn.softmax(tf.matmul(h_fc1_drop, w_fc2)+b_fc2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Loss and Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training \n",
    "Loss = tf.reduce_mean(tf.square(output - y_target))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate, epsilon = epsilon).minimize(Loss)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y_target,1), tf.argmax(output,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set: (60000, 784)\n",
      "Testing set: (9000, 784)\n",
      "Validation set: (1000, 784)\n"
     ]
    }
   ],
   "source": [
    "# Dataset for train, test, validation\n",
    "\n",
    "test_len = mnist.test.images.shape[0]\n",
    "validation_len = int(test_len * validation_ratio)\n",
    "\n",
    "train_x = mnist.train.images\n",
    "test_x = mnist.test.images[validation_len : test_len, :]\n",
    "validation_x = mnist.test.images[ : validation_len, :]\n",
    "\n",
    "# Reshape\n",
    "validation_x_resize = np.reshape(validation_x, (validation_x.shape[0], img_size, img_size, 1))\n",
    "\n",
    "train_y = mnist.train.labels\n",
    "test_y = mnist.test.labels[validation_len : test_len]\n",
    "validation_y = mnist.test.labels[ : validation_len]\n",
    "\n",
    "print(\"Training set: \" + str(train_x.shape))\n",
    "print(\"Testing set: \" + str(test_x.shape))\n",
    "print(\"Validation set: \" + str(validation_x.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = gpu_fraction\n",
    "\n",
    "sess = tf.InteractiveSession(config=config)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 / Batch: 0/60000 / Loss: 0.089460276 / Training Accuracy: 0.13671875 / Validation Accuracy: 0.126\n",
      "Epoch: 1 / Batch: 512/60000 / Loss: 0.089491 / Training Accuracy: 0.18359375 / Validation Accuracy: 0.177\n",
      "Epoch: 1 / Batch: 1024/60000 / Loss: 0.089114584 / Training Accuracy: 0.25390625 / Validation Accuracy: 0.235\n",
      "Epoch: 1 / Batch: 1536/60000 / Loss: 0.08880108 / Training Accuracy: 0.3359375 / Validation Accuracy: 0.33\n",
      "Epoch: 1 / Batch: 2048/60000 / Loss: 0.08836613 / Training Accuracy: 0.49023438 / Validation Accuracy: 0.447\n",
      "Epoch: 1 / Batch: 2560/60000 / Loss: 0.088054135 / Training Accuracy: 0.46484375 / Validation Accuracy: 0.448\n",
      "Epoch: 1 / Batch: 3072/60000 / Loss: 0.087538496 / Training Accuracy: 0.5253906 / Validation Accuracy: 0.519\n",
      "Epoch: 1 / Batch: 3584/60000 / Loss: 0.086510405 / Training Accuracy: 0.47070312 / Validation Accuracy: 0.423\n",
      "Epoch: 1 / Batch: 4096/60000 / Loss: 0.08562286 / Training Accuracy: 0.36914062 / Validation Accuracy: 0.328\n",
      "Epoch: 1 / Batch: 4608/60000 / Loss: 0.08485019 / Training Accuracy: 0.24804688 / Validation Accuracy: 0.244\n",
      "Epoch: 1 / Batch: 5120/60000 / Loss: 0.08323009 / Training Accuracy: 0.25976562 / Validation Accuracy: 0.25\n",
      "Epoch: 1 / Batch: 5632/60000 / Loss: 0.082756065 / Training Accuracy: 0.28710938 / Validation Accuracy: 0.287\n",
      "Epoch: 1 / Batch: 6144/60000 / Loss: 0.08103781 / Training Accuracy: 0.33789062 / Validation Accuracy: 0.321\n",
      "Epoch: 1 / Batch: 6656/60000 / Loss: 0.0809629 / Training Accuracy: 0.38671875 / Validation Accuracy: 0.409\n",
      "Epoch: 1 / Batch: 7168/60000 / Loss: 0.0783303 / Training Accuracy: 0.4765625 / Validation Accuracy: 0.483\n",
      "Epoch: 1 / Batch: 7680/60000 / Loss: 0.075905 / Training Accuracy: 0.5292969 / Validation Accuracy: 0.504\n",
      "Epoch: 1 / Batch: 8192/60000 / Loss: 0.07209122 / Training Accuracy: 0.5214844 / Validation Accuracy: 0.491\n",
      "Epoch: 1 / Batch: 8704/60000 / Loss: 0.07118952 / Training Accuracy: 0.47851562 / Validation Accuracy: 0.484\n",
      "Epoch: 1 / Batch: 9216/60000 / Loss: 0.06984359 / Training Accuracy: 0.5449219 / Validation Accuracy: 0.539\n",
      "Epoch: 1 / Batch: 9728/60000 / Loss: 0.067012966 / Training Accuracy: 0.5917969 / Validation Accuracy: 0.619\n",
      "Epoch: 1 / Batch: 10240/60000 / Loss: 0.0629119 / Training Accuracy: 0.6171875 / Validation Accuracy: 0.603\n",
      "Epoch: 1 / Batch: 10752/60000 / Loss: 0.060890354 / Training Accuracy: 0.60546875 / Validation Accuracy: 0.61\n",
      "Epoch: 1 / Batch: 11264/60000 / Loss: 0.056629576 / Training Accuracy: 0.67578125 / Validation Accuracy: 0.634\n",
      "Epoch: 1 / Batch: 11776/60000 / Loss: 0.053899806 / Training Accuracy: 0.7050781 / Validation Accuracy: 0.676\n",
      "Epoch: 1 / Batch: 12288/60000 / Loss: 0.049781807 / Training Accuracy: 0.7441406 / Validation Accuracy: 0.695\n",
      "Epoch: 1 / Batch: 12800/60000 / Loss: 0.048995413 / Training Accuracy: 0.72265625 / Validation Accuracy: 0.721\n",
      "Epoch: 1 / Batch: 13312/60000 / Loss: 0.043899518 / Training Accuracy: 0.7597656 / Validation Accuracy: 0.737\n",
      "Epoch: 1 / Batch: 13824/60000 / Loss: 0.04182223 / Training Accuracy: 0.7734375 / Validation Accuracy: 0.73\n",
      "Epoch: 1 / Batch: 14336/60000 / Loss: 0.041944522 / Training Accuracy: 0.7246094 / Validation Accuracy: 0.753\n",
      "Epoch: 1 / Batch: 14848/60000 / Loss: 0.03726421 / Training Accuracy: 0.75 / Validation Accuracy: 0.757\n",
      "Epoch: 1 / Batch: 15360/60000 / Loss: 0.036880113 / Training Accuracy: 0.7636719 / Validation Accuracy: 0.754\n",
      "Epoch: 1 / Batch: 15872/60000 / Loss: 0.03484555 / Training Accuracy: 0.7636719 / Validation Accuracy: 0.771\n",
      "Epoch: 1 / Batch: 16384/60000 / Loss: 0.031335965 / Training Accuracy: 0.7890625 / Validation Accuracy: 0.755\n",
      "Epoch: 1 / Batch: 16896/60000 / Loss: 0.031283654 / Training Accuracy: 0.7988281 / Validation Accuracy: 0.749\n",
      "Epoch: 1 / Batch: 17408/60000 / Loss: 0.031116435 / Training Accuracy: 0.80859375 / Validation Accuracy: 0.784\n",
      "Epoch: 1 / Batch: 17920/60000 / Loss: 0.029285211 / Training Accuracy: 0.7988281 / Validation Accuracy: 0.81\n",
      "Epoch: 1 / Batch: 18432/60000 / Loss: 0.030669829 / Training Accuracy: 0.7832031 / Validation Accuracy: 0.807\n",
      "Epoch: 1 / Batch: 18944/60000 / Loss: 0.02843808 / Training Accuracy: 0.8144531 / Validation Accuracy: 0.81\n",
      "Epoch: 1 / Batch: 19456/60000 / Loss: 0.029107193 / Training Accuracy: 0.8222656 / Validation Accuracy: 0.82\n",
      "Epoch: 1 / Batch: 19968/60000 / Loss: 0.027447138 / Training Accuracy: 0.8222656 / Validation Accuracy: 0.824\n",
      "Epoch: 1 / Batch: 20480/60000 / Loss: 0.020550448 / Training Accuracy: 0.8730469 / Validation Accuracy: 0.828\n",
      "Epoch: 1 / Batch: 20992/60000 / Loss: 0.02042683 / Training Accuracy: 0.87109375 / Validation Accuracy: 0.845\n",
      "Epoch: 1 / Batch: 21504/60000 / Loss: 0.0208822 / Training Accuracy: 0.8691406 / Validation Accuracy: 0.857\n",
      "Epoch: 1 / Batch: 22016/60000 / Loss: 0.023312291 / Training Accuracy: 0.85546875 / Validation Accuracy: 0.861\n",
      "Epoch: 1 / Batch: 22528/60000 / Loss: 0.02059767 / Training Accuracy: 0.875 / Validation Accuracy: 0.857\n",
      "Epoch: 1 / Batch: 23040/60000 / Loss: 0.01969732 / Training Accuracy: 0.87890625 / Validation Accuracy: 0.862\n",
      "Epoch: 1 / Batch: 23552/60000 / Loss: 0.018473163 / Training Accuracy: 0.8847656 / Validation Accuracy: 0.866\n",
      "Epoch: 1 / Batch: 24064/60000 / Loss: 0.02048332 / Training Accuracy: 0.8691406 / Validation Accuracy: 0.865\n",
      "Epoch: 1 / Batch: 24576/60000 / Loss: 0.017119668 / Training Accuracy: 0.8925781 / Validation Accuracy: 0.864\n",
      "Epoch: 1 / Batch: 25088/60000 / Loss: 0.018036077 / Training Accuracy: 0.8847656 / Validation Accuracy: 0.873\n",
      "Epoch: 1 / Batch: 25600/60000 / Loss: 0.016260693 / Training Accuracy: 0.90625 / Validation Accuracy: 0.879\n",
      "Epoch: 1 / Batch: 26112/60000 / Loss: 0.016420739 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.884\n",
      "Epoch: 1 / Batch: 26624/60000 / Loss: 0.017854076 / Training Accuracy: 0.8847656 / Validation Accuracy: 0.885\n",
      "Epoch: 1 / Batch: 27136/60000 / Loss: 0.015385278 / Training Accuracy: 0.91015625 / Validation Accuracy: 0.892\n",
      "Epoch: 1 / Batch: 27648/60000 / Loss: 0.017127115 / Training Accuracy: 0.88671875 / Validation Accuracy: 0.884\n",
      "Epoch: 1 / Batch: 28160/60000 / Loss: 0.014856574 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.885\n",
      "Epoch: 1 / Batch: 28672/60000 / Loss: 0.014999849 / Training Accuracy: 0.90625 / Validation Accuracy: 0.896\n",
      "Epoch: 1 / Batch: 29184/60000 / Loss: 0.01395181 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.903\n",
      "Epoch: 1 / Batch: 29696/60000 / Loss: 0.016474152 / Training Accuracy: 0.88671875 / Validation Accuracy: 0.891\n",
      "Epoch: 1 / Batch: 30208/60000 / Loss: 0.012100659 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.896\n",
      "Epoch: 1 / Batch: 30720/60000 / Loss: 0.012602441 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.904\n",
      "Epoch: 1 / Batch: 31232/60000 / Loss: 0.015595096 / Training Accuracy: 0.9003906 / Validation Accuracy: 0.908\n",
      "Epoch: 1 / Batch: 31744/60000 / Loss: 0.011748565 / Training Accuracy: 0.921875 / Validation Accuracy: 0.907\n",
      "Epoch: 1 / Batch: 32256/60000 / Loss: 0.013236605 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.908\n",
      "Epoch: 1 / Batch: 32768/60000 / Loss: 0.012752505 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.91\n",
      "Epoch: 1 / Batch: 33280/60000 / Loss: 0.011045181 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.915\n",
      "Epoch: 1 / Batch: 33792/60000 / Loss: 0.012288475 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.914\n",
      "Epoch: 1 / Batch: 34304/60000 / Loss: 0.0106699895 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.913\n",
      "Epoch: 1 / Batch: 34816/60000 / Loss: 0.011354673 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.92\n",
      "Epoch: 1 / Batch: 35328/60000 / Loss: 0.013009265 / Training Accuracy: 0.9082031 / Validation Accuracy: 0.928\n",
      "Epoch: 1 / Batch: 35840/60000 / Loss: 0.0124487085 / Training Accuracy: 0.921875 / Validation Accuracy: 0.926\n",
      "Epoch: 1 / Batch: 36352/60000 / Loss: 0.00994648 / Training Accuracy: 0.9375 / Validation Accuracy: 0.922\n",
      "Epoch: 1 / Batch: 36864/60000 / Loss: 0.008231169 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.915\n",
      "Epoch: 1 / Batch: 37376/60000 / Loss: 0.010565777 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.917\n",
      "Epoch: 1 / Batch: 37888/60000 / Loss: 0.009208707 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.928\n",
      "Epoch: 1 / Batch: 38400/60000 / Loss: 0.011181186 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.924\n",
      "Epoch: 1 / Batch: 38912/60000 / Loss: 0.009486641 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.922\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 / Batch: 39424/60000 / Loss: 0.009545557 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.928\n",
      "Epoch: 1 / Batch: 39936/60000 / Loss: 0.009830497 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.931\n",
      "Epoch: 1 / Batch: 40448/60000 / Loss: 0.011532742 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.928\n",
      "Epoch: 1 / Batch: 40960/60000 / Loss: 0.008951267 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.926\n",
      "Epoch: 1 / Batch: 41472/60000 / Loss: 0.009232595 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.934\n",
      "Epoch: 1 / Batch: 41984/60000 / Loss: 0.0075394013 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.933\n",
      "Epoch: 1 / Batch: 42496/60000 / Loss: 0.008877648 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.932\n",
      "Epoch: 1 / Batch: 43008/60000 / Loss: 0.009929956 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.926\n",
      "Epoch: 1 / Batch: 43520/60000 / Loss: 0.007069759 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.934\n",
      "Epoch: 1 / Batch: 44032/60000 / Loss: 0.0071208403 / Training Accuracy: 0.953125 / Validation Accuracy: 0.941\n",
      "Epoch: 1 / Batch: 44544/60000 / Loss: 0.010230939 / Training Accuracy: 0.9375 / Validation Accuracy: 0.939\n",
      "Epoch: 1 / Batch: 45056/60000 / Loss: 0.009724525 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.933\n",
      "Epoch: 1 / Batch: 45568/60000 / Loss: 0.009074115 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.935\n",
      "Epoch: 1 / Batch: 46080/60000 / Loss: 0.0084305145 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.941\n",
      "Epoch: 1 / Batch: 46592/60000 / Loss: 0.0060468363 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.939\n",
      "Epoch: 1 / Batch: 47104/60000 / Loss: 0.00833928 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.934\n",
      "Epoch: 1 / Batch: 47616/60000 / Loss: 0.010285991 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.932\n",
      "Epoch: 1 / Batch: 48128/60000 / Loss: 0.008036848 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.936\n",
      "Epoch: 1 / Batch: 48640/60000 / Loss: 0.006443868 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.942\n",
      "Epoch: 1 / Batch: 49152/60000 / Loss: 0.007905652 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.938\n",
      "Epoch: 1 / Batch: 49664/60000 / Loss: 0.008888787 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.937\n",
      "Epoch: 1 / Batch: 50176/60000 / Loss: 0.007954704 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.935\n",
      "Epoch: 1 / Batch: 50688/60000 / Loss: 0.008941509 / Training Accuracy: 0.9375 / Validation Accuracy: 0.934\n",
      "Epoch: 1 / Batch: 51200/60000 / Loss: 0.007998181 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.935\n",
      "Epoch: 1 / Batch: 51712/60000 / Loss: 0.0074513024 / Training Accuracy: 0.953125 / Validation Accuracy: 0.938\n",
      "Epoch: 1 / Batch: 52224/60000 / Loss: 0.009627499 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.943\n",
      "Epoch: 1 / Batch: 52736/60000 / Loss: 0.007861155 / Training Accuracy: 0.953125 / Validation Accuracy: 0.947\n",
      "Epoch: 1 / Batch: 53248/60000 / Loss: 0.007364341 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.947\n",
      "Epoch: 1 / Batch: 53760/60000 / Loss: 0.0076066516 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.948\n",
      "Epoch: 1 / Batch: 54272/60000 / Loss: 0.006346108 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.94\n",
      "Epoch: 1 / Batch: 54784/60000 / Loss: 0.007754291 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.937\n",
      "Epoch: 1 / Batch: 55296/60000 / Loss: 0.008296364 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.941\n",
      "Epoch: 1 / Batch: 55808/60000 / Loss: 0.008002447 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.945\n",
      "Epoch: 1 / Batch: 56320/60000 / Loss: 0.007082184 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.942\n",
      "Epoch: 1 / Batch: 56832/60000 / Loss: 0.006146606 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.947\n",
      "Epoch: 1 / Batch: 57344/60000 / Loss: 0.008899064 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.944\n",
      "Epoch: 1 / Batch: 57856/60000 / Loss: 0.0069773695 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.945\n",
      "Epoch: 1 / Batch: 58368/60000 / Loss: 0.0066417335 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.945\n",
      "Epoch: 1 / Batch: 58880/60000 / Loss: 0.0065592825 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.943\n",
      "Epoch: 1 / Batch: 59392/60000 / Loss: 0.0070257797 / Training Accuracy: 0.953125 / Validation Accuracy: 0.941\n",
      "Epoch: 1 / Batch: 59904/60000 / Loss: 0.009174121 / Training Accuracy: 0.9375 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 0/60000 / Loss: 0.0065467255 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.944\n",
      "Epoch: 2 / Batch: 512/60000 / Loss: 0.005360175 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.946\n",
      "Epoch: 2 / Batch: 1024/60000 / Loss: 0.004946007 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.945\n",
      "Epoch: 2 / Batch: 1536/60000 / Loss: 0.0075312024 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 2048/60000 / Loss: 0.007687087 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.945\n",
      "Epoch: 2 / Batch: 2560/60000 / Loss: 0.005664792 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.945\n",
      "Epoch: 2 / Batch: 3072/60000 / Loss: 0.007355696 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.947\n",
      "Epoch: 2 / Batch: 3584/60000 / Loss: 0.0063162423 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.948\n",
      "Epoch: 2 / Batch: 4096/60000 / Loss: 0.00652166 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.947\n",
      "Epoch: 2 / Batch: 4608/60000 / Loss: 0.0051552737 / Training Accuracy: 0.96875 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 5120/60000 / Loss: 0.0049170386 / Training Accuracy: 0.96875 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 5632/60000 / Loss: 0.008314733 / Training Accuracy: 0.953125 / Validation Accuracy: 0.944\n",
      "Epoch: 2 / Batch: 6144/60000 / Loss: 0.006409579 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.949\n",
      "Epoch: 2 / Batch: 6656/60000 / Loss: 0.0075250217 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.948\n",
      "Epoch: 2 / Batch: 7168/60000 / Loss: 0.006356991 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.941\n",
      "Epoch: 2 / Batch: 7680/60000 / Loss: 0.006080365 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.945\n",
      "Epoch: 2 / Batch: 8192/60000 / Loss: 0.0055110045 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.949\n",
      "Epoch: 2 / Batch: 8704/60000 / Loss: 0.007034843 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.951\n",
      "Epoch: 2 / Batch: 9216/60000 / Loss: 0.0067054913 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.953\n",
      "Epoch: 2 / Batch: 9728/60000 / Loss: 0.0045993817 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.954\n",
      "Epoch: 2 / Batch: 10240/60000 / Loss: 0.008790502 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.952\n",
      "Epoch: 2 / Batch: 10752/60000 / Loss: 0.0058693723 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.955\n",
      "Epoch: 2 / Batch: 11264/60000 / Loss: 0.00572784 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.956\n",
      "Epoch: 2 / Batch: 11776/60000 / Loss: 0.007193552 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.955\n",
      "Epoch: 2 / Batch: 12288/60000 / Loss: 0.0056178225 / Training Accuracy: 0.96875 / Validation Accuracy: 0.955\n",
      "Epoch: 2 / Batch: 12800/60000 / Loss: 0.0068908944 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.952\n",
      "Epoch: 2 / Batch: 13312/60000 / Loss: 0.0061042113 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.954\n",
      "Epoch: 2 / Batch: 13824/60000 / Loss: 0.005796472 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.952\n",
      "Epoch: 2 / Batch: 14336/60000 / Loss: 0.0057861535 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.953\n",
      "Epoch: 2 / Batch: 14848/60000 / Loss: 0.00595513 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.958\n",
      "Epoch: 2 / Batch: 15360/60000 / Loss: 0.005034713 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.961\n",
      "Epoch: 2 / Batch: 15872/60000 / Loss: 0.0062251645 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.961\n",
      "Epoch: 2 / Batch: 16384/60000 / Loss: 0.0049486705 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.956\n",
      "Epoch: 2 / Batch: 16896/60000 / Loss: 0.0062271166 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.956\n",
      "Epoch: 2 / Batch: 17408/60000 / Loss: 0.004853692 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.953\n",
      "Epoch: 2 / Batch: 17920/60000 / Loss: 0.0058280188 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.956\n",
      "Epoch: 2 / Batch: 18432/60000 / Loss: 0.0048538 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.956\n",
      "Epoch: 2 / Batch: 18944/60000 / Loss: 0.0067218593 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.955\n",
      "Epoch: 2 / Batch: 19456/60000 / Loss: 0.0043040407 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.955\n",
      "Epoch: 2 / Batch: 19968/60000 / Loss: 0.0064929174 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.95\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 / Batch: 20480/60000 / Loss: 0.006727631 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.953\n",
      "Epoch: 2 / Batch: 20992/60000 / Loss: 0.0050890255 / Training Accuracy: 0.96875 / Validation Accuracy: 0.956\n",
      "Epoch: 2 / Batch: 21504/60000 / Loss: 0.007051371 / Training Accuracy: 0.953125 / Validation Accuracy: 0.957\n",
      "Epoch: 2 / Batch: 22016/60000 / Loss: 0.004466276 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.953\n",
      "Epoch: 2 / Batch: 22528/60000 / Loss: 0.006744653 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.96\n",
      "Epoch: 2 / Batch: 23040/60000 / Loss: 0.0049846144 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.962\n",
      "Epoch: 2 / Batch: 23552/60000 / Loss: 0.0033623963 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.959\n",
      "Epoch: 2 / Batch: 24064/60000 / Loss: 0.0055456124 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.956\n",
      "Epoch: 2 / Batch: 24576/60000 / Loss: 0.008279244 / Training Accuracy: 0.9375 / Validation Accuracy: 0.961\n",
      "Epoch: 2 / Batch: 25088/60000 / Loss: 0.0053533544 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.963\n",
      "Epoch: 2 / Batch: 25600/60000 / Loss: 0.0061477115 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.956\n",
      "Epoch: 2 / Batch: 26112/60000 / Loss: 0.0072773183 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.957\n",
      "Epoch: 2 / Batch: 26624/60000 / Loss: 0.005792267 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.96\n",
      "Epoch: 2 / Batch: 27136/60000 / Loss: 0.0064079673 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.962\n",
      "Epoch: 2 / Batch: 27648/60000 / Loss: 0.004978412 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.965\n",
      "Epoch: 2 / Batch: 28160/60000 / Loss: 0.005399848 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.965\n",
      "Epoch: 2 / Batch: 28672/60000 / Loss: 0.005109405 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.965\n",
      "Epoch: 2 / Batch: 29184/60000 / Loss: 0.004284173 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.966\n",
      "Epoch: 2 / Batch: 29696/60000 / Loss: 0.0050094617 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.963\n",
      "Epoch: 2 / Batch: 30208/60000 / Loss: 0.0057356176 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.964\n",
      "Epoch: 2 / Batch: 30720/60000 / Loss: 0.0053022215 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.964\n",
      "Epoch: 2 / Batch: 31232/60000 / Loss: 0.002954098 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.965\n",
      "Epoch: 2 / Batch: 31744/60000 / Loss: 0.003963257 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 2 / Batch: 32256/60000 / Loss: 0.0051946463 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.963\n",
      "Epoch: 2 / Batch: 32768/60000 / Loss: 0.006095237 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.962\n",
      "Epoch: 2 / Batch: 33280/60000 / Loss: 0.005874589 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.96\n",
      "Epoch: 2 / Batch: 33792/60000 / Loss: 0.0070359483 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.962\n",
      "Epoch: 2 / Batch: 34304/60000 / Loss: 0.0036792636 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.958\n",
      "Epoch: 2 / Batch: 34816/60000 / Loss: 0.0059582237 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.956\n",
      "Epoch: 2 / Batch: 35328/60000 / Loss: 0.0052105277 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.958\n",
      "Epoch: 2 / Batch: 35840/60000 / Loss: 0.0063605867 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.964\n",
      "Epoch: 2 / Batch: 36352/60000 / Loss: 0.00449483 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 2 / Batch: 36864/60000 / Loss: 0.0043586083 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.967\n",
      "Epoch: 2 / Batch: 37376/60000 / Loss: 0.0063757314 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.965\n",
      "Epoch: 2 / Batch: 37888/60000 / Loss: 0.005082498 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.967\n",
      "Epoch: 2 / Batch: 38400/60000 / Loss: 0.004901565 / Training Accuracy: 0.96875 / Validation Accuracy: 0.969\n",
      "Epoch: 2 / Batch: 38912/60000 / Loss: 0.004254188 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.966\n",
      "Epoch: 2 / Batch: 39424/60000 / Loss: 0.004681213 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.965\n",
      "Epoch: 2 / Batch: 39936/60000 / Loss: 0.004820652 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.963\n",
      "Epoch: 2 / Batch: 40448/60000 / Loss: 0.004179621 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.962\n",
      "Epoch: 2 / Batch: 40960/60000 / Loss: 0.003932538 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.967\n",
      "Epoch: 2 / Batch: 41472/60000 / Loss: 0.005001113 / Training Accuracy: 0.96875 / Validation Accuracy: 0.968\n",
      "Epoch: 2 / Batch: 41984/60000 / Loss: 0.0051165773 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.965\n",
      "Epoch: 2 / Batch: 42496/60000 / Loss: 0.0035077794 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.964\n",
      "Epoch: 2 / Batch: 43008/60000 / Loss: 0.0037421263 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 2 / Batch: 43520/60000 / Loss: 0.0055757337 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.963\n",
      "Epoch: 2 / Batch: 44032/60000 / Loss: 0.003869918 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 2 / Batch: 44544/60000 / Loss: 0.0040836623 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.965\n",
      "Epoch: 2 / Batch: 45056/60000 / Loss: 0.004511156 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.965\n",
      "Epoch: 2 / Batch: 45568/60000 / Loss: 0.005714373 / Training Accuracy: 0.96875 / Validation Accuracy: 0.968\n",
      "Epoch: 2 / Batch: 46080/60000 / Loss: 0.003627325 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.971\n",
      "Epoch: 2 / Batch: 46592/60000 / Loss: 0.0030243532 / Training Accuracy: 0.984375 / Validation Accuracy: 0.969\n",
      "Epoch: 2 / Batch: 47104/60000 / Loss: 0.004482332 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.968\n",
      "Epoch: 2 / Batch: 47616/60000 / Loss: 0.0035826452 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.968\n",
      "Epoch: 2 / Batch: 48128/60000 / Loss: 0.0032865487 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.966\n",
      "Epoch: 2 / Batch: 48640/60000 / Loss: 0.0036662538 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.967\n",
      "Epoch: 2 / Batch: 49152/60000 / Loss: 0.002892483 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 2 / Batch: 49664/60000 / Loss: 0.0039897277 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 2 / Batch: 50176/60000 / Loss: 0.0039482093 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.966\n",
      "Epoch: 2 / Batch: 50688/60000 / Loss: 0.005327564 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.968\n",
      "Epoch: 2 / Batch: 51200/60000 / Loss: 0.0042024506 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.968\n",
      "Epoch: 2 / Batch: 51712/60000 / Loss: 0.0034459098 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.967\n",
      "Epoch: 2 / Batch: 52224/60000 / Loss: 0.0042329673 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.968\n",
      "Epoch: 2 / Batch: 52736/60000 / Loss: 0.0039227055 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.968\n",
      "Epoch: 2 / Batch: 53248/60000 / Loss: 0.004187723 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.969\n",
      "Epoch: 2 / Batch: 53760/60000 / Loss: 0.005965928 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.968\n",
      "Epoch: 2 / Batch: 54272/60000 / Loss: 0.0048716255 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 2 / Batch: 54784/60000 / Loss: 0.004188217 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.973\n",
      "Epoch: 2 / Batch: 55296/60000 / Loss: 0.0049188747 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.971\n",
      "Epoch: 2 / Batch: 55808/60000 / Loss: 0.0034837672 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.972\n",
      "Epoch: 2 / Batch: 56320/60000 / Loss: 0.0036128904 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.977\n",
      "Epoch: 2 / Batch: 56832/60000 / Loss: 0.0038355733 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.978\n",
      "Epoch: 2 / Batch: 57344/60000 / Loss: 0.0033190101 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 2 / Batch: 57856/60000 / Loss: 0.0031895502 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.97\n",
      "Epoch: 2 / Batch: 58368/60000 / Loss: 0.0041755266 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.969\n",
      "Epoch: 2 / Batch: 58880/60000 / Loss: 0.0041422425 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 2 / Batch: 59392/60000 / Loss: 0.004882599 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.973\n",
      "Epoch: 2 / Batch: 59904/60000 / Loss: 0.0053671915 / Training Accuracy: 0.9583333 / Validation Accuracy: 0.975\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 / Batch: 0/60000 / Loss: 0.0040280484 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.969\n",
      "Epoch: 3 / Batch: 512/60000 / Loss: 0.004011091 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.969\n",
      "Epoch: 3 / Batch: 1024/60000 / Loss: 0.0045425138 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.968\n",
      "Epoch: 3 / Batch: 1536/60000 / Loss: 0.004199448 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.967\n",
      "Epoch: 3 / Batch: 2048/60000 / Loss: 0.003459402 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 3 / Batch: 2560/60000 / Loss: 0.0039837705 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n",
      "Epoch: 3 / Batch: 3072/60000 / Loss: 0.0051104403 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.97\n",
      "Epoch: 3 / Batch: 3584/60000 / Loss: 0.0031764128 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 3 / Batch: 4096/60000 / Loss: 0.0049010688 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.972\n",
      "Epoch: 3 / Batch: 4608/60000 / Loss: 0.0045449743 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.973\n",
      "Epoch: 3 / Batch: 5120/60000 / Loss: 0.004206571 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.972\n",
      "Epoch: 3 / Batch: 5632/60000 / Loss: 0.00418436 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 6144/60000 / Loss: 0.0052701323 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 6656/60000 / Loss: 0.0046931044 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 7168/60000 / Loss: 0.002761687 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 7680/60000 / Loss: 0.0047716536 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 8192/60000 / Loss: 0.004175554 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 8704/60000 / Loss: 0.0030208104 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 9216/60000 / Loss: 0.003345816 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.982\n",
      "Epoch: 3 / Batch: 9728/60000 / Loss: 0.00258919 / Training Accuracy: 0.984375 / Validation Accuracy: 0.982\n",
      "Epoch: 3 / Batch: 10240/60000 / Loss: 0.004080505 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 10752/60000 / Loss: 0.0034692641 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 11264/60000 / Loss: 0.0029065947 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 11776/60000 / Loss: 0.003567189 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 12288/60000 / Loss: 0.004008243 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 12800/60000 / Loss: 0.004274412 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.974\n",
      "Epoch: 3 / Batch: 13312/60000 / Loss: 0.0030680606 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 13824/60000 / Loss: 0.0028849132 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 14336/60000 / Loss: 0.0022401647 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 14848/60000 / Loss: 0.0028907817 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 15360/60000 / Loss: 0.0018186662 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 15872/60000 / Loss: 0.0044587515 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 16384/60000 / Loss: 0.004517432 / Training Accuracy: 0.96875 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 16896/60000 / Loss: 0.0035909128 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 17408/60000 / Loss: 0.0026864246 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 17920/60000 / Loss: 0.0037320536 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 18432/60000 / Loss: 0.0045986422 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.974\n",
      "Epoch: 3 / Batch: 18944/60000 / Loss: 0.0027772835 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.974\n",
      "Epoch: 3 / Batch: 19456/60000 / Loss: 0.0032254823 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.972\n",
      "Epoch: 3 / Batch: 19968/60000 / Loss: 0.0037396606 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 3 / Batch: 20480/60000 / Loss: 0.0023908198 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 20992/60000 / Loss: 0.0024081946 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 21504/60000 / Loss: 0.0028678111 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 22016/60000 / Loss: 0.0029971641 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 22528/60000 / Loss: 0.004071765 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 23040/60000 / Loss: 0.0041586845 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 23552/60000 / Loss: 0.0034579528 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 24064/60000 / Loss: 0.0055812094 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 24576/60000 / Loss: 0.0028927205 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 3 / Batch: 25088/60000 / Loss: 0.0035861065 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 25600/60000 / Loss: 0.0025999933 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 26112/60000 / Loss: 0.0032358146 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 26624/60000 / Loss: 0.0024850566 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 27136/60000 / Loss: 0.0057993447 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.973\n",
      "Epoch: 3 / Batch: 27648/60000 / Loss: 0.0050196378 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 28160/60000 / Loss: 0.0027381172 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.982\n",
      "Epoch: 3 / Batch: 28672/60000 / Loss: 0.004594675 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.983\n",
      "Epoch: 3 / Batch: 29184/60000 / Loss: 0.004503556 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 29696/60000 / Loss: 0.0032886297 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 30208/60000 / Loss: 0.0027845628 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.983\n",
      "Epoch: 3 / Batch: 30720/60000 / Loss: 0.0029822574 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 31232/60000 / Loss: 0.003278988 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 31744/60000 / Loss: 0.0038716171 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 32256/60000 / Loss: 0.002811785 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 32768/60000 / Loss: 0.002833544 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 33280/60000 / Loss: 0.0030956657 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 33792/60000 / Loss: 0.0042572133 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 34304/60000 / Loss: 0.0032543887 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 34816/60000 / Loss: 0.0034234305 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 35328/60000 / Loss: 0.0037726979 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 35840/60000 / Loss: 0.0024561756 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 36352/60000 / Loss: 0.0028302884 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 36864/60000 / Loss: 0.0027960534 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 37376/60000 / Loss: 0.0033995106 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 37888/60000 / Loss: 0.0038475501 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 38400/60000 / Loss: 0.0026749042 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 38912/60000 / Loss: 0.004606028 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.981\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 / Batch: 39424/60000 / Loss: 0.0034442164 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.982\n",
      "Epoch: 3 / Batch: 39936/60000 / Loss: 0.00457698 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.983\n",
      "Epoch: 3 / Batch: 40448/60000 / Loss: 0.00294836 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.983\n",
      "Epoch: 3 / Batch: 40960/60000 / Loss: 0.0025534933 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.983\n",
      "Epoch: 3 / Batch: 41472/60000 / Loss: 0.0037134402 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.982\n",
      "Epoch: 3 / Batch: 41984/60000 / Loss: 0.0025918852 / Training Accuracy: 0.984375 / Validation Accuracy: 0.985\n",
      "Epoch: 3 / Batch: 42496/60000 / Loss: 0.0036871482 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.983\n",
      "Epoch: 3 / Batch: 43008/60000 / Loss: 0.001608541 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 43520/60000 / Loss: 0.0020856028 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 44032/60000 / Loss: 0.0033983383 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 44544/60000 / Loss: 0.0032363296 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 45056/60000 / Loss: 0.0038825304 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 45568/60000 / Loss: 0.0024440996 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 3 / Batch: 46080/60000 / Loss: 0.0032561715 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 46592/60000 / Loss: 0.0036921918 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.978\n",
      "Epoch: 3 / Batch: 47104/60000 / Loss: 0.0032081925 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.984\n",
      "Epoch: 3 / Batch: 47616/60000 / Loss: 0.002061094 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 48128/60000 / Loss: 0.0031805546 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 48640/60000 / Loss: 0.0019491432 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.984\n",
      "Epoch: 3 / Batch: 49152/60000 / Loss: 0.0019300109 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.983\n",
      "Epoch: 3 / Batch: 49664/60000 / Loss: 0.0038578548 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.982\n",
      "Epoch: 3 / Batch: 50176/60000 / Loss: 0.002601054 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 50688/60000 / Loss: 0.0034522365 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 51200/60000 / Loss: 0.0023179266 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 51712/60000 / Loss: 0.0033982384 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.982\n",
      "Epoch: 3 / Batch: 52224/60000 / Loss: 0.0026900002 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.982\n",
      "Epoch: 3 / Batch: 52736/60000 / Loss: 0.002440399 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 53248/60000 / Loss: 0.00266498 / Training Accuracy: 0.984375 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 53760/60000 / Loss: 0.0016406073 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 54272/60000 / Loss: 0.0018640801 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 54784/60000 / Loss: 0.003801959 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 55296/60000 / Loss: 0.0023854515 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 3 / Batch: 55808/60000 / Loss: 0.0028327585 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.979\n",
      "Epoch: 3 / Batch: 56320/60000 / Loss: 0.0012674986 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.984\n",
      "Epoch: 3 / Batch: 56832/60000 / Loss: 0.0035536462 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.982\n",
      "Epoch: 3 / Batch: 57344/60000 / Loss: 0.002545702 / Training Accuracy: 0.984375 / Validation Accuracy: 0.981\n",
      "Epoch: 3 / Batch: 57856/60000 / Loss: 0.003222778 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.98\n",
      "Epoch: 3 / Batch: 58368/60000 / Loss: 0.003516833 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.983\n",
      "Epoch: 3 / Batch: 58880/60000 / Loss: 0.00253831 / Training Accuracy: 0.984375 / Validation Accuracy: 0.983\n",
      "Epoch: 3 / Batch: 59392/60000 / Loss: 0.0034098357 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.984\n",
      "Epoch: 3 / Batch: 59904/60000 / Loss: 0.0010384284 / Training Accuracy: 1.0 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 0/60000 / Loss: 0.0018475596 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 512/60000 / Loss: 0.0024932846 / Training Accuracy: 0.984375 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 1024/60000 / Loss: 0.0035956644 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 1536/60000 / Loss: 0.003118673 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 2048/60000 / Loss: 0.0015669577 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 2560/60000 / Loss: 0.0025250623 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.979\n",
      "Epoch: 4 / Batch: 3072/60000 / Loss: 0.002267128 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 3584/60000 / Loss: 0.0023734136 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 4096/60000 / Loss: 0.0021830178 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 4608/60000 / Loss: 0.0026020315 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.978\n",
      "Epoch: 4 / Batch: 5120/60000 / Loss: 0.0020025612 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 5632/60000 / Loss: 0.004208392 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 6144/60000 / Loss: 0.0030038108 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 6656/60000 / Loss: 0.002315289 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 7168/60000 / Loss: 0.0024104272 / Training Accuracy: 0.984375 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 7680/60000 / Loss: 0.0024990947 / Training Accuracy: 0.984375 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 8192/60000 / Loss: 0.002460388 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 8704/60000 / Loss: 0.0024315205 / Training Accuracy: 0.984375 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 9216/60000 / Loss: 0.0044933045 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 9728/60000 / Loss: 0.0027616206 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 10240/60000 / Loss: 0.003420479 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 10752/60000 / Loss: 0.0018346937 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 11264/60000 / Loss: 0.0022996136 / Training Accuracy: 0.984375 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 11776/60000 / Loss: 0.0022472786 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 12288/60000 / Loss: 0.0020987256 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.979\n",
      "Epoch: 4 / Batch: 12800/60000 / Loss: 0.0024870671 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 13312/60000 / Loss: 0.002675486 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 13824/60000 / Loss: 0.0024788529 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 14336/60000 / Loss: 0.0014160248 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 4 / Batch: 14848/60000 / Loss: 0.0016232489 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 15360/60000 / Loss: 0.0026522805 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 15872/60000 / Loss: 0.0020633703 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 16384/60000 / Loss: 0.0027996395 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 16896/60000 / Loss: 0.002869423 / Training Accuracy: 0.984375 / Validation Accuracy: 0.982\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 / Batch: 17408/60000 / Loss: 0.0028134293 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 17920/60000 / Loss: 0.0026079006 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.979\n",
      "Epoch: 4 / Batch: 18432/60000 / Loss: 0.0020828922 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 18944/60000 / Loss: 0.0026903264 / Training Accuracy: 0.984375 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 19456/60000 / Loss: 0.0013696719 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 19968/60000 / Loss: 0.0031181567 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 20480/60000 / Loss: 0.0018946875 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 20992/60000 / Loss: 0.0025222558 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 21504/60000 / Loss: 0.00146947 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 22016/60000 / Loss: 0.0019320942 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 22528/60000 / Loss: 0.0017725282 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 23040/60000 / Loss: 0.0036386545 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 23552/60000 / Loss: 0.0023825471 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 24064/60000 / Loss: 0.0010746134 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 24576/60000 / Loss: 0.0027521972 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 25088/60000 / Loss: 0.002466964 / Training Accuracy: 0.984375 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 25600/60000 / Loss: 0.0016251008 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 26112/60000 / Loss: 0.0018429153 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 26624/60000 / Loss: 0.0033907644 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 27136/60000 / Loss: 0.0028929634 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 27648/60000 / Loss: 0.0028310684 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 28160/60000 / Loss: 0.0021176233 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 28672/60000 / Loss: 0.002772357 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 29184/60000 / Loss: 0.0022732348 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 29696/60000 / Loss: 0.0036862441 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.978\n",
      "Epoch: 4 / Batch: 30208/60000 / Loss: 0.0047250954 / Training Accuracy: 0.96875 / Validation Accuracy: 0.978\n",
      "Epoch: 4 / Batch: 30720/60000 / Loss: 0.0025201403 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 4 / Batch: 31232/60000 / Loss: 0.0031630315 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 31744/60000 / Loss: 0.0014265069 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 32256/60000 / Loss: 0.0040368717 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 32768/60000 / Loss: 0.003574305 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 33280/60000 / Loss: 0.002176914 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 33792/60000 / Loss: 0.0034592147 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 34304/60000 / Loss: 0.002850707 / Training Accuracy: 0.984375 / Validation Accuracy: 0.987\n",
      "Epoch: 4 / Batch: 34816/60000 / Loss: 0.002976194 / Training Accuracy: 0.984375 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 35328/60000 / Loss: 0.002701906 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 35840/60000 / Loss: 0.00336088 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 36352/60000 / Loss: 0.0017229464 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 36864/60000 / Loss: 0.0025529286 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 4 / Batch: 37376/60000 / Loss: 0.0012704224 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 37888/60000 / Loss: 0.0031233537 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 38400/60000 / Loss: 0.003347795 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 38912/60000 / Loss: 0.0033804744 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.979\n",
      "Epoch: 4 / Batch: 39424/60000 / Loss: 0.0023305386 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 39936/60000 / Loss: 0.0014872145 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 40448/60000 / Loss: 0.0022080513 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 40960/60000 / Loss: 0.0022835082 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 41472/60000 / Loss: 0.0034236596 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 41984/60000 / Loss: 0.003071518 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 42496/60000 / Loss: 0.0032734438 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 43008/60000 / Loss: 0.0034970343 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 43520/60000 / Loss: 0.0019554086 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 44032/60000 / Loss: 0.0039646574 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 44544/60000 / Loss: 0.0017537496 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 45056/60000 / Loss: 0.0013604454 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 45568/60000 / Loss: 0.0030191555 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.979\n",
      "Epoch: 4 / Batch: 46080/60000 / Loss: 0.0020108738 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 46592/60000 / Loss: 0.0023780067 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.979\n",
      "Epoch: 4 / Batch: 47104/60000 / Loss: 0.0021852744 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 47616/60000 / Loss: 0.0028340828 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 48128/60000 / Loss: 0.0013781777 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 48640/60000 / Loss: 0.0016435761 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 49152/60000 / Loss: 0.0020638607 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 4 / Batch: 49664/60000 / Loss: 0.0017569397 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 50176/60000 / Loss: 0.0017719846 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 50688/60000 / Loss: 0.002949416 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 51200/60000 / Loss: 0.0029584225 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 51712/60000 / Loss: 0.0025636237 / Training Accuracy: 0.984375 / Validation Accuracy: 0.986\n",
      "Epoch: 4 / Batch: 52224/60000 / Loss: 0.0022730676 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 4 / Batch: 52736/60000 / Loss: 0.0026951514 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.984\n",
      "Epoch: 4 / Batch: 53248/60000 / Loss: 0.0020076022 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.981\n",
      "Epoch: 4 / Batch: 53760/60000 / Loss: 0.00254662 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.977\n",
      "Epoch: 4 / Batch: 54272/60000 / Loss: 0.0035083697 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.98\n",
      "Epoch: 4 / Batch: 54784/60000 / Loss: 0.0024358935 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 4 / Batch: 55296/60000 / Loss: 0.0019172516 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 55808/60000 / Loss: 0.002408709 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 56320/60000 / Loss: 0.0020689308 / Training Accuracy: 0.984375 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 56832/60000 / Loss: 0.002034489 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.983\n",
      "Epoch: 4 / Batch: 57344/60000 / Loss: 0.0021498874 / Training Accuracy: 0.984375 / Validation Accuracy: 0.981\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 / Batch: 57856/60000 / Loss: 0.0042553497 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.982\n",
      "Epoch: 4 / Batch: 58368/60000 / Loss: 0.0016884981 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 58880/60000 / Loss: 0.0022678783 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 59392/60000 / Loss: 0.002447368 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.985\n",
      "Epoch: 4 / Batch: 59904/60000 / Loss: 0.0017643484 / Training Accuracy: 0.9895833 / Validation Accuracy: 0.982\n",
      "Epoch: 5 / Batch: 0/60000 / Loss: 0.0016422154 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.979\n",
      "Epoch: 5 / Batch: 512/60000 / Loss: 0.0030284326 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 5 / Batch: 1024/60000 / Loss: 0.0036827996 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.982\n",
      "Epoch: 5 / Batch: 1536/60000 / Loss: 0.002349008 / Training Accuracy: 0.984375 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 2048/60000 / Loss: 0.001610403 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.981\n",
      "Epoch: 5 / Batch: 2560/60000 / Loss: 0.0031767294 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 5 / Batch: 3072/60000 / Loss: 0.0026993805 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 5 / Batch: 3584/60000 / Loss: 0.0026959332 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.979\n",
      "Epoch: 5 / Batch: 4096/60000 / Loss: 0.003436071 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 4608/60000 / Loss: 0.002065945 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 5120/60000 / Loss: 0.0021323494 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.98\n",
      "Epoch: 5 / Batch: 5632/60000 / Loss: 0.003060342 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 6144/60000 / Loss: 0.0035899647 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 6656/60000 / Loss: 0.0014758112 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 7168/60000 / Loss: 0.002188275 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 7680/60000 / Loss: 0.002271462 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 8192/60000 / Loss: 0.002045378 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 5 / Batch: 8704/60000 / Loss: 0.0014884187 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 5 / Batch: 9216/60000 / Loss: 0.0009126192 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.98\n",
      "Epoch: 5 / Batch: 9728/60000 / Loss: 0.0026059796 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 5 / Batch: 10240/60000 / Loss: 0.0035802189 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.98\n",
      "Epoch: 5 / Batch: 10752/60000 / Loss: 0.0026472185 / Training Accuracy: 0.984375 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 11264/60000 / Loss: 0.0016408978 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 11776/60000 / Loss: 0.0016222876 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 12288/60000 / Loss: 0.0013949967 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 12800/60000 / Loss: 0.00080862147 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 13312/60000 / Loss: 0.0015041612 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 13824/60000 / Loss: 0.0023161615 / Training Accuracy: 0.984375 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 14336/60000 / Loss: 0.00074158155 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 14848/60000 / Loss: 0.0028522662 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 15360/60000 / Loss: 0.0020938779 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 15872/60000 / Loss: 0.0006272764 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 16384/60000 / Loss: 0.0016346804 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 16896/60000 / Loss: 0.0018215349 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 17408/60000 / Loss: 0.0020819358 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 17920/60000 / Loss: 0.0014391039 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 18432/60000 / Loss: 0.0027031368 / Training Accuracy: 0.984375 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 18944/60000 / Loss: 0.0022441004 / Training Accuracy: 0.984375 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 19456/60000 / Loss: 0.002036672 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 19968/60000 / Loss: 0.0030955507 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 20480/60000 / Loss: 0.001978806 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 20992/60000 / Loss: 0.0016209984 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 21504/60000 / Loss: 0.0021158915 / Training Accuracy: 0.984375 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 22016/60000 / Loss: 0.0014314785 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 22528/60000 / Loss: 0.0022706292 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 23040/60000 / Loss: 0.001736946 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 23552/60000 / Loss: 0.0016409907 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 24064/60000 / Loss: 0.0021656537 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 24576/60000 / Loss: 0.003858819 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 25088/60000 / Loss: 0.0011927165 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 25600/60000 / Loss: 0.0024878744 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 26112/60000 / Loss: 0.0018209789 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 26624/60000 / Loss: 0.003174365 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 27136/60000 / Loss: 0.00439731 / Training Accuracy: 0.96875 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 27648/60000 / Loss: 0.0019640676 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 28160/60000 / Loss: 0.0022430022 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.989\n",
      "Epoch: 5 / Batch: 28672/60000 / Loss: 0.0029266167 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 29184/60000 / Loss: 0.0022493703 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 29696/60000 / Loss: 0.0024600902 / Training Accuracy: 0.984375 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 30208/60000 / Loss: 0.002285068 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 30720/60000 / Loss: 0.0029415127 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 31232/60000 / Loss: 0.0024096295 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 31744/60000 / Loss: 0.0018411751 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 32256/60000 / Loss: 0.002733515 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 32768/60000 / Loss: 0.0019839385 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 33280/60000 / Loss: 0.0023221401 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 33792/60000 / Loss: 0.0013242675 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.982\n",
      "Epoch: 5 / Batch: 34304/60000 / Loss: 0.0012256161 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 34816/60000 / Loss: 0.0017702341 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 35328/60000 / Loss: 0.0013761886 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.984\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 / Batch: 35840/60000 / Loss: 0.0021947029 / Training Accuracy: 0.984375 / Validation Accuracy: 0.981\n",
      "Epoch: 5 / Batch: 36352/60000 / Loss: 0.0017215215 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.98\n",
      "Epoch: 5 / Batch: 36864/60000 / Loss: 0.0017329121 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 37376/60000 / Loss: 0.0018906761 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 37888/60000 / Loss: 0.0020698127 / Training Accuracy: 0.984375 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 38400/60000 / Loss: 0.0020266438 / Training Accuracy: 0.984375 / Validation Accuracy: 0.982\n",
      "Epoch: 5 / Batch: 38912/60000 / Loss: 0.0019931197 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.982\n",
      "Epoch: 5 / Batch: 39424/60000 / Loss: 0.0008597417 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.982\n",
      "Epoch: 5 / Batch: 39936/60000 / Loss: 0.0029178534 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 40448/60000 / Loss: 0.0013143986 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 40960/60000 / Loss: 0.0014126522 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 41472/60000 / Loss: 0.0028097199 / Training Accuracy: 0.984375 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 41984/60000 / Loss: 0.0026525739 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.984\n",
      "Epoch: 5 / Batch: 42496/60000 / Loss: 0.002789937 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.983\n",
      "Epoch: 5 / Batch: 43008/60000 / Loss: 0.0025838162 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 43520/60000 / Loss: 0.0018399371 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 44032/60000 / Loss: 0.0026600421 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 44544/60000 / Loss: 0.0011570636 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 45056/60000 / Loss: 0.0016322685 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 45568/60000 / Loss: 0.0018653491 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 46080/60000 / Loss: 0.002282538 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.985\n",
      "Epoch: 5 / Batch: 46592/60000 / Loss: 0.0024858885 / Training Accuracy: 0.984375 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 47104/60000 / Loss: 0.0028882972 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 47616/60000 / Loss: 0.0013229373 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 48128/60000 / Loss: 0.0011930596 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 48640/60000 / Loss: 0.0019301844 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 49152/60000 / Loss: 0.0019210756 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 49664/60000 / Loss: 0.0019272016 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 50176/60000 / Loss: 0.0021321145 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.989\n",
      "Epoch: 5 / Batch: 50688/60000 / Loss: 0.0014376941 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 5 / Batch: 51200/60000 / Loss: 0.0023392916 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.989\n",
      "Epoch: 5 / Batch: 51712/60000 / Loss: 0.0008936666 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 52224/60000 / Loss: 0.0015851639 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 52736/60000 / Loss: 0.0018508326 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 53248/60000 / Loss: 0.0017295347 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 53760/60000 / Loss: 0.0025092799 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 54272/60000 / Loss: 0.0011723998 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 54784/60000 / Loss: 0.00096667575 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 5 / Batch: 55296/60000 / Loss: 0.0023453608 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.986\n",
      "Epoch: 5 / Batch: 55808/60000 / Loss: 0.0020712672 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.989\n",
      "Epoch: 5 / Batch: 56320/60000 / Loss: 0.001887793 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.991\n",
      "Epoch: 5 / Batch: 56832/60000 / Loss: 0.0008740205 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 5 / Batch: 57344/60000 / Loss: 0.0011339135 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 5 / Batch: 57856/60000 / Loss: 0.0013968705 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 58368/60000 / Loss: 0.0012337894 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 58880/60000 / Loss: 0.0029999153 / Training Accuracy: 0.984375 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 59392/60000 / Loss: 0.0019866293 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 5 / Batch: 59904/60000 / Loss: 0.00043144773 / Training Accuracy: 1.0 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 0/60000 / Loss: 0.001971725 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 512/60000 / Loss: 0.0017057594 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 1024/60000 / Loss: 0.0011393859 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 1536/60000 / Loss: 0.0007425164 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 2048/60000 / Loss: 0.0018880826 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 2560/60000 / Loss: 0.0013156714 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 3072/60000 / Loss: 0.001803281 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.984\n",
      "Epoch: 6 / Batch: 3584/60000 / Loss: 0.0016353221 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 4096/60000 / Loss: 0.0004053985 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 4608/60000 / Loss: 0.0015836576 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 5120/60000 / Loss: 0.0020516955 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.99\n",
      "Epoch: 6 / Batch: 5632/60000 / Loss: 0.001742508 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.991\n",
      "Epoch: 6 / Batch: 6144/60000 / Loss: 0.0015759533 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 6656/60000 / Loss: 0.0012894941 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 7168/60000 / Loss: 0.001763509 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 7680/60000 / Loss: 0.0019423455 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 8192/60000 / Loss: 0.0017976552 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 8704/60000 / Loss: 0.0024066896 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 9216/60000 / Loss: 0.002314353 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 9728/60000 / Loss: 0.0018163892 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.991\n",
      "Epoch: 6 / Batch: 10240/60000 / Loss: 0.0016475465 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.99\n",
      "Epoch: 6 / Batch: 10752/60000 / Loss: 0.001696899 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.99\n",
      "Epoch: 6 / Batch: 11264/60000 / Loss: 0.0024990703 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.99\n",
      "Epoch: 6 / Batch: 11776/60000 / Loss: 0.001175192 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 12288/60000 / Loss: 0.002416003 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 12800/60000 / Loss: 0.0011945765 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 13312/60000 / Loss: 0.0023027482 / Training Accuracy: 0.984375 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 13824/60000 / Loss: 0.0014261013 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 14336/60000 / Loss: 0.0010254554 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 14848/60000 / Loss: 0.0022963607 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.99\n",
      "Epoch: 6 / Batch: 15360/60000 / Loss: 0.0030766462 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 15872/60000 / Loss: 0.0024018805 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.989\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 / Batch: 16384/60000 / Loss: 0.00113018 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 6 / Batch: 16896/60000 / Loss: 0.0020322355 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 17408/60000 / Loss: 0.001680872 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 17920/60000 / Loss: 0.003146403 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 18432/60000 / Loss: 0.0014872162 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 18944/60000 / Loss: 0.001909757 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 19456/60000 / Loss: 0.0022639683 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 19968/60000 / Loss: 0.00132536 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 20480/60000 / Loss: 0.0013699352 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 20992/60000 / Loss: 0.0014823137 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 21504/60000 / Loss: 0.002432757 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 22016/60000 / Loss: 0.0016509896 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 22528/60000 / Loss: 0.0018466616 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 23040/60000 / Loss: 0.0017389089 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 23552/60000 / Loss: 0.0018069946 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 24064/60000 / Loss: 0.0015865744 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 24576/60000 / Loss: 0.0021400577 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 25088/60000 / Loss: 0.00095518294 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 6 / Batch: 25600/60000 / Loss: 0.0018683819 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 26112/60000 / Loss: 0.0015015475 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 26624/60000 / Loss: 0.0012412904 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 27136/60000 / Loss: 0.0016604787 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 27648/60000 / Loss: 0.0024954532 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 28160/60000 / Loss: 0.0018381979 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 28672/60000 / Loss: 0.0011905979 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 29184/60000 / Loss: 0.0013426794 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 29696/60000 / Loss: 0.0022768443 / Training Accuracy: 0.984375 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 30208/60000 / Loss: 0.0019893826 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 30720/60000 / Loss: 0.00089801336 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 31232/60000 / Loss: 0.0006669323 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 6 / Batch: 31744/60000 / Loss: 0.0023291234 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.99\n",
      "Epoch: 6 / Batch: 32256/60000 / Loss: 0.0004496911 / Training Accuracy: 1.0 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 32768/60000 / Loss: 0.002508893 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 33280/60000 / Loss: 0.0013738391 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 33792/60000 / Loss: 0.0030090723 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.984\n",
      "Epoch: 6 / Batch: 34304/60000 / Loss: 0.00068993645 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.983\n",
      "Epoch: 6 / Batch: 34816/60000 / Loss: 0.0017358446 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.982\n",
      "Epoch: 6 / Batch: 35328/60000 / Loss: 0.0015546089 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.983\n",
      "Epoch: 6 / Batch: 35840/60000 / Loss: 0.0020700977 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.983\n",
      "Epoch: 6 / Batch: 36352/60000 / Loss: 0.0012510127 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 36864/60000 / Loss: 0.0009610161 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 37376/60000 / Loss: 0.0015155775 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 37888/60000 / Loss: 0.0017975731 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 38400/60000 / Loss: 0.0017383214 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 38912/60000 / Loss: 0.0017053824 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 39424/60000 / Loss: 0.00204444 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 39936/60000 / Loss: 0.0016745335 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 40448/60000 / Loss: 0.0025226693 / Training Accuracy: 0.984375 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 40960/60000 / Loss: 0.0017318468 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.984\n",
      "Epoch: 6 / Batch: 41472/60000 / Loss: 0.0013415818 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 41984/60000 / Loss: 0.0012690739 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 42496/60000 / Loss: 0.0025084005 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 43008/60000 / Loss: 0.0015966197 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.982\n",
      "Epoch: 6 / Batch: 43520/60000 / Loss: 0.0018727935 / Training Accuracy: 0.984375 / Validation Accuracy: 0.981\n",
      "Epoch: 6 / Batch: 44032/60000 / Loss: 0.0010388462 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.98\n",
      "Epoch: 6 / Batch: 44544/60000 / Loss: 0.0013165222 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.982\n",
      "Epoch: 6 / Batch: 45056/60000 / Loss: 0.0019773692 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 45568/60000 / Loss: 0.0020514852 / Training Accuracy: 0.984375 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 46080/60000 / Loss: 0.001180819 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 46592/60000 / Loss: 0.0010146337 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 47104/60000 / Loss: 0.0013761857 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 47616/60000 / Loss: 0.0021759719 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 48128/60000 / Loss: 0.0010046031 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 48640/60000 / Loss: 0.00056597556 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 49152/60000 / Loss: 0.0011594273 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 49664/60000 / Loss: 0.001433572 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 50176/60000 / Loss: 0.0017286654 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 50688/60000 / Loss: 0.0010067069 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 51200/60000 / Loss: 0.0018697387 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 51712/60000 / Loss: 0.0013546206 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 52224/60000 / Loss: 0.0010305735 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 52736/60000 / Loss: 0.0015319735 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 53248/60000 / Loss: 0.0013435029 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 53760/60000 / Loss: 0.001415475 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 6 / Batch: 54272/60000 / Loss: 0.0034871097 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 54784/60000 / Loss: 0.0032256697 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 55296/60000 / Loss: 0.0016562497 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 6 / Batch: 55808/60000 / Loss: 0.0019725845 / Training Accuracy: 0.984375 / Validation Accuracy: 0.987\n",
      "Epoch: 6 / Batch: 56320/60000 / Loss: 0.0018785277 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 56832/60000 / Loss: 0.0019429408 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 / Batch: 57344/60000 / Loss: 0.00049977965 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 57856/60000 / Loss: 0.0014160179 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 6 / Batch: 58368/60000 / Loss: 0.0014424857 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 58880/60000 / Loss: 0.0020624357 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 59392/60000 / Loss: 0.0014284102 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 6 / Batch: 59904/60000 / Loss: 0.0010309152 / Training Accuracy: 0.9895833 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 0/60000 / Loss: 0.0022394783 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 512/60000 / Loss: 0.001601322 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 1024/60000 / Loss: 0.00086414686 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 1536/60000 / Loss: 0.0014593707 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 2048/60000 / Loss: 0.0014168009 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.991\n",
      "Epoch: 7 / Batch: 2560/60000 / Loss: 0.0010441977 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 3072/60000 / Loss: 0.0030581919 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 3584/60000 / Loss: 0.0010478772 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 4096/60000 / Loss: 0.0008195324 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 7 / Batch: 4608/60000 / Loss: 0.001611116 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 7 / Batch: 5120/60000 / Loss: 0.0012571502 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 5632/60000 / Loss: 0.0013034156 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.991\n",
      "Epoch: 7 / Batch: 6144/60000 / Loss: 0.0013092097 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 7 / Batch: 6656/60000 / Loss: 0.0015007758 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 7168/60000 / Loss: 0.0013136134 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 7680/60000 / Loss: 0.002438493 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 8192/60000 / Loss: 0.0013127096 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 8704/60000 / Loss: 0.0025148154 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 9216/60000 / Loss: 0.0005457472 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 9728/60000 / Loss: 0.0013734675 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 10240/60000 / Loss: 0.0012606565 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.983\n",
      "Epoch: 7 / Batch: 10752/60000 / Loss: 0.0010402993 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.983\n",
      "Epoch: 7 / Batch: 11264/60000 / Loss: 0.002996265 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 11776/60000 / Loss: 0.0016252231 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 12288/60000 / Loss: 0.00057993166 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 12800/60000 / Loss: 0.0015774649 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 13312/60000 / Loss: 0.0012332832 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 13824/60000 / Loss: 0.00071527564 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.983\n",
      "Epoch: 7 / Batch: 14336/60000 / Loss: 0.001230283 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.981\n",
      "Epoch: 7 / Batch: 14848/60000 / Loss: 0.0011550833 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.982\n",
      "Epoch: 7 / Batch: 15360/60000 / Loss: 0.0016712971 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 15872/60000 / Loss: 0.0012872177 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 16384/60000 / Loss: 0.0010588067 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 16896/60000 / Loss: 0.0018736597 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 17408/60000 / Loss: 0.001099095 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.982\n",
      "Epoch: 7 / Batch: 17920/60000 / Loss: 0.0014335973 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.98\n",
      "Epoch: 7 / Batch: 18432/60000 / Loss: 0.0032356374 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.982\n",
      "Epoch: 7 / Batch: 18944/60000 / Loss: 0.0017747184 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 19456/60000 / Loss: 0.0014758402 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 19968/60000 / Loss: 0.001100838 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 20480/60000 / Loss: 0.0022659362 / Training Accuracy: 0.984375 / Validation Accuracy: 0.983\n",
      "Epoch: 7 / Batch: 20992/60000 / Loss: 0.0022719249 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.983\n",
      "Epoch: 7 / Batch: 21504/60000 / Loss: 0.0017592717 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.983\n",
      "Epoch: 7 / Batch: 22016/60000 / Loss: 0.0016361813 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 22528/60000 / Loss: 0.0015901143 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 23040/60000 / Loss: 0.0008672078 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 23552/60000 / Loss: 0.0010545393 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 24064/60000 / Loss: 0.00088457076 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 24576/60000 / Loss: 0.002031113 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 25088/60000 / Loss: 0.0011287135 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 25600/60000 / Loss: 0.00055796636 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.982\n",
      "Epoch: 7 / Batch: 26112/60000 / Loss: 0.00241879 / Training Accuracy: 0.984375 / Validation Accuracy: 0.983\n",
      "Epoch: 7 / Batch: 26624/60000 / Loss: 0.0015416958 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 27136/60000 / Loss: 0.00085328147 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 27648/60000 / Loss: 0.0012171541 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 28160/60000 / Loss: 0.0010324885 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 28672/60000 / Loss: 0.0012153495 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.991\n",
      "Epoch: 7 / Batch: 29184/60000 / Loss: 0.0019692995 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.99\n",
      "Epoch: 7 / Batch: 29696/60000 / Loss: 0.0011077663 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.991\n",
      "Epoch: 7 / Batch: 30208/60000 / Loss: 0.0016049596 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 30720/60000 / Loss: 0.0017074322 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 31232/60000 / Loss: 0.0015921651 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 31744/60000 / Loss: 0.0011341054 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 32256/60000 / Loss: 0.0023816854 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.983\n",
      "Epoch: 7 / Batch: 32768/60000 / Loss: 0.0015524395 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.983\n",
      "Epoch: 7 / Batch: 33280/60000 / Loss: 0.00055965595 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 33792/60000 / Loss: 0.0021628826 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 34304/60000 / Loss: 0.0011543129 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 34816/60000 / Loss: 0.0013266837 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 35328/60000 / Loss: 0.0008925478 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 35840/60000 / Loss: 0.0019747405 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 36352/60000 / Loss: 0.003174875 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.983\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 / Batch: 36864/60000 / Loss: 0.0015717056 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 37376/60000 / Loss: 0.0016040907 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 37888/60000 / Loss: 0.001309769 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 38400/60000 / Loss: 0.0012701394 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 38912/60000 / Loss: 0.0028616306 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 39424/60000 / Loss: 0.0018934238 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 39936/60000 / Loss: 0.002871123 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 40448/60000 / Loss: 0.0013366755 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 40960/60000 / Loss: 0.00063695264 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 41472/60000 / Loss: 0.001349542 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 7 / Batch: 41984/60000 / Loss: 0.0012944413 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 42496/60000 / Loss: 0.0014552837 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 43008/60000 / Loss: 0.001769793 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 43520/60000 / Loss: 0.0024697483 / Training Accuracy: 0.984375 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 44032/60000 / Loss: 0.001435408 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 44544/60000 / Loss: 0.0009910057 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 45056/60000 / Loss: 0.0012729533 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 45568/60000 / Loss: 0.0015818484 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 46080/60000 / Loss: 0.001739631 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.991\n",
      "Epoch: 7 / Batch: 46592/60000 / Loss: 0.0022364922 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.992\n",
      "Epoch: 7 / Batch: 47104/60000 / Loss: 0.0019939325 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.993\n",
      "Epoch: 7 / Batch: 47616/60000 / Loss: 0.0007803428 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 7 / Batch: 48128/60000 / Loss: 0.0011619318 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.99\n",
      "Epoch: 7 / Batch: 48640/60000 / Loss: 0.0013147744 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 49152/60000 / Loss: 0.002093021 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 49664/60000 / Loss: 0.000751263 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.99\n",
      "Epoch: 7 / Batch: 50176/60000 / Loss: 0.0012791698 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 7 / Batch: 50688/60000 / Loss: 0.0011111834 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 51200/60000 / Loss: 0.00205016 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 51712/60000 / Loss: 0.0027740146 / Training Accuracy: 0.984375 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 52224/60000 / Loss: 0.0011420192 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 52736/60000 / Loss: 0.0009984319 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 53248/60000 / Loss: 0.0014147656 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 7 / Batch: 53760/60000 / Loss: 0.0013973094 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 54272/60000 / Loss: 0.0010731099 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 54784/60000 / Loss: 0.0019932378 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 55296/60000 / Loss: 0.0022246689 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.99\n",
      "Epoch: 7 / Batch: 55808/60000 / Loss: 0.0016225639 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 56320/60000 / Loss: 0.0026220884 / Training Accuracy: 0.984375 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 56832/60000 / Loss: 0.00092867826 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 7 / Batch: 57344/60000 / Loss: 0.0016802566 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.991\n",
      "Epoch: 7 / Batch: 57856/60000 / Loss: 0.0010545417 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.992\n",
      "Epoch: 7 / Batch: 58368/60000 / Loss: 0.0012094523 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 7 / Batch: 58880/60000 / Loss: 0.0009292259 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 7 / Batch: 59392/60000 / Loss: 0.0017699979 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 7 / Batch: 59904/60000 / Loss: 1.8363928e-05 / Training Accuracy: 1.0 / Validation Accuracy: 0.984\n",
      "Epoch: 8 / Batch: 0/60000 / Loss: 0.0012329681 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 512/60000 / Loss: 0.001972077 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 1024/60000 / Loss: 0.00092023227 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 1536/60000 / Loss: 0.0023923288 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 2048/60000 / Loss: 0.0016304019 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 2560/60000 / Loss: 0.00057638594 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 3072/60000 / Loss: 0.0010607964 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 3584/60000 / Loss: 0.0011630526 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 4096/60000 / Loss: 0.0013678672 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 4608/60000 / Loss: 0.0019608364 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 5120/60000 / Loss: 0.0010883803 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 5632/60000 / Loss: 0.0017080763 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 6144/60000 / Loss: 0.0015539177 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 6656/60000 / Loss: 0.00077793276 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 7168/60000 / Loss: 0.00040358322 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.985\n",
      "Epoch: 8 / Batch: 7680/60000 / Loss: 0.0014839976 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 8192/60000 / Loss: 0.0019521031 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 8704/60000 / Loss: 0.0011319716 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 9216/60000 / Loss: 0.0010664257 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 9728/60000 / Loss: 0.0018052487 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.991\n",
      "Epoch: 8 / Batch: 10240/60000 / Loss: 0.0014805641 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 10752/60000 / Loss: 0.0008605857 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 11264/60000 / Loss: 0.000995717 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 11776/60000 / Loss: 0.0015830321 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 12288/60000 / Loss: 0.0006295537 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 12800/60000 / Loss: 0.0013876092 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 13312/60000 / Loss: 0.0020938993 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 13824/60000 / Loss: 0.001184359 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 14336/60000 / Loss: 0.0007007447 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 14848/60000 / Loss: 0.00127727 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 15360/60000 / Loss: 0.0016507267 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.991\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 / Batch: 15872/60000 / Loss: 0.0016172372 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 16384/60000 / Loss: 0.0012826246 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 16896/60000 / Loss: 0.0017640777 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 17408/60000 / Loss: 0.0012041299 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 8 / Batch: 17920/60000 / Loss: 0.0009429526 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 18432/60000 / Loss: 0.00080554746 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 18944/60000 / Loss: 0.0007360315 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 19456/60000 / Loss: 0.00054091687 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 19968/60000 / Loss: 0.0012031416 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 20480/60000 / Loss: 0.0011492772 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.991\n",
      "Epoch: 8 / Batch: 20992/60000 / Loss: 0.0007561236 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 8 / Batch: 21504/60000 / Loss: 0.0013393458 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 22016/60000 / Loss: 0.0015530586 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 22528/60000 / Loss: 0.0015970438 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 23040/60000 / Loss: 0.000942312 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 23552/60000 / Loss: 0.00078268535 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 24064/60000 / Loss: 0.0011607322 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 24576/60000 / Loss: 0.0010322396 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 25088/60000 / Loss: 0.0017520853 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 25600/60000 / Loss: 0.0010768907 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 26112/60000 / Loss: 0.0021112137 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 26624/60000 / Loss: 0.0013531317 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 27136/60000 / Loss: 0.0012703631 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 27648/60000 / Loss: 0.0016924699 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 8 / Batch: 28160/60000 / Loss: 0.0017119019 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.985\n",
      "Epoch: 8 / Batch: 28672/60000 / Loss: 0.0020622744 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 29184/60000 / Loss: 0.0013252082 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 29696/60000 / Loss: 0.0016820595 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 30208/60000 / Loss: 0.0014205336 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 30720/60000 / Loss: 0.0008588692 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 31232/60000 / Loss: 0.0025089667 / Training Accuracy: 0.984375 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 31744/60000 / Loss: 0.0007464472 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 32256/60000 / Loss: 0.00117755 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 32768/60000 / Loss: 0.0017013258 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 33280/60000 / Loss: 0.0018192406 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 33792/60000 / Loss: 0.0011312229 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 34304/60000 / Loss: 0.0008692721 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 34816/60000 / Loss: 0.0009501016 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 35328/60000 / Loss: 0.0024003987 / Training Accuracy: 0.984375 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 35840/60000 / Loss: 0.0005191561 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.992\n",
      "Epoch: 8 / Batch: 36352/60000 / Loss: 0.0016413305 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.992\n",
      "Epoch: 8 / Batch: 36864/60000 / Loss: 0.0015309654 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 37376/60000 / Loss: 0.0018029714 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 37888/60000 / Loss: 0.0005493027 / Training Accuracy: 1.0 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 38400/60000 / Loss: 0.00096705696 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 38912/60000 / Loss: 0.0013352262 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 39424/60000 / Loss: 0.0015720103 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 39936/60000 / Loss: 0.0009983878 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 40448/60000 / Loss: 0.0017473089 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 40960/60000 / Loss: 0.00044026476 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 41472/60000 / Loss: 0.0011441362 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 8 / Batch: 41984/60000 / Loss: 0.0011448269 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.985\n",
      "Epoch: 8 / Batch: 42496/60000 / Loss: 0.0012047819 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 43008/60000 / Loss: 0.0020404775 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 43520/60000 / Loss: 0.000254777 / Training Accuracy: 1.0 / Validation Accuracy: 0.985\n",
      "Epoch: 8 / Batch: 44032/60000 / Loss: 0.0013479932 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 44544/60000 / Loss: 0.0011626368 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 8 / Batch: 45056/60000 / Loss: 0.0011374345 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.984\n",
      "Epoch: 8 / Batch: 45568/60000 / Loss: 0.00086569955 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 46080/60000 / Loss: 0.000984623 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 46592/60000 / Loss: 0.00054007594 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.991\n",
      "Epoch: 8 / Batch: 47104/60000 / Loss: 0.0015901907 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.99\n",
      "Epoch: 8 / Batch: 47616/60000 / Loss: 0.0023596284 / Training Accuracy: 0.984375 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 48128/60000 / Loss: 0.0018786596 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 48640/60000 / Loss: 0.0012686584 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 49152/60000 / Loss: 0.0015287104 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 49664/60000 / Loss: 0.0021718033 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 50176/60000 / Loss: 0.0009185731 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 50688/60000 / Loss: 0.00047485606 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 51200/60000 / Loss: 0.003188795 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 51712/60000 / Loss: 0.0014282146 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 52224/60000 / Loss: 0.0010168994 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 52736/60000 / Loss: 0.0007904916 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.985\n",
      "Epoch: 8 / Batch: 53248/60000 / Loss: 0.0008853617 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 53760/60000 / Loss: 0.0016038923 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 54272/60000 / Loss: 0.0019625947 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 54784/60000 / Loss: 0.00064127543 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 55296/60000 / Loss: 0.0010764597 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 55808/60000 / Loss: 0.0014369427 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 56320/60000 / Loss: 0.0010468392 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 / Batch: 56832/60000 / Loss: 0.0014319866 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 57344/60000 / Loss: 0.0011470186 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 8 / Batch: 57856/60000 / Loss: 0.0004752925 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 8 / Batch: 58368/60000 / Loss: 0.0015731469 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 8 / Batch: 58880/60000 / Loss: 0.0010326732 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.986\n",
      "Epoch: 8 / Batch: 59392/60000 / Loss: 0.0015511743 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 8 / Batch: 59904/60000 / Loss: 0.0002199812 / Training Accuracy: 1.0 / Validation Accuracy: 0.985\n",
      "Epoch: 9 / Batch: 0/60000 / Loss: 0.0013297771 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.983\n",
      "Epoch: 9 / Batch: 512/60000 / Loss: 0.0011725437 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 9 / Batch: 1024/60000 / Loss: 0.001893843 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 1536/60000 / Loss: 0.0007036553 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 2048/60000 / Loss: 0.0012563276 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 2560/60000 / Loss: 0.00092156825 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 3072/60000 / Loss: 0.0016263311 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 3584/60000 / Loss: 0.00091122824 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 9 / Batch: 4096/60000 / Loss: 0.0019212797 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 4608/60000 / Loss: 0.00055970985 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 5120/60000 / Loss: 0.0016092224 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 5632/60000 / Loss: 0.00091824 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 6144/60000 / Loss: 0.0011019426 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 6656/60000 / Loss: 0.00054046523 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.985\n",
      "Epoch: 9 / Batch: 7168/60000 / Loss: 0.0012138324 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 7680/60000 / Loss: 0.00094700156 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 9 / Batch: 8192/60000 / Loss: 0.0013672726 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 8704/60000 / Loss: 0.001569751 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 9216/60000 / Loss: 0.0019852065 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 9728/60000 / Loss: 0.0016166199 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 10240/60000 / Loss: 0.0005408687 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 10752/60000 / Loss: 0.0008212408 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 11264/60000 / Loss: 0.0010254956 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 11776/60000 / Loss: 0.00085768814 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 12288/60000 / Loss: 0.0010844573 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 12800/60000 / Loss: 0.0011234463 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 13312/60000 / Loss: 0.000719258 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 13824/60000 / Loss: 0.0006968662 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 14336/60000 / Loss: 0.0011784589 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 14848/60000 / Loss: 0.0014137977 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 15360/60000 / Loss: 0.001123925 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 15872/60000 / Loss: 0.0014291834 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 16384/60000 / Loss: 0.00043888934 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 16896/60000 / Loss: 0.0010640172 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 17408/60000 / Loss: 0.0013523634 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 17920/60000 / Loss: 0.00093328656 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 18432/60000 / Loss: 0.0004385672 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 18944/60000 / Loss: 0.0010497535 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 19456/60000 / Loss: 0.00067720667 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 19968/60000 / Loss: 0.0017102171 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 20480/60000 / Loss: 0.000721216 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 20992/60000 / Loss: 0.00074569415 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 21504/60000 / Loss: 0.00075375877 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 22016/60000 / Loss: 0.001988842 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 9 / Batch: 22528/60000 / Loss: 0.0013807536 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 9 / Batch: 23040/60000 / Loss: 0.0012997532 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 23552/60000 / Loss: 0.0022030477 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 24064/60000 / Loss: 0.0010324683 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.985\n",
      "Epoch: 9 / Batch: 24576/60000 / Loss: 0.0009376549 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.985\n",
      "Epoch: 9 / Batch: 25088/60000 / Loss: 0.00080992375 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.984\n",
      "Epoch: 9 / Batch: 25600/60000 / Loss: 0.0013354901 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.984\n",
      "Epoch: 9 / Batch: 26112/60000 / Loss: 0.0016996015 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 26624/60000 / Loss: 0.0014088991 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 27136/60000 / Loss: 0.0011869414 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.992\n",
      "Epoch: 9 / Batch: 27648/60000 / Loss: 0.0016609237 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.992\n",
      "Epoch: 9 / Batch: 28160/60000 / Loss: 0.0019737135 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.991\n",
      "Epoch: 9 / Batch: 28672/60000 / Loss: 0.0010025073 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 29184/60000 / Loss: 0.0015618785 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 29696/60000 / Loss: 0.00047824852 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 30208/60000 / Loss: 0.0007548878 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 30720/60000 / Loss: 0.0010960053 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 31232/60000 / Loss: 0.0015398216 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 31744/60000 / Loss: 0.00047179777 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 32256/60000 / Loss: 0.0004339929 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.991\n",
      "Epoch: 9 / Batch: 32768/60000 / Loss: 0.0030988827 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 33280/60000 / Loss: 0.00065003603 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 9 / Batch: 33792/60000 / Loss: 0.0017548507 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 34304/60000 / Loss: 0.0006493373 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 34816/60000 / Loss: 0.0011603796 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 35328/60000 / Loss: 0.00083456235 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 / Batch: 35840/60000 / Loss: 0.00064954924 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 36352/60000 / Loss: 0.0022070569 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 36864/60000 / Loss: 0.0006017593 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 37376/60000 / Loss: 0.0012660192 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 37888/60000 / Loss: 0.0014851411 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 38400/60000 / Loss: 0.0015999526 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 38912/60000 / Loss: 0.0013980276 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 39424/60000 / Loss: 0.0019919449 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 39936/60000 / Loss: 0.0011840216 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 40448/60000 / Loss: 0.0019000309 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 40960/60000 / Loss: 0.001018401 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 41472/60000 / Loss: 0.0007004546 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 41984/60000 / Loss: 0.0012707231 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 42496/60000 / Loss: 0.001808919 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 43008/60000 / Loss: 0.0010819098 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 43520/60000 / Loss: 0.0015458523 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 44032/60000 / Loss: 0.0014080314 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 44544/60000 / Loss: 0.0022065868 / Training Accuracy: 0.984375 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 45056/60000 / Loss: 0.00065847376 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 45568/60000 / Loss: 0.0007938255 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 46080/60000 / Loss: 0.0012773147 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 46592/60000 / Loss: 0.0005979849 / Training Accuracy: 1.0 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 47104/60000 / Loss: 0.0011088507 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 47616/60000 / Loss: 0.0013487292 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 48128/60000 / Loss: 0.00082417147 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 48640/60000 / Loss: 0.00078479695 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 9 / Batch: 49152/60000 / Loss: 0.0012146676 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 49664/60000 / Loss: 0.0008224209 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 50176/60000 / Loss: 0.0014436045 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 50688/60000 / Loss: 0.0007267017 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 51200/60000 / Loss: 0.0010338665 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 51712/60000 / Loss: 0.0008541091 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 52224/60000 / Loss: 0.0021341683 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 52736/60000 / Loss: 0.00032540032 / Training Accuracy: 1.0 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 53248/60000 / Loss: 0.0005579479 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 53760/60000 / Loss: 0.0020789027 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 54272/60000 / Loss: 0.0011991776 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 54784/60000 / Loss: 0.0008740883 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 55296/60000 / Loss: 0.0013343409 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 55808/60000 / Loss: 0.0013629718 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 56320/60000 / Loss: 0.0011553979 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 56832/60000 / Loss: 0.0006126262 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 9 / Batch: 57344/60000 / Loss: 0.00075346173 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.991\n",
      "Epoch: 9 / Batch: 57856/60000 / Loss: 0.0020425105 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.99\n",
      "Epoch: 9 / Batch: 58368/60000 / Loss: 0.00069915 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 9 / Batch: 58880/60000 / Loss: 0.00059540634 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.988\n",
      "Epoch: 9 / Batch: 59392/60000 / Loss: 0.001774533 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.985\n",
      "Epoch: 9 / Batch: 59904/60000 / Loss: 0.00023534911 / Training Accuracy: 1.0 / Validation Accuracy: 0.983\n",
      "Epoch: 10 / Batch: 0/60000 / Loss: 0.0008762431 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.984\n",
      "Epoch: 10 / Batch: 512/60000 / Loss: 0.0017635773 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.984\n",
      "Epoch: 10 / Batch: 1024/60000 / Loss: 0.0015895611 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 10 / Batch: 1536/60000 / Loss: 0.00059378904 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.986\n",
      "Epoch: 10 / Batch: 2048/60000 / Loss: 0.0012366842 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.985\n",
      "Epoch: 10 / Batch: 2560/60000 / Loss: 0.0013446201 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.986\n",
      "Epoch: 10 / Batch: 3072/60000 / Loss: 0.002472099 / Training Accuracy: 0.984375 / Validation Accuracy: 0.985\n",
      "Epoch: 10 / Batch: 3584/60000 / Loss: 0.0016106612 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.984\n",
      "Epoch: 10 / Batch: 4096/60000 / Loss: 0.0006026623 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 4608/60000 / Loss: 0.0005527828 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 5120/60000 / Loss: 0.0014249543 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 5632/60000 / Loss: 0.00060988206 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 6144/60000 / Loss: 0.0012152919 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 6656/60000 / Loss: 0.0011976252 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 7168/60000 / Loss: 0.001127329 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 7680/60000 / Loss: 0.0012334824 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 8192/60000 / Loss: 0.00043483818 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.986\n",
      "Epoch: 10 / Batch: 8704/60000 / Loss: 0.0010955092 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 9216/60000 / Loss: 0.001242421 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.986\n",
      "Epoch: 10 / Batch: 9728/60000 / Loss: 0.002133469 / Training Accuracy: 0.984375 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 10240/60000 / Loss: 0.0007416901 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 10752/60000 / Loss: 0.00034869282 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 11264/60000 / Loss: 0.00068054145 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 11776/60000 / Loss: 0.00042331632 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 12288/60000 / Loss: 0.00030815636 / Training Accuracy: 1.0 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 12800/60000 / Loss: 0.00038860945 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.99\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 / Batch: 13312/60000 / Loss: 0.0014286505 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 13824/60000 / Loss: 0.0010126103 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 14336/60000 / Loss: 0.00063861656 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 14848/60000 / Loss: 0.00067817344 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 15360/60000 / Loss: 0.00064508215 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 15872/60000 / Loss: 0.000867429 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 16384/60000 / Loss: 0.00087672536 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 16896/60000 / Loss: 0.000605299 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 17408/60000 / Loss: 0.0010777999 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 10 / Batch: 17920/60000 / Loss: 0.0015015427 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.985\n",
      "Epoch: 10 / Batch: 18432/60000 / Loss: 0.0007709923 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.985\n",
      "Epoch: 10 / Batch: 18944/60000 / Loss: 0.001504211 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.986\n",
      "Epoch: 10 / Batch: 19456/60000 / Loss: 0.0023364357 / Training Accuracy: 0.984375 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 19968/60000 / Loss: 0.0006839164 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.993\n",
      "Epoch: 10 / Batch: 20480/60000 / Loss: 0.0017128491 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 20992/60000 / Loss: 0.0009369748 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 21504/60000 / Loss: 0.001437376 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 22016/60000 / Loss: 0.00081983546 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 22528/60000 / Loss: 0.000765531 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.994\n",
      "Epoch: 10 / Batch: 23040/60000 / Loss: 0.00086562184 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 23552/60000 / Loss: 0.0007073774 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 24064/60000 / Loss: 0.0015284875 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 24576/60000 / Loss: 0.0012618356 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 25088/60000 / Loss: 0.00059306587 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 25600/60000 / Loss: 0.0008864626 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 26112/60000 / Loss: 0.0013455006 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 26624/60000 / Loss: 0.0015005482 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 27136/60000 / Loss: 0.0016559433 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 27648/60000 / Loss: 0.00018810545 / Training Accuracy: 1.0 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 28160/60000 / Loss: 0.00094376627 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 28672/60000 / Loss: 0.00062439276 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 29184/60000 / Loss: 0.0013883461 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 29696/60000 / Loss: 0.002373244 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 30208/60000 / Loss: 0.0009385489 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 30720/60000 / Loss: 0.0009672657 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 31232/60000 / Loss: 0.0010576338 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 31744/60000 / Loss: 0.0011026683 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 32256/60000 / Loss: 0.0011916393 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 32768/60000 / Loss: 0.0009301918 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 33280/60000 / Loss: 0.0010447117 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 33792/60000 / Loss: 0.00046095927 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 34304/60000 / Loss: 0.00096054154 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 34816/60000 / Loss: 0.0006886625 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 35328/60000 / Loss: 0.00067505107 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 35840/60000 / Loss: 0.0004065223 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 36352/60000 / Loss: 0.0012487172 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 36864/60000 / Loss: 0.001100006 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 37376/60000 / Loss: 0.001178197 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 37888/60000 / Loss: 0.0009948993 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 38400/60000 / Loss: 0.00029613444 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 38912/60000 / Loss: 0.0016789058 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.993\n",
      "Epoch: 10 / Batch: 39424/60000 / Loss: 0.0005626271 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 39936/60000 / Loss: 0.0013984183 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 40448/60000 / Loss: 0.00055206846 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 40960/60000 / Loss: 0.00063505454 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 41472/60000 / Loss: 0.0010662144 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 41984/60000 / Loss: 0.0010719313 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 42496/60000 / Loss: 0.0011353268 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 43008/60000 / Loss: 0.0012455478 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 43520/60000 / Loss: 0.00074894016 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 44032/60000 / Loss: 0.0006978314 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 44544/60000 / Loss: 0.0004539701 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 45056/60000 / Loss: 0.0007566943 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 45568/60000 / Loss: 0.00073949515 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 46080/60000 / Loss: 0.00094500586 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 46592/60000 / Loss: 0.0017370345 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 47104/60000 / Loss: 0.0008242422 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 47616/60000 / Loss: 0.00016581948 / Training Accuracy: 1.0 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 48128/60000 / Loss: 0.0014617529 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 48640/60000 / Loss: 0.00091534463 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 49152/60000 / Loss: 0.0009777197 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 49664/60000 / Loss: 0.0018980608 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 50176/60000 / Loss: 0.0006229718 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 50688/60000 / Loss: 0.0011168163 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 51200/60000 / Loss: 0.0014221531 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 51712/60000 / Loss: 0.0010487477 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 52224/60000 / Loss: 0.0017903078 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 52736/60000 / Loss: 0.0003731627 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 53248/60000 / Loss: 0.0003266118 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 53760/60000 / Loss: 0.0013490142 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.992\n",
      "Epoch: 10 / Batch: 54272/60000 / Loss: 0.00089563633 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.991\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 / Batch: 54784/60000 / Loss: 0.0017231617 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.991\n",
      "Epoch: 10 / Batch: 55296/60000 / Loss: 0.0009373136 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 55808/60000 / Loss: 0.0012185862 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.99\n",
      "Epoch: 10 / Batch: 56320/60000 / Loss: 0.0014448576 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 56832/60000 / Loss: 0.00073188357 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 57344/60000 / Loss: 0.0004597144 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 57856/60000 / Loss: 0.00071444974 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.989\n",
      "Epoch: 10 / Batch: 58368/60000 / Loss: 0.0010265325 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 58880/60000 / Loss: 0.00044424474 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 59392/60000 / Loss: 0.0023486223 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 59904/60000 / Loss: 0.001131507 / Training Accuracy: 0.9895833 / Validation Accuracy: 0.988\n"
     ]
    }
   ],
   "source": [
    "data = np.insert(train_x, img_flat_size, train_y, axis = 1)\n",
    "len_data = data.shape[0]\n",
    "\n",
    "# Training\n",
    "for i in range(num_epoch):\n",
    "    # Shuffle the data \n",
    "    np.random.shuffle(data)\n",
    "\n",
    "    # Split data to input can label\n",
    "    data_x = data[:, :img_flat_size]\n",
    "    data_y = data[:, img_flat_size]\n",
    "\n",
    "    # Make label data as one-hot\n",
    "    data_y_onehot = np.zeros([data_y.shape[0], num_label])\n",
    "    for j in range(data_y.shape[0]):\n",
    "        data_y_onehot[j, int(data_y[j])] = 1\n",
    "    \n",
    "    data_y_onehot_val = np.zeros([validation_y.shape[0], num_label])\n",
    "    for j in range(validation_y.shape[0]):\n",
    "        data_y_onehot_val[j, int(validation_y[j])] = 1\n",
    "        \n",
    "    # Making mini-batch\n",
    "    for j in range(0, len_data, batch_size):\n",
    "        if j + batch_size < len_data:\n",
    "            data_x_in = data_x[j : j + batch_size, :]\n",
    "            data_y_in = data_y_onehot[j : j + batch_size, :]\n",
    "        else:\n",
    "            data_x_in = data_x[j : len_data, :]\n",
    "            data_y_in = data_y_onehot[j : len_data, :]\n",
    "\n",
    "        # Reshape input data as image\n",
    "        data_x_in_resize = np.reshape(data_x_in, (data_x_in.shape[0], img_size, img_size, 1))\n",
    "\n",
    "        # Run Optimizer!\n",
    "        optimizer.run(feed_dict = {x_image: data_x_in_resize, y_target: data_y_in, keep_prob: 0.5})\n",
    "        loss, acc = sess.run([Loss, accuracy], feed_dict = {x_image: data_x_in_resize, y_target: data_y_in, keep_prob: 1})\n",
    "        val_acc = sess.run(accuracy, feed_dict = {x_image: validation_x_resize, y_target: data_y_onehot_val, keep_prob: 1})\n",
    "        \n",
    "        print(\"Epoch: \" + str(i+1) + ' / ' + \"Batch: \" + str(j) + '/' + str(len_data) + ' / ' + \n",
    "              \"Loss: \" + str(loss) + ' / ' + \"Training Accuracy: \" + str(acc) + ' / ' + \n",
    "              \"Validation Accuracy: \" + str(val_acc))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.9912222222222222\n"
     ]
    }
   ],
   "source": [
    "# Testing\n",
    "\n",
    "test_y_onehot = np.zeros([test_y.shape[0], num_label])\n",
    "for i in range(test_y.shape[0]):\n",
    "    test_y_onehot[i, int(test_y[i])] = 1\n",
    "\n",
    "# Resize testset as image \n",
    "test_x_resize = np.reshape(test_x, (test_x.shape[0], img_size, img_size, 1))\n",
    "\n",
    "test_result = output.eval(feed_dict = {x_image: test_x_resize, keep_prob: 1})\n",
    "\n",
    "count_correct = 0\n",
    "for i in range(test_result.shape[0]):\n",
    "    prediction_y = np.argmax(test_result[i,:])\n",
    "    \n",
    "    if prediction_y == test_y[i]:\n",
    "        count_correct += 1\n",
    "\n",
    "test_acc = count_correct / test_result.shape[0]\n",
    "\n",
    "print(\"Test Accuracy: \" + str(test_acc))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}