0_ANN_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Artificial Neural Network Code with MNIST\n",
    "\n",
    "This is jupyter notebook for basic ANN and dense layer using `Tensorflow`. \n",
    "<br> Dataset is MNIST dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# Import modules\n",
    "\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-086d30c7fab6>:4: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Users\\Q\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import MNIST data\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', validation_size=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters \n",
    "img_size = 28\n",
    "data_size = 28 * 28\n",
    "\n",
    "# labels: 0 - 9\n",
    "num_label = 10\n",
    "\n",
    "# Parameters for optimizer\n",
    "num_epoch = 10\n",
    "\n",
    "learning_rate = 5e-4\n",
    "batch_size = 512\n",
    "\n",
    "validation_ratio = 0.1\n",
    "gpu_fraction = 0.2\n",
    "\n",
    "# Fully Connected [kernel size, kernel size, # input, # output]\n",
    "first_fc  = [data_size, 256]\n",
    "second_fc = [256, 1024]\n",
    "third_fc  = [1024, num_label]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADgpJREFUeJzt3X+MVfWZx/HPs1j+kKI4aQRCYSnEYJW4082IjSWrxkzVDQZHrekkJjQapn8wiU02ZA3/VNNgyCrslmiamaZYSFpKE3VB0iw0otLGZuKIWC0srTFsO3IDNTjywx9kmGf/mEMzxbnfe+fec++5zPN+JeT+eM6558kNnznn3O+592vuLgDx/EPRDQAoBuEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxDUZc3cmJlxOSHQYO5u1SxX157fzO40syNm9q6ZPVrPawFoLqv12n4zmybpj5I6JQ1Jel1St7sfSqzDnh9osGbs+ZdJetfd33P3c5J+IWllHa8HoInqCf88SX8Z93goe+7vmFmPmQ2a2WAd2wKQs3o+8Jvo0OJzh/Xu3i+pX+KwH2gl9ez5hyTNH/f4y5KO1dcOgGapJ/yvS7rGzL5iZtMlfVvSrnzaAtBoNR/2u/uImfVK2iNpmqQt7v6H3DoD0FA1D/XVtDHO+YGGa8pFPgAuXYQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EVfMU3ZJkZkclnZZ0XtKIu3fk0RTyM23atGT9yiuvbOj2e3t7y9Yuv/zy5LpLlixJ1tesWZOsP/XUU2Vr3d3dyXU//fTTZH3Dhg3J+uOPP56st4K6wp+5zd0/yOF1ADQRh/1AUPWG3yXtNbM3zKwnj4YANEe9h/3fcPdjZna1pF+b2f+6+/7xC2R/FPjDALSYuvb87n4suz0h6QVJyyZYpt/dO/gwEGgtNYffzGaY2cwL9yV9U9I7eTUGoLHqOeyfLekFM7vwOj939//JpSsADVdz+N39PUn/lGMvU9aCBQuS9enTpyfrN998c7K+fPnysrVZs2Yl173vvvuS9SINDQ0l65s3b07Wu7q6ytZOnz6dXPett95K1l999dVk/VLAUB8QFOEHgiL8QFCEHwiK8ANBEX4gKHP35m3MrHkba6L29vZkfd++fcl6o79W26pGR0eT9YceeihZP3PmTM3bLpVKyfqHH36YrB85cqTmbTeau1s1y7HnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgGOfPQVtbW7I+MDCQrC9atCjPdnJVqffh4eFk/bbbbitbO3fuXHLdqNc/1ItxfgBJhB8IivADQRF+ICjCDwRF+IGgCD8QVB6z9IZ38uTJZH3t2rXJ+ooVK5L1N998M1mv9BPWKQcPHkzWOzs7k/WzZ88m69dff33Z2iOPPJJcF43Fnh8IivADQRF+ICjCDwRF+IGgCD8QFOEHgqr4fX4z2yJphaQT7r40e65N0g5JCyUdlfSAu6d/6FxT9/v89briiiuS9UrTSff19ZWtPfzww8l1H3zwwWR9+/btyTpaT57f5/+ppDsveu5RSS+5+zWSXsoeA7iEVAy/u++XdPElbCslbc3ub5V0T859AWiwWs/5Z7t7SZKy26vzawlAMzT82n4z65HU0+jtAJicWvf8x81sriRltyfKLeju/e7e4e4dNW4LQAPUGv5dklZl91dJ2plPOwCapWL4zWy7pN9JWmJmQ2b2sKQNkjrN7E+SOrPHAC4hFc/53b27TOn2nHsJ69SpU3Wt/9FHH9W87urVq5P1HTt2JOujo6M1bxvF4go/ICjCDwRF+IGgCD8QFOEHgiL8QFBM0T0FzJgxo2ztxRdfTK57yy23JOt33XVXsr53795kHc3HFN0Akgg/EBThB4Ii/EBQhB8IivADQRF+ICjG+ae4xYsXJ+sHDhxI1oeHh5P1l19+OVkfHBwsW3vmmWeS6zbz/+ZUwjg/gCTCDwRF+IGgCD8QFOEHgiL8QFCEHwiKcf7gurq6kvVnn302WZ85c2bN2163bl2yvm3btmS9VCrVvO2pjHF+AEmEHwiK8ANBEX4gKMIPBEX4gaAIPxBUxXF+M9siaYWkE+6+NHvuMUmrJf01W2ydu/+q4sYY57/kLF26NFnftGlTsn777bXP5N7X15esr1+/Pll///33a972pSzPcf6fSrpzguf/093bs38Vgw+gtVQMv7vvl3SyCb0AaKJ6zvl7zez3ZrbFzK7KrSMATVFr+H8kabGkdkklSRvLLWhmPWY2aGblf8wNQNPVFH53P+7u5919VNKPJS1LLNvv7h3u3lFrkwDyV1P4zWzuuIddkt7Jpx0AzXJZpQXMbLukWyV9ycyGJH1f0q1m1i7JJR2V9N0G9gigAfg+P+oya9asZP3uu+8uW6v0WwFm6eHqffv2JeudnZ3J+lTF9/kBJBF+ICjCDwRF+IGgCD8QFOEHgmKoD4X57LPPkvXLLktfhjIyMpKs33HHHWVrr7zySnLdSxlDfQCSCD8QFOEHgiL8QFCEHwiK8ANBEX4gqIrf50dsN9xwQ7J+//33J+s33nhj2VqlcfxKDh06lKzv37+/rtef6tjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQjPNPcUuWLEnWe3t7k/V77703WZ8zZ86ke6rW+fPnk/VSqZSsj46O5tnOlMOeHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCqjjOb2bzJW2TNEfSqKR+d/+hmbVJ2iFpoaSjkh5w9w8b12pclcbSu7u7y9YqjeMvXLiwlpZyMTg4mKyvX78+Wd+1a1ee7YRTzZ5/RNK/uftXJX1d0hozu07So5JecvdrJL2UPQZwiagYfncvufuB7P5pSYclzZO0UtLWbLGtku5pVJMA8jepc34zWyjpa5IGJM1295I09gdC0tV5Nwegcaq+tt/MvijpOUnfc/dTZlVNByYz65HUU1t7ABqlqj2/mX1BY8H/mbs/nz193MzmZvW5kk5MtK6797t7h7t35NEwgHxUDL+N7eJ/Iumwu28aV9olaVV2f5Wknfm3B6BRKk7RbWbLJf1G0tsaG+qTpHUaO+//paQFkv4s6VvufrLCa4Wconv27NnJ+nXXXZesP/3008n6tddeO+me8jIwMJCsP/nkk2VrO3em9xd8Jbc21U7RXfGc391/K6nci90+maYAtA6u8AOCIvxAUIQfCIrwA0ERfiAowg8ExU93V6mtra1sra+vL7lue3t7sr5o0aKaesrDa6+9lqxv3LgxWd+zZ0+y/sknn0y6JzQHe34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCCrMOP9NN92UrK9duzZZX7ZsWdnavHnzauopLx9//HHZ2ubNm5PrPvHEE8n62bNna+oJrY89PxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EFWacv6urq656PQ4dOpSs7969O1kfGRlJ1lPfuR8eHk6ui7jY8wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUObu6QXM5kvaJmmOpFFJ/e7+QzN7TNJqSX/NFl3n7r+q8FrpjQGom7tbNctVE/65kua6+wEzmynpDUn3SHpA0hl3f6rapgg/0HjVhr/iFX7uXpJUyu6fNrPDkor96RoAdZvUOb+ZLZT0NUkD2VO9ZvZ7M9tiZleVWafHzAbNbLCuTgHkquJh/98WNPuipFclrXf3581stqQPJLmkH2js1OChCq/BYT/QYLmd80uSmX1B0m5Je9x90wT1hZJ2u/vSCq9D+IEGqzb8FQ/7zcwk/UTS4fHBzz4IvKBL0juTbRJAcar5tH+5pN9IeltjQ32StE5St6R2jR32H5X03ezDwdRrsecHGizXw/68EH6g8XI77AcwNRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCavYU3R9I+r9xj7+UPdeKWrW3Vu1Lorda5dnbP1a7YFO/z/+5jZsNuntHYQ0ktGpvrdqXRG+1Kqo3DvuBoAg/EFTR4e8vePsprdpbq/Yl0VutCumt0HN+AMUpes8PoCCFhN/M7jSzI2b2rpk9WkQP5ZjZUTN728wOFj3FWDYN2gkze2fcc21m9msz+1N2O+E0aQX19piZvZ+9dwfN7F8L6m2+mb1sZofN7A9m9kj2fKHvXaKvQt63ph/2m9k0SX+U1ClpSNLrkrrd/VBTGynDzI5K6nD3wseEzexfJJ2RtO3CbEhm9h+STrr7huwP51Xu/u8t0ttjmuTMzQ3qrdzM0t9Rge9dnjNe56GIPf8ySe+6+3vufk7SLyStLKCPlufu+yWdvOjplZK2Zve3auw/T9OV6a0luHvJ3Q9k909LujCzdKHvXaKvQhQR/nmS/jLu8ZBaa8pvl7TXzN4ws56im5nA7AszI2W3Vxfcz8UqztzcTBfNLN0y710tM17nrYjwTzSbSCsNOXzD3f9Z0l2S1mSHt6jOjyQt1tg0biVJG4tsJptZ+jlJ33P3U0X2Mt4EfRXyvhUR/iFJ88c9/rKkYwX0MSF3P5bdnpD0gsZOU1rJ8QuTpGa3Jwru52/c/bi7n3f3UUk/VoHvXTaz9HOSfubuz2dPF/7eTdRXUe9bEeF/XdI1ZvYVM5su6duSdhXQx+eY2YzsgxiZ2QxJ31TrzT68S9Kq7P4qSTsL7OXvtMrMzeVmllbB712rzXhdyEU+2VDGf0maJmmLu69vehMTMLNFGtvbS2PfePx5kb2Z2XZJt2rsW1/HJX1f0n9L+qWkBZL+LOlb7t70D97K9HarJjlzc4N6Kzez9IAKfO/ynPE6l364wg+IiSv8gKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8E9f/Ex0YKZYOZcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting example image\n",
    "img = mnist.train.images[0]\n",
    "img_resize = img.reshape((img_size, img_size))\n",
    "plt.imshow(img_resize, cmap = 'gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize weights and bias \n",
    "def weight_variable(name, shape):\n",
    "    return tf.get_variable(name, shape=shape, initializer=tf.contrib.layers.xavier_initializer())\n",
    "\n",
    "def bias_variable(name, shape):\n",
    "    return tf.get_variable(name, shape=shape, initializer=tf.contrib.layers.xavier_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reset graph\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# Input \n",
    "x_input  = tf.placeholder(tf.float32, shape = [None, data_size])\n",
    "y_target = tf.placeholder(tf.float32, shape=[None, num_label])\n",
    "\n",
    "# Keep Prob\n",
    "keep_prob = tf.placeholder(\"float\")\n",
    "\n",
    "# Densely connect layer variables \n",
    "w_fc1 = weight_variable('W_fc1', first_fc)\n",
    "b_fc1 = bias_variable('b_fc1', first_fc[1])\n",
    "\n",
    "w_fc2 = weight_variable('W_fc2', second_fc)\n",
    "b_fc2 = bias_variable('b_fc2', second_fc[1])\n",
    "\n",
    "w_fc3 = weight_variable('W_fc3', third_fc)\n",
    "b_fc3 = bias_variable('b_fc3', third_fc[1])\n",
    "\n",
    "# Network\n",
    "h_fc1 = tf.nn.relu(tf.matmul(x_input, w_fc1)+b_fc1)\n",
    "h_fc2 = tf.nn.relu(tf.matmul(h_fc1, w_fc2)+b_fc2)\n",
    "\n",
    "# Dropout\n",
    "h_fc1_drop = tf.nn.dropout(h_fc2, keep_prob)\n",
    "\n",
    "# Output\n",
    "output = tf.nn.softmax(tf.matmul(h_fc1_drop, w_fc3)+b_fc3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Loss and Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Training \n",
    "Loss = tf.reduce_mean(tf.square(output - y_target))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(Loss)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y_target,1), tf.argmax(output,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set: (60000, 784)\n",
      "Testing set: (9000, 784)\n",
      "Validation set: (1000, 784)\n"
     ]
    }
   ],
   "source": [
    "# Dataset for train, test, validation\n",
    "\n",
    "test_len = mnist.test.images.shape[0]\n",
    "validation_len = int(test_len * validation_ratio)\n",
    "\n",
    "train_x = mnist.train.images\n",
    "test_x = mnist.test.images[validation_len : test_len, :]\n",
    "validation_x = mnist.test.images[ : validation_len, :]\n",
    "\n",
    "train_y = mnist.train.labels\n",
    "test_y = mnist.test.labels[validation_len : test_len]\n",
    "validation_y = mnist.test.labels[ : validation_len]\n",
    "\n",
    "print(\"Training set: \" + str(train_x.shape))\n",
    "print(\"Testing set: \" + str(test_x.shape))\n",
    "print(\"Validation set: \" + str(validation_x.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = gpu_fraction\n",
    "\n",
    "sess = tf.InteractiveSession(config=config)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 / Batch: 0/60000 / Loss: 0.088776484 / Training Accuracy: 0.21679688 / Validation Accuracy: 0.186\n",
      "Epoch: 1 / Batch: 512/60000 / Loss: 0.08743241 / Training Accuracy: 0.23632812 / Validation Accuracy: 0.241\n",
      "Epoch: 1 / Batch: 1024/60000 / Loss: 0.08548719 / Training Accuracy: 0.296875 / Validation Accuracy: 0.33\n",
      "Epoch: 1 / Batch: 1536/60000 / Loss: 0.0824879 / Training Accuracy: 0.4453125 / Validation Accuracy: 0.421\n",
      "Epoch: 1 / Batch: 2048/60000 / Loss: 0.07955234 / Training Accuracy: 0.5332031 / Validation Accuracy: 0.517\n",
      "Epoch: 1 / Batch: 2560/60000 / Loss: 0.07745081 / Training Accuracy: 0.56640625 / Validation Accuracy: 0.592\n",
      "Epoch: 1 / Batch: 3072/60000 / Loss: 0.07544744 / Training Accuracy: 0.6074219 / Validation Accuracy: 0.64\n",
      "Epoch: 1 / Batch: 3584/60000 / Loss: 0.072384514 / Training Accuracy: 0.6347656 / Validation Accuracy: 0.668\n",
      "Epoch: 1 / Batch: 4096/60000 / Loss: 0.066252485 / Training Accuracy: 0.6875 / Validation Accuracy: 0.674\n",
      "Epoch: 1 / Batch: 4608/60000 / Loss: 0.062949024 / Training Accuracy: 0.69140625 / Validation Accuracy: 0.688\n",
      "Epoch: 1 / Batch: 5120/60000 / Loss: 0.06125184 / Training Accuracy: 0.6972656 / Validation Accuracy: 0.692\n",
      "Epoch: 1 / Batch: 5632/60000 / Loss: 0.056821745 / Training Accuracy: 0.6777344 / Validation Accuracy: 0.7\n",
      "Epoch: 1 / Batch: 6144/60000 / Loss: 0.05299536 / Training Accuracy: 0.734375 / Validation Accuracy: 0.715\n",
      "Epoch: 1 / Batch: 6656/60000 / Loss: 0.0492454 / Training Accuracy: 0.7402344 / Validation Accuracy: 0.733\n",
      "Epoch: 1 / Batch: 7168/60000 / Loss: 0.045407716 / Training Accuracy: 0.734375 / Validation Accuracy: 0.742\n",
      "Epoch: 1 / Batch: 7680/60000 / Loss: 0.040615696 / Training Accuracy: 0.7949219 / Validation Accuracy: 0.768\n",
      "Epoch: 1 / Batch: 8192/60000 / Loss: 0.038307715 / Training Accuracy: 0.7910156 / Validation Accuracy: 0.778\n",
      "Epoch: 1 / Batch: 8704/60000 / Loss: 0.036645826 / Training Accuracy: 0.7988281 / Validation Accuracy: 0.792\n",
      "Epoch: 1 / Batch: 9216/60000 / Loss: 0.03575941 / Training Accuracy: 0.7949219 / Validation Accuracy: 0.814\n",
      "Epoch: 1 / Batch: 9728/60000 / Loss: 0.030040275 / Training Accuracy: 0.85546875 / Validation Accuracy: 0.829\n",
      "Epoch: 1 / Batch: 10240/60000 / Loss: 0.02891829 / Training Accuracy: 0.83984375 / Validation Accuracy: 0.832\n",
      "Epoch: 1 / Batch: 10752/60000 / Loss: 0.028283719 / Training Accuracy: 0.84375 / Validation Accuracy: 0.833\n",
      "Epoch: 1 / Batch: 11264/60000 / Loss: 0.025075708 / Training Accuracy: 0.85546875 / Validation Accuracy: 0.836\n",
      "Epoch: 1 / Batch: 11776/60000 / Loss: 0.02671953 / Training Accuracy: 0.8417969 / Validation Accuracy: 0.839\n",
      "Epoch: 1 / Batch: 12288/60000 / Loss: 0.021012131 / Training Accuracy: 0.8769531 / Validation Accuracy: 0.833\n",
      "Epoch: 1 / Batch: 12800/60000 / Loss: 0.023414034 / Training Accuracy: 0.8457031 / Validation Accuracy: 0.838\n",
      "Epoch: 1 / Batch: 13312/60000 / Loss: 0.023975242 / Training Accuracy: 0.83984375 / Validation Accuracy: 0.835\n",
      "Epoch: 1 / Batch: 13824/60000 / Loss: 0.02199566 / Training Accuracy: 0.87109375 / Validation Accuracy: 0.838\n",
      "Epoch: 1 / Batch: 14336/60000 / Loss: 0.020216865 / Training Accuracy: 0.8730469 / Validation Accuracy: 0.847\n",
      "Epoch: 1 / Batch: 14848/60000 / Loss: 0.020486318 / Training Accuracy: 0.8730469 / Validation Accuracy: 0.855\n",
      "Epoch: 1 / Batch: 15360/60000 / Loss: 0.018509123 / Training Accuracy: 0.890625 / Validation Accuracy: 0.853\n",
      "Epoch: 1 / Batch: 15872/60000 / Loss: 0.019100597 / Training Accuracy: 0.890625 / Validation Accuracy: 0.853\n",
      "Epoch: 1 / Batch: 16384/60000 / Loss: 0.021827554 / Training Accuracy: 0.84765625 / Validation Accuracy: 0.861\n",
      "Epoch: 1 / Batch: 16896/60000 / Loss: 0.019455504 / Training Accuracy: 0.86328125 / Validation Accuracy: 0.862\n",
      "Epoch: 1 / Batch: 17408/60000 / Loss: 0.017058657 / Training Accuracy: 0.8886719 / Validation Accuracy: 0.869\n",
      "Epoch: 1 / Batch: 17920/60000 / Loss: 0.019576376 / Training Accuracy: 0.8691406 / Validation Accuracy: 0.866\n",
      "Epoch: 1 / Batch: 18432/60000 / Loss: 0.016371077 / Training Accuracy: 0.8925781 / Validation Accuracy: 0.865\n",
      "Epoch: 1 / Batch: 18944/60000 / Loss: 0.015960664 / Training Accuracy: 0.89453125 / Validation Accuracy: 0.859\n",
      "Epoch: 1 / Batch: 19456/60000 / Loss: 0.015664773 / Training Accuracy: 0.8925781 / Validation Accuracy: 0.864\n",
      "Epoch: 1 / Batch: 19968/60000 / Loss: 0.013664849 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.868\n",
      "Epoch: 1 / Batch: 20480/60000 / Loss: 0.01609137 / Training Accuracy: 0.8847656 / Validation Accuracy: 0.871\n",
      "Epoch: 1 / Batch: 20992/60000 / Loss: 0.016389508 / Training Accuracy: 0.9003906 / Validation Accuracy: 0.874\n",
      "Epoch: 1 / Batch: 21504/60000 / Loss: 0.014619231 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.878\n",
      "Epoch: 1 / Batch: 22016/60000 / Loss: 0.018829737 / Training Accuracy: 0.87890625 / Validation Accuracy: 0.881\n",
      "Epoch: 1 / Batch: 22528/60000 / Loss: 0.018375382 / Training Accuracy: 0.87109375 / Validation Accuracy: 0.883\n",
      "Epoch: 1 / Batch: 23040/60000 / Loss: 0.012963144 / Training Accuracy: 0.9140625 / Validation Accuracy: 0.882\n",
      "Epoch: 1 / Batch: 23552/60000 / Loss: 0.0142082 / Training Accuracy: 0.9003906 / Validation Accuracy: 0.884\n",
      "Epoch: 1 / Batch: 24064/60000 / Loss: 0.015774757 / Training Accuracy: 0.8886719 / Validation Accuracy: 0.884\n",
      "Epoch: 1 / Batch: 24576/60000 / Loss: 0.014832499 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.883\n",
      "Epoch: 1 / Batch: 25088/60000 / Loss: 0.016195556 / Training Accuracy: 0.890625 / Validation Accuracy: 0.88\n",
      "Epoch: 1 / Batch: 25600/60000 / Loss: 0.017022759 / Training Accuracy: 0.8769531 / Validation Accuracy: 0.885\n",
      "Epoch: 1 / Batch: 26112/60000 / Loss: 0.01449396 / Training Accuracy: 0.90625 / Validation Accuracy: 0.887\n",
      "Epoch: 1 / Batch: 26624/60000 / Loss: 0.015562715 / Training Accuracy: 0.8925781 / Validation Accuracy: 0.891\n",
      "Epoch: 1 / Batch: 27136/60000 / Loss: 0.01340692 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.891\n",
      "Epoch: 1 / Batch: 27648/60000 / Loss: 0.012157327 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.892\n",
      "Epoch: 1 / Batch: 28160/60000 / Loss: 0.013816426 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.891\n",
      "Epoch: 1 / Batch: 28672/60000 / Loss: 0.014324456 / Training Accuracy: 0.9082031 / Validation Accuracy: 0.892\n",
      "Epoch: 1 / Batch: 29184/60000 / Loss: 0.010503653 / Training Accuracy: 0.9375 / Validation Accuracy: 0.89\n",
      "Epoch: 1 / Batch: 29696/60000 / Loss: 0.011909334 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.897\n",
      "Epoch: 1 / Batch: 30208/60000 / Loss: 0.015938036 / Training Accuracy: 0.890625 / Validation Accuracy: 0.895\n",
      "Epoch: 1 / Batch: 30720/60000 / Loss: 0.013243857 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.898\n",
      "Epoch: 1 / Batch: 31232/60000 / Loss: 0.015412201 / Training Accuracy: 0.89453125 / Validation Accuracy: 0.897\n",
      "Epoch: 1 / Batch: 31744/60000 / Loss: 0.009873988 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.9\n",
      "Epoch: 1 / Batch: 32256/60000 / Loss: 0.013573977 / Training Accuracy: 0.91015625 / Validation Accuracy: 0.898\n",
      "Epoch: 1 / Batch: 32768/60000 / Loss: 0.0136665385 / Training Accuracy: 0.90625 / Validation Accuracy: 0.897\n",
      "Epoch: 1 / Batch: 33280/60000 / Loss: 0.011487288 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.896\n",
      "Epoch: 1 / Batch: 33792/60000 / Loss: 0.0134799285 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.902\n",
      "Epoch: 1 / Batch: 34304/60000 / Loss: 0.013766443 / Training Accuracy: 0.90234375 / Validation Accuracy: 0.9\n",
      "Epoch: 1 / Batch: 34816/60000 / Loss: 0.011720495 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.901\n",
      "Epoch: 1 / Batch: 35328/60000 / Loss: 0.013813118 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.899\n",
      "Epoch: 1 / Batch: 35840/60000 / Loss: 0.013699284 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.894\n",
      "Epoch: 1 / Batch: 36352/60000 / Loss: 0.011235626 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.898\n",
      "Epoch: 1 / Batch: 36864/60000 / Loss: 0.013733061 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.893\n",
      "Epoch: 1 / Batch: 37376/60000 / Loss: 0.010200129 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.9\n",
      "Epoch: 1 / Batch: 37888/60000 / Loss: 0.015108114 / Training Accuracy: 0.90234375 / Validation Accuracy: 0.901\n",
      "Epoch: 1 / Batch: 38400/60000 / Loss: 0.012541649 / Training Accuracy: 0.9199219 / Validation Accuracy: 0.902\n",
      "Epoch: 1 / Batch: 38912/60000 / Loss: 0.014874128 / Training Accuracy: 0.9042969 / Validation Accuracy: 0.899\n",
      "Epoch: 1 / Batch: 39424/60000 / Loss: 0.010613038 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.897\n",
      "Epoch: 1 / Batch: 39936/60000 / Loss: 0.010958487 / Training Accuracy: 0.921875 / Validation Accuracy: 0.897\n",
      "Epoch: 1 / Batch: 40448/60000 / Loss: 0.012907031 / Training Accuracy: 0.91015625 / Validation Accuracy: 0.897\n",
      "Epoch: 1 / Batch: 40960/60000 / Loss: 0.011809629 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.901\n",
      "Epoch: 1 / Batch: 41472/60000 / Loss: 0.011573959 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.903\n",
      "Epoch: 1 / Batch: 41984/60000 / Loss: 0.015299296 / Training Accuracy: 0.890625 / Validation Accuracy: 0.904\n",
      "Epoch: 1 / Batch: 42496/60000 / Loss: 0.009546548 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.907\n",
      "Epoch: 1 / Batch: 43008/60000 / Loss: 0.012741676 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.908\n",
      "Epoch: 1 / Batch: 43520/60000 / Loss: 0.013957113 / Training Accuracy: 0.91015625 / Validation Accuracy: 0.91\n",
      "Epoch: 1 / Batch: 44032/60000 / Loss: 0.011288705 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.909\n",
      "Epoch: 1 / Batch: 44544/60000 / Loss: 0.0126759885 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.912\n",
      "Epoch: 1 / Batch: 45056/60000 / Loss: 0.01255005 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.915\n",
      "Epoch: 1 / Batch: 45568/60000 / Loss: 0.012526719 / Training Accuracy: 0.91796875 / Validation Accuracy: 0.918\n",
      "Epoch: 1 / Batch: 46080/60000 / Loss: 0.010682126 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.919\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 / Batch: 46592/60000 / Loss: 0.012062493 / Training Accuracy: 0.9199219 / Validation Accuracy: 0.917\n",
      "Epoch: 1 / Batch: 47104/60000 / Loss: 0.010683266 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.917\n",
      "Epoch: 1 / Batch: 47616/60000 / Loss: 0.014199575 / Training Accuracy: 0.9121094 / Validation Accuracy: 0.915\n",
      "Epoch: 1 / Batch: 48128/60000 / Loss: 0.009058681 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.914\n",
      "Epoch: 1 / Batch: 48640/60000 / Loss: 0.0099741295 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.913\n",
      "Epoch: 1 / Batch: 49152/60000 / Loss: 0.01105151 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.915\n",
      "Epoch: 1 / Batch: 49664/60000 / Loss: 0.011359194 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.916\n",
      "Epoch: 1 / Batch: 50176/60000 / Loss: 0.009952257 / Training Accuracy: 0.9375 / Validation Accuracy: 0.914\n",
      "Epoch: 1 / Batch: 50688/60000 / Loss: 0.010178453 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.915\n",
      "Epoch: 1 / Batch: 51200/60000 / Loss: 0.011056225 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.913\n",
      "Epoch: 1 / Batch: 51712/60000 / Loss: 0.010302329 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.913\n",
      "Epoch: 1 / Batch: 52224/60000 / Loss: 0.01240533 / Training Accuracy: 0.9160156 / Validation Accuracy: 0.91\n",
      "Epoch: 1 / Batch: 52736/60000 / Loss: 0.012205128 / Training Accuracy: 0.921875 / Validation Accuracy: 0.913\n",
      "Epoch: 1 / Batch: 53248/60000 / Loss: 0.012300206 / Training Accuracy: 0.9160156 / Validation Accuracy: 0.913\n",
      "Epoch: 1 / Batch: 53760/60000 / Loss: 0.011982343 / Training Accuracy: 0.921875 / Validation Accuracy: 0.916\n",
      "Epoch: 1 / Batch: 54272/60000 / Loss: 0.012264857 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.917\n",
      "Epoch: 1 / Batch: 54784/60000 / Loss: 0.010550736 / Training Accuracy: 0.9375 / Validation Accuracy: 0.918\n",
      "Epoch: 1 / Batch: 55296/60000 / Loss: 0.011396533 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.923\n",
      "Epoch: 1 / Batch: 55808/60000 / Loss: 0.011325928 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.924\n",
      "Epoch: 1 / Batch: 56320/60000 / Loss: 0.010143342 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.921\n",
      "Epoch: 1 / Batch: 56832/60000 / Loss: 0.009669105 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.92\n",
      "Epoch: 1 / Batch: 57344/60000 / Loss: 0.010672878 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.919\n",
      "Epoch: 1 / Batch: 57856/60000 / Loss: 0.010430155 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.919\n",
      "Epoch: 1 / Batch: 58368/60000 / Loss: 0.011503445 / Training Accuracy: 0.9199219 / Validation Accuracy: 0.923\n",
      "Epoch: 1 / Batch: 58880/60000 / Loss: 0.012659995 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.924\n",
      "Epoch: 1 / Batch: 59392/60000 / Loss: 0.009669907 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.926\n",
      "Epoch: 1 / Batch: 59904/60000 / Loss: 0.0065661925 / Training Accuracy: 0.9583333 / Validation Accuracy: 0.928\n",
      "Epoch: 2 / Batch: 0/60000 / Loss: 0.008159791 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.924\n",
      "Epoch: 2 / Batch: 512/60000 / Loss: 0.0085815145 / Training Accuracy: 0.953125 / Validation Accuracy: 0.93\n",
      "Epoch: 2 / Batch: 1024/60000 / Loss: 0.0096643 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.928\n",
      "Epoch: 2 / Batch: 1536/60000 / Loss: 0.009854346 / Training Accuracy: 0.92578125 / Validation Accuracy: 0.929\n",
      "Epoch: 2 / Batch: 2048/60000 / Loss: 0.009962788 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.931\n",
      "Epoch: 2 / Batch: 2560/60000 / Loss: 0.009928965 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.929\n",
      "Epoch: 2 / Batch: 3072/60000 / Loss: 0.009747234 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.93\n",
      "Epoch: 2 / Batch: 3584/60000 / Loss: 0.009338634 / Training Accuracy: 0.9375 / Validation Accuracy: 0.931\n",
      "Epoch: 2 / Batch: 4096/60000 / Loss: 0.00833041 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.929\n",
      "Epoch: 2 / Batch: 4608/60000 / Loss: 0.009886944 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.927\n",
      "Epoch: 2 / Batch: 5120/60000 / Loss: 0.010307158 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.928\n",
      "Epoch: 2 / Batch: 5632/60000 / Loss: 0.0076822266 / Training Accuracy: 0.953125 / Validation Accuracy: 0.928\n",
      "Epoch: 2 / Batch: 6144/60000 / Loss: 0.0095168045 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.928\n",
      "Epoch: 2 / Batch: 6656/60000 / Loss: 0.010453845 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.93\n",
      "Epoch: 2 / Batch: 7168/60000 / Loss: 0.009438752 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.932\n",
      "Epoch: 2 / Batch: 7680/60000 / Loss: 0.0082964 / Training Accuracy: 0.953125 / Validation Accuracy: 0.931\n",
      "Epoch: 2 / Batch: 8192/60000 / Loss: 0.008667708 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.932\n",
      "Epoch: 2 / Batch: 8704/60000 / Loss: 0.007513234 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.931\n",
      "Epoch: 2 / Batch: 9216/60000 / Loss: 0.010937845 / Training Accuracy: 0.9238281 / Validation Accuracy: 0.932\n",
      "Epoch: 2 / Batch: 9728/60000 / Loss: 0.00944371 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.933\n",
      "Epoch: 2 / Batch: 10240/60000 / Loss: 0.011388225 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.932\n",
      "Epoch: 2 / Batch: 10752/60000 / Loss: 0.0060055084 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.931\n",
      "Epoch: 2 / Batch: 11264/60000 / Loss: 0.008246165 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.927\n",
      "Epoch: 2 / Batch: 11776/60000 / Loss: 0.010194157 / Training Accuracy: 0.9375 / Validation Accuracy: 0.926\n",
      "Epoch: 2 / Batch: 12288/60000 / Loss: 0.0078578815 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.925\n",
      "Epoch: 2 / Batch: 12800/60000 / Loss: 0.0090279 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.928\n",
      "Epoch: 2 / Batch: 13312/60000 / Loss: 0.010529948 / Training Accuracy: 0.9277344 / Validation Accuracy: 0.927\n",
      "Epoch: 2 / Batch: 13824/60000 / Loss: 0.008681292 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.933\n",
      "Epoch: 2 / Batch: 14336/60000 / Loss: 0.010300846 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.937\n",
      "Epoch: 2 / Batch: 14848/60000 / Loss: 0.011543137 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.936\n",
      "Epoch: 2 / Batch: 15360/60000 / Loss: 0.010321587 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.936\n",
      "Epoch: 2 / Batch: 15872/60000 / Loss: 0.009109906 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.936\n",
      "Epoch: 2 / Batch: 16384/60000 / Loss: 0.009974236 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.938\n",
      "Epoch: 2 / Batch: 16896/60000 / Loss: 0.010183033 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.938\n",
      "Epoch: 2 / Batch: 17408/60000 / Loss: 0.0063819727 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.938\n",
      "Epoch: 2 / Batch: 17920/60000 / Loss: 0.010611804 / Training Accuracy: 0.9296875 / Validation Accuracy: 0.939\n",
      "Epoch: 2 / Batch: 18432/60000 / Loss: 0.008660693 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.94\n",
      "Epoch: 2 / Batch: 18944/60000 / Loss: 0.008509925 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.935\n",
      "Epoch: 2 / Batch: 19456/60000 / Loss: 0.007337193 / Training Accuracy: 0.953125 / Validation Accuracy: 0.933\n",
      "Epoch: 2 / Batch: 19968/60000 / Loss: 0.007962857 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.933\n",
      "Epoch: 2 / Batch: 20480/60000 / Loss: 0.01060194 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.934\n",
      "Epoch: 2 / Batch: 20992/60000 / Loss: 0.012657541 / Training Accuracy: 0.9199219 / Validation Accuracy: 0.936\n",
      "Epoch: 2 / Batch: 21504/60000 / Loss: 0.0084555885 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.936\n",
      "Epoch: 2 / Batch: 22016/60000 / Loss: 0.010521819 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.933\n",
      "Epoch: 2 / Batch: 22528/60000 / Loss: 0.008505784 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.933\n",
      "Epoch: 2 / Batch: 23040/60000 / Loss: 0.010002991 / Training Accuracy: 0.9316406 / Validation Accuracy: 0.932\n",
      "Epoch: 2 / Batch: 23552/60000 / Loss: 0.009041423 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.931\n",
      "Epoch: 2 / Batch: 24064/60000 / Loss: 0.007672798 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.935\n",
      "Epoch: 2 / Batch: 24576/60000 / Loss: 0.008660903 / Training Accuracy: 0.9375 / Validation Accuracy: 0.939\n",
      "Epoch: 2 / Batch: 25088/60000 / Loss: 0.009111999 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.938\n",
      "Epoch: 2 / Batch: 25600/60000 / Loss: 0.009857084 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.939\n",
      "Epoch: 2 / Batch: 26112/60000 / Loss: 0.008959241 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.939\n",
      "Epoch: 2 / Batch: 26624/60000 / Loss: 0.0077706664 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.94\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 2 / Batch: 27136/60000 / Loss: 0.007937275 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.94\n",
      "Epoch: 2 / Batch: 27648/60000 / Loss: 0.008717937 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.94\n",
      "Epoch: 2 / Batch: 28160/60000 / Loss: 0.007901097 / Training Accuracy: 0.953125 / Validation Accuracy: 0.94\n",
      "Epoch: 2 / Batch: 28672/60000 / Loss: 0.007812259 / Training Accuracy: 0.953125 / Validation Accuracy: 0.94\n",
      "Epoch: 2 / Batch: 29184/60000 / Loss: 0.0073101693 / Training Accuracy: 0.953125 / Validation Accuracy: 0.937\n",
      "Epoch: 2 / Batch: 29696/60000 / Loss: 0.007525664 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.938\n",
      "Epoch: 2 / Batch: 30208/60000 / Loss: 0.006929419 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.939\n",
      "Epoch: 2 / Batch: 30720/60000 / Loss: 0.008496662 / Training Accuracy: 0.953125 / Validation Accuracy: 0.938\n",
      "Epoch: 2 / Batch: 31232/60000 / Loss: 0.008259493 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.936\n",
      "Epoch: 2 / Batch: 31744/60000 / Loss: 0.008205043 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.935\n",
      "Epoch: 2 / Batch: 32256/60000 / Loss: 0.0076515437 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.939\n",
      "Epoch: 2 / Batch: 32768/60000 / Loss: 0.0067186086 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.941\n",
      "Epoch: 2 / Batch: 33280/60000 / Loss: 0.005815956 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.941\n",
      "Epoch: 2 / Batch: 33792/60000 / Loss: 0.0098902555 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.944\n",
      "Epoch: 2 / Batch: 34304/60000 / Loss: 0.007859813 / Training Accuracy: 0.953125 / Validation Accuracy: 0.944\n",
      "Epoch: 2 / Batch: 34816/60000 / Loss: 0.009098026 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 35328/60000 / Loss: 0.008515576 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.941\n",
      "Epoch: 2 / Batch: 35840/60000 / Loss: 0.007049404 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.941\n",
      "Epoch: 2 / Batch: 36352/60000 / Loss: 0.007557149 / Training Accuracy: 0.953125 / Validation Accuracy: 0.939\n",
      "Epoch: 2 / Batch: 36864/60000 / Loss: 0.009688047 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.94\n",
      "Epoch: 2 / Batch: 37376/60000 / Loss: 0.0077217454 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.943\n",
      "Epoch: 2 / Batch: 37888/60000 / Loss: 0.008446958 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.941\n",
      "Epoch: 2 / Batch: 38400/60000 / Loss: 0.0076631634 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 38912/60000 / Loss: 0.008031537 / Training Accuracy: 0.953125 / Validation Accuracy: 0.943\n",
      "Epoch: 2 / Batch: 39424/60000 / Loss: 0.008018189 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.944\n",
      "Epoch: 2 / Batch: 39936/60000 / Loss: 0.0076761893 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.943\n",
      "Epoch: 2 / Batch: 40448/60000 / Loss: 0.006075146 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.944\n",
      "Epoch: 2 / Batch: 40960/60000 / Loss: 0.010018875 / Training Accuracy: 0.93359375 / Validation Accuracy: 0.943\n",
      "Epoch: 2 / Batch: 41472/60000 / Loss: 0.0074806726 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.945\n",
      "Epoch: 2 / Batch: 41984/60000 / Loss: 0.0074836267 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.946\n",
      "Epoch: 2 / Batch: 42496/60000 / Loss: 0.006419029 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.948\n",
      "Epoch: 2 / Batch: 43008/60000 / Loss: 0.0069962763 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.946\n",
      "Epoch: 2 / Batch: 43520/60000 / Loss: 0.007530385 / Training Accuracy: 0.953125 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 44032/60000 / Loss: 0.009665811 / Training Accuracy: 0.9355469 / Validation Accuracy: 0.939\n",
      "Epoch: 2 / Batch: 44544/60000 / Loss: 0.007997183 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.937\n",
      "Epoch: 2 / Batch: 45056/60000 / Loss: 0.011262591 / Training Accuracy: 0.9160156 / Validation Accuracy: 0.939\n",
      "Epoch: 2 / Batch: 45568/60000 / Loss: 0.008123623 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.94\n",
      "Epoch: 2 / Batch: 46080/60000 / Loss: 0.008662182 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 46592/60000 / Loss: 0.0090432335 / Training Accuracy: 0.9394531 / Validation Accuracy: 0.943\n",
      "Epoch: 2 / Batch: 47104/60000 / Loss: 0.008596094 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.946\n",
      "Epoch: 2 / Batch: 47616/60000 / Loss: 0.009326233 / Training Accuracy: 0.9375 / Validation Accuracy: 0.948\n",
      "Epoch: 2 / Batch: 48128/60000 / Loss: 0.0072310762 / Training Accuracy: 0.953125 / Validation Accuracy: 0.951\n",
      "Epoch: 2 / Batch: 48640/60000 / Loss: 0.0068299687 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.954\n",
      "Epoch: 2 / Batch: 49152/60000 / Loss: 0.00832084 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.951\n",
      "Epoch: 2 / Batch: 49664/60000 / Loss: 0.007470978 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.948\n",
      "Epoch: 2 / Batch: 50176/60000 / Loss: 0.0064421734 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.945\n",
      "Epoch: 2 / Batch: 50688/60000 / Loss: 0.006653721 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 51200/60000 / Loss: 0.00818156 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 51712/60000 / Loss: 0.0059309592 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 52224/60000 / Loss: 0.0078824535 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.943\n",
      "Epoch: 2 / Batch: 52736/60000 / Loss: 0.007797127 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 53248/60000 / Loss: 0.009293748 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.944\n",
      "Epoch: 2 / Batch: 53760/60000 / Loss: 0.0069749104 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.94\n",
      "Epoch: 2 / Batch: 54272/60000 / Loss: 0.008036406 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.94\n",
      "Epoch: 2 / Batch: 54784/60000 / Loss: 0.0059300545 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 55296/60000 / Loss: 0.007565514 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.944\n",
      "Epoch: 2 / Batch: 55808/60000 / Loss: 0.0068787993 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.944\n",
      "Epoch: 2 / Batch: 56320/60000 / Loss: 0.006251015 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.941\n",
      "Epoch: 2 / Batch: 56832/60000 / Loss: 0.0088044 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.942\n",
      "Epoch: 2 / Batch: 57344/60000 / Loss: 0.006112226 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.945\n",
      "Epoch: 2 / Batch: 57856/60000 / Loss: 0.007920056 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.946\n",
      "Epoch: 2 / Batch: 58368/60000 / Loss: 0.006311354 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.946\n",
      "Epoch: 2 / Batch: 58880/60000 / Loss: 0.0047706477 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.947\n",
      "Epoch: 2 / Batch: 59392/60000 / Loss: 0.008162463 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.948\n",
      "Epoch: 2 / Batch: 59904/60000 / Loss: 0.008248943 / Training Accuracy: 0.9583333 / Validation Accuracy: 0.95\n",
      "Epoch: 3 / Batch: 0/60000 / Loss: 0.006466216 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 512/60000 / Loss: 0.006058584 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.95\n",
      "Epoch: 3 / Batch: 1024/60000 / Loss: 0.0060358015 / Training Accuracy: 0.96875 / Validation Accuracy: 0.949\n",
      "Epoch: 3 / Batch: 1536/60000 / Loss: 0.0054380074 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.952\n",
      "Epoch: 3 / Batch: 2048/60000 / Loss: 0.007174676 / Training Accuracy: 0.953125 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 2560/60000 / Loss: 0.0068221674 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 3072/60000 / Loss: 0.007760306 / Training Accuracy: 0.953125 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 3584/60000 / Loss: 0.0059640575 / Training Accuracy: 0.96875 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 4096/60000 / Loss: 0.005661148 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 4608/60000 / Loss: 0.006187562 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 5120/60000 / Loss: 0.0068047484 / Training Accuracy: 0.953125 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 5632/60000 / Loss: 0.006608525 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 6144/60000 / Loss: 0.0059827585 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 6656/60000 / Loss: 0.0071447315 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.952\n",
      "Epoch: 3 / Batch: 7168/60000 / Loss: 0.0049694628 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 7680/60000 / Loss: 0.0078060017 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.95\n",
      "Epoch: 3 / Batch: 8192/60000 / Loss: 0.005014945 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.948\n",
      "Epoch: 3 / Batch: 8704/60000 / Loss: 0.0065584765 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 9216/60000 / Loss: 0.006196859 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 9728/60000 / Loss: 0.0077120215 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.954\n",
      "Epoch: 3 / Batch: 10240/60000 / Loss: 0.005060184 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 10752/60000 / Loss: 0.006976726 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 11264/60000 / Loss: 0.0050109564 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.954\n",
      "Epoch: 3 / Batch: 11776/60000 / Loss: 0.005384042 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.955\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 / Batch: 12288/60000 / Loss: 0.00750601 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 12800/60000 / Loss: 0.006545274 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 13312/60000 / Loss: 0.0064585684 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 13824/60000 / Loss: 0.0061347964 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 14336/60000 / Loss: 0.0068942695 / Training Accuracy: 0.953125 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 14848/60000 / Loss: 0.0072009056 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 15360/60000 / Loss: 0.008709556 / Training Accuracy: 0.94140625 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 15872/60000 / Loss: 0.00726704 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.952\n",
      "Epoch: 3 / Batch: 16384/60000 / Loss: 0.007033974 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 16896/60000 / Loss: 0.0065653883 / Training Accuracy: 0.96875 / Validation Accuracy: 0.95\n",
      "Epoch: 3 / Batch: 17408/60000 / Loss: 0.0063579353 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.949\n",
      "Epoch: 3 / Batch: 17920/60000 / Loss: 0.0069730254 / Training Accuracy: 0.953125 / Validation Accuracy: 0.95\n",
      "Epoch: 3 / Batch: 18432/60000 / Loss: 0.0054677418 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 18944/60000 / Loss: 0.006044706 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 19456/60000 / Loss: 0.0047969287 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 19968/60000 / Loss: 0.0039832657 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 20480/60000 / Loss: 0.006981281 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.954\n",
      "Epoch: 3 / Batch: 20992/60000 / Loss: 0.006791456 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 21504/60000 / Loss: 0.006157602 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.95\n",
      "Epoch: 3 / Batch: 22016/60000 / Loss: 0.007738398 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 22528/60000 / Loss: 0.0049591023 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 23040/60000 / Loss: 0.0053137997 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 23552/60000 / Loss: 0.004344199 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 24064/60000 / Loss: 0.006252884 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.952\n",
      "Epoch: 3 / Batch: 24576/60000 / Loss: 0.0056532314 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 25088/60000 / Loss: 0.0045337947 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 25600/60000 / Loss: 0.008694319 / Training Accuracy: 0.9453125 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 26112/60000 / Loss: 0.006089189 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.952\n",
      "Epoch: 3 / Batch: 26624/60000 / Loss: 0.0053708935 / Training Accuracy: 0.96875 / Validation Accuracy: 0.954\n",
      "Epoch: 3 / Batch: 27136/60000 / Loss: 0.006406208 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 27648/60000 / Loss: 0.008159675 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.954\n",
      "Epoch: 3 / Batch: 28160/60000 / Loss: 0.007041672 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 28672/60000 / Loss: 0.005892641 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 29184/60000 / Loss: 0.0072996295 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.954\n",
      "Epoch: 3 / Batch: 29696/60000 / Loss: 0.0051288726 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 30208/60000 / Loss: 0.0058498904 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.954\n",
      "Epoch: 3 / Batch: 30720/60000 / Loss: 0.007141535 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 31232/60000 / Loss: 0.007551742 / Training Accuracy: 0.953125 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 31744/60000 / Loss: 0.005592494 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 32256/60000 / Loss: 0.0074324063 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.952\n",
      "Epoch: 3 / Batch: 32768/60000 / Loss: 0.006818543 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 33280/60000 / Loss: 0.007279642 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 33792/60000 / Loss: 0.0053673 / Training Accuracy: 0.96875 / Validation Accuracy: 0.958\n",
      "Epoch: 3 / Batch: 34304/60000 / Loss: 0.0054849447 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 34816/60000 / Loss: 0.006444598 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.96\n",
      "Epoch: 3 / Batch: 35328/60000 / Loss: 0.0070605204 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.959\n",
      "Epoch: 3 / Batch: 35840/60000 / Loss: 0.005344791 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.961\n",
      "Epoch: 3 / Batch: 36352/60000 / Loss: 0.0027595223 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.961\n",
      "Epoch: 3 / Batch: 36864/60000 / Loss: 0.00689911 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.959\n",
      "Epoch: 3 / Batch: 37376/60000 / Loss: 0.0059630047 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.96\n",
      "Epoch: 3 / Batch: 37888/60000 / Loss: 0.004904732 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 38400/60000 / Loss: 0.0071587726 / Training Accuracy: 0.953125 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 38912/60000 / Loss: 0.007200022 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.96\n",
      "Epoch: 3 / Batch: 39424/60000 / Loss: 0.005242467 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.959\n",
      "Epoch: 3 / Batch: 39936/60000 / Loss: 0.0042022 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.959\n",
      "Epoch: 3 / Batch: 40448/60000 / Loss: 0.0052659763 / Training Accuracy: 0.96875 / Validation Accuracy: 0.958\n",
      "Epoch: 3 / Batch: 40960/60000 / Loss: 0.0059633865 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 41472/60000 / Loss: 0.00454479 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.954\n",
      "Epoch: 3 / Batch: 41984/60000 / Loss: 0.005912262 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 42496/60000 / Loss: 0.006074206 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 43008/60000 / Loss: 0.0052020303 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 43520/60000 / Loss: 0.005253094 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 44032/60000 / Loss: 0.007110817 / Training Accuracy: 0.953125 / Validation Accuracy: 0.958\n",
      "Epoch: 3 / Batch: 44544/60000 / Loss: 0.0053887344 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.959\n",
      "Epoch: 3 / Batch: 45056/60000 / Loss: 0.005651597 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.959\n",
      "Epoch: 3 / Batch: 45568/60000 / Loss: 0.0061930735 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 46080/60000 / Loss: 0.0073967874 / Training Accuracy: 0.9472656 / Validation Accuracy: 0.955\n",
      "Epoch: 3 / Batch: 46592/60000 / Loss: 0.0044058263 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 47104/60000 / Loss: 0.0055644987 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 47616/60000 / Loss: 0.006044209 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 48128/60000 / Loss: 0.0049972576 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 48640/60000 / Loss: 0.004784459 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 49152/60000 / Loss: 0.005514546 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.958\n",
      "Epoch: 3 / Batch: 49664/60000 / Loss: 0.004988177 / Training Accuracy: 0.96875 / Validation Accuracy: 0.959\n",
      "Epoch: 3 / Batch: 50176/60000 / Loss: 0.0065207295 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.958\n",
      "Epoch: 3 / Batch: 50688/60000 / Loss: 0.0044514337 / Training Accuracy: 0.96875 / Validation Accuracy: 0.959\n",
      "Epoch: 3 / Batch: 51200/60000 / Loss: 0.007233155 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.96\n",
      "Epoch: 3 / Batch: 51712/60000 / Loss: 0.005182208 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.96\n",
      "Epoch: 3 / Batch: 52224/60000 / Loss: 0.008311262 / Training Accuracy: 0.9433594 / Validation Accuracy: 0.962\n",
      "Epoch: 3 / Batch: 52736/60000 / Loss: 0.0050782776 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 3 / Batch: 53248/60000 / Loss: 0.0039052528 / Training Accuracy: 0.96875 / Validation Accuracy: 0.963\n",
      "Epoch: 3 / Batch: 53760/60000 / Loss: 0.0069262846 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.96\n",
      "Epoch: 3 / Batch: 54272/60000 / Loss: 0.0057336735 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.961\n",
      "Epoch: 3 / Batch: 54784/60000 / Loss: 0.0061390004 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.963\n",
      "Epoch: 3 / Batch: 55296/60000 / Loss: 0.0060426933 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.962\n",
      "Epoch: 3 / Batch: 55808/60000 / Loss: 0.004688081 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.964\n",
      "Epoch: 3 / Batch: 56320/60000 / Loss: 0.004547064 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.963\n",
      "Epoch: 3 / Batch: 56832/60000 / Loss: 0.0052517825 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.961\n",
      "Epoch: 3 / Batch: 57344/60000 / Loss: 0.007230182 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.96\n",
      "Epoch: 3 / Batch: 57856/60000 / Loss: 0.0047048978 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.96\n",
      "Epoch: 3 / Batch: 58368/60000 / Loss: 0.0064249197 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.96\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 3 / Batch: 58880/60000 / Loss: 0.005247043 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.965\n",
      "Epoch: 3 / Batch: 59392/60000 / Loss: 0.0043766415 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.965\n",
      "Epoch: 3 / Batch: 59904/60000 / Loss: 0.007875714 / Training Accuracy: 0.9479167 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 0/60000 / Loss: 0.0058203945 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 512/60000 / Loss: 0.00501726 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 1024/60000 / Loss: 0.0054653003 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 1536/60000 / Loss: 0.0041576996 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 2048/60000 / Loss: 0.0050699813 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 2560/60000 / Loss: 0.0039702267 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 3072/60000 / Loss: 0.0051037716 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 3584/60000 / Loss: 0.0056306543 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 4096/60000 / Loss: 0.005697117 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 4608/60000 / Loss: 0.0039267605 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 5120/60000 / Loss: 0.004118794 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 5632/60000 / Loss: 0.0057037165 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 6144/60000 / Loss: 0.0053935773 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 6656/60000 / Loss: 0.0063573346 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 7168/60000 / Loss: 0.005695842 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 7680/60000 / Loss: 0.0044653257 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 8192/60000 / Loss: 0.0038910713 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 8704/60000 / Loss: 0.0037879895 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 9216/60000 / Loss: 0.004425771 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 9728/60000 / Loss: 0.0066930437 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 10240/60000 / Loss: 0.0067980634 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 10752/60000 / Loss: 0.005083491 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 11264/60000 / Loss: 0.0036626775 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 11776/60000 / Loss: 0.0033053725 / Training Accuracy: 0.984375 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 12288/60000 / Loss: 0.0047725243 / Training Accuracy: 0.96875 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 12800/60000 / Loss: 0.0059809787 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 13312/60000 / Loss: 0.0049003493 / Training Accuracy: 0.96875 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 13824/60000 / Loss: 0.004608941 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 14336/60000 / Loss: 0.0062448783 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 14848/60000 / Loss: 0.005113684 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 15360/60000 / Loss: 0.0037589972 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 15872/60000 / Loss: 0.006807305 / Training Accuracy: 0.95703125 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 16384/60000 / Loss: 0.0045678797 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 16896/60000 / Loss: 0.003932625 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 17408/60000 / Loss: 0.0046708686 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 17920/60000 / Loss: 0.004149008 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 18432/60000 / Loss: 0.005663107 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 18944/60000 / Loss: 0.00613377 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 19456/60000 / Loss: 0.0029786986 / Training Accuracy: 0.984375 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 19968/60000 / Loss: 0.0049603647 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 20480/60000 / Loss: 0.003789906 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 20992/60000 / Loss: 0.0046465024 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 21504/60000 / Loss: 0.0036546662 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 22016/60000 / Loss: 0.005781939 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 22528/60000 / Loss: 0.0050781155 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 23040/60000 / Loss: 0.0057313973 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 23552/60000 / Loss: 0.005899532 / Training Accuracy: 0.9589844 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 24064/60000 / Loss: 0.0061577316 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 24576/60000 / Loss: 0.0032153712 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 25088/60000 / Loss: 0.005541441 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 25600/60000 / Loss: 0.0054305578 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 26112/60000 / Loss: 0.004966874 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 26624/60000 / Loss: 0.0064360155 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 27136/60000 / Loss: 0.0048149717 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 27648/60000 / Loss: 0.0037739612 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 28160/60000 / Loss: 0.006131209 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 28672/60000 / Loss: 0.0058587804 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 29184/60000 / Loss: 0.004148421 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 29696/60000 / Loss: 0.0051686643 / Training Accuracy: 0.96875 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 30208/60000 / Loss: 0.0033255282 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 30720/60000 / Loss: 0.0048886603 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 31232/60000 / Loss: 0.004405409 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 31744/60000 / Loss: 0.0037305919 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 32256/60000 / Loss: 0.0032099083 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 32768/60000 / Loss: 0.0053676115 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 33280/60000 / Loss: 0.003704948 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 33792/60000 / Loss: 0.004846728 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 34304/60000 / Loss: 0.0050065164 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 34816/60000 / Loss: 0.0043309466 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 35328/60000 / Loss: 0.0035726961 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 35840/60000 / Loss: 0.0053936215 / Training Accuracy: 0.96875 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 36352/60000 / Loss: 0.0039383294 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 36864/60000 / Loss: 0.0042597926 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 37376/60000 / Loss: 0.0054112985 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 37888/60000 / Loss: 0.0060928427 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 38400/60000 / Loss: 0.005622579 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.967\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 4 / Batch: 38912/60000 / Loss: 0.0042992258 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 39424/60000 / Loss: 0.0048915017 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 39936/60000 / Loss: 0.0051926887 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 40448/60000 / Loss: 0.004359176 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 40960/60000 / Loss: 0.0024505598 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 41472/60000 / Loss: 0.003482379 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 41984/60000 / Loss: 0.0040159533 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 42496/60000 / Loss: 0.0060154647 / Training Accuracy: 0.9550781 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 43008/60000 / Loss: 0.0047682086 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 43520/60000 / Loss: 0.004560364 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 44032/60000 / Loss: 0.0046341224 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 44544/60000 / Loss: 0.005255622 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 45056/60000 / Loss: 0.0030989726 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 45568/60000 / Loss: 0.004588874 / Training Accuracy: 0.96875 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 46080/60000 / Loss: 0.005440096 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 46592/60000 / Loss: 0.004670434 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 47104/60000 / Loss: 0.004318627 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 47616/60000 / Loss: 0.0061132615 / Training Accuracy: 0.9609375 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 48128/60000 / Loss: 0.0054638092 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 48640/60000 / Loss: 0.0065850792 / Training Accuracy: 0.953125 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 49152/60000 / Loss: 0.0046203695 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 49664/60000 / Loss: 0.004919693 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 50176/60000 / Loss: 0.003876172 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 50688/60000 / Loss: 0.004915853 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 51200/60000 / Loss: 0.005319803 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 51712/60000 / Loss: 0.003473614 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 52224/60000 / Loss: 0.0038579125 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 52736/60000 / Loss: 0.0055048876 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 53248/60000 / Loss: 0.0050102524 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.96\n",
      "Epoch: 4 / Batch: 53760/60000 / Loss: 0.004386065 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 54272/60000 / Loss: 0.0054319315 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.96\n",
      "Epoch: 4 / Batch: 54784/60000 / Loss: 0.0036579284 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.964\n",
      "Epoch: 4 / Batch: 55296/60000 / Loss: 0.003998908 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 55808/60000 / Loss: 0.0030744649 / Training Accuracy: 0.984375 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 56320/60000 / Loss: 0.0067398297 / Training Accuracy: 0.94921875 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 56832/60000 / Loss: 0.004831803 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 57344/60000 / Loss: 0.004308906 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.968\n",
      "Epoch: 4 / Batch: 57856/60000 / Loss: 0.0041821934 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 58368/60000 / Loss: 0.004505106 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 58880/60000 / Loss: 0.0043198904 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 59392/60000 / Loss: 0.0051812064 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.967\n",
      "Epoch: 4 / Batch: 59904/60000 / Loss: 0.006064935 / Training Accuracy: 0.9583333 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 0/60000 / Loss: 0.0030040361 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 512/60000 / Loss: 0.005447292 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 1024/60000 / Loss: 0.0042908527 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 1536/60000 / Loss: 0.0048183873 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 2048/60000 / Loss: 0.0028254702 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 2560/60000 / Loss: 0.0028973373 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 3072/60000 / Loss: 0.004592336 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 3584/60000 / Loss: 0.0024530874 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 4096/60000 / Loss: 0.0040506567 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 4608/60000 / Loss: 0.0025222076 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 5120/60000 / Loss: 0.0049647572 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 5632/60000 / Loss: 0.004471424 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 5 / Batch: 6144/60000 / Loss: 0.004692108 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 6656/60000 / Loss: 0.003493558 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 7168/60000 / Loss: 0.0026149205 / Training Accuracy: 0.984375 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 7680/60000 / Loss: 0.0040134545 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 8192/60000 / Loss: 0.0071566454 / Training Accuracy: 0.9511719 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 8704/60000 / Loss: 0.0047092615 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 9216/60000 / Loss: 0.004455547 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 9728/60000 / Loss: 0.003490199 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 10240/60000 / Loss: 0.0053064646 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 10752/60000 / Loss: 0.00432027 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 11264/60000 / Loss: 0.0042850254 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 11776/60000 / Loss: 0.0026153796 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 12288/60000 / Loss: 0.002470322 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 12800/60000 / Loss: 0.004821119 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 13312/60000 / Loss: 0.004957239 / Training Accuracy: 0.96875 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 13824/60000 / Loss: 0.0046656067 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 14336/60000 / Loss: 0.002026092 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 14848/60000 / Loss: 0.003890726 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 15360/60000 / Loss: 0.0046614283 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 15872/60000 / Loss: 0.003782253 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 16384/60000 / Loss: 0.0032386095 / Training Accuracy: 0.984375 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 16896/60000 / Loss: 0.0033015765 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.964\n",
      "Epoch: 5 / Batch: 17408/60000 / Loss: 0.003922077 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.964\n",
      "Epoch: 5 / Batch: 17920/60000 / Loss: 0.0046328283 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 18432/60000 / Loss: 0.002818946 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 18944/60000 / Loss: 0.0037098906 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 19456/60000 / Loss: 0.0043146014 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 19968/60000 / Loss: 0.003923958 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 20480/60000 / Loss: 0.004829918 / Training Accuracy: 0.96875 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 20992/60000 / Loss: 0.0034272515 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 21504/60000 / Loss: 0.004185387 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 22016/60000 / Loss: 0.004273902 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 22528/60000 / Loss: 0.0033342466 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 23040/60000 / Loss: 0.00419064 / Training Accuracy: 0.96875 / Validation Accuracy: 0.968\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 5 / Batch: 23552/60000 / Loss: 0.0042928113 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 24064/60000 / Loss: 0.0027063189 / Training Accuracy: 0.984375 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 24576/60000 / Loss: 0.0024459881 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 25088/60000 / Loss: 0.0024057482 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 25600/60000 / Loss: 0.0037020412 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 26112/60000 / Loss: 0.0034703356 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 26624/60000 / Loss: 0.0039209365 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 27136/60000 / Loss: 0.004139169 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 27648/60000 / Loss: 0.004154346 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 28160/60000 / Loss: 0.0043758964 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 28672/60000 / Loss: 0.0047471696 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 29184/60000 / Loss: 0.005229679 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 29696/60000 / Loss: 0.004891213 / Training Accuracy: 0.96875 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 30208/60000 / Loss: 0.003889172 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 30720/60000 / Loss: 0.0041121696 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 31232/60000 / Loss: 0.0036211922 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.973\n",
      "Epoch: 5 / Batch: 31744/60000 / Loss: 0.0029119956 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 32256/60000 / Loss: 0.0044736853 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 32768/60000 / Loss: 0.004830025 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 33280/60000 / Loss: 0.0048012924 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 33792/60000 / Loss: 0.0032168333 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 34304/60000 / Loss: 0.004059914 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 34816/60000 / Loss: 0.003468529 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 35328/60000 / Loss: 0.003639859 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 35840/60000 / Loss: 0.005412139 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 36352/60000 / Loss: 0.004661408 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 36864/60000 / Loss: 0.0036437165 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 37376/60000 / Loss: 0.0036653038 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 37888/60000 / Loss: 0.004240236 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 38400/60000 / Loss: 0.0055977786 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 38912/60000 / Loss: 0.005522794 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.973\n",
      "Epoch: 5 / Batch: 39424/60000 / Loss: 0.0026906629 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 39936/60000 / Loss: 0.0057889903 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 40448/60000 / Loss: 0.0032840215 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 40960/60000 / Loss: 0.005769772 / Training Accuracy: 0.9628906 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 41472/60000 / Loss: 0.0032621883 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 41984/60000 / Loss: 0.004014959 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 42496/60000 / Loss: 0.0032085418 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 43008/60000 / Loss: 0.0045540477 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 43520/60000 / Loss: 0.0038007367 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 44032/60000 / Loss: 0.003359766 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 44544/60000 / Loss: 0.0049249735 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 45056/60000 / Loss: 0.003987991 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.974\n",
      "Epoch: 5 / Batch: 45568/60000 / Loss: 0.0044816495 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.974\n",
      "Epoch: 5 / Batch: 46080/60000 / Loss: 0.0035280534 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 5 / Batch: 46592/60000 / Loss: 0.0039730794 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 5 / Batch: 47104/60000 / Loss: 0.0039336914 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 47616/60000 / Loss: 0.0036648016 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 48128/60000 / Loss: 0.003847917 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 48640/60000 / Loss: 0.0041485084 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 49152/60000 / Loss: 0.0036061672 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 49664/60000 / Loss: 0.0027875795 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 50176/60000 / Loss: 0.0035406977 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 50688/60000 / Loss: 0.0049008485 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 51200/60000 / Loss: 0.0043153884 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 51712/60000 / Loss: 0.0026995633 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 52224/60000 / Loss: 0.0038820393 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 52736/60000 / Loss: 0.0045028436 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 53248/60000 / Loss: 0.0029367437 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 53760/60000 / Loss: 0.0020038947 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 54272/60000 / Loss: 0.0039504766 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 54784/60000 / Loss: 0.0028756922 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 55296/60000 / Loss: 0.003507198 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 55808/60000 / Loss: 0.004052355 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 56320/60000 / Loss: 0.004472633 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.973\n",
      "Epoch: 5 / Batch: 56832/60000 / Loss: 0.0035640053 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 5 / Batch: 57344/60000 / Loss: 0.0029635585 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 57856/60000 / Loss: 0.004308308 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.973\n",
      "Epoch: 5 / Batch: 58368/60000 / Loss: 0.0041590417 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 58880/60000 / Loss: 0.0040668477 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.972\n",
      "Epoch: 5 / Batch: 59392/60000 / Loss: 0.0025435793 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.971\n",
      "Epoch: 5 / Batch: 59904/60000 / Loss: 0.0011697771 / Training Accuracy: 1.0 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 0/60000 / Loss: 0.0020753827 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 512/60000 / Loss: 0.0024169025 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 1024/60000 / Loss: 0.004078888 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 1536/60000 / Loss: 0.003350538 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 2048/60000 / Loss: 0.0040807175 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 2560/60000 / Loss: 0.003510955 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 3072/60000 / Loss: 0.0024231053 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 3584/60000 / Loss: 0.0026189447 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 4096/60000 / Loss: 0.0031824633 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 4608/60000 / Loss: 0.0035635543 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 5120/60000 / Loss: 0.0030446576 / Training Accuracy: 0.984375 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 5632/60000 / Loss: 0.0038022317 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 6144/60000 / Loss: 0.0035947456 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 6656/60000 / Loss: 0.003990545 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 7168/60000 / Loss: 0.0033742427 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 7680/60000 / Loss: 0.003972269 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 8192/60000 / Loss: 0.0031963047 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 8704/60000 / Loss: 0.002935752 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 9216/60000 / Loss: 0.0030062902 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 9728/60000 / Loss: 0.0035258397 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 10240/60000 / Loss: 0.0021291473 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 10752/60000 / Loss: 0.0019010047 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 11264/60000 / Loss: 0.0041798716 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 11776/60000 / Loss: 0.0032657098 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 12288/60000 / Loss: 0.0024586413 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.97\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 / Batch: 12800/60000 / Loss: 0.002709229 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 13312/60000 / Loss: 0.0046110367 / Training Accuracy: 0.96875 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 13824/60000 / Loss: 0.0031481553 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 14336/60000 / Loss: 0.004691325 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 14848/60000 / Loss: 0.0025415842 / Training Accuracy: 0.984375 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 15360/60000 / Loss: 0.004067072 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 15872/60000 / Loss: 0.0032108028 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 16384/60000 / Loss: 0.0038362953 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 16896/60000 / Loss: 0.0034227888 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 17408/60000 / Loss: 0.0017424576 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 17920/60000 / Loss: 0.004606976 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 18432/60000 / Loss: 0.002807765 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 18944/60000 / Loss: 0.0036289922 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 19456/60000 / Loss: 0.0039930227 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 19968/60000 / Loss: 0.0023974744 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 20480/60000 / Loss: 0.0028340034 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 20992/60000 / Loss: 0.0035592467 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 21504/60000 / Loss: 0.003198902 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 22016/60000 / Loss: 0.004472845 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 22528/60000 / Loss: 0.0034380038 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 23040/60000 / Loss: 0.0034788332 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 23552/60000 / Loss: 0.0027383775 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 24064/60000 / Loss: 0.0035258655 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 24576/60000 / Loss: 0.002719957 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 25088/60000 / Loss: 0.0029795016 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 25600/60000 / Loss: 0.0029788525 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 26112/60000 / Loss: 0.0026949414 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 26624/60000 / Loss: 0.0033157095 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 27136/60000 / Loss: 0.003584079 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 27648/60000 / Loss: 0.0030076965 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 28160/60000 / Loss: 0.004053115 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 28672/60000 / Loss: 0.0028778685 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 29184/60000 / Loss: 0.002399228 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 29696/60000 / Loss: 0.0042627407 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 30208/60000 / Loss: 0.0051317858 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 30720/60000 / Loss: 0.0025919282 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 31232/60000 / Loss: 0.0032569193 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 31744/60000 / Loss: 0.0027218168 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n",
      "Epoch: 6 / Batch: 32256/60000 / Loss: 0.004000586 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.978\n",
      "Epoch: 6 / Batch: 32768/60000 / Loss: 0.0037726625 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.979\n",
      "Epoch: 6 / Batch: 33280/60000 / Loss: 0.0032709553 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.978\n",
      "Epoch: 6 / Batch: 33792/60000 / Loss: 0.0025967907 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.979\n",
      "Epoch: 6 / Batch: 34304/60000 / Loss: 0.0029667134 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.978\n",
      "Epoch: 6 / Batch: 34816/60000 / Loss: 0.004399319 / Training Accuracy: 0.96875 / Validation Accuracy: 0.978\n",
      "Epoch: 6 / Batch: 35328/60000 / Loss: 0.0032041047 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.978\n",
      "Epoch: 6 / Batch: 35840/60000 / Loss: 0.0030979663 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 6 / Batch: 36352/60000 / Loss: 0.0017579533 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 36864/60000 / Loss: 0.003611913 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 37376/60000 / Loss: 0.0024902725 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 37888/60000 / Loss: 0.0030123217 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 38400/60000 / Loss: 0.002509437 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 38912/60000 / Loss: 0.0027940092 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 39424/60000 / Loss: 0.0023844433 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 39936/60000 / Loss: 0.00379448 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 40448/60000 / Loss: 0.002512808 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 40960/60000 / Loss: 0.0055234474 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 41472/60000 / Loss: 0.0027748265 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 41984/60000 / Loss: 0.0047501978 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 42496/60000 / Loss: 0.004700244 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 43008/60000 / Loss: 0.0039552925 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 43520/60000 / Loss: 0.00389807 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 44032/60000 / Loss: 0.0041007893 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 44544/60000 / Loss: 0.003915033 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 45056/60000 / Loss: 0.004358707 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 45568/60000 / Loss: 0.0033910207 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 46080/60000 / Loss: 0.0024859577 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 46592/60000 / Loss: 0.0021558423 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 47104/60000 / Loss: 0.0034887902 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 47616/60000 / Loss: 0.0039794818 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 48128/60000 / Loss: 0.003151133 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 48640/60000 / Loss: 0.0036300526 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.977\n",
      "Epoch: 6 / Batch: 49152/60000 / Loss: 0.0024583067 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 49664/60000 / Loss: 0.0034570848 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 50176/60000 / Loss: 0.00413374 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 50688/60000 / Loss: 0.0027676476 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 51200/60000 / Loss: 0.004411106 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 51712/60000 / Loss: 0.002249256 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 52224/60000 / Loss: 0.003383788 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 52736/60000 / Loss: 0.003475146 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 53248/60000 / Loss: 0.0015327291 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 53760/60000 / Loss: 0.0032072514 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 54272/60000 / Loss: 0.0026943947 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 54784/60000 / Loss: 0.002571383 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 55296/60000 / Loss: 0.004433432 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 55808/60000 / Loss: 0.0035482817 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 56320/60000 / Loss: 0.0020240366 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 56832/60000 / Loss: 0.00301954 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 57344/60000 / Loss: 0.004071024 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 57856/60000 / Loss: 0.0029251776 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 58368/60000 / Loss: 0.0050197076 / Training Accuracy: 0.9667969 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 58880/60000 / Loss: 0.0022553224 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.979\n",
      "Epoch: 6 / Batch: 59392/60000 / Loss: 0.002924358 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 6 / Batch: 59904/60000 / Loss: 0.0014416494 / Training Accuracy: 1.0 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 0/60000 / Loss: 0.0028563174 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 512/60000 / Loss: 0.001686102 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 1024/60000 / Loss: 0.003381522 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 1536/60000 / Loss: 0.0039754543 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 2048/60000 / Loss: 0.003749929 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 2560/60000 / Loss: 0.004314534 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 3072/60000 / Loss: 0.0023256917 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 3584/60000 / Loss: 0.0016118679 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 4096/60000 / Loss: 0.002667218 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 4608/60000 / Loss: 0.0016612075 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 5120/60000 / Loss: 0.0030186924 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 5632/60000 / Loss: 0.0030823438 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 6144/60000 / Loss: 0.003375947 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 6656/60000 / Loss: 0.00484217 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 7168/60000 / Loss: 0.0020587775 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 7680/60000 / Loss: 0.0030955046 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 8192/60000 / Loss: 0.0035817765 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 8704/60000 / Loss: 0.002318425 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 9216/60000 / Loss: 0.0035335342 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 9728/60000 / Loss: 0.0032362286 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 10240/60000 / Loss: 0.002105372 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 10752/60000 / Loss: 0.0036899273 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 11264/60000 / Loss: 0.003457085 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 11776/60000 / Loss: 0.0025701735 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 12288/60000 / Loss: 0.0026917076 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 12800/60000 / Loss: 0.003624798 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 13312/60000 / Loss: 0.0014797449 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 13824/60000 / Loss: 0.003515534 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 14336/60000 / Loss: 0.0027690325 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 14848/60000 / Loss: 0.0039277854 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 15360/60000 / Loss: 0.0033940107 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 15872/60000 / Loss: 0.0025341061 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 16384/60000 / Loss: 0.0024001363 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 16896/60000 / Loss: 0.003870767 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 17408/60000 / Loss: 0.0021193223 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 17920/60000 / Loss: 0.0030640787 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 18432/60000 / Loss: 0.0031841297 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 18944/60000 / Loss: 0.0027615956 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 19456/60000 / Loss: 0.0017470734 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 19968/60000 / Loss: 0.0022669607 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 20480/60000 / Loss: 0.0031154645 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 20992/60000 / Loss: 0.0025060214 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 21504/60000 / Loss: 0.0038420416 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 22016/60000 / Loss: 0.0018849038 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 22528/60000 / Loss: 0.0017249167 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 23040/60000 / Loss: 0.0025584423 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 23552/60000 / Loss: 0.0025038957 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 24064/60000 / Loss: 0.0022612729 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 24576/60000 / Loss: 0.0013232374 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 25088/60000 / Loss: 0.0019177751 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 25600/60000 / Loss: 0.002499324 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 26112/60000 / Loss: 0.001696509 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 26624/60000 / Loss: 0.0024318616 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 27136/60000 / Loss: 0.0038036301 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 27648/60000 / Loss: 0.0017167164 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 28160/60000 / Loss: 0.003471069 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 28672/60000 / Loss: 0.0024292446 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 29184/60000 / Loss: 0.003946168 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 29696/60000 / Loss: 0.0028389783 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 30208/60000 / Loss: 0.002416387 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 30720/60000 / Loss: 0.0037232835 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 31232/60000 / Loss: 0.004172647 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 31744/60000 / Loss: 0.0036758326 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 32256/60000 / Loss: 0.0032501959 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 32768/60000 / Loss: 0.0034203574 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 33280/60000 / Loss: 0.0028130882 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 33792/60000 / Loss: 0.0038053144 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 34304/60000 / Loss: 0.0029608575 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 34816/60000 / Loss: 0.0016678115 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 35328/60000 / Loss: 0.003674578 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 35840/60000 / Loss: 0.004489782 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 36352/60000 / Loss: 0.0035199032 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 36864/60000 / Loss: 0.0023389757 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 37376/60000 / Loss: 0.002477438 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 37888/60000 / Loss: 0.0024233444 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 7 / Batch: 38400/60000 / Loss: 0.003129155 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 38912/60000 / Loss: 0.0020483932 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 39424/60000 / Loss: 0.002857883 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 39936/60000 / Loss: 0.0045945635 / Training Accuracy: 0.9707031 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 40448/60000 / Loss: 0.002736377 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 40960/60000 / Loss: 0.0050546103 / Training Accuracy: 0.96484375 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 41472/60000 / Loss: 0.003264803 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 41984/60000 / Loss: 0.0031346537 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 42496/60000 / Loss: 0.0026223394 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 43008/60000 / Loss: 0.0022676487 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 43520/60000 / Loss: 0.0029825722 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 44032/60000 / Loss: 0.00211975 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 44544/60000 / Loss: 0.002428237 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 45056/60000 / Loss: 0.0015421418 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 45568/60000 / Loss: 0.0019855439 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 46080/60000 / Loss: 0.0040351218 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 46592/60000 / Loss: 0.0026564335 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 47104/60000 / Loss: 0.0021272437 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.979\n",
      "Epoch: 7 / Batch: 47616/60000 / Loss: 0.003383812 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 48128/60000 / Loss: 0.0035954104 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 48640/60000 / Loss: 0.002674259 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 49152/60000 / Loss: 0.0020580604 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 49664/60000 / Loss: 0.0023363132 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 50176/60000 / Loss: 0.0020266678 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 50688/60000 / Loss: 0.0031711874 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 51200/60000 / Loss: 0.002131352 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 51712/60000 / Loss: 0.003980573 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 52224/60000 / Loss: 0.0015063996 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 52736/60000 / Loss: 0.002847535 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 53248/60000 / Loss: 0.0029378945 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 53760/60000 / Loss: 0.00239871 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 54272/60000 / Loss: 0.0031130186 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 54784/60000 / Loss: 0.0025799812 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 55296/60000 / Loss: 0.002581635 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 55808/60000 / Loss: 0.0013053404 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 56320/60000 / Loss: 0.002786566 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 56832/60000 / Loss: 0.0023006839 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 57344/60000 / Loss: 0.0035793863 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 57856/60000 / Loss: 0.0037504197 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 58368/60000 / Loss: 0.0030455221 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 58880/60000 / Loss: 0.002962074 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 59392/60000 / Loss: 0.0019808998 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 59904/60000 / Loss: 0.0029405246 / Training Accuracy: 0.9895833 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 0/60000 / Loss: 0.0036701863 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 512/60000 / Loss: 0.0028941163 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 1024/60000 / Loss: 0.0029640978 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 1536/60000 / Loss: 0.002792313 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 2048/60000 / Loss: 0.002347065 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 2560/60000 / Loss: 0.0029291916 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 3072/60000 / Loss: 0.00185888 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 3584/60000 / Loss: 0.0021400014 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 4096/60000 / Loss: 0.003297308 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 4608/60000 / Loss: 0.0032899748 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 5120/60000 / Loss: 0.002994785 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 5632/60000 / Loss: 0.0025918416 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 6144/60000 / Loss: 0.003995147 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 6656/60000 / Loss: 0.002163161 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 7168/60000 / Loss: 0.0017075915 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 7680/60000 / Loss: 0.0015775819 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 8192/60000 / Loss: 0.0029909543 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 8704/60000 / Loss: 0.0038468582 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 9216/60000 / Loss: 0.0027022355 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 9728/60000 / Loss: 0.00178685 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 10240/60000 / Loss: 0.0033877823 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 10752/60000 / Loss: 0.0025715732 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 11264/60000 / Loss: 0.0017137404 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 11776/60000 / Loss: 0.0016448591 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 12288/60000 / Loss: 0.0025330535 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 12800/60000 / Loss: 0.0031158314 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 13312/60000 / Loss: 0.003075558 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 13824/60000 / Loss: 0.0023039822 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 14336/60000 / Loss: 0.0019724744 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 14848/60000 / Loss: 0.001629032 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 15360/60000 / Loss: 0.0013508922 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 15872/60000 / Loss: 0.002852985 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 16384/60000 / Loss: 0.002406604 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 16896/60000 / Loss: 0.0015589062 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 17408/60000 / Loss: 0.0028337375 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 17920/60000 / Loss: 0.002440557 / Training Accuracy: 0.984375 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 18432/60000 / Loss: 0.0020079454 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 18944/60000 / Loss: 0.0018509362 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 19456/60000 / Loss: 0.0024806603 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 19968/60000 / Loss: 0.001259186 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 20480/60000 / Loss: 0.0018933152 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 20992/60000 / Loss: 0.0030065621 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 21504/60000 / Loss: 0.0013343097 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 22016/60000 / Loss: 0.002834355 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 22528/60000 / Loss: 0.002079925 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 23040/60000 / Loss: 0.0013501013 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 23552/60000 / Loss: 0.0022182318 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 8 / Batch: 24064/60000 / Loss: 0.0018829845 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 24576/60000 / Loss: 0.0010880785 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 25088/60000 / Loss: 0.002938812 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 25600/60000 / Loss: 0.0021891969 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 26112/60000 / Loss: 0.0025948114 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 26624/60000 / Loss: 0.0012638399 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 27136/60000 / Loss: 0.002203291 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 27648/60000 / Loss: 0.0034042194 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 28160/60000 / Loss: 0.0018378958 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 28672/60000 / Loss: 0.001991442 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 29184/60000 / Loss: 0.0021397069 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 29696/60000 / Loss: 0.0033857576 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 30208/60000 / Loss: 0.0022554924 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 30720/60000 / Loss: 0.0013443758 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 31232/60000 / Loss: 0.0032138675 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 31744/60000 / Loss: 0.004478653 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 32256/60000 / Loss: 0.0040705083 / Training Accuracy: 0.9746094 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 32768/60000 / Loss: 0.0030321013 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 33280/60000 / Loss: 0.001920162 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 33792/60000 / Loss: 0.0017723754 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 34304/60000 / Loss: 0.0019608357 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 34816/60000 / Loss: 0.0031697955 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 35328/60000 / Loss: 0.0024440926 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 35840/60000 / Loss: 0.001842799 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 36352/60000 / Loss: 0.0015224814 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 36864/60000 / Loss: 0.0027604098 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 37376/60000 / Loss: 0.0032282905 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 37888/60000 / Loss: 0.0041755917 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 38400/60000 / Loss: 0.0026855443 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 38912/60000 / Loss: 0.002661333 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 39424/60000 / Loss: 0.0017663892 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 39936/60000 / Loss: 0.0035379692 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 40448/60000 / Loss: 0.002490775 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 40960/60000 / Loss: 0.0026822984 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 41472/60000 / Loss: 0.0018596959 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 41984/60000 / Loss: 0.0023704811 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 42496/60000 / Loss: 0.0025778234 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 43008/60000 / Loss: 0.0021466871 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 43520/60000 / Loss: 0.0016240699 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 44032/60000 / Loss: 0.0011810759 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 44544/60000 / Loss: 0.002790685 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 45056/60000 / Loss: 0.001647418 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 45568/60000 / Loss: 0.0031338178 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 46080/60000 / Loss: 0.0021802294 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 46592/60000 / Loss: 0.0028959406 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 47104/60000 / Loss: 0.002238639 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 47616/60000 / Loss: 0.0017338298 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 48128/60000 / Loss: 0.0020980346 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 48640/60000 / Loss: 0.002190365 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 49152/60000 / Loss: 0.0015430321 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 49664/60000 / Loss: 0.0030823485 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 50176/60000 / Loss: 0.0019921816 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 50688/60000 / Loss: 0.0030123447 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 51200/60000 / Loss: 0.0033797182 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 51712/60000 / Loss: 0.0022282167 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 52224/60000 / Loss: 0.0022108916 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 52736/60000 / Loss: 0.0023837946 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 53248/60000 / Loss: 0.0016658105 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 53760/60000 / Loss: 0.0018662723 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 54272/60000 / Loss: 0.0015456695 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 54784/60000 / Loss: 0.0012165768 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 55296/60000 / Loss: 0.002575921 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 55808/60000 / Loss: 0.0021821915 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 56320/60000 / Loss: 0.0016924299 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 56832/60000 / Loss: 0.0033998315 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 57344/60000 / Loss: 0.0018844472 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 57856/60000 / Loss: 0.0030649067 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 58368/60000 / Loss: 0.00208764 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 58880/60000 / Loss: 0.002823392 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 59392/60000 / Loss: 0.0013684633 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 59904/60000 / Loss: 0.002187654 / Training Accuracy: 0.9791667 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 0/60000 / Loss: 0.002057039 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 512/60000 / Loss: 0.0021644442 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 1024/60000 / Loss: 0.0025815733 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 1536/60000 / Loss: 0.0026352368 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 2048/60000 / Loss: 0.0016096995 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 2560/60000 / Loss: 0.0028723043 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 / Batch: 3072/60000 / Loss: 0.0015402512 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 3584/60000 / Loss: 0.0018061867 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 4096/60000 / Loss: 0.0026345241 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 4608/60000 / Loss: 0.0018962396 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 5120/60000 / Loss: 0.0010639032 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 5632/60000 / Loss: 0.0022450476 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 6144/60000 / Loss: 0.0022853245 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 6656/60000 / Loss: 0.0019332251 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 7168/60000 / Loss: 0.0027895295 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 7680/60000 / Loss: 0.0015447071 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 8192/60000 / Loss: 0.0027434654 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 8704/60000 / Loss: 0.0014142368 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 9216/60000 / Loss: 0.002618368 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 9728/60000 / Loss: 0.0015640461 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 10240/60000 / Loss: 0.002427158 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 10752/60000 / Loss: 0.0015193295 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 11264/60000 / Loss: 0.001133263 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 11776/60000 / Loss: 0.0025207258 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 12288/60000 / Loss: 0.0023638112 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 12800/60000 / Loss: 0.0027849139 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 13312/60000 / Loss: 0.0013732944 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 13824/60000 / Loss: 0.0015958339 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 14336/60000 / Loss: 0.0022502982 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 14848/60000 / Loss: 0.0012793689 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 15360/60000 / Loss: 0.0016038042 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 15872/60000 / Loss: 0.0021181542 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 16384/60000 / Loss: 0.0021834446 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 16896/60000 / Loss: 0.0012257961 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 17408/60000 / Loss: 0.0016985291 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 17920/60000 / Loss: 0.0018588727 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 18432/60000 / Loss: 0.0015644245 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 18944/60000 / Loss: 0.0022175182 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 19456/60000 / Loss: 0.002961343 / Training Accuracy: 0.984375 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 19968/60000 / Loss: 0.0026515778 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 20480/60000 / Loss: 0.0018035717 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 20992/60000 / Loss: 0.0021668249 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.967\n",
      "Epoch: 9 / Batch: 21504/60000 / Loss: 0.0016342271 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.967\n",
      "Epoch: 9 / Batch: 22016/60000 / Loss: 0.0035923454 / Training Accuracy: 0.97265625 / Validation Accuracy: 0.967\n",
      "Epoch: 9 / Batch: 22528/60000 / Loss: 0.0026436124 / Training Accuracy: 0.984375 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 23040/60000 / Loss: 0.0026780684 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 23552/60000 / Loss: 0.0017928283 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 24064/60000 / Loss: 0.0013432148 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 24576/60000 / Loss: 0.0029216332 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 25088/60000 / Loss: 0.0017652208 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 25600/60000 / Loss: 0.0023587751 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 26112/60000 / Loss: 0.0018617952 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 26624/60000 / Loss: 0.0018301092 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 27136/60000 / Loss: 0.00075685454 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 27648/60000 / Loss: 0.0032894698 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 28160/60000 / Loss: 0.0020022315 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 28672/60000 / Loss: 0.0029474762 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 29184/60000 / Loss: 0.0019230142 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 29696/60000 / Loss: 0.001542782 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 30208/60000 / Loss: 0.0014394844 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 30720/60000 / Loss: 0.0016896181 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 31232/60000 / Loss: 0.0029273105 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 31744/60000 / Loss: 0.00093567616 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 32256/60000 / Loss: 0.0023227115 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 32768/60000 / Loss: 0.0032503232 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 33280/60000 / Loss: 0.001743334 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 33792/60000 / Loss: 0.0016957022 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 34304/60000 / Loss: 0.0018918492 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 34816/60000 / Loss: 0.0026688394 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 35328/60000 / Loss: 0.0028922218 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 35840/60000 / Loss: 0.0019396093 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 36352/60000 / Loss: 0.0017499328 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 36864/60000 / Loss: 0.0019264184 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 37376/60000 / Loss: 0.0014533928 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 37888/60000 / Loss: 0.0018898135 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 38400/60000 / Loss: 0.0027694409 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 38912/60000 / Loss: 0.0022344163 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 39424/60000 / Loss: 0.002606939 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 39936/60000 / Loss: 0.0030368657 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 40448/60000 / Loss: 0.0018961467 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 40960/60000 / Loss: 0.0020383527 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 41472/60000 / Loss: 0.0016182316 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 41984/60000 / Loss: 0.0023160535 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 42496/60000 / Loss: 0.0026172823 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 43008/60000 / Loss: 0.0027896725 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 43520/60000 / Loss: 0.0030552277 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 44032/60000 / Loss: 0.001892545 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 44544/60000 / Loss: 0.002364639 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 45056/60000 / Loss: 0.0013080545 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 45568/60000 / Loss: 0.0017799579 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 46080/60000 / Loss: 0.00127705 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 46592/60000 / Loss: 0.002675467 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 47104/60000 / Loss: 0.002259656 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 47616/60000 / Loss: 0.0028821987 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 48128/60000 / Loss: 0.0009616666 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 48640/60000 / Loss: 0.0020171986 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 49152/60000 / Loss: 0.0023328133 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 49664/60000 / Loss: 0.002096187 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.973\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 9 / Batch: 50176/60000 / Loss: 0.0014701815 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 50688/60000 / Loss: 0.0023837448 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 51200/60000 / Loss: 0.0022161533 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 51712/60000 / Loss: 0.0014857627 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 52224/60000 / Loss: 0.0008291969 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 52736/60000 / Loss: 0.0030863634 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 53248/60000 / Loss: 0.0011743745 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 53760/60000 / Loss: 0.0008619397 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 54272/60000 / Loss: 0.002292648 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 54784/60000 / Loss: 0.0015514499 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 55296/60000 / Loss: 0.0016171817 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 55808/60000 / Loss: 0.0018344515 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.982\n",
      "Epoch: 9 / Batch: 56320/60000 / Loss: 0.0021349557 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.981\n",
      "Epoch: 9 / Batch: 56832/60000 / Loss: 0.00086627045 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 57344/60000 / Loss: 0.0015620065 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 57856/60000 / Loss: 0.003560152 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 58368/60000 / Loss: 0.0040557175 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 58880/60000 / Loss: 0.0015257165 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 59392/60000 / Loss: 0.002694393 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 59904/60000 / Loss: 0.002073904 / Training Accuracy: 0.9895833 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 0/60000 / Loss: 0.0017667748 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 512/60000 / Loss: 0.001608308 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 1024/60000 / Loss: 0.0016481455 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 1536/60000 / Loss: 0.0013046926 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 2048/60000 / Loss: 0.0021571382 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 2560/60000 / Loss: 0.0029543848 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 3072/60000 / Loss: 0.0017289236 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 3584/60000 / Loss: 0.0011618552 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 4096/60000 / Loss: 0.0020978195 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 4608/60000 / Loss: 0.002587077 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 5120/60000 / Loss: 0.0012736786 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 5632/60000 / Loss: 0.0017369833 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 6144/60000 / Loss: 0.0015503329 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 6656/60000 / Loss: 0.0025318207 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 7168/60000 / Loss: 0.0015744155 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 7680/60000 / Loss: 0.002025383 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 8192/60000 / Loss: 0.0010890396 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 8704/60000 / Loss: 0.0015872193 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 9216/60000 / Loss: 0.0018077228 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 9728/60000 / Loss: 0.0021031764 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 10240/60000 / Loss: 0.00082007627 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 10752/60000 / Loss: 0.0028784026 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 11264/60000 / Loss: 0.0013588503 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 11776/60000 / Loss: 0.001969093 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 12288/60000 / Loss: 0.0018170562 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 12800/60000 / Loss: 0.0016804121 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 13312/60000 / Loss: 0.0021214487 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 13824/60000 / Loss: 0.002735971 / Training Accuracy: 0.9824219 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 14336/60000 / Loss: 0.0026715233 / Training Accuracy: 0.98046875 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 14848/60000 / Loss: 0.0009591818 / Training Accuracy: 0.9980469 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 15360/60000 / Loss: 0.0019647598 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 15872/60000 / Loss: 0.0022441738 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 16384/60000 / Loss: 0.0024547805 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 16896/60000 / Loss: 0.0011993095 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 17408/60000 / Loss: 0.002328992 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 17920/60000 / Loss: 0.0019816733 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 18432/60000 / Loss: 0.0018665392 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 18944/60000 / Loss: 0.0010854657 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 19456/60000 / Loss: 0.0012048718 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 19968/60000 / Loss: 0.0016165692 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 20480/60000 / Loss: 0.0018112296 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 20992/60000 / Loss: 0.0009785251 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 21504/60000 / Loss: 0.00224035 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 22016/60000 / Loss: 0.0016134847 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 22528/60000 / Loss: 0.0014681556 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 23040/60000 / Loss: 0.0011474459 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 23552/60000 / Loss: 0.0011376545 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 24064/60000 / Loss: 0.0026646892 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 24576/60000 / Loss: 0.001619008 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 25088/60000 / Loss: 0.00213603 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 25600/60000 / Loss: 0.0012304184 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 26112/60000 / Loss: 0.0018714961 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 26624/60000 / Loss: 0.0013674775 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 27136/60000 / Loss: 0.0020152046 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 27648/60000 / Loss: 0.0027621822 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 28160/60000 / Loss: 0.0008382773 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 28672/60000 / Loss: 0.001743714 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 29184/60000 / Loss: 0.0025339627 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 29696/60000 / Loss: 0.0022306836 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 30208/60000 / Loss: 0.0016200558 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 30720/60000 / Loss: 0.0017883785 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.978\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 / Batch: 31232/60000 / Loss: 0.00305557 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 31744/60000 / Loss: 0.0010887687 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 32256/60000 / Loss: 0.0011172306 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 32768/60000 / Loss: 0.0017961016 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 33280/60000 / Loss: 0.0025853415 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 33792/60000 / Loss: 0.0018851564 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 34304/60000 / Loss: 0.0017144184 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 34816/60000 / Loss: 0.0012838539 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 35328/60000 / Loss: 0.0010039408 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 35840/60000 / Loss: 0.0012700745 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 36352/60000 / Loss: 0.0017462944 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 36864/60000 / Loss: 0.0018587068 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 37376/60000 / Loss: 0.0016520539 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 37888/60000 / Loss: 0.001428351 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 38400/60000 / Loss: 0.0015504174 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 38912/60000 / Loss: 0.0018514475 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 39424/60000 / Loss: 0.0022570032 / Training Accuracy: 0.984375 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 39936/60000 / Loss: 0.0014129234 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 40448/60000 / Loss: 0.0020378744 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 40960/60000 / Loss: 0.002127322 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 41472/60000 / Loss: 0.0025620922 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 41984/60000 / Loss: 0.0014024021 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 42496/60000 / Loss: 0.0022554665 / Training Accuracy: 0.984375 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 43008/60000 / Loss: 0.0016538322 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 43520/60000 / Loss: 0.002624048 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 44032/60000 / Loss: 0.0021006176 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 44544/60000 / Loss: 0.0033840605 / Training Accuracy: 0.9765625 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 45056/60000 / Loss: 0.0025511265 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 45568/60000 / Loss: 0.0011653475 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 46080/60000 / Loss: 0.0024748575 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 46592/60000 / Loss: 0.0013054565 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 47104/60000 / Loss: 0.0015227336 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 47616/60000 / Loss: 0.0023368574 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 48128/60000 / Loss: 0.0008892018 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 48640/60000 / Loss: 0.0012555465 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 49152/60000 / Loss: 0.001627565 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 49664/60000 / Loss: 0.001679441 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 50176/60000 / Loss: 0.0012857241 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 50688/60000 / Loss: 0.0012221446 / Training Accuracy: 0.99609375 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 51200/60000 / Loss: 0.0014090607 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 51712/60000 / Loss: 0.0011151957 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 52224/60000 / Loss: 0.0015115872 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 52736/60000 / Loss: 0.0015257148 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 53248/60000 / Loss: 0.0015648438 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 53760/60000 / Loss: 0.001443369 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 54272/60000 / Loss: 0.00091110764 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 54784/60000 / Loss: 0.0019576456 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 55296/60000 / Loss: 0.0018191707 / Training Accuracy: 0.9902344 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 55808/60000 / Loss: 0.0015507775 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 56320/60000 / Loss: 0.0013579294 / Training Accuracy: 0.9941406 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 56832/60000 / Loss: 0.001803587 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 57344/60000 / Loss: 0.0015708532 / Training Accuracy: 0.9921875 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 57856/60000 / Loss: 0.0020059482 / Training Accuracy: 0.98828125 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 58368/60000 / Loss: 0.0022298885 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 58880/60000 / Loss: 0.0021142834 / Training Accuracy: 0.9863281 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 59392/60000 / Loss: 0.0029129963 / Training Accuracy: 0.9785156 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 59904/60000 / Loss: 0.0004995836 / Training Accuracy: 1.0 / Validation Accuracy: 0.981\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "data = np.insert(train_x, data_size, train_y, axis = 1)\n",
    "len_data = data.shape[0]\n",
    "\n",
    "for i in range(num_epoch):\n",
    "    # Shuffle the data \n",
    "    np.random.shuffle(data)\n",
    "    \n",
    "    # Split data to input can label\n",
    "    data_x = data[:, : data_size]\n",
    "    data_y = data[:, data_size]\n",
    "    \n",
    "    # Make label data as one-hot\n",
    "    data_y_onehot = np.zeros([data_y.shape[0], num_label])\n",
    "    for j in range(data_y.shape[0]):\n",
    "        data_y_onehot[j, int(data_y[j])] = 1\n",
    "    \n",
    "    data_y_onehot_val = np.zeros([validation_y.shape[0], num_label])\n",
    "    for j in range(validation_y.shape[0]):\n",
    "        data_y_onehot_val[j, int(validation_y[j])] = 1\n",
    "    \n",
    "    # Making mini-batch\n",
    "    for j in range(0, len_data, batch_size):\n",
    "        if j + batch_size < len_data:\n",
    "            data_x_in = data_x[j : j + batch_size, :]\n",
    "            data_y_in = data_y_onehot[j : j + batch_size, :]\n",
    "        else:\n",
    "            data_x_in = data_x[j : len_data, :]\n",
    "            data_y_in = data_y_onehot[j : len_data, :]\n",
    "\n",
    "        # Run Optimizer!\n",
    "        optimizer.run(feed_dict = {x_input: data_x_in, y_target: data_y_in, keep_prob: 0.5})\n",
    "        \n",
    "        \n",
    "        loss, acc = sess.run([Loss, accuracy], feed_dict = {x_input: data_x_in, y_target: data_y_in, keep_prob: 1})\n",
    "        val_loss, val_acc = sess.run([Loss, accuracy], feed_dict = {x_input: validation_x, y_target: data_y_onehot_val, keep_prob: 1})\n",
    "        \n",
    "        print(\"Epoch: \" + str(i+1) + ' / ' + \"Batch: \" + str(j) + '/' + str(len_data) + ' / ' + \"Loss: \" + str(loss) + ' / ' + \\\n",
    "              \"Training Accuracy: \" + str(acc) + ' / ' + \"Validation Accuracy: \" + str(val_acc))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.9774444444444444\n"
     ]
    }
   ],
   "source": [
    "# Testing\n",
    "test_y_onehot = np.zeros([test_y.shape[0], num_label])\n",
    "for i in range(test_y.shape[0]):\n",
    "    test_y_onehot[i, int(test_y[i])] = 1\n",
    "\n",
    "test_result = output.eval(feed_dict = {x_input: test_x, keep_prob: 1})\n",
    "\n",
    "count_correct = 0\n",
    "for i in range(test_result.shape[0]):\n",
    "    prediction_y = np.argmax(test_result[i,:])\n",
    "    \n",
    "    if prediction_y == test_y[i]:\n",
    "        count_correct += 1\n",
    "\n",
    "test_acc = count_correct / test_result.shape[0]\n",
    "\n",
    "print(\"Test Accuracy: \" + str(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}