4_Making_Own_LSTM_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Making Own LSTM\n",
    "\n",
    "This is jupyter notebook for LSTM making. It will not use `tf.contrib.rnn.BasicLSTMCell` and `tf.nn.dynamic_rnn`, etc.\n",
    "This LSTM will be tested by `MNIST dataset`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import modules\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import MNIST data\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('MNIST_data', validation_size=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters \n",
    "\n",
    "img_size = 28\n",
    "img_flat_size = img_size * img_size\n",
    "\n",
    "# labels: 0 - 9\n",
    "num_label = 10\n",
    "\n",
    "num_epoch = 10\n",
    "\n",
    "# Parameters for optimizer\n",
    "learning_rate = 5e-4\n",
    "epsilon = 1e-8\n",
    "\n",
    "batch_size = 512\n",
    "\n",
    "# Parameter for LSTM\n",
    "lstm_size = 512\n",
    "step_size = img_size\n",
    "flatten_size = img_size\n",
    "\n",
    "validation_ratio = 0.1\n",
    "gpu_fraction = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFfCAYAAACfj30KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztnX2MvVtV3797Zs7L/Ob3+92bQnMx0lT02ihpNBZbSpRy\nW5po+QPxH4w1QWpMgy+NMakSElIQTIwaDI1KYxoLmlYTEm1RA1wr4gsioigqGiXQi6hwryAv9/cy\nZ86cmd0/Zta566xZa+99zpwzzzlnvp9k59nPPm/P85yZ71nP2mutnXLOIIQQ0g07XR8AIYRcZyjC\nhBDSIRRhQgjpEIowIYR0CEWYEEI6hCJMCCEdQhEmhJAOoQgTQkiHUIQJIaRD9ro+gJTS0wB8HYCP\nARh1ezSEELIUhgC+CMCjOee/Lz1xZSKcUvouAP8ZwDMA/DGA/5Rz/n3nqV8H4H+t6jgIIaRDvgXA\nz5WesBJ3RErpmwC8AcBrAHwVzkT40ZTS052nf2wVx0AIIWvAx2pPWJVP+HsB/FTO+Wdzzn8B4BUA\n7gP4Nue5dEEQQraVqr4tXYRTSj0AzwHwLhnLZ6Xafg3A85b9eYQQssmswhJ+OoBdAE+Y8Sdw5h8m\nhBByDkPUCCGkQ1Yhwp8GcALgITP+EIDHV/B5hBCysSxdhHPOxwA+AOCFMpZSSuf771325xFCyCaz\nqjjhHwPwlpTSBwC8H2fREjcAvGVFn0cIIRvJSkQ45/zW85jg1+HMDfFBAF+Xc/7UKj6PEEI2ldT1\nQp8ppX+GM/cFIYRsG8/JOf9h6QmMjiCEkA6hCBNCSIdQhAkhpEMowoQQ0iEUYUII6RCKMCGEdAhF\nmBBCOoQiTAghHUIRJoSQDqEIE0JIh1CECSGkQyjChBDSIRRhQgjpEIowIYR0CEWYEEI6hCJMCCEd\nQhEmhJAOoQgTQkiHUIQJIaRDKMKEENIhFGFCCOkQijAhhHQIRZgQQjqEIkwIIR1CESaEkA6hCBNC\nSIdQhAkhpEMowoQQ0iEUYUII6RCKMCGEdAhFmBBCOoQiTAghHUIRJoSQDqEIE0JIh1CECSGkQyjC\nhBDSIRRhQgjpEIowIYR0yF7XB0DIJpJScrfR2Lzv2/JYzjnc2rHSMeutdx6l/rx4x1U6j5btpkMR\nJhtLJAaXEYlWdnZ2pqKl+95+6zHNI+wpJZyens60nHO4L8e0s7NT7XvnELUW7I+DHFdpGz3mjW06\nFGGycZSssmVZbDV2d3enwmWbfax0/HbMWqVWyPX+yclJU5PjlSbHGI3pY9cC7e17RBaqiOjJycnM\n1htr2Z6cnCCltPEWMUWYbBStVuKqj8ETsr29PVfcvGOK9ksWtW2TyQTHx8eYTCYXmozL+8rxtTQt\nyrWtxoqh3dc/DJPJpNj3tpPJBDs7O5hMJtP3166XTYUiTDaCFtGNfJurOJaaqGlRrvlW9Zi2Nq0Q\n27HxeIzj4+PpVvf1NTg9PcXe3h56vV5Ts8df+qGRz/B8vHZfxFT/cNT6cl4i/FqAd3Z2ppb+JkMR\nJmtNze/r3a6XXrcMxAoUYbMCp/f39vYuHE/tx6PFHSAifHR0hKOjo2lfW9/AmQBrAe33+xfaYDCY\n2bdCXGrWHVDqn5ycuD8Yti/74/F4ehzywyLvJS6Jq/D/r5qli3BK6TUAXmOG/yLn/OxlfxbZbiLf\n6Tx+01WgXREibL1eb0bEpK9FuMWCj/zMXhuNRhiNRuj1ehcEWG7TxfrUPwyDwWDahsPhhX70Y+L1\ntQjXIh4mk8lUXOWHw9s/Ojqafs54PJ7+6Mj7yA9L5JPeNFZlCX8IwAsByH/CZEWfQ7aUmoiWZutX\nLcTWEhZh01albEWoSi4TvdUTY7X+4eHhjNXqWYraf6t/HIbDIYbDIfb39y9s5bjl+Xpr+/oa10LJ\njo+PMRqNpta7NDtmf1S88xJ/Ny3hmEnO+VMrem9yDbG38y1tVYgYyi25vq0Xa1K2/X5/5pijvnZF\nRBELNnpBC7AVK+1/tVa7HN/+/j5u3LiBGzduTPv7+/sz7gnrqrDN+oRLMcvj8XhqvY9GIxweHs7s\ny5hcV+0P1yFqk8mElnADX5pS+lsAIwC/C+BVOee/XtFnkS2jFElQclFclRB7lrC2LrWVKUIVWen2\neO1EWNREhMUC1pNwJycnU9+qiJm12rUIHxwc4ODgYNrXPyDadeGNaRGuJY2Mx2McHh7i/v37ODw8\nvNDkR6Vk2YtrRfvIN51ViPD7ALwcwF8C+AIArwXwWymlf5pzvreCzyPXkJLwrvqf0/MJa7/q/v7+\ntIlQ1dwmWoR1BIKNRoiiEwDMCJX4V+1rtSUsVvDBwQFu3rw5bfIj4ln29jHrEy71j46OcP/+/QtN\n/NAtAiyTdvbHZ5NZugjnnB9Vux9KKb0fwF8BeCmANy/788jyWOQPOnrNPOOtlq8ntFH21yr/Qff2\n9qYia2/l7Zi2FmsWuxX3Un9nZ2cmO07H0npRBvpHQv9QaEtYRFj8w/r5JREGZmN2vQy5nDP6/f6F\nH0gtsjpWWI7di19e9fd71aw8RC3n/PmU0ocBPLzqzyKzlMSs9vzS4zX/5qJ973PsWGuW2iqt4d3d\nXXdCy9sOBoOZcy25JvS5RNlt+lytL1oSG3R6r2CtXRFecUPI8WpXhA1Xs5Z3KR3ZttPT05kfB/0D\nYUPV9JhOQLHntulJGsLKRTildBNnAvyzq/4s8hQlofNEzr4ues+WmgLRc2qvLR2jZy3W+qsU4dpt\nuhazSHS9fulHxY6JL3owGOD4+HhqSUYiLEJsxVc3HbJmRdj+uMn71+pAyNaKcCTINmZYZ9TJ+W1L\nthywmjjhHwXwyzhzQXwhgB8AcAzg55f9WcQnuv2NxqLX27FWN4CXaFDqRyLs9b1JqshvukoRthNU\n0b4XolY619Zrl1KaWqvaUjw5OZkRJ3lv7XYQAdaREdYS1jHBNgLDuiBqxYOklUTYE2VrBesfmG0o\n3COswhJ+JoCfA/A0AJ8C8B4A/zLn/Pcr+CwSULNWS0Jsx2qugFb3QPRY6Rbd9nUWl83osvurFGEd\nc+s1HU+rr2HN6p/n7kE+w96uawGW672/vz8TAaG3VoglycQTYR0Wpt0RthCP1zyfdc094bkjtskK\nBlYzMffNy35PshjRP7MnfPZ13v48hV0890DkPihZ6nZcBFDEodRflQiLK6C1DoN3nVvcL6UxANNw\nMyu++rlahLX1W7KEJfRNC7D3XWlLuKU6Wk18o326I8hG4gmwt9XPr/VbY1hr7gIv5rXmNpEW1Wvw\n2ipFWFvcWvy9Vrqu0bVuIXJByDHq669D0qKoDhFh/X15SSIppQuTblp0bblJ2dei2iLGWoDtxJz9\n0dlkKMJbiCdm1rdYsoYjsbCiOs+2NNYy2adFuJRWqx/b3Z0ttbjM69saxyvHULrbsI/V0n91yJcV\nJH3HIK3X612IX9bNCnHJlaSP1Qpxra6xJ76lSnCeJczoCLJReP7F0j9V7bY5Ktc4b7OWY0mE7WNa\nfG0lMDu+ShG2/m0tXJ7P23sPbz8K8YrCvjwB1pa6zebTYmu30uwdU+TGinzBXt3gki+4JMrWH+y5\nXjYdivCWUhJgO8lS809KX4uu54u1Y3Y8qsZV+4fXfR2WZbd2bFUiDMR1f70ohuj78Whd0scTYOuC\n0CI8GAwuiK3XBoPBzHGXfqRL7gidPFITYSu8tYk5+oTJ2tPiipB/WPv80tYKrFfacJFm3SOeJSb9\nUilG21YpwvYal/q19xC8sC4rcF6Ilr5G2gXR7/engtYS0yxb63LQWztWEmAtxF6h9kiISxNz9gdp\nG6AIbxFaAPQ/ZMuEmH29bHW/JKTaF1sa9/y5kVXpjdl4XCsgenyVItxKznk6kQVg2tdjmmhiy76n\nvJf+QdXWob6rkLhmG0Jns+G8iTf5vCgbLhJab0yXq7QF6UuJGqXoiG2AIrxBtFis4rttCeOy4VNR\nX7aRiyEaK1nJejxyPXiCrP2++lxqPtiusGLrWZYaETt7G+6tIzfPUkFW2G1EwtHR0fQHTo6jJR25\ndGy2jUYj3Lt3b6ZJBTUpZSnCHAkx3RGkMzyRjCayrFBFKz/oMK7abbWIe2toVkn4rWjPMzFnoyOs\nJdfiBrhK5hEJG2VQ8qOWFvn0WlTkp9frTQVYWjQJ6O17roeoPxqN3CpqIsLWMmbGHFlLalbjzs6O\nWwc2qgsLxFlqdsyLipgngy0aK/2o2HOVW2trVVshXhcit0OEtoS9mgradxpZy17fCvB4PA7dSPo4\nvNTkKCzN+zy9lVKWYv3ausKj0Sgs3uP5hbfBCgYowhuDJ0h2Jl58g6XZcFsjIBJcr9XiYUsxs15f\nttHneccik07WurY+zXUR4nmFQoubFky7JNDR0dGMMNX68l6liBXZ1oTXWsLe53pjspSRFl27uoa2\nhL33ZJww6RTP6rUhZ3K7LkIbZUjduHFjWg+2VYi9LKqW/dp49LnecdhstXV3R8xjCZcE2C4DNBqN\nqskRusmP3vHxcTXOuyVEzlrCLckadnkjK77aHaGrwtnP2DYhpghvCCXx1VsRYbGEpXCLraAl2xZX\ngP1sm4xQ2q8lM4gI63OUvjcmIuxZ2Os4MTePAMtW11rQImyXA7Li56UKWxH2fgTtuBf/6wlyKVXZ\nO5bIoo9EOKpHsU2TcgBFeKPQYhhZljo4XyxhXdD71q1b074W4ZbUYRuB4GXfeVl5tcfk3Fq2+py9\n67CJlrAnwNZ/KyKsJ7SiYjle37te0daLV47GrEh6xyDb4+PjmfC0aLl78XeXYqa3RYABivBGEAmi\nZ9HYDCkR4Vu3buHWrVu4ffv2tH/z5k3X11yaGJu33zIm56jPtzQWifkm+oTt45ElLBbj/fv3p+Fd\nJZGyrfUHcWdnJxS9eRJJvP3JZOJmx3n9yWQSukCsn3rToQhvEN4/j01T9XzCYvnevn0bDzzwwHR7\n69atqgiXhFmOqdRanyPPs+cbXYPaD8Qm4AmIZwlbd4QnwpG7QPql79Tue+6GeQS/JNQt9SNkMs4L\nlbNtG6AIbwiRb9YKcMkSvn37Nm7fvo0HH3wQDz74IG7fvl0NebMWqxyL3raO1Z7vnXPpWtj3aXnP\ndcETEO2LtSFq1h1x9+7dYtSCZy22XLOUUtXKLbkISvst2XUyfnp6OnOd5kl42TQowhuEZ6V6VrH1\nC3ti/MADD+CBBx4oug2sC2HTWMY/afQelxH6SIBlq8O7tDtCRPjevXu4e/du0UK8zARWiwh7dRys\n+HqP1WKJpb8tAtsCRXiD8P7RvH8ML67Ss1rkPcQPKEJrLadNx1pOeuv5ZEvvIUTWZGT9t3yO/k69\nyTkbrtZyu74MEW6xfktWuPRL8b7b5mKYB4rwBhFZPFaI7Qy1FWP9Oi3AWpzENyj9TWUekfJueaP9\nFp+5vqYt7zuPCB8eHl74MWk5rxZKLoWS1VuyxEt/j9Ztct2gCG8I9h/Oiqj9A2/14wGYisU2ia9Q\nssw88ZDX6NfbMWC2nnDU19fTHlPUt3cyesLKxtbaY4725xW3kjuhdP0iy1Y/Xku6uI5CTBHeICIh\nTinh5ORkOqlSitn0rA/btkF8gfh6RRadfb59D9lG/nh5nRd2p98j6luxanVHeMfs9ee5bpEQtwpu\nJMI199h1hCK8YXh/2HL761nCNYtYo63gy1hS64QnAN6PU2RJen0RYZ00knOeqV8s19JawqW+tYR1\ntTMbL3x4eOgem90u8t2VhLYmvvr1djzyJW/T39siUIQ3gJJlYf2QrS4J/Xr7ftv0j2AtuSittlVQ\nhL29vem19IrHaxePvI8+Jm9M/1i0WsKl97Nj81yzeZt+nX0Pe352S0uYbBRWMLUQA7ggNCUxlvcB\nLlrB8lmbTmQJe8VhWgVHrrsUu9HXSb4L8dfLMXhbOzavNTyPwC963Wo/SN7fS2lb8iMveqybDkV4\ng4gsYe1WqEVGeO6IyA0hn7mp2H9+66bRMaott8hWhO21slln8wiUPs553BHee3jXYNHrZrfzno/3\n+pqwXzcowhtCJAjWr1uLEbZWoU5T1f7LbfmnsNfKipxeeaLVB+qJrJ6sq8361yzFVneEfV973pe5\nZtGxRmO1/VZB34a/uXmhCG8gnvgKJavXExrtzthGi6QkwFrorEui1LchfV4G4zxWoxWhFmtYW8It\n12AZ13Ge9235cViG1b4NUIQ3CO9WTotxztn1d5Ya8JQVV7sNXxXzhMTNcxw1C1jH4MrknDdxZMVY\n3zHozDgdrlayhqOtd5yRO+Lo6Kj5OpD1hiK8IdjbYBmTrVho+lbWW5lB6g8MBoMLywyVliDSEQAt\nfsB5qnYtej2i45G+1F7wml1qXeoVtAjxzo6/mKrXtytal7YnJyczxxWtNEy2C4rwBmKtsZolJaJz\neHg4XeRTlpovLcJp9z3LOBrTVqHe2rFFRLjl83POoQXp7UeTc5El7K1k7W31atK12sqnp6euCHsR\nHGR7oAhvENrK1P+I4tcVl8Lp6alrCR8eHl4Qh9py9Z4Il8KMpNkym9qqzjlPLcRFq7N57hJ7HPYH\nSN8N2EUmpXxizScsIiyLjerVir2+XYS01BcRPjo6mqmt600cku2BIryBRP+EdmbdirC9TdYiXFuF\nVy8AWQq4l619j16vN42tBbCwAEfC6x2DXhpIVqXQS63rJdfH43H1B8azhEut3+9fWIy0tCI1gKol\nTHfE9kER3jBqVlAkwiLEvV5vaoEBmFk+XguIt1+LuND72jdqi7XYWgvzuiTshFt0PNYSllUp9BJB\n0hcRbrH0tSUcNX39vLsLbx94SoS9lSZoCW8nFOENwc7G239EEQcrwtoa1LfG8j41S05uh3u9XnER\nRzs2GAwwGAwuCEhKaWr5XcaqsyLpHYu9E9CrUty9e3daHP3u3bs4Ojqqujf0xFzLj5Z3PaOxyWSC\nlNLMisN6Uo4ivL1QhDcMzy+srUkb6C8iZH2T8no7kWSFQYtxVHfBG7Nrhckx7+7uYjKZTN0bi14D\nawl7x6HLP4oPWIT3zp070+2dO3dwdHTkiq4nyGIJey4Xb0zuCGRSVLfj4+Npf3d3N4yOsFY+2R4o\nwhtI9E8oAqFFaDwez/gdtQCfnp5eCKvSwjCZTKbuBC3CNuHBG/MEWPugF/VveuLo/Sjo89eheeKG\nuHv3Lp588slp06UhS2Is19i6FDy3g4iw3BVIGw6H7p2CFmHritCWMNkuKMIbhLZ4S+4IGx2hBVie\nm/NZYodYaNoHqd0QVmC9tcGi9cJEZHXpRxF8LdCLXovIGpbP96JDxB1x584dPPnkk/j85z+Pz33u\nc8XlgiKfcC2qRERYVr6WNh6PMRwOL/h79/b2muKEKcTbBUV4wyj9A8pj2iesXRDyHO0zHQwGGI/H\n01tlEWQrrrVmnyciqwVYhGkwGFzKv1mygvXn24k5vUimFuDPfe5z0+WCalawtoSjZq3h/f39mWYj\nH+Qze73ejE9YZ/PRJ7y9UIQ3HPsPaSfmrA/YC2MT4ZXbY1tdTARAi7P0ozEdu6wF2IrPoudcsoRF\ngG2ImvUJixB/9rOfnYpwSxMRtskstu3u7qLf7+PGjRszwmrFF8DUjeS5I/QdCQV4+6AIbyGe2NpJ\nOXle6ZZeRKDf78/M1lvR9VqUCOJNXEmMrE1ltgXr9TF7Vca0S+X4+HgaB6wz42q3+brpz7PjEiVh\nr50OtxPBti4h7/qfnJyg1+vhzp0709C5w8PDmUgJCvF2QhHeQrzbdRFGKxLe82xxG5m8i1wT3riI\nsGct2kks+wNR2kbWvFcfQmKDxRK2mWj2Ft/zs5du/+31E2tWk1LC8fHx1N9rf1DEfy8RIzp+WY5b\n/3AwYWP7oAhvIS0CHD1PJuX07bwOWatNxsm21RLWt+5ROi+Amey6SIBtXQhrCddqMsg1iQTZ7lu3\nSJR0Ir55uwKK/eHb29ubxjJbS1j7hmkJbxcU4S3E3ubu7OxgMpm4j1shkNjgXq83FeCjo6MZS7gU\npraICItLIkrlBRDewosQ60pxNk1ZW8KRX9aKsO3b62uPxRNgeZ6eGAXi9PK9vb2p9W5/PLQlTBHe\nLijCW4oWqsgHLMK5t7c344bQFqqIw97enhuLGyVttLghZCvHYGtU6EkrXe84ygrUoWi2NkQUf+uJ\nsN62jMkxRc/3XBD6+HU9D/2DEbkjKMLbBUV4y7BCZf2UkQiLEEcz/pJm7GWmef15LGGZlJIiP6en\nZ7UnrPjacxAhs0v/iABHE3M24sC6I/TntF5r7zE5dj1mJz7ljkOusa51rLd0R2wvc4twSun5AL4P\nwHMAfAGAl+Scf8k853UAvh3AgwB+B8B35Jw/cvnDJS1YIfbGxFoVX6TnCtATa7u7uzORBFpwvTEr\n5qX0XpuSK1EF2i+sRdKbRIxE2E5w1dwR8hm2H7kldJSEvs7e4qnWAtY/cHt7Z1XtbISHtd45Mbd9\nLGIJHwD4IICfBvCL9sGU0isBfDeAlwH4GIAfBPBoSunLc87jxQ+VtOLdHutVN2xhdZk4sk0e1wV3\nbEiWHZMWWdSeCFv3A/CUH1WO27v91/HA3uohniXsuSO0gOprVrvGgvYJi/jqZe+9cEHvGovvvpYU\nQ0t4u5hbhHPO7wTwTgBI/nTw9wB4fc75V86f8zIATwB4CYC3Ln6opBUtVLqvQ70mk8m0oE7LKhha\nDK3gemNahGu1iuWYBVtpzVqrNp655o7QPmGvME4pFK0kePYxcf/YWOednZ3pj1+piesocvVY1wnZ\nDpbqE04pPQvAMwC8S8Zyzk+mlH4PwPNAEb4S9Iy9WGRe4oNdIViLtDeuhb221VlyVoyl2LlYxXbh\nTG0dirvCuiMkvtZOzOlJuZboiJJPuHaNdd+uF6f70dp6pWSU2vUl28OyJ+aeASDjzPLVPHH+GLki\nIssOuCgU3tpnXt9ajdba9kQ4qqegRViw4muz2fR52QkuW7LS+oNboyMWuc56G1EqXO/Fb0efQ7YP\nRkdcQ+w/tLaUxX+sfZtiTVsBLrXITWF9yp5f2b6XRv8o6GZ9zrpam5581C4WbYFe9TVvfYxsP8sW\n4ccBJAAPYdYafgjAHy35s8iKiERhHhHW7zPP66L30niFgUR0pU7v/v7+hR8CPcElFvdVCjEhHksV\n4ZzzYymlxwG8EMCfAEBK6TaA5wL4yWV+Flke8/hBS+JYeu9IkFusXz2mLWEtwnr1iuFwODPppl0X\nMpFnS3wS0hWLxAkfAHgYZxYvAHxxSukrAXwm5/zXAN4I4NUppY/gLETt9QD+BsDblnLEZKnoSSU7\n7o1ZS7clqsB7fuRLts/10JEb2g2hi9PbVGovNveqXRKEeCxiCX81gHfjbAIuA3jD+fjPAPi2nPOP\npJRuAPgpnCVr/DaAf5cZI7z2aNHTEQv6sUgoIwtW92uTeiWXhj6uyBLWNZFtGJut0UARJuvCInHC\nvwmgeA+Xc34tgNcudkikC7RFXJtE8lwMtefXJvBqPmXBC2Gzq0Pr+FodxhZVbZP35QQZ6QJGR5AZ\nItdCyTL2trXPmFeINZ4QSySEDjuTjDpJ0tDZet7EHIWYdAFFmFwQ2UiMPBG2ffv8RSIhSr7myB3h\nLRmkRfjo6GimWlk0MUchJlcNRZgA8N0RWpBK/t7W/VZ3RHR88pidmJNax7YA0MnJyVSAB4PBBSs4\n8glTiMlVQhEmU0oTc7Xnl57jWbgtE3OeMHuWsJTC1M+T+hiSRSchbNYSjibmKMTkqqAIE5dlCFDk\nQ/bEuNUiBi6mN9t6CrpI0Wg0wmAwmAqwZw0T0iUUYbIStMDaxUPFRSCRCtq9YKMeZCv1HgBc8P1a\ny1jGxuPxVIC1Jey1yWRy4YdC+naMkGVCESYrQQTYrqM2Go2mwmtdAbqimLV2RYxTSu4knJS/1EKp\nRVcLsde8lZettS7nRcgyoQiTlRCJsA0N89wI2qq1scBSm1eLpl4GSRYHTSldsIBLIiyWsFcjWY6R\nQkxWAUWYrIRIhL36xF5lNG81jn6/P7NShV49RF6v+y0WsBZhW+HNVpDjZB1ZBRRhshI8EbYuCBG8\nmgtCxFQsac9nq+sfixhLllyLNWwz7STETZ8PBZisAoowWQmeCFsBlscAXJiYsxNng8FgWvdBiPzJ\nQqsVrEVYln2yYWu25jIhy4IiTFaGFlodCqaL64g4lyzgwWAwTbjQPmC7PJAd8yzgyBqWCI5oqSG9\nojIhy4QiTFaCtoS1BSk1HUSAJeLB+oC1QEqh9vH4rBCfXYDUW2lD4oTnmZizFrCNY2a1NbIKKMJk\nJWgRtvt7e3vT2r4SKWEF2FrBsj6cPFfQxXe8NepaJuek/KW1gkV8rYVNyDKhCJOVoEXXCpq4J7Q1\na9eG06tkyCKe4/G4ukip+JNFhFt9wmJlW+tXzoGZdWRVUITJStAJDnIrL64JK5xahLX7YTAYYDQa\nYTgcTstR2tfa6AhtUYsIW+s6anLcWoRFgGkFk1VBESYrQSc4aJ+wxN3q23udyqyXrR8MBtOtiKie\n1PPSlqNUZqk5rC3s4XCIGzduYDweI+c8U/Rd15ew+xLRoc+ztt+6JdcPijC5MiKh0VEUWowPDw8v\nVD2TcLfBYDAjxiLoIpQ2nVmvQyerMeu16HZ3d6efLVZ3tC8i7FWI8/o6HVpn4zElmgAUYXIFaEG0\nYwAuxBNLjQkrwACmoWRe2rK4I8Rils/U4/1+H8PhcFoQSER8b2/PFV4rwuPxeGYl51qzvmVvX64B\nRfh6QhEmK0VEUvoeniXs1f0FcKFuBIALk3Keq8KuyGxFPBJhb0yLcMtWzk2vAG2z8hiDfH2hCJMr\nwYqxzjwTodKlLkej0YwAa0HzqqdFyxzZx8US1rUnZGKwJMLWHaEtWtvsxJ5Y77rZUDhO+l1fKMJk\nZWixLaX72prDR0dHzQJsXQ3aStXP2d3dvVCyEsDMYyVfsG7Hx8cX3Aulvvy46PoZ+hpRhK83FGFy\nJZQE2boj9AKcWoBF/LQfWFwQ/X5/xs+rIzDsWnRagGW8JLqeCGu3iHUz2DEpPGRD3SR70AozuV5Q\nhMlKKbn/igOWAAAgAElEQVQhrJUrlqJnAYuVnHOeyYyzk21exISIrRVwHVMsadHzinDk79UFgfQ5\nCXJeEvJGEb6+UITJyqlNztlCP5EAHx0dTQu3awGW1OaaJWxdEHrpJPFFtwjweDy+ILSlvrXsdf0M\nW9yIXD8owuRK0OJrBVmLbSTA4/EYvV4PAGaSL2xtCesT1qFrsi8CrEtYlkTYPiafo/3Y3uSbLuEp\n52sn7Gz0B7l+UITJlWOtYa/Qj0zW6UI/IqTaBTEcDrG/vz8jjjr+VixhaxXbiAbxR9cEWMbks7QA\n2zF5rT4vbQFHa+2R6wVFmHSOjhDwxFj7Tnu9Hg4PD6dtf38fo9HIbTZxQhAx1rUnRBDtis9SSF7X\nOPbE1tvqiUYrslEihz7OlhRnJnhsPhRhshZ4mWaehahdB7rOxP3792dqTOzt7WEwGMyUutTv5/W1\n68KuX2cflx+Emkvi+Ph4Kupa5HXdY1uQyKY4R2Ny3fQ19PpkvaEIk7XBCrAXP6vFTQvx/fv3ZwR4\nd3cXw+HQLfauV3TWReF3d3enIif+Z+DiKtAtE3Lan61FV7tGvJVBSkkgtsk1k60Xi00xXn8owqRz\nvOI32hLVQuIJ8OHh4QUB3tnZmboSxLVgt8BTk4RaJDVWtK3QRqFpevUQK/5a1K1FbmOMo3177bQA\n15JjyHpBESZrg73t9h73RNgTYCmRqX25ugFPxSnbovCCtYBlVZCaQFpRtuKrt/I5AKZV4qw1rd/H\nltH0fN7yGNkMKMJkLbACrBfW1I95IhwV+zk+Pp6GsUnKso0V1skbOolEi7IW2JY0ZRuG5lnCkW/a\n8yfLexwfH4fXq3ZtyfpCESZrgxVgLU6RCEcCnHPG8fExhsPhTMlKeT+bwGFD2XS8soh1S8EeL/TN\niq9tgljv0nZ3d4v1JvT18sSY4rsZUIRJ53hxw1oItaVqRTgSYF0UKBJg8a3q97dRCl6EghXB0tYu\nIOqJr0bOza4mLeel4421D9i7e6BfeDOgCJO1wvNtajGR8DA96eYJsPanagGWELN+v38hs64Wl2sn\nD0tN3lvqBltLODpvXUEuEmAJcxPBtXcPFODNgiJM1oJIfIHZFTl0Ft3R0dGMuNUE2KYr64I+IpQe\nkWhaUfb6IsLW7WDPW7YisPY1+tyk8ppG3z1oAaYYrz8UYbI2tIRY2dhbLVQ6IkEqnQEXBdjWmLCW\nqrymlODhHbe3b90RtXOPSl7alGcrwvrHxBNjsr5QhMlaEomHFll7i299psBTlqVNPe73+zg6Oppu\nbeyu3gK4MLkG+BayHRPfs/inS5N70efI9bATgDpqQ8cY29dRjNcbijDZOKzY6qQILV5S7rLmOz45\nOcFwOHRTi71+yUqOLGn9QyBhczq+WF7jJZZEzYsn1payTuqQc671ydVDESYbhbUIbSyuft7Ozk6T\nCE8mEwyHwxnxk6b3tTBqP7IW2ygJQ68CIkXkrQDrMpsl8ZUWldOUpA59rnbrWccU426gCJONQwuo\nCI4XSyuREJ742nrFw+FwKtg6s87uS9acl4KsXRfyWQBcS1jXf9DWcmuT5A0dU2zdNPp8o8lDfb1I\nN1CEycZhhVRnk+nHrGWqHxcXhsTlDofDafpzv9+faTJ2cnIyFVBdAEhn3wkixNpCFktYi6IWcW0B\nW2vYc43oGseS2BGdqw6fs9fSi9YgVwdFmGwUWkxs+JcVZ2B2FQ95jViMIsCj0QjD4RCDwWCmie9W\nuw50rK6O2RW0K0KLrC4mb+OW9VJL1hViJxb1mHa1eNEi+lpEiSe2hjG5euYW4ZTS8wF8H4DnAPgC\nAC/JOf+SevzNAL7VvOydOecXXeZACRFEPFIqF4HXz5fHo1KYskqHbXryTN/Oa0G1Imb9rTpJJBrX\n7o+SC0I37Rax/nAbU2xdEdoVol9Hrp5FLOEDAB8E8NMAfjF4zjsAvByAfMNHC3wOIRewImLHtIvC\nWnw1Ad7f3y8ulQQ8JVqnp6dTUbXCKhayHrMuC+0nluLw/X7/gouj1KxvWrAxxTq7Tq6Frc9BAe6O\nuUU45/xOAO8EgBRHoB/lnD91mQMjJMIrVCPCIgIsYlgSYO1yGA6HFxYMtRawCKe2LGVchNbe8mtB\nledqF4Qte1mzfksV2bykDnkPfa3kGjK7bj1YlU/4kZTSEwA+C+DXAbw65/yZFX0WuUZoEZF9m7Ir\nTURUC7BO0pAJt16vF1rAnv+2JMLaAteP68gJsUxtsoYnwiVBtv5uK8B6kVQhulYU4O5YhQi/A8Av\nAHgMwJcA+CEAb08pPS/zWyZLQP6MdHaYvimTvl1mqNfrTctf6vCzvb09jEajJgG2omajH7So6iw4\neY71zern6Ym+SHz1vjfpKOdr46Oj69iSUk1Wy9JFOOf8VrX7ZymlPwXwUQCPAHj3sj+PXE9sAoKH\n547Y29ubWog6AeP4+HhGEG0Chs2Y0xN2VnS97DlBJ3DYMW+STz/HHpN+jfWJ61RpqaMh7hJ5XF9L\na92Tq2PlIWo558dSSp8G8DAowuSK8UKybI2FlNK0PKaErHmTXvJ6cWvoJv5lL8bYS+zw+nIsWvDF\nWtfWuWdNC96PhvzIeKtBa6G2vnZyNaxchFNKzwTwNACfXPVnEaLxwrJ0WJtGEj5qheJPTk4wHo+r\nAixjvV6vKQPOs3R10SFb6Ed8uNb6tq4LEWHxd8vEpGx1tAdFuBsWiRM+wJlVK/dYX5xS+koAnzlv\nr8GZT/jx8+f9MIAPA3h0GQdMyDxYK9i6CuQ5Ozs7TYXiRai18FoRtvteIobsW7+xtYS9DDttOQMX\nBdhL8pCIEPmRkcpxcn7ivuAE3dWziCX81ThzK+Tz9obz8Z8B8J0AvgLAywA8COATOBPf/5JzPr74\nVoSsFivAXmysiJstFC+P2TTnkgjrsDcv9Vm20aoesi+WsLVQtUjrscgNIZOQdkFUe35ejWJyNSwS\nJ/ybAErf1tcvfjiELJ9SgodYwSkljMfjqgCPx+MLQtuyHQwGmEwmGAwGF4rJi9CKOOpIC7GWPSu5\nJL62frIIsD0/HU/MKIluYO0IstVYd0T0mIhZTYDnEV2dCCIRClaARTQ9365k2FmhFYH1hFk/rq3v\nkgBL5TVawt1AESZbiwiYTdXVwiuPyb68zhNgW2EtEl09NhgMLggwgBlBjWKS9b4VbR11Ebkg9PFG\nLgid0MGkjW6gCJOtRguKTdXVPmKdMKEFuNfrTSfsdIKHV2ktsoJLFrCUyLQiLGgr2C5t5FnHdgmn\n0WiEfr8fCrCNBiFXD0WYbD12Ak7HB+sQLW0BizDpiAkdseAJr7c/Ho+LAqwXHJVj0jHD8pi4Jmw5\nTR3KpovG6zYej6fnbwVYojfoE+4OijDZauyttRY7QcROF2sXP6mXNry3txeKrh0fj8cXXBBipeo0\naXtcIro6M0620vd8wGK5y+dLUSItwDoppSW9mawWijC5FkRirBHfq14uyW4llE2ETCxN3SSEbTwe\nT2NvRXxFgCNf8Ty1HLSrIoqIkOPzSnjq8DmvKDy5GijChJxjJ+esr1gsRV1/QS806iWBpJRmLFKd\nPiyvt6nIAJrEMMqSs5N9Oeep2NrJRW85J22Ze2nS+jjJ5aEIE6KwQmwfk60IsRVgnU4sIXEyQadT\nh3W1Nm+JIX0cJWz4mrzf3t7sv3ZJeK04TyYTt+aGrVVBIV4OFGFCDJG46OgKHaEgLgcd8qbXuZN0\nYXFXaCs4EmE5jpIQ21oTOrbYvp8V2pJVbM9DR5LoHxeK8HKgCBMSUBJjvYKHfr4NIwOA0Wg0447Q\nlrBdCdmKbkmIPVeEV9QnpeRWeIuanqiUHwr94+OJPFkcijAhCi0uUeKCFVk7Lm0ymQDAjBVsfcLW\nEl5EiG2GnYzLdmdnJ7SAPWsYwIz46rKfErFBEV4eFGFCzvH8wVaIRYSsCFs3hNRjyDnPWMLREkr2\nM+admJP3sGUxxUouia8VYuAsUkTEVyJGShY7WRyKMCEGGxlhx20lNivAOrY45zzjE/aiI1om5iLh\n02Jrx7SFXBNf3Qcw4+fW56l9wmQ5UIQJUWgBLgmx9LVlqFOJtQiPRqMZK1jig7U7wr63t2/xBBjA\n1FLXqc4t4it9m1loBdhGgpDLQREmxBAJsEZHCthkDi2Op6enRUvYW7Jonlt9/VlaGOUHQN6/1QqW\nIvKRAHvx0ORyUIQJcSgJsBZKb6v7ngjb6IgoRM1+lkXGtU9Y93Wbxx1hkzO0ADOrbvlQhAlpIIqa\naLkdj8TXhqfpz/KEzoq/92PgHVfOeSadOepL0z8M2s+tJ+ooxMuDIky2HluspzRmLVlv33ufaNvv\n9/HAAw/g1q1buHnzJm7cuIH9/X0Mh8OZCmZ2Yq3Wt9aq7nv73urK8qOgayZbl0mUWEJf8PKgCJOt\npkVUrR/X+li9sajZ9+33+7h9+zZu3749FeHhcHhhNWbvx6GG53aImhVeK8ZR01a7l7pMLg9FmFwL\nIgG1YuuVr/RaJNS2ry3hg4ODqSUsIuyt7iy0CrL229q+bFuE10Zw2AnEmv+aLAZFmGw1nuh6ZSq9\n5eJFHL0x77XevljC4o44ODhw3RE1AY4E2RNd7TawCSSREGvxLbkjrIVNLg9FmGwtkQBHTQTRbr0x\n+9porNfruT5h647Qq2l45+FhBdGmTWvr1QpwZBGLNazdEC3FhsjiUITJ1uMJsBVNu0SQLpDujev3\nsFazHuv1elNLuOSOiCIOai4JT4St+0CLsBZgXejdWsO2ZrK1sCnEy4MiTLYazxL2RNML1YrGPEs5\nar1eDzdv3py2aGLOW1poHp+wZwFrMZ7HEhYRtkJu/cxkOVCEydYTuSSsWErCQm3rCXHkwuj1ejg4\nOJi2WojaZaMjrAjr1T/m8QmXfMwU4OVCESZbS018PcHUa8DpymJ6P3JR2KXn5T1FePf392fcEfJ5\nJXdEDc8frC1gT4BLQixF5220hReBQZYDRZhsNVaIvYgHLZi11ZO1L1eLcak/HA4vNC9EbVG0QHoC\nrIXYRkB4Quyte+ftk+VAESZbjyfE0XLx2vIVsbRbsWKtzzja1yJuLWubrCHHG6VJ6/3IHVFzRZQy\n5iRBwxNZCu9qoAiTtSJK09X7LZlqKaUZN4O1Uu02soCt+C5iCXtuDPkR8M7Bil1pP3IVeOFqXoRD\nVFTeQgFeHRRhshbU6jNIvzVJQkc8tBSvsRaqdUPoscgnHPmFIwH2qpLNI3aeu8BL3PDqPpRa9Dlk\nNVCEydpQq8tQCjPz4nS9ELOoRYte2gk67UKoRUjYCTotxPZHY95JuVYBrrXWzyGrgyJMOqdUz8Hu\nRxls1sr0BNgudFkKSyuNiQi3tMha1vUiFomKAOICPq2uCFrA6wFFmKwNXsUy625o8fNaN0Or0HoC\n7Y3rjLmWrLkojngRSzgSxtpEnRfr60U91D6HLB+KMFkLSkV2bHpxi3B6qwhbl4Ldr/mNtdC31o7w\nCgAtYglHlmpr7Yh5hJgCfLVQhMla4Pl/PWHTouj5a0u+3NK4zoSr1Y3Y29trnhz0BNmO1YS45CpY\nhktCvw+5eijCpHNsFETJorRhXzaMzAs18yIf7FZPtnmRDrpv3Qi1fkmcbXSEpUUYl+ETJt1BESZr\nQWsUhM1si7LRbGyvJ9A27MxLZ/bG7MRh1OS8ShON9vktlKxgr5awFxvM2sDrA0WYrBXWSvQm5Gx6\n8XA4nNZmkOI4so0SMGzTImz9uPbHQFuv82xLYx4tEQulULVSJTQvUYNC3A0UYbIW1Cxh7RLQ7ggR\nXCmMo7faMrbWse3rGg61ybaa1bpoyFkNL1NO9+eJD45cEhTiq4ciTC5FJDg1y0/3o8kvrw0GgwsV\nyWxfW8SRn9irERxZ4Z5Pt5Uo7CsKCytNtNmx8XiMo6OjafUzb1/anTt3cPfuXdy/fx+Hh4cYjUbT\n5+p15MjVQxEmc6OF1+uXki9s/G9KKazp4I2JCGuXg+eG0KtXeEVz5D09a9dOvC3it7WUwsEWsWRP\nTk5C4dX70r9z5w6efPJJ3L17F/fu3cPh4SFFeE2gCJO5aPV5tliV0mrpxHo/mozzWpT5VqrjEEU6\n2POfh5pFq8d0+cla0yLrCa/u3717d9ru37+P+/fvYzQaVSunkdVDESZz47kVvDAzLznBG/My2KJ+\nyb/rVTuzoh7VcdAVzawVLOe1KFZsvQgG6UcF2L1SlDUR1mP37t3DvXv3cP/+/QuWsIgwV83oBoow\nWZhSeFZrgZvd3d1iIoXnUogqnUUCXPIxt4adyfkuSktW2+npqbvuW9Ss2Ja2h4eH06b9wuKOmEwm\ndEd0xFwinFJ6FYBvBPBlAA4BvBfAK3POHzbPex2AbwfwIIDfAfAdOeePLOWISWdEAuUJmC2kU8pE\nKwmptz9vtbPSD4BXw6EkxItgBViLrl0Jw5tUs01bwdIi8ZX+aDSaTsZJn+6I9WBeS/j5AH4cwB+c\nv/aHAPxqSunLc86HAJBSeiWA7wbwMgAfA/CDAB49f854WQdOuqM26aYtYc8VoP2y2s/bEkpWKshj\nt7bQTsk1YmN/lyXAgifE3lJEVjy10EYteo3tRxN52h1Brp65RDjn/CK9n1J6OYC/A/AcAO85H/4e\nAK/POf/K+XNeBuAJAC8B8NZLHi/pmFoEhGy9kpKRrzfKeIsm21rrBJeK6nhNn5fXX5RSEoVdgkgL\npbZYbauJsd0vrSunoyNoDV89l/UJPwggA/gMAKSUngXgGQDeJU/IOT+ZUvo9AM8DRXgr8CbhdESE\n9LX7Iarf0O/3p2FlNsTM21o/b9QXd4P3AxFt7bl557soLQIsAikCrP244sPV+yUBtuM64iLq0xLu\nhoVFOJ39Rb4RwHtyzn9+PvwMnInyE+bpT5w/RrYEzxK2FqdX68FrNsmi1Gx4WakfRTnUJt2s2C4z\nPljEOBJgbQXriTTbrAjXWqmmhE1fJlfLZSzhNwF4NoCvWdKxkDWnNnEV1XooFd0ZDAZuyrE3JiJc\nKq6jWy2+dxm+3la8iTkrxFaER6PRNKTMhpjdu3dvLhFuydYj3bCQCKeUfgLAiwA8P+f8SfXQ4wAS\ngIcwaw0/BOCPFj1Ishwi0SlZfrrfuqTP7u7uzGRa5Ov1iu94qcd2qXmvulpUaKeVWnqx7nsZb9G+\nDTvzfLIyLvG82uLV+7qvLeEoNli/N1lf5hbhcwH+BgAvyDl/XD+Wc34spfQ4gBcC+JPz598G8FwA\nP3n5wyXzEIlp5O/0/KK631LbQSzTeSIeonKUOt5XpxhHGW6XnUiL0om9sZbECxv7q4XRa+PxeCaO\n1/a1b1gnWojQWv8urdvNYN444TcB+GYALwZwL6X00PlDn885j877bwTw6pTSR3AWovZ6AH8D4G1L\nOWLSRDTRFAltSytFItiIhcj/K5XP9ASdzXTTj9s6Dzau1wqxPrdFsGnFUZqxDTGLws5OTk5cEfb6\nOp5XxDbqS8SDvFaLsPX1kvVmXkv4FTibePsNM/4fAPwsAOScfySldAPAT+EseuK3Afw7xghfPZ7Q\n6n0AYZyvFwPcul5bbRWLlq0tuDNPcsWieKFkXt9GFuitN6ZFVve9rQ4/08kVdswLO9OWsLXkyfoy\nb5xwk5Mt5/xaAK9d4HjIEqhFA3gTal6Ime3XMtrsY7U13bTI1rZeuUnPEl6GCNdSi0VcrQB6omiF\n1gqxt99aD8LWk9CWOK3gzYG1I7aYeYQ3Ejcd7WCL5cxTwSyymFtXOdY+YS/Gd9n+YOtm0Ftr3Zbc\nC1ZsW1OSo/eyLojIKqcQbw4U4S0mEt4opKzWF+u1JZ7XCqzej6xcG+/rxf9Gbgiv7OS81LLatI93\nXqvV2/fGdbiat/VcD/Y4Gfu7WVCEtxTrjogKq9uIg1Lolw4pk1jeqLWkFev04lrJSx37W2qXIarv\nYC1NPYnm+Wq9tOKW2r8iwlFWW2Tt2sk4WsGbBUV4CylFQHguh3lif7UIHxwchM1bFSNaMaNUz8Fz\nlbSG1S2CJ8SlWr5eBIPdjwrreNuSG8Ru9WSht6UlvBlQhLeUVgG2IqyjEGzfE+GbN2+6zavlEJWz\nbHExlPy+UX8eakV2vPoOOr3YxvPatdxaC+6URDUS2VI8M0V4/aEIbzGlSblIgEs1GSIRvnXr1ky7\nefPmjJVb8/la69UK6TJEtoVSfQfrirA1Hrz6DnoFi9aSlF5Wnj42PWb7pTGyvlCEN4BSyJk3Virb\naMdaJ8X29vamAnvz5s2p20HXedAVz7zi6VFh9XmEtSW9WO9HImYf10kVLStaeKJbKrRTKjOp3RG1\ncybbB0V4TahZf60+Uy+yoTTWUo1Mmli+2vd748aNmbA0O9HmHdey/La635JeXNp6NR2iNOOjo6ML\nqcSeS0L7g6MU49okGgV4+6EIrwElP6cIVs1na/dLEQ+RH9iKrxXhKBJCLy/vLaAZRTUsSiS0UV2H\n0sSWF/vrCbDetwXX7aScTS/Wsb5RUgXdCtcXinDHeLP7spW+iJkXWWC3XllHLwJCl5uMtrav3Q22\n0tm8xXb0uc6DrVBWKp4ThZh5Bc1LiRZRjYdaeJpew81byaJUaIcCfH2gCK8BpbArcUV4ywR5WWhe\n3G2tlYRYb2tlKbUIt4aZLcI86cVekkPUr9V2qKUXe1vrgmhJMaYAXy8owh3iRQVEIixiWKrXIELo\nuSpqYy0TaV6xHTsWre3mhZstinU5RFXMbO2GmquhVtOhVISnJN5RwoV1R1B8rycU4TWglgWm3RF2\ndQq7PttgMAhFtDZWe24t/Vi7S1ZZ4yGK59UWpk0vbkkbLtV5sGJri/V4+9L3UotZ95cIFOE1oiTC\n4pfVIuwtATQcDqv+XTtWm+TzJulKiRh7e3tNiReLEmW1eSUkS3G52n87b5EdW7bSa9bijdKLKcTX\nG4pwx8wT86sLpsukmA4Vu3nz5lSES/G+niiX3Bbec2qTf6VMt2UKcbRqsV25uNZKQmwfE+u2ZOHq\n5k0c2jGK8PWFItwhkT/YWo+eT1gsYZs6fOPGjaKF6glxq9/Yxit78cv6uPU5XsYFoYkm5bQQ24k2\nu3qx16xFXHJdtFq3ns83auT6QhFeAzwr0YqcuAAiEb516xZu376Ng4OD0GXguRDmsXhrwloS2mWm\nG0fVzmrLx5cy2yScrKXa2fHxcTVWORLZKLOPXF8ownPSUtegZasFtmRRSh3fUsUy20qWr1e/t7WS\n2jzMIy5eGnG0jQqqe2OyZLxkspVSi2suCVtuspSpZ8+JkBIU4QZKMbyesNqJKC9SYJ7W7/dnXA62\nhoNMzEm8rucH9ibjagkVy7ReNZ416KUSe2PzrFRRc0F45SZtpIOXXuxZt4zzJYtCEW4kKqsYiasV\nOC10pTHv8V6v59ZtsEV0rAi3TLLZz4x8upellBUWJVt4+6VECTsWpRJ7rSTAUWIFxZcsA4pwhSha\nwbNwW2Jtbb9U40H6/X4/rNtgLeHhcNjsXmhJqpBrcBlKflBt8VrR89KLS2FnUe3elhRjG3oWpReX\nXCelcyYkgiLcgBXgqEkYmed/tf1WkZT39NZx0yUktTuipXiPZ/0uO7MNqJee1FlvpZhbad5S8Nql\nYLfzpBdHsb+RFVzaEtIKRbgBL2bXcyHY+g5eX/bnidPt9XpuhpxtssCm5+LwfjAii34VfuGSWHkh\nZqVavq1uBi8TLupHhX1s5bVSxAMFmCwCRbgRT4g98ZQaCrp5Y7bqWWkrCRq1NhwO0ev1mmN5rdiu\nygpu2Yo7orZUvNTy1ZNuUV/7eSNxt64HrwaFF/Orj7/UJ6QGRbiC54qIrFYtuiKKpUI7LZltYglb\nEffaYDBAr9dritSIRDdqy8Kb1PIsYev71TG/tnh6VFRdLNyWNm/ShXdOtk9ICxThBiIhtuIpYhm5\nC2zpx9ZYXu3msG4NO+bVbaiJa7S/DDzR9fpiCXuJFnZVYy24Nt5X96WCWeRmiNKLa1t9Xt65EjIP\nFOEG7C27N3GmU4r1gphRE6s1msCLJvVa0pCjuObSmD7X6LFFiWJqtQDb4uo6gkH7faOEi3v37l3Y\nPz4+DlfS8Pa9xIt5st8IWQSKcAUtRnZSzrNUdVqxRDB4Tfy3+rXW4tUCPE/ImSecpUy/lvHLEiU4\n6PRjzxLW67lZkZUW7YsIR4kfdquPrdYnZFlQhBuIfMJWNLUVrC1hG9N7cHBQFWHdjxIrvH5JRCMR\nsZEKLa+pYTPfoky4nPOMn9dzK9gxLbRagK11fHx8PD2WVuuWkKuGItyAdUVYy1d8sy0RDLrV3A0t\nacXR5FkpJKx02x2NzytUpWWHbIuiG6Ixr96DuDG0f3dZ50LIKqEIV/B8wZ4LQrshWrZahKOoiFJW\nW+R2AGKR9cZaJ6PmvS2v1djVrbXmr/YN69hgne0W1XnwIhooxmQdoAg34MUHewJcit/1ti1pzSUr\nuJRUEbkBPF9s1Kz16hEJWbTqhDdWSz/2Uo11QoZYw/J+kQBTeMk6QhGu4CVpaEvYiwv2rF6vtdZ1\nEF/vPBltLVaurcdrV4mwj7X4lAVvrbWo35paHBXqidwRkQBzso2sExThBrwCPV5ExLzuiFpNBy3E\nteQK4KnIBk+AvcQDWzTH6+sxoSVGNlp5eJ7VjWv7duXkyB2hj88TYkK6hCLcgGcJz+OOiFpUzyGq\nalbLcLN4AmxdDK0ZZZPJ5MJ7e58neCtSRCtWlGpF1FKNvfKTrPFANgmKcAUvMsJmyM1rBQ+Hw5lC\nO9bVEBXU0cfT4oqwvl9bD0GSIzxRs2OymoT9HO+zATSVmpR9LaD686J9b2VlW/+3xf1AMSbrAEW4\ngZIlbIW4tYkIW1FtcTm0ZLVFk29erV57W6+bHtfv7fX1/jzVziJhjdwiUaEdr9hO7TgJ6RqKcIVa\niFpLaJrX+v1+KKylOg4tfS8SIiqWXqtaph+T99afo9H7Ucyv10oVzKKKZp7P2/YpvGQToAg34Lkk\ntEH2YBgAAAiaSURBVAjXhDiyhPX7e31vPxqz1CzhaGXi0vJB+r1r/VJxHTuul5D3BLVUSrLWJ2Td\noQgvSM0nK9vIJ6ujDZZJztmNyY36tRCwVhG2Y7Ul5nWzBdOtpesljBCyLVCEGxDh1FbjaDSaiWaQ\n52nr0lYBkzoHN27cQK/XW8mxiuuhFnom/daVi8UdIZ9R69u0Y5terF0QnvuEE2jkukARriBWmBbh\no6OjqX9Yx+aKCNtSjIeHhxeWJVq1CHvN+ll1NIQXd2vH5P3t53n90lJDVoBLIWUUYLLtUIQr6Nth\nEa2SAOvb+NFoVKwbsSq8yawWIa4tAyTnaq+Ptx+tiKzTi1stYUK2GYpwBWsJj8fjmbhd/bhYwIPB\nYCrAdp05vbTRqo83StLQfS8GN4rNlfe3n+cdg1wLO9Gn04ujQjsUY3KdoAg3oEVY+4CtAI/HY/T7\nfYxGo2l5S2/V5X6/j93d3ZUcqxe6VWrRpJ23jQTRGy+5NvRn2Uk3uiXIdWMuEU4pvQrANwL4MgCH\nAN4L4JU55w+r57wZwLeal74z5/yiSx5rJ1hLOLKARWC8JYq8rQj5KqjFz+pta7nJRQv4WKu6ZAnL\ne3kRF4RsK/Naws8H8OMA/uD8tT8E4FdTSl+ecz5Uz3sHgJcDkBiuI2woWqw8H7COmPCWq4+Wsl+l\nCNuwLm/yywpxzW1RC6mzgtma/WYt32hLyLYylwhbazal9HIAfwfgOQDeox46yjl/6tJHtwaIQIgI\nedESXglKvfX6VyHCLc1zXURWdMvnClGtiij7zb6eIWrkunBZn/CDADKAz5jxR1JKTwD4LIBfB/Dq\nnLN9zsYgAqQF2FY58yqflfqrWlBTjlO2nmXp+V8jcV5kgiwS+Jq4t0z6EbJtLCzC6UxF3gjgPTnn\nP1cPvQPALwB4DMCX4Mxl8faU0vPyBv5XWTGJqpnVaj9EtSBWfewt+63beT87mmyLJt8IuY5cxhJ+\nE4BnA/gaPZhzfqva/bOU0p8C+CiARwC8+xKf1xmcnSeErIqFHJMppZ8A8CIAj+ScP1l6bs75MQCf\nBvDwIp9FCCHbzNyW8LkAfwOAF+ScP97w/GcCeBqAolgTQsh1ZC5LOKX0JgDfAuDfA7iXUnrovA3P\nHz9IKf1ISum5KaV/nFJ6IYD/A+DDAB5d9sETQsimM6874hUAbgP4DQCfUO2l54+fAPgKAG8D8JcA\n/juA3wfwr3LOx/bNCCHkujNvnHBRtHPOIwBff6kjIoSQa8TqMgYIIYRUoQgTQkiHUIQJIaRDKMKE\nENIhFGFCCOkQijAhhHQIRZgQQjqEIkwIIR1CESaEkA6hCBNCSIdQhAkhpEMowoQQ0iEUYUII6RCK\nMCGEdAhFmBBCOoQiTAghHUIRJoSQDqEIE0JIh1CECSGkQyjChBDSIRRhQgjpkHUQ4WHXB0AIISui\nqm/rIMJf1PUBEELIivii2hNSzvkKjqNwACk9DcDXAfgYgFGnB0MIIcthiDMBfjTn/PelJ3YuwoQQ\ncp1ZB3cEIYRcWyjChBDSIRRhQgjpEIowIYR0yFqKcErpu1JKj6WUDlNK70sp/fOuj2kZpJRek1I6\nNe3Puz6uRUgpPT+l9Esppb89P48XO895XUrpEyml+yml/5tSeriLY12E2vmllN7sfJdv7+p4W0kp\nvSql9P6U0pMppSdSSv87pfRPnOdt5HfXcn7r9t2tnQinlL4JwBsAvAbAVwH4YwCPppSe3umBLY8P\nAXgIwDPO29d2ezgLcwDggwC+E8CFEJuU0isBfDeA/wjgXwC4h7PvsX+VB3kJiud3zjsw+11+89Uc\n2qV4PoAfB/BcAP8WQA/Ar6aU9uUJG/7dVc/vnPX57nLOa9UAvA/Af1X7CcDfAPj+ro9tCef2GgB/\n2PVxrOC8TgG82Ix9AsD3qv3bAA4BvLTr413S+b0ZwC92fWxLOLenn5/f127pd+ed31p9d2tlCaeU\negCeA+BdMpbPrtqvAXheV8e1ZL70/Bb3oyml/5lS+kddH9CySSk9C2fWhf4enwTwe9ie7xEAHjm/\n5f2LlNKbUkr/oOsDWoAHcWbpfwbYyu9u5vwUa/PdrZUI4+xXaxfAE2b8CZz9YWw67wPwcpxlCL4C\nwLMA/FZK6aDLg1oBz8DZH/62fo/A2e3sywD8GwDfD+AFAN6eUkqdHtUcnB/rGwG8J+cscxNb890F\n5wes2Xe318WHXldyzo+q3Q+llN4P4K8AvBRnt0hkQ8g5v1Xt/llK6U8BfBTAIwDe3clBzc+bADwb\nwNd0fSArwj2/dfvu1s0S/jSAE5w5zDUPAXj86g9nteScPw/gwwA2YuZ5Dh7HmS//WnyPAJBzfgxn\nf78b8V2mlH4CwIsAPJJz/qR6aCu+u8L5XaDr726tRDjnfAzgAwBeKGPntwgvBPDero5rVaSUbuLs\niy/+kWwa53/Uj2P2e7yNsxnrrfseASCl9EwAT8MGfJfnAvUNAP51zvnj+rFt+O5K5xc8v9Pvbh3d\nET8G4C0ppQ8AeD+A7wVwA8BbujyoZZBS+lEAv4wzF8QXAvgBAMcAfr7L41qEcz/2wzizmgDgi1NK\nXwngMznnv8aZL+7VKaWP4KxC3utxFuXytg4Od25K53feXgPgF3AmWA8D+GGc3dU8evHd1oeU0ptw\nFo71YgD3Ukpi8X4+5yxVDDf2u6ud3/n3ul7fXdfhGUFYyXfi7Ms/BPC7AL6662Na0nn9PM7+mA8B\nfBzAzwF4VtfHteC5vABnoT8npv0P9ZzX4izc6T7O/sAf7vq4l3F+OCtT+E6c/ROPAPw/AP8NwD/s\n+rgbzss7pxMALzPP28jvrnZ+6/jdsZQlIYR0yFr5hAkh5LpBESaEkA6hCBNCSIdQhAkhpEMowoQQ\n0iEUYUII6RCKMCGEdAhFmBBCOoQiTAghHUIRJoSQDqEIE0JIh1CECSGkQ/4/dWtzaJOhYVoAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11f87e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting example image\n",
    "img = mnist.train.images[0]\n",
    "img_resize = img.reshape((img_size, img_size))\n",
    "plt.imshow(img_resize, cmap = 'gray')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Initialize weights and bias \n",
    "# Convolution and pooling\n",
    "def conv2d(x,w, stride):\n",
    "\treturn tf.nn.conv2d(x,w,strides=[1, stride, stride, 1], padding='SAME')\n",
    "\n",
    "# Get Variables\n",
    "def weight_variable(name, shape):\n",
    "    return tf.get_variable(name,shape=shape, initializer=tf.contrib.layers.xavier_initializer())\n",
    "\n",
    "def bias_variable(name, shape):\n",
    "    return tf.get_variable(name,shape=shape, initializer=tf.contrib.layers.xavier_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM Function!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reset the graph\n",
    "tf.reset_default_graph()\n",
    "\n",
    "#LSTM Variables\n",
    "Wf = weight_variable('Wf', [flatten_size + lstm_size, lstm_size])\n",
    "Wi = weight_variable('Wi', [flatten_size + lstm_size, lstm_size])\n",
    "Wc = weight_variable('Wc', [flatten_size + lstm_size, lstm_size])\n",
    "Wo = weight_variable('Wo', [flatten_size + lstm_size, lstm_size])\n",
    "\n",
    "bf = bias_variable('bf', [lstm_size])\n",
    "bi = bias_variable('bi', [lstm_size])\n",
    "bc = bias_variable('bc', [lstm_size])\n",
    "bo = bias_variable('bo', [lstm_size]) \n",
    "\n",
    "# LSTM function\n",
    "def LSTM_cell(C_prev, h_prev, x_lstm):\n",
    "    # C_prev: Cell state from lstm of previous time step (shape: [batch_size, lstm_size])\n",
    "    # h_prev: output from lstm of previous time step (shape: [batch_size, lstm_size])\n",
    "    # x_lstm: input of lstm (shape: [batch_size, data_flatten_size])\n",
    "\n",
    "    input_concat = tf.concat([x_lstm, h_prev], 1)\n",
    "    f = tf.sigmoid(tf.matmul(input_concat, Wf) + bf)\n",
    "    i = tf.sigmoid(tf.matmul(input_concat, Wi) + bi)\n",
    "    c = tf.tanh(tf.matmul(input_concat, Wc) + bc)\n",
    "    o = tf.sigmoid(tf.matmul(input_concat, Wo) + bo)\n",
    "    \n",
    "    C_t = tf.multiply(f, C_prev) + tf.multiply(i, c) \n",
    "    h_t = tf.multiply(o, tf.tanh(C_t))\n",
    "    \n",
    "    return C_t, h_t # Cell state, Output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN Network with LSTM function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Network\n",
    "\n",
    "# Input \n",
    "x_image  = tf.placeholder(tf.float32, shape = [None, img_flat_size])\n",
    "y_target = tf.placeholder(tf.float32, shape=[None, num_label])\n",
    "\n",
    "input_flat = tf.reshape(x_image,[-1, step_size , flatten_size])\n",
    "input_unstack = tf.unstack(input_flat, axis = 1)\n",
    "################################### LSTM ###################################\n",
    "rnn_batch_size = tf.shape(input_flat)[0]\n",
    "rnn_step_size = tf.shape(input_flat)[1]\n",
    "\n",
    "rnn_state = tf.zeros([rnn_batch_size, lstm_size], tf.float32)\n",
    "rnn_out = tf.zeros([rnn_batch_size, lstm_size], tf.float32)\n",
    "\n",
    "for i in range(len(input_unstack)):\n",
    "    rnn_state, rnn_out = LSTM_cell(rnn_state, rnn_out, input_unstack[i])\n",
    "############################################################################\n",
    "\n",
    "rnn_out = tf.reshape(rnn_out ,shape = [-1, lstm_size])\n",
    "\n",
    "# Densely connect layer variables \n",
    "w_fc1 = weight_variable('W_fc1', [lstm_size, 256])\n",
    "b_fc1 = bias_variable('b_fc1', [256])\n",
    "\n",
    "w_fc2 = weight_variable('W_fc2',[256, num_label])\n",
    "b_fc2 = bias_variable('b_fc2',[num_label])\n",
    "\n",
    "# Fully Connected Layer\n",
    "h_fc1 = tf.nn.relu(tf.matmul(rnn_out, w_fc1)+b_fc1)\n",
    "output = tf.matmul(h_fc1, w_fc2)+b_fc2\n",
    "\n",
    "# Training \n",
    "Loss = tf.nn.sigmoid_cross_entropy_with_logits(labels = y_target, logits = output)\n",
    "Cost = tf.reduce_mean(Loss)\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate, epsilon = epsilon).minimize(Cost)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y_target,1), tf.argmax(output,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set: (60000, 784)\n",
      "Testing set: (9000, 784)\n",
      "Validation set: (1000, 784)\n"
     ]
    }
   ],
   "source": [
    "# Dataset for train, test, validation\n",
    "\n",
    "test_len = mnist.test.images.shape[0]\n",
    "validation_len = int(test_len * validation_ratio)\n",
    "\n",
    "train_x = mnist.train.images\n",
    "test_x = mnist.test.images[validation_len : test_len, :]\n",
    "validation_x = mnist.test.images[ : validation_len, :]\n",
    "\n",
    "train_y = mnist.train.labels\n",
    "test_y = mnist.test.labels[validation_len : test_len]\n",
    "validation_y = mnist.test.labels[ : validation_len]\n",
    "\n",
    "print(\"Training set: \" + str(train_x.shape))\n",
    "print(\"Testing set: \" + str(test_x.shape))\n",
    "print(\"Validation set: \" + str(validation_x.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create Session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = gpu_fraction\n",
    "\n",
    "sess = tf.InteractiveSession(config=config)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "\n",
    "data = np.insert(train_x, img_flat_size, train_y, axis = 1)\n",
    "len_data = data.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 / Batch: 0/60000 / Cost: 0.704195 / Training Accuracy: 0.119141 / Validation Accuracy: 0.126\n",
      "Epoch: 1 / Batch: 512/60000 / Cost: 0.681971 / Training Accuracy: 0.0917969 / Validation Accuracy: 0.126\n",
      "Epoch: 1 / Batch: 1024/60000 / Cost: 0.656538 / Training Accuracy: 0.0957031 / Validation Accuracy: 0.126\n",
      "Epoch: 1 / Batch: 1536/60000 / Cost: 0.626169 / Training Accuracy: 0.105469 / Validation Accuracy: 0.126\n",
      "Epoch: 1 / Batch: 2048/60000 / Cost: 0.580511 / Training Accuracy: 0.107422 / Validation Accuracy: 0.126\n",
      "Epoch: 1 / Batch: 2560/60000 / Cost: 0.508731 / Training Accuracy: 0.117188 / Validation Accuracy: 0.137\n",
      "Epoch: 1 / Batch: 3072/60000 / Cost: 0.407777 / Training Accuracy: 0.126953 / Validation Accuracy: 0.14\n",
      "Epoch: 1 / Batch: 3584/60000 / Cost: 0.339261 / Training Accuracy: 0.107422 / Validation Accuracy: 0.089\n",
      "Epoch: 1 / Batch: 4096/60000 / Cost: 0.3495 / Training Accuracy: 0.0976563 / Validation Accuracy: 0.089\n",
      "Epoch: 1 / Batch: 4608/60000 / Cost: 0.36137 / Training Accuracy: 0.162109 / Validation Accuracy: 0.177\n",
      "Epoch: 1 / Batch: 5120/60000 / Cost: 0.359946 / Training Accuracy: 0.0996094 / Validation Accuracy: 0.126\n",
      "Epoch: 1 / Batch: 5632/60000 / Cost: 0.352014 / Training Accuracy: 0.125 / Validation Accuracy: 0.172\n",
      "Epoch: 1 / Batch: 6144/60000 / Cost: 0.337382 / Training Accuracy: 0.0898438 / Validation Accuracy: 0.107\n",
      "Epoch: 1 / Batch: 6656/60000 / Cost: 0.33329 / Training Accuracy: 0.121094 / Validation Accuracy: 0.125\n",
      "Epoch: 1 / Batch: 7168/60000 / Cost: 0.329558 / Training Accuracy: 0.103516 / Validation Accuracy: 0.094\n",
      "Epoch: 1 / Batch: 7680/60000 / Cost: 0.330435 / Training Accuracy: 0.0820313 / Validation Accuracy: 0.094\n",
      "Epoch: 1 / Batch: 8192/60000 / Cost: 0.334595 / Training Accuracy: 0.0996094 / Validation Accuracy: 0.094\n",
      "Epoch: 1 / Batch: 8704/60000 / Cost: 0.332966 / Training Accuracy: 0.136719 / Validation Accuracy: 0.104\n",
      "Epoch: 1 / Batch: 9216/60000 / Cost: 0.332123 / Training Accuracy: 0.117188 / Validation Accuracy: 0.115\n",
      "Epoch: 1 / Batch: 9728/60000 / Cost: 0.328292 / Training Accuracy: 0.113281 / Validation Accuracy: 0.099\n",
      "Epoch: 1 / Batch: 10240/60000 / Cost: 0.326386 / Training Accuracy: 0.0898438 / Validation Accuracy: 0.1\n",
      "Epoch: 1 / Batch: 10752/60000 / Cost: 0.325491 / Training Accuracy: 0.111328 / Validation Accuracy: 0.128\n",
      "Epoch: 1 / Batch: 11264/60000 / Cost: 0.32428 / Training Accuracy: 0.119141 / Validation Accuracy: 0.127\n",
      "Epoch: 1 / Batch: 11776/60000 / Cost: 0.325364 / Training Accuracy: 0.107422 / Validation Accuracy: 0.127\n",
      "Epoch: 1 / Batch: 12288/60000 / Cost: 0.325565 / Training Accuracy: 0.117188 / Validation Accuracy: 0.131\n",
      "Epoch: 1 / Batch: 12800/60000 / Cost: 0.324557 / Training Accuracy: 0.160156 / Validation Accuracy: 0.158\n",
      "Epoch: 1 / Batch: 13312/60000 / Cost: 0.325942 / Training Accuracy: 0.191406 / Validation Accuracy: 0.209\n",
      "Epoch: 1 / Batch: 13824/60000 / Cost: 0.324399 / Training Accuracy: 0.203125 / Validation Accuracy: 0.202\n",
      "Epoch: 1 / Batch: 14336/60000 / Cost: 0.32532 / Training Accuracy: 0.138672 / Validation Accuracy: 0.157\n",
      "Epoch: 1 / Batch: 14848/60000 / Cost: 0.323988 / Training Accuracy: 0.119141 / Validation Accuracy: 0.139\n",
      "Epoch: 1 / Batch: 15360/60000 / Cost: 0.322001 / Training Accuracy: 0.150391 / Validation Accuracy: 0.158\n",
      "Epoch: 1 / Batch: 15872/60000 / Cost: 0.321171 / Training Accuracy: 0.199219 / Validation Accuracy: 0.218\n",
      "Epoch: 1 / Batch: 16384/60000 / Cost: 0.321526 / Training Accuracy: 0.126953 / Validation Accuracy: 0.128\n",
      "Epoch: 1 / Batch: 16896/60000 / Cost: 0.31997 / Training Accuracy: 0.121094 / Validation Accuracy: 0.112\n",
      "Epoch: 1 / Batch: 17408/60000 / Cost: 0.320938 / Training Accuracy: 0.111328 / Validation Accuracy: 0.118\n",
      "Epoch: 1 / Batch: 17920/60000 / Cost: 0.318668 / Training Accuracy: 0.150391 / Validation Accuracy: 0.128\n",
      "Epoch: 1 / Batch: 18432/60000 / Cost: 0.318278 / Training Accuracy: 0.158203 / Validation Accuracy: 0.11\n",
      "Epoch: 1 / Batch: 18944/60000 / Cost: 0.318398 / Training Accuracy: 0.103516 / Validation Accuracy: 0.118\n",
      "Epoch: 1 / Batch: 19456/60000 / Cost: 0.316009 / Training Accuracy: 0.222656 / Validation Accuracy: 0.214\n",
      "Epoch: 1 / Batch: 19968/60000 / Cost: 0.315386 / Training Accuracy: 0.197266 / Validation Accuracy: 0.227\n",
      "Epoch: 1 / Batch: 20480/60000 / Cost: 0.311484 / Training Accuracy: 0.222656 / Validation Accuracy: 0.208\n",
      "Epoch: 1 / Batch: 20992/60000 / Cost: 0.310938 / Training Accuracy: 0.191406 / Validation Accuracy: 0.21\n",
      "Epoch: 1 / Batch: 21504/60000 / Cost: 0.310464 / Training Accuracy: 0.203125 / Validation Accuracy: 0.205\n",
      "Epoch: 1 / Batch: 22016/60000 / Cost: 0.310282 / Training Accuracy: 0.197266 / Validation Accuracy: 0.234\n",
      "Epoch: 1 / Batch: 22528/60000 / Cost: 0.311473 / Training Accuracy: 0.195313 / Validation Accuracy: 0.228\n",
      "Epoch: 1 / Batch: 23040/60000 / Cost: 0.304398 / Training Accuracy: 0.224609 / Validation Accuracy: 0.21\n",
      "Epoch: 1 / Batch: 23552/60000 / Cost: 0.309314 / Training Accuracy: 0.177734 / Validation Accuracy: 0.218\n",
      "Epoch: 1 / Batch: 24064/60000 / Cost: 0.304092 / Training Accuracy: 0.199219 / Validation Accuracy: 0.221\n",
      "Epoch: 1 / Batch: 24576/60000 / Cost: 0.305856 / Training Accuracy: 0.195313 / Validation Accuracy: 0.216\n",
      "Epoch: 1 / Batch: 25088/60000 / Cost: 0.299874 / Training Accuracy: 0.302734 / Validation Accuracy: 0.266\n",
      "Epoch: 1 / Batch: 25600/60000 / Cost: 0.300307 / Training Accuracy: 0.267578 / Validation Accuracy: 0.248\n",
      "Epoch: 1 / Batch: 26112/60000 / Cost: 0.297288 / Training Accuracy: 0.261719 / Validation Accuracy: 0.276\n",
      "Epoch: 1 / Batch: 26624/60000 / Cost: 0.299975 / Training Accuracy: 0.246094 / Validation Accuracy: 0.27\n",
      "Epoch: 1 / Batch: 27136/60000 / Cost: 0.293117 / Training Accuracy: 0.296875 / Validation Accuracy: 0.263\n",
      "Epoch: 1 / Batch: 27648/60000 / Cost: 0.300733 / Training Accuracy: 0.228516 / Validation Accuracy: 0.229\n",
      "Epoch: 1 / Batch: 28160/60000 / Cost: 0.290645 / Training Accuracy: 0.279297 / Validation Accuracy: 0.274\n",
      "Epoch: 1 / Batch: 28672/60000 / Cost: 0.301649 / Training Accuracy: 0.205078 / Validation Accuracy: 0.258\n",
      "Epoch: 1 / Batch: 29184/60000 / Cost: 0.295432 / Training Accuracy: 0.248047 / Validation Accuracy: 0.238\n",
      "Epoch: 1 / Batch: 29696/60000 / Cost: 0.299242 / Training Accuracy: 0.275391 / Validation Accuracy: 0.263\n",
      "Epoch: 1 / Batch: 30208/60000 / Cost: 0.291191 / Training Accuracy: 0.316406 / Validation Accuracy: 0.302\n",
      "Epoch: 1 / Batch: 30720/60000 / Cost: 0.286161 / Training Accuracy: 0.300781 / Validation Accuracy: 0.318\n",
      "Epoch: 1 / Batch: 31232/60000 / Cost: 0.28483 / Training Accuracy: 0.298828 / Validation Accuracy: 0.288\n",
      "Epoch: 1 / Batch: 31744/60000 / Cost: 0.287925 / Training Accuracy: 0.253906 / Validation Accuracy: 0.264\n",
      "Epoch: 1 / Batch: 32256/60000 / Cost: 0.287629 / Training Accuracy: 0.267578 / Validation Accuracy: 0.263\n",
      "Epoch: 1 / Batch: 32768/60000 / Cost: 0.279028 / Training Accuracy: 0.314453 / Validation Accuracy: 0.302\n",
      "Epoch: 1 / Batch: 33280/60000 / Cost: 0.28414 / Training Accuracy: 0.28125 / Validation Accuracy: 0.251\n",
      "Epoch: 1 / Batch: 33792/60000 / Cost: 0.278299 / Training Accuracy: 0.326172 / Validation Accuracy: 0.278\n",
      "Epoch: 1 / Batch: 34304/60000 / Cost: 0.285365 / Training Accuracy: 0.351563 / Validation Accuracy: 0.37\n",
      "Epoch: 1 / Batch: 34816/60000 / Cost: 0.271907 / Training Accuracy: 0.369141 / Validation Accuracy: 0.349\n",
      "Epoch: 1 / Batch: 35328/60000 / Cost: 0.278023 / Training Accuracy: 0.357422 / Validation Accuracy: 0.315\n",
      "Epoch: 1 / Batch: 35840/60000 / Cost: 0.266348 / Training Accuracy: 0.365234 / Validation Accuracy: 0.328\n",
      "Epoch: 1 / Batch: 36352/60000 / Cost: 0.277738 / Training Accuracy: 0.3125 / Validation Accuracy: 0.356\n",
      "Epoch: 1 / Batch: 36864/60000 / Cost: 0.267578 / Training Accuracy: 0.367188 / Validation Accuracy: 0.342\n",
      "Epoch: 1 / Batch: 37376/60000 / Cost: 0.268749 / Training Accuracy: 0.341797 / Validation Accuracy: 0.313\n",
      "Epoch: 1 / Batch: 37888/60000 / Cost: 0.266999 / Training Accuracy: 0.369141 / Validation Accuracy: 0.355\n",
      "Epoch: 1 / Batch: 38400/60000 / Cost: 0.257329 / Training Accuracy: 0.402344 / Validation Accuracy: 0.366\n",
      "Epoch: 1 / Batch: 38912/60000 / Cost: 0.254954 / Training Accuracy: 0.382813 / Validation Accuracy: 0.398\n",
      "Epoch: 1 / Batch: 39424/60000 / Cost: 0.253556 / Training Accuracy: 0.412109 / Validation Accuracy: 0.428\n",
      "Epoch: 1 / Batch: 39936/60000 / Cost: 0.250889 / Training Accuracy: 0.431641 / Validation Accuracy: 0.407\n",
      "Epoch: 1 / Batch: 40448/60000 / Cost: 0.247721 / Training Accuracy: 0.513672 / Validation Accuracy: 0.458\n",
      "Epoch: 1 / Batch: 40960/60000 / Cost: 0.246301 / Training Accuracy: 0.482422 / Validation Accuracy: 0.48\n",
      "Epoch: 1 / Batch: 41472/60000 / Cost: 0.240272 / Training Accuracy: 0.498047 / Validation Accuracy: 0.455\n",
      "Epoch: 1 / Batch: 41984/60000 / Cost: 0.234505 / Training Accuracy: 0.515625 / Validation Accuracy: 0.445\n",
      "Epoch: 1 / Batch: 42496/60000 / Cost: 0.234 / Training Accuracy: 0.492188 / Validation Accuracy: 0.505\n",
      "Epoch: 1 / Batch: 43008/60000 / Cost: 0.22939 / Training Accuracy: 0.511719 / Validation Accuracy: 0.491\n",
      "Epoch: 1 / Batch: 43520/60000 / Cost: 0.229628 / Training Accuracy: 0.533203 / Validation Accuracy: 0.487\n",
      "Epoch: 1 / Batch: 44032/60000 / Cost: 0.223887 / Training Accuracy: 0.53125 / Validation Accuracy: 0.504\n",
      "Epoch: 1 / Batch: 44544/60000 / Cost: 0.210618 / Training Accuracy: 0.529297 / Validation Accuracy: 0.483\n",
      "Epoch: 1 / Batch: 45056/60000 / Cost: 0.216307 / Training Accuracy: 0.53125 / Validation Accuracy: 0.52\n",
      "Epoch: 1 / Batch: 45568/60000 / Cost: 0.203283 / Training Accuracy: 0.595703 / Validation Accuracy: 0.555\n",
      "Epoch: 1 / Batch: 46080/60000 / Cost: 0.208636 / Training Accuracy: 0.603516 / Validation Accuracy: 0.586\n",
      "Epoch: 1 / Batch: 46592/60000 / Cost: 0.19764 / Training Accuracy: 0.607422 / Validation Accuracy: 0.559\n",
      "Epoch: 1 / Batch: 47104/60000 / Cost: 0.187811 / Training Accuracy: 0.630859 / Validation Accuracy: 0.59\n",
      "Epoch: 1 / Batch: 47616/60000 / Cost: 0.195836 / Training Accuracy: 0.638672 / Validation Accuracy: 0.567\n",
      "Epoch: 1 / Batch: 48128/60000 / Cost: 0.195513 / Training Accuracy: 0.591797 / Validation Accuracy: 0.558\n",
      "Epoch: 1 / Batch: 48640/60000 / Cost: 0.201788 / Training Accuracy: 0.541016 / Validation Accuracy: 0.524\n",
      "Epoch: 1 / Batch: 49152/60000 / Cost: 0.178397 / Training Accuracy: 0.628906 / Validation Accuracy: 0.612\n",
      "Epoch: 1 / Batch: 49664/60000 / Cost: 0.177806 / Training Accuracy: 0.619141 / Validation Accuracy: 0.584\n",
      "Epoch: 1 / Batch: 50176/60000 / Cost: 0.182993 / Training Accuracy: 0.595703 / Validation Accuracy: 0.582\n",
      "Epoch: 1 / Batch: 50688/60000 / Cost: 0.175881 / Training Accuracy: 0.597656 / Validation Accuracy: 0.604\n",
      "Epoch: 1 / Batch: 51200/60000 / Cost: 0.177589 / Training Accuracy: 0.640625 / Validation Accuracy: 0.607\n",
      "Epoch: 1 / Batch: 51712/60000 / Cost: 0.173464 / Training Accuracy: 0.638672 / Validation Accuracy: 0.635\n",
      "Epoch: 1 / Batch: 52224/60000 / Cost: 0.1658 / Training Accuracy: 0.654297 / Validation Accuracy: 0.636\n",
      "Epoch: 1 / Batch: 52736/60000 / Cost: 0.163741 / Training Accuracy: 0.667969 / Validation Accuracy: 0.618\n",
      "Epoch: 1 / Batch: 53248/60000 / Cost: 0.163475 / Training Accuracy: 0.662109 / Validation Accuracy: 0.633\n",
      "Epoch: 1 / Batch: 53760/60000 / Cost: 0.150481 / Training Accuracy: 0.695313 / Validation Accuracy: 0.647\n",
      "Epoch: 1 / Batch: 54272/60000 / Cost: 0.15793 / Training Accuracy: 0.667969 / Validation Accuracy: 0.622\n",
      "Epoch: 1 / Batch: 54784/60000 / Cost: 0.158948 / Training Accuracy: 0.664063 / Validation Accuracy: 0.643\n",
      "Epoch: 1 / Batch: 55296/60000 / Cost: 0.161326 / Training Accuracy: 0.636719 / Validation Accuracy: 0.64\n",
      "Epoch: 1 / Batch: 55808/60000 / Cost: 0.149142 / Training Accuracy: 0.710938 / Validation Accuracy: 0.652\n",
      "Epoch: 1 / Batch: 56320/60000 / Cost: 0.138975 / Training Accuracy: 0.738281 / Validation Accuracy: 0.682\n",
      "Epoch: 1 / Batch: 56832/60000 / Cost: 0.147329 / Training Accuracy: 0.695313 / Validation Accuracy: 0.677\n",
      "Epoch: 1 / Batch: 57344/60000 / Cost: 0.146701 / Training Accuracy: 0.712891 / Validation Accuracy: 0.678\n",
      "Epoch: 1 / Batch: 57856/60000 / Cost: 0.144576 / Training Accuracy: 0.720703 / Validation Accuracy: 0.676\n",
      "Epoch: 1 / Batch: 58368/60000 / Cost: 0.140411 / Training Accuracy: 0.724609 / Validation Accuracy: 0.676\n",
      "Epoch: 1 / Batch: 58880/60000 / Cost: 0.140781 / Training Accuracy: 0.740234 / Validation Accuracy: 0.694\n",
      "Epoch: 1 / Batch: 59392/60000 / Cost: 0.130255 / Training Accuracy: 0.753906 / Validation Accuracy: 0.69\n",
      "Epoch: 1 / Batch: 59904/60000 / Cost: 0.119518 / Training Accuracy: 0.791667 / Validation Accuracy: 0.691\n",
      "Epoch: 2 / Batch: 0/60000 / Cost: 0.148402 / Training Accuracy: 0.703125 / Validation Accuracy: 0.699\n",
      "Epoch: 2 / Batch: 512/60000 / Cost: 0.131755 / Training Accuracy: 0.730469 / Validation Accuracy: 0.702\n",
      "Epoch: 2 / Batch: 1024/60000 / Cost: 0.124842 / Training Accuracy: 0.783203 / Validation Accuracy: 0.689\n",
      "Epoch: 2 / Batch: 1536/60000 / Cost: 0.143843 / Training Accuracy: 0.697266 / Validation Accuracy: 0.657\n",
      "Epoch: 2 / Batch: 2048/60000 / Cost: 0.124911 / Training Accuracy: 0.777344 / Validation Accuracy: 0.71\n",
      "Epoch: 2 / Batch: 2560/60000 / Cost: 0.13894 / Training Accuracy: 0.703125 / Validation Accuracy: 0.696\n",
      "Epoch: 2 / Batch: 3072/60000 / Cost: 0.124559 / Training Accuracy: 0.748047 / Validation Accuracy: 0.714\n",
      "Epoch: 2 / Batch: 3584/60000 / Cost: 0.126984 / Training Accuracy: 0.732422 / Validation Accuracy: 0.713\n",
      "Epoch: 2 / Batch: 4096/60000 / Cost: 0.121315 / Training Accuracy: 0.78125 / Validation Accuracy: 0.735\n",
      "Epoch: 2 / Batch: 4608/60000 / Cost: 0.125485 / Training Accuracy: 0.757813 / Validation Accuracy: 0.741\n",
      "Epoch: 2 / Batch: 5120/60000 / Cost: 0.128271 / Training Accuracy: 0.746094 / Validation Accuracy: 0.729\n",
      "Epoch: 2 / Batch: 5632/60000 / Cost: 0.128177 / Training Accuracy: 0.779297 / Validation Accuracy: 0.737\n",
      "Epoch: 2 / Batch: 6144/60000 / Cost: 0.116979 / Training Accuracy: 0.757813 / Validation Accuracy: 0.72\n",
      "Epoch: 2 / Batch: 6656/60000 / Cost: 0.117151 / Training Accuracy: 0.791016 / Validation Accuracy: 0.75\n",
      "Epoch: 2 / Batch: 7168/60000 / Cost: 0.127939 / Training Accuracy: 0.740234 / Validation Accuracy: 0.725\n",
      "Epoch: 2 / Batch: 7680/60000 / Cost: 0.123156 / Training Accuracy: 0.767578 / Validation Accuracy: 0.772\n",
      "Epoch: 2 / Batch: 8192/60000 / Cost: 0.111877 / Training Accuracy: 0.808594 / Validation Accuracy: 0.759\n",
      "Epoch: 2 / Batch: 8704/60000 / Cost: 0.117157 / Training Accuracy: 0.791016 / Validation Accuracy: 0.749\n",
      "Epoch: 2 / Batch: 9216/60000 / Cost: 0.10187 / Training Accuracy: 0.8125 / Validation Accuracy: 0.768\n",
      "Epoch: 2 / Batch: 9728/60000 / Cost: 0.107599 / Training Accuracy: 0.787109 / Validation Accuracy: 0.77\n",
      "Epoch: 2 / Batch: 10240/60000 / Cost: 0.104224 / Training Accuracy: 0.839844 / Validation Accuracy: 0.778\n",
      "Epoch: 2 / Batch: 10752/60000 / Cost: 0.107233 / Training Accuracy: 0.820313 / Validation Accuracy: 0.778\n",
      "Epoch: 2 / Batch: 11264/60000 / Cost: 0.100304 / Training Accuracy: 0.84375 / Validation Accuracy: 0.784\n",
      "Epoch: 2 / Batch: 11776/60000 / Cost: 0.10263 / Training Accuracy: 0.818359 / Validation Accuracy: 0.774\n",
      "Epoch: 2 / Batch: 12288/60000 / Cost: 0.102779 / Training Accuracy: 0.824219 / Validation Accuracy: 0.798\n",
      "Epoch: 2 / Batch: 12800/60000 / Cost: 0.108305 / Training Accuracy: 0.810547 / Validation Accuracy: 0.804\n",
      "Epoch: 2 / Batch: 13312/60000 / Cost: 0.0982766 / Training Accuracy: 0.839844 / Validation Accuracy: 0.812\n",
      "Epoch: 2 / Batch: 13824/60000 / Cost: 0.0943591 / Training Accuracy: 0.84375 / Validation Accuracy: 0.781\n",
      "Epoch: 2 / Batch: 14336/60000 / Cost: 0.0953618 / Training Accuracy: 0.822266 / Validation Accuracy: 0.813\n",
      "Epoch: 2 / Batch: 14848/60000 / Cost: 0.0952763 / Training Accuracy: 0.818359 / Validation Accuracy: 0.797\n",
      "Epoch: 2 / Batch: 15360/60000 / Cost: 0.0807767 / Training Accuracy: 0.857422 / Validation Accuracy: 0.815\n",
      "Epoch: 2 / Batch: 15872/60000 / Cost: 0.0904045 / Training Accuracy: 0.837891 / Validation Accuracy: 0.83\n",
      "Epoch: 2 / Batch: 16384/60000 / Cost: 0.0887101 / Training Accuracy: 0.863281 / Validation Accuracy: 0.82\n",
      "Epoch: 2 / Batch: 16896/60000 / Cost: 0.099181 / Training Accuracy: 0.808594 / Validation Accuracy: 0.807\n",
      "Epoch: 2 / Batch: 17408/60000 / Cost: 0.0813373 / Training Accuracy: 0.884766 / Validation Accuracy: 0.85\n",
      "Epoch: 2 / Batch: 17920/60000 / Cost: 0.0834089 / Training Accuracy: 0.853516 / Validation Accuracy: 0.82\n",
      "Epoch: 2 / Batch: 18432/60000 / Cost: 0.0983645 / Training Accuracy: 0.810547 / Validation Accuracy: 0.811\n",
      "Epoch: 2 / Batch: 18944/60000 / Cost: 0.0861094 / Training Accuracy: 0.832031 / Validation Accuracy: 0.803\n",
      "Epoch: 2 / Batch: 19456/60000 / Cost: 0.0717293 / Training Accuracy: 0.888672 / Validation Accuracy: 0.835\n",
      "Epoch: 2 / Batch: 19968/60000 / Cost: 0.0789219 / Training Accuracy: 0.871094 / Validation Accuracy: 0.838\n",
      "Epoch: 2 / Batch: 20480/60000 / Cost: 0.0710062 / Training Accuracy: 0.880859 / Validation Accuracy: 0.846\n",
      "Epoch: 2 / Batch: 20992/60000 / Cost: 0.0857748 / Training Accuracy: 0.84375 / Validation Accuracy: 0.836\n",
      "Epoch: 2 / Batch: 21504/60000 / Cost: 0.0773093 / Training Accuracy: 0.875 / Validation Accuracy: 0.834\n",
      "Epoch: 2 / Batch: 22016/60000 / Cost: 0.0823006 / Training Accuracy: 0.859375 / Validation Accuracy: 0.839\n",
      "Epoch: 2 / Batch: 22528/60000 / Cost: 0.0829092 / Training Accuracy: 0.853516 / Validation Accuracy: 0.839\n",
      "Epoch: 2 / Batch: 23040/60000 / Cost: 0.0892078 / Training Accuracy: 0.84375 / Validation Accuracy: 0.82\n",
      "Epoch: 2 / Batch: 23552/60000 / Cost: 0.072897 / Training Accuracy: 0.873047 / Validation Accuracy: 0.848\n",
      "Epoch: 2 / Batch: 24064/60000 / Cost: 0.0858255 / Training Accuracy: 0.845703 / Validation Accuracy: 0.836\n",
      "Epoch: 2 / Batch: 24576/60000 / Cost: 0.0747738 / Training Accuracy: 0.869141 / Validation Accuracy: 0.845\n",
      "Epoch: 2 / Batch: 25088/60000 / Cost: 0.074429 / Training Accuracy: 0.867188 / Validation Accuracy: 0.832\n",
      "Epoch: 2 / Batch: 25600/60000 / Cost: 0.0665285 / Training Accuracy: 0.894531 / Validation Accuracy: 0.836\n",
      "Epoch: 2 / Batch: 26112/60000 / Cost: 0.0746185 / Training Accuracy: 0.857422 / Validation Accuracy: 0.841\n",
      "Epoch: 2 / Batch: 26624/60000 / Cost: 0.0773416 / Training Accuracy: 0.873047 / Validation Accuracy: 0.836\n",
      "Epoch: 2 / Batch: 27136/60000 / Cost: 0.0867626 / Training Accuracy: 0.859375 / Validation Accuracy: 0.817\n",
      "Epoch: 2 / Batch: 27648/60000 / Cost: 0.0734547 / Training Accuracy: 0.882813 / Validation Accuracy: 0.836\n",
      "Epoch: 2 / Batch: 28160/60000 / Cost: 0.0711156 / Training Accuracy: 0.882813 / Validation Accuracy: 0.852\n",
      "Epoch: 2 / Batch: 28672/60000 / Cost: 0.0830747 / Training Accuracy: 0.855469 / Validation Accuracy: 0.84\n",
      "Epoch: 2 / Batch: 29184/60000 / Cost: 0.0640011 / Training Accuracy: 0.892578 / Validation Accuracy: 0.867\n",
      "Epoch: 2 / Batch: 29696/60000 / Cost: 0.0671121 / Training Accuracy: 0.898438 / Validation Accuracy: 0.866\n",
      "Epoch: 2 / Batch: 30208/60000 / Cost: 0.0741759 / Training Accuracy: 0.876953 / Validation Accuracy: 0.841\n",
      "Epoch: 2 / Batch: 30720/60000 / Cost: 0.0774723 / Training Accuracy: 0.853516 / Validation Accuracy: 0.849\n",
      "Epoch: 2 / Batch: 31232/60000 / Cost: 0.0649835 / Training Accuracy: 0.884766 / Validation Accuracy: 0.873\n",
      "Epoch: 2 / Batch: 31744/60000 / Cost: 0.0716323 / Training Accuracy: 0.880859 / Validation Accuracy: 0.844\n",
      "Epoch: 2 / Batch: 32256/60000 / Cost: 0.0691069 / Training Accuracy: 0.884766 / Validation Accuracy: 0.862\n",
      "Epoch: 2 / Batch: 32768/60000 / Cost: 0.0736822 / Training Accuracy: 0.865234 / Validation Accuracy: 0.855\n",
      "Epoch: 2 / Batch: 33280/60000 / Cost: 0.0715302 / Training Accuracy: 0.876953 / Validation Accuracy: 0.856\n",
      "Epoch: 2 / Batch: 33792/60000 / Cost: 0.0621689 / Training Accuracy: 0.900391 / Validation Accuracy: 0.88\n",
      "Epoch: 2 / Batch: 34304/60000 / Cost: 0.0621084 / Training Accuracy: 0.898438 / Validation Accuracy: 0.864\n",
      "Epoch: 2 / Batch: 34816/60000 / Cost: 0.058501 / Training Accuracy: 0.908203 / Validation Accuracy: 0.871\n",
      "Epoch: 2 / Batch: 35328/60000 / Cost: 0.0635168 / Training Accuracy: 0.908203 / Validation Accuracy: 0.87\n",
      "Epoch: 2 / Batch: 35840/60000 / Cost: 0.0759857 / Training Accuracy: 0.861328 / Validation Accuracy: 0.873\n",
      "Epoch: 2 / Batch: 36352/60000 / Cost: 0.0604676 / Training Accuracy: 0.902344 / Validation Accuracy: 0.878\n",
      "Epoch: 2 / Batch: 36864/60000 / Cost: 0.0775096 / Training Accuracy: 0.861328 / Validation Accuracy: 0.849\n",
      "Epoch: 2 / Batch: 37376/60000 / Cost: 0.0749719 / Training Accuracy: 0.876953 / Validation Accuracy: 0.866\n",
      "Epoch: 2 / Batch: 37888/60000 / Cost: 0.0508936 / Training Accuracy: 0.912109 / Validation Accuracy: 0.889\n",
      "Epoch: 2 / Batch: 38400/60000 / Cost: 0.0671002 / Training Accuracy: 0.878906 / Validation Accuracy: 0.88\n",
      "Epoch: 2 / Batch: 38912/60000 / Cost: 0.0563724 / Training Accuracy: 0.910156 / Validation Accuracy: 0.871\n",
      "Epoch: 2 / Batch: 39424/60000 / Cost: 0.0629007 / Training Accuracy: 0.892578 / Validation Accuracy: 0.88\n",
      "Epoch: 2 / Batch: 39936/60000 / Cost: 0.0703801 / Training Accuracy: 0.875 / Validation Accuracy: 0.878\n",
      "Epoch: 2 / Batch: 40448/60000 / Cost: 0.0508142 / Training Accuracy: 0.912109 / Validation Accuracy: 0.899\n",
      "Epoch: 2 / Batch: 40960/60000 / Cost: 0.0566193 / Training Accuracy: 0.912109 / Validation Accuracy: 0.886\n",
      "Epoch: 2 / Batch: 41472/60000 / Cost: 0.0587463 / Training Accuracy: 0.898438 / Validation Accuracy: 0.869\n",
      "Epoch: 2 / Batch: 41984/60000 / Cost: 0.066925 / Training Accuracy: 0.888672 / Validation Accuracy: 0.881\n",
      "Epoch: 2 / Batch: 42496/60000 / Cost: 0.0587236 / Training Accuracy: 0.916016 / Validation Accuracy: 0.891\n",
      "Epoch: 2 / Batch: 43008/60000 / Cost: 0.0670711 / Training Accuracy: 0.880859 / Validation Accuracy: 0.891\n",
      "Epoch: 2 / Batch: 43520/60000 / Cost: 0.0514611 / Training Accuracy: 0.914063 / Validation Accuracy: 0.901\n",
      "Epoch: 2 / Batch: 44032/60000 / Cost: 0.0484266 / Training Accuracy: 0.923828 / Validation Accuracy: 0.895\n",
      "Epoch: 2 / Batch: 44544/60000 / Cost: 0.0564054 / Training Accuracy: 0.900391 / Validation Accuracy: 0.877\n",
      "Epoch: 2 / Batch: 45056/60000 / Cost: 0.0587778 / Training Accuracy: 0.902344 / Validation Accuracy: 0.899\n",
      "Epoch: 2 / Batch: 45568/60000 / Cost: 0.0540006 / Training Accuracy: 0.912109 / Validation Accuracy: 0.882\n",
      "Epoch: 2 / Batch: 46080/60000 / Cost: 0.0519978 / Training Accuracy: 0.919922 / Validation Accuracy: 0.88\n",
      "Epoch: 2 / Batch: 46592/60000 / Cost: 0.0664737 / Training Accuracy: 0.886719 / Validation Accuracy: 0.876\n",
      "Epoch: 2 / Batch: 47104/60000 / Cost: 0.063695 / Training Accuracy: 0.886719 / Validation Accuracy: 0.903\n",
      "Epoch: 2 / Batch: 47616/60000 / Cost: 0.0590501 / Training Accuracy: 0.912109 / Validation Accuracy: 0.902\n",
      "Epoch: 2 / Batch: 48128/60000 / Cost: 0.0561289 / Training Accuracy: 0.892578 / Validation Accuracy: 0.884\n",
      "Epoch: 2 / Batch: 48640/60000 / Cost: 0.0542111 / Training Accuracy: 0.894531 / Validation Accuracy: 0.894\n",
      "Epoch: 2 / Batch: 49152/60000 / Cost: 0.0481683 / Training Accuracy: 0.921875 / Validation Accuracy: 0.907\n",
      "Epoch: 2 / Batch: 49664/60000 / Cost: 0.0452329 / Training Accuracy: 0.929688 / Validation Accuracy: 0.911\n",
      "Epoch: 2 / Batch: 50176/60000 / Cost: 0.0527109 / Training Accuracy: 0.912109 / Validation Accuracy: 0.918\n",
      "Epoch: 2 / Batch: 50688/60000 / Cost: 0.047348 / Training Accuracy: 0.917969 / Validation Accuracy: 0.921\n",
      "Epoch: 2 / Batch: 51200/60000 / Cost: 0.0456855 / Training Accuracy: 0.925781 / Validation Accuracy: 0.909\n",
      "Epoch: 2 / Batch: 51712/60000 / Cost: 0.0513004 / Training Accuracy: 0.908203 / Validation Accuracy: 0.901\n",
      "Epoch: 2 / Batch: 52224/60000 / Cost: 0.0503902 / Training Accuracy: 0.919922 / Validation Accuracy: 0.912\n",
      "Epoch: 2 / Batch: 52736/60000 / Cost: 0.0428875 / Training Accuracy: 0.9375 / Validation Accuracy: 0.914\n",
      "Epoch: 2 / Batch: 53248/60000 / Cost: 0.054129 / Training Accuracy: 0.916016 / Validation Accuracy: 0.917\n",
      "Epoch: 2 / Batch: 53760/60000 / Cost: 0.0457873 / Training Accuracy: 0.927734 / Validation Accuracy: 0.911\n",
      "Epoch: 2 / Batch: 54272/60000 / Cost: 0.0485064 / Training Accuracy: 0.916016 / Validation Accuracy: 0.913\n",
      "Epoch: 2 / Batch: 54784/60000 / Cost: 0.0547869 / Training Accuracy: 0.912109 / Validation Accuracy: 0.904\n",
      "Epoch: 2 / Batch: 55296/60000 / Cost: 0.0520726 / Training Accuracy: 0.919922 / Validation Accuracy: 0.914\n",
      "Epoch: 2 / Batch: 55808/60000 / Cost: 0.0430402 / Training Accuracy: 0.923828 / Validation Accuracy: 0.932\n",
      "Epoch: 2 / Batch: 56320/60000 / Cost: 0.0433055 / Training Accuracy: 0.931641 / Validation Accuracy: 0.925\n",
      "Epoch: 2 / Batch: 56832/60000 / Cost: 0.0386288 / Training Accuracy: 0.947266 / Validation Accuracy: 0.907\n",
      "Epoch: 2 / Batch: 57344/60000 / Cost: 0.0438609 / Training Accuracy: 0.927734 / Validation Accuracy: 0.914\n",
      "Epoch: 2 / Batch: 57856/60000 / Cost: 0.0423844 / Training Accuracy: 0.921875 / Validation Accuracy: 0.919\n",
      "Epoch: 2 / Batch: 58368/60000 / Cost: 0.0463242 / Training Accuracy: 0.931641 / Validation Accuracy: 0.916\n",
      "Epoch: 2 / Batch: 58880/60000 / Cost: 0.0428336 / Training Accuracy: 0.933594 / Validation Accuracy: 0.911\n",
      "Epoch: 2 / Batch: 59392/60000 / Cost: 0.0501107 / Training Accuracy: 0.908203 / Validation Accuracy: 0.914\n",
      "Epoch: 2 / Batch: 59904/60000 / Cost: 0.0416991 / Training Accuracy: 0.9375 / Validation Accuracy: 0.895\n",
      "Epoch: 3 / Batch: 0/60000 / Cost: 0.0522587 / Training Accuracy: 0.914063 / Validation Accuracy: 0.89\n",
      "Epoch: 3 / Batch: 512/60000 / Cost: 0.0550598 / Training Accuracy: 0.900391 / Validation Accuracy: 0.878\n",
      "Epoch: 3 / Batch: 1024/60000 / Cost: 0.06998 / Training Accuracy: 0.890625 / Validation Accuracy: 0.855\n",
      "Epoch: 3 / Batch: 1536/60000 / Cost: 0.0432473 / Training Accuracy: 0.933594 / Validation Accuracy: 0.919\n",
      "Epoch: 3 / Batch: 2048/60000 / Cost: 0.0631197 / Training Accuracy: 0.886719 / Validation Accuracy: 0.875\n",
      "Epoch: 3 / Batch: 2560/60000 / Cost: 0.0585412 / Training Accuracy: 0.90625 / Validation Accuracy: 0.902\n",
      "Epoch: 3 / Batch: 3072/60000 / Cost: 0.0498706 / Training Accuracy: 0.921875 / Validation Accuracy: 0.905\n",
      "Epoch: 3 / Batch: 3584/60000 / Cost: 0.0478378 / Training Accuracy: 0.916016 / Validation Accuracy: 0.894\n",
      "Epoch: 3 / Batch: 4096/60000 / Cost: 0.0464775 / Training Accuracy: 0.921875 / Validation Accuracy: 0.895\n",
      "Epoch: 3 / Batch: 4608/60000 / Cost: 0.0448772 / Training Accuracy: 0.923828 / Validation Accuracy: 0.905\n",
      "Epoch: 3 / Batch: 5120/60000 / Cost: 0.0520203 / Training Accuracy: 0.916016 / Validation Accuracy: 0.91\n",
      "Epoch: 3 / Batch: 5632/60000 / Cost: 0.0410071 / Training Accuracy: 0.927734 / Validation Accuracy: 0.911\n",
      "Epoch: 3 / Batch: 6144/60000 / Cost: 0.0420816 / Training Accuracy: 0.9375 / Validation Accuracy: 0.915\n",
      "Epoch: 3 / Batch: 6656/60000 / Cost: 0.0584608 / Training Accuracy: 0.912109 / Validation Accuracy: 0.911\n",
      "Epoch: 3 / Batch: 7168/60000 / Cost: 0.0472734 / Training Accuracy: 0.931641 / Validation Accuracy: 0.919\n",
      "Epoch: 3 / Batch: 7680/60000 / Cost: 0.0425759 / Training Accuracy: 0.9375 / Validation Accuracy: 0.908\n",
      "Epoch: 3 / Batch: 8192/60000 / Cost: 0.0464404 / Training Accuracy: 0.927734 / Validation Accuracy: 0.918\n",
      "Epoch: 3 / Batch: 8704/60000 / Cost: 0.0449448 / Training Accuracy: 0.933594 / Validation Accuracy: 0.909\n",
      "Epoch: 3 / Batch: 9216/60000 / Cost: 0.0413272 / Training Accuracy: 0.935547 / Validation Accuracy: 0.904\n",
      "Epoch: 3 / Batch: 9728/60000 / Cost: 0.0394106 / Training Accuracy: 0.929688 / Validation Accuracy: 0.892\n",
      "Epoch: 3 / Batch: 10240/60000 / Cost: 0.043165 / Training Accuracy: 0.935547 / Validation Accuracy: 0.903\n",
      "Epoch: 3 / Batch: 10752/60000 / Cost: 0.0474863 / Training Accuracy: 0.914063 / Validation Accuracy: 0.914\n",
      "Epoch: 3 / Batch: 11264/60000 / Cost: 0.036559 / Training Accuracy: 0.941406 / Validation Accuracy: 0.922\n",
      "Epoch: 3 / Batch: 11776/60000 / Cost: 0.0400813 / Training Accuracy: 0.933594 / Validation Accuracy: 0.922\n",
      "Epoch: 3 / Batch: 12288/60000 / Cost: 0.0461261 / Training Accuracy: 0.935547 / Validation Accuracy: 0.905\n",
      "Epoch: 3 / Batch: 12800/60000 / Cost: 0.0404516 / Training Accuracy: 0.941406 / Validation Accuracy: 0.9\n",
      "Epoch: 3 / Batch: 13312/60000 / Cost: 0.043171 / Training Accuracy: 0.925781 / Validation Accuracy: 0.92\n",
      "Epoch: 3 / Batch: 13824/60000 / Cost: 0.0400418 / Training Accuracy: 0.933594 / Validation Accuracy: 0.924\n",
      "Epoch: 3 / Batch: 14336/60000 / Cost: 0.0361744 / Training Accuracy: 0.927734 / Validation Accuracy: 0.919\n",
      "Epoch: 3 / Batch: 14848/60000 / Cost: 0.03744 / Training Accuracy: 0.939453 / Validation Accuracy: 0.914\n",
      "Epoch: 3 / Batch: 15360/60000 / Cost: 0.0439921 / Training Accuracy: 0.941406 / Validation Accuracy: 0.921\n",
      "Epoch: 3 / Batch: 15872/60000 / Cost: 0.0417715 / Training Accuracy: 0.923828 / Validation Accuracy: 0.924\n",
      "Epoch: 3 / Batch: 16384/60000 / Cost: 0.0343889 / Training Accuracy: 0.941406 / Validation Accuracy: 0.923\n",
      "Epoch: 3 / Batch: 16896/60000 / Cost: 0.0434107 / Training Accuracy: 0.9375 / Validation Accuracy: 0.935\n",
      "Epoch: 3 / Batch: 17408/60000 / Cost: 0.0383509 / Training Accuracy: 0.939453 / Validation Accuracy: 0.934\n",
      "Epoch: 3 / Batch: 17920/60000 / Cost: 0.0423323 / Training Accuracy: 0.917969 / Validation Accuracy: 0.933\n",
      "Epoch: 3 / Batch: 18432/60000 / Cost: 0.0317906 / Training Accuracy: 0.945313 / Validation Accuracy: 0.935\n",
      "Epoch: 3 / Batch: 18944/60000 / Cost: 0.0321116 / Training Accuracy: 0.951172 / Validation Accuracy: 0.932\n",
      "Epoch: 3 / Batch: 19456/60000 / Cost: 0.0379212 / Training Accuracy: 0.947266 / Validation Accuracy: 0.923\n",
      "Epoch: 3 / Batch: 19968/60000 / Cost: 0.0275327 / Training Accuracy: 0.953125 / Validation Accuracy: 0.928\n",
      "Epoch: 3 / Batch: 20480/60000 / Cost: 0.0385875 / Training Accuracy: 0.929688 / Validation Accuracy: 0.926\n",
      "Epoch: 3 / Batch: 20992/60000 / Cost: 0.040254 / Training Accuracy: 0.935547 / Validation Accuracy: 0.93\n",
      "Epoch: 3 / Batch: 21504/60000 / Cost: 0.0291187 / Training Accuracy: 0.945313 / Validation Accuracy: 0.94\n",
      "Epoch: 3 / Batch: 22016/60000 / Cost: 0.0327011 / Training Accuracy: 0.939453 / Validation Accuracy: 0.94\n",
      "Epoch: 3 / Batch: 22528/60000 / Cost: 0.0361419 / Training Accuracy: 0.953125 / Validation Accuracy: 0.938\n",
      "Epoch: 3 / Batch: 23040/60000 / Cost: 0.0388265 / Training Accuracy: 0.935547 / Validation Accuracy: 0.943\n",
      "Epoch: 3 / Batch: 23552/60000 / Cost: 0.0316697 / Training Accuracy: 0.951172 / Validation Accuracy: 0.942\n",
      "Epoch: 3 / Batch: 24064/60000 / Cost: 0.03046 / Training Accuracy: 0.955078 / Validation Accuracy: 0.942\n",
      "Epoch: 3 / Batch: 24576/60000 / Cost: 0.0423248 / Training Accuracy: 0.9375 / Validation Accuracy: 0.946\n",
      "Epoch: 3 / Batch: 25088/60000 / Cost: 0.0351789 / Training Accuracy: 0.945313 / Validation Accuracy: 0.94\n",
      "Epoch: 3 / Batch: 25600/60000 / Cost: 0.0378161 / Training Accuracy: 0.945313 / Validation Accuracy: 0.934\n",
      "Epoch: 3 / Batch: 26112/60000 / Cost: 0.0437904 / Training Accuracy: 0.935547 / Validation Accuracy: 0.939\n",
      "Epoch: 3 / Batch: 26624/60000 / Cost: 0.0306544 / Training Accuracy: 0.951172 / Validation Accuracy: 0.938\n",
      "Epoch: 3 / Batch: 27136/60000 / Cost: 0.0301195 / Training Accuracy: 0.945313 / Validation Accuracy: 0.938\n",
      "Epoch: 3 / Batch: 27648/60000 / Cost: 0.0290558 / Training Accuracy: 0.949219 / Validation Accuracy: 0.943\n",
      "Epoch: 3 / Batch: 28160/60000 / Cost: 0.0413863 / Training Accuracy: 0.929688 / Validation Accuracy: 0.946\n",
      "Epoch: 3 / Batch: 28672/60000 / Cost: 0.0313197 / Training Accuracy: 0.955078 / Validation Accuracy: 0.946\n",
      "Epoch: 3 / Batch: 29184/60000 / Cost: 0.0313177 / Training Accuracy: 0.953125 / Validation Accuracy: 0.946\n",
      "Epoch: 3 / Batch: 29696/60000 / Cost: 0.0327339 / Training Accuracy: 0.945313 / Validation Accuracy: 0.949\n",
      "Epoch: 3 / Batch: 30208/60000 / Cost: 0.0348987 / Training Accuracy: 0.943359 / Validation Accuracy: 0.949\n",
      "Epoch: 3 / Batch: 30720/60000 / Cost: 0.0264875 / Training Accuracy: 0.957031 / Validation Accuracy: 0.943\n",
      "Epoch: 3 / Batch: 31232/60000 / Cost: 0.0287933 / Training Accuracy: 0.962891 / Validation Accuracy: 0.936\n",
      "Epoch: 3 / Batch: 31744/60000 / Cost: 0.0263665 / Training Accuracy: 0.958984 / Validation Accuracy: 0.935\n",
      "Epoch: 3 / Batch: 32256/60000 / Cost: 0.0321693 / Training Accuracy: 0.947266 / Validation Accuracy: 0.941\n",
      "Epoch: 3 / Batch: 32768/60000 / Cost: 0.0405434 / Training Accuracy: 0.931641 / Validation Accuracy: 0.938\n",
      "Epoch: 3 / Batch: 33280/60000 / Cost: 0.0266803 / Training Accuracy: 0.957031 / Validation Accuracy: 0.942\n",
      "Epoch: 3 / Batch: 33792/60000 / Cost: 0.0272089 / Training Accuracy: 0.960938 / Validation Accuracy: 0.941\n",
      "Epoch: 3 / Batch: 34304/60000 / Cost: 0.0268304 / Training Accuracy: 0.951172 / Validation Accuracy: 0.941\n",
      "Epoch: 3 / Batch: 34816/60000 / Cost: 0.0422096 / Training Accuracy: 0.925781 / Validation Accuracy: 0.943\n",
      "Epoch: 3 / Batch: 35328/60000 / Cost: 0.0294517 / Training Accuracy: 0.949219 / Validation Accuracy: 0.944\n",
      "Epoch: 3 / Batch: 35840/60000 / Cost: 0.0372235 / Training Accuracy: 0.929688 / Validation Accuracy: 0.947\n",
      "Epoch: 3 / Batch: 36352/60000 / Cost: 0.034818 / Training Accuracy: 0.949219 / Validation Accuracy: 0.945\n",
      "Epoch: 3 / Batch: 36864/60000 / Cost: 0.0316501 / Training Accuracy: 0.935547 / Validation Accuracy: 0.937\n",
      "Epoch: 3 / Batch: 37376/60000 / Cost: 0.0338678 / Training Accuracy: 0.949219 / Validation Accuracy: 0.942\n",
      "Epoch: 3 / Batch: 37888/60000 / Cost: 0.028747 / Training Accuracy: 0.949219 / Validation Accuracy: 0.949\n",
      "Epoch: 3 / Batch: 38400/60000 / Cost: 0.0383412 / Training Accuracy: 0.941406 / Validation Accuracy: 0.939\n",
      "Epoch: 3 / Batch: 38912/60000 / Cost: 0.0347182 / Training Accuracy: 0.949219 / Validation Accuracy: 0.945\n",
      "Epoch: 3 / Batch: 39424/60000 / Cost: 0.0373675 / Training Accuracy: 0.935547 / Validation Accuracy: 0.94\n",
      "Epoch: 3 / Batch: 39936/60000 / Cost: 0.0305432 / Training Accuracy: 0.943359 / Validation Accuracy: 0.938\n",
      "Epoch: 3 / Batch: 40448/60000 / Cost: 0.038622 / Training Accuracy: 0.9375 / Validation Accuracy: 0.937\n",
      "Epoch: 3 / Batch: 40960/60000 / Cost: 0.0335194 / Training Accuracy: 0.939453 / Validation Accuracy: 0.936\n",
      "Epoch: 3 / Batch: 41472/60000 / Cost: 0.0321309 / Training Accuracy: 0.949219 / Validation Accuracy: 0.931\n",
      "Epoch: 3 / Batch: 41984/60000 / Cost: 0.0270055 / Training Accuracy: 0.960938 / Validation Accuracy: 0.942\n",
      "Epoch: 3 / Batch: 42496/60000 / Cost: 0.0260781 / Training Accuracy: 0.958984 / Validation Accuracy: 0.946\n",
      "Epoch: 3 / Batch: 43008/60000 / Cost: 0.029023 / Training Accuracy: 0.955078 / Validation Accuracy: 0.938\n",
      "Epoch: 3 / Batch: 43520/60000 / Cost: 0.0415881 / Training Accuracy: 0.933594 / Validation Accuracy: 0.941\n",
      "Epoch: 3 / Batch: 44032/60000 / Cost: 0.0359906 / Training Accuracy: 0.941406 / Validation Accuracy: 0.949\n",
      "Epoch: 3 / Batch: 44544/60000 / Cost: 0.0237915 / Training Accuracy: 0.966797 / Validation Accuracy: 0.938\n",
      "Epoch: 3 / Batch: 45056/60000 / Cost: 0.0355255 / Training Accuracy: 0.949219 / Validation Accuracy: 0.938\n",
      "Epoch: 3 / Batch: 45568/60000 / Cost: 0.0264327 / Training Accuracy: 0.966797 / Validation Accuracy: 0.948\n",
      "Epoch: 3 / Batch: 46080/60000 / Cost: 0.0348846 / Training Accuracy: 0.935547 / Validation Accuracy: 0.949\n",
      "Epoch: 3 / Batch: 46592/60000 / Cost: 0.0327509 / Training Accuracy: 0.953125 / Validation Accuracy: 0.946\n",
      "Epoch: 3 / Batch: 47104/60000 / Cost: 0.0251084 / Training Accuracy: 0.960938 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 47616/60000 / Cost: 0.0252231 / Training Accuracy: 0.964844 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 48128/60000 / Cost: 0.0242181 / Training Accuracy: 0.958984 / Validation Accuracy: 0.95\n",
      "Epoch: 3 / Batch: 48640/60000 / Cost: 0.0330683 / Training Accuracy: 0.943359 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 49152/60000 / Cost: 0.0252595 / Training Accuracy: 0.966797 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 49664/60000 / Cost: 0.0328791 / Training Accuracy: 0.943359 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 50176/60000 / Cost: 0.0278224 / Training Accuracy: 0.953125 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 50688/60000 / Cost: 0.0288243 / Training Accuracy: 0.957031 / Validation Accuracy: 0.961\n",
      "Epoch: 3 / Batch: 51200/60000 / Cost: 0.0278883 / Training Accuracy: 0.958984 / Validation Accuracy: 0.956\n",
      "Epoch: 3 / Batch: 51712/60000 / Cost: 0.0270381 / Training Accuracy: 0.949219 / Validation Accuracy: 0.958\n",
      "Epoch: 3 / Batch: 52224/60000 / Cost: 0.0331775 / Training Accuracy: 0.951172 / Validation Accuracy: 0.949\n",
      "Epoch: 3 / Batch: 52736/60000 / Cost: 0.0313656 / Training Accuracy: 0.957031 / Validation Accuracy: 0.952\n",
      "Epoch: 3 / Batch: 53248/60000 / Cost: 0.0180043 / Training Accuracy: 0.978516 / Validation Accuracy: 0.948\n",
      "Epoch: 3 / Batch: 53760/60000 / Cost: 0.03189 / Training Accuracy: 0.951172 / Validation Accuracy: 0.947\n",
      "Epoch: 3 / Batch: 54272/60000 / Cost: 0.025957 / Training Accuracy: 0.951172 / Validation Accuracy: 0.953\n",
      "Epoch: 3 / Batch: 54784/60000 / Cost: 0.030542 / Training Accuracy: 0.947266 / Validation Accuracy: 0.948\n",
      "Epoch: 3 / Batch: 55296/60000 / Cost: 0.0224534 / Training Accuracy: 0.957031 / Validation Accuracy: 0.947\n",
      "Epoch: 3 / Batch: 55808/60000 / Cost: 0.0267084 / Training Accuracy: 0.962891 / Validation Accuracy: 0.944\n",
      "Epoch: 3 / Batch: 56320/60000 / Cost: 0.0231643 / Training Accuracy: 0.972656 / Validation Accuracy: 0.95\n",
      "Epoch: 3 / Batch: 56832/60000 / Cost: 0.02433 / Training Accuracy: 0.964844 / Validation Accuracy: 0.944\n",
      "Epoch: 3 / Batch: 57344/60000 / Cost: 0.0233457 / Training Accuracy: 0.962891 / Validation Accuracy: 0.945\n",
      "Epoch: 3 / Batch: 57856/60000 / Cost: 0.0324818 / Training Accuracy: 0.939453 / Validation Accuracy: 0.949\n",
      "Epoch: 3 / Batch: 58368/60000 / Cost: 0.0316287 / Training Accuracy: 0.949219 / Validation Accuracy: 0.954\n",
      "Epoch: 3 / Batch: 58880/60000 / Cost: 0.0237406 / Training Accuracy: 0.966797 / Validation Accuracy: 0.951\n",
      "Epoch: 3 / Batch: 59392/60000 / Cost: 0.0190354 / Training Accuracy: 0.978516 / Validation Accuracy: 0.957\n",
      "Epoch: 3 / Batch: 59904/60000 / Cost: 0.0294573 / Training Accuracy: 0.947917 / Validation Accuracy: 0.949\n",
      "Epoch: 4 / Batch: 0/60000 / Cost: 0.0354799 / Training Accuracy: 0.941406 / Validation Accuracy: 0.94\n",
      "Epoch: 4 / Batch: 512/60000 / Cost: 0.0290894 / Training Accuracy: 0.947266 / Validation Accuracy: 0.951\n",
      "Epoch: 4 / Batch: 1024/60000 / Cost: 0.0271495 / Training Accuracy: 0.958984 / Validation Accuracy: 0.95\n",
      "Epoch: 4 / Batch: 1536/60000 / Cost: 0.0337442 / Training Accuracy: 0.941406 / Validation Accuracy: 0.941\n",
      "Epoch: 4 / Batch: 2048/60000 / Cost: 0.0237095 / Training Accuracy: 0.958984 / Validation Accuracy: 0.942\n",
      "Epoch: 4 / Batch: 2560/60000 / Cost: 0.0296162 / Training Accuracy: 0.953125 / Validation Accuracy: 0.949\n",
      "Epoch: 4 / Batch: 3072/60000 / Cost: 0.0212854 / Training Accuracy: 0.960938 / Validation Accuracy: 0.951\n",
      "Epoch: 4 / Batch: 3584/60000 / Cost: 0.026977 / Training Accuracy: 0.960938 / Validation Accuracy: 0.95\n",
      "Epoch: 4 / Batch: 4096/60000 / Cost: 0.0240404 / Training Accuracy: 0.962891 / Validation Accuracy: 0.948\n",
      "Epoch: 4 / Batch: 4608/60000 / Cost: 0.0263147 / Training Accuracy: 0.958984 / Validation Accuracy: 0.948\n",
      "Epoch: 4 / Batch: 5120/60000 / Cost: 0.0226806 / Training Accuracy: 0.957031 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 5632/60000 / Cost: 0.0270414 / Training Accuracy: 0.949219 / Validation Accuracy: 0.951\n",
      "Epoch: 4 / Batch: 6144/60000 / Cost: 0.0319771 / Training Accuracy: 0.947266 / Validation Accuracy: 0.944\n",
      "Epoch: 4 / Batch: 6656/60000 / Cost: 0.0291212 / Training Accuracy: 0.962891 / Validation Accuracy: 0.95\n",
      "Epoch: 4 / Batch: 7168/60000 / Cost: 0.0217036 / Training Accuracy: 0.960938 / Validation Accuracy: 0.947\n",
      "Epoch: 4 / Batch: 7680/60000 / Cost: 0.0268488 / Training Accuracy: 0.957031 / Validation Accuracy: 0.943\n",
      "Epoch: 4 / Batch: 8192/60000 / Cost: 0.0243269 / Training Accuracy: 0.957031 / Validation Accuracy: 0.94\n",
      "Epoch: 4 / Batch: 8704/60000 / Cost: 0.0308833 / Training Accuracy: 0.955078 / Validation Accuracy: 0.936\n",
      "Epoch: 4 / Batch: 9216/60000 / Cost: 0.0265813 / Training Accuracy: 0.953125 / Validation Accuracy: 0.947\n",
      "Epoch: 4 / Batch: 9728/60000 / Cost: 0.0223651 / Training Accuracy: 0.962891 / Validation Accuracy: 0.946\n",
      "Epoch: 4 / Batch: 10240/60000 / Cost: 0.0267851 / Training Accuracy: 0.955078 / Validation Accuracy: 0.951\n",
      "Epoch: 4 / Batch: 10752/60000 / Cost: 0.0229326 / Training Accuracy: 0.957031 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 11264/60000 / Cost: 0.0251533 / Training Accuracy: 0.957031 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 11776/60000 / Cost: 0.0264823 / Training Accuracy: 0.947266 / Validation Accuracy: 0.947\n",
      "Epoch: 4 / Batch: 12288/60000 / Cost: 0.0226479 / Training Accuracy: 0.962891 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 12800/60000 / Cost: 0.0218753 / Training Accuracy: 0.966797 / Validation Accuracy: 0.96\n",
      "Epoch: 4 / Batch: 13312/60000 / Cost: 0.0263134 / Training Accuracy: 0.958984 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 13824/60000 / Cost: 0.0235694 / Training Accuracy: 0.964844 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 14336/60000 / Cost: 0.0259135 / Training Accuracy: 0.955078 / Validation Accuracy: 0.954\n",
      "Epoch: 4 / Batch: 14848/60000 / Cost: 0.0249685 / Training Accuracy: 0.955078 / Validation Accuracy: 0.954\n",
      "Epoch: 4 / Batch: 15360/60000 / Cost: 0.0334742 / Training Accuracy: 0.949219 / Validation Accuracy: 0.949\n",
      "Epoch: 4 / Batch: 15872/60000 / Cost: 0.0293863 / Training Accuracy: 0.949219 / Validation Accuracy: 0.946\n",
      "Epoch: 4 / Batch: 16384/60000 / Cost: 0.023134 / Training Accuracy: 0.964844 / Validation Accuracy: 0.95\n",
      "Epoch: 4 / Batch: 16896/60000 / Cost: 0.0209256 / Training Accuracy: 0.962891 / Validation Accuracy: 0.951\n",
      "Epoch: 4 / Batch: 17408/60000 / Cost: 0.0268025 / Training Accuracy: 0.957031 / Validation Accuracy: 0.945\n",
      "Epoch: 4 / Batch: 17920/60000 / Cost: 0.0232746 / Training Accuracy: 0.96875 / Validation Accuracy: 0.946\n",
      "Epoch: 4 / Batch: 18432/60000 / Cost: 0.018303 / Training Accuracy: 0.970703 / Validation Accuracy: 0.952\n",
      "Epoch: 4 / Batch: 18944/60000 / Cost: 0.0230677 / Training Accuracy: 0.964844 / Validation Accuracy: 0.954\n",
      "Epoch: 4 / Batch: 19456/60000 / Cost: 0.0161246 / Training Accuracy: 0.980469 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 19968/60000 / Cost: 0.0245177 / Training Accuracy: 0.966797 / Validation Accuracy: 0.96\n",
      "Epoch: 4 / Batch: 20480/60000 / Cost: 0.0226458 / Training Accuracy: 0.957031 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 20992/60000 / Cost: 0.0249278 / Training Accuracy: 0.955078 / Validation Accuracy: 0.96\n",
      "Epoch: 4 / Batch: 21504/60000 / Cost: 0.0229547 / Training Accuracy: 0.972656 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 22016/60000 / Cost: 0.0260597 / Training Accuracy: 0.958984 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 22528/60000 / Cost: 0.0256922 / Training Accuracy: 0.955078 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 23040/60000 / Cost: 0.0219822 / Training Accuracy: 0.958984 / Validation Accuracy: 0.96\n",
      "Epoch: 4 / Batch: 23552/60000 / Cost: 0.0326829 / Training Accuracy: 0.943359 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 24064/60000 / Cost: 0.0223408 / Training Accuracy: 0.972656 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 24576/60000 / Cost: 0.0186588 / Training Accuracy: 0.974609 / Validation Accuracy: 0.954\n",
      "Epoch: 4 / Batch: 25088/60000 / Cost: 0.0240736 / Training Accuracy: 0.966797 / Validation Accuracy: 0.954\n",
      "Epoch: 4 / Batch: 25600/60000 / Cost: 0.0184217 / Training Accuracy: 0.974609 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 26112/60000 / Cost: 0.020366 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 26624/60000 / Cost: 0.0206787 / Training Accuracy: 0.972656 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 27136/60000 / Cost: 0.0162962 / Training Accuracy: 0.972656 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 27648/60000 / Cost: 0.0162599 / Training Accuracy: 0.976563 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 28160/60000 / Cost: 0.0211092 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 28672/60000 / Cost: 0.0204832 / Training Accuracy: 0.964844 / Validation Accuracy: 0.951\n",
      "Epoch: 4 / Batch: 29184/60000 / Cost: 0.0261534 / Training Accuracy: 0.958984 / Validation Accuracy: 0.953\n",
      "Epoch: 4 / Batch: 29696/60000 / Cost: 0.0246684 / Training Accuracy: 0.966797 / Validation Accuracy: 0.949\n",
      "Epoch: 4 / Batch: 30208/60000 / Cost: 0.0258106 / Training Accuracy: 0.955078 / Validation Accuracy: 0.952\n",
      "Epoch: 4 / Batch: 30720/60000 / Cost: 0.0245844 / Training Accuracy: 0.964844 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 31232/60000 / Cost: 0.0270652 / Training Accuracy: 0.949219 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 31744/60000 / Cost: 0.0264776 / Training Accuracy: 0.958984 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 32256/60000 / Cost: 0.0267557 / Training Accuracy: 0.960938 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 32768/60000 / Cost: 0.0323782 / Training Accuracy: 0.949219 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 33280/60000 / Cost: 0.0204182 / Training Accuracy: 0.96875 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 33792/60000 / Cost: 0.0221203 / Training Accuracy: 0.966797 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 34304/60000 / Cost: 0.0261825 / Training Accuracy: 0.953125 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 34816/60000 / Cost: 0.0182318 / Training Accuracy: 0.96875 / Validation Accuracy: 0.966\n",
      "Epoch: 4 / Batch: 35328/60000 / Cost: 0.0295554 / Training Accuracy: 0.949219 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 35840/60000 / Cost: 0.023565 / Training Accuracy: 0.962891 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 36352/60000 / Cost: 0.0157313 / Training Accuracy: 0.980469 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 36864/60000 / Cost: 0.0248945 / Training Accuracy: 0.96875 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 37376/60000 / Cost: 0.0202514 / Training Accuracy: 0.966797 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 37888/60000 / Cost: 0.0203624 / Training Accuracy: 0.96875 / Validation Accuracy: 0.954\n",
      "Epoch: 4 / Batch: 38400/60000 / Cost: 0.0253773 / Training Accuracy: 0.957031 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 38912/60000 / Cost: 0.0251733 / Training Accuracy: 0.96875 / Validation Accuracy: 0.954\n",
      "Epoch: 4 / Batch: 39424/60000 / Cost: 0.020781 / Training Accuracy: 0.970703 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 39936/60000 / Cost: 0.0240639 / Training Accuracy: 0.962891 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 40448/60000 / Cost: 0.0221802 / Training Accuracy: 0.962891 / Validation Accuracy: 0.954\n",
      "Epoch: 4 / Batch: 40960/60000 / Cost: 0.0236294 / Training Accuracy: 0.970703 / Validation Accuracy: 0.953\n",
      "Epoch: 4 / Batch: 41472/60000 / Cost: 0.0160615 / Training Accuracy: 0.982422 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 41984/60000 / Cost: 0.0262936 / Training Accuracy: 0.953125 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 42496/60000 / Cost: 0.0228964 / Training Accuracy: 0.962891 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 43008/60000 / Cost: 0.0210159 / Training Accuracy: 0.974609 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 43520/60000 / Cost: 0.0217471 / Training Accuracy: 0.962891 / Validation Accuracy: 0.962\n",
      "Epoch: 4 / Batch: 44032/60000 / Cost: 0.0176615 / Training Accuracy: 0.974609 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 44544/60000 / Cost: 0.0191178 / Training Accuracy: 0.96875 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 45056/60000 / Cost: 0.0176558 / Training Accuracy: 0.96875 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 45568/60000 / Cost: 0.02005 / Training Accuracy: 0.964844 / Validation Accuracy: 0.95\n",
      "Epoch: 4 / Batch: 46080/60000 / Cost: 0.0266402 / Training Accuracy: 0.96875 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 46592/60000 / Cost: 0.0215713 / Training Accuracy: 0.964844 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 47104/60000 / Cost: 0.0233608 / Training Accuracy: 0.957031 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 47616/60000 / Cost: 0.0214102 / Training Accuracy: 0.964844 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 48128/60000 / Cost: 0.0225419 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 48640/60000 / Cost: 0.0152865 / Training Accuracy: 0.974609 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 49152/60000 / Cost: 0.0166003 / Training Accuracy: 0.974609 / Validation Accuracy: 0.96\n",
      "Epoch: 4 / Batch: 49664/60000 / Cost: 0.0190606 / Training Accuracy: 0.970703 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 50176/60000 / Cost: 0.0223549 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 4 / Batch: 50688/60000 / Cost: 0.0237627 / Training Accuracy: 0.96875 / Validation Accuracy: 0.959\n",
      "Epoch: 4 / Batch: 51200/60000 / Cost: 0.0192981 / Training Accuracy: 0.966797 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 51712/60000 / Cost: 0.0219233 / Training Accuracy: 0.964844 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 52224/60000 / Cost: 0.0230241 / Training Accuracy: 0.958984 / Validation Accuracy: 0.955\n",
      "Epoch: 4 / Batch: 52736/60000 / Cost: 0.0204997 / Training Accuracy: 0.962891 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 53248/60000 / Cost: 0.0215773 / Training Accuracy: 0.960938 / Validation Accuracy: 0.965\n",
      "Epoch: 4 / Batch: 53760/60000 / Cost: 0.0250054 / Training Accuracy: 0.957031 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 54272/60000 / Cost: 0.0255122 / Training Accuracy: 0.958984 / Validation Accuracy: 0.951\n",
      "Epoch: 4 / Batch: 54784/60000 / Cost: 0.027749 / Training Accuracy: 0.955078 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 55296/60000 / Cost: 0.0203873 / Training Accuracy: 0.96875 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 55808/60000 / Cost: 0.0242329 / Training Accuracy: 0.960938 / Validation Accuracy: 0.963\n",
      "Epoch: 4 / Batch: 56320/60000 / Cost: 0.021684 / Training Accuracy: 0.96875 / Validation Accuracy: 0.958\n",
      "Epoch: 4 / Batch: 56832/60000 / Cost: 0.0262141 / Training Accuracy: 0.964844 / Validation Accuracy: 0.953\n",
      "Epoch: 4 / Batch: 57344/60000 / Cost: 0.0210766 / Training Accuracy: 0.970703 / Validation Accuracy: 0.956\n",
      "Epoch: 4 / Batch: 57856/60000 / Cost: 0.0207002 / Training Accuracy: 0.970703 / Validation Accuracy: 0.96\n",
      "Epoch: 4 / Batch: 58368/60000 / Cost: 0.0207879 / Training Accuracy: 0.964844 / Validation Accuracy: 0.96\n",
      "Epoch: 4 / Batch: 58880/60000 / Cost: 0.0162915 / Training Accuracy: 0.978516 / Validation Accuracy: 0.95\n",
      "Epoch: 4 / Batch: 59392/60000 / Cost: 0.0232372 / Training Accuracy: 0.964844 / Validation Accuracy: 0.957\n",
      "Epoch: 4 / Batch: 59904/60000 / Cost: 0.0150782 / Training Accuracy: 0.979167 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 0/60000 / Cost: 0.0112047 / Training Accuracy: 0.984375 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 512/60000 / Cost: 0.0194529 / Training Accuracy: 0.972656 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 1024/60000 / Cost: 0.022033 / Training Accuracy: 0.964844 / Validation Accuracy: 0.952\n",
      "Epoch: 5 / Batch: 1536/60000 / Cost: 0.0276301 / Training Accuracy: 0.958984 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 2048/60000 / Cost: 0.0187436 / Training Accuracy: 0.974609 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 2560/60000 / Cost: 0.020446 / Training Accuracy: 0.966797 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 3072/60000 / Cost: 0.0200005 / Training Accuracy: 0.966797 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 3584/60000 / Cost: 0.0173463 / Training Accuracy: 0.966797 / Validation Accuracy: 0.954\n",
      "Epoch: 5 / Batch: 4096/60000 / Cost: 0.0225144 / Training Accuracy: 0.960938 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 4608/60000 / Cost: 0.0193365 / Training Accuracy: 0.976563 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 5120/60000 / Cost: 0.0217291 / Training Accuracy: 0.960938 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 5632/60000 / Cost: 0.027981 / Training Accuracy: 0.957031 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 6144/60000 / Cost: 0.0230398 / Training Accuracy: 0.960938 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 6656/60000 / Cost: 0.0189811 / Training Accuracy: 0.972656 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 7168/60000 / Cost: 0.0255839 / Training Accuracy: 0.966797 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 7680/60000 / Cost: 0.0176212 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 5 / Batch: 8192/60000 / Cost: 0.0216626 / Training Accuracy: 0.958984 / Validation Accuracy: 0.951\n",
      "Epoch: 5 / Batch: 8704/60000 / Cost: 0.0272667 / Training Accuracy: 0.960938 / Validation Accuracy: 0.945\n",
      "Epoch: 5 / Batch: 9216/60000 / Cost: 0.0218664 / Training Accuracy: 0.958984 / Validation Accuracy: 0.952\n",
      "Epoch: 5 / Batch: 9728/60000 / Cost: 0.0164924 / Training Accuracy: 0.976563 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 10240/60000 / Cost: 0.0212425 / Training Accuracy: 0.964844 / Validation Accuracy: 0.953\n",
      "Epoch: 5 / Batch: 10752/60000 / Cost: 0.0250562 / Training Accuracy: 0.962891 / Validation Accuracy: 0.933\n",
      "Epoch: 5 / Batch: 11264/60000 / Cost: 0.0204784 / Training Accuracy: 0.96875 / Validation Accuracy: 0.952\n",
      "Epoch: 5 / Batch: 11776/60000 / Cost: 0.0174414 / Training Accuracy: 0.972656 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 12288/60000 / Cost: 0.0152825 / Training Accuracy: 0.970703 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 12800/60000 / Cost: 0.0233609 / Training Accuracy: 0.966797 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 13312/60000 / Cost: 0.0237883 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 5 / Batch: 13824/60000 / Cost: 0.020069 / Training Accuracy: 0.96875 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 14336/60000 / Cost: 0.0262701 / Training Accuracy: 0.958984 / Validation Accuracy: 0.954\n",
      "Epoch: 5 / Batch: 14848/60000 / Cost: 0.0161002 / Training Accuracy: 0.972656 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 15360/60000 / Cost: 0.0226177 / Training Accuracy: 0.964844 / Validation Accuracy: 0.951\n",
      "Epoch: 5 / Batch: 15872/60000 / Cost: 0.0181812 / Training Accuracy: 0.972656 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 16384/60000 / Cost: 0.0153209 / Training Accuracy: 0.970703 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 16896/60000 / Cost: 0.0266978 / Training Accuracy: 0.960938 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 17408/60000 / Cost: 0.0165448 / Training Accuracy: 0.974609 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 17920/60000 / Cost: 0.0173817 / Training Accuracy: 0.980469 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 18432/60000 / Cost: 0.0165687 / Training Accuracy: 0.970703 / Validation Accuracy: 0.954\n",
      "Epoch: 5 / Batch: 18944/60000 / Cost: 0.0231814 / Training Accuracy: 0.960938 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 19456/60000 / Cost: 0.014011 / Training Accuracy: 0.978516 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 19968/60000 / Cost: 0.0214774 / Training Accuracy: 0.960938 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 20480/60000 / Cost: 0.0187717 / Training Accuracy: 0.96875 / Validation Accuracy: 0.964\n",
      "Epoch: 5 / Batch: 20992/60000 / Cost: 0.0186698 / Training Accuracy: 0.96875 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 21504/60000 / Cost: 0.0193734 / Training Accuracy: 0.966797 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 22016/60000 / Cost: 0.0222805 / Training Accuracy: 0.951172 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 22528/60000 / Cost: 0.0181699 / Training Accuracy: 0.96875 / Validation Accuracy: 0.961\n",
      "Epoch: 5 / Batch: 23040/60000 / Cost: 0.018741 / Training Accuracy: 0.970703 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 23552/60000 / Cost: 0.0200601 / Training Accuracy: 0.966797 / Validation Accuracy: 0.963\n",
      "Epoch: 5 / Batch: 24064/60000 / Cost: 0.0194847 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 24576/60000 / Cost: 0.0141482 / Training Accuracy: 0.976563 / Validation Accuracy: 0.963\n",
      "Epoch: 5 / Batch: 25088/60000 / Cost: 0.0214907 / Training Accuracy: 0.96875 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 25600/60000 / Cost: 0.0154835 / Training Accuracy: 0.970703 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 26112/60000 / Cost: 0.0264239 / Training Accuracy: 0.949219 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 26624/60000 / Cost: 0.0229616 / Training Accuracy: 0.962891 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 27136/60000 / Cost: 0.0206672 / Training Accuracy: 0.962891 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 27648/60000 / Cost: 0.0156748 / Training Accuracy: 0.978516 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 28160/60000 / Cost: 0.0194842 / Training Accuracy: 0.960938 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 28672/60000 / Cost: 0.0177775 / Training Accuracy: 0.976563 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 29184/60000 / Cost: 0.0170173 / Training Accuracy: 0.970703 / Validation Accuracy: 0.963\n",
      "Epoch: 5 / Batch: 29696/60000 / Cost: 0.0154271 / Training Accuracy: 0.980469 / Validation Accuracy: 0.963\n",
      "Epoch: 5 / Batch: 30208/60000 / Cost: 0.0172363 / Training Accuracy: 0.970703 / Validation Accuracy: 0.951\n",
      "Epoch: 5 / Batch: 30720/60000 / Cost: 0.0247174 / Training Accuracy: 0.957031 / Validation Accuracy: 0.954\n",
      "Epoch: 5 / Batch: 31232/60000 / Cost: 0.0151084 / Training Accuracy: 0.974609 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 31744/60000 / Cost: 0.0221518 / Training Accuracy: 0.957031 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 32256/60000 / Cost: 0.0202291 / Training Accuracy: 0.970703 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 32768/60000 / Cost: 0.016624 / Training Accuracy: 0.972656 / Validation Accuracy: 0.955\n",
      "Epoch: 5 / Batch: 33280/60000 / Cost: 0.0177129 / Training Accuracy: 0.966797 / Validation Accuracy: 0.961\n",
      "Epoch: 5 / Batch: 33792/60000 / Cost: 0.0186822 / Training Accuracy: 0.964844 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 34304/60000 / Cost: 0.017893 / Training Accuracy: 0.970703 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 34816/60000 / Cost: 0.0201358 / Training Accuracy: 0.958984 / Validation Accuracy: 0.961\n",
      "Epoch: 5 / Batch: 35328/60000 / Cost: 0.0212342 / Training Accuracy: 0.970703 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 35840/60000 / Cost: 0.0208462 / Training Accuracy: 0.976563 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 36352/60000 / Cost: 0.0217738 / Training Accuracy: 0.960938 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 36864/60000 / Cost: 0.0196086 / Training Accuracy: 0.970703 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 37376/60000 / Cost: 0.0201839 / Training Accuracy: 0.962891 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 37888/60000 / Cost: 0.0214407 / Training Accuracy: 0.962891 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 38400/60000 / Cost: 0.0217544 / Training Accuracy: 0.966797 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 38912/60000 / Cost: 0.0176196 / Training Accuracy: 0.96875 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 39424/60000 / Cost: 0.0166871 / Training Accuracy: 0.976563 / Validation Accuracy: 0.963\n",
      "Epoch: 5 / Batch: 39936/60000 / Cost: 0.0177966 / Training Accuracy: 0.960938 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 40448/60000 / Cost: 0.017557 / Training Accuracy: 0.96875 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 40960/60000 / Cost: 0.015409 / Training Accuracy: 0.974609 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 41472/60000 / Cost: 0.0142939 / Training Accuracy: 0.978516 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 41984/60000 / Cost: 0.0163943 / Training Accuracy: 0.976563 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 42496/60000 / Cost: 0.0187776 / Training Accuracy: 0.976563 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 43008/60000 / Cost: 0.0188099 / Training Accuracy: 0.974609 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 43520/60000 / Cost: 0.0185144 / Training Accuracy: 0.972656 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 44032/60000 / Cost: 0.0187856 / Training Accuracy: 0.966797 / Validation Accuracy: 0.961\n",
      "Epoch: 5 / Batch: 44544/60000 / Cost: 0.0218682 / Training Accuracy: 0.960938 / Validation Accuracy: 0.957\n",
      "Epoch: 5 / Batch: 45056/60000 / Cost: 0.0161465 / Training Accuracy: 0.974609 / Validation Accuracy: 0.964\n",
      "Epoch: 5 / Batch: 45568/60000 / Cost: 0.0207536 / Training Accuracy: 0.96875 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 46080/60000 / Cost: 0.0238445 / Training Accuracy: 0.957031 / Validation Accuracy: 0.963\n",
      "Epoch: 5 / Batch: 46592/60000 / Cost: 0.015946 / Training Accuracy: 0.978516 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 47104/60000 / Cost: 0.0223981 / Training Accuracy: 0.964844 / Validation Accuracy: 0.961\n",
      "Epoch: 5 / Batch: 47616/60000 / Cost: 0.0232893 / Training Accuracy: 0.964844 / Validation Accuracy: 0.956\n",
      "Epoch: 5 / Batch: 48128/60000 / Cost: 0.0228235 / Training Accuracy: 0.962891 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 48640/60000 / Cost: 0.0182554 / Training Accuracy: 0.972656 / Validation Accuracy: 0.969\n",
      "Epoch: 5 / Batch: 49152/60000 / Cost: 0.0180623 / Training Accuracy: 0.970703 / Validation Accuracy: 0.958\n",
      "Epoch: 5 / Batch: 49664/60000 / Cost: 0.0218801 / Training Accuracy: 0.958984 / Validation Accuracy: 0.949\n",
      "Epoch: 5 / Batch: 50176/60000 / Cost: 0.0165465 / Training Accuracy: 0.96875 / Validation Accuracy: 0.959\n",
      "Epoch: 5 / Batch: 50688/60000 / Cost: 0.0125749 / Training Accuracy: 0.982422 / Validation Accuracy: 0.967\n",
      "Epoch: 5 / Batch: 51200/60000 / Cost: 0.0196128 / Training Accuracy: 0.966797 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 51712/60000 / Cost: 0.0200895 / Training Accuracy: 0.964844 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 52224/60000 / Cost: 0.0171468 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 52736/60000 / Cost: 0.0133314 / Training Accuracy: 0.984375 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 53248/60000 / Cost: 0.0100544 / Training Accuracy: 0.986328 / Validation Accuracy: 0.964\n",
      "Epoch: 5 / Batch: 53760/60000 / Cost: 0.0189181 / Training Accuracy: 0.970703 / Validation Accuracy: 0.964\n",
      "Epoch: 5 / Batch: 54272/60000 / Cost: 0.0171832 / Training Accuracy: 0.976563 / Validation Accuracy: 0.964\n",
      "Epoch: 5 / Batch: 54784/60000 / Cost: 0.0149269 / Training Accuracy: 0.976563 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 55296/60000 / Cost: 0.0155405 / Training Accuracy: 0.972656 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 55808/60000 / Cost: 0.0242558 / Training Accuracy: 0.955078 / Validation Accuracy: 0.962\n",
      "Epoch: 5 / Batch: 56320/60000 / Cost: 0.0160045 / Training Accuracy: 0.96875 / Validation Accuracy: 0.96\n",
      "Epoch: 5 / Batch: 56832/60000 / Cost: 0.0188228 / Training Accuracy: 0.966797 / Validation Accuracy: 0.964\n",
      "Epoch: 5 / Batch: 57344/60000 / Cost: 0.017799 / Training Accuracy: 0.966797 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 57856/60000 / Cost: 0.0147826 / Training Accuracy: 0.972656 / Validation Accuracy: 0.97\n",
      "Epoch: 5 / Batch: 58368/60000 / Cost: 0.0170671 / Training Accuracy: 0.970703 / Validation Accuracy: 0.968\n",
      "Epoch: 5 / Batch: 58880/60000 / Cost: 0.0165707 / Training Accuracy: 0.972656 / Validation Accuracy: 0.966\n",
      "Epoch: 5 / Batch: 59392/60000 / Cost: 0.0200645 / Training Accuracy: 0.964844 / Validation Accuracy: 0.965\n",
      "Epoch: 5 / Batch: 59904/60000 / Cost: 0.00408557 / Training Accuracy: 1.0 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 0/60000 / Cost: 0.0151045 / Training Accuracy: 0.972656 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 512/60000 / Cost: 0.0234888 / Training Accuracy: 0.964844 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 1024/60000 / Cost: 0.0142422 / Training Accuracy: 0.980469 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 1536/60000 / Cost: 0.0147505 / Training Accuracy: 0.974609 / Validation Accuracy: 0.953\n",
      "Epoch: 6 / Batch: 2048/60000 / Cost: 0.0224929 / Training Accuracy: 0.960938 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 2560/60000 / Cost: 0.0166883 / Training Accuracy: 0.96875 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 3072/60000 / Cost: 0.0166281 / Training Accuracy: 0.978516 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 3584/60000 / Cost: 0.0170099 / Training Accuracy: 0.96875 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 4096/60000 / Cost: 0.0165911 / Training Accuracy: 0.96875 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 4608/60000 / Cost: 0.0132818 / Training Accuracy: 0.976563 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 5120/60000 / Cost: 0.0153902 / Training Accuracy: 0.982422 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 5632/60000 / Cost: 0.0241936 / Training Accuracy: 0.962891 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 6144/60000 / Cost: 0.0174682 / Training Accuracy: 0.970703 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 6656/60000 / Cost: 0.0134142 / Training Accuracy: 0.978516 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 7168/60000 / Cost: 0.0119229 / Training Accuracy: 0.990234 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 7680/60000 / Cost: 0.0212048 / Training Accuracy: 0.962891 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 8192/60000 / Cost: 0.017921 / Training Accuracy: 0.980469 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 8704/60000 / Cost: 0.0163564 / Training Accuracy: 0.974609 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 9216/60000 / Cost: 0.0187272 / Training Accuracy: 0.974609 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 9728/60000 / Cost: 0.0175804 / Training Accuracy: 0.980469 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 10240/60000 / Cost: 0.0152587 / Training Accuracy: 0.980469 / Validation Accuracy: 0.954\n",
      "Epoch: 6 / Batch: 10752/60000 / Cost: 0.0142932 / Training Accuracy: 0.972656 / Validation Accuracy: 0.959\n",
      "Epoch: 6 / Batch: 11264/60000 / Cost: 0.0109099 / Training Accuracy: 0.984375 / Validation Accuracy: 0.959\n",
      "Epoch: 6 / Batch: 11776/60000 / Cost: 0.0213964 / Training Accuracy: 0.960938 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 12288/60000 / Cost: 0.012692 / Training Accuracy: 0.976563 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 12800/60000 / Cost: 0.0107937 / Training Accuracy: 0.990234 / Validation Accuracy: 0.96\n",
      "Epoch: 6 / Batch: 13312/60000 / Cost: 0.0163015 / Training Accuracy: 0.96875 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 13824/60000 / Cost: 0.0179634 / Training Accuracy: 0.976563 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 14336/60000 / Cost: 0.0201366 / Training Accuracy: 0.974609 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 14848/60000 / Cost: 0.0200282 / Training Accuracy: 0.972656 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 15360/60000 / Cost: 0.0196139 / Training Accuracy: 0.964844 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 15872/60000 / Cost: 0.0169552 / Training Accuracy: 0.974609 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 16384/60000 / Cost: 0.0205168 / Training Accuracy: 0.970703 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 16896/60000 / Cost: 0.0126593 / Training Accuracy: 0.980469 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 17408/60000 / Cost: 0.0197959 / Training Accuracy: 0.970703 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 17920/60000 / Cost: 0.0180165 / Training Accuracy: 0.970703 / Validation Accuracy: 0.959\n",
      "Epoch: 6 / Batch: 18432/60000 / Cost: 0.0155108 / Training Accuracy: 0.974609 / Validation Accuracy: 0.962\n",
      "Epoch: 6 / Batch: 18944/60000 / Cost: 0.0169217 / Training Accuracy: 0.96875 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 19456/60000 / Cost: 0.0220756 / Training Accuracy: 0.958984 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 19968/60000 / Cost: 0.0214974 / Training Accuracy: 0.960938 / Validation Accuracy: 0.959\n",
      "Epoch: 6 / Batch: 20480/60000 / Cost: 0.0233493 / Training Accuracy: 0.962891 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 20992/60000 / Cost: 0.0182728 / Training Accuracy: 0.978516 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 21504/60000 / Cost: 0.020128 / Training Accuracy: 0.966797 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 22016/60000 / Cost: 0.0153444 / Training Accuracy: 0.972656 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 22528/60000 / Cost: 0.0208166 / Training Accuracy: 0.966797 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 23040/60000 / Cost: 0.010185 / Training Accuracy: 0.990234 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 23552/60000 / Cost: 0.017908 / Training Accuracy: 0.96875 / Validation Accuracy: 0.967\n",
      "Epoch: 6 / Batch: 24064/60000 / Cost: 0.0132403 / Training Accuracy: 0.982422 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 24576/60000 / Cost: 0.022012 / Training Accuracy: 0.962891 / Validation Accuracy: 0.967\n",
      "Epoch: 6 / Batch: 25088/60000 / Cost: 0.0136278 / Training Accuracy: 0.978516 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 25600/60000 / Cost: 0.0206338 / Training Accuracy: 0.964844 / Validation Accuracy: 0.965\n",
      "Epoch: 6 / Batch: 26112/60000 / Cost: 0.0172727 / Training Accuracy: 0.974609 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 26624/60000 / Cost: 0.0112438 / Training Accuracy: 0.978516 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 27136/60000 / Cost: 0.0206004 / Training Accuracy: 0.972656 / Validation Accuracy: 0.959\n",
      "Epoch: 6 / Batch: 27648/60000 / Cost: 0.0167756 / Training Accuracy: 0.970703 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 28160/60000 / Cost: 0.00906248 / Training Accuracy: 0.988281 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 28672/60000 / Cost: 0.01568 / Training Accuracy: 0.974609 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 29184/60000 / Cost: 0.0180874 / Training Accuracy: 0.970703 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 29696/60000 / Cost: 0.0180928 / Training Accuracy: 0.966797 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 30208/60000 / Cost: 0.016137 / Training Accuracy: 0.966797 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 30720/60000 / Cost: 0.0199515 / Training Accuracy: 0.976563 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 31232/60000 / Cost: 0.0157831 / Training Accuracy: 0.970703 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 31744/60000 / Cost: 0.0123817 / Training Accuracy: 0.978516 / Validation Accuracy: 0.972\n",
      "Epoch: 6 / Batch: 32256/60000 / Cost: 0.0153831 / Training Accuracy: 0.974609 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 32768/60000 / Cost: 0.0118232 / Training Accuracy: 0.988281 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 33280/60000 / Cost: 0.0166379 / Training Accuracy: 0.978516 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 33792/60000 / Cost: 0.0101929 / Training Accuracy: 0.986328 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 34304/60000 / Cost: 0.0153502 / Training Accuracy: 0.972656 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 34816/60000 / Cost: 0.0110937 / Training Accuracy: 0.986328 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 35328/60000 / Cost: 0.0149906 / Training Accuracy: 0.978516 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 35840/60000 / Cost: 0.0156993 / Training Accuracy: 0.970703 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 36352/60000 / Cost: 0.0104241 / Training Accuracy: 0.990234 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 36864/60000 / Cost: 0.0148172 / Training Accuracy: 0.970703 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 37376/60000 / Cost: 0.00958528 / Training Accuracy: 0.982422 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 37888/60000 / Cost: 0.0174876 / Training Accuracy: 0.974609 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 38400/60000 / Cost: 0.0132661 / Training Accuracy: 0.976563 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 38912/60000 / Cost: 0.011417 / Training Accuracy: 0.980469 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 39424/60000 / Cost: 0.0178856 / Training Accuracy: 0.978516 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 39936/60000 / Cost: 0.014778 / Training Accuracy: 0.978516 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 40448/60000 / Cost: 0.0140853 / Training Accuracy: 0.978516 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 40960/60000 / Cost: 0.0130488 / Training Accuracy: 0.978516 / Validation Accuracy: 0.956\n",
      "Epoch: 6 / Batch: 41472/60000 / Cost: 0.0138197 / Training Accuracy: 0.980469 / Validation Accuracy: 0.958\n",
      "Epoch: 6 / Batch: 41984/60000 / Cost: 0.0116768 / Training Accuracy: 0.984375 / Validation Accuracy: 0.961\n",
      "Epoch: 6 / Batch: 42496/60000 / Cost: 0.0136434 / Training Accuracy: 0.980469 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 43008/60000 / Cost: 0.0153493 / Training Accuracy: 0.974609 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 43520/60000 / Cost: 0.0170447 / Training Accuracy: 0.96875 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 44032/60000 / Cost: 0.0122684 / Training Accuracy: 0.982422 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 44544/60000 / Cost: 0.0149923 / Training Accuracy: 0.980469 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 45056/60000 / Cost: 0.0106923 / Training Accuracy: 0.982422 / Validation Accuracy: 0.971\n",
      "Epoch: 6 / Batch: 45568/60000 / Cost: 0.0149782 / Training Accuracy: 0.972656 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 46080/60000 / Cost: 0.0128597 / Training Accuracy: 0.974609 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 46592/60000 / Cost: 0.0169376 / Training Accuracy: 0.972656 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 47104/60000 / Cost: 0.014179 / Training Accuracy: 0.978516 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 47616/60000 / Cost: 0.0104709 / Training Accuracy: 0.984375 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 48128/60000 / Cost: 0.0172628 / Training Accuracy: 0.966797 / Validation Accuracy: 0.968\n",
      "Epoch: 6 / Batch: 48640/60000 / Cost: 0.0203968 / Training Accuracy: 0.962891 / Validation Accuracy: 0.957\n",
      "Epoch: 6 / Batch: 49152/60000 / Cost: 0.012105 / Training Accuracy: 0.986328 / Validation Accuracy: 0.954\n",
      "Epoch: 6 / Batch: 49664/60000 / Cost: 0.0159662 / Training Accuracy: 0.972656 / Validation Accuracy: 0.956\n",
      "Epoch: 6 / Batch: 50176/60000 / Cost: 0.0131032 / Training Accuracy: 0.978516 / Validation Accuracy: 0.955\n",
      "Epoch: 6 / Batch: 50688/60000 / Cost: 0.0121529 / Training Accuracy: 0.976563 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 51200/60000 / Cost: 0.0145667 / Training Accuracy: 0.982422 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 51712/60000 / Cost: 0.0152744 / Training Accuracy: 0.982422 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 52224/60000 / Cost: 0.0155274 / Training Accuracy: 0.976563 / Validation Accuracy: 0.964\n",
      "Epoch: 6 / Batch: 52736/60000 / Cost: 0.0164103 / Training Accuracy: 0.966797 / Validation Accuracy: 0.966\n",
      "Epoch: 6 / Batch: 53248/60000 / Cost: 0.0119561 / Training Accuracy: 0.982422 / Validation Accuracy: 0.963\n",
      "Epoch: 6 / Batch: 53760/60000 / Cost: 0.0145724 / Training Accuracy: 0.976563 / Validation Accuracy: 0.974\n",
      "Epoch: 6 / Batch: 54272/60000 / Cost: 0.0129726 / Training Accuracy: 0.978516 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 54784/60000 / Cost: 0.0128992 / Training Accuracy: 0.978516 / Validation Accuracy: 0.97\n",
      "Epoch: 6 / Batch: 55296/60000 / Cost: 0.0166209 / Training Accuracy: 0.96875 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 55808/60000 / Cost: 0.0122708 / Training Accuracy: 0.976563 / Validation Accuracy: 0.979\n",
      "Epoch: 6 / Batch: 56320/60000 / Cost: 0.0150249 / Training Accuracy: 0.986328 / Validation Accuracy: 0.98\n",
      "Epoch: 6 / Batch: 56832/60000 / Cost: 0.0109987 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 6 / Batch: 57344/60000 / Cost: 0.0116924 / Training Accuracy: 0.978516 / Validation Accuracy: 0.975\n",
      "Epoch: 6 / Batch: 57856/60000 / Cost: 0.0188557 / Training Accuracy: 0.966797 / Validation Accuracy: 0.973\n",
      "Epoch: 6 / Batch: 58368/60000 / Cost: 0.0111419 / Training Accuracy: 0.986328 / Validation Accuracy: 0.969\n",
      "Epoch: 6 / Batch: 58880/60000 / Cost: 0.0123639 / Training Accuracy: 0.976563 / Validation Accuracy: 0.967\n",
      "Epoch: 6 / Batch: 59392/60000 / Cost: 0.013195 / Training Accuracy: 0.978516 / Validation Accuracy: 0.967\n",
      "Epoch: 6 / Batch: 59904/60000 / Cost: 0.00288726 / Training Accuracy: 1.0 / Validation Accuracy: 0.963\n",
      "Epoch: 7 / Batch: 0/60000 / Cost: 0.0176181 / Training Accuracy: 0.970703 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 512/60000 / Cost: 0.0104088 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 1024/60000 / Cost: 0.0107344 / Training Accuracy: 0.986328 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 1536/60000 / Cost: 0.00926383 / Training Accuracy: 0.980469 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 2048/60000 / Cost: 0.019732 / Training Accuracy: 0.972656 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 2560/60000 / Cost: 0.0127668 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 3072/60000 / Cost: 0.0133332 / Training Accuracy: 0.972656 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 3584/60000 / Cost: 0.0111951 / Training Accuracy: 0.980469 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 4096/60000 / Cost: 0.00972871 / Training Accuracy: 0.984375 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 4608/60000 / Cost: 0.0123945 / Training Accuracy: 0.982422 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 5120/60000 / Cost: 0.0131041 / Training Accuracy: 0.978516 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 5632/60000 / Cost: 0.0112228 / Training Accuracy: 0.986328 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 6144/60000 / Cost: 0.00672447 / Training Accuracy: 0.988281 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 6656/60000 / Cost: 0.0119975 / Training Accuracy: 0.978516 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 7168/60000 / Cost: 0.00871782 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 7680/60000 / Cost: 0.0147248 / Training Accuracy: 0.978516 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 8192/60000 / Cost: 0.0140028 / Training Accuracy: 0.980469 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 8704/60000 / Cost: 0.00915533 / Training Accuracy: 0.990234 / Validation Accuracy: 0.982\n",
      "Epoch: 7 / Batch: 9216/60000 / Cost: 0.0139243 / Training Accuracy: 0.978516 / Validation Accuracy: 0.981\n",
      "Epoch: 7 / Batch: 9728/60000 / Cost: 0.0124808 / Training Accuracy: 0.982422 / Validation Accuracy: 0.982\n",
      "Epoch: 7 / Batch: 10240/60000 / Cost: 0.0161535 / Training Accuracy: 0.978516 / Validation Accuracy: 0.984\n",
      "Epoch: 7 / Batch: 10752/60000 / Cost: 0.016539 / Training Accuracy: 0.974609 / Validation Accuracy: 0.982\n",
      "Epoch: 7 / Batch: 11264/60000 / Cost: 0.00855037 / Training Accuracy: 0.990234 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 11776/60000 / Cost: 0.00847128 / Training Accuracy: 0.986328 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 12288/60000 / Cost: 0.0118854 / Training Accuracy: 0.980469 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 12800/60000 / Cost: 0.00824155 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 13312/60000 / Cost: 0.00880099 / Training Accuracy: 0.986328 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 13824/60000 / Cost: 0.00568328 / Training Accuracy: 0.994141 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 14336/60000 / Cost: 0.0103357 / Training Accuracy: 0.988281 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 14848/60000 / Cost: 0.0138089 / Training Accuracy: 0.980469 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 15360/60000 / Cost: 0.0113612 / Training Accuracy: 0.986328 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 15872/60000 / Cost: 0.0163155 / Training Accuracy: 0.970703 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 16384/60000 / Cost: 0.0112074 / Training Accuracy: 0.980469 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 16896/60000 / Cost: 0.0137886 / Training Accuracy: 0.978516 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 17408/60000 / Cost: 0.00920404 / Training Accuracy: 0.988281 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 17920/60000 / Cost: 0.011982 / Training Accuracy: 0.976563 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 18432/60000 / Cost: 0.0127577 / Training Accuracy: 0.980469 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 18944/60000 / Cost: 0.0190981 / Training Accuracy: 0.972656 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 19456/60000 / Cost: 0.0145878 / Training Accuracy: 0.982422 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 19968/60000 / Cost: 0.0132968 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 20480/60000 / Cost: 0.0118487 / Training Accuracy: 0.980469 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 20992/60000 / Cost: 0.00857795 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 21504/60000 / Cost: 0.00780483 / Training Accuracy: 0.986328 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 22016/60000 / Cost: 0.0176213 / Training Accuracy: 0.974609 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 22528/60000 / Cost: 0.00964539 / Training Accuracy: 0.980469 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 23040/60000 / Cost: 0.0128063 / Training Accuracy: 0.982422 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 23552/60000 / Cost: 0.0113404 / Training Accuracy: 0.984375 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 24064/60000 / Cost: 0.00918552 / Training Accuracy: 0.982422 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 24576/60000 / Cost: 0.0161607 / Training Accuracy: 0.96875 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 25088/60000 / Cost: 0.00841158 / Training Accuracy: 0.990234 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 25600/60000 / Cost: 0.0151005 / Training Accuracy: 0.978516 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 26112/60000 / Cost: 0.0157971 / Training Accuracy: 0.96875 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 26624/60000 / Cost: 0.0107208 / Training Accuracy: 0.980469 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 27136/60000 / Cost: 0.0148815 / Training Accuracy: 0.972656 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 27648/60000 / Cost: 0.0100165 / Training Accuracy: 0.980469 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 28160/60000 / Cost: 0.0109551 / Training Accuracy: 0.980469 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 28672/60000 / Cost: 0.016242 / Training Accuracy: 0.974609 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 29184/60000 / Cost: 0.0125491 / Training Accuracy: 0.978516 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 29696/60000 / Cost: 0.0141967 / Training Accuracy: 0.976563 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 30208/60000 / Cost: 0.0161856 / Training Accuracy: 0.974609 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 30720/60000 / Cost: 0.0127681 / Training Accuracy: 0.978516 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 31232/60000 / Cost: 0.0116367 / Training Accuracy: 0.972656 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 31744/60000 / Cost: 0.00688029 / Training Accuracy: 0.988281 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 32256/60000 / Cost: 0.00712716 / Training Accuracy: 0.990234 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 32768/60000 / Cost: 0.00754444 / Training Accuracy: 0.994141 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 33280/60000 / Cost: 0.00926768 / Training Accuracy: 0.988281 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 33792/60000 / Cost: 0.00970412 / Training Accuracy: 0.984375 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 34304/60000 / Cost: 0.0114711 / Training Accuracy: 0.980469 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 34816/60000 / Cost: 0.0101067 / Training Accuracy: 0.984375 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 35328/60000 / Cost: 0.0125837 / Training Accuracy: 0.980469 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 35840/60000 / Cost: 0.0101446 / Training Accuracy: 0.982422 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 36352/60000 / Cost: 0.0115523 / Training Accuracy: 0.984375 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 36864/60000 / Cost: 0.0101261 / Training Accuracy: 0.984375 / Validation Accuracy: 0.966\n",
      "Epoch: 7 / Batch: 37376/60000 / Cost: 0.0118172 / Training Accuracy: 0.982422 / Validation Accuracy: 0.963\n",
      "Epoch: 7 / Batch: 37888/60000 / Cost: 0.0127231 / Training Accuracy: 0.984375 / Validation Accuracy: 0.96\n",
      "Epoch: 7 / Batch: 38400/60000 / Cost: 0.0135492 / Training Accuracy: 0.978516 / Validation Accuracy: 0.962\n",
      "Epoch: 7 / Batch: 38912/60000 / Cost: 0.0112717 / Training Accuracy: 0.982422 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 39424/60000 / Cost: 0.00713187 / Training Accuracy: 0.992188 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 39936/60000 / Cost: 0.0092601 / Training Accuracy: 0.980469 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 40448/60000 / Cost: 0.0111231 / Training Accuracy: 0.978516 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 40960/60000 / Cost: 0.0138049 / Training Accuracy: 0.978516 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 41472/60000 / Cost: 0.0131502 / Training Accuracy: 0.980469 / Validation Accuracy: 0.971\n",
      "Epoch: 7 / Batch: 41984/60000 / Cost: 0.0115594 / Training Accuracy: 0.982422 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 42496/60000 / Cost: 0.0151663 / Training Accuracy: 0.974609 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 43008/60000 / Cost: 0.0106968 / Training Accuracy: 0.982422 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 43520/60000 / Cost: 0.00820598 / Training Accuracy: 0.986328 / Validation Accuracy: 0.97\n",
      "Epoch: 7 / Batch: 44032/60000 / Cost: 0.0128169 / Training Accuracy: 0.978516 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 44544/60000 / Cost: 0.0187191 / Training Accuracy: 0.964844 / Validation Accuracy: 0.965\n",
      "Epoch: 7 / Batch: 45056/60000 / Cost: 0.0142399 / Training Accuracy: 0.982422 / Validation Accuracy: 0.964\n",
      "Epoch: 7 / Batch: 45568/60000 / Cost: 0.0108579 / Training Accuracy: 0.986328 / Validation Accuracy: 0.961\n",
      "Epoch: 7 / Batch: 46080/60000 / Cost: 0.0149998 / Training Accuracy: 0.976563 / Validation Accuracy: 0.967\n",
      "Epoch: 7 / Batch: 46592/60000 / Cost: 0.00663222 / Training Accuracy: 0.994141 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 47104/60000 / Cost: 0.00956932 / Training Accuracy: 0.988281 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 47616/60000 / Cost: 0.0121009 / Training Accuracy: 0.982422 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 48128/60000 / Cost: 0.013802 / Training Accuracy: 0.982422 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 48640/60000 / Cost: 0.00873093 / Training Accuracy: 0.986328 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 49152/60000 / Cost: 0.00842395 / Training Accuracy: 0.990234 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 49664/60000 / Cost: 0.0048758 / Training Accuracy: 0.998047 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 50176/60000 / Cost: 0.0157762 / Training Accuracy: 0.978516 / Validation Accuracy: 0.978\n",
      "Epoch: 7 / Batch: 50688/60000 / Cost: 0.0126084 / Training Accuracy: 0.980469 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 51200/60000 / Cost: 0.00882686 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 51712/60000 / Cost: 0.00609525 / Training Accuracy: 0.994141 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 52224/60000 / Cost: 0.0123205 / Training Accuracy: 0.976563 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 52736/60000 / Cost: 0.0146611 / Training Accuracy: 0.972656 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 53248/60000 / Cost: 0.0116732 / Training Accuracy: 0.980469 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 53760/60000 / Cost: 0.00955252 / Training Accuracy: 0.988281 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 54272/60000 / Cost: 0.00920194 / Training Accuracy: 0.978516 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 54784/60000 / Cost: 0.0157657 / Training Accuracy: 0.980469 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 55296/60000 / Cost: 0.0077481 / Training Accuracy: 0.986328 / Validation Accuracy: 0.976\n",
      "Epoch: 7 / Batch: 55808/60000 / Cost: 0.00877299 / Training Accuracy: 0.990234 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 56320/60000 / Cost: 0.0148895 / Training Accuracy: 0.976563 / Validation Accuracy: 0.975\n",
      "Epoch: 7 / Batch: 56832/60000 / Cost: 0.00907235 / Training Accuracy: 0.986328 / Validation Accuracy: 0.977\n",
      "Epoch: 7 / Batch: 57344/60000 / Cost: 0.0131972 / Training Accuracy: 0.982422 / Validation Accuracy: 0.972\n",
      "Epoch: 7 / Batch: 57856/60000 / Cost: 0.0124468 / Training Accuracy: 0.978516 / Validation Accuracy: 0.974\n",
      "Epoch: 7 / Batch: 58368/60000 / Cost: 0.0055618 / Training Accuracy: 0.992188 / Validation Accuracy: 0.973\n",
      "Epoch: 7 / Batch: 58880/60000 / Cost: 0.00977054 / Training Accuracy: 0.990234 / Validation Accuracy: 0.969\n",
      "Epoch: 7 / Batch: 59392/60000 / Cost: 0.0104599 / Training Accuracy: 0.984375 / Validation Accuracy: 0.968\n",
      "Epoch: 7 / Batch: 59904/60000 / Cost: 0.00594827 / Training Accuracy: 0.989583 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 0/60000 / Cost: 0.018016 / Training Accuracy: 0.974609 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 512/60000 / Cost: 0.0191339 / Training Accuracy: 0.964844 / Validation Accuracy: 0.954\n",
      "Epoch: 8 / Batch: 1024/60000 / Cost: 0.0141724 / Training Accuracy: 0.978516 / Validation Accuracy: 0.959\n",
      "Epoch: 8 / Batch: 1536/60000 / Cost: 0.0174302 / Training Accuracy: 0.972656 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 2048/60000 / Cost: 0.0101123 / Training Accuracy: 0.980469 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 2560/60000 / Cost: 0.0165689 / Training Accuracy: 0.974609 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 3072/60000 / Cost: 0.0135798 / Training Accuracy: 0.974609 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 3584/60000 / Cost: 0.0124069 / Training Accuracy: 0.978516 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 4096/60000 / Cost: 0.016104 / Training Accuracy: 0.980469 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 4608/60000 / Cost: 0.0129284 / Training Accuracy: 0.978516 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 5120/60000 / Cost: 0.0162064 / Training Accuracy: 0.976563 / Validation Accuracy: 0.966\n",
      "Epoch: 8 / Batch: 5632/60000 / Cost: 0.0112521 / Training Accuracy: 0.980469 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 6144/60000 / Cost: 0.0118844 / Training Accuracy: 0.976563 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 6656/60000 / Cost: 0.00957867 / Training Accuracy: 0.982422 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 7168/60000 / Cost: 0.0124576 / Training Accuracy: 0.978516 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 7680/60000 / Cost: 0.00929146 / Training Accuracy: 0.986328 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 8192/60000 / Cost: 0.0113997 / Training Accuracy: 0.978516 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 8704/60000 / Cost: 0.00915795 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 9216/60000 / Cost: 0.0122097 / Training Accuracy: 0.974609 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 9728/60000 / Cost: 0.0117453 / Training Accuracy: 0.978516 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 10240/60000 / Cost: 0.00781351 / Training Accuracy: 0.986328 / Validation Accuracy: 0.965\n",
      "Epoch: 8 / Batch: 10752/60000 / Cost: 0.0176839 / Training Accuracy: 0.972656 / Validation Accuracy: 0.963\n",
      "Epoch: 8 / Batch: 11264/60000 / Cost: 0.0140306 / Training Accuracy: 0.976563 / Validation Accuracy: 0.964\n",
      "Epoch: 8 / Batch: 11776/60000 / Cost: 0.0122145 / Training Accuracy: 0.974609 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 12288/60000 / Cost: 0.00931065 / Training Accuracy: 0.990234 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 12800/60000 / Cost: 0.0108131 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 13312/60000 / Cost: 0.0100967 / Training Accuracy: 0.986328 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 13824/60000 / Cost: 0.0115678 / Training Accuracy: 0.978516 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 14336/60000 / Cost: 0.0124572 / Training Accuracy: 0.980469 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 14848/60000 / Cost: 0.00959198 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 15360/60000 / Cost: 0.0102057 / Training Accuracy: 0.988281 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 15872/60000 / Cost: 0.0106803 / Training Accuracy: 0.980469 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 16384/60000 / Cost: 0.0109356 / Training Accuracy: 0.982422 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 16896/60000 / Cost: 0.00706938 / Training Accuracy: 0.986328 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 17408/60000 / Cost: 0.00670818 / Training Accuracy: 0.992188 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 17920/60000 / Cost: 0.0117453 / Training Accuracy: 0.986328 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 18432/60000 / Cost: 0.0102097 / Training Accuracy: 0.984375 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 18944/60000 / Cost: 0.0103596 / Training Accuracy: 0.982422 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 19456/60000 / Cost: 0.0150786 / Training Accuracy: 0.978516 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 19968/60000 / Cost: 0.00976537 / Training Accuracy: 0.986328 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 20480/60000 / Cost: 0.0128313 / Training Accuracy: 0.978516 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 20992/60000 / Cost: 0.0135005 / Training Accuracy: 0.976563 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 21504/60000 / Cost: 0.00663629 / Training Accuracy: 0.994141 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 22016/60000 / Cost: 0.0117142 / Training Accuracy: 0.982422 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 22528/60000 / Cost: 0.0108239 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 23040/60000 / Cost: 0.0154025 / Training Accuracy: 0.982422 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 23552/60000 / Cost: 0.00962521 / Training Accuracy: 0.988281 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 24064/60000 / Cost: 0.014687 / Training Accuracy: 0.978516 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 24576/60000 / Cost: 0.0119004 / Training Accuracy: 0.990234 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 25088/60000 / Cost: 0.0102752 / Training Accuracy: 0.986328 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 25600/60000 / Cost: 0.0149764 / Training Accuracy: 0.972656 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 26112/60000 / Cost: 0.00553857 / Training Accuracy: 0.992188 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 26624/60000 / Cost: 0.0200404 / Training Accuracy: 0.966797 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 27136/60000 / Cost: 0.0132943 / Training Accuracy: 0.986328 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 27648/60000 / Cost: 0.0119564 / Training Accuracy: 0.974609 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 28160/60000 / Cost: 0.00709826 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 28672/60000 / Cost: 0.0137296 / Training Accuracy: 0.982422 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 29184/60000 / Cost: 0.0090345 / Training Accuracy: 0.986328 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 29696/60000 / Cost: 0.0095521 / Training Accuracy: 0.982422 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 30208/60000 / Cost: 0.0144382 / Training Accuracy: 0.978516 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 30720/60000 / Cost: 0.00977214 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 31232/60000 / Cost: 0.0138431 / Training Accuracy: 0.974609 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 31744/60000 / Cost: 0.0112358 / Training Accuracy: 0.974609 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 32256/60000 / Cost: 0.00966411 / Training Accuracy: 0.986328 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 32768/60000 / Cost: 0.0102353 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 33280/60000 / Cost: 0.00798002 / Training Accuracy: 0.988281 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 33792/60000 / Cost: 0.00980571 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 34304/60000 / Cost: 0.00995298 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 34816/60000 / Cost: 0.0095354 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 35328/60000 / Cost: 0.0122815 / Training Accuracy: 0.986328 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 35840/60000 / Cost: 0.00835373 / Training Accuracy: 0.988281 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 36352/60000 / Cost: 0.0064775 / Training Accuracy: 0.994141 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 36864/60000 / Cost: 0.0122865 / Training Accuracy: 0.980469 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 37376/60000 / Cost: 0.00950897 / Training Accuracy: 0.984375 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 37888/60000 / Cost: 0.0126071 / Training Accuracy: 0.978516 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 38400/60000 / Cost: 0.00650357 / Training Accuracy: 0.990234 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 38912/60000 / Cost: 0.00732076 / Training Accuracy: 0.990234 / Validation Accuracy: 0.967\n",
      "Epoch: 8 / Batch: 39424/60000 / Cost: 0.0113684 / Training Accuracy: 0.978516 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 39936/60000 / Cost: 0.010623 / Training Accuracy: 0.982422 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 40448/60000 / Cost: 0.00600888 / Training Accuracy: 0.994141 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 40960/60000 / Cost: 0.00830127 / Training Accuracy: 0.986328 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 41472/60000 / Cost: 0.010717 / Training Accuracy: 0.982422 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 41984/60000 / Cost: 0.0108303 / Training Accuracy: 0.978516 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 42496/60000 / Cost: 0.0129905 / Training Accuracy: 0.980469 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 43008/60000 / Cost: 0.0103735 / Training Accuracy: 0.988281 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 43520/60000 / Cost: 0.00782004 / Training Accuracy: 0.988281 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 44032/60000 / Cost: 0.0103236 / Training Accuracy: 0.980469 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 44544/60000 / Cost: 0.0104681 / Training Accuracy: 0.986328 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 45056/60000 / Cost: 0.00488815 / Training Accuracy: 0.994141 / Validation Accuracy: 0.967\n",
      "Epoch: 8 / Batch: 45568/60000 / Cost: 0.0107042 / Training Accuracy: 0.990234 / Validation Accuracy: 0.968\n",
      "Epoch: 8 / Batch: 46080/60000 / Cost: 0.0110884 / Training Accuracy: 0.980469 / Validation Accuracy: 0.969\n",
      "Epoch: 8 / Batch: 46592/60000 / Cost: 0.0087708 / Training Accuracy: 0.986328 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 47104/60000 / Cost: 0.00837984 / Training Accuracy: 0.990234 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 47616/60000 / Cost: 0.00827486 / Training Accuracy: 0.990234 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 48128/60000 / Cost: 0.0107769 / Training Accuracy: 0.988281 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 48640/60000 / Cost: 0.00715247 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 49152/60000 / Cost: 0.0093819 / Training Accuracy: 0.986328 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 49664/60000 / Cost: 0.0062276 / Training Accuracy: 0.990234 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 50176/60000 / Cost: 0.00561174 / Training Accuracy: 0.992188 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 50688/60000 / Cost: 0.00781405 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 51200/60000 / Cost: 0.00640938 / Training Accuracy: 0.994141 / Validation Accuracy: 0.98\n",
      "Epoch: 8 / Batch: 51712/60000 / Cost: 0.0126237 / Training Accuracy: 0.982422 / Validation Accuracy: 0.979\n",
      "Epoch: 8 / Batch: 52224/60000 / Cost: 0.0127535 / Training Accuracy: 0.976563 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 52736/60000 / Cost: 0.0107118 / Training Accuracy: 0.986328 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 53248/60000 / Cost: 0.00874562 / Training Accuracy: 0.986328 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 53760/60000 / Cost: 0.00906559 / Training Accuracy: 0.986328 / Validation Accuracy: 0.976\n",
      "Epoch: 8 / Batch: 54272/60000 / Cost: 0.00538225 / Training Accuracy: 0.994141 / Validation Accuracy: 0.978\n",
      "Epoch: 8 / Batch: 54784/60000 / Cost: 0.00658938 / Training Accuracy: 0.990234 / Validation Accuracy: 0.975\n",
      "Epoch: 8 / Batch: 55296/60000 / Cost: 0.00971121 / Training Accuracy: 0.980469 / Validation Accuracy: 0.974\n",
      "Epoch: 8 / Batch: 55808/60000 / Cost: 0.00684974 / Training Accuracy: 0.994141 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 56320/60000 / Cost: 0.00804325 / Training Accuracy: 0.986328 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 56832/60000 / Cost: 0.0099379 / Training Accuracy: 0.986328 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 57344/60000 / Cost: 0.00860447 / Training Accuracy: 0.990234 / Validation Accuracy: 0.971\n",
      "Epoch: 8 / Batch: 57856/60000 / Cost: 0.00920679 / Training Accuracy: 0.984375 / Validation Accuracy: 0.97\n",
      "Epoch: 8 / Batch: 58368/60000 / Cost: 0.00682039 / Training Accuracy: 0.992188 / Validation Accuracy: 0.973\n",
      "Epoch: 8 / Batch: 58880/60000 / Cost: 0.0104969 / Training Accuracy: 0.986328 / Validation Accuracy: 0.972\n",
      "Epoch: 8 / Batch: 59392/60000 / Cost: 0.01195 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 8 / Batch: 59904/60000 / Cost: 0.00563013 / Training Accuracy: 1.0 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 0/60000 / Cost: 0.0131056 / Training Accuracy: 0.982422 / Validation Accuracy: 0.956\n",
      "Epoch: 9 / Batch: 512/60000 / Cost: 0.0108667 / Training Accuracy: 0.984375 / Validation Accuracy: 0.963\n",
      "Epoch: 9 / Batch: 1024/60000 / Cost: 0.0107669 / Training Accuracy: 0.984375 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 1536/60000 / Cost: 0.0135716 / Training Accuracy: 0.974609 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 2048/60000 / Cost: 0.0116555 / Training Accuracy: 0.988281 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 2560/60000 / Cost: 0.0165624 / Training Accuracy: 0.974609 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 3072/60000 / Cost: 0.0116149 / Training Accuracy: 0.978516 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 3584/60000 / Cost: 0.00838894 / Training Accuracy: 0.992188 / Validation Accuracy: 0.968\n",
      "Epoch: 9 / Batch: 4096/60000 / Cost: 0.00680227 / Training Accuracy: 0.994141 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 4608/60000 / Cost: 0.00722807 / Training Accuracy: 0.986328 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 5120/60000 / Cost: 0.00786086 / Training Accuracy: 0.990234 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 5632/60000 / Cost: 0.0117428 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 6144/60000 / Cost: 0.00683147 / Training Accuracy: 0.992188 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 6656/60000 / Cost: 0.00984374 / Training Accuracy: 0.988281 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 7168/60000 / Cost: 0.00694287 / Training Accuracy: 0.988281 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 7680/60000 / Cost: 0.0086735 / Training Accuracy: 0.990234 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 8192/60000 / Cost: 0.00756554 / Training Accuracy: 0.992188 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 8704/60000 / Cost: 0.00922801 / Training Accuracy: 0.986328 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 9216/60000 / Cost: 0.0155541 / Training Accuracy: 0.978516 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 9728/60000 / Cost: 0.00943803 / Training Accuracy: 0.982422 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 10240/60000 / Cost: 0.0138508 / Training Accuracy: 0.978516 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 10752/60000 / Cost: 0.00896694 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 11264/60000 / Cost: 0.00515921 / Training Accuracy: 0.994141 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 11776/60000 / Cost: 0.00807188 / Training Accuracy: 0.990234 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 12288/60000 / Cost: 0.0104402 / Training Accuracy: 0.986328 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 12800/60000 / Cost: 0.00661537 / Training Accuracy: 0.992188 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 13312/60000 / Cost: 0.0106294 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 13824/60000 / Cost: 0.00825006 / Training Accuracy: 0.986328 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 14336/60000 / Cost: 0.00781567 / Training Accuracy: 0.988281 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 14848/60000 / Cost: 0.00873911 / Training Accuracy: 0.988281 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 15360/60000 / Cost: 0.0111569 / Training Accuracy: 0.986328 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 15872/60000 / Cost: 0.0153113 / Training Accuracy: 0.978516 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 16384/60000 / Cost: 0.0064525 / Training Accuracy: 0.992188 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 16896/60000 / Cost: 0.00549019 / Training Accuracy: 0.992188 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 17408/60000 / Cost: 0.00856387 / Training Accuracy: 0.986328 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 17920/60000 / Cost: 0.00946694 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 18432/60000 / Cost: 0.00548819 / Training Accuracy: 0.992188 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 18944/60000 / Cost: 0.00504655 / Training Accuracy: 0.990234 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 19456/60000 / Cost: 0.00970033 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 19968/60000 / Cost: 0.0097697 / Training Accuracy: 0.994141 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 20480/60000 / Cost: 0.00763645 / Training Accuracy: 0.988281 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 20992/60000 / Cost: 0.00808475 / Training Accuracy: 0.992188 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 21504/60000 / Cost: 0.00684653 / Training Accuracy: 0.990234 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 22016/60000 / Cost: 0.00898124 / Training Accuracy: 0.990234 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 22528/60000 / Cost: 0.00996352 / Training Accuracy: 0.986328 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 23040/60000 / Cost: 0.00747587 / Training Accuracy: 0.986328 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 23552/60000 / Cost: 0.00483378 / Training Accuracy: 0.994141 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 24064/60000 / Cost: 0.00832935 / Training Accuracy: 0.992188 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 24576/60000 / Cost: 0.0130077 / Training Accuracy: 0.980469 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 25088/60000 / Cost: 0.00931751 / Training Accuracy: 0.978516 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 25600/60000 / Cost: 0.0104451 / Training Accuracy: 0.986328 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 26112/60000 / Cost: 0.00994628 / Training Accuracy: 0.982422 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 26624/60000 / Cost: 0.00586512 / Training Accuracy: 0.994141 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 27136/60000 / Cost: 0.00787268 / Training Accuracy: 0.990234 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 27648/60000 / Cost: 0.00863714 / Training Accuracy: 0.986328 / Validation Accuracy: 0.981\n",
      "Epoch: 9 / Batch: 28160/60000 / Cost: 0.0079193 / Training Accuracy: 0.990234 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 28672/60000 / Cost: 0.00563166 / Training Accuracy: 0.994141 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 29184/60000 / Cost: 0.0101584 / Training Accuracy: 0.984375 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 29696/60000 / Cost: 0.00761589 / Training Accuracy: 0.986328 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 30208/60000 / Cost: 0.00877643 / Training Accuracy: 0.984375 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 30720/60000 / Cost: 0.0123121 / Training Accuracy: 0.980469 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 31232/60000 / Cost: 0.0112153 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 31744/60000 / Cost: 0.0115108 / Training Accuracy: 0.982422 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 32256/60000 / Cost: 0.00673011 / Training Accuracy: 0.994141 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 32768/60000 / Cost: 0.0117451 / Training Accuracy: 0.976563 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 33280/60000 / Cost: 0.0076171 / Training Accuracy: 0.988281 / Validation Accuracy: 0.981\n",
      "Epoch: 9 / Batch: 33792/60000 / Cost: 0.0108532 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 9 / Batch: 34304/60000 / Cost: 0.00837609 / Training Accuracy: 0.986328 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 34816/60000 / Cost: 0.0109359 / Training Accuracy: 0.988281 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 35328/60000 / Cost: 0.0037384 / Training Accuracy: 0.998047 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 35840/60000 / Cost: 0.0172511 / Training Accuracy: 0.976563 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 36352/60000 / Cost: 0.00726572 / Training Accuracy: 0.986328 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 36864/60000 / Cost: 0.00705256 / Training Accuracy: 0.990234 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 37376/60000 / Cost: 0.00997864 / Training Accuracy: 0.978516 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 37888/60000 / Cost: 0.00682406 / Training Accuracy: 0.992188 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 38400/60000 / Cost: 0.00413201 / Training Accuracy: 0.996094 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 38912/60000 / Cost: 0.00717462 / Training Accuracy: 0.990234 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 39424/60000 / Cost: 0.00521513 / Training Accuracy: 0.992188 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 39936/60000 / Cost: 0.0112021 / Training Accuracy: 0.980469 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 40448/60000 / Cost: 0.0110351 / Training Accuracy: 0.986328 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 40960/60000 / Cost: 0.0090585 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 41472/60000 / Cost: 0.00497657 / Training Accuracy: 0.996094 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 41984/60000 / Cost: 0.00766181 / Training Accuracy: 0.994141 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 42496/60000 / Cost: 0.00734674 / Training Accuracy: 0.980469 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 43008/60000 / Cost: 0.00574119 / Training Accuracy: 0.990234 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 43520/60000 / Cost: 0.00777206 / Training Accuracy: 0.990234 / Validation Accuracy: 0.981\n",
      "Epoch: 9 / Batch: 44032/60000 / Cost: 0.0113854 / Training Accuracy: 0.986328 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 44544/60000 / Cost: 0.00447737 / Training Accuracy: 0.996094 / Validation Accuracy: 0.983\n",
      "Epoch: 9 / Batch: 45056/60000 / Cost: 0.0087363 / Training Accuracy: 0.984375 / Validation Accuracy: 0.981\n",
      "Epoch: 9 / Batch: 45568/60000 / Cost: 0.00963039 / Training Accuracy: 0.986328 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 46080/60000 / Cost: 0.00666259 / Training Accuracy: 0.992188 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 46592/60000 / Cost: 0.00781688 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 9 / Batch: 47104/60000 / Cost: 0.0106985 / Training Accuracy: 0.982422 / Validation Accuracy: 0.981\n",
      "Epoch: 9 / Batch: 47616/60000 / Cost: 0.0117661 / Training Accuracy: 0.982422 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 48128/60000 / Cost: 0.00839887 / Training Accuracy: 0.988281 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 48640/60000 / Cost: 0.00826773 / Training Accuracy: 0.986328 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 49152/60000 / Cost: 0.011733 / Training Accuracy: 0.984375 / Validation Accuracy: 0.97\n",
      "Epoch: 9 / Batch: 49664/60000 / Cost: 0.0086542 / Training Accuracy: 0.990234 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 50176/60000 / Cost: 0.0062604 / Training Accuracy: 0.992188 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 50688/60000 / Cost: 0.00748654 / Training Accuracy: 0.992188 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 51200/60000 / Cost: 0.00619599 / Training Accuracy: 0.990234 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 51712/60000 / Cost: 0.0123487 / Training Accuracy: 0.980469 / Validation Accuracy: 0.971\n",
      "Epoch: 9 / Batch: 52224/60000 / Cost: 0.0107866 / Training Accuracy: 0.986328 / Validation Accuracy: 0.969\n",
      "Epoch: 9 / Batch: 52736/60000 / Cost: 0.00784793 / Training Accuracy: 0.990234 / Validation Accuracy: 0.966\n",
      "Epoch: 9 / Batch: 53248/60000 / Cost: 0.00651237 / Training Accuracy: 0.988281 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 53760/60000 / Cost: 0.00808672 / Training Accuracy: 0.988281 / Validation Accuracy: 0.973\n",
      "Epoch: 9 / Batch: 54272/60000 / Cost: 0.0115341 / Training Accuracy: 0.980469 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 54784/60000 / Cost: 0.00687953 / Training Accuracy: 0.988281 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 55296/60000 / Cost: 0.00759613 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 55808/60000 / Cost: 0.00940045 / Training Accuracy: 0.980469 / Validation Accuracy: 0.974\n",
      "Epoch: 9 / Batch: 56320/60000 / Cost: 0.00812116 / Training Accuracy: 0.986328 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 56832/60000 / Cost: 0.0071301 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 9 / Batch: 57344/60000 / Cost: 0.00832222 / Training Accuracy: 0.992188 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 57856/60000 / Cost: 0.0140222 / Training Accuracy: 0.984375 / Validation Accuracy: 0.975\n",
      "Epoch: 9 / Batch: 58368/60000 / Cost: 0.0119703 / Training Accuracy: 0.982422 / Validation Accuracy: 0.977\n",
      "Epoch: 9 / Batch: 58880/60000 / Cost: 0.00881046 / Training Accuracy: 0.988281 / Validation Accuracy: 0.976\n",
      "Epoch: 9 / Batch: 59392/60000 / Cost: 0.00515563 / Training Accuracy: 0.994141 / Validation Accuracy: 0.972\n",
      "Epoch: 9 / Batch: 59904/60000 / Cost: 0.00503053 / Training Accuracy: 1.0 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 0/60000 / Cost: 0.00834384 / Training Accuracy: 0.982422 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 512/60000 / Cost: 0.0125426 / Training Accuracy: 0.980469 / Validation Accuracy: 0.965\n",
      "Epoch: 10 / Batch: 1024/60000 / Cost: 0.0118744 / Training Accuracy: 0.982422 / Validation Accuracy: 0.969\n",
      "Epoch: 10 / Batch: 1536/60000 / Cost: 0.00729743 / Training Accuracy: 0.992188 / Validation Accuracy: 0.965\n",
      "Epoch: 10 / Batch: 2048/60000 / Cost: 0.00703705 / Training Accuracy: 0.986328 / Validation Accuracy: 0.965\n",
      "Epoch: 10 / Batch: 2560/60000 / Cost: 0.0167772 / Training Accuracy: 0.974609 / Validation Accuracy: 0.967\n",
      "Epoch: 10 / Batch: 3072/60000 / Cost: 0.008935 / Training Accuracy: 0.984375 / Validation Accuracy: 0.969\n",
      "Epoch: 10 / Batch: 3584/60000 / Cost: 0.0130077 / Training Accuracy: 0.972656 / Validation Accuracy: 0.969\n",
      "Epoch: 10 / Batch: 4096/60000 / Cost: 0.00923939 / Training Accuracy: 0.986328 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 4608/60000 / Cost: 0.0141591 / Training Accuracy: 0.978516 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 5120/60000 / Cost: 0.00556505 / Training Accuracy: 0.992188 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 5632/60000 / Cost: 0.00637685 / Training Accuracy: 0.994141 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 6144/60000 / Cost: 0.00676772 / Training Accuracy: 0.988281 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 6656/60000 / Cost: 0.0082267 / Training Accuracy: 0.990234 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 7168/60000 / Cost: 0.00692181 / Training Accuracy: 0.992188 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 7680/60000 / Cost: 0.0117407 / Training Accuracy: 0.978516 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 8192/60000 / Cost: 0.00641564 / Training Accuracy: 0.986328 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 8704/60000 / Cost: 0.00861843 / Training Accuracy: 0.980469 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 9216/60000 / Cost: 0.0074746 / Training Accuracy: 0.988281 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 9728/60000 / Cost: 0.00728863 / Training Accuracy: 0.988281 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 10240/60000 / Cost: 0.00818903 / Training Accuracy: 0.982422 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 10752/60000 / Cost: 0.00683063 / Training Accuracy: 0.986328 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 11264/60000 / Cost: 0.00976266 / Training Accuracy: 0.984375 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 11776/60000 / Cost: 0.00995631 / Training Accuracy: 0.978516 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 12288/60000 / Cost: 0.00633057 / Training Accuracy: 0.994141 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 12800/60000 / Cost: 0.00515155 / Training Accuracy: 0.996094 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 13312/60000 / Cost: 0.0109222 / Training Accuracy: 0.986328 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 13824/60000 / Cost: 0.0111259 / Training Accuracy: 0.982422 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 14336/60000 / Cost: 0.00583199 / Training Accuracy: 0.996094 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 14848/60000 / Cost: 0.00838608 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 15360/60000 / Cost: 0.0123677 / Training Accuracy: 0.974609 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 15872/60000 / Cost: 0.00693837 / Training Accuracy: 0.990234 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 16384/60000 / Cost: 0.00859606 / Training Accuracy: 0.982422 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 16896/60000 / Cost: 0.00873121 / Training Accuracy: 0.986328 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 17408/60000 / Cost: 0.0105645 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 17920/60000 / Cost: 0.0044682 / Training Accuracy: 0.994141 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 18432/60000 / Cost: 0.00891724 / Training Accuracy: 0.982422 / Validation Accuracy: 0.97\n",
      "Epoch: 10 / Batch: 18944/60000 / Cost: 0.0115852 / Training Accuracy: 0.978516 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 19456/60000 / Cost: 0.00881334 / Training Accuracy: 0.986328 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 19968/60000 / Cost: 0.00748146 / Training Accuracy: 0.990234 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 20480/60000 / Cost: 0.00750343 / Training Accuracy: 0.986328 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 20992/60000 / Cost: 0.00958573 / Training Accuracy: 0.986328 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 21504/60000 / Cost: 0.0104899 / Training Accuracy: 0.982422 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 22016/60000 / Cost: 0.00666982 / Training Accuracy: 0.992188 / Validation Accuracy: 0.973\n",
      "Epoch: 10 / Batch: 22528/60000 / Cost: 0.00856855 / Training Accuracy: 0.994141 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 23040/60000 / Cost: 0.00949807 / Training Accuracy: 0.982422 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 23552/60000 / Cost: 0.00812803 / Training Accuracy: 0.986328 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 24064/60000 / Cost: 0.00857138 / Training Accuracy: 0.990234 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 24576/60000 / Cost: 0.0065283 / Training Accuracy: 0.986328 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 25088/60000 / Cost: 0.0100234 / Training Accuracy: 0.984375 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 25600/60000 / Cost: 0.00681432 / Training Accuracy: 0.988281 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 26112/60000 / Cost: 0.00906749 / Training Accuracy: 0.990234 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 26624/60000 / Cost: 0.00640992 / Training Accuracy: 0.990234 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 27136/60000 / Cost: 0.00800375 / Training Accuracy: 0.980469 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 27648/60000 / Cost: 0.00855758 / Training Accuracy: 0.986328 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 28160/60000 / Cost: 0.00881773 / Training Accuracy: 0.982422 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 28672/60000 / Cost: 0.00238588 / Training Accuracy: 1.0 / Validation Accuracy: 0.975\n",
      "Epoch: 10 / Batch: 29184/60000 / Cost: 0.00642537 / Training Accuracy: 0.992188 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 29696/60000 / Cost: 0.00740092 / Training Accuracy: 0.992188 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 30208/60000 / Cost: 0.00802521 / Training Accuracy: 0.986328 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 30720/60000 / Cost: 0.00824324 / Training Accuracy: 0.986328 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 31232/60000 / Cost: 0.00703686 / Training Accuracy: 0.988281 / Validation Accuracy: 0.972\n",
      "Epoch: 10 / Batch: 31744/60000 / Cost: 0.00687154 / Training Accuracy: 0.994141 / Validation Accuracy: 0.971\n",
      "Epoch: 10 / Batch: 32256/60000 / Cost: 0.00444686 / Training Accuracy: 0.990234 / Validation Accuracy: 0.974\n",
      "Epoch: 10 / Batch: 32768/60000 / Cost: 0.00748283 / Training Accuracy: 0.990234 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 33280/60000 / Cost: 0.00740649 / Training Accuracy: 0.988281 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 33792/60000 / Cost: 0.00530353 / Training Accuracy: 0.996094 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 34304/60000 / Cost: 0.00915513 / Training Accuracy: 0.982422 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 34816/60000 / Cost: 0.00914805 / Training Accuracy: 0.980469 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 35328/60000 / Cost: 0.0061514 / Training Accuracy: 0.990234 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 35840/60000 / Cost: 0.0111719 / Training Accuracy: 0.984375 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 36352/60000 / Cost: 0.00499058 / Training Accuracy: 0.992188 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 36864/60000 / Cost: 0.00865105 / Training Accuracy: 0.994141 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 37376/60000 / Cost: 0.00584836 / Training Accuracy: 0.990234 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 37888/60000 / Cost: 0.00773558 / Training Accuracy: 0.990234 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 38400/60000 / Cost: 0.0108759 / Training Accuracy: 0.984375 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 38912/60000 / Cost: 0.00870689 / Training Accuracy: 0.986328 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 39424/60000 / Cost: 0.0113204 / Training Accuracy: 0.978516 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 39936/60000 / Cost: 0.00642147 / Training Accuracy: 0.990234 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 40448/60000 / Cost: 0.0116859 / Training Accuracy: 0.982422 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 40960/60000 / Cost: 0.00894217 / Training Accuracy: 0.990234 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 41472/60000 / Cost: 0.00636169 / Training Accuracy: 0.988281 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 41984/60000 / Cost: 0.010522 / Training Accuracy: 0.980469 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 42496/60000 / Cost: 0.00688684 / Training Accuracy: 0.990234 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 43008/60000 / Cost: 0.00444311 / Training Accuracy: 0.998047 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 43520/60000 / Cost: 0.00880017 / Training Accuracy: 0.982422 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 44032/60000 / Cost: 0.00716894 / Training Accuracy: 0.984375 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 44544/60000 / Cost: 0.00557433 / Training Accuracy: 0.994141 / Validation Accuracy: 0.983\n",
      "Epoch: 10 / Batch: 45056/60000 / Cost: 0.00523433 / Training Accuracy: 0.990234 / Validation Accuracy: 0.984\n",
      "Epoch: 10 / Batch: 45568/60000 / Cost: 0.00867931 / Training Accuracy: 0.986328 / Validation Accuracy: 0.983\n",
      "Epoch: 10 / Batch: 46080/60000 / Cost: 0.00593155 / Training Accuracy: 0.992188 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 46592/60000 / Cost: 0.00923421 / Training Accuracy: 0.988281 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 47104/60000 / Cost: 0.00995634 / Training Accuracy: 0.976563 / Validation Accuracy: 0.983\n",
      "Epoch: 10 / Batch: 47616/60000 / Cost: 0.00734896 / Training Accuracy: 0.990234 / Validation Accuracy: 0.985\n",
      "Epoch: 10 / Batch: 48128/60000 / Cost: 0.00475906 / Training Accuracy: 0.994141 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 48640/60000 / Cost: 0.00891937 / Training Accuracy: 0.986328 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 49152/60000 / Cost: 0.0052944 / Training Accuracy: 0.992188 / Validation Accuracy: 0.98\n",
      "Epoch: 10 / Batch: 49664/60000 / Cost: 0.00780466 / Training Accuracy: 0.988281 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 50176/60000 / Cost: 0.00362746 / Training Accuracy: 0.996094 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 50688/60000 / Cost: 0.00692403 / Training Accuracy: 0.990234 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 51200/60000 / Cost: 0.00880548 / Training Accuracy: 0.986328 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 51712/60000 / Cost: 0.00876181 / Training Accuracy: 0.984375 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 52224/60000 / Cost: 0.00831791 / Training Accuracy: 0.988281 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 52736/60000 / Cost: 0.00501268 / Training Accuracy: 0.994141 / Validation Accuracy: 0.976\n",
      "Epoch: 10 / Batch: 53248/60000 / Cost: 0.00826209 / Training Accuracy: 0.986328 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 53760/60000 / Cost: 0.00945547 / Training Accuracy: 0.992188 / Validation Accuracy: 0.979\n",
      "Epoch: 10 / Batch: 54272/60000 / Cost: 0.015137 / Training Accuracy: 0.980469 / Validation Accuracy: 0.982\n",
      "Epoch: 10 / Batch: 54784/60000 / Cost: 0.00768042 / Training Accuracy: 0.986328 / Validation Accuracy: 0.985\n",
      "Epoch: 10 / Batch: 55296/60000 / Cost: 0.00956871 / Training Accuracy: 0.984375 / Validation Accuracy: 0.987\n",
      "Epoch: 10 / Batch: 55808/60000 / Cost: 0.0109317 / Training Accuracy: 0.982422 / Validation Accuracy: 0.986\n",
      "Epoch: 10 / Batch: 56320/60000 / Cost: 0.00607355 / Training Accuracy: 0.988281 / Validation Accuracy: 0.983\n",
      "Epoch: 10 / Batch: 56832/60000 / Cost: 0.00674596 / Training Accuracy: 0.990234 / Validation Accuracy: 0.984\n",
      "Epoch: 10 / Batch: 57344/60000 / Cost: 0.00841928 / Training Accuracy: 0.988281 / Validation Accuracy: 0.985\n",
      "Epoch: 10 / Batch: 57856/60000 / Cost: 0.00873855 / Training Accuracy: 0.984375 / Validation Accuracy: 0.988\n",
      "Epoch: 10 / Batch: 58368/60000 / Cost: 0.00587709 / Training Accuracy: 0.990234 / Validation Accuracy: 0.981\n",
      "Epoch: 10 / Batch: 58880/60000 / Cost: 0.00612561 / Training Accuracy: 0.994141 / Validation Accuracy: 0.978\n",
      "Epoch: 10 / Batch: 59392/60000 / Cost: 0.00877517 / Training Accuracy: 0.988281 / Validation Accuracy: 0.977\n",
      "Epoch: 10 / Batch: 59904/60000 / Cost: 0.00260344 / Training Accuracy: 1.0 / Validation Accuracy: 0.973\n"
     ]
    }
   ],
   "source": [
    "# Training\n",
    "for i in range(num_epoch):\n",
    "    np.random.shuffle(data)\n",
    "\n",
    "    data_x = data[:, :img_flat_size]\n",
    "    data_y = data[:, img_flat_size]\n",
    "\n",
    "    data_y_onehot = np.zeros([data_y.shape[0], num_label])\n",
    "    for j in range(data_y.shape[0]):\n",
    "        data_y_onehot[j, int(data_y[j])] = 1\n",
    "    \n",
    "    data_y_onehot_val = np.zeros([validation_y.shape[0], num_label])\n",
    "    for j in range(validation_y.shape[0]):\n",
    "        data_y_onehot_val[j, int(validation_y[j])] = 1\n",
    "        \n",
    "    batch_count = 1\n",
    "    for j in range(0, len_data, batch_size):\n",
    "        if j + batch_size < len_data:\n",
    "            data_x_in = data_x[j : j + batch_size, :]\n",
    "            data_y_in = data_y_onehot[j : j + batch_size, :]\n",
    "        else:\n",
    "            data_x_in = data_x[j : len_data, :]\n",
    "            data_y_in = data_y_onehot[j : len_data, :]\n",
    "\n",
    "        optimizer.run(feed_dict = {x_image: data_x_in, y_target: data_y_in})\n",
    "        cost = sess.run(Cost, feed_dict = {x_image: data_x_in, y_target: data_y_in})\n",
    "        acc = sess.run(accuracy, feed_dict = {x_image: data_x_in, y_target: data_y_in})\n",
    "        val_acc = sess.run(accuracy, feed_dict = {x_image: validation_x, y_target: data_y_onehot_val})\n",
    "        \n",
    "        print(\"Epoch: \" + str(i+1) + ' / ' + \"Batch: \" + str(j) + '/' + str(len_data) + ' / ' + \"Cost: \" + str(cost) + ' / ' + \\\n",
    "              \"Training Accuracy: \" + str(acc) + ' / ' + \"Validation Accuracy: \" + str(val_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy: 0.9783333333333334\n"
     ]
    }
   ],
   "source": [
    "# Testing\n",
    "\n",
    "test_y_onehot = np.zeros([test_y.shape[0], num_label])\n",
    "for i in range(test_y.shape[0]):\n",
    "    test_y_onehot[i, int(test_y[i])] = 1\n",
    "\n",
    "test_result = sess.run(output, feed_dict = {x_image: test_x, rnn_batch_size: test_x.shape[0], rnn_step_size: step_size})\n",
    "\n",
    "count_correct = 0\n",
    "for i in range(test_result.shape[0]):\n",
    "    prediction_y = np.argmax(test_result[i,:])\n",
    "    \n",
    "    if prediction_y == test_y[i]:\n",
    "        count_correct += 1\n",
    "\n",
    "test_acc = count_correct / test_result.shape[0]\n",
    "\n",
    "print(\"Test Accuracy: \" + str(test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}