MNIST-CDELM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-02-09T20:16:44.088892Z",
     "start_time": "2018-02-09T20:16:44.081883Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.5.0\n",
      "2.1.3\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import scipy\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "print( keras.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# To install Keras with Tensorflow backend:\n",
    "In this notebook, tf 1.5.0 and keras 2.1.3 are used.  To install Tensorflow 1.5.0, follow the prompts, being careful to install CUDA 9.0 and **NOT** CUDA 9.1.  Then, download CuDNN and follow their install instructions - simple unzip the folder, navigate to your CUDA installation, and paste the cudnn_7.0.dll into the bin folder there.  That's it!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-02-09T20:05:29.619372Z",
     "start_time": "2018-02-09T20:05:28.875406Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "(55000, 784)\n",
      "(55000, 10)\n"
     ]
    }
   ],
   "source": [
    "# load up the data\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)\n",
    "\n",
    "\n",
    "X_train = np.vstack([img.reshape(-1,) for img in mnist.train.images])\n",
    "Y_train = mnist.train.labels\n",
    "\n",
    "X_test = np.vstack([img.reshape(-1,) for img in mnist.test.images])\n",
    "Y_test = mnist.test.labels\n",
    "\n",
    "del mnist\n",
    "print(X_train.shape)\n",
    "print(Y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-02-09T20:05:29.852208Z",
     "start_time": "2018-02-09T20:05:29.622375Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000, 28, 28, 1)\n",
      "(10000, 28, 28, 1) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "num_classes = 10\n",
    "conv = 1\n",
    "input_shape = (1, X_train.shape[1])\n",
    "\n",
    "img_rows, img_cols = 28, 28\n",
    "# If using a convolutional network, reshape the data into 28x28 with channels first\n",
    "if conv == 1: \n",
    "    X_train = X_train.reshape(X_train.shape[0],  img_rows, img_cols, 1)\n",
    "    X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)\n",
    "# cast to 32-bit floats and normalize (0~1)\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')    \n",
    "X_train /= 255\n",
    "X_test /= 255\n",
    "\n",
    "# limit X_train to 10k and 15k samples\n",
    "\n",
    "source = np.random.permutation(55000)\n",
    "list_10k = list(source[:10000])\n",
    "list_15k = list(source[:15000])\n",
    "\n",
    "X_train10k = X_train[list_10k,:,:,:]\n",
    "X_train15k = X_train[list_15k,:,:,:]\n",
    "Y_train10k = Y_train[list_10k,:]\n",
    "Y_train15k = Y_train[list_15k,:]\n",
    "\n",
    "print(X_train10k.shape)\n",
    "print(X_test.shape, Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-02-09T20:22:02.167869Z",
     "start_time": "2018-02-09T20:20:27.587057Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "10000/10000 [==============================] - 2s 248us/step - loss: 2.3035 - acc: 0.1091 - val_loss: 2.3058 - val_acc: 0.0980\n",
      "Epoch 2/100\n",
      "10000/10000 [==============================] - 1s 88us/step - loss: 2.3041 - acc: 0.1032 - val_loss: 2.3032 - val_acc: 0.0974\n",
      "Epoch 3/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 2.3035 - acc: 0.1059 - val_loss: 2.3089 - val_acc: 0.0958\n",
      "Epoch 4/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 2.3044 - acc: 0.1043 - val_loss: 2.3046 - val_acc: 0.1135\n",
      "Epoch 5/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 2.3041 - acc: 0.1038 - val_loss: 2.3052 - val_acc: 0.0980\n",
      "Epoch 6/100\n",
      "10000/10000 [==============================] - 1s 98us/step - loss: 2.3040 - acc: 0.1000 - val_loss: 2.3015 - val_acc: 0.1135\n",
      "Epoch 7/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 2.3031 - acc: 0.1054 - val_loss: 2.3087 - val_acc: 0.0982\n",
      "Epoch 8/100\n",
      "10000/10000 [==============================] - 1s 96us/step - loss: 2.3040 - acc: 0.1032 - val_loss: 2.3048 - val_acc: 0.1135\n",
      "Epoch 9/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 2.3034 - acc: 0.1090 - val_loss: 2.3036 - val_acc: 0.1135\n",
      "Epoch 10/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 2.3025 - acc: 0.1076 - val_loss: 2.3012 - val_acc: 0.1028\n",
      "Epoch 11/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 2.2994 - acc: 0.1143 - val_loss: 2.3065 - val_acc: 0.0974\n",
      "Epoch 12/100\n",
      "10000/10000 [==============================] - 1s 88us/step - loss: 2.2706 - acc: 0.1402 - val_loss: 2.2513 - val_acc: 0.1567\n",
      "Epoch 13/100\n",
      "10000/10000 [==============================] - 1s 86us/step - loss: 2.0983 - acc: 0.2409 - val_loss: 2.1860 - val_acc: 0.1752\n",
      "Epoch 14/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 1.5414 - acc: 0.4672 - val_loss: 1.2650 - val_acc: 0.5660\n",
      "Epoch 15/100\n",
      "10000/10000 [==============================] - 1s 95us/step - loss: 1.1301 - acc: 0.6125 - val_loss: 1.2969 - val_acc: 0.5638\n",
      "Epoch 16/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 0.9524 - acc: 0.6689 - val_loss: 1.0704 - val_acc: 0.6455\n",
      "Epoch 17/100\n",
      "10000/10000 [==============================] - 1s 87us/step - loss: 0.8191 - acc: 0.7246 - val_loss: 0.8680 - val_acc: 0.7007\n",
      "Epoch 18/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 0.7306 - acc: 0.7593 - val_loss: 0.6622 - val_acc: 0.7778\n",
      "Epoch 19/100\n",
      "10000/10000 [==============================] - 1s 95us/step - loss: 0.6618 - acc: 0.7812 - val_loss: 0.9308 - val_acc: 0.6882\n",
      "Epoch 20/100\n",
      "10000/10000 [==============================] - 1s 87us/step - loss: 0.6192 - acc: 0.7974 - val_loss: 0.6270 - val_acc: 0.7841\n",
      "Epoch 21/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.5752 - acc: 0.8155 - val_loss: 0.6374 - val_acc: 0.7874\n",
      "Epoch 22/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.5534 - acc: 0.8180 - val_loss: 0.6162 - val_acc: 0.7931\n",
      "Epoch 23/100\n",
      "10000/10000 [==============================] - 1s 85us/step - loss: 0.5189 - acc: 0.8272 - val_loss: 0.5298 - val_acc: 0.8205\n",
      "Epoch 24/100\n",
      "10000/10000 [==============================] - 1s 87us/step - loss: 0.4967 - acc: 0.8359 - val_loss: 0.9637 - val_acc: 0.6875\n",
      "Epoch 25/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.4731 - acc: 0.8462 - val_loss: 0.5004 - val_acc: 0.8348\n",
      "Epoch 26/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.4445 - acc: 0.8530 - val_loss: 0.4840 - val_acc: 0.8377\n",
      "Epoch 27/100\n",
      "10000/10000 [==============================] - 1s 96us/step - loss: 0.4267 - acc: 0.8599 - val_loss: 0.4657 - val_acc: 0.8407\n",
      "Epoch 28/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 0.4146 - acc: 0.8631 - val_loss: 0.4462 - val_acc: 0.8535\n",
      "Epoch 29/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.3952 - acc: 0.8681 - val_loss: 0.4483 - val_acc: 0.8536\n",
      "Epoch 30/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.3841 - acc: 0.8734 - val_loss: 0.3714 - val_acc: 0.8794\n",
      "Epoch 31/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.3717 - acc: 0.8816 - val_loss: 0.3927 - val_acc: 0.8712\n",
      "Epoch 32/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.3592 - acc: 0.8812 - val_loss: 0.3807 - val_acc: 0.8765\n",
      "Epoch 33/100\n",
      "10000/10000 [==============================] - 1s 98us/step - loss: 0.3455 - acc: 0.8876 - val_loss: 0.4446 - val_acc: 0.8560\n",
      "Epoch 34/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.3353 - acc: 0.8913 - val_loss: 0.4223 - val_acc: 0.8584\n",
      "Epoch 35/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 0.3234 - acc: 0.8946 - val_loss: 0.4300 - val_acc: 0.8496\n",
      "Epoch 36/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.3216 - acc: 0.8956 - val_loss: 0.3862 - val_acc: 0.8772\n",
      "Epoch 37/100\n",
      "10000/10000 [==============================] - 1s 93us/step - loss: 0.3167 - acc: 0.8997 - val_loss: 0.3363 - val_acc: 0.8891\n",
      "Epoch 38/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.3100 - acc: 0.8987 - val_loss: 0.4948 - val_acc: 0.8407\n",
      "Epoch 39/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.3027 - acc: 0.9021 - val_loss: 0.4050 - val_acc: 0.8591\n",
      "Epoch 40/100\n",
      "10000/10000 [==============================] - 1s 95us/step - loss: 0.2863 - acc: 0.9074 - val_loss: 0.3560 - val_acc: 0.8846\n",
      "Epoch 41/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 0.2877 - acc: 0.9069 - val_loss: 0.4205 - val_acc: 0.8667\n",
      "Epoch 42/100\n",
      "10000/10000 [==============================] - 1s 85us/step - loss: 0.2835 - acc: 0.9066 - val_loss: 0.3314 - val_acc: 0.8882\n",
      "Epoch 43/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.2753 - acc: 0.9095 - val_loss: 0.4123 - val_acc: 0.8658\n",
      "Epoch 44/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.2669 - acc: 0.9133 - val_loss: 0.2951 - val_acc: 0.9021\n",
      "Epoch 45/100\n",
      "10000/10000 [==============================] - 1s 93us/step - loss: 0.2659 - acc: 0.9140 - val_loss: 0.2993 - val_acc: 0.9038\n",
      "Epoch 46/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 0.2652 - acc: 0.9133 - val_loss: 0.3245 - val_acc: 0.8989\n",
      "Epoch 47/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.2575 - acc: 0.9156 - val_loss: 0.3842 - val_acc: 0.8789\n",
      "Epoch 48/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 0.2540 - acc: 0.9184 - val_loss: 0.3442 - val_acc: 0.8892\n",
      "Epoch 49/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.2469 - acc: 0.9196 - val_loss: 0.3094 - val_acc: 0.8999\n",
      "Epoch 50/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 0.2424 - acc: 0.9206 - val_loss: 0.3962 - val_acc: 0.8747\n",
      "Epoch 51/100\n",
      "10000/10000 [==============================] - 1s 97us/step - loss: 0.2382 - acc: 0.9218 - val_loss: 0.3684 - val_acc: 0.8813\n",
      "Epoch 52/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.2355 - acc: 0.9226 - val_loss: 0.3048 - val_acc: 0.9040\n",
      "Epoch 53/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.2333 - acc: 0.9232 - val_loss: 0.2678 - val_acc: 0.9174\n",
      "Epoch 54/100\n",
      "10000/10000 [==============================] - 1s 93us/step - loss: 0.2286 - acc: 0.9253 - val_loss: 0.2622 - val_acc: 0.9191\n",
      "Epoch 55/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 0.2273 - acc: 0.9246 - val_loss: 0.3406 - val_acc: 0.8925\n",
      "Epoch 56/100\n",
      "10000/10000 [==============================] - 1s 93us/step - loss: 0.2256 - acc: 0.9254 - val_loss: 0.2681 - val_acc: 0.9166\n",
      "Epoch 57/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.2229 - acc: 0.9249 - val_loss: 0.3368 - val_acc: 0.8984\n",
      "Epoch 58/100\n",
      "10000/10000 [==============================] - 1s 88us/step - loss: 0.2193 - acc: 0.9268 - val_loss: 0.2652 - val_acc: 0.9142\n",
      "Epoch 59/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.2129 - acc: 0.9302 - val_loss: 0.3234 - val_acc: 0.8985\n",
      "Epoch 60/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.2111 - acc: 0.9319 - val_loss: 0.2453 - val_acc: 0.9217\n",
      "Epoch 61/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.2099 - acc: 0.9299 - val_loss: 0.3425 - val_acc: 0.8932\n",
      "Epoch 62/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.2076 - acc: 0.9307 - val_loss: 0.3832 - val_acc: 0.8791\n",
      "Epoch 63/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.2087 - acc: 0.9338 - val_loss: 0.3492 - val_acc: 0.8936\n",
      "Epoch 64/100\n",
      "10000/10000 [==============================] - 1s 86us/step - loss: 0.1995 - acc: 0.9351 - val_loss: 0.3325 - val_acc: 0.8966\n",
      "Epoch 65/100\n",
      "10000/10000 [==============================] - 1s 88us/step - loss: 0.1969 - acc: 0.9363 - val_loss: 0.2278 - val_acc: 0.9289\n",
      "Epoch 66/100\n",
      "10000/10000 [==============================] - 1s 99us/step - loss: 0.1931 - acc: 0.9347 - val_loss: 0.2856 - val_acc: 0.9077\n",
      "Epoch 67/100\n",
      "10000/10000 [==============================] - 1s 97us/step - loss: 0.1993 - acc: 0.9337 - val_loss: 0.2740 - val_acc: 0.9175\n",
      "Epoch 68/100\n",
      "10000/10000 [==============================] - 1s 95us/step - loss: 0.1912 - acc: 0.9372 - val_loss: 0.3564 - val_acc: 0.8885\n",
      "Epoch 69/100\n",
      "10000/10000 [==============================] - 1s 87us/step - loss: 0.1910 - acc: 0.9371 - val_loss: 0.2626 - val_acc: 0.9189\n",
      "Epoch 70/100\n",
      "10000/10000 [==============================] - 1s 88us/step - loss: 0.1917 - acc: 0.9376 - val_loss: 0.3163 - val_acc: 0.8997\n",
      "Epoch 71/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.1844 - acc: 0.9422 - val_loss: 0.3144 - val_acc: 0.9064\n",
      "Epoch 72/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 0.1841 - acc: 0.9384 - val_loss: 0.2251 - val_acc: 0.9297\n",
      "Epoch 73/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.1846 - acc: 0.9398 - val_loss: 0.2402 - val_acc: 0.9275\n",
      "Epoch 74/100\n",
      "10000/10000 [==============================] - 1s 88us/step - loss: 0.1824 - acc: 0.9376 - val_loss: 0.2856 - val_acc: 0.9104\n",
      "Epoch 75/100\n",
      "10000/10000 [==============================] - 1s 83us/step - loss: 0.1770 - acc: 0.9414 - val_loss: 0.3115 - val_acc: 0.9010\n",
      "Epoch 76/100\n",
      "10000/10000 [==============================] - 1s 87us/step - loss: 0.1764 - acc: 0.9420 - val_loss: 0.3678 - val_acc: 0.8914\n",
      "Epoch 77/100\n",
      "10000/10000 [==============================] - 1s 92us/step - loss: 0.1736 - acc: 0.9424 - val_loss: 0.2180 - val_acc: 0.9330\n",
      "Epoch 78/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.1747 - acc: 0.9406 - val_loss: 0.2956 - val_acc: 0.9060\n",
      "Epoch 79/100\n",
      "10000/10000 [==============================] - 1s 87us/step - loss: 0.1692 - acc: 0.9453 - val_loss: 0.3258 - val_acc: 0.9055\n",
      "Epoch 80/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.1697 - acc: 0.9456 - val_loss: 0.2320 - val_acc: 0.9268\n",
      "Epoch 81/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.1676 - acc: 0.9453 - val_loss: 0.4480 - val_acc: 0.8474\n",
      "Epoch 82/100\n",
      "10000/10000 [==============================] - 1s 95us/step - loss: 0.1660 - acc: 0.9474 - val_loss: 0.2260 - val_acc: 0.9311\n",
      "Epoch 83/100\n",
      "10000/10000 [==============================] - 1s 87us/step - loss: 0.1679 - acc: 0.9465 - val_loss: 0.2459 - val_acc: 0.9247\n",
      "Epoch 84/100\n",
      "10000/10000 [==============================] - 1s 89us/step - loss: 0.1627 - acc: 0.9477 - val_loss: 0.2639 - val_acc: 0.9177\n",
      "Epoch 85/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 0.1644 - acc: 0.9463 - val_loss: 0.2621 - val_acc: 0.9201\n",
      "Epoch 86/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 0.1627 - acc: 0.9474 - val_loss: 0.5109 - val_acc: 0.8529\n",
      "Epoch 87/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 0.1616 - acc: 0.9473 - val_loss: 0.3185 - val_acc: 0.9017\n",
      "Epoch 88/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 0.1608 - acc: 0.9467 - val_loss: 0.3098 - val_acc: 0.9062\n",
      "Epoch 89/100\n",
      "10000/10000 [==============================] - 1s 93us/step - loss: 0.1523 - acc: 0.9496 - val_loss: 0.4068 - val_acc: 0.8786\n",
      "Epoch 90/100\n",
      "10000/10000 [==============================] - 1s 87us/step - loss: 0.1564 - acc: 0.9486 - val_loss: 0.2499 - val_acc: 0.9215\n",
      "Epoch 91/100\n",
      "10000/10000 [==============================] - 1s 88us/step - loss: 0.1538 - acc: 0.9484 - val_loss: 0.2586 - val_acc: 0.9205\n",
      "Epoch 92/100\n",
      "10000/10000 [==============================] - 1s 84us/step - loss: 0.1505 - acc: 0.9507 - val_loss: 0.2594 - val_acc: 0.9222\n",
      "Epoch 93/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 0.1519 - acc: 0.9495 - val_loss: 0.2203 - val_acc: 0.9337\n",
      "Epoch 94/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 0.1508 - acc: 0.9504 - val_loss: 0.2055 - val_acc: 0.9365\n",
      "Epoch 95/100\n",
      "10000/10000 [==============================] - 1s 90us/step - loss: 0.1467 - acc: 0.9521 - val_loss: 0.2212 - val_acc: 0.9334\n",
      "Epoch 96/100\n",
      "10000/10000 [==============================] - 1s 91us/step - loss: 0.1461 - acc: 0.9525 - val_loss: 0.2345 - val_acc: 0.9277\n",
      "Epoch 97/100\n",
      "10000/10000 [==============================] - 1s 94us/step - loss: 0.1511 - acc: 0.9505 - val_loss: 0.3287 - val_acc: 0.9010\n",
      "Epoch 98/100\n",
      "10000/10000 [==============================] - 1s 88us/step - loss: 0.1486 - acc: 0.9514 - val_loss: 0.2396 - val_acc: 0.9280\n",
      "Epoch 99/100\n",
      "10000/10000 [==============================] - 1s 97us/step - loss: 0.1426 - acc: 0.9548 - val_loss: 0.2189 - val_acc: 0.9339\n",
      "Epoch 100/100\n",
      "10000/10000 [==============================] - 1s 97us/step - loss: 0.1440 - acc: 0.9546 - val_loss: 0.2574 - val_acc: 0.9270\n",
      "Test loss: 0.25735176564753054\n",
      "Test accuracy: 0.927\n"
     ]
    }
   ],
   "source": [
    "# Convolutional Neural Network - all layers trained\n",
    "batch_size = 128\n",
    "epochs = 100\n",
    "\n",
    "model = Sequential()\n",
    "# 1st conv layer\n",
    "model.add(Conv2D(5, kernel_size=(3, 3),\n",
    "                 activation='selu',\n",
    "                 padding='same',\n",
    "                 input_shape=input_shape))\n",
    "# model.add(AveragePooling2D(pool_size=(5, 5)))\n",
    "model.add(MaxPooling2D(pool_size=(5, 5)))\n",
    "# 2nd conv layer\n",
    "model.add(Conv2D(10, (3, 3), padding='same', activation='selu'))\n",
    "# model.add(AveragePooling2D(pool_size=(2, 2)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# 3rd conv layer\n",
    "model.add(Conv2D(15, (3, 3), padding='same', activation='selu'))\n",
    "# model.add(AveragePooling2D(pool_size=(2, 2)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(3000, activation='selu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train10k, Y_train10k,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(X_test, Y_test))\n",
    "\n",
    "score = model.evaluate(X_test, Y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2018-02-09T20:31:47.442Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 55000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "55000/55000 [==============================] - 4s 64us/step - loss: 2.3007 - acc: 0.1137 - val_loss: 2.2983 - val_acc: 0.1135\n",
      "Epoch 2/100\n",
      "55000/55000 [==============================] - 2s 34us/step - loss: 2.2496 - acc: 0.1642 - val_loss: 2.0302 - val_acc: 0.3191\n",
      "Epoch 3/100\n",
      "55000/55000 [==============================] - 2s 34us/step - loss: 1.5390 - acc: 0.4882 - val_loss: 1.1120 - val_acc: 0.6076\n",
      "Epoch 4/100\n",
      "55000/55000 [==============================] - 2s 35us/step - loss: 0.9917 - acc: 0.6599 - val_loss: 0.8517 - val_acc: 0.7108\n",
      "Epoch 5/100\n",
      "55000/55000 [==============================] - 2s 35us/step - loss: 0.8050 - acc: 0.7271 - val_loss: 0.7114 - val_acc: 0.7632\n",
      "Epoch 6/100\n",
      "55000/55000 [==============================] - 2s 34us/step - loss: 0.6923 - acc: 0.7697 - val_loss: 0.6341 - val_acc: 0.7896\n",
      "Epoch 7/100\n",
      "55000/55000 [==============================] - 2s 34us/step - loss: 0.6152 - acc: 0.7990 - val_loss: 0.5652 - val_acc: 0.8152\n",
      "Epoch 8/100\n",
      "55000/55000 [==============================] - 2s 34us/step - loss: 0.5586 - acc: 0.8185 - val_loss: 0.5236 - val_acc: 0.8305\n",
      "Epoch 9/100\n",
      "55000/55000 [==============================] - 2s 33us/step - loss: 0.5168 - acc: 0.8328 - val_loss: 0.4844 - val_acc: 0.8459\n",
      "Epoch 10/100\n",
      "55000/55000 [==============================] - 2s 34us/step - loss: 0.4814 - acc: 0.8447 - val_loss: 0.4499 - val_acc: 0.8558\n",
      "Epoch 11/100\n",
      "55000/55000 [==============================] - 2s 33us/step - loss: 0.4535 - acc: 0.8540 - val_loss: 0.4193 - val_acc: 0.8669\n",
      "Epoch 12/100\n",
      "55000/55000 [==============================] - 2s 34us/step - loss: 0.4296 - acc: 0.8623 - val_loss: 0.4298 - val_acc: 0.8670\n",
      "Epoch 13/100\n",
      "55000/55000 [==============================] - 2s 35us/step - loss: 0.4075 - acc: 0.8681 - val_loss: 0.3749 - val_acc: 0.8805\n",
      "Epoch 14/100\n",
      "55000/55000 [==============================] - 2s 34us/step - loss: 0.3890 - acc: 0.8737 - val_loss: 0.3583 - val_acc: 0.8872\n",
      "Epoch 15/100\n",
      "55000/55000 [==============================] - 2s 36us/step - loss: 0.3727 - acc: 0.8789 - val_loss: 0.3458 - val_acc: 0.8924\n",
      "Epoch 16/100\n",
      "55000/55000 [==============================] - 2s 33us/step - loss: 0.3572 - acc: 0.8841 - val_loss: 0.3306 - val_acc: 0.8956\n",
      "Epoch 17/100\n",
      "55000/55000 [==============================] - 2s 36us/step - loss: 0.3454 - acc: 0.8889 - val_loss: 0.3345 - val_acc: 0.8951\n",
      "Epoch 18/100\n",
      "55000/55000 [==============================] - 2s 35us/step - loss: 0.3319 - acc: 0.8927 - val_loss: 0.3084 - val_acc: 0.9030\n",
      "Epoch 19/100\n",
      "55000/55000 [==============================] - 2s 35us/step - loss: 0.3220 - acc: 0.8954 - val_loss: 0.2996 - val_acc: 0.9060\n",
      "Epoch 20/100\n",
      "55000/55000 [==============================] - 2s 34us/step - loss: 0.3115 - acc: 0.8994 - val_loss: 0.2940 - val_acc: 0.9084\n",
      "Epoch 21/100\n",
      "55000/55000 [==============================] - 2s 32us/step - loss: 0.3028 - acc: 0.9014 - val_loss: 0.2850 - val_acc: 0.9077\n",
      "Epoch 22/100\n",
      "48896/55000 [=========================>....] - ETA: 0s - loss: 0.2963 - acc: 0.9045"
     ]
    }
   ],
   "source": [
    "# Convolutional ELM - only final values trained\n",
    "batch_size = 256\n",
    "epochs = 100\n",
    "\n",
    "model = Sequential()\n",
    "# 1st conv layer\n",
    "model.add(Conv2D(5, kernel_size=(3, 3),\n",
    "                 activation='selu',\n",
    "                 padding='same',\n",
    "                 input_shape=input_shape))\n",
    "# model.add(AveragePooling2D(pool_size=(5, 5)))\n",
    "model.add(MaxPooling2D(pool_size=(5, 5)))\n",
    "# 2nd conv layer\n",
    "model.add(Conv2D(10, (3, 3), padding='same', activation='selu'))\n",
    "# model.add(AveragePooling2D(pool_size=(2, 2)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# 3rd conv layer\n",
    "model.add(Conv2D(15, (3, 3), padding='same', activation='selu'))\n",
    "# model.add(AveragePooling2D(pool_size=(2, 2)))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "# model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "# model.add(Dense(128, activation='relu'))\n",
    "# model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.Adadelta(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train, Y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(X_test, Y_test))\n",
    "\n",
    "score = model.evaluate(X_test, Y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}