cnn3 - NoDropout.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MODEL CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\DELL\\Anaconda3\\envs\\tensorflowGPU\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import helpers\n",
    "# import brewer2mpl\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "# set3 = brewer2mpl.get_map('Set3', 'qualitative', 4).mpl_colors\n",
    "%matplotlib inline\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten\n",
    "from keras.models import Sequential\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.tools import freeze_graph\n",
    "from tensorflow.python.tools import optimize_for_inference_lib\n",
    "from keras.models import model_from_json\n",
    "from keras.optimizers import SGD\n",
    "from tensorflow.python.framework import graph_util\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start loading data...\n",
      "\n",
      "Data has been loaded\n",
      "Data latih : 14316\n",
      "Data Uji :  3579 \n",
      "\n",
      "Total Data :  17895\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\DELL\\Anaconda3\\envs\\tensorflowGPU\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "num_classes = 36\n",
    "epochs = 250\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "print('Start loading data...\\n')\n",
    "files, labels = helpers.load_chars74k_data()\n",
    "X, y = helpers.create_dataset(files, labels)\n",
    "print('Data has been loaded')\n",
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, y, random_state=2, train_size=0.8)\n",
    "\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "\n",
    "x_train/=255.\n",
    "x_test/=255.\n",
    "\n",
    "print('Data latih :', x_train.shape[0])\n",
    "print('Data Uji : ', x_test.shape[0],'\\n')\n",
    "\n",
    "print('Total Data : ', x_train.shape[0]+ x_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if K.image_data_format() == 'channels_first':\n",
    "    x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)\n",
    "    x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)\n",
    "    input_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "    x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)\n",
    "    x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)\n",
    "    input_shape = (img_rows, img_cols, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training the model.\n",
      "WARNING:tensorflow:Variable *= will be deprecated. Use variable.assign_mul if you want assignment to the variable value or 'x = x * y' if you want a new python Tensor object.\n",
      "Train on 14316 samples, validate on 3579 samples\n",
      "Epoch 1/250\n",
      "14316/14316 [==============================] - 8s 563us/step - loss: 1.8244 - acc: 0.4924 - val_loss: 0.6250 - val_acc: 0.8268\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.62499, saving model to model/model.h5\n",
      "Epoch 2/250\n",
      "14316/14316 [==============================] - 5s 320us/step - loss: 0.4532 - acc: 0.8713 - val_loss: 0.4291 - val_acc: 0.8779\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.62499 to 0.42913, saving model to model/model.h5\n",
      "Epoch 3/250\n",
      "14316/14316 [==============================] - 5s 319us/step - loss: 0.2677 - acc: 0.9204 - val_loss: 0.3388 - val_acc: 0.9056\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.42913 to 0.33880, saving model to model/model.h5\n",
      "Epoch 4/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 0.1778 - acc: 0.9466 - val_loss: 0.2991 - val_acc: 0.9170\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.33880 to 0.29914, saving model to model/model.h5\n",
      "Epoch 5/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 0.1185 - acc: 0.9647 - val_loss: 0.3263 - val_acc: 0.9156\n",
      "\n",
      "Epoch 00005: val_loss did not improve\n",
      "Epoch 6/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 0.0788 - acc: 0.9776 - val_loss: 0.3019 - val_acc: 0.9257\n",
      "\n",
      "Epoch 00006: val_loss did not improve\n",
      "Epoch 7/250\n",
      "14316/14316 [==============================] - 4s 314us/step - loss: 0.0598 - acc: 0.9828 - val_loss: 0.3471 - val_acc: 0.9179\n",
      "\n",
      "Epoch 00007: val_loss did not improve\n",
      "Epoch 8/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 0.0468 - acc: 0.9871 - val_loss: 0.3228 - val_acc: 0.9237\n",
      "\n",
      "Epoch 00008: val_loss did not improve\n",
      "Epoch 9/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 0.0301 - acc: 0.9913 - val_loss: 0.3232 - val_acc: 0.9301\n",
      "\n",
      "Epoch 00009: val_loss did not improve\n",
      "Epoch 10/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 0.0264 - acc: 0.9917 - val_loss: 0.3263 - val_acc: 0.9313\n",
      "\n",
      "Epoch 00010: val_loss did not improve\n",
      "Epoch 11/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 0.0246 - acc: 0.9934 - val_loss: 0.3130 - val_acc: 0.9321\n",
      "\n",
      "Epoch 00011: val_loss did not improve\n",
      "Epoch 12/250\n",
      "14316/14316 [==============================] - 5s 324us/step - loss: 0.0254 - acc: 0.9931 - val_loss: 0.3306 - val_acc: 0.9299\n",
      "\n",
      "Epoch 00012: val_loss did not improve\n",
      "Epoch 13/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 0.0208 - acc: 0.9939 - val_loss: 0.3186 - val_acc: 0.9296\n",
      "\n",
      "Epoch 00013: val_loss did not improve\n",
      "Epoch 14/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 0.0153 - acc: 0.9957 - val_loss: 0.3382 - val_acc: 0.9355\n",
      "\n",
      "Epoch 00014: val_loss did not improve\n",
      "Epoch 15/250\n",
      "14316/14316 [==============================] - 5s 320us/step - loss: 0.0123 - acc: 0.9964 - val_loss: 0.2972 - val_acc: 0.9374\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.29914 to 0.29715, saving model to model/model.h5\n",
      "Epoch 16/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 0.0074 - acc: 0.9984 - val_loss: 0.3400 - val_acc: 0.9335\n",
      "\n",
      "Epoch 00016: val_loss did not improve\n",
      "Epoch 17/250\n",
      "14316/14316 [==============================] - 5s 316us/step - loss: 0.0097 - acc: 0.9980 - val_loss: 0.3229 - val_acc: 0.9408\n",
      "\n",
      "Epoch 00017: val_loss did not improve\n",
      "Epoch 18/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 0.0109 - acc: 0.9973 - val_loss: 0.2988 - val_acc: 0.9424\n",
      "\n",
      "Epoch 00018: val_loss did not improve\n",
      "Epoch 19/250\n",
      "14316/14316 [==============================] - 5s 316us/step - loss: 0.0029 - acc: 0.9994 - val_loss: 0.3054 - val_acc: 0.9430\n",
      "\n",
      "Epoch 00019: val_loss did not improve\n",
      "Epoch 20/250\n",
      "14316/14316 [==============================] - 5s 316us/step - loss: 0.0016 - acc: 0.9999 - val_loss: 0.3112 - val_acc: 0.9433\n",
      "\n",
      "Epoch 00020: val_loss did not improve\n",
      "Epoch 21/250\n",
      "14316/14316 [==============================] - 5s 316us/step - loss: 9.5539e-04 - acc: 0.9997 - val_loss: 0.3119 - val_acc: 0.9427\n",
      "\n",
      "Epoch 00021: val_loss did not improve\n",
      "Epoch 22/250\n",
      "14316/14316 [==============================] - 5s 319us/step - loss: 7.9366e-04 - acc: 0.9999 - val_loss: 0.3174 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00022: val_loss did not improve\n",
      "Epoch 23/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 0.0011 - acc: 0.9999 - val_loss: 0.3222 - val_acc: 0.9416\n",
      "\n",
      "Epoch 00023: val_loss did not improve\n",
      "Epoch 24/250\n",
      "14316/14316 [==============================] - 5s 316us/step - loss: 2.2615e-04 - acc: 1.0000 - val_loss: 0.3260 - val_acc: 0.9433\n",
      "\n",
      "Epoch 00024: val_loss did not improve\n",
      "Epoch 25/250\n",
      "14316/14316 [==============================] - 4s 314us/step - loss: 2.3269e-04 - acc: 1.0000 - val_loss: 0.3311 - val_acc: 0.9433\n",
      "\n",
      "Epoch 00025: val_loss did not improve\n",
      "Epoch 26/250\n",
      "14316/14316 [==============================] - 5s 324us/step - loss: 7.1787e-04 - acc: 0.9999 - val_loss: 0.3311 - val_acc: 0.9430\n",
      "\n",
      "Epoch 00026: val_loss did not improve\n",
      "Epoch 27/250\n",
      "14316/14316 [==============================] - 5s 333us/step - loss: 3.3493e-04 - acc: 0.9999 - val_loss: 0.3341 - val_acc: 0.9452\n",
      "\n",
      "Epoch 00027: val_loss did not improve\n",
      "Epoch 28/250\n",
      "14316/14316 [==============================] - 5s 323us/step - loss: 5.7080e-04 - acc: 0.9999 - val_loss: 0.3354 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00028: val_loss did not improve\n",
      "Epoch 29/250\n",
      "14316/14316 [==============================] - 5s 325us/step - loss: 2.5074e-04 - acc: 0.9999 - val_loss: 0.3373 - val_acc: 0.9430\n",
      "\n",
      "Epoch 00029: val_loss did not improve\n",
      "Epoch 30/250\n",
      "14316/14316 [==============================] - 5s 321us/step - loss: 1.4988e-04 - acc: 1.0000 - val_loss: 0.3417 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00030: val_loss did not improve\n",
      "Epoch 31/250\n",
      "14316/14316 [==============================] - 5s 321us/step - loss: 1.0019e-04 - acc: 1.0000 - val_loss: 0.3436 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00031: val_loss did not improve\n",
      "Epoch 32/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 8.2584e-05 - acc: 1.0000 - val_loss: 0.3465 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00032: val_loss did not improve\n",
      "Epoch 33/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 7.4695e-05 - acc: 1.0000 - val_loss: 0.3486 - val_acc: 0.9447\n",
      "\n",
      "Epoch 00033: val_loss did not improve\n",
      "Epoch 34/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 6.8715e-05 - acc: 1.0000 - val_loss: 0.3505 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00034: val_loss did not improve\n",
      "Epoch 35/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 6.3909e-05 - acc: 1.0000 - val_loss: 0.3525 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00035: val_loss did not improve\n",
      "Epoch 36/250\n",
      "14316/14316 [==============================] - 5s 319us/step - loss: 5.9775e-05 - acc: 1.0000 - val_loss: 0.3542 - val_acc: 0.9452\n",
      "\n",
      "Epoch 00036: val_loss did not improve\n",
      "Epoch 37/250\n",
      "14316/14316 [==============================] - 5s 323us/step - loss: 5.6066e-05 - acc: 1.0000 - val_loss: 0.3556 - val_acc: 0.9452\n",
      "\n",
      "Epoch 00037: val_loss did not improve\n",
      "Epoch 38/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 5.3072e-05 - acc: 1.0000 - val_loss: 0.3572 - val_acc: 0.9452\n",
      "\n",
      "Epoch 00038: val_loss did not improve\n",
      "Epoch 39/250\n",
      "14316/14316 [==============================] - 5s 316us/step - loss: 5.0252e-05 - acc: 1.0000 - val_loss: 0.3586 - val_acc: 0.9452\n",
      "\n",
      "Epoch 00039: val_loss did not improve\n",
      "Epoch 40/250\n",
      "14316/14316 [==============================] - 5s 322us/step - loss: 4.7787e-05 - acc: 1.0000 - val_loss: 0.3600 - val_acc: 0.9452\n",
      "\n",
      "Epoch 00040: val_loss did not improve\n",
      "Epoch 41/250\n",
      "14316/14316 [==============================] - 5s 325us/step - loss: 4.5494e-05 - acc: 1.0000 - val_loss: 0.3616 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00041: val_loss did not improve\n",
      "Epoch 42/250\n",
      "14316/14316 [==============================] - 5s 319us/step - loss: 4.3515e-05 - acc: 1.0000 - val_loss: 0.3627 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00042: val_loss did not improve\n",
      "Epoch 43/250\n",
      "14316/14316 [==============================] - 5s 320us/step - loss: 4.1744e-05 - acc: 1.0000 - val_loss: 0.3637 - val_acc: 0.9450\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00043: val_loss did not improve\n",
      "Epoch 44/250\n",
      "14316/14316 [==============================] - 5s 320us/step - loss: 4.0099e-05 - acc: 1.0000 - val_loss: 0.3651 - val_acc: 0.9452\n",
      "\n",
      "Epoch 00044: val_loss did not improve\n",
      "Epoch 45/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 3.8456e-05 - acc: 1.0000 - val_loss: 0.3662 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00045: val_loss did not improve\n",
      "Epoch 46/250\n",
      "14316/14316 [==============================] - 5s 320us/step - loss: 3.6929e-05 - acc: 1.0000 - val_loss: 0.3670 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00046: val_loss did not improve\n",
      "Epoch 47/250\n",
      "14316/14316 [==============================] - 5s 323us/step - loss: 3.5670e-05 - acc: 1.0000 - val_loss: 0.3681 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00047: val_loss did not improve\n",
      "Epoch 48/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 3.4462e-05 - acc: 1.0000 - val_loss: 0.3692 - val_acc: 0.9447\n",
      "\n",
      "Epoch 00048: val_loss did not improve\n",
      "Epoch 49/250\n",
      "14316/14316 [==============================] - 5s 316us/step - loss: 3.3270e-05 - acc: 1.0000 - val_loss: 0.3700 - val_acc: 0.9447\n",
      "\n",
      "Epoch 00049: val_loss did not improve\n",
      "Epoch 50/250\n",
      "14316/14316 [==============================] - 5s 321us/step - loss: 3.2250e-05 - acc: 1.0000 - val_loss: 0.3710 - val_acc: 0.9447\n",
      "\n",
      "Epoch 00050: val_loss did not improve\n",
      "Epoch 51/250\n",
      "14316/14316 [==============================] - 5s 322us/step - loss: 3.1258e-05 - acc: 1.0000 - val_loss: 0.3719 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00051: val_loss did not improve\n",
      "Epoch 52/250\n",
      "14316/14316 [==============================] - 5s 322us/step - loss: 3.0345e-05 - acc: 1.0000 - val_loss: 0.3728 - val_acc: 0.9450\n",
      "\n",
      "Epoch 00052: val_loss did not improve\n",
      "Epoch 53/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 2.9329e-05 - acc: 1.0000 - val_loss: 0.3736 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00053: val_loss did not improve\n",
      "Epoch 54/250\n",
      "14316/14316 [==============================] - 5s 319us/step - loss: 2.8620e-05 - acc: 1.0000 - val_loss: 0.3744 - val_acc: 0.9447\n",
      "\n",
      "Epoch 00054: val_loss did not improve\n",
      "Epoch 55/250\n",
      "14316/14316 [==============================] - 5s 319us/step - loss: 2.7786e-05 - acc: 1.0000 - val_loss: 0.3753 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00055: val_loss did not improve\n",
      "Epoch 56/250\n",
      "14316/14316 [==============================] - 5s 322us/step - loss: 2.6999e-05 - acc: 1.0000 - val_loss: 0.3760 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00056: val_loss did not improve\n",
      "Epoch 57/250\n",
      "14316/14316 [==============================] - 5s 321us/step - loss: 2.6312e-05 - acc: 1.0000 - val_loss: 0.3767 - val_acc: 0.9447\n",
      "\n",
      "Epoch 00057: val_loss did not improve\n",
      "Epoch 58/250\n",
      "14316/14316 [==============================] - 5s 319us/step - loss: 2.5621e-05 - acc: 1.0000 - val_loss: 0.3775 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00058: val_loss did not improve\n",
      "Epoch 59/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 2.5018e-05 - acc: 1.0000 - val_loss: 0.3782 - val_acc: 0.9447\n",
      "\n",
      "Epoch 00059: val_loss did not improve\n",
      "Epoch 60/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 2.4385e-05 - acc: 1.0000 - val_loss: 0.3789 - val_acc: 0.9447\n",
      "\n",
      "Epoch 00060: val_loss did not improve\n",
      "Epoch 61/250\n",
      "14316/14316 [==============================] - 5s 317us/step - loss: 2.3796e-05 - acc: 1.0000 - val_loss: 0.3797 - val_acc: 0.9447\n",
      "\n",
      "Epoch 00061: val_loss did not improve\n",
      "Epoch 62/250\n",
      "14316/14316 [==============================] - 5s 321us/step - loss: 2.3278e-05 - acc: 1.0000 - val_loss: 0.3803 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00062: val_loss did not improve\n",
      "Epoch 63/250\n",
      "14316/14316 [==============================] - 5s 318us/step - loss: 2.2661e-05 - acc: 1.0000 - val_loss: 0.3810 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00063: val_loss did not improve\n",
      "Epoch 64/250\n",
      "14316/14316 [==============================] - 5s 323us/step - loss: 2.2248e-05 - acc: 1.0000 - val_loss: 0.3816 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00064: val_loss did not improve\n",
      "Epoch 65/250\n",
      "14316/14316 [==============================] - 5s 321us/step - loss: 2.1762e-05 - acc: 1.0000 - val_loss: 0.3821 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00065: val_loss did not improve\n",
      "Epoch 66/250\n",
      "14316/14316 [==============================] - 5s 319us/step - loss: 2.1294e-05 - acc: 1.0000 - val_loss: 0.3828 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00066: val_loss did not improve\n",
      "Epoch 67/250\n",
      "14316/14316 [==============================] - 5s 338us/step - loss: 2.0857e-05 - acc: 1.0000 - val_loss: 0.3833 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00067: val_loss did not improve\n",
      "Epoch 68/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 2.0423e-05 - acc: 1.0000 - val_loss: 0.3838 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00068: val_loss did not improve\n",
      "Epoch 69/250\n",
      "14316/14316 [==============================] - 4s 307us/step - loss: 2.0054e-05 - acc: 1.0000 - val_loss: 0.3845 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00069: val_loss did not improve\n",
      "Epoch 70/250\n",
      "14316/14316 [==============================] - 4s 306us/step - loss: 1.9671e-05 - acc: 1.0000 - val_loss: 0.3849 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00070: val_loss did not improve\n",
      "Epoch 71/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 1.9225e-05 - acc: 1.0000 - val_loss: 0.3855 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00071: val_loss did not improve\n",
      "Epoch 72/250\n",
      "14316/14316 [==============================] - 4s 307us/step - loss: 1.8914e-05 - acc: 1.0000 - val_loss: 0.3861 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00072: val_loss did not improve\n",
      "Epoch 73/250\n",
      "14316/14316 [==============================] - 4s 309us/step - loss: 1.8531e-05 - acc: 1.0000 - val_loss: 0.3867 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00073: val_loss did not improve\n",
      "Epoch 74/250\n",
      "14316/14316 [==============================] - 4s 311us/step - loss: 1.8200e-05 - acc: 1.0000 - val_loss: 0.3871 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00074: val_loss did not improve\n",
      "Epoch 75/250\n",
      "14316/14316 [==============================] - 4s 310us/step - loss: 1.7884e-05 - acc: 1.0000 - val_loss: 0.3876 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00075: val_loss did not improve\n",
      "Epoch 76/250\n",
      "14316/14316 [==============================] - 4s 311us/step - loss: 1.7598e-05 - acc: 1.0000 - val_loss: 0.3881 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00076: val_loss did not improve\n",
      "Epoch 77/250\n",
      "14316/14316 [==============================] - 4s 306us/step - loss: 1.7274e-05 - acc: 1.0000 - val_loss: 0.3887 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00077: val_loss did not improve\n",
      "Epoch 78/250\n",
      "14316/14316 [==============================] - 4s 309us/step - loss: 1.6990e-05 - acc: 1.0000 - val_loss: 0.3891 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00078: val_loss did not improve\n",
      "Epoch 79/250\n",
      "14316/14316 [==============================] - 4s 310us/step - loss: 1.6705e-05 - acc: 1.0000 - val_loss: 0.3896 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00079: val_loss did not improve\n",
      "Epoch 80/250\n",
      "14316/14316 [==============================] - 4s 307us/step - loss: 1.6445e-05 - acc: 1.0000 - val_loss: 0.3902 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00080: val_loss did not improve\n",
      "Epoch 81/250\n",
      "14316/14316 [==============================] - 4s 306us/step - loss: 1.6157e-05 - acc: 1.0000 - val_loss: 0.3906 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00081: val_loss did not improve\n",
      "Epoch 82/250\n",
      "14316/14316 [==============================] - 4s 309us/step - loss: 1.5912e-05 - acc: 1.0000 - val_loss: 0.3910 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00082: val_loss did not improve\n",
      "Epoch 83/250\n",
      "14316/14316 [==============================] - 4s 307us/step - loss: 1.5683e-05 - acc: 1.0000 - val_loss: 0.3915 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00083: val_loss did not improve\n",
      "Epoch 84/250\n",
      "14316/14316 [==============================] - 4s 308us/step - loss: 1.5435e-05 - acc: 1.0000 - val_loss: 0.3919 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00084: val_loss did not improve\n",
      "Epoch 85/250\n",
      "14316/14316 [==============================] - 4s 310us/step - loss: 1.5154e-05 - acc: 1.0000 - val_loss: 0.3924 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00085: val_loss did not improve\n",
      "Epoch 86/250\n",
      "14316/14316 [==============================] - 4s 310us/step - loss: 1.4988e-05 - acc: 1.0000 - val_loss: 0.3927 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00086: val_loss did not improve\n",
      "Epoch 87/250\n",
      "14316/14316 [==============================] - 4s 307us/step - loss: 1.4757e-05 - acc: 1.0000 - val_loss: 0.3931 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00087: val_loss did not improve\n",
      "Epoch 88/250\n",
      "14316/14316 [==============================] - 4s 310us/step - loss: 1.4529e-05 - acc: 1.0000 - val_loss: 0.3936 - val_acc: 0.9441\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00088: val_loss did not improve\n",
      "Epoch 89/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 1.4327e-05 - acc: 1.0000 - val_loss: 0.3940 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00089: val_loss did not improve\n",
      "Epoch 90/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 1.4126e-05 - acc: 1.0000 - val_loss: 0.3943 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00090: val_loss did not improve\n",
      "Epoch 91/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 1.3925e-05 - acc: 1.0000 - val_loss: 0.3947 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00091: val_loss did not improve\n",
      "Epoch 92/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 1.3741e-05 - acc: 1.0000 - val_loss: 0.3951 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00092: val_loss did not improve\n",
      "Epoch 93/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 1.3561e-05 - acc: 1.0000 - val_loss: 0.3955 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00093: val_loss did not improve\n",
      "Epoch 94/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 1.3375e-05 - acc: 1.0000 - val_loss: 0.3958 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00094: val_loss did not improve\n",
      "Epoch 95/250\n",
      "14316/14316 [==============================] - 4s 297us/step - loss: 1.3200e-05 - acc: 1.0000 - val_loss: 0.3962 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00095: val_loss did not improve\n",
      "Epoch 96/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 1.3019e-05 - acc: 1.0000 - val_loss: 0.3966 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00096: val_loss did not improve\n",
      "Epoch 97/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 1.2864e-05 - acc: 1.0000 - val_loss: 0.3970 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00097: val_loss did not improve\n",
      "Epoch 98/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 1.2705e-05 - acc: 1.0000 - val_loss: 0.3974 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00098: val_loss did not improve\n",
      "Epoch 99/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 1.2543e-05 - acc: 1.0000 - val_loss: 0.3978 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00099: val_loss did not improve\n",
      "Epoch 100/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 1.2387e-05 - acc: 1.0000 - val_loss: 0.3981 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00100: val_loss did not improve\n",
      "Epoch 101/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 1.2241e-05 - acc: 1.0000 - val_loss: 0.3984 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00101: val_loss did not improve\n",
      "Epoch 102/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 1.2099e-05 - acc: 1.0000 - val_loss: 0.3988 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00102: val_loss did not improve\n",
      "Epoch 103/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 1.1952e-05 - acc: 1.0000 - val_loss: 0.3991 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00103: val_loss did not improve\n",
      "Epoch 104/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 1.1788e-05 - acc: 1.0000 - val_loss: 0.3994 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00104: val_loss did not improve\n",
      "Epoch 105/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 1.1676e-05 - acc: 1.0000 - val_loss: 0.3997 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00105: val_loss did not improve\n",
      "Epoch 106/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 1.1544e-05 - acc: 1.0000 - val_loss: 0.4000 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00106: val_loss did not improve\n",
      "Epoch 107/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 1.1415e-05 - acc: 1.0000 - val_loss: 0.4004 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00107: val_loss did not improve\n",
      "Epoch 108/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 1.1293e-05 - acc: 1.0000 - val_loss: 0.4007 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00108: val_loss did not improve\n",
      "Epoch 109/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 1.1164e-05 - acc: 1.0000 - val_loss: 0.4010 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00109: val_loss did not improve\n",
      "Epoch 110/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 1.1052e-05 - acc: 1.0000 - val_loss: 0.4013 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00110: val_loss did not improve\n",
      "Epoch 111/250\n",
      "14316/14316 [==============================] - 4s 298us/step - loss: 1.0898e-05 - acc: 1.0000 - val_loss: 0.4016 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00111: val_loss did not improve\n",
      "Epoch 112/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 1.0810e-05 - acc: 1.0000 - val_loss: 0.4019 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00112: val_loss did not improve\n",
      "Epoch 113/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 1.0676e-05 - acc: 1.0000 - val_loss: 0.4023 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00113: val_loss did not improve\n",
      "Epoch 114/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 1.0557e-05 - acc: 1.0000 - val_loss: 0.4025 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00114: val_loss did not improve\n",
      "Epoch 115/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 1.0469e-05 - acc: 1.0000 - val_loss: 0.4028 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00115: val_loss did not improve\n",
      "Epoch 116/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 1.0363e-05 - acc: 1.0000 - val_loss: 0.4031 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00116: val_loss did not improve\n",
      "Epoch 117/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 1.0259e-05 - acc: 1.0000 - val_loss: 0.4034 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00117: val_loss did not improve\n",
      "Epoch 118/250\n",
      "14316/14316 [==============================] - 4s 306us/step - loss: 1.0156e-05 - acc: 1.0000 - val_loss: 0.4037 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00118: val_loss did not improve\n",
      "Epoch 119/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 1.0040e-05 - acc: 1.0000 - val_loss: 0.4040 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00119: val_loss did not improve\n",
      "Epoch 120/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 9.9396e-06 - acc: 1.0000 - val_loss: 0.4042 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00120: val_loss did not improve\n",
      "Epoch 121/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 9.8514e-06 - acc: 1.0000 - val_loss: 0.4045 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00121: val_loss did not improve\n",
      "Epoch 122/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 9.7484e-06 - acc: 1.0000 - val_loss: 0.4048 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00122: val_loss did not improve\n",
      "Epoch 123/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 9.6558e-06 - acc: 1.0000 - val_loss: 0.4050 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00123: val_loss did not improve\n",
      "Epoch 124/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 9.5768e-06 - acc: 1.0000 - val_loss: 0.4053 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00124: val_loss did not improve\n",
      "Epoch 125/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 9.4670e-06 - acc: 1.0000 - val_loss: 0.4056 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00125: val_loss did not improve\n",
      "Epoch 126/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 9.3971e-06 - acc: 1.0000 - val_loss: 0.4058 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00126: val_loss did not improve\n",
      "Epoch 127/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 9.3103e-06 - acc: 1.0000 - val_loss: 0.4061 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00127: val_loss did not improve\n",
      "Epoch 128/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 9.2209e-06 - acc: 1.0000 - val_loss: 0.4063 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00128: val_loss did not improve\n",
      "Epoch 129/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 9.1331e-06 - acc: 1.0000 - val_loss: 0.4066 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00129: val_loss did not improve\n",
      "Epoch 130/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 9.0556e-06 - acc: 1.0000 - val_loss: 0.4068 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00130: val_loss did not improve\n",
      "Epoch 131/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 8.9823e-06 - acc: 1.0000 - val_loss: 0.4070 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00131: val_loss did not improve\n",
      "Epoch 132/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 8.8894e-06 - acc: 1.0000 - val_loss: 0.4073 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00132: val_loss did not improve\n",
      "Epoch 133/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14316/14316 [==============================] - 4s 299us/step - loss: 8.7978e-06 - acc: 1.0000 - val_loss: 0.4076 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00133: val_loss did not improve\n",
      "Epoch 134/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 8.7311e-06 - acc: 1.0000 - val_loss: 0.4078 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00134: val_loss did not improve\n",
      "Epoch 135/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 8.6705e-06 - acc: 1.0000 - val_loss: 0.4080 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00135: val_loss did not improve\n",
      "Epoch 136/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 8.5915e-06 - acc: 1.0000 - val_loss: 0.4082 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00136: val_loss did not improve\n",
      "Epoch 137/250\n",
      "14316/14316 [==============================] - 4s 298us/step - loss: 8.5173e-06 - acc: 1.0000 - val_loss: 0.4085 - val_acc: 0.9441\n",
      "\n",
      "Epoch 00137: val_loss did not improve\n",
      "Epoch 138/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 8.4370e-06 - acc: 1.0000 - val_loss: 0.4087 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00138: val_loss did not improve\n",
      "Epoch 139/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 8.3850e-06 - acc: 1.0000 - val_loss: 0.4089 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00139: val_loss did not improve\n",
      "Epoch 140/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 8.3087e-06 - acc: 1.0000 - val_loss: 0.4091 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00140: val_loss did not improve\n",
      "Epoch 141/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 8.2261e-06 - acc: 1.0000 - val_loss: 0.4094 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00141: val_loss did not improve\n",
      "Epoch 142/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 8.1673e-06 - acc: 1.0000 - val_loss: 0.4096 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00142: val_loss did not improve\n",
      "Epoch 143/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 8.1019e-06 - acc: 1.0000 - val_loss: 0.4099 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00143: val_loss did not improve\n",
      "Epoch 144/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 8.0405e-06 - acc: 1.0000 - val_loss: 0.4101 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00144: val_loss did not improve\n",
      "Epoch 145/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 7.9619e-06 - acc: 1.0000 - val_loss: 0.4103 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00145: val_loss did not improve\n",
      "Epoch 146/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 7.9065e-06 - acc: 1.0000 - val_loss: 0.4105 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00146: val_loss did not improve\n",
      "Epoch 147/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 7.8485e-06 - acc: 1.0000 - val_loss: 0.4107 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00147: val_loss did not improve\n",
      "Epoch 148/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 7.7938e-06 - acc: 1.0000 - val_loss: 0.4109 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00148: val_loss did not improve\n",
      "Epoch 149/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 7.7273e-06 - acc: 1.0000 - val_loss: 0.4112 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00149: val_loss did not improve\n",
      "Epoch 150/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 7.6765e-06 - acc: 1.0000 - val_loss: 0.4113 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00150: val_loss did not improve\n",
      "Epoch 151/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 7.6166e-06 - acc: 1.0000 - val_loss: 0.4115 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00151: val_loss did not improve\n",
      "Epoch 152/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 7.5515e-06 - acc: 1.0000 - val_loss: 0.4117 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00152: val_loss did not improve\n",
      "Epoch 153/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 7.5001e-06 - acc: 1.0000 - val_loss: 0.4119 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00153: val_loss did not improve\n",
      "Epoch 154/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 7.4386e-06 - acc: 1.0000 - val_loss: 0.4121 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00154: val_loss did not improve\n",
      "Epoch 155/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 7.3862e-06 - acc: 1.0000 - val_loss: 0.4123 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00155: val_loss did not improve\n",
      "Epoch 156/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 7.3318e-06 - acc: 1.0000 - val_loss: 0.4125 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00156: val_loss did not improve\n",
      "Epoch 157/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 7.2817e-06 - acc: 1.0000 - val_loss: 0.4127 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00157: val_loss did not improve\n",
      "Epoch 158/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 7.2376e-06 - acc: 1.0000 - val_loss: 0.4129 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00158: val_loss did not improve\n",
      "Epoch 159/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 7.1631e-06 - acc: 1.0000 - val_loss: 0.4131 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00159: val_loss did not improve\n",
      "Epoch 160/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 7.1261e-06 - acc: 1.0000 - val_loss: 0.4133 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00160: val_loss did not improve\n",
      "Epoch 161/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 7.0577e-06 - acc: 1.0000 - val_loss: 0.4135 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00161: val_loss did not improve\n",
      "Epoch 162/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 7.0259e-06 - acc: 1.0000 - val_loss: 0.4137 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00162: val_loss did not improve\n",
      "Epoch 163/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 6.9817e-06 - acc: 1.0000 - val_loss: 0.4139 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00163: val_loss did not improve\n",
      "Epoch 164/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 6.9290e-06 - acc: 1.0000 - val_loss: 0.4141 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00164: val_loss did not improve\n",
      "Epoch 165/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 6.8796e-06 - acc: 1.0000 - val_loss: 0.4142 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00165: val_loss did not improve\n",
      "Epoch 166/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 6.8235e-06 - acc: 1.0000 - val_loss: 0.4144 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00166: val_loss did not improve\n",
      "Epoch 167/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 6.7904e-06 - acc: 1.0000 - val_loss: 0.4146 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00167: val_loss did not improve\n",
      "Epoch 168/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 6.7476e-06 - acc: 1.0000 - val_loss: 0.4148 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00168: val_loss did not improve\n",
      "Epoch 169/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 6.6951e-06 - acc: 1.0000 - val_loss: 0.4149 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00169: val_loss did not improve\n",
      "Epoch 170/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 6.6520e-06 - acc: 1.0000 - val_loss: 0.4151 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00170: val_loss did not improve\n",
      "Epoch 171/250\n",
      "14316/14316 [==============================] - 4s 306us/step - loss: 6.6095e-06 - acc: 1.0000 - val_loss: 0.4153 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00171: val_loss did not improve\n",
      "Epoch 172/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 6.5650e-06 - acc: 1.0000 - val_loss: 0.4154 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00172: val_loss did not improve\n",
      "Epoch 173/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 6.5189e-06 - acc: 1.0000 - val_loss: 0.4156 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00173: val_loss did not improve\n",
      "Epoch 174/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 6.4806e-06 - acc: 1.0000 - val_loss: 0.4158 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00174: val_loss did not improve\n",
      "Epoch 175/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 6.4415e-06 - acc: 1.0000 - val_loss: 0.4160 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00175: val_loss did not improve\n",
      "Epoch 176/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 6.3955e-06 - acc: 1.0000 - val_loss: 0.4161 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00176: val_loss did not improve\n",
      "Epoch 177/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 6.3568e-06 - acc: 1.0000 - val_loss: 0.4162 - val_acc: 0.9438\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 00177: val_loss did not improve\n",
      "Epoch 178/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 6.3180e-06 - acc: 1.0000 - val_loss: 0.4164 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00178: val_loss did not improve\n",
      "Epoch 179/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 6.2794e-06 - acc: 1.0000 - val_loss: 0.4166 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00179: val_loss did not improve\n",
      "Epoch 180/250\n",
      "14316/14316 [==============================] - 4s 307us/step - loss: 6.2400e-06 - acc: 1.0000 - val_loss: 0.4168 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00180: val_loss did not improve\n",
      "Epoch 181/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 6.1991e-06 - acc: 1.0000 - val_loss: 0.4170 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00181: val_loss did not improve\n",
      "Epoch 182/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 6.1629e-06 - acc: 1.0000 - val_loss: 0.4171 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00182: val_loss did not improve\n",
      "Epoch 183/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 6.1250e-06 - acc: 1.0000 - val_loss: 0.4173 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00183: val_loss did not improve\n",
      "Epoch 184/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 6.0904e-06 - acc: 1.0000 - val_loss: 0.4174 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00184: val_loss did not improve\n",
      "Epoch 185/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 6.0497e-06 - acc: 1.0000 - val_loss: 0.4176 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00185: val_loss did not improve\n",
      "Epoch 186/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 6.0090e-06 - acc: 1.0000 - val_loss: 0.4177 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00186: val_loss did not improve\n",
      "Epoch 187/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 5.9677e-06 - acc: 1.0000 - val_loss: 0.4179 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00187: val_loss did not improve\n",
      "Epoch 188/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 5.9410e-06 - acc: 1.0000 - val_loss: 0.4181 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00188: val_loss did not improve\n",
      "Epoch 189/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 5.9104e-06 - acc: 1.0000 - val_loss: 0.4182 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00189: val_loss did not improve\n",
      "Epoch 190/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 5.8701e-06 - acc: 1.0000 - val_loss: 0.4184 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00190: val_loss did not improve\n",
      "Epoch 191/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 5.8375e-06 - acc: 1.0000 - val_loss: 0.4185 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00191: val_loss did not improve\n",
      "Epoch 192/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 5.7997e-06 - acc: 1.0000 - val_loss: 0.4187 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00192: val_loss did not improve\n",
      "Epoch 193/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 5.7673e-06 - acc: 1.0000 - val_loss: 0.4188 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00193: val_loss did not improve\n",
      "Epoch 194/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 5.7380e-06 - acc: 1.0000 - val_loss: 0.4190 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00194: val_loss did not improve\n",
      "Epoch 195/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 5.7107e-06 - acc: 1.0000 - val_loss: 0.4191 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00195: val_loss did not improve\n",
      "Epoch 196/250\n",
      "14316/14316 [==============================] - 4s 294us/step - loss: 5.6733e-06 - acc: 1.0000 - val_loss: 0.4192 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00196: val_loss did not improve\n",
      "Epoch 197/250\n",
      "14316/14316 [==============================] - 4s 295us/step - loss: 5.6431e-06 - acc: 1.0000 - val_loss: 0.4194 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00197: val_loss did not improve\n",
      "Epoch 198/250\n",
      "14316/14316 [==============================] - 4s 298us/step - loss: 5.6107e-06 - acc: 1.0000 - val_loss: 0.4196 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00198: val_loss did not improve\n",
      "Epoch 199/250\n",
      "14316/14316 [==============================] - 4s 296us/step - loss: 5.5797e-06 - acc: 1.0000 - val_loss: 0.4197 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00199: val_loss did not improve\n",
      "Epoch 200/250\n",
      "14316/14316 [==============================] - 4s 297us/step - loss: 5.5447e-06 - acc: 1.0000 - val_loss: 0.4198 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00200: val_loss did not improve\n",
      "Epoch 201/250\n",
      "14316/14316 [==============================] - 4s 295us/step - loss: 5.5163e-06 - acc: 1.0000 - val_loss: 0.4200 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00201: val_loss did not improve\n",
      "Epoch 202/250\n",
      "14316/14316 [==============================] - 4s 295us/step - loss: 5.4866e-06 - acc: 1.0000 - val_loss: 0.4201 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00202: val_loss did not improve\n",
      "Epoch 203/250\n",
      "14316/14316 [==============================] - 4s 296us/step - loss: 5.4550e-06 - acc: 1.0000 - val_loss: 0.4202 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00203: val_loss did not improve\n",
      "Epoch 204/250\n",
      "14316/14316 [==============================] - 4s 295us/step - loss: 5.4249e-06 - acc: 1.0000 - val_loss: 0.4204 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00204: val_loss did not improve\n",
      "Epoch 205/250\n",
      "14316/14316 [==============================] - 4s 296us/step - loss: 5.3967e-06 - acc: 1.0000 - val_loss: 0.4205 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00205: val_loss did not improve\n",
      "Epoch 206/250\n",
      "14316/14316 [==============================] - 4s 298us/step - loss: 5.3696e-06 - acc: 1.0000 - val_loss: 0.4206 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00206: val_loss did not improve\n",
      "Epoch 207/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 5.3404e-06 - acc: 1.0000 - val_loss: 0.4208 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00207: val_loss did not improve\n",
      "Epoch 208/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 5.3117e-06 - acc: 1.0000 - val_loss: 0.4209 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00208: val_loss did not improve\n",
      "Epoch 209/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 5.2866e-06 - acc: 1.0000 - val_loss: 0.4211 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00209: val_loss did not improve\n",
      "Epoch 210/250\n",
      "14316/14316 [==============================] - 4s 306us/step - loss: 5.2564e-06 - acc: 1.0000 - val_loss: 0.4212 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00210: val_loss did not improve\n",
      "Epoch 211/250\n",
      "14316/14316 [==============================] - 5s 315us/step - loss: 5.2274e-06 - acc: 1.0000 - val_loss: 0.4213 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00211: val_loss did not improve\n",
      "Epoch 212/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 5.2037e-06 - acc: 1.0000 - val_loss: 0.4215 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00212: val_loss did not improve\n",
      "Epoch 213/250\n",
      "14316/14316 [==============================] - 4s 304us/step - loss: 5.1745e-06 - acc: 1.0000 - val_loss: 0.4216 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00213: val_loss did not improve\n",
      "Epoch 214/250\n",
      "14316/14316 [==============================] - 4s 296us/step - loss: 5.1481e-06 - acc: 1.0000 - val_loss: 0.4217 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00214: val_loss did not improve\n",
      "Epoch 215/250\n",
      "14316/14316 [==============================] - 4s 305us/step - loss: 5.1212e-06 - acc: 1.0000 - val_loss: 0.4219 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00215: val_loss did not improve\n",
      "Epoch 216/250\n",
      "14316/14316 [==============================] - 4s 308us/step - loss: 5.0968e-06 - acc: 1.0000 - val_loss: 0.4220 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00216: val_loss did not improve\n",
      "Epoch 217/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 5.0727e-06 - acc: 1.0000 - val_loss: 0.4222 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00217: val_loss did not improve\n",
      "Epoch 218/250\n",
      "14316/14316 [==============================] - 4s 297us/step - loss: 5.0420e-06 - acc: 1.0000 - val_loss: 0.4223 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00218: val_loss did not improve\n",
      "Epoch 219/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 5.0190e-06 - acc: 1.0000 - val_loss: 0.4224 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00219: val_loss did not improve\n",
      "Epoch 220/250\n",
      "14316/14316 [==============================] - 4s 296us/step - loss: 4.9922e-06 - acc: 1.0000 - val_loss: 0.4225 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00220: val_loss did not improve\n",
      "Epoch 221/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.9666e-06 - acc: 1.0000 - val_loss: 0.4226 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00221: val_loss did not improve\n",
      "Epoch 222/250\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14316/14316 [==============================] - 4s 296us/step - loss: 4.9431e-06 - acc: 1.0000 - val_loss: 0.4227 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00222: val_loss did not improve\n",
      "Epoch 223/250\n",
      "14316/14316 [==============================] - 4s 294us/step - loss: 4.9171e-06 - acc: 1.0000 - val_loss: 0.4229 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00223: val_loss did not improve\n",
      "Epoch 224/250\n",
      "14316/14316 [==============================] - 4s 298us/step - loss: 4.8968e-06 - acc: 1.0000 - val_loss: 0.4230 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00224: val_loss did not improve\n",
      "Epoch 225/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 4.8702e-06 - acc: 1.0000 - val_loss: 0.4231 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00225: val_loss did not improve\n",
      "Epoch 226/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 4.8491e-06 - acc: 1.0000 - val_loss: 0.4233 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00226: val_loss did not improve\n",
      "Epoch 227/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 4.8289e-06 - acc: 1.0000 - val_loss: 0.4234 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00227: val_loss did not improve\n",
      "Epoch 228/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 4.7965e-06 - acc: 1.0000 - val_loss: 0.4235 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00228: val_loss did not improve\n",
      "Epoch 229/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.7794e-06 - acc: 1.0000 - val_loss: 0.4236 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00229: val_loss did not improve\n",
      "Epoch 230/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.7526e-06 - acc: 1.0000 - val_loss: 0.4237 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00230: val_loss did not improve\n",
      "Epoch 231/250\n",
      "14316/14316 [==============================] - 4s 303us/step - loss: 4.7315e-06 - acc: 1.0000 - val_loss: 0.4239 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00231: val_loss did not improve\n",
      "Epoch 232/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.7109e-06 - acc: 1.0000 - val_loss: 0.4240 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00232: val_loss did not improve\n",
      "Epoch 233/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 4.6890e-06 - acc: 1.0000 - val_loss: 0.4241 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00233: val_loss did not improve\n",
      "Epoch 234/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 4.6678e-06 - acc: 1.0000 - val_loss: 0.4242 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00234: val_loss did not improve\n",
      "Epoch 235/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.6413e-06 - acc: 1.0000 - val_loss: 0.4243 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00235: val_loss did not improve\n",
      "Epoch 236/250\n",
      "14316/14316 [==============================] - 4s 295us/step - loss: 4.6231e-06 - acc: 1.0000 - val_loss: 0.4245 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00236: val_loss did not improve\n",
      "Epoch 237/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 4.6040e-06 - acc: 1.0000 - val_loss: 0.4245 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00237: val_loss did not improve\n",
      "Epoch 238/250\n",
      "14316/14316 [==============================] - 4s 302us/step - loss: 4.5802e-06 - acc: 1.0000 - val_loss: 0.4247 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00238: val_loss did not improve\n",
      "Epoch 239/250\n",
      "14316/14316 [==============================] - 4s 298us/step - loss: 4.5623e-06 - acc: 1.0000 - val_loss: 0.4248 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00239: val_loss did not improve\n",
      "Epoch 240/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.5380e-06 - acc: 1.0000 - val_loss: 0.4249 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00240: val_loss did not improve\n",
      "Epoch 241/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 4.5186e-06 - acc: 1.0000 - val_loss: 0.4250 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00241: val_loss did not improve\n",
      "Epoch 242/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.5006e-06 - acc: 1.0000 - val_loss: 0.4251 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00242: val_loss did not improve\n",
      "Epoch 243/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 4.4809e-06 - acc: 1.0000 - val_loss: 0.4252 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00243: val_loss did not improve\n",
      "Epoch 244/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 4.4603e-06 - acc: 1.0000 - val_loss: 0.4253 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00244: val_loss did not improve\n",
      "Epoch 245/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.4397e-06 - acc: 1.0000 - val_loss: 0.4255 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00245: val_loss did not improve\n",
      "Epoch 246/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.4203e-06 - acc: 1.0000 - val_loss: 0.4256 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00246: val_loss did not improve\n",
      "Epoch 247/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 4.3977e-06 - acc: 1.0000 - val_loss: 0.4257 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00247: val_loss did not improve\n",
      "Epoch 248/250\n",
      "14316/14316 [==============================] - 4s 300us/step - loss: 4.3830e-06 - acc: 1.0000 - val_loss: 0.4258 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00248: val_loss did not improve\n",
      "Epoch 249/250\n",
      "14316/14316 [==============================] - 4s 299us/step - loss: 4.3600e-06 - acc: 1.0000 - val_loss: 0.4259 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00249: val_loss did not improve\n",
      "Epoch 250/250\n",
      "14316/14316 [==============================] - 4s 301us/step - loss: 4.3413e-06 - acc: 1.0000 - val_loss: 0.4260 - val_acc: 0.9438\n",
      "\n",
      "Epoch 00250: val_loss did not improve\n"
     ]
    }
   ],
   "source": [
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "# train_generator, validation_generator = helpers.create_datagenerator(x_train, x_test, y_train, y_test)\n",
    "\n",
    "# Convolutional network will be build with Keras.\n",
    "print('Start training the model.')\n",
    "model = Sequential()\n",
    "model.add(Conv2D(filters=64, kernel_size=3, strides=1, \\\n",
    "        padding='same', activation='relu', \\\n",
    "        input_shape=[28, 28, 1]))\n",
    "# 28*28*64\n",
    "model.add(MaxPooling2D(pool_size=2, strides=2, padding='same'))\n",
    "# 14*14*64\n",
    "\n",
    "model.add(Conv2D(filters=128, kernel_size=3, strides=1, \\\n",
    "        padding='same', activation='relu'))\n",
    "# 14*14*128\n",
    "model.add(MaxPooling2D(pool_size=2, strides=2, padding='same'))\n",
    "# 7*7*128\n",
    "\n",
    "model.add(Conv2D(filters=256, kernel_size=3, strides=1, \\\n",
    "        padding='same', activation='relu'))\n",
    "# 7*7*256\n",
    "model.add(MaxPooling2D(pool_size=2, strides=2, padding='same'))\n",
    "# 4*4*256\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(1024, activation='relu'))\n",
    "# model.add(Dropout(0.5))\n",
    "model.add(Dense(36, activation='softmax'))\n",
    "\n",
    "sgd = SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)\n",
    "\n",
    "model.compile(loss=keras.losses.categorical_crossentropy,\n",
    "              optimizer=sgd,\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "'''\n",
    "saves the model weights after each epoch if the validation loss decreased\n",
    "'''\n",
    "checkpointer = ModelCheckpoint(filepath='model/model.h5', verbose=2, save_best_only=True)\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test), callbacks=[checkpointer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['val_loss', 'val_acc', 'acc', 'loss'])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1cd0dc43390>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAACgCAYAAADjNXB5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGntJREFUeJzt3Xt8XWWd7/HPN5c2TRt6S4u9UFoQEASnQEEYGA8cbxSGmzqMMjheZqzXEc4II4wK6GvOeTHnpRxnvICgHUGQi1wso0VLnYI6gECxcikwLQzYtNCWQtokbS5779/5Y62E3TTZ2U2zs5Ps7/v1Srv2uv6erHb99vM8az1LEYGZmRlAVbkDMDOzkcNJwczMejgpmJlZDycFMzPr4aRgZmY9nBTMzKyHk4JVFEk/lPRPRa77oqR3lToms5HEScHMzHo4KZiNQpJqyh2DjU1OCjbipM02l0h6QlKbpB9I2l/SvZJaJK2UNDVv/bMkPS2pWdL9kg7PW3a0pMfT7W4D6nod688lrUm3fVDS24qM8QxJv5e0Q9IGSVf2Wn5yur/mdPlH0/kTJH1D0kuStkv6bTrvFElNffwe3pVOXynpDkk3SdoBfFTS8ZIeSo/xsqRvSxqXt/1bJd0n6TVJmyX9o6Q3SdopaXreesdK2iqptpiy29jmpGAj1fuBdwOHAmcC9wL/CDSS/Lv9PICkQ4FbgIuAGcBy4N8ljUsvkD8FfgRMA36S7pd022OApcAngenA94B7JI0vIr424K+BKcAZwKclnZPud14a77fSmBYCa9Ltvg4cC/xpGtM/ALkifydnA3ekx7wZyAL/K/2dnAi8E/hMGkMDsBL4BTAbeDPwq4h4BbgfOC9vvxcAt0ZEV5Fx2BjmpGAj1bciYnNEbAR+A/wuIn4fER3A3cDR6Xp/Cfw8Iu5LL2pfByaQXHRPAGqBb0ZEV0TcATyad4xPAN+LiN9FRDYibgA60u0Kioj7I+LJiMhFxBMkiel/pIv/ClgZEbekx90WEWskVQEfBy6MiI3pMR9My1SMhyLip+kxd0XE6oh4OCIyEfEiSVLrjuHPgVci4hsR0R4RLRHxu3TZDSSJAEnVwIdIEqeZk4KNWJvzpnf18XlSOj0beKl7QUTkgA3AnHTZxth91MeX8qYPBL6QNr80S2oGDki3K0jS2yWtSptdtgOfIvnGTrqP5/vYrJGk+aqvZcXY0CuGQyX9TNIraZPS/ykiBoBlwBGSDiKpjW2PiEcGGZONMU4KNtptIrm4AyBJJBfEjcDLwJx0Xrd5edMbgP8dEVPyfuoj4pYijvtj4B7ggIiYDFwLdB9nA3BwH9u8CrT3s6wNqM8rRzVJ01O+3kMaXwM8CxwSEfuRNK8NFAMR0Q7cTlKj+TCuJVgeJwUb7W4HzpD0zrSj9AskTUAPAg8BGeDzkmokvQ84Pm/b64FPpd/6JWli2oHcUMRxG4DXIqJd0vHA+XnLbgbeJem89LjTJS1MazFLgaslzZZULenEtA/jv4C69Pi1wJeBgfo2GoAdQKuktwCfzlv2M+BNki6SNF5Sg6S35y2/EfgocBZwUxHltQrhpGCjWkQ8R9I+/i2Sb+JnAmdGRGdEdALvI7n4vU7S/3BX3raPkfQrfDtdvj5dtxifAb4mqQW4nCQ5de/3j8DpJAnqNZJO5j9JF18MPEnSt/Ea8M9AVURsT/f5fZJaThuw291IfbiYJBm1kCS42/JiaCFpGjoTeAVYB5yat/w/STq4H0/7I8wAkF+yY1aZJP0H8OOI+H65Y7GRw0nBrAJJOg64j6RPpKXc8djI4eYjswoj6QaSZxguckKw3lxTMDOzHq4pmJlZDycFMzPrMepGWmxsbIz58+eXOwwzs1Fl9erVr0ZE7wci91CypCBpKcn4K1si4sg+lgv4F5L7uXcCH42Ixwfa7/z583nssceGOlwzszFN0ksDr1Xa5qMfAqcVWL4YOCT9WULyyL6ZmZVRyWoKEfFrSfMLrHI2cGM6WNnDkqZImhURL5cqpqESEXQPp9PelWX9llY6Mjm6sjlyuSAXkI0gF0H+3V1Cb4xMs9sO8yd9N5iZ9e3Q/RuYO7V+4BX3QTn7FOaw+6iPTem8PZKCpCUktQnmzZvXe3FJRASbd3RQXSWad3ayZkMzT2/awfotrTz64mvUVImpE8exrbWTXV3ZYYnJzCrbP51zJBeccODAK+6DciaFAb4z582MuA64DmDRokV7rNPV1UVTUxPt7e37FFAugtb2DO2ZHNlckM29cagpwJ/NgFP3H8dnj54NiFwEVYLxNdWouxKQ/tFX4fZZdS0xcRqqGnX3B5jZEJgzdULJj1HOq0sTyRDH3eaSDIO89ztqaqKhoYH58+f3NOvsrR27utjUvIsJ2RyN42sYV13FhHHVANRUifG11dTVVA16//sqIti2bRstLS0sOHBBWWIws7GvnEnhHuBzkm4F3k7yoo9B9Se0t7cPOiHkItjUvIvX2jqpq63m4GmTmDh+5H0Tl8T06dPZunVruUMxszGslLek3gKcAjSmLyS/guTViETEtSTv0j2dZLjincDH9vF4g9pua0sHr7V1MqNhPPvvV0dVmWoCxShXLcXMKkfJbkmNiA9FxKyIqI2IuRHxg4i4Nk0IROKzEXFwRByVjm0/rDLZHK+2dDB5Qi2zJk8YdEJobm7mu9/97l5vd/rpp9Pc3DyoY5qZlUJFD3OxtbWDXAT771e3T/vpLylks4XvSlq+fDlTpkzZp2ObmQ2lkdd4PkxyEbze1kVDXS11tdX7tK9LL72U559/noULF1JbW8ukSZOYNWsWa9asYe3atZxzzjls2LCB9vZ2LrzwQpYsWQK88XR2a2srixcv5uSTT+bBBx9kzpw5LFu2jAkTSn+ngZlZvjGXFL7670+zdtOOAdfL5oL2rix1tdVUVxVuNjpi9n5cceZb+11+1VVX8dRTT7FmzRruv/9+zjjjDJ566ikWLEjuElq6dCnTpk1j165dHHfccbz//e9n+vTpu+1j3bp13HLLLVx//fWcd9553HnnnVxwwQVFlNjMbOiMuaRQrK5sDkkDJoSCIgfZLujclUx37oT2HRy/6BgWzDsAOloB+Nerv87dy5ZBBBs2NLHu6TVMX7Qw2aZ1C7S2seDAeSw8aCZs38ixb30zLz77B2h+x57H3PUa/Ozvk2mJnsc9+pru7iNRFdRNhpq9bCYrqo+liHWK7qsZqn0N436K3tdQ7ce/7xEX03D+vuccC9MPLvJ4gzPmkkKhb/TdchGs3bSDqfW1zNmbR8ZzOch1QVUtNP8R2puBgNc3QbYTXn0OWjczsRbY/CQA9z/4GCtX3MtDP72e+gn1nPKBv6V9ywvQPA1yGWh9Bdp2Mb62KkkQEtXZDna174T27XvG0LkL1i5LjtszhEb3dPR8fGM6ILLQtbP4cprZyHTG1U4KpbCrM0sugkl1tX2vELnkIp/NQO2E5Jt2y8vQuhXIJUkh1wX1jTCunobcZFp2dsLUBTBlM9TWQ8MsqK1nezzL1Bn7U3/gsTy77gUefvwp2G8OzDgcqsfBzLdCW1vyLX72wuT4+82CqlZ401F7xvZ6DfzD83tf6ExnUqaiFTEGU1Fv7StyLKeh2tdQvUmw6P0M4+9p1P6+h/K8jbSYhvL3XYT66QOvs48qMim0dGQQMDF9YplcFqqqk5rA9j/Crtfz1q5Kl3VB3ZQkSbS9CpMPgImNAEyvn85JJ5/MkcedzIQJE9h///2h4U0AnHbW+7h26Y287ZjjOOywwzjhhBNgXD3Upk05VdVJ0im1mnHJj5lZAaPuHc2LFi2K3u9TeOaZZzj88MOL3sfzW1oJ4M0zJ0HbVti+EaYcADu3QWdbTw0A1UBnS1JjGD8pydJlfoBsb8tqZgYgaXVELBpovYqrKeQi2NmZ5YAJ7bBje9okRNJHgGDKgVA/7Y0NJkwuS5xmZuVQcUmhM5NjHJ1M7ngZOgKqamD6m5NO3vppML6h3CGamZVNRSaF2dqWtOPPOCxp06+qgamlHaPczGw0qLhhLjJdHTRoF7mJM6FmfJIQzMwMqMCkoM7kgbIq9xWYme2h4pJCTaaNLFVob5/uNTOrABWXFMbndtJRVT+kt5YOduhsgG9+85vs3Omnjc1sZKiopJDLdDCODJmaiUO6XycFMxsrKqqXNduxM8mC4/ZivKMi5A+d/e53v5uZM2dy++2309HRwbnnnstXv/pV2traOO+882hqaiKbzfKVr3yFzZs3s2nTJk499VQaGxtZtWrVkMZlZra3xl5SuPdSeOXJPhdVZzoh18HEmolQtReVpDcdBYuv6ndx/tDZK1as4I477uCRRx4hIjjrrLP49a9/zdatW5k9ezY///nPAdi+fTuTJ0/m6quvZtWqVTQ2Nu5VMc3MSqGimo8gRxQ7PO8grVixghUrVnD00UdzzDHH8Oyzz7Ju3TqOOuooVq5cyRe/+EV+85vfMHmy734ys5GnqJqCpDuBpcC9EZErbUj7qMA3+tzW9XR1dpBrPIyJ40tTSYoILrvsMj75yU/usWz16tUsX76cyy67jPe85z1cfvnlJYnBzGywiq0pXAOcD6yTdJWkt5QwppJRtoNOaoZ8TLuGhgZaWloAeO9738vSpUtpbU2eh9i4cSNbtmxh06ZN1NfXc8EFF3DxxRfz+OOP77GtmVm5FfV1OSJWAislTQY+BNwnaQNwPXBTRHSVMMahEUFVrosOJjDUA0hPnz6dk046iSOPPJLFixdz/vnnc+KJJwIwadIkbrrpJtavX88ll1xCVVUVtbW1XHPNNQAsWbKExYsXM2vWLHc0m1nZFT10tqTpwAXAh4FNwM3AycBREXFKqQLsbdBDZ2c6YcvTNEUjjTNnU1dbXcIoS8dDZ5vZYAzp0NmS7gLeAvwIODMiXk4X3Sbpsf63HEGyHQB0xti74crMbKgUe4X8dkT8R18Lisk8I0ImeRVlJ7Xlfk+OmdmIVWxH8+GSpnR/kDRV0mdKFFOJBFnV0EUNKvFtqWZmo1WxSeETEdHc/SEiXgc+MdBGkk6T9Jyk9ZIu7WP5PEmrJP1e0hOSTi8+9N0N2DcysZHmhkMJyv5GzUEbba9ONbPRp9ikUCW9cSmVVA2Fb+JJ1/kOsBg4AviQpCN6rfZl4PaIOBr4IDCoAYTq6urYtm3bgBfN7qWjMSdEBNu2baOuzqO7mlnpFNun8EvgdknXklxbPwX8YoBtjgfWR8QLAJJuBc4G1uatE8B+6fRkkrua9trcuXNpampi69atBddrac+wfVcX1TvqqBqF1YW6ujrmzp1b7jDMbAwrNil8Efgk8GmSL9orgO8PsM0cYEPe5ybg7b3WuRJYIenvgInAu4qMZze1tbUsWLBgwPWufeB5rrr3WdZ+7b3Uj/NdSGZmvRX78FqO5Knma/Zi3319Fe/dvvMh4IcR8Q1JJwI/knRk76E0JC0BlgDMmzdvL0LYXSab7LZmbwbDMzOrIEVdHSUdIukOSWslvdD9M8BmTcABeZ/nsmfz0N8AtwNExENAHbDHcKERcV1ELIqIRTNmzCgm5D5lcklOqqkafU1HZmbDodivzP9GUkvIAKcCN5I8yFbIo8AhkhZIGkfSkXxPr3X+CLwTQNLhJEmhcMfAPshkgypBlZOCmVmfik0KEyLiVyTDYrwUEVcC/7PQBhGRAT5H0kn9DMldRk9L+pqks9LVvgB8QtIfgFuAj0YJ77vsyuWoqXbTkZlZf4rtbW2XVEUySurngI3AzIE2iojlwPJe8y7Pm14LnFR8uPsmmw03HZmZFVDs1+aLgHrg88CxJAPjfaRUQZVKJuekYGZWyIA1hfQhtPMi4hKgFfhYyaMqka5sjlo3H5mZ9WvAK2REZIFj859oHq2yuaDaNQUzs34V26fwe2CZpJ8Abd0zI+KukkRVIl3ZcE3BzKyAYpPCNGAbu99xFMCoSgqZXI6aatcUzMz6U+wTzaO2HyFfxs1HZmYFFfvmtX9jzyEqiIiPD3lEJZTJ5qj1EBdmZv0qtvnoZ3nTdcC5DHJE03LKZMPNR2ZmBRTbfHRn/mdJtwArSxJRCfk5BTOzwgbblnIIMPjhSssk42EuzMwKKrZPoYXd+xReIXnHwqjS5WEuzMwKKrb5qKHUgQyHbC6oq3VNwcysP8W+T+FcSZPzPk+RdE7pwiqNTDbnF+yYmRVQ7BXyiojY3v0hIpqBK0oTUukkTzS7+cjMrD/FJoW+1ht1Lzn22EdmZoUVmxQek3S1pIMlHSTp/wGrSxlYKfglO2ZmhRV7hfw7oBO4jeSdyruAz5YqqFLJZINa1xTMzPpV7N1HbcClJY6l5JLmI9cUzMz6U+zdR/dJmpL3eaqkX5YurNJIXrLjmoKZWX+K/drcmN5xBEBEvE4R72geaTI5j31kZlZIsUkhJ6lnWAtJ8+lj1NSRzs8pmJkVVuxtpV8CfivpgfTzO4AlpQmpdDwgnplZYcV2NP9C0iKSRLAGWEZyB9Kokgyd7ZqCmVl/ih0Q72+BC4G5JEnhBOAhdn8954iXyeVcUzAzK6DYr80XAscBL0XEqcDRwNaSRVUCuVyQC9zRbGZWQLFJoT0i2gEkjY+IZ4HDShfW0Mvkkn7xWjcfmZn1q9grZFP6nMJPgfskLaOI13FKOk3Sc5LWS+rz4TdJ50laK+lpST8uPvS9k8nlADz2kZlZAcV2NJ+bTl4paRUwGfhFoW0kVQPfAd4NNAGPSronItbmrXMIcBlwUkS8Lqlkzz50ZZOagvsUzMz6t9cjnUbEAwOvBcDxwPqIeAFA0q3A2cDavHU+AXwnfRiOiNiyt/EUK+vmIzOzAZXyCjkH2JD3uSmdl+9Q4FBJ/ynpYUmnlSqYTNbNR2ZmAynlOxH6uvr2fgq6BjgEOIXkdtffSDoyf0gNAElLSB+WmzdvHoPR1VNTcFIwM+tPKWsKTcABeZ/nsmfndBOwLCK6IuK/gedIksRuIuK6iFgUEYtmzJgxqGCyPX0Kbj4yM+tPKa+QjwKHSFogaRzwQeCeXuv8FDgVQFIjSXPSC6UIpiu9+8jPKZiZ9a9kSSEiMsDngF8CzwC3R8TTkr4m6ax0tV8C2yStBVYBl0TEtlLEk3FNwcxsQCV9z3JELAeW95p3ed50AH+f/pRUxjUFM7MBVczX5oyfUzAzG1DlJIWemkLFFNnMbK9VzBWyu6ZQ65qCmVm/KicppM8p+OE1M7P+VUxS6Mq6+cjMbCAVc4XM+olmM7MBVUxS6B4l1c1HZmb9q5ik0H33kUdJNTPrX8VcIbubj/ycgplZ/yomKXR5mAszswFVzBUyk/UwF2ZmA6mcpNDdfOSkYGbWr8pJCt01BTcfmZn1q2KukK4pmJkNrOKSQq1rCmZm/Srp+xRGko+ftIDz3z6PulonBTOz/lRMUhhXU8W4GicEM7NCfJU0M7MeTgpmZtZDyWuSRw9JW4GXBrl5I/DqEIYzGlRimaEyy+0yV4bBlvnAiJgx0EqjLinsC0mPRcSicscxnCqxzFCZ5XaZK0Opy+zmIzMz6+GkYGZmPSotKVxX7gDKoBLLDJVZbpe5MpS0zBXVp2BmZoVVWk3BzMwKqJikIOk0Sc9JWi/p0nLHUyqSXpT0pKQ1kh5L502TdJ+kdenfU8sd576QtFTSFklP5c3rs4xK/Gt63p+QdEz5Ih+8fsp8paSN6bleI+n0vGWXpWV+TtJ7yxP1vpF0gKRVkp6R9LSkC9P5Y/ZcFyjz8J3riBjzP0A18DxwEDAO+ANwRLnjKlFZXwQae837v8Cl6fSlwD+XO859LOM7gGOApwYqI3A6cC8g4ATgd+WOfwjLfCVwcR/rHpH+Gx8PLEj/7VeXuwyDKPMs4Jh0ugH4r7RsY/ZcFyjzsJ3rSqkpHA+sj4gXIqITuBU4u8wxDaezgRvS6RuAc8oYyz6LiF8Dr/Wa3V8ZzwZujMTDwBRJs4Yn0qHTT5n7czZwa0R0RMR/A+tJ/g+MKhHxckQ8nk63AM8AcxjD57pAmfsz5Oe6UpLCHGBD3ucmCv+iR7MAVkhaLWlJOm//iHgZkn90wMyyRVc6/ZVxrJ/7z6VNJUvzmgXHXJklzQeOBn5HhZzrXmWGYTrXlZIU+nqzzli97eqkiDgGWAx8VtI7yh1QmY3lc38NcDCwEHgZ+EY6f0yVWdIk4E7goojYUWjVPuaNynL3UeZhO9eVkhSagAPyPs8FNpUplpKKiE3p31uAu0mqkpu7q9Hp31vKF2HJ9FfGMXvuI2JzRGQjIgdczxvNBmOmzJJqSS6ON0fEXensMX2u+yrzcJ7rSkkKjwKHSFogaRzwQeCeMsc05CRNlNTQPQ28B3iKpKwfSVf7CLCsPBGWVH9lvAf46/TOlBOA7d1ND6Ndr/byc0nONSRl/qCk8ZIWAIcAjwx3fPtKkoAfAM9ExNV5i8bsue6vzMN6rsvd2z6Mvfqnk/TkPw98qdzxlKiMB5HcifAH4OnucgLTgV8B69K/p5U71n0s5y0kVegukm9Kf9NfGUmq199Jz/uTwKJyxz+EZf5RWqYn0ovDrLz1v5SW+TlgcbnjH2SZTyZpCnkCWJP+nD6Wz3WBMg/bufYTzWZm1qNSmo/MzKwITgpmZtbDScHMzHo4KZiZWQ8nBTMz6+GkYDaMJJ0i6WfljsOsP04KZmbWw0nBrA+SLpD0SDp2/fckVUtqlfQNSY9L+pWkGem6CyU9nA5Wdnfe+P5vlrRS0h/SbQ5Odz9J0h2SnpV0c/oUq9mI4KRg1oukw4G/JBlccCGQBf4KmAg8HsmAgw8AV6Sb3Ah8MSLeRvLUaff8m4HvRMSfAH9K8kQyJCNfXkQyFv5BwEklL5RZkWrKHYDZCPRO4Fjg0fRL/ASSQddywG3pOjcBd0maDEyJiAfS+TcAP0nHoJoTEXcDREQ7QLq/RyKiKf28BpgP/Lb0xTIbmJOC2Z4E3BARl+02U/pKr/UKjRFTqEmoI286i/8f2gji5iOzPf0K+ICkmdDzTuADSf6/fCBd53zgtxGxHXhd0p+l8z8MPBDJGPhNks5J9zFeUv2wlsJsEPwNxayXiFgr6cskb7CrIhmZ9LNAG/BWSauB7ST9DpAM33xtetF/AfhYOv/DwPckfS3dx18MYzHMBsWjpJoVSVJrREwqdxxmpeTmIzMz6+GagpmZ9XBNwczMejgpmJlZDycFMzPr4aRgZmY9nBTMzKyHk4KZmfX4/9vE5KPkBZDXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(history.history.keys())  \n",
    "\n",
    "plt.figure(1)  \n",
    "\n",
    "# summarize history for accuracy  \n",
    "\n",
    "plt.subplot(211)  \n",
    "plt.plot(history.history['acc'])  \n",
    "plt.plot(history.history['val_acc'])  \n",
    "plt.title('model accuracy')  \n",
    "plt.ylabel('accuracy')  \n",
    "plt.xlabel('epoch')  \n",
    "plt.legend(['train', 'test'], loc='upper left')  \n",
    "\n",
    "# summarize history for loss  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAACgCAYAAAAPbNcqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAF59JREFUeJzt3XucVPV9//HXZ2Znb+yyC7uLcpFLjDF4CwIaicYmTUXBJmpNMInY/GwexfTXPmp+j+gv0tS06aOX9JI0Pxtj1J+0phoTo/EXW/UnIQEvVUSgqFw0oIWwgNx3ZYGFvXz6xzmzDMvO7OxldmbOvJ+Pxz7mzLnM+Xz3wHvOfs+Z75i7IyIi0RfLdwEiIjIyFPgiIiVCgS8iUiIU+CIiJUKBLyJSIhT4IiIlQoEvApjZv5jZX2a57lYz+62hvo7ISFPgi4iUCAW+iEiJUOBL0Qi7Um43s9fN7LCZPWBmp5nZM2Z2yMyWmdmYlPU/ZWYbzKzFzFaY2fSUZRea2dpwux8Dlb329dtmti7c9iUzu2CQNf++mW0xswNm9qSZTQjnm5n9o5ntMbPWsE3nhcvmm9nGsLYdZnbboH5hIr0o8KXYXA9cAXwA+CTwDPAnQCPBv+c/BjCzDwCPAF8GmoCngX8zs3IzKwf+H/CvwFjgJ+HrEm47E1gC3AI0APcCT5pZxUAKNbPfBP4GWACMB7YBPwoXzwUuD9tRD9wA7A+XPQDc4u61wHnALweyX5F0FPhSbP7J3Xe7+w7gBeAVd/9Pdz8GPAFcGK53A/CUu//c3TuAfwCqgI8AlwAJ4Dvu3uHujwGvpuzj94F73f0Vd+9y9weBY+F2A3EjsMTd14b1LQbmmNlUoAOoBT4ImLtvcvdd4XYdwDlmNtrdD7r72gHuV6RPCnwpNrtTpo/28bwmnJ5AcEYNgLt3A9uBieGyHX7yyIHbUqanAF8Ju3NazKwFOCPcbiB619BGcBY/0d1/CXwXuBvYbWb3mdnocNXrgfnANjN7zszmDHC/In1S4EtU7SQIbiDoMycI7R3ALmBiOC9pcsr0duCv3L0+5afa3R8ZYg2jCLqIdgC4+13uPgs4l6Br5/Zw/qvufg0wjqDr6dEB7lekTwp8iapHgavN7BNmlgC+QtAt8xLwMtAJ/LGZlZnZ7wAXp2x7P/AlM/tweHF1lJldbWa1A6zhh8DNZjYj7P//a4IuqK1mdlH4+gngMNAOdIXXGG40s7qwK+o9oGsIvweRHgp8iSR3fwtYCPwTsI/gAu8n3f24ux8Hfgf4H8BBgv7+n6Zsu5qgH/+74fIt4boDreEXwJ3A4wR/VZwJfDZcPJrgjeUgQbfPfoLrDAA3AVvN7D3gS2E7RIbM9AUoIiKlQWf4IiIlQoEvIlIiFPgiIiVCgS8iUiIU+CIiJaIs3wWkamxs9KlTp+a7DBGRorFmzZp97t6UzboFFfhTp05l9erV+S5DRKRomNm2/tcKqEtHRKRERCLwl7+1h827D+W7DBGRghaJwP+fD63l0dXb812GiEhBK6g+/L50dHTQ3NxMe3t72nW+O38c1eWdbNq0aQQrG16VlZVMmjSJRCKR71JEJKIKPvCbm5upra1l6tSpnDya7Qm+s5W6qnImjqka4eqGh7uzf/9+mpubmTZtWr7LEZGIKvgunfb2dhoaGtKGPQBmOMU7CJyZ0dDQkPGvGBGRoSr4wAcyhz1gQLEP+tlfG0VEhqooAr8/uczKlpYWvve97w14u/nz59PS0pKDikREBicagY/l7Aw/XeB3dWX+EqKnn36a+vr63BQlIjIIBX/RNhsGOevDv+OOO3j77beZMWMGiUSCmpoaxo8fz7p169i4cSPXXnst27dvp729nVtvvZVFixYBJz413NbWxrx587jssst46aWXmDhxIj/72c+oqirOC8wiUryKKvC/8W8b2LjzvVPmH+3owoDKRHzAr3nOhNH82SfPTbv8m9/8JuvXr2fdunWsWLGCq6++mvXr1/fcTbNkyRLGjh3L0aNHueiii7j++utpaGg46TU2b97MI488wv3338+CBQt4/PHHWbhQ31onIiOrqAK/EFx88cUn3Tp511138cQTTwCwfft2Nm/efErgT5s2jRkzZgAwa9Ystm7dOmL1iogkFVXgpzsT37KnjZjB+5pqcl7DqFGjeqZXrFjBsmXLePnll6muruZjH/tYn7dWVlRU9EzH43GOHj2a8zpFRHqLyEXb3KmtreXQob7H6WltbWXMmDFUV1fz5ptvsnLlyhxWIiIyNEV1hp+OWe7uw29oaODSSy/lvPPOo6qqitNOO61n2VVXXcX3v/99LrjgAs4++2wuueSS3BQhIjIMzAvoE0uzZ8/23uPhb9q0ienTp2fc7p29bXQ7vH9c7rt0cimbtoqIpDKzNe4+O5t1I9GlEzOjkN64REQKUSQCHyjikXREREZGJAI/l334IiJREY3Az+l9OiIi0RCNwDfUhy8i0o9oBD7qwxcR6U8kAp8c9uEPdnhkgO985zscOXJkmCsSERmcSAS+5fAbrxT4IhIV0fikLeSsTyd1eOQrrriCcePG8eijj3Ls2DGuu+46vvGNb3D48GEWLFhAc3MzXV1d3HnnnezevZudO3fy8Y9/nMbGRpYvX56bAkVEslRcgf/MHfDuG6fMbujqoq7LoXwQzTn9fJj3zbSLU4dHXrp0KY899hirVq3C3fnUpz7F888/z969e5kwYQJPPfUUEIyxU1dXx7e//W2WL19OY2PjwOsSERlm0ejSYWTuw1+6dClLly7lwgsvZObMmbz55pts3ryZ888/n2XLlvHVr36VF154gbq6utwXIyIyQMV1hp/mTPxAazt7DrVz/sS6nH4ZuLuzePFibrnlllOWrVmzhqeffprFixczd+5cvv71r+esDhGRwYjGGX4OP3eVOjzylVdeyZIlS2hrawNgx44d7Nmzh507d1JdXc3ChQu57bbbWLt27SnbiojkW3Gd4aeRDHz34Q//1OGR582bx+c//3nmzJkDQE1NDQ899BBbtmzh9ttvJxaLkUgkuOeeewBYtGgR8+bNY/z48bpoKyJ5F4nhkfceamdXazvnTqgjHiveYRY0PLKIDFTJDY+cHEunkN68REQKTTQCP9mlk98yREQKWiQCv4cSX0QkraII/P66apK3YuZqeIWRoO4oEcm1gg/8yspK9u/fnzEQU+/SKUbuzv79+6msrMx3KSISYQV/W+akSZNobm5m7969adc5cryLA4ePQ0sFiXjBv4f1qbKykkmTJuW7DBGJsIIP/EQiwbRp0zKu88wbu/iDJ9fyzK0fZfr40SNUmYhIcSnO0+Fekvfed3UXaZ+OiMgIyFngm9kSM9tjZutztY+kZDdOR1d3rnclIlK0sgp8M7vVzEZb4AEzW2tmc/vZ7F+Aq4ZcYRbK4sEZfqfO8EVE0sr2DP/33P09YC7QBNwMpB9EHnD354EDQysvO8kunc4uBb6ISDrZBn5ygJr5wD+7+2sp8/Iu2aXT2a0uHRGRdLIN/DVmtpQg8J81s1pgWNLVzBaZ2WozW53p1stMymLq0hER6U+2gf9F4A7gInc/AiQIunWGzN3vc/fZ7j67qalpUK9RFgvP8NWlIyKSVraBPwd4y91bzGwh8KdAa+7KGpiei7a6S0dEJK1sA/8e4IiZfQj438A24AeZNjCzR4CXgbPNrNnMvjikSjNQl46ISP+y/aRtp7u7mV0D/B93f8DMvpBpA3f/3NDLy06ZLtqKiPQr28A/ZGaLgZuAj5pZnKAfvyAkz/A71IcvIpJWtl06NwDHCO7HfxeYCPx9zqoaoGQfvoZWEBFJL6vAD0P+YaDOzH4baHf3jH34I+nEXTrq0hERSSfboRUWAKuAzwALgFfM7NO5LGwgEhpaQUSkX9n24X+N4B78PQBm1gQsAx7LVWEDoaEVRET6l20ffiwZ9qH9A9g253pGy9RdOiIiaWV7hv//zexZ4JHw+Q3A07kpaeB6xsPXGb6ISFpZBb67325m1wOXEgyadp+7P5HTygag57ZM9eGLiKSV9VccuvvjwOM5rGXQzIyymOkuHRGRDDIGvpkdAvo6bTbA3b1gvkA2HjPdhy8ikkHGwHf32pEqZKgS8Zg+aSsikkHB3GkzVGVx01g6IiIZFH/gd3fDznVMsT364JWISAbFH/gAD8zlM75UF21FRDIo/sCPxaB+MhPRGb6ISCbFH/gA9ZOZwB4NrSAikkE0An/MFMb7Hl20FRHJIBqBXz+Z0X6Iso62fFciIlKwIhL4U4KH4+/muRARkcIVscDfledCREQKV0QCfzIAjR0KfBGRdKIR+KMaabcKGjp257sSEZGClfVomQXNjAOJ06k/vjPflYiUHvfwpxsIH707mNfzvPc0WayTMu3dKftK7iebaQa4vmdf06Da02tfyTYlqmDmTTk8SIFoBD7QOupMph14je6ubmLxaPzhUvLcobsLujvBu4Jp7wqG0/DulHm9p7tPrNsz3d3Husn53b3WTS737PZx0vLetQ1kH73r7DWd+gN9z08GyinTvcM4dZ2+5qeun267lPVkaEaNU+APROuEy5h+8Jfs/fUGmqadn+9yCkN3F3S2Q+ex8DE5Hf50HYfuDujqDB67O6Gr12PPdEf42JUy3Xt5+Dx1eV/rdXee2Gd/r1m0DGJxsHj4GAunYynT4XKLhfOT0/Fe66YssxiYnZiOpcwnZf4p6/a1LHWb3svTvFaf+0jdxk59Tayfaetnnd77tvBX3Nf8TNN91NfvNFm2IYv29FXPSevlXmQCP3bmx2HDX3B447PFE/jucPwwtLfA0YNwtCWYbn8POo4Ey056PBI89gR4ymPXsVPnd3fmqHCDeAJiCYiVQbwsmI6Hz2NlJ6Z75iWgvDrNNokguJLT8bIT28TDZbGyE8GXGnInhWbvoOwrbONptov1Wje53NLsI1x2yj6S05aj373I4EUm8E+bcjbvdJ9O1bYVwG0nFrz7BuzfAudel37jzuNQVh4EZccRqBozuCK6OuDgVjjwzokAP3rw5EDv/Tybs9iyqiAsE6OCx7IKKKsMHitHn/y8rPLk6Xh5r2XJx/JgWcagThfo8cH9fkQkryIT+BPqq/ihX8CN+56D1mZ4+5cw5VL40Y3Qsg22vwqnnQtt78LkjwTzfr0Sxk6D5X8Dn7gTtr4I76yAK/8KZt186lla217Y+Z+wez3s3gBtu4NukcP7gp9jrX0XV1EHVXXBG0llPYyeAFX1wXTVmGA6uaxqDFTUQnlNcCEnUR2cmYqIDFFkAj8Rj/FU9bV87vgKuPc34Mg+iFcEXR1TLoWVd5+6kcWDC2XlNfDzrwddIPWT4d//F+zbErwZbH0RWrdD257gMaluMtRNCs6Yx38IRjXBqMZgXsNZUD02DO/RwRmyiEieRSqJyhrP5NGWz7Hw8INwwWdhy89hwuVw40+CrhaAyjpY98Oga+JDN8D2VdD4AfjeHBgzFb70H/Ds4hNvEKMnQdPZwbKLF8HEmXDaecFZuYhIETH3wrmlavbs2b569epBb7/4p6+zbMMuXv29Jhg/I+iPjyeCs/D+7FwH1Q1Qf0ZwMfXXK6H29CDodQFORAqUma1x99nZrBupzuGzxtWy93AnO0ZND/q9K2qyC3uACTOCsIcg4KfMCbp0FPYiEhGRCvyPntUIwPO/2pvnSkRECk+kAv/942oYX1epwBcR6UOkAt/MuPysJl7csk9faC4i0kukAh/g8g80cai9k1f+60C+SxERKSiRC/xPTB9HfXWCh1Zuy3cpIiIFJXKBX5mIc8PsM1i6cTe7Wo/muxwRkYIRucAHWHjJFNydu5dvyXcpIiIFI5KBf8bYar7wkak8/MqvWbPtYL7LEREpCJEMfICvzD2b8aMrufmfV/HMG7sopE8Ui4jkQ04D38yuMrO3zGyLmd2Ry331VlNRxo9vmcOUhlH8wcNrWXDvy2zZc2gkSxARKSg5G0vHzOLAr4ArgGbgVeBz7r4x3TZDHUunLx1d3fz41e18a+lbtB3rZFrjKE6vq2JUeZxEPMbZp9fywdNr+eD40Uyoq8Q0lIKIFJGBjKWTy9EyLwa2uPs7YVE/Aq4B0gZ+LiTiMRZeMoUrzz2d//vCO2zdf5hdre3sauniaEcXT7524ovPR1eW0VRbQevRTo4c72TWlDGc2VRDRVmM8rIY5fHYiW9X6+ONoWcZ1vPcUpalzhcRSaoqj3Pjh6fkfD+5DPyJQMoA8jQDH+69kpktAhYBTJ48OWfFNNVWsHj+9FPmH2rv4Fe7D7Fp1yE27XqPg0eOU1dVTlnMeOntfby2vYVjnd0c7+pGlwFEJBcaayqKPvD7Oo89JTLd/T7gPgi6dHJYT59qKxPMmjKWWVPGZlzP3ens9nA6ZT6nzks+T13mPfP1riEi+ZHLwG8Gzkh5PgnYmWbdgmdmJOLqixGR4pXLu3ReBc4ys2lmVg58Fngyh/sTEZEMcnaG7+6dZvZHwLNAHFji7htytT8REcmsoL7i0Mz2AoMd9awR2DeM5RQDtbk0qM2lYbBtnuLuTdmsWFCBPxRmtjrbe1GjQm0uDWpzaRiJNkd2aAURETmZAl9EpEREKfDvy3cBeaA2lwa1uTTkvM2R6cMXEZHMonSGLyIiGRR94OdzCOaRZGZbzewNM1tnZqvDeWPN7Odmtjl8HJPvOofKzJaY2R4zW58yr892WuCu8Ni/bmYz81f54KVp85+b2Y7weK8zs/kpyxaHbX7LzK7MT9VDY2ZnmNlyM9tkZhvM7NZwfmSPdYY2j9yxdvei/SH4QNfbwPuAcuA14Jx815Wjtm4FGnvN+zvgjnD6DuBv813nMLTzcmAmsL6/dgLzgWcIxm26BHgl3/UPY5v/HLitj3XPCf+dVwDTwn//8Xy3YRBtHg/MDKdrCYZSPyfKxzpDm0fsWBf7GX7PEMzufhxIDsFcKq4BHgynHwSuzWMtw8LdnwcO9Jqdrp3XAD/wwEqg3szGj0ylwydNm9O5BviRux9z9/8CthD8Pygq7r7L3deG04eATQQj7Eb2WGdoczrDfqyLPfD7GoI50y+wmDmw1MzWhENKA5zm7rsg+McEjMtbdbmVrp1RP/5/FHZfLEnprotcm81sKnAh8Aolcqx7tRlG6FgXe+BnNQRzRFzq7jOBecAfmtnl+S6oAET5+N8DnAnMAHYB3wrnR6rNZlYDPA582d3fy7RqH/OKst19tHnEjnWxB36khmDOxN13ho97gCcI/rTbnfyzNnzck78KcypdOyN7/N19t7t3uXs3cD8n/pSPTJvNLEEQfA+7+0/D2ZE+1n21eSSPdbEHfkkMwWxmo8ysNjkNzAXWE7T1C+FqXwB+lp8Kcy5dO58Efje8g+MSoDXZHVDsevVPX0dwvCFo82fNrMLMpgFnAatGur6hsuA7Qh8ANrn7t1MWRfZYp2vziB7rfF+5HoYr3/MJrna/DXwt3/XkqI3vI7ha/xqwIdlOoAH4BbA5fByb71qHoa2PEPxZ20FwhvPFdO0k+JP37vDYvwHMznf9w9jmfw3b9Hr4H398yvpfC9v8FjAv3/UPss2XEXRPvA6sC3/mR/lYZ2jziB1rfdJWRKREFHuXjoiIZEmBLyJSIhT4IiIlQoEvIlIiFPgiIiVCgS8yDMzsY2b27/muQyQTBb6ISIlQ4EtJMbOFZrYqHHf8XjOLm1mbmX3LzNaa2S/MrClcd4aZrQwHtXoiZWz295vZMjN7LdzmzPDla8zsMTN708weDj9ZKVIwFPhSMsxsOnADwUB0M4Au4EZgFLDWg8HpngP+LNzkB8BX3f0Cgk9CJuc/DNzt7h8CPkLwKVkIRj/8MsE45u8DLs15o0QGoCzfBYiMoE8As4BXw5PvKoLBubqBH4frPAT81MzqgHp3fy6c/yDwk3BMo4nu/gSAu7cDhK+3yt2bw+frgKnAi7lvlkh2FPhSSgx40N0XnzTT7M5e62UabyRTN82xlOku9P9LCoy6dKSU/AL4tJmNg57vT51C8P/g0+E6nwdedPdW4KCZfTScfxPwnAfjlzeb2bXha1SYWfWItkJkkHQGIiXD3Tea2Z8SfHNYjGB0yj8EDgPnmtkaoJWgnx+C4Xm/Hwb6O8DN4fybgHvN7C/C1/jMCDZDZNA0WqaUPDNrc/eafNchkmvq0hERKRE6wxcRKRE6wxcRKREKfBGREqHAFxEpEQp8EZESocAXESkRCnwRkRLx3ywGjmeqiho9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model has been trained.\n",
      "Test loss: 0.4259992352588655\n",
      "Test accuracy:  94.38390613568166\n"
     ]
    }
   ],
   "source": [
    "# print(\"SGD\")\n",
    " \n",
    "plt.subplot(212)  \n",
    "plt.plot(history.history['loss'])  \n",
    "plt.plot(history.history['val_loss'])  \n",
    "plt.title('model loss')  \n",
    "plt.ylabel('loss')  \n",
    "plt.xlabel('epoch')  \n",
    "plt.legend(['train', 'test'], loc='upper left')  \n",
    "plt.show()  \n",
    "\n",
    "# Calculate loss and accuracy.\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Model has been trained.')\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy: ', score[1]*100.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def plot_filters(layer, x, y):\n",
    "#     filters = layer.W.get_value()\n",
    "#     fig = plt.figure(figsize=(8, 4))\n",
    "#     for j in range(len(filters)):\n",
    "#         ax = fig.add_subplot(y, x, j+1)\n",
    "#         ax.matshow(filters[j][0], cmap = matplotlib.cm.Blues)\n",
    "#         plt.xticks(np.array([]))\n",
    "#         plt.yticks(np.array([]))\n",
    "#     plt.tight_layout()\n",
    "#     return plt\n",
    "\n",
    "# plot_filters(model.layers[0], 8, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer 0 \t conv2d_1 \t\t (None, 28, 28, 1) \t\t\t (None, 28, 28, 64)\n",
      "Layer 1 \t max_pooling2d_1 \t\t (None, 28, 28, 64) \t\t\t (None, 14, 14, 64)\n",
      "Layer 2 \t conv2d_2 \t\t (None, 14, 14, 64) \t\t\t (None, 14, 14, 128)\n",
      "Layer 3 \t max_pooling2d_2 \t\t (None, 14, 14, 128) \t\t\t (None, 7, 7, 128)\n",
      "Layer 4 \t conv2d_3 \t\t (None, 7, 7, 128) \t\t\t (None, 7, 7, 256)\n",
      "Layer 5 \t max_pooling2d_3 \t\t (None, 7, 7, 256) \t\t\t (None, 4, 4, 256)\n",
      "Layer 6 \t flatten_1 \t\t (None, 4, 4, 256) \t\t\t (None, 4096)\n",
      "Layer 7 \t dense_1 \t\t (None, 4096) \t\t\t (None, 1024)\n",
      "Layer 8 \t dense_2 \t\t (None, 1024) \t\t\t (None, 36)\n"
     ]
    }
   ],
   "source": [
    "for i, layer in enumerate(model.layers):\n",
    "    print (\"Layer\", i, \"\\t\", layer.name, \"\\t\\t\", layer.input_shape, \"\\t\\t\\t\", layer.output_shape)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 28, 28, 64)        640       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 14, 14, 128)       73856     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 7, 7, 256)         295168    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1024)              4195328   \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 36)                36900     \n",
      "=================================================================\n",
      "Total params: 4,601,892\n",
      "Trainable params: 4,601,892\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model : model.h5 telah disimpan\n"
     ]
    }
   ],
   "source": [
    "#Save the model\n",
    "# serialize model to JSON\n",
    "model_json = model.to_json()\n",
    "with open(\"model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "# serialize weights to HDF5\n",
    "\n",
    "namaModel = \"model.h5\"\n",
    "model.save_weights(namaModel)\n",
    "print(\"Model : \" + namaModel + \" telah disimpan\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Uji"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jumlah data uji : 3579\n",
      "[[220   0   0 ...   0   0   0]\n",
      " [  1 227   0 ...   0   0   0]\n",
      " [  0   0 205 ...   0   0   0]\n",
      " ...\n",
      " [  0   0   0 ...  17   1   0]\n",
      " [  0   0   0 ...   1  25   0]\n",
      " [  0   0   0 ...   0   0  15]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.colorbar.Colorbar at 0x1cd0dad9400>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQUAAAD3CAYAAAAUu0E3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGDdJREFUeJzt3X+sHWWdx/H3t7ctKL9rodS2WmDrAovL1VTo2s0GimhlNykkamCziqYRzEJcomusmKxotklNVMSsslsiUrIiEJTQIAvWgmuMCm2xlh9FWkqhl9bWQot1TUrvvd/9Y+bcnnNnzj3POTNzzsy5n1cyufdM5zzPM7f3fs8z33nmeczdERGpmdLrBohIuSgoiEgDBQURaaCgICINFBREpIGCgog0UFAQkQYKCiLSQEFBRBooKIhIg6m9boBIv/vAxcf5q6+NBB27acvhR9x9acFNmpCCgkjB9r82wuOPzA06dtrsF2YW3JyWFBRECueM+GivGxFMQUGkYA6MUp2nkRUURArmOEc8LKdQBl2/+2BmS83sd2a23cxWFFTHTjN7ysw2m9nGnMq83cz2mdnTdftmmNk6M9sWfz2lgDpuMrNX4nPZbGaXZSh/npk9ZmZbzewZM/uXvM9jgjryPI9jzewJM/ttXMeX4/1nmNnj8XncY2bTcy7/DjN7se4cBkPLHMWDtjLoalAwswHg28AHgXOBq8zs3IKqu9jdB919YU7l3QGMzwqvANa7+wJgffw67zoAbo7PZdDdH8pQ/jDwWXc/B1gEXBf//PM8j2Z1QH7ncRhY4u7nA4PAUjNbBHw1rmMBcABYnnP5AJ+rO4fNIYU5MIIHbWXQ7Z7CBcB2d9/h7m8AdwPLutyGjrj7z4HXxu1eBqyJv18DXF5AHblx9z3u/mT8/SFgKzCHHM9jgjpy45E/xS+nxZsDS4D74v0dn8cE5XdMPYXm5gC76l4PkfMvTMyBn5jZJjO7poDya2a5+x6I/hiA0wqq53oz2xJfXmS6RKkxs/nAu4DHKeg8xtUBOZ6HmQ2Y2WZgH7AOeAE46O7D8SGZfrfGl+/utXNYGZ/DzWZ2TEhZDoy4B21l0O2gYCn7ivhJLHb3dxNdplxnZn9XQB3dcitwFlE3dg/w9awFmtnxwA+BG9z9j1nLC6wj1/Nw9xF3HwTmEvVAz0k7LK/yzew84AvA2cB7gBnA50PLGw3cyqDbQWEImFf3ei6wO+9K3H13/HUfcD/RL00R9prZbID46768K3D3vfEv6ChwGxnPxcymEf2xft/dfxTvzvU80urI+zxq3P0g8DOi/MXJZla7o5bL71Zd+UvjSyN398PA9wg8Bw/MJ0zWnMIGYEGcJZ4OXAmszbMCMzvOzE6ofQ+8H3h64nd1bC1wdfz91cADeVdQ+2ONXUGGczEzA74LbHX3b9T9U27n0ayOnM/jVDM7Of7+TcD7iHIXjwEfig/r+DyalP9cXeA0onxF0Dm4w5HArQy6Ok7B3YfN7HrgEWAAuN3dn8m5mlnA/dH/G1OBu9z94ayFmtkPgIuAmWY2BHwJWAXca2bLgZeBDxdQx0XxrS8HdgLXZqhiMfBR4Kn4ehngRvI9j2Z1XJXjecwG1sR3s6YA97r7g2b2LHC3mf078Bui4JRn+Y+a2alEl8GbgU+FFWeMpF45l5Np3QeRYp3319P9hz8Oe6Th7Lft2ZTjbfSOaESjSBdUqaegoCBSsGjwkoKCiNQZdQUFEYmppyAiDRzjiA/0uhnBejJHY8FDj1VHyeroh3PIUketpxCylUGvJm4t/D9QdZSqjn44hwx1GCM+JWgrg0ytsC7MjSBSddHMS1OCtjLoOKdQNzfCpUTPNGwws7Xu/myz97xlxhSfN28qc+cMMHj+dAfYseX4TpswoWN5MyfajEJHZqmOcpTfizoOcWC/u58a+t68Lg3MbB5wJ3A60TNUq939FjObAdwDzCcaMfoRdz8QD8m+BbgM+DPw8dqj7c1kSTSOzY0QN7Y2N0LToDBv3lTWP9T4c7xy3nszNEFyZSm/uKEjXqekJNJGqzMFWbt+6ve9FHqsu+V5aVCbxObJ+BmfTWa2Dvg40UQ5q+Je+wqipzg/CCyItwuJnla9cKIKsrQ0aG4EM7vGzDaa2cZXXy3Lw6Ei3TWKBW2tdDBRzjLgzvjpzl8TPUk6mwlk6SkEzY3g7quB1cDYJYPIZOIYb3j+d/8nmijHzGoT5TT78N7TrNwsLW17boQdW47nyrctbtj3yO7fJI77wFuD58PsXGh3N0uXuhd61d4+vlTIqpZoDDTTGicbXh1/sDYYP4mNpf2/x4c2aVJTWYLC2NwIwCtEcyP8Y4byRPrWSPgw5/2tnpKcaKKcuJdQP1FO2x/eHecU4rnwanMjbCV65jzvuRFEKs8xRpgStLXSwUQ5a4GPWWQR8HrtMqOZTBc68TTdWabqFpkURvO7+9DuRDkPEd2O3E50S/ITrSrQsw8iBYuGOecTFNz9F6TnCQAuSTnegevaqaP7QWFc0istqfjFHck1NlaemZJ8zJJUC02MlTmpmCZLe6t2rhVRtQei1FMQKZg7pXmuIYSCgkjhwgYmlYWCgkjBohWi1FMQkTp5JRq7oZRBYeVZ70rs+7cdmxL7vnLmu5NvzvBgjk1LrlzuR95IObBioxylpxzTHI0i0kg9BREZo1uSItLAyXVEY+EUFES6oCyTsoYoZ1BISdqlJRWXP/9iYt9333FG59WmJRVTD+zTpKISqIVwN/UURKSRximIyJhokhVdPojImFwnbi1cpqBgZjuBQ8AIMNxqxhiRychh0t2SvNjd9+dQTtvSkorf3PnLxL4b5msa+SBKKhZCIxpFJKEsqz+FyNpSB35iZpu6scCnSBVF8ylY0FYGWXsKi919dzzH/Doze87df15/QBwsroFo2S2RyahKlw+Zegruvjv+ug+4n2gpufHHrHb3he6+cBrHZKlOpJKinMKUoK0Msiwwexwwxd0Pxd+/H/hKwBsbX2dJbqWMwLvhjMWJfZ/evjWx71t/cXbn9Yq0abIMc54F3B+vTDMVuMvdH86lVSJ9xDGGRyfBLcl4tenzc2yLSN/SiEYRGVO7+1AVCgoiXVCWJGKIni8Gk2VOxdAkZVpSMXjBGSmPij7arRGNIpKgnIKIjImmY1NQEJEanyS3JEUkjCZZaVdoUjEDOyY5vDotqbjkqf9L7Hv0/BOTBaa1OUvCdLILTSBWIKnYjC4fRGSMcgoikqCgICJjNE5BRBo5DGtE4wTGJ+S6kIzzN8JWjn70nccl9n3npf9N7Pvnt/9tsjwlFTtX4QRiCOUURCRBQUFExiinICIJXqGg0DL7YWa3m9k+M3u6bt8MM1tnZtvir6cU20yRahvFgrYyCOkp3AH8B3Bn3b4VwHp3X2VmK+LXnw+qsRcJudBEVkryMS2puOu+8xL75n3o6cS+0ujGI8ehIzrT2pKmj5KP7vnlFMzsduAfgH3ufl687ybgk8Af4sNudPeH4n/7ArCcaBW3T7v7I63qaNlTiKdsf23c7mXAmvj7NcDlrcoRmbyMkdEpQVuAO4ClKftvdvfBeKsFhHOBK4G/it/zHTNr+WRWpzdPZ7n7HoD462kdliMyKbhb0Na6nNQP6WaWAXe7+2F3fxHYTsoyDOMVPqLCzK4xs41mtvEIh4uuTqR0auMUQrYMrjezLXEOsJbjmwPsqjtmKN43oU6Dwl4zmw0Qf93X7EAtBiOTnkd5hZANmFn7EI23kOUYbwXOAgaBPcDX4/1pUaZlsqbTW5JrgauBVfHXBzosp5LSkooD574jsW/k2eeDyrNp0xP7/EjKKMzUN+e4uE5I+c3q8NHO3xuafKywNu4s7Hf3he2U7e57a9+b2W3Ag/HLIWBe3aFzgd2tygu5JfkD4FfAX5rZkJktJwoGl5rZNuDS+LWIpHDyyymkqfXaY1cAtU+ttcCVZnaMmZ0BLACeaFVey56Cu1/V5J8uafVeEQFyHNEYf0hfRHSZMQR8CbjIzAaJ4s9O4FoAd3/GzO4FngWGgevcveWYAI1oFOmC0dF8gkKTD+nvTnD8SmBlO3UoKIgULEoiVidvoqBQL0OSbmTrtsS+w3//nsS+Y368IVnt8JGwSjqdBzJL8jH0vd2oo6KLwYCekhSRcSoSuwAFBZGu0OWDiIxxOr/d2AsKCiJdUKGrBwWFjgQm/NKSilPOS66APfr0c2H1dvrYeZaRhYEXw2kL7vjhnJ91qdKFeT0Hz+mWZDcoKIh0gS4fRKRBlTo5CgoiBas9+1AVCgoiRXNAQaENZZqzL+8RcynlpSUVn/+v5MjHd1ybTFJ2rAujEv3IcMfvrfJIxVBVOp3eBwWRyUBBQUSOMt2SFJE6FXtKstPFYG4ys1fMbHO8XVZsM0UqzgO3Euh0MRiI5pn/WuYWlCmBGNqW0JGFgeWlJRWnzk1Oujs89EpYvb2QZZGfLPM25pwILu73sY96Cm3OMy8iaSrUU8iy7kPaPPMikmYSBIVm88wnaDEYmfTiB6JCtjLoKCi4+153H3H3UeA2JliKSovBiFCpnkJHtyTNbHZtLUka55kvp14NJ8uQyEpLKk4ZPDexb3Tzs4W2I1gXHs/Ovc3d/L2o0C3JlkGhnXnmRSSdlaQXEKLTxWCazjMvIuOU6NIghEY0ihTO+uvyQURyoJ5Cn+vRo75pScU/X3Fhw+s33/944e3oeFEa6F1SMU2m5GibdaUsyl1WCgoiRdMkKyIyXl/dfRCRHCgotKGfp+LKMngn8LjxOYTX/2lR4piT/vvXTZvYUOW06ckqj7yRPDDLE5Fl0i+/ZznrfVAQmQR0+SAijZRoFJExjm5JikgjXT60o4rJnixtLnjwzkl3Jad2GzglOQfOyMGDySrTkopZdGEdjeDyev10ZoV+zXsfFEQmAwUFEakx1+WDiIynuw8i0qCfegpmNo9ozYfTiW6srHb3W8xsBnAPMJ9o9qWPuPuB4prax/JMyKWMNhw5kPxvGThnQfK4rds6q7OZKk6fVtAIW6vQLcmQiVuHgc+6+znAIuA6MzsXWAGsd/cFwPr4tYiM50fzCq22Vpqs2DbDzNaZ2bb46ynxfjOzb5nZ9ng5hneHNDdkMZg97v5k/P0hYCswB1gGrIkPWwNcHlKhyKSU32zOdwBLx+1r9gH9QWBBvF1DtDRDS21N8W5m84F3AY8Ds2ozOsdfT2unLJFJJaeg0GTFtmYf0MuAOz3ya+BkM5vdqo7goGBmxwM/BG5w9z+28T4tBiOTXhuXDzNrfy/xdk1A8c0+oOcAu+qOG4r3TSjo7oOZTSMKCN939x/Fu/fW1n+Io8++tPe6+2pgNcCJNqNCOdhqmnLCCYl9o4cOJfalJRVt4XmJfb6x3Et6BKvOyNn97r4wp7LS7oO2/EGELEVvRFO6b3X3b9T901rg6vj7q4EHAhopMjkVu0LU3tplwbgP6CFgXt1xc4HdrQoLuXxYDHwUWGJmm+PtMmAVcKmZbQMujV+LyHge3ZIM2TrU7AN6LfCx+C7EIuD1upXdmgpZDOYXpHdDAC5p3V4RyWvwUpMV21YB95rZcuBl4MPx4Q8BlwHbgT8DnwipQyMaRQpm5PfsQ5MV2yDlA9rdHbiu3ToUFMogx8eJ05KKoaP00pKKqcnHJ7cmy0ubt7FX82/mvS5FHus+VCbPqaAgUjw9JSkiCQoKIlKvSg9EKSiIdIN6CtI1Nm6oiack1DIk99KSj7v/9b2JfW/92i87rzfvhGSWpGcRydFsA5O6TkFBpAuUaBSRRgoKIlJPPQURaaSg0ISBTW2s0oeHu9qEnssy2i5ND1aATksq7vpiMvk4b2VK8jFNN0Y5dnPhl3E0xbuIJCkoiEg99RREpJGCgog06KegMMFiMDcBnwT+EB96o7s/NGFhXsHEYjdG26XJOyEZIsOov9Sk4gXvTO7bkDLn4/hRmdCTBGph+jDRWFsM5kkzOwHYZGbr4n+72d2/VlzzRPpEPwWFeE632vTRh8ysthiMiASq0lOSWRaDAbg+Xo7q9tpSVSnv0boPMunltWxcN2RZDOZW4CxgkKgn8fW097n7andf6O4Lp3FMDk0WqZjQ6d1LEhQ6XgzG3ffW/fttwIOFtLDKQhN3vUgqpsn7UeInnkrsGjj5pMS+kdeDFxxL6uUj0e0oyR98iI4Xgxm3Jt0VQJ8sJSSSr9pszlW5fAjpKdQWg3nKzDbH+24ErjKzQaIYuBO4tpAWivSDkvzBh8iyGMzEYxJEZIxVZy1LjWgUKZxX65akgkK9qiSt8hB6Din7xj/+DuEjVUcOvp7Yt3Pl3yT2zf/ir4LKy/2R6KL+byv066GgINIFZUkihlBQEOkGBQURGVOi240hFBREukFBoYeyJIp6dlxgarrTc8t5xGTej7+nJRV3rEomH89cEZh8zKKIORpRT0FExrHR6kQFBQWRopXoYacQCgoiXaDBSyLSSD2FHqrayELIN3GZmozs0cdUhhGiaUnF39+QXHDm9G8GLjgTKq3Nadr8NVOiUUSOcir1YaWgINIFyimIyJiqjVMImXnpWDN7wsx+a2bPmNmX4/1nmNnjZrbNzO4xs+nFN1ekgtzDtxII6SkcBpa4+5/iuRp/YWb/A3yGaN2Hu83sP4HlRJO5dk+W0Yv98PhzqCznVaIRoqffkkw+Tp2bXG1geOiVsHq7+DuQZ0/BzHYCh4ARYNjdF5rZDOAeYD7RTGgfcfcDnZTfsqfgkT/FL6fFmwNLgPvi/WuAyztpgMikkP9szhe7+6C7L4xfrwDWu/sCYH38uiNBU7yb2UA8P+M+YB3wAnDQ3WuD4IfQAjEiTXVh4tZlRB/OkPFDOigouPuIuw8Cc4ELgHPSDkt7rxaDkUnPgVEP28JL/ImZbTKza+J9s+LV3Gqrup3WaXPbuvvg7gfN7GfAIuBkM5sa9xbmArubvGc1sBrgRJvRpxftIhNr45bkTDPbWPd6dfw3VG+xu+82s9OAdWb2XB5trAlZdfpU4EgcEN4EvA/4KvAY8CHgbuBq4IE8GxYkS1JIScXelJez4VeSn0VTz3h78ridLyff3NXFYILr2l+XJ2hSlO+Ov+4zs/uJeu97zWy2u++J12TZ12lTQy4fZgOPmdkWYAOwzt0fBD4PfMbMtgNvIVowRkRS5JVTMLPj4tXfMbPjgPcTLcS0lujDGTJ+SIes+7CFaFHZ8ft3EEUoEZlIvo9OzwLujxZuYypwl7s/bGYbgHvNbDnwMvDhTivQiEaRgkUjGvOJCvGH8fkp+18FLsmjDgUFkW7Qsw/SN8o08jOw3uEXX0rsm3rm/ORxO3ZmbFA4LRsnIkd5W2MQek5BQaQLqvSUpIKCSDfo8kFExmjVaclFmRJ8fSAtqTh1zluTx+3ek3xzLqtOV+f/TkFBpBuqExMUFES6QbckReQoB0YUFEQkZrh6CoXol8Rb6HnkeW7dmGexVzKcW1pScWDBmYl9I8+/0HazEsr+c6xTnaAgUmUKCiIyxtEDUSLSqEo5hSyLwdxhZi+a2eZ4Gyy+uSIVNUkWgwH4nLvfN8F781OSH1hmaecxZSDluJT+Zqc/gzL97Mq0gE/Ke9OSigOzUiZG/n2b9YxW5/ohZDo2B9IWgxGRUNWJCZ0tBuPuj8f/tNLMtpjZzWZ2TGGtFKk4cw/ayqCjxWDM7DzgC8DZwHuAGUSzOydoMRgRKpVTCAoKNe5+EPgZsNTd98TrTB4GvkeTmZ3dfbW7L3T3hdNQZ0ImofxXiCpUx4vB1C08YUTr1j3dqqxDHNj/U7/vJWAmsD9j21upTh0jXahjYkXXcbT8LL/3E7+3uHM4mlSsryO54kxT5ekFhAi5+zAbWGNmA0Q9i3vd/UEzezQOGAZsBj7VqiB3PxXAzDa2WgUnK9VRnjr64Rwy19FPQWGCxWCWFNIikX7jwEh1bj9oRKNI4Tx93ElJ9SoojF9FV3X0dx39cA7Z6qjQ5YN5hRorUkUnTZ/l7z39qqBjH951y6aicyOt6PJBpBsq9OGroCDSDQoKIjLGHUYmHoxSJgoKIt2gnoKINFBQEJGjyvNcQwgFBZGiObgGL4lIA/UURKSBcgoiMka3JEVkPO+niVtFJKv+m2RFRLKoTcdWEW3N0SgiHfLRsK0FM1tqZr8zs+1mtqKIpqqnIFIwBzyHnkI8JeK3gUuBIWCDma1192czF15HPQWRornn1VO4ANju7jvc/Q3gbmBZ3s1VT0GkCzyfW5JzgF11r4eAC/MouJ6CgkjBDnHgkZ/6fTMDDz/WzDbWvV7t7rVp4FIW08x/CUcFBZGCufvSnIoaAubVvZ4L7M6p7DHKKYhUxwZggZmdYWbTgSuBtXlXop6CSEW4+7CZXQ88AgwAt7v7M3nXo9mcRaSBLh9EpIGCgog0UFAQkQYKCiLSQEFBRBooKIhIAwUFEWmgoCAiDf4fVWZ/t8Le2fIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_prob = model.predict(x_test, batch_size=batch_size, verbose=0)\n",
    "y_pred = [np.argmax(prob) for prob in y_prob]\n",
    "y_true = [np.argmax(true) for true in y_test]\n",
    "\n",
    "print('jumlah data uji :', len(y_prob))\n",
    "\n",
    "cm = confusion_matrix(y_true, y_pred)\n",
    "print(cm)\n",
    "\n",
    "plt.matshow(cm)\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['-----0-----', '-----1-----', '-----2-----', '-----3-----', '-----4-----', '-----5-----', '-----6-----', '-----7-----',\n",
    "         '-----8-----', '-----9-----', '-----a-----', '-----b-----', '-----c-----', '-----d-----', '-----e-----', '-----f-----',\n",
    "         '-----g-----', '-----h-----', '-----i-----', '-----j-----', '-----k-----', '-----l-----', '-----m-----', '-----n-----', \n",
    "         '-----o-----', '-----p-----', '-----q-----', '-----r-----', '-----s-----', '-----t-----', '-----u-----', '-----v-----', \n",
    "         '-----w-----', '-----x-----', '-----y-----', '-----z-----']\n",
    "\n",
    "def class_precision(y_true, y_pred, character):\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    i = [i for i, label in enumerate(labels) if label == character][0]\n",
    "    col = [cm[j,i] for j in range(0,36)]\n",
    "    return float(col[i])/sum(col)\n",
    "\n",
    "def class_recall(y_true, y_pred, character):\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    i = [i for i, label in enumerate(labels) if label == character][0]\n",
    "    row = [cm[i,j] for j in range(0,36)]\n",
    "    return float(row[i])/sum(row)\n",
    "\n",
    "def class_accuracy(y_true, y_pred, character):\n",
    "    cm = confusion_matrix(y_true, y_pred)\n",
    "    i = [i for i, label in enumerate(labels) if label == character][0]\n",
    "    tp = cm[i,i]\n",
    "    fn = sum([cm[i,j] for j in range(0,36) if j != i])\n",
    "    fp = sum([cm[j,i] for j in range(0,36) if j != i])\n",
    "    tn = sum([cm[i,j] for j in range(0,36) for i in range(0,4)]) -(tp+fp+fn)\n",
    "    return float(tp + tn)/sum([tp, fn, fp, tn])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----0-----\n",
      "   acc = 0.99\n",
      "  prec = 0.97\n",
      "recall = 0.99\n",
      "\n",
      "-----1-----\n",
      "   acc = 0.99\n",
      "  prec = 0.98\n",
      "recall = 0.97\n",
      "\n",
      "-----2-----\n",
      "   acc = 1.00\n",
      "  prec = 0.99\n",
      "recall = 1.00\n",
      "\n",
      "-----3-----\n",
      "   acc = 0.99\n",
      "  prec = 1.00\n",
      "recall = 0.96\n",
      "\n",
      "-----4-----\n",
      "   acc = 1.00\n",
      "  prec = 0.98\n",
      "recall = 0.99\n",
      "\n",
      "-----5-----\n",
      "   acc = 1.00\n",
      "  prec = 0.98\n",
      "recall = 1.00\n",
      "\n",
      "-----6-----\n",
      "   acc = 0.99\n",
      "  prec = 0.97\n",
      "recall = 0.99\n",
      "\n",
      "-----7-----\n",
      "   acc = 0.99\n",
      "  prec = 0.98\n",
      "recall = 0.99\n",
      "\n",
      "-----8-----\n",
      "   acc = 0.98\n",
      "  prec = 0.96\n",
      "recall = 0.95\n",
      "\n",
      "-----9-----\n",
      "   acc = 0.99\n",
      "  prec = 0.98\n",
      "recall = 0.98\n",
      "\n",
      "-----A-----\n",
      "   acc = 0.98\n",
      "  prec = 0.95\n",
      "recall = 0.96\n",
      "\n",
      "-----B-----\n",
      "   acc = 0.98\n",
      "  prec = 0.90\n",
      "recall = 0.72\n",
      "\n",
      "-----C-----\n",
      "   acc = 0.99\n",
      "  prec = 0.91\n",
      "recall = 0.91\n",
      "\n",
      "-----D-----\n",
      "   acc = 0.99\n",
      "  prec = 0.88\n",
      "recall = 0.77\n",
      "\n",
      "-----E-----\n",
      "   acc = 0.97\n",
      "  prec = 0.90\n",
      "recall = 0.93\n",
      "\n",
      "-----F-----\n",
      "   acc = 0.99\n",
      "  prec = 0.78\n",
      "recall = 0.82\n",
      "\n",
      "-----G-----\n",
      "   acc = 0.99\n",
      "  prec = 0.91\n",
      "recall = 0.86\n",
      "\n",
      "-----H-----\n",
      "   acc = 0.99\n",
      "  prec = 0.93\n",
      "recall = 0.89\n",
      "\n",
      "-----I-----\n",
      "   acc = 0.97\n",
      "  prec = 0.84\n",
      "recall = 0.88\n",
      "\n",
      "-----J-----\n",
      "   acc = 0.98\n",
      "  prec = 0.52\n",
      "recall = 0.65\n",
      "\n",
      "-----K-----\n",
      "   acc = 0.99\n",
      "  prec = 0.93\n",
      "recall = 0.90\n",
      "\n",
      "-----L-----\n",
      "   acc = 0.98\n",
      "  prec = 0.80\n",
      "recall = 0.78\n",
      "\n",
      "-----M-----\n",
      "   acc = 0.99\n",
      "  prec = 0.88\n",
      "recall = 0.81\n",
      "\n",
      "-----N-----\n",
      "   acc = 0.99\n",
      "  prec = 0.92\n",
      "recall = 0.95\n",
      "\n",
      "-----O-----\n",
      "   acc = 0.98\n",
      "  prec = 0.88\n",
      "recall = 0.95\n",
      "\n",
      "-----P-----\n",
      "   acc = 0.99\n",
      "  prec = 0.90\n",
      "recall = 0.82\n",
      "\n",
      "-----Q-----\n",
      "   acc = 0.99\n",
      "  prec = 0.71\n",
      "recall = 0.56\n",
      "\n",
      "-----R-----\n",
      "   acc = 0.98\n",
      "  prec = 0.92\n",
      "recall = 0.91\n",
      "\n",
      "-----S-----\n",
      "   acc = 0.99\n",
      "  prec = 0.98\n",
      "recall = 0.96\n",
      "\n",
      "-----T-----\n",
      "   acc = 0.99\n",
      "  prec = 0.92\n",
      "recall = 0.93\n",
      "\n",
      "-----U-----\n",
      "   acc = 0.99\n",
      "  prec = 0.86\n",
      "recall = 0.75\n",
      "\n",
      "-----V-----\n",
      "   acc = 0.99\n",
      "  prec = 0.92\n",
      "recall = 0.88\n",
      "\n",
      "-----W-----\n",
      "   acc = 0.99\n",
      "  prec = 0.82\n",
      "recall = 0.96\n",
      "\n",
      "-----X-----\n",
      "   acc = 0.99\n",
      "  prec = 0.81\n",
      "recall = 0.85\n",
      "\n",
      "-----Y-----\n",
      "   acc = 0.99\n",
      "  prec = 0.81\n",
      "recall = 0.83\n",
      "\n",
      "-----Z-----\n",
      "   acc = 0.99\n",
      "  prec = 0.79\n",
      "recall = 0.75\n",
      "\n",
      "===============================================================\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "-----0-----       0.97      0.99      0.98       223\n",
      "-----1-----       0.98      0.97      0.98       233\n",
      "-----2-----       0.99      1.00      0.99       206\n",
      "-----3-----       1.00      0.96      0.98       214\n",
      "-----4-----       0.98      0.99      0.99       198\n",
      "-----5-----       0.98      1.00      0.99       212\n",
      "-----6-----       0.97      0.99      0.98       204\n",
      "-----7-----       0.98      0.99      0.98       208\n",
      "-----8-----       0.96      0.95      0.95       196\n",
      "-----9-----       0.98      0.98      0.98       218\n",
      "-----a-----       0.95      0.96      0.95       147\n",
      "-----b-----       0.90      0.72      0.80        39\n",
      "-----c-----       0.91      0.91      0.91        58\n",
      "-----d-----       0.88      0.77      0.82        39\n",
      "-----e-----       0.90      0.93      0.92       133\n",
      "-----f-----       0.78      0.82      0.80        22\n",
      "-----g-----       0.91      0.86      0.89        37\n",
      "-----h-----       0.93      0.89      0.91        64\n",
      "-----i-----       0.84      0.88      0.86        76\n",
      "-----j-----       0.52      0.65      0.58        17\n",
      "-----k-----       0.93      0.90      0.92        30\n",
      "-----l-----       0.80      0.78      0.79        45\n",
      "-----m-----       0.88      0.81      0.84        43\n",
      "-----n-----       0.92      0.95      0.93        93\n",
      "-----o-----       0.88      0.95      0.91       100\n",
      "-----p-----       0.90      0.82      0.86        45\n",
      "-----q-----       0.71      0.56      0.63        18\n",
      "-----r-----       0.92      0.91      0.92       114\n",
      "-----s-----       0.98      0.96      0.97       113\n",
      "-----t-----       0.92      0.93      0.93        91\n",
      "-----u-----       0.86      0.75      0.80        24\n",
      "-----v-----       0.92      0.88      0.90        25\n",
      "-----w-----       0.82      0.96      0.88        24\n",
      "-----x-----       0.81      0.85      0.83        20\n",
      "-----y-----       0.81      0.83      0.82        30\n",
      "-----z-----       0.79      0.75      0.77        20\n",
      "\n",
      "avg / total       0.94      0.94      0.94      3579\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for character in labels:\n",
    "    print(character.upper())\n",
    "    print('   acc = {0:.2f}'.format(class_accuracy(y_true, y_pred, character)))\n",
    "    print('  prec = {0:.2f}'.format(class_precision(y_true, y_pred, character)))\n",
    "    print('recall = {0:.2f}\\n'.format(class_recall(y_true, y_pred, character)))\n",
    "    \n",
    "print('===============================================================')\n",
    "print(classification_report(y_true, y_pred, target_names=labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Latih"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "jumlah data latih : 14316\n",
      "[[898   0   0 ...   0   0   0]\n",
      " [  0 860   0 ...   0   0   0]\n",
      " [  0   0 880 ...   0   0   0]\n",
      " ...\n",
      " [  0   0   0 ...  95   0   0]\n",
      " [  0   0   0 ...   0  80   0]\n",
      " [  0   0   0 ...   0   0  70]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.colorbar.Colorbar at 0x1cd1fa9e0b8>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAD3CAYAAAD7eSoJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFyZJREFUeJzt3XuwHGWdxvHvQxIIwQvGAHKJBCSrsKxEljtbFhcvkd0SqXVd0LLUSm2wCnZxVZZA1S7oulVYXhBLSysKAhYQ2WBKClkgcinKFQIJxBBANIQAIZEQbuKiITnnt3/0O3Mmczs9Z7pnzpzzfKq6zpmeOe/bjZmf3W93v48iAjMzgF36vQFmNn64IJhZlQuCmVW5IJhZlQuCmVW5IJhZlQuCmVW5IJhZlQuCmVW5IJhZ1dR+b4DZRPfBk/eIF14cyvXZVWu23RYR80vepJZcEMxKtvXFIVbcdkCuz07b94lZJW9OWy4IZqULhmK43xuRiwuCWckCGGYwnip2QTArWRBsj3xjCP3W86sMkuZLelzSOkmLSupjg6SHJa2WtLKgNq+UtEXS2pp1MyUtl/S79PMtJfRxiaRn076slnRaF+3PlnSXpMckPSLpvKL3o00fRe7HdEn3S/p16uNLaf1Bklak/fiJpF0Lbv8qSU/W7MO8vG0OE7mWfutpQZA0Bfgu8CHgMOAsSYeV1N3JETEvIo4qqL2rgPrR30XAHRExF7gjvS66D4DL0r7Mi4hbumh/B/CFiDgUOA44J/33L3I/WvUBxe3HNuCUiDgCmAfMl3Qc8NXUx1zgJWBBwe0DnF+zD6vzNBbAEJFr6bdeHyEcA6yLiPUR8TqwBDi9x9swJhFxD/Bi3erTgavT71cDHymhj8JExOaIeDD9/irwGLA/Be5Hmz4KE5k/ppfT0hLAKcDStH7M+9Gm/THzEUJz+wPP1LzeSMH/WJIAbpe0StLCEtqv2CciNkP2RQD2LqmfcyWtSacUXZ2WVEiaA7wHWEFJ+1HXBxS4H5KmSFoNbAGWA08AL0fEjvSRrv5t1bcfEZV9+K+0D5dJ2i1PWwEMReRa+q3XBUFN1pXxX+HEiDiS7NTkHEnvLaGPXvke8A6yQ9fNwDe6bVDSG4Abgc9FxB+6bS9nH4XuR0QMRcQ84ACyI89Dm32sqPYlHQ5cCLwLOBqYCVyQt73hnEu/9bogbARm17w+ANhUdCcRsSn93AIsI/sHU4bnJO0LkH5uKbqDiHgu/eMcBn5Al/siaRrZF/XaiPhpWl3ofjTro+j9qIiIl4G7ycYr9pRUuXJWyL+tmvbnp9OhiIhtwI/IuQ+Rc/xgMo4hPADMTaPBuwJnAjcV2YGkPSS9sfI78AFgbfu/GrObgE+l3z8F/KzoDipf1OQMutgXSQKuAB6LiG/WvFXYfrTqo+D92EvSnun33YH3kY1V3AV8NH1szPvRov3f1BRNkY1P5NqHCNiec+m3nt6HEBE7JJ0L3AZMAa6MiEcK7mYfYFn2vxlTgesi4tZuG5V0PXASMEvSRuBi4FLgBkkLgKeBfyihj5PS5a0ANgBnd9HFicAngYfT+THARRS7H636OKvA/dgXuDpdtdoFuCEibpb0KLBE0leAh8gKU5Ht3ylpL7JT39XAZ/M1J4aani2PP3Iug1m5Dn/3rnHjz/M9ovCut29eVeCl8o75TkWzHhiUIwQXBLOSZTcmuSCYWTIcLghmho8QzKxGILbHlH5vRi59mVOx5NuJ3cc462Mi7EM3fVSOEPIs/davSVZL/x/PfYyrPibCPnTRhxiKXXIto7Yk/Wt6JHutpOvTo9pNH/uWtFt6vS69P2e09rsqCOrB3AZmgy6bMWmXXEs7kvYH/gU4KiIOJ7u570xaP/a9AHgpIg4BLkufa2vMYwg1cxu8n+wZhQck3RQRj7b6m1kzp8Sc2dN4+/5TOeqI6QHw2zUzxroJbU1nBm/SzFLvunIf46P9fvTxKi9tjYi98v5tgacDU4HdJW0HZpA9KHYK8PH0/tXAJWQPk52efofssfDvSFK0uRuxm0HF6twGAJIqcxu0LAhzZk/j/ttm77Tug/vlnnTGbNz4RSx9Ku9nI5TrdGD0duJZSV8nu738T8DtwCpaP/ZdnW4gPTbwCvBWYGurPrrZylxzG0haKGmlpJXPvzAY88qZFW0Y5VrInmNZWbNUxy3SHBKnAwcB+wF7kD3iX69yBNDxdAPdHCHk6iwiFgOLgeppgtlkEojXI/dXbWubZxneBzwZEc8DSPopcALpse90lFD72HdluoGN6bHwNzPKjFzdFISO5zb47ZoZDacIVz39y4bPffrtf9PFZpmNL5VBxQI8DRwnaQbZKcOpwEpGHvtews6PfVcea783vX9nu/ED6K4gVOc2AJ4lG+38ePs/MZuchgq4dTkiVkhaCjxINpntQ2RH3z+n+WPfVwA/lrSO7MjgzNH6GHNB6NHcBmYDLxBDBd3yExEXk82TUWs9TWZviog/0+HcFl3dupym0u5mOm2zSWG4gKsMveBnGcxKlt267IKQS7MBxBs33tew7u8POK5hndkgGKSHm/peEMwmuggKuTGpF1wQzEpXvelo3HNBMCtZltzkIwQzSzyo2IVmA4jff6rxjsbPHug7Gm38C+Q5Fc1shI8QzAzwZUczqxH4TkUzqzEeJlDNY2AKQrMBxB8/878N6z45+8RebI5ZbhHyEYKZjfB9CGYGVCZI8SmDmQGVXIZB0G0uwwZJD0taLWllURtlNpEEsD2m5FpGI+md6ftWWf4g6XOSZkpansJalqcJWVHm2yk7ZY2kI9u1X8QRwskR0XJa5zI1G0C8tslA4yc80Gh9VOSdihHxODAPqtkozwLLgEXAHRFxaQpNWgRcQDYr89y0HEuW13Bsq/YH4zjGbMAVkdzUxKnAExHxFNn07Fen9VcDH0m/nw5cE5n7yGZo3rdVg90WhABul7SqF2GbZoMomw9BuZYOnQlcn37fJyI2Z/3FZmDvtD5XfkpFt6cMJ0bEJkl7A8sl/SYi7qn9QCoUCyGLwjKbjDo4ZZhVNx63OGWb7CQFun4YuHCU9joKa+l2ktVN6ecWScvIZn69p+4z1aCWsrP3zMajbAwh98F4u6CWWh8CHoyI59Lr5yTtGxGb0ynBlrS+o/yUbsJe9wB2iYhX0+8fAL481vaK0mwA8YaN9zas+9gBx/dic8yAUm5dPouR0wUYCWW5lMawlnNT9uqxwCuVU4tmujlC2AdYJqnSznURcWsX7ZlNSIHYMVzc044puen9wNk1qy8FbpC0gCzhqZLHcAtwGrAOeA34TLu2uwlqWQ8cMda/N5tMirxTMSJeI0txrl33AtlVh/rPBnBO3rZ9p6JZySpXGQaBC4JZD/hpx3Gk2QDiFU1Spxc4ddpK4DkVzWwnftrRzIDKFGouCGYGEMVediyTC4JZyTxBygBoNoB407MPNKz78P5H92JzbILzKYOZAR5DMLM6LghmBvg+BDOrFbDDdyoOnmYDiF/bcF/DuvPnNKZTm7XiMQQz24kLgpkBHkMwszoxIAVh1JEOSVdK2iJpbc26pqEQZtbcMMq15CFpT0lLJf1G0mOSju9lUMtVwHeAa2rWtQqFmHCaDSBuWnZYw7r9zni0F5tjAyii8DGEy4FbI+KjafblGcBF9CKoJU2r/mLd6lahEGbWQAwN75JrGbUl6U3Ae4ErACLi9Yh4mT4HtbQKhTCzJiKUa8nhYOB54EeSHpL0wzTreSFBLaXfLSFpoaSVklZuZ1vZ3ZmNO5X7EPIspKCWmqU+EW0qcCTwvYh4D/B/ZKcHrfQkqKVVKERjzw5qsckusnGEnEYLatkIbIyIFen1UrKC0NegllahEJNCswHEKX/5zoZ1Q4883ovNsQFQ1HwIEfF7Sc9IemdKgj4VeDQt5Qe1SLoeOInsUGYjcDGtQyHMrE5Q+H0I/wxcm64wrCcLX9mFXgS1RMRZLd5qCIUws2aKvVMxIlYDzU4rHNRiNgiGhwfjTkUXBLOSRQzOrcsuCAVpNoD4p9OPaVi3+8/u78Xm2Djjh5vMrKqDy4595YJg1gM+ZTAzIJsPwQXBzKoG5IzBBaFMzQYQX5/fOG/jrrc2BsTYBBIQvuxoZhU+ZTCzKl9lMDOglGcZSuOCYFa2AFwQrJlmA4jrr5vXsO7gj6/uxeZYj/iUwcxGuCCYWUa+7GhmyQA97TjWoJZLJD0raXVaTit3M80GXORccpC0QdLD6bu3Mq3ra1ALwGUR8fV8u2DtNBtAnHLIQQ3rhtY92YvNsVIUfoRwckRsrXndKjypJ0EtZtaJAo8QWuhrUAtkM7muSacUznY0a6fYghDA7ZJW1eQ29DWo5XvAO4B5wGbgG60+6KAWm/TSw015FkYPagE4MSKOJDsdOEfSe9v0Xn5QS0Q8V+1N+gFwc5vPOqjFrLigFiJiU/q5RdIy4Bj6GdRS6Ti9PANY2+7z1rlmA4g6+q8a1sUDD/dic6xbBV12TDmOu0TEq+n3DwBfpnV4Uk+CWk6SNI+s7m0Azh7T3plNEiru2HgfYJkkyL6/10XErZIeoI9BLVfk3nyzya77KwgjTUWsB45osv4FHNRiNgjkpx3NrMaADKe7IAyQZgOIO075651eT71zVa82xzox3O8NyMcFwaxsniDFzGoVeJWhVC4IZr3ggmC9UD9msOn8Exo+s9/XftWrzbEB54Jg1gM+ZTCzER5UNDMgGz/wZUczq/Apg/VFswFET8c2DrggmFmVC4KZQXa64FMGMxsxIFcZuplk1czyKjaXYYqkhyTdnF4fJGlFymT4iaRd0/rd0ut16f05o7WdZ8ak2WSZDG8ju3iyOCIulzQT+Akwh2zWpI9FxEv5dsl6qdkA4pS5Bzd+7nfre7E5k5KKvex4HvAY8Kb0+qtkOSlLJH0fWEA2EfIC4KWIOETSmelz/9iu4TxHCDuAL0TEocBxZLO8HsZIMMRc4I702szqxcg4wmjLaCQdAPwt8MP0WsApwNL0kfpMhkpWw1Lg1PT5lvIEtWyOiAfT76+SVab9aR0MYWb1ijtl+Bbwb4zc6vRW4OWI2JFe1+YuVDMZ0vuvpM+31NEYQjoHeQ+wgtbBEGZWL39BaJnLIOnvgC0RUftEW7vchY4yGaCDqwyS3gDcCHwuIv4wypFH7d8tBBYCTGdG3u7MJpQOLju2y2U4EfhwCleeTjaG8C2yeLap6SigNnehksmwUdJU4M2MEsuYqyBImkZWDK6NiJ+m1a2CIXbioJbxqdkA4mtnNGaAzli2ohebYzlExIXAhQCSTgK+GBGfkPTfwEeBJTRmMnwKuDe9f2eahbmlPHHwIpt2/bGI+GbNW5XOqNsIM6tXbtjrBcDnJa0jGyOoxCRcAbw1rf88OQb+8xwhnAh8EnhYUiW3/CKyhJhmwRBmVisKv+xIRNwN3J1+X08W51b/mT/T4fcyT1DLL2kdbt8QDGFmTQzIybJvXTYrmfCzDDaAmg4gHvfuxnX3rSl/YyYaFwQzA6p3Kg4CFwSzXnBBMLOKoq8ylMUFwawXfIRgE0KTAcQN/3l8w7o5/35vL7ZmMHV301FPuSCY9YAHFc1shAuCmVX4CMHMRrgg2ETVbABx/VcbBxoPvsADjeBp2M2snguCmVX4CMHMRgxIQXBQi1kvFDRjkqTpku6X9GtJj0j6UlpfSFhLN0EtlwD/BDyfPnpRRNwy+i7ZRNRsADGOP6Jhne79dS82Z3wpdlBxG3BKRPwxzXX6S0n/QzZFWtdhLd0EtZA2YF5aXAzMWinoCCEyf0wvp6UlKCispZugFjPLScP5llxtZdmOq8lmOl8OPEFBYS3dBLUAnCtpjaQrJb2lxd8srIRObGdbJ92ZTRgdRLm1DGqpiIihiJhHlsFwDHBoky7HFNaSuyDUB7WQnZ+8A5gHbAa+0ezvImJxRBwVEUdNY7e83ZlNHHlPF7Kv6dbK9yUti1s2G/Ey2czLx5HCWtJbzcJayBPWMuagloh4rub9HwA352nLJo9mA4hTD2g829yx8dlebE5/FTSoKGkvYHtEvCxpd+B9ZAOFd1FAWEueqwxNg1oqqU3p5RnA2g73zWxSKHjW5X2BqyVNITvCvyEibpb0KLBE0leAh9g5rOXHKazlReDMdo13E9RylqR5ZLVvA3B2R7tlNpkUVBAiYg3ZOF79+kLCWroJavFlRrOc1D5ScdzwrctmZSshyq0sLgjWU80GEJ/6cuOj0wf+xwR7dHowDhBcEMx6wU87mtkIFwQzAxzlZmZ1XBDM8mk2gLjp/BMa1u33tV/1YnMK5zh4M9uJhgejIrggmJXNUW5mVss3JpnZCB8hmI1dswHE35/XOND4tssHY6DRg4pmlgnADzeZWYXHEMwMGKz7EEadU7HTYAgzqxORfxmFpNmS7pL0WPo+npfWz5S0PH0fl1cmPVbm2ymoZY2kI9u1n+cIodNgCLNSNBtAnHLYXzSsG3r0t73YnI4UeIRQyUl5UNIbgVWSlgOfBu6IiEslLQIWARcAHwLmpuVYsu/osa0az5PL0GkwhJnVKy6opVVOSm0gS31QyzXpe3wf2ezM+7ZqP9c07B0GQ5hZnQ5yGfK3uXNOyj6VSY/Tz73Tx6pBLUnb72quQcWIGALmSdoTWEb7YIj6jV4ILASYzow83ZlNLAHkf5ZhlqSVNa8XN8tmqM9JaZPO1lFQS0dXGdJc8HdTEwyRjhJqgyHq/2YxsBjgTZo5IGOtZsXq4LLj1og4qm1bTXJSgOcq0QjplGBLWl8NaklaflchXy5Dp8EQZj3TbABxyiEHNX5u3ZO92JzWCroxqVVOCiOBLJfSGNRyrqQlZIOJr9TkqTTIc4TQaTCEmdUp8CpDq5yUS4EbJC0AnmYki+EW4DRgHfAa8Jl2jefJZegoGMLM6hT4+HObnBSAU5t8PoBz8rbvOxXNSpbdqTgYw2cuCGa94GcZzPqj2QDi1ANnN6zb8dQzDevK4iMEM8tEdHIfQl+5IJj1wKA87eiCYNYLPmUwM8Dpz2bjTbMBxClzD25YN/S79eVsgI8QzKxqMOqBC4JZL/iyo5llAhhyQTAzQISPEMzGu2YDiKXd0eiCYGZVLghmBqQp1Pq9Efm4IJj1wKCMIXQT1HKVpCclrU7LvPI312xAFRfUcqWkLZLW1qwrJKQFugtqATg/Ipa2+VuzgdJsAHHq/vs1fnBjB41GwHBh5wxXAd8BrqlZt4gCQlqgu6AWM8trOOcyioi4B3ixbnUhIS0wxqCWiFiR3vqvdChymaTd8rRlNhkpItcyRoWEtEDOghARQxExj2xO92MkHQ5cCLwLOBqYSXaI0kDSQkkrJa3czrY83ZlNPPnHEGZVvi9pWdhFrx2FtMDYg1rmR8TX0+ptkn4EfLHF3zioxSa3zpKbRg1qaaKQkBboIqilZgNEds6ytm1DwKu8tPUXsfQpYBawdbTPd8l9jJ8+BnsfRgYQa/s4MH8D+a4gdKGQkBboLqjlzlQsBKwGPjtaQxGxF4CklWOogh1xH+Onj4mwD133UVxy0/XASWSnFhuBiykopAW6C2o5Jd8umE1yAQwVc9kxIs5q8VbXIS3gOxXNeiAgBuPe5X4VhIZ4a/cxofuYCPvQXR8DcuuyYkA21GxQvXnXfeKEt7U60t/Zrc9cvqrssZB2fMpg1gsD8n+8LghmveCCYGZAVgyGhvq9Fbm4IJj1go8QzKzKBcHMMk5/NrOKgPCNSWZW5SMEM6vyGIKZAb7saGY7i+ImWS2VC4JZ6UqfIKUwLghmZetsCrW+yjXJqpl1KYbzLaOQNF/S4yl8ZVHRm+kjBLOSBRAFHCGkaQy/C7yfbKbHByTdFBGPdt144iMEs7JFFHWEcAywLiLWR8TrwBKyMJbC+AjBrAeimMuOzYJX2kazdcoFwaxkr/LSbb+IpbNyfny6pJU1rxenbBMYQ/BKp1wQzEoWEfMLaqrj4JVOeQzBbHA8AMyVdJCkXYEzycJYCuMjBLMBERE7JJ0L3AZMAa6MiEeK7MOzLptZlU8ZzKzKBcHMqlwQzKzKBcHMqlwQzKzKBcHMqlwQzKzKBcHMqv4fIE9rDET7pZcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_prob = model.predict(x_train, batch_size=batch_size, verbose=0)\n",
    "y_pred = [np.argmax(prob) for prob in y_prob]\n",
    "y_true = [np.argmax(true) for true in y_train]\n",
    "\n",
    "print('jumlah data latih :', len(y_prob))\n",
    "\n",
    "cm = confusion_matrix(y_true, y_pred)\n",
    "print(cm)\n",
    "\n",
    "plt.matshow(cm)\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----0-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----1-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----2-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----3-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----4-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----5-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----6-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----7-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----8-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----9-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----A-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----B-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----C-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----D-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----E-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----F-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----G-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----H-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----I-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----J-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----K-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----L-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----M-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----N-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----O-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----P-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----Q-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----R-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----S-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----T-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----U-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----V-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----W-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----X-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----Y-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "-----Z-----\n",
      "   acc = 1.00\n",
      "  prec = 1.00\n",
      "recall = 1.00\n",
      "\n",
      "===============================================================\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "-----0-----       1.00      1.00      1.00       898\n",
      "-----1-----       1.00      1.00      1.00       860\n",
      "-----2-----       1.00      1.00      1.00       880\n",
      "-----3-----       1.00      1.00      1.00       852\n",
      "-----4-----       1.00      1.00      1.00       865\n",
      "-----5-----       1.00      1.00      1.00       868\n",
      "-----6-----       1.00      1.00      1.00       876\n",
      "-----7-----       1.00      1.00      1.00       856\n",
      "-----8-----       1.00      1.00      1.00       884\n",
      "-----9-----       1.00      1.00      1.00       832\n",
      "-----a-----       1.00      1.00      1.00       569\n",
      "-----b-----       1.00      1.00      1.00       114\n",
      "-----c-----       1.00      1.00      1.00       220\n",
      "-----d-----       1.00      1.00      1.00       198\n",
      "-----e-----       1.00      1.00      1.00       540\n",
      "-----f-----       1.00      1.00      1.00        94\n",
      "-----g-----       1.00      1.00      1.00       142\n",
      "-----h-----       1.00      1.00      1.00       181\n",
      "-----i-----       1.00      1.00      1.00       353\n",
      "-----j-----       1.00      1.00      1.00        93\n",
      "-----k-----       1.00      1.00      1.00        96\n",
      "-----l-----       1.00      1.00      1.00       228\n",
      "-----m-----       1.00      1.00      1.00       149\n",
      "-----n-----       1.00      1.00      1.00       405\n",
      "-----o-----       1.00      1.00      1.00       430\n",
      "-----p-----       1.00      1.00      1.00       151\n",
      "-----q-----       1.00      1.00      1.00        71\n",
      "-----r-----       1.00      1.00      1.00       401\n",
      "-----s-----       1.00      1.00      1.00       353\n",
      "-----t-----       1.00      1.00      1.00       332\n",
      "-----u-----       1.00      1.00      1.00       109\n",
      "-----v-----       1.00      1.00      1.00        92\n",
      "-----w-----       1.00      1.00      1.00        79\n",
      "-----x-----       1.00      1.00      1.00        95\n",
      "-----y-----       1.00      1.00      1.00        80\n",
      "-----z-----       1.00      1.00      1.00        70\n",
      "\n",
      "avg / total       1.00      1.00      1.00     14316\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for character in labels:\n",
    "    print(character.upper())\n",
    "    print('   acc = {0:.2f}'.format(class_accuracy(y_true, y_pred, character)))\n",
    "    print('  prec = {0:.2f}'.format(class_precision(y_true, y_pred, character)))\n",
    "    print('recall = {0:.2f}\\n'.format(class_recall(y_true, y_pred, character)))\n",
    "    \n",
    "print('===============================================================')\n",
    "print(classification_report(y_true, y_pred, target_names=labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# K.set_learning_phase(0)\n",
    "# sess = K.get_session()\n",
    "\n",
    "# output_node_name = \"dense_2/Softmax\" # Name of your output node\n",
    "\n",
    "# with sess as sess:\n",
    "#     init_op = tf.global_variables_initializer()\n",
    "#     sess.run(init_op)\n",
    "#     graph_def = sess.graph.as_graph_def()\n",
    "#     output_graph_def = graph_util.convert_variables_to_constants(\n",
    "#                                                                  sess,\n",
    "#                                                                  sess.graph.as_graph_def(),\n",
    "#                                                                  output_node_name.split(\",\"))\n",
    "#     tf.train.write_graph(output_graph_def,\n",
    "#                          logdir=\"model\",\n",
    "#                          name=\"my_model.pb\",\n",
    "#                          as_text=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}