Day087HW.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Work\n",
    "1. 請改變 reduce_lr 的 patience 和 factor 並比較不同設定下，對訓練/驗證集的影響\n",
    "2. 請將 optimizer 換成 Adam、RMSprop 搭配 reduce_lr 並比較訓練結果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import keras\n",
    "\n",
    "# Disable GPU\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test = keras.datasets.cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 資料前處理\n",
    "def preproc_x(x, flatten=True):\n",
    "    x = x / 255.\n",
    "    if flatten:\n",
    "        x = x.reshape((len(x), -1))\n",
    "    return x\n",
    "\n",
    "def preproc_y(y, num_classes=10):\n",
    "    if y.shape[-1] == 1:\n",
    "        y = keras.utils.to_categorical(y, num_classes)\n",
    "    return y    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, y_train = train\n",
    "x_test, y_test = test\n",
    "\n",
    "# Preproc the inputs\n",
    "x_train = preproc_x(x_train)\n",
    "x_test = preproc_x(x_test)\n",
    "\n",
    "# Preprc the outputs\n",
    "y_train = preproc_y(y_train)\n",
    "y_test = preproc_y(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import BatchNormalization\n",
    "\n",
    "\"\"\"\n",
    "建立神經網路，並加入 BN layer\n",
    "\"\"\"\n",
    "def build_mlp(input_shape, output_units=10, num_neurons=[512, 256, 128]):\n",
    "    input_layer = keras.layers.Input(input_shape)\n",
    "    \n",
    "    for i, n_units in enumerate(num_neurons):\n",
    "        if i == 0:\n",
    "            x = keras.layers.Dense(units=n_units, \n",
    "                                   activation=\"relu\", \n",
    "                                   name=\"hidden_layer\"+str(i+1))(input_layer)\n",
    "            x = BatchNormalization()(x)\n",
    "        else:\n",
    "            x = keras.layers.Dense(units=n_units, \n",
    "                                   activation=\"relu\", \n",
    "                                   name=\"hidden_layer\"+str(i+1))(x)\n",
    "            x = BatchNormalization()(x)\n",
    "    \n",
    "    out = keras.layers.Dense(units=output_units, activation=\"softmax\", name=\"output\")(x)\n",
    "    \n",
    "    model = keras.models.Model(inputs=[input_layer], outputs=[out])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "LEARNING_RATE = 1e-3\n",
    "EPOCHS = 40 # IF you feel too run to finish, try to make it smaller\n",
    "BATCH_SIZE = 1024 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 載入 Callbacks\n",
    "from keras.callbacks import ReduceLROnPlateau\n",
    "\n",
    "optimizer_set = [keras.optimizers.SGD(lr=LEARNING_RATE, nesterov=True, momentum=0.95),\n",
    "                 keras.optimizers.Adam(lr=LEARNING_RATE),\n",
    "                 keras.optimizers.RMSprop(lr=LEARNING_RATE)]\n",
    "\n",
    "\"\"\"Code Here\n",
    "建立實驗的比較組合\n",
    "\"\"\"\n",
    "reduce_lr_factor = [0.3,0.5,0.8]\n",
    "redice_lr_patient = [5,10,20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Numbers of exp: 0, reduce_factor: 0.30, reduce_patient: 5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 3072)              0         \n",
      "_________________________________________________________________\n",
      "hidden_layer1 (Dense)        (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "hidden_layer2 (Dense)        (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "hidden_layer3 (Dense)        (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 1,742,474\n",
      "Trainable params: 1,740,682\n",
      "Non-trainable params: 1,792\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/40\n",
      "50000/50000 [==============================] - 6s 126us/step - loss: 2.2567 - acc: 0.2593 - val_loss: 2.1192 - val_acc: 0.3128\n",
      "Epoch 2/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.7477 - acc: 0.3917 - val_loss: 1.8176 - val_acc: 0.3843\n",
      "Epoch 3/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.6192 - acc: 0.4345 - val_loss: 1.6786 - val_acc: 0.4119\n",
      "Epoch 4/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.5457 - acc: 0.4596 - val_loss: 1.6287 - val_acc: 0.4272\n",
      "Epoch 5/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.4899 - acc: 0.4791 - val_loss: 1.5956 - val_acc: 0.4407\n",
      "Epoch 6/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.4437 - acc: 0.4966 - val_loss: 1.5618 - val_acc: 0.4487\n",
      "Epoch 7/40\n",
      "50000/50000 [==============================] - 6s 110us/step - loss: 1.4064 - acc: 0.5121 - val_loss: 1.5515 - val_acc: 0.4590\n",
      "Epoch 8/40\n",
      "50000/50000 [==============================] - 5s 103us/step - loss: 1.3714 - acc: 0.5215 - val_loss: 1.5258 - val_acc: 0.4628\n",
      "Epoch 9/40\n",
      "50000/50000 [==============================] - 5s 103us/step - loss: 1.3384 - acc: 0.5342 - val_loss: 1.5164 - val_acc: 0.4680\n",
      "Epoch 10/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.3095 - acc: 0.5459 - val_loss: 1.5032 - val_acc: 0.4692\n",
      "Epoch 11/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.2817 - acc: 0.5555 - val_loss: 1.5026 - val_acc: 0.4707\n",
      "Epoch 12/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.2542 - acc: 0.5657 - val_loss: 1.4895 - val_acc: 0.4758\n",
      "Epoch 13/40\n",
      "50000/50000 [==============================] - 6s 113us/step - loss: 1.2299 - acc: 0.5714 - val_loss: 1.4884 - val_acc: 0.4751\n",
      "Epoch 14/40\n",
      "50000/50000 [==============================] - 6s 113us/step - loss: 1.2049 - acc: 0.5854 - val_loss: 1.4776 - val_acc: 0.4792\n",
      "Epoch 15/40\n",
      "50000/50000 [==============================] - 6s 128us/step - loss: 1.1802 - acc: 0.5928 - val_loss: 1.4674 - val_acc: 0.4858\n",
      "Epoch 16/40\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 1.1570 - acc: 0.6023 - val_loss: 1.4590 - val_acc: 0.4931\n",
      "Epoch 17/40\n",
      "50000/50000 [==============================] - 6s 116us/step - loss: 1.1334 - acc: 0.6091 - val_loss: 1.4616 - val_acc: 0.4858\n",
      "Epoch 18/40\n",
      "50000/50000 [==============================] - 6s 110us/step - loss: 1.1113 - acc: 0.6185 - val_loss: 1.4547 - val_acc: 0.4947\n",
      "Epoch 19/40\n",
      "50000/50000 [==============================] - 6s 114us/step - loss: 1.0881 - acc: 0.6277 - val_loss: 1.4558 - val_acc: 0.4941\n",
      "Epoch 20/40\n",
      "50000/50000 [==============================] - 6s 125us/step - loss: 1.0662 - acc: 0.6358 - val_loss: 1.4698 - val_acc: 0.4899\n",
      "Epoch 21/40\n",
      "50000/50000 [==============================] - 6s 118us/step - loss: 1.0456 - acc: 0.6437 - val_loss: 1.4672 - val_acc: 0.4889\n",
      "Epoch 22/40\n",
      "50000/50000 [==============================] - 6s 116us/step - loss: 1.0247 - acc: 0.6512 - val_loss: 1.4698 - val_acc: 0.4850\n",
      "Epoch 23/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.0034 - acc: 0.6585 - val_loss: 1.4592 - val_acc: 0.4914\n",
      "\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 0.0003000000142492354.\n",
      "Epoch 24/40\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 0.9776 - acc: 0.6699 - val_loss: 1.4474 - val_acc: 0.4975\n",
      "Epoch 25/40\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 0.9552 - acc: 0.6801 - val_loss: 1.4462 - val_acc: 0.4981\n",
      "Epoch 26/40\n",
      "50000/50000 [==============================] - 6s 123us/step - loss: 0.9472 - acc: 0.6834 - val_loss: 1.4476 - val_acc: 0.4995\n",
      "Epoch 27/40\n",
      "50000/50000 [==============================] - 6s 115us/step - loss: 0.9397 - acc: 0.6866 - val_loss: 1.4458 - val_acc: 0.5020\n",
      "Epoch 28/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 0.9315 - acc: 0.6915 - val_loss: 1.4501 - val_acc: 0.4983\n",
      "Epoch 29/40\n",
      "50000/50000 [==============================] - 7s 135us/step - loss: 0.9250 - acc: 0.6931 - val_loss: 1.4496 - val_acc: 0.4992\n",
      "Epoch 30/40\n",
      "50000/50000 [==============================] - 7s 139us/step - loss: 0.9185 - acc: 0.6931 - val_loss: 1.4512 - val_acc: 0.4985\n",
      "Epoch 31/40\n",
      "50000/50000 [==============================] - 7s 130us/step - loss: 0.9109 - acc: 0.6972 - val_loss: 1.4559 - val_acc: 0.4993\n",
      "Epoch 32/40\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 0.9041 - acc: 0.6998 - val_loss: 1.4501 - val_acc: 0.5021\n",
      "\n",
      "Epoch 00032: ReduceLROnPlateau reducing learning rate to 9.000000427477062e-05.\n",
      "Epoch 33/40\n",
      "50000/50000 [==============================] - 6s 127us/step - loss: 0.8939 - acc: 0.7044 - val_loss: 1.4510 - val_acc: 0.5025\n",
      "Epoch 34/40\n",
      "50000/50000 [==============================] - 6s 124us/step - loss: 0.8875 - acc: 0.7086 - val_loss: 1.4483 - val_acc: 0.5044\n",
      "Epoch 35/40\n",
      "50000/50000 [==============================] - 6s 118us/step - loss: 0.8847 - acc: 0.7098 - val_loss: 1.4500 - val_acc: 0.5028\n",
      "Epoch 36/40\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 0.8837 - acc: 0.7101 - val_loss: 1.4516 - val_acc: 0.5032\n",
      "Epoch 37/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 0.8808 - acc: 0.7111 - val_loss: 1.4529 - val_acc: 0.5023\n",
      "\n",
      "Epoch 00037: ReduceLROnPlateau reducing learning rate to 2.700000040931627e-05.\n",
      "Epoch 38/40\n",
      "50000/50000 [==============================] - 6s 114us/step - loss: 0.8780 - acc: 0.7122 - val_loss: 1.4512 - val_acc: 0.5028\n",
      "Epoch 39/40\n",
      "50000/50000 [==============================] - 6s 115us/step - loss: 0.8749 - acc: 0.7132 - val_loss: 1.4521 - val_acc: 0.5021\n",
      "Epoch 40/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 0.8746 - acc: 0.7123 - val_loss: 1.4517 - val_acc: 0.5032\n",
      "Numbers of exp: 1, reduce_factor: 0.30, reduce_patient: 10\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 3072)              0         \n",
      "_________________________________________________________________\n",
      "hidden_layer1 (Dense)        (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "hidden_layer2 (Dense)        (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "hidden_layer3 (Dense)        (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 1,742,474\n",
      "Trainable params: 1,740,682\n",
      "Non-trainable params: 1,792\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/40\n",
      "50000/50000 [==============================] - 7s 141us/step - loss: 3.0072 - acc: 0.1117 - val_loss: 2.8495 - val_acc: 0.1511\n",
      "Epoch 2/40\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 2.6556 - acc: 0.1666 - val_loss: 2.6346 - val_acc: 0.1883\n",
      "Epoch 3/40\n",
      "50000/50000 [==============================] - 6s 126us/step - loss: 2.4580 - acc: 0.2054 - val_loss: 2.4570 - val_acc: 0.2172\n",
      "Epoch 4/40\n",
      "50000/50000 [==============================] - 6s 129us/step - loss: 2.3417 - acc: 0.2307 - val_loss: 2.3472 - val_acc: 0.2375\n",
      "Epoch 5/40\n",
      "50000/50000 [==============================] - 6s 127us/step - loss: 2.2617 - acc: 0.2483 - val_loss: 2.2763 - val_acc: 0.2530\n",
      "Epoch 6/40\n",
      "50000/50000 [==============================] - 6s 118us/step - loss: 2.2040 - acc: 0.2634 - val_loss: 2.2202 - val_acc: 0.2640\n",
      "Epoch 7/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 2.1552 - acc: 0.2744 - val_loss: 2.1769 - val_acc: 0.2729\n",
      "Epoch 8/40\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 2.1150 - acc: 0.2847 - val_loss: 2.1405 - val_acc: 0.2836\n",
      "Epoch 9/40\n",
      "50000/50000 [==============================] - 6s 115us/step - loss: 2.0812 - acc: 0.2924 - val_loss: 2.1093 - val_acc: 0.2922\n",
      "Epoch 10/40\n",
      "50000/50000 [==============================] - 6s 123us/step - loss: 2.0515 - acc: 0.3011 - val_loss: 2.0801 - val_acc: 0.2984\n",
      "Epoch 11/40\n",
      "50000/50000 [==============================] - 6s 128us/step - loss: 2.0267 - acc: 0.3081 - val_loss: 2.0560 - val_acc: 0.3033\n",
      "Epoch 12/40\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 2.0011 - acc: 0.3140 - val_loss: 2.0362 - val_acc: 0.3084\n",
      "Epoch 13/40\n",
      "50000/50000 [==============================] - 6s 114us/step - loss: 1.9796 - acc: 0.3204 - val_loss: 2.0173 - val_acc: 0.3131\n",
      "Epoch 14/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.9617 - acc: 0.3247 - val_loss: 2.0005 - val_acc: 0.3180\n",
      "Epoch 15/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 1.9449 - acc: 0.3306 - val_loss: 1.9852 - val_acc: 0.3214\n",
      "Epoch 16/40\n",
      "50000/50000 [==============================] - 5s 109us/step - loss: 1.9265 - acc: 0.3351 - val_loss: 1.9708 - val_acc: 0.3265\n",
      "Epoch 17/40\n",
      "50000/50000 [==============================] - 6s 115us/step - loss: 1.9124 - acc: 0.3408 - val_loss: 1.9576 - val_acc: 0.3294\n",
      "Epoch 18/40\n",
      "50000/50000 [==============================] - 6s 114us/step - loss: 1.8971 - acc: 0.3448 - val_loss: 1.9453 - val_acc: 0.3316\n",
      "Epoch 19/40\n",
      "50000/50000 [==============================] - 6s 121us/step - loss: 1.8837 - acc: 0.3484 - val_loss: 1.9335 - val_acc: 0.3339\n",
      "Epoch 20/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.8718 - acc: 0.3522 - val_loss: 1.9236 - val_acc: 0.3359\n",
      "Epoch 21/40\n",
      "50000/50000 [==============================] - 6s 121us/step - loss: 1.8600 - acc: 0.3559 - val_loss: 1.9142 - val_acc: 0.3370\n",
      "Epoch 22/40\n",
      "50000/50000 [==============================] - 6s 114us/step - loss: 1.8489 - acc: 0.3599 - val_loss: 1.9041 - val_acc: 0.3403\n",
      "Epoch 23/40\n",
      "50000/50000 [==============================] - 6s 120us/step - loss: 1.8388 - acc: 0.3625 - val_loss: 1.8954 - val_acc: 0.3429\n",
      "Epoch 24/40\n",
      "50000/50000 [==============================] - 7s 134us/step - loss: 1.8283 - acc: 0.3662 - val_loss: 1.8863 - val_acc: 0.3448\n",
      "Epoch 25/40\n",
      "50000/50000 [==============================] - 7s 143us/step - loss: 1.8193 - acc: 0.3693 - val_loss: 1.8784 - val_acc: 0.3472\n",
      "Epoch 26/40\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 1.8109 - acc: 0.3722 - val_loss: 1.8710 - val_acc: 0.3502\n",
      "Epoch 27/40\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 1.8017 - acc: 0.3749 - val_loss: 1.8640 - val_acc: 0.3512\n",
      "Epoch 28/40\n",
      "50000/50000 [==============================] - 6s 110us/step - loss: 1.7935 - acc: 0.3778 - val_loss: 1.8568 - val_acc: 0.3537\n",
      "Epoch 29/40\n",
      "50000/50000 [==============================] - 6s 113us/step - loss: 1.7847 - acc: 0.3806 - val_loss: 1.8514 - val_acc: 0.3558\n",
      "Epoch 30/40\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 1.7780 - acc: 0.3835 - val_loss: 1.8451 - val_acc: 0.3579\n",
      "Epoch 31/40\n",
      "50000/50000 [==============================] - 6s 118us/step - loss: 1.7702 - acc: 0.3851 - val_loss: 1.8387 - val_acc: 0.3596\n",
      "Epoch 32/40\n",
      "50000/50000 [==============================] - 6s 113us/step - loss: 1.7630 - acc: 0.3880 - val_loss: 1.8331 - val_acc: 0.3600\n",
      "Epoch 33/40\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 1.7567 - acc: 0.3906 - val_loss: 1.8280 - val_acc: 0.3636\n",
      "Epoch 34/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.7494 - acc: 0.3923 - val_loss: 1.8228 - val_acc: 0.3650\n",
      "Epoch 35/40\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 1.7434 - acc: 0.3941 - val_loss: 1.8175 - val_acc: 0.3667\n",
      "Epoch 36/40\n",
      "50000/50000 [==============================] - 6s 122us/step - loss: 1.7362 - acc: 0.3968 - val_loss: 1.8123 - val_acc: 0.3686\n",
      "Epoch 37/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.7305 - acc: 0.3981 - val_loss: 1.8081 - val_acc: 0.3707\n",
      "Epoch 38/40\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 1.7253 - acc: 0.4002 - val_loss: 1.8035 - val_acc: 0.3719\n",
      "Epoch 39/40\n",
      "50000/50000 [==============================] - 6s 116us/step - loss: 1.7199 - acc: 0.4022 - val_loss: 1.7999 - val_acc: 0.3737\n",
      "Epoch 40/40\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 1.7140 - acc: 0.4033 - val_loss: 1.7950 - val_acc: 0.3744\n",
      "Numbers of exp: 2, reduce_factor: 0.30, reduce_patient: 20\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         (None, 3072)              0         \n",
      "_________________________________________________________________\n",
      "hidden_layer1 (Dense)        (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "hidden_layer2 (Dense)        (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "batch_normalization_11 (Batc (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "hidden_layer3 (Dense)        (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "batch_normalization_12 (Batc (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 1,742,474\n",
      "Trainable params: 1,740,682\n",
      "Non-trainable params: 1,792\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/40\n",
      "50000/50000 [==============================] - 7s 133us/step - loss: 2.8666 - acc: 0.1263 - val_loss: 2.7307 - val_acc: 0.1575\n",
      "Epoch 2/40\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 2.5839 - acc: 0.1775 - val_loss: 2.5293 - val_acc: 0.2004\n",
      "Epoch 3/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 2.4200 - acc: 0.2127 - val_loss: 2.3914 - val_acc: 0.2229\n",
      "Epoch 4/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 2.3163 - acc: 0.2346 - val_loss: 2.3038 - val_acc: 0.2408\n",
      "Epoch 5/40\n",
      "50000/50000 [==============================] - 5s 104us/step - loss: 2.2411 - acc: 0.2521 - val_loss: 2.2445 - val_acc: 0.2523\n",
      "Epoch 6/40\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 2.1855 - acc: 0.2656 - val_loss: 2.1965 - val_acc: 0.2631\n",
      "Epoch 7/40\n",
      "50000/50000 [==============================] - 6s 118us/step - loss: 2.1414 - acc: 0.2784 - val_loss: 2.1548 - val_acc: 0.2745\n",
      "Epoch 8/40\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 2.1043 - acc: 0.2868 - val_loss: 2.1202 - val_acc: 0.2836\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/40\n",
      "50000/50000 [==============================] - 6s 115us/step - loss: 2.0715 - acc: 0.2957 - val_loss: 2.0940 - val_acc: 0.2918\n",
      "Epoch 10/40\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 2.0428 - acc: 0.3036 - val_loss: 2.0711 - val_acc: 0.2989\n",
      "Epoch 11/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 2.0182 - acc: 0.3109 - val_loss: 2.0505 - val_acc: 0.3046\n",
      "Epoch 12/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.9973 - acc: 0.3162 - val_loss: 2.0312 - val_acc: 0.3101\n",
      "Epoch 13/40\n",
      "50000/50000 [==============================] - 6s 115us/step - loss: 1.9775 - acc: 0.3209 - val_loss: 2.0140 - val_acc: 0.3131\n",
      "Epoch 14/40\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 1.9594 - acc: 0.3266 - val_loss: 2.0001 - val_acc: 0.3191\n",
      "Epoch 15/40\n",
      "50000/50000 [==============================] - 5s 109us/step - loss: 1.9422 - acc: 0.3317 - val_loss: 1.9859 - val_acc: 0.3238\n",
      "Epoch 16/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9252 - acc: 0.3377 - val_loss: 1.9738 - val_acc: 0.3285\n",
      "Epoch 17/40\n",
      "50000/50000 [==============================] - 5s 109us/step - loss: 1.9113 - acc: 0.3413 - val_loss: 1.9618 - val_acc: 0.3304\n",
      "Epoch 18/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.8967 - acc: 0.3452 - val_loss: 1.9506 - val_acc: 0.3334\n",
      "Epoch 19/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 1.8842 - acc: 0.3505 - val_loss: 1.9393 - val_acc: 0.3384\n",
      "Epoch 20/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.8712 - acc: 0.3542 - val_loss: 1.9286 - val_acc: 0.3391\n",
      "Epoch 21/40\n",
      "50000/50000 [==============================] - 5s 104us/step - loss: 1.8598 - acc: 0.3574 - val_loss: 1.9194 - val_acc: 0.3425\n",
      "Epoch 22/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.8493 - acc: 0.3610 - val_loss: 1.9096 - val_acc: 0.3487\n",
      "Epoch 23/40\n",
      "50000/50000 [==============================] - 6s 113us/step - loss: 1.8389 - acc: 0.3635 - val_loss: 1.9008 - val_acc: 0.3496\n",
      "Epoch 24/40\n",
      "50000/50000 [==============================] - 6s 116us/step - loss: 1.8285 - acc: 0.3666 - val_loss: 1.8925 - val_acc: 0.3506\n",
      "Epoch 25/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.8175 - acc: 0.3702 - val_loss: 1.8845 - val_acc: 0.3525\n",
      "Epoch 26/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.8092 - acc: 0.3736 - val_loss: 1.8769 - val_acc: 0.3545\n",
      "Epoch 27/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.8004 - acc: 0.3764 - val_loss: 1.8697 - val_acc: 0.3546\n",
      "Epoch 28/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 1.7919 - acc: 0.3783 - val_loss: 1.8633 - val_acc: 0.3574\n",
      "Epoch 29/40\n",
      "50000/50000 [==============================] - 6s 116us/step - loss: 1.7841 - acc: 0.3817 - val_loss: 1.8567 - val_acc: 0.3587\n",
      "Epoch 30/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 1.7767 - acc: 0.3831 - val_loss: 1.8501 - val_acc: 0.3613\n",
      "Epoch 31/40\n",
      "50000/50000 [==============================] - 6s 110us/step - loss: 1.7694 - acc: 0.3859 - val_loss: 1.8438 - val_acc: 0.3622\n",
      "Epoch 32/40\n",
      "50000/50000 [==============================] - 6s 124us/step - loss: 1.7618 - acc: 0.3878 - val_loss: 1.8391 - val_acc: 0.3648\n",
      "Epoch 33/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.7551 - acc: 0.3903 - val_loss: 1.8335 - val_acc: 0.3669\n",
      "Epoch 34/40\n",
      "50000/50000 [==============================] - 5s 109us/step - loss: 1.7476 - acc: 0.3937 - val_loss: 1.8278 - val_acc: 0.3684\n",
      "Epoch 35/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 1.7424 - acc: 0.3942 - val_loss: 1.8225 - val_acc: 0.3695\n",
      "Epoch 36/40\n",
      "50000/50000 [==============================] - 6s 124us/step - loss: 1.7356 - acc: 0.3970 - val_loss: 1.8174 - val_acc: 0.3705\n",
      "Epoch 37/40\n",
      "50000/50000 [==============================] - 7s 137us/step - loss: 1.7292 - acc: 0.3985 - val_loss: 1.8127 - val_acc: 0.3707\n",
      "Epoch 38/40\n",
      "50000/50000 [==============================] - 7s 132us/step - loss: 1.7234 - acc: 0.4009 - val_loss: 1.8081 - val_acc: 0.3721\n",
      "Epoch 39/40\n",
      "50000/50000 [==============================] - 7s 135us/step - loss: 1.7174 - acc: 0.4036 - val_loss: 1.8032 - val_acc: 0.3742\n",
      "Epoch 40/40\n",
      "50000/50000 [==============================] - 6s 116us/step - loss: 1.7113 - acc: 0.4060 - val_loss: 1.7979 - val_acc: 0.3755\n",
      "Numbers of exp: 3, reduce_factor: 0.50, reduce_patient: 5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         (None, 3072)              0         \n",
      "_________________________________________________________________\n",
      "hidden_layer1 (Dense)        (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "batch_normalization_13 (Batc (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "hidden_layer2 (Dense)        (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "batch_normalization_14 (Batc (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "hidden_layer3 (Dense)        (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "batch_normalization_15 (Batc (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 1,742,474\n",
      "Trainable params: 1,740,682\n",
      "Non-trainable params: 1,792\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/40\n",
      "50000/50000 [==============================] - 6s 128us/step - loss: 2.9323 - acc: 0.1133 - val_loss: 2.8144 - val_acc: 0.1453\n",
      "Epoch 2/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 2.6435 - acc: 0.1668 - val_loss: 2.6337 - val_acc: 0.1869\n",
      "Epoch 3/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 2.4781 - acc: 0.2065 - val_loss: 2.4821 - val_acc: 0.2181\n",
      "Epoch 4/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 2.3726 - acc: 0.2319 - val_loss: 2.3871 - val_acc: 0.2340\n",
      "Epoch 5/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 2.2978 - acc: 0.2500 - val_loss: 2.3214 - val_acc: 0.2505\n",
      "Epoch 6/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 2.2388 - acc: 0.2643 - val_loss: 2.2658 - val_acc: 0.2644\n",
      "Epoch 7/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 2.1886 - acc: 0.2754 - val_loss: 2.2204 - val_acc: 0.2741\n",
      "Epoch 8/40\n",
      "50000/50000 [==============================] - 5s 104us/step - loss: 2.1457 - acc: 0.2884 - val_loss: 2.1835 - val_acc: 0.2839\n",
      "Epoch 9/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 2.1096 - acc: 0.2977 - val_loss: 2.1509 - val_acc: 0.2930\n",
      "Epoch 10/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 2.0777 - acc: 0.3042 - val_loss: 2.1222 - val_acc: 0.2996\n",
      "Epoch 11/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 2.0491 - acc: 0.3119 - val_loss: 2.0959 - val_acc: 0.3076\n",
      "Epoch 12/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 2.0211 - acc: 0.3188 - val_loss: 2.0723 - val_acc: 0.3156\n",
      "Epoch 13/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9988 - acc: 0.3247 - val_loss: 2.0492 - val_acc: 0.3198\n",
      "Epoch 14/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.9771 - acc: 0.3318 - val_loss: 2.0293 - val_acc: 0.3242\n",
      "Epoch 15/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9576 - acc: 0.3374 - val_loss: 2.0110 - val_acc: 0.3284\n",
      "Epoch 16/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9387 - acc: 0.3407 - val_loss: 1.9944 - val_acc: 0.3300\n",
      "Epoch 17/40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9225 - acc: 0.3447 - val_loss: 1.9799 - val_acc: 0.3342\n",
      "Epoch 18/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9070 - acc: 0.3515 - val_loss: 1.9652 - val_acc: 0.3388\n",
      "Epoch 19/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.8924 - acc: 0.3538 - val_loss: 1.9525 - val_acc: 0.3414\n",
      "Epoch 20/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.8787 - acc: 0.3587 - val_loss: 1.9408 - val_acc: 0.3434\n",
      "Epoch 21/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.8666 - acc: 0.3611 - val_loss: 1.9295 - val_acc: 0.3470\n",
      "Epoch 22/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.8544 - acc: 0.3666 - val_loss: 1.9187 - val_acc: 0.3481\n",
      "Epoch 23/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.8438 - acc: 0.3693 - val_loss: 1.9093 - val_acc: 0.3520\n",
      "Epoch 24/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.8324 - acc: 0.3707 - val_loss: 1.9006 - val_acc: 0.3545\n",
      "Epoch 25/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.8237 - acc: 0.3739 - val_loss: 1.8917 - val_acc: 0.3563\n",
      "Epoch 26/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.8129 - acc: 0.3761 - val_loss: 1.8841 - val_acc: 0.3573\n",
      "Epoch 27/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.8040 - acc: 0.3793 - val_loss: 1.8766 - val_acc: 0.3589\n",
      "Epoch 28/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7953 - acc: 0.3827 - val_loss: 1.8683 - val_acc: 0.3602\n",
      "Epoch 29/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 1.7867 - acc: 0.3849 - val_loss: 1.8605 - val_acc: 0.3616\n",
      "Epoch 30/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.7791 - acc: 0.3872 - val_loss: 1.8545 - val_acc: 0.3620\n",
      "Epoch 31/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7706 - acc: 0.3898 - val_loss: 1.8484 - val_acc: 0.3632\n",
      "Epoch 32/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7641 - acc: 0.3916 - val_loss: 1.8429 - val_acc: 0.3665\n",
      "Epoch 33/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.7572 - acc: 0.3937 - val_loss: 1.8365 - val_acc: 0.3680\n",
      "Epoch 34/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7503 - acc: 0.3939 - val_loss: 1.8309 - val_acc: 0.3700\n",
      "Epoch 35/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7436 - acc: 0.3964 - val_loss: 1.8259 - val_acc: 0.3719\n",
      "Epoch 36/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.7372 - acc: 0.4001 - val_loss: 1.8205 - val_acc: 0.3747\n",
      "Epoch 37/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.7307 - acc: 0.4002 - val_loss: 1.8166 - val_acc: 0.3752\n",
      "Epoch 38/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.7243 - acc: 0.4043 - val_loss: 1.8120 - val_acc: 0.3776\n",
      "Epoch 39/40\n",
      "50000/50000 [==============================] - 5s 109us/step - loss: 1.7188 - acc: 0.4042 - val_loss: 1.8073 - val_acc: 0.3813\n",
      "Epoch 40/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7139 - acc: 0.4075 - val_loss: 1.8032 - val_acc: 0.3811\n",
      "Numbers of exp: 4, reduce_factor: 0.50, reduce_patient: 10\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         (None, 3072)              0         \n",
      "_________________________________________________________________\n",
      "hidden_layer1 (Dense)        (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "batch_normalization_16 (Batc (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "hidden_layer2 (Dense)        (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "batch_normalization_17 (Batc (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "hidden_layer3 (Dense)        (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "batch_normalization_18 (Batc (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 1,742,474\n",
      "Trainable params: 1,740,682\n",
      "Non-trainable params: 1,792\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/40\n",
      "50000/50000 [==============================] - 7s 133us/step - loss: 2.8401 - acc: 0.1245 - val_loss: 2.6761 - val_acc: 0.1736\n",
      "Epoch 2/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 2.5529 - acc: 0.1815 - val_loss: 2.5020 - val_acc: 0.2105\n",
      "Epoch 3/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 2.3994 - acc: 0.2148 - val_loss: 2.3770 - val_acc: 0.2347\n",
      "Epoch 4/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 2.3041 - acc: 0.2378 - val_loss: 2.3011 - val_acc: 0.2500\n",
      "Epoch 5/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 2.2330 - acc: 0.2557 - val_loss: 2.2415 - val_acc: 0.2624\n",
      "Epoch 6/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 2.1786 - acc: 0.2683 - val_loss: 2.1922 - val_acc: 0.2728\n",
      "Epoch 7/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 2.1338 - acc: 0.2797 - val_loss: 2.1526 - val_acc: 0.2835\n",
      "Epoch 8/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 2.0961 - acc: 0.2887 - val_loss: 2.1187 - val_acc: 0.2894\n",
      "Epoch 9/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 2.0639 - acc: 0.2997 - val_loss: 2.0891 - val_acc: 0.2956\n",
      "Epoch 10/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 2.0344 - acc: 0.3074 - val_loss: 2.0656 - val_acc: 0.2994\n",
      "Epoch 11/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 2.0096 - acc: 0.3120 - val_loss: 2.0427 - val_acc: 0.3049\n",
      "Epoch 12/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9869 - acc: 0.3201 - val_loss: 2.0215 - val_acc: 0.3144\n",
      "Epoch 13/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9665 - acc: 0.3261 - val_loss: 2.0041 - val_acc: 0.3169\n",
      "Epoch 14/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.9485 - acc: 0.3307 - val_loss: 1.9862 - val_acc: 0.3214\n",
      "Epoch 15/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9300 - acc: 0.3357 - val_loss: 1.9710 - val_acc: 0.3247\n",
      "Epoch 16/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.9151 - acc: 0.3413 - val_loss: 1.9566 - val_acc: 0.3305\n",
      "Epoch 17/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.8996 - acc: 0.3459 - val_loss: 1.9446 - val_acc: 0.3339\n",
      "Epoch 18/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.8861 - acc: 0.3504 - val_loss: 1.9337 - val_acc: 0.3368\n",
      "Epoch 19/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.8721 - acc: 0.3534 - val_loss: 1.9216 - val_acc: 0.3426\n",
      "Epoch 20/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.8605 - acc: 0.3569 - val_loss: 1.9125 - val_acc: 0.3466\n",
      "Epoch 21/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.8484 - acc: 0.3608 - val_loss: 1.9034 - val_acc: 0.3501\n",
      "Epoch 22/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.8370 - acc: 0.3646 - val_loss: 1.8938 - val_acc: 0.3503\n",
      "Epoch 23/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.8263 - acc: 0.3679 - val_loss: 1.8855 - val_acc: 0.3529\n",
      "Epoch 24/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.8164 - acc: 0.3709 - val_loss: 1.8770 - val_acc: 0.3568\n",
      "Epoch 25/40\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.8067 - acc: 0.3743 - val_loss: 1.8692 - val_acc: 0.3586\n",
      "Epoch 26/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.7971 - acc: 0.3781 - val_loss: 1.8612 - val_acc: 0.3603\n",
      "Epoch 27/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7891 - acc: 0.3804 - val_loss: 1.8546 - val_acc: 0.3629\n",
      "Epoch 28/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.7804 - acc: 0.3833 - val_loss: 1.8481 - val_acc: 0.3651\n",
      "Epoch 29/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.7726 - acc: 0.3852 - val_loss: 1.8418 - val_acc: 0.3650\n",
      "Epoch 30/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7643 - acc: 0.3881 - val_loss: 1.8362 - val_acc: 0.3683\n",
      "Epoch 31/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7564 - acc: 0.3916 - val_loss: 1.8301 - val_acc: 0.3704\n",
      "Epoch 32/40\n",
      "50000/50000 [==============================] - 5s 106us/step - loss: 1.7500 - acc: 0.3928 - val_loss: 1.8249 - val_acc: 0.3727\n",
      "Epoch 33/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.7423 - acc: 0.3956 - val_loss: 1.8191 - val_acc: 0.3743\n",
      "Epoch 34/40\n",
      "50000/50000 [==============================] - 6s 127us/step - loss: 1.7353 - acc: 0.3964 - val_loss: 1.8144 - val_acc: 0.3762\n",
      "Epoch 35/40\n",
      "50000/50000 [==============================] - 6s 129us/step - loss: 1.7288 - acc: 0.4002 - val_loss: 1.8093 - val_acc: 0.3764\n",
      "Epoch 36/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.7231 - acc: 0.4019 - val_loss: 1.8048 - val_acc: 0.3777\n",
      "Epoch 37/40\n",
      "50000/50000 [==============================] - 6s 122us/step - loss: 1.7174 - acc: 0.4038 - val_loss: 1.7999 - val_acc: 0.3804\n",
      "Epoch 38/40\n",
      "50000/50000 [==============================] - 6s 113us/step - loss: 1.7110 - acc: 0.4073 - val_loss: 1.7955 - val_acc: 0.3812\n",
      "Epoch 39/40\n",
      "50000/50000 [==============================] - 6s 119us/step - loss: 1.7052 - acc: 0.4078 - val_loss: 1.7911 - val_acc: 0.3819\n",
      "Epoch 40/40\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.6994 - acc: 0.4114 - val_loss: 1.7873 - val_acc: 0.3825\n",
      "Numbers of exp: 5, reduce_factor: 0.50, reduce_patient: 20\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_7 (InputLayer)         (None, 3072)              0         \n",
      "_________________________________________________________________\n",
      "hidden_layer1 (Dense)        (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "batch_normalization_19 (Batc (None, 512)               2048      \n",
      "_________________________________________________________________\n",
      "hidden_layer2 (Dense)        (None, 256)               131328    \n",
      "_________________________________________________________________\n",
      "batch_normalization_20 (Batc (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "hidden_layer3 (Dense)        (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "batch_normalization_21 (Batc (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "output (Dense)               (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 1,742,474\n",
      "Trainable params: 1,740,682\n",
      "Non-trainable params: 1,792\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/40\n",
      "50000/50000 [==============================] - 7s 132us/step - loss: 2.8659 - acc: 0.1161 - val_loss: 2.6860 - val_acc: 0.1547\n",
      "Epoch 2/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 2.5903 - acc: 0.1642 - val_loss: 2.5016 - val_acc: 0.1990\n",
      "Epoch 3/40\n",
      "50000/50000 [==============================] - 5s 109us/step - loss: 2.4132 - acc: 0.2009 - val_loss: 2.3949 - val_acc: 0.2214\n",
      "Epoch 4/40\n",
      "50000/50000 [==============================] - 5s 109us/step - loss: 2.3090 - acc: 0.2278 - val_loss: 2.3109 - val_acc: 0.2388\n",
      "Epoch 5/40\n",
      "50000/50000 [==============================] - 6s 114us/step - loss: 2.2382 - acc: 0.2459 - val_loss: 2.2497 - val_acc: 0.2525\n",
      "Epoch 6/40\n",
      "50000/50000 [==============================] - 6s 115us/step - loss: 2.1819 - acc: 0.2627 - val_loss: 2.2007 - val_acc: 0.2659\n",
      "Epoch 7/40\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 2.1380 - acc: 0.2744 - val_loss: 2.1595 - val_acc: 0.2766\n",
      "Epoch 8/40\n",
      "50000/50000 [==============================] - 6s 122us/step - loss: 2.0992 - acc: 0.2860 - val_loss: 2.1254 - val_acc: 0.2855\n",
      "Epoch 9/40\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 2.0676 - acc: 0.2942 - val_loss: 2.0949 - val_acc: 0.2924\n",
      "Epoch 10/40\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 2.0387 - acc: 0.3022 - val_loss: 2.0700 - val_acc: 0.3014\n",
      "Epoch 11/40\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 2.0132 - acc: 0.3094 - val_loss: 2.0472 - val_acc: 0.3081\n",
      "Epoch 12/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.9891 - acc: 0.3175 - val_loss: 2.0258 - val_acc: 0.3146\n",
      "Epoch 13/40\n",
      "50000/50000 [==============================] - 6s 113us/step - loss: 1.9700 - acc: 0.3214 - val_loss: 2.0068 - val_acc: 0.3190\n",
      "Epoch 14/40\n",
      "50000/50000 [==============================] - 7s 149us/step - loss: 1.9503 - acc: 0.3281 - val_loss: 1.9903 - val_acc: 0.3257\n",
      "Epoch 15/40\n",
      "50000/50000 [==============================] - 8s 170us/step - loss: 1.9334 - acc: 0.3332 - val_loss: 1.9739 - val_acc: 0.3283\n",
      "Epoch 16/40\n",
      "50000/50000 [==============================] - 6s 117us/step - loss: 1.9180 - acc: 0.3378 - val_loss: 1.9608 - val_acc: 0.3316\n",
      "Epoch 17/40\n",
      "50000/50000 [==============================] - 6s 118us/step - loss: 1.9032 - acc: 0.3425 - val_loss: 1.9471 - val_acc: 0.3347\n",
      "Epoch 18/40\n",
      "50000/50000 [==============================] - 6s 121us/step - loss: 1.8890 - acc: 0.3462 - val_loss: 1.9348 - val_acc: 0.3365\n",
      "Epoch 19/40\n",
      "50000/50000 [==============================] - 6s 121us/step - loss: 1.8767 - acc: 0.3497 - val_loss: 1.9241 - val_acc: 0.3422\n",
      "Epoch 20/40\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.8638 - acc: 0.3549 - val_loss: 1.9133 - val_acc: 0.3425\n",
      "Epoch 21/40\n",
      "50000/50000 [==============================] - 6s 114us/step - loss: 1.8527 - acc: 0.3573 - val_loss: 1.9034 - val_acc: 0.3459\n",
      "Epoch 22/40\n",
      "50000/50000 [==============================] - 8s 159us/step - loss: 1.8415 - acc: 0.3616 - val_loss: 1.8934 - val_acc: 0.3499\n",
      "Epoch 23/40\n",
      "50000/50000 [==============================] - 8s 163us/step - loss: 1.8317 - acc: 0.3639 - val_loss: 1.8839 - val_acc: 0.3532\n",
      "Epoch 24/40\n",
      "50000/50000 [==============================] - 9s 170us/step - loss: 1.8210 - acc: 0.3678 - val_loss: 1.8754 - val_acc: 0.3549: 3s \n",
      "Epoch 25/40\n",
      "50000/50000 [==============================] - 9s 181us/step - loss: 1.8119 - acc: 0.3711 - val_loss: 1.8673 - val_acc: 0.3587\n",
      "Epoch 26/40\n",
      "50000/50000 [==============================] - 9s 179us/step - loss: 1.8033 - acc: 0.3737 - val_loss: 1.8595 - val_acc: 0.3606\n",
      "Epoch 27/40\n",
      "50000/50000 [==============================] - 9s 182us/step - loss: 1.7938 - acc: 0.3771 - val_loss: 1.8527 - val_acc: 0.3624\n",
      "Epoch 28/40\n",
      "50000/50000 [==============================] - 9s 180us/step - loss: 1.7860 - acc: 0.3792 - val_loss: 1.8462 - val_acc: 0.3645\n",
      "Epoch 29/40\n",
      "50000/50000 [==============================] - 10s 193us/step - loss: 1.7782 - acc: 0.3808 - val_loss: 1.8399 - val_acc: 0.3668\n",
      "Epoch 30/40\n",
      "50000/50000 [==============================] - 9s 173us/step - loss: 1.7697 - acc: 0.3846 - val_loss: 1.8325 - val_acc: 0.3672\n",
      "Epoch 31/40\n",
      "50000/50000 [==============================] - 11s 211us/step - loss: 1.7626 - acc: 0.3855 - val_loss: 1.8268 - val_acc: 0.3703\n",
      "Epoch 32/40\n",
      " 3072/50000 [>.............................] - ETA: 6s - loss: 1.7348 - acc: 0.4004"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "results = {}\n",
    "for i, (optim, reduce_factor, reduce_patient) in enumerate(itertools.product(optimizer_set, reduce_lr_factor, redice_lr_patient)):\n",
    "    print(\"Numbers of exp: %i, reduce_factor: %.2f, reduce_patient: %i\" % (i, reduce_factor, reduce_patient))\n",
    "    model = build_mlp(input_shape=x_train.shape[1:])\n",
    "    model.summary()\n",
    "    model.compile(loss=\"categorical_crossentropy\", metrics=[\"accuracy\"], optimizer=optim)\n",
    "    \n",
    "    \"\"\"Code Here\n",
    "    設定 reduce learning rate 的 callback function\n",
    "    \"\"\"\n",
    "    reduce_lr = ReduceLROnPlateau(factor=reduce_factor, \n",
    "                              min_lr=1e-12, \n",
    "                              monitor='val_loss', \n",
    "                              patience=reduce_patient, \n",
    "                              verbose=1)\n",
    "    \n",
    "    model.fit(x_train, y_train, \n",
    "              epochs=EPOCHS, \n",
    "              batch_size=BATCH_SIZE, \n",
    "              validation_data=(x_test, y_test), \n",
    "              shuffle=True,\n",
    "              callbacks=[reduce_lr]\n",
    "             )\n",
    "\n",
    "    # Collect results\n",
    "    exp_name_tag = (\"exp-%s\" % (i))\n",
    "    results[exp_name_tag] = {'train-loss': model.history.history[\"loss\"],\n",
    "                             'valid-loss': model.history.history[\"val_loss\"],\n",
    "                             'train-acc': model.history.history[\"acc\"],\n",
    "                             'valid-acc': model.history.history[\"val_acc\"]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as mplcm\n",
    "import matplotlib.colors as colors\n",
    "%matplotlib inline\n",
    "NUM_COLORS = len(results.keys())\n",
    "\n",
    "cm = plt.get_cmap('gist_rainbow')\n",
    "cNorm  = colors.Normalize(vmin=0, vmax=NUM_COLORS-1)\n",
    "scalarMap = mplcm.ScalarMappable(norm=cNorm, cmap=cm)\n",
    "color_bar = [scalarMap.to_rgba(i) for i in range(NUM_COLORS)]\n",
    "\n",
    "plt.figure(figsize=(8,6))\n",
    "for i, cond in enumerate(results.keys()):\n",
    "    plt.plot(range(len(results[cond]['train-loss'])),results[cond]['train-loss'], '-', label=cond, color=color_bar[i])\n",
    "    plt.plot(range(len(results[cond]['valid-loss'])),results[cond]['valid-loss'], '--', label=cond, color=color_bar[i])\n",
    "plt.title(\"Loss\")\n",
    "plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))\n",
    "plt.show()\n",
    "\n",
    "plt.figure(figsize=(8,6))\n",
    "for i, cond in enumerate(results.keys()):\n",
    "    plt.plot(range(len(results[cond]['train-acc'])),results[cond]['train-acc'], '-', label=cond, color=color_bar[i])\n",
    "    plt.plot(range(len(results[cond]['valid-acc'])),results[cond]['valid-acc'], '--', label=cond, color=color_bar[i])\n",
    "plt.title(\"Accuracy\")\n",
    "plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}