Day_097_Keras_CNN_vs_DNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128 # batch 的大小，如果出現 OOM error，請降低這個值\n",
    "num_classes = 10 # 類別的數量，Cifar 10 共有 10 個類別\n",
    "epochs = 10 # 訓練的 epochs 數量\n",
    "\n",
    "# 讀取資料並檢視\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# 對 label 進行 one-hot encoding (y_trian 原本是純數字)\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 首先我們使用一般的 DNN (MLP) 來訓練\n",
    "由於 DNN 只能輸入一維的資料，我們要先將影像進行攤平，若 (50000, 32, 32, 3) 的影像，攤平後會變成 (50000, 32*32*3) = (50000, 3072)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "# 將資料攤平成一維資料\n",
    "x_train = x_train.reshape(50000, 3072) \n",
    "x_test = x_test.reshape(10000, 3072)\n",
    "\n",
    "# 將資料變為 float32 並標準化\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:68: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:504: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3828: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:126: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3135: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 1,841,162\n",
      "Trainable params: 1,841,162\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:From C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\keras\\optimizers.py:744: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3005: The name tf.log is deprecated. Please use tf.math.log instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 2.1897 - acc: 0.2513 - val_loss: 1.8012 - val_acc: 0.3598\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 2s 44us/step - loss: 1.8452 - acc: 0.3341 - val_loss: 1.8543 - val_acc: 0.3277\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 2s 42us/step - loss: 1.7770 - acc: 0.3625 - val_loss: 1.7014 - val_acc: 0.3948\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 2s 42us/step - loss: 1.7303 - acc: 0.3787 - val_loss: 1.6534 - val_acc: 0.4099\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 2s 43us/step - loss: 1.6858 - acc: 0.3969 - val_loss: 1.6475 - val_acc: 0.4049\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 2s 42us/step - loss: 1.6614 - acc: 0.4039 - val_loss: 1.5802 - val_acc: 0.4432\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 2s 42us/step - loss: 1.6359 - acc: 0.4133 - val_loss: 1.5892 - val_acc: 0.4284\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 2s 43us/step - loss: 1.6186 - acc: 0.4207 - val_loss: 1.5693 - val_acc: 0.4439\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 2s 41us/step - loss: 1.6065 - acc: 0.4241 - val_loss: 1.5681 - val_acc: 0.4483\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 2s 42us/step - loss: 1.5834 - acc: 0.4336 - val_loss: 1.5352 - val_acc: 0.4584\n",
      "Test loss: 1.5352456148147584\n",
      "Test accuracy: 0.4584\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(512, activation='relu', input_shape=(3072,)))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(512, activation='relu'))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=epochs,\n",
    "                    verbose=1,\n",
    "                    validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 接下來我們使用 CNN 來訓練神經網路\n",
    "CNN 的原理非常適合處理影像類的資料，就讓我們來看看，同樣的訓練條件，CNN 是否顯著優於 DNN 呢?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\cerbe\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3652: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 30, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 15, 15, 64)        18496     \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 15, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 13, 13, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 13, 13, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 512)               1180160   \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                5130      \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 1,250,858\n",
      "Trainable params: 1,250,858\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 9s 185us/step - loss: 1.7147 - acc: 0.3829 - val_loss: 1.3087 - val_acc: 0.5328\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 7s 133us/step - loss: 1.2805 - acc: 0.5460 - val_loss: 1.0902 - val_acc: 0.6135\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 7s 133us/step - loss: 1.0737 - acc: 0.6234 - val_loss: 0.9196 - val_acc: 0.6822\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 7s 135us/step - loss: 0.9483 - acc: 0.6682 - val_loss: 0.8813 - val_acc: 0.6960\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 7s 136us/step - loss: 0.8596 - acc: 0.7006 - val_loss: 0.9571 - val_acc: 0.6719\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 7s 133us/step - loss: 0.7960 - acc: 0.7234 - val_loss: 0.7896 - val_acc: 0.7300\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 7s 134us/step - loss: 0.7490 - acc: 0.7411 - val_loss: 0.8079 - val_acc: 0.7206\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 7s 137us/step - loss: 0.7099 - acc: 0.7544 - val_loss: 0.7086 - val_acc: 0.7598\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 7s 134us/step - loss: 0.6734 - acc: 0.7672 - val_loss: 0.6824 - val_acc: 0.7685\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 7s 134us/step - loss: 0.6525 - acc: 0.7744 - val_loss: 0.7256 - val_acc: 0.7524\n",
      "Test loss: 0.7256334943771362\n",
      "Test accuracy: 0.7524\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3, 3), padding='same',\n",
    "                 input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(32, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(64, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=epochs,\n",
    "                    verbose=1,\n",
    "                    validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 同樣運算 10 個 epochs，但 CNN 在 test data 的準確率顯著優於 DNN!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 作業\n",
    "1. 請試著調整各個超參數，並說明那些超參數對於結果有明顯的影響?\n",
    "2. CNN 與 DNN 哪個模型的參數數量比較多? 造成參數的數量不同的原因在哪?\n",
    "1,841,162是DNN參數數量，1,250,858是CNN參數數量，因為DNN會被迫將原本2D或是3D的部分攤平成一維才可以進行分析。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_13 (Conv2D)           (None, 32, 32, 32)        1568      \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "activation_20 (Activation)   (None, 30, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_7 (MaxPooling2 (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_12 (Dropout)         (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, 15, 15, 64)        18496     \n",
      "_________________________________________________________________\n",
      "activation_21 (Activation)   (None, 15, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_16 (Conv2D)           (None, 13, 13, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_22 (Activation)   (None, 13, 13, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_8 (MaxPooling2 (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_13 (Dropout)         (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_4 (Flatten)          (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 512)               1180160   \n",
      "_________________________________________________________________\n",
      "activation_23 (Activation)   (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 10)                5130      \n",
      "_________________________________________________________________\n",
      "activation_24 (Activation)   (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 1,251,530\n",
      "Trainable params: 1,251,530\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 6s 114us/step - loss: 2.0428 - acc: 0.2657 - val_loss: 1.7637 - val_acc: 0.3747\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 5s 104us/step - loss: 1.7134 - acc: 0.3852 - val_loss: 1.6927 - val_acc: 0.4328\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.5450 - acc: 0.4454 - val_loss: 1.4846 - val_acc: 0.4792\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 5s 107us/step - loss: 1.4286 - acc: 0.4895 - val_loss: 1.2883 - val_acc: 0.5350\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 5s 105us/step - loss: 1.3330 - acc: 0.5228 - val_loss: 1.2304 - val_acc: 0.5585\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 5s 110us/step - loss: 1.2492 - acc: 0.5580 - val_loss: 1.1268 - val_acc: 0.5951\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.1770 - acc: 0.5828 - val_loss: 1.2349 - val_acc: 0.5708\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 6s 111us/step - loss: 1.1247 - acc: 0.6024 - val_loss: 1.1116 - val_acc: 0.6075\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 6s 112us/step - loss: 1.0587 - acc: 0.6247 - val_loss: 0.9137 - val_acc: 0.6766\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 5s 108us/step - loss: 1.0263 - acc: 0.6388 - val_loss: 0.8880 - val_acc: 0.6919\n",
      "Test loss: 0.8880370895385742\n",
      "Test accuracy: 0.6919\n"
     ]
    }
   ],
   "source": [
    "batch_size =512 # batch 的大小，如果出現 OOM error，請降低這個值\n",
    "num_classes = 10 # 類別的數量，Cifar 10 共有 10 個類別\n",
    "epochs = 10 # 訓練的 epochs 數量\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Conv2D(32, (4, 4), padding='same',\n",
    "                 input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(32, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "\n",
    "model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(64, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=epochs,\n",
    "                    verbose=1,\n",
    "                    validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. 提升dropout的比例--> 準確度下降\n",
    "2. 放大filter的大小便成4*4 準確度也有機會下降"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}