Day097_Keras_CNN_vs_DNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "c:\\users\\henry\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.datasets import cifar10\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "batch_size = 128 # batch 的大小，如果出現 OOM error，請降低這個值\n",
    "num_classes = 10 # 類別的數量，Cifar 10 共有 10 個類別\n",
    "epochs = 10 # 訓練的 epochs 數量\n",
    "\n",
    "# 讀取資料並檢視\n",
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "\n",
    "# 對 label 進行 one-hot encoding (y_trian 原本是純數字)\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 首先我們使用一般的 DNN (MLP) 來訓練\n",
    "由於 DNN 只能輸入一維的資料，我們要先將影像進行攤平，若 (50000, 32, 32, 3) 的影像，攤平後會變成 (50000, 32*32*3) = (50000, 3072)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "# 將資料攤平成一維資料\n",
    "x_train = x_train.reshape(50000, 3072) \n",
    "x_test = x_test.reshape(10000, 3072)\n",
    "\n",
    "# 將資料變為 float32 並標準化\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_10 (Dense)             (None, 512)               1573376   \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 512)               262656    \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 1,841,162\n",
      "Trainable params: 1,841,162\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 12s 243us/step - loss: 2.1960 - acc: 0.1918 - val_loss: 1.9509 - val_acc: 0.3026\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 12s 230us/step - loss: 2.0202 - acc: 0.2502 - val_loss: 1.9856 - val_acc: 0.2811\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 12s 233us/step - loss: 1.9700 - acc: 0.2717 - val_loss: 1.9194 - val_acc: 0.3082\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 11s 228us/step - loss: 1.9449 - acc: 0.2818 - val_loss: 1.9070 - val_acc: 0.3368\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 11s 227us/step - loss: 1.9203 - acc: 0.2937 - val_loss: 1.8797 - val_acc: 0.3522\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 12s 234us/step - loss: 1.9048 - acc: 0.2991 - val_loss: 1.9082 - val_acc: 0.3142\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 12s 244us/step - loss: 1.8928 - acc: 0.3045 - val_loss: 1.8916 - val_acc: 0.3542\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 12s 237us/step - loss: 1.8877 - acc: 0.3092 - val_loss: 1.8861 - val_acc: 0.3472\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 11s 226us/step - loss: 1.8765 - acc: 0.3115 - val_loss: 1.8905 - val_acc: 0.3420\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 11s 227us/step - loss: 1.8671 - acc: 0.3158 - val_loss: 1.9062 - val_acc: 0.3395\n",
      "Test loss: 1.9061515918731688\n",
      "Test accuracy: 0.3395\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(512, activation='relu', input_shape=(3072,)))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(512, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=epochs,\n",
    "                    verbose=1,\n",
    "                    validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 接下來我們使用 CNN 來訓練神經網路\n",
    "CNN 的原理非常適合處理影像類的資料，就讓我們來看看，同樣的訓練條件，CNN 是否顯著優於 DNN 呢?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train shape: (50000, 32, 32, 3)\n",
      "50000 train samples\n",
      "10000 test samples\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
    "print('x_train shape:', x_train.shape)\n",
    "print(x_train.shape[0], 'train samples')\n",
    "print(x_test.shape[0], 'test samples')\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "# Convert class vectors to binary class matrices.\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 30, 30, 32)        9248      \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 30, 30, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 15, 15, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 15, 15, 64)        18496     \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 15, 15, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 13, 13, 64)        36928     \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 13, 13, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 6, 6, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 2304)              0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 512)               1180160   \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 10)                5130      \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 1,250,858\n",
      "Trainable params: 1,250,858\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 50000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "50000/50000 [==============================] - 141s 3ms/step - loss: 2.0560 - acc: 0.2567 - val_loss: 1.6529 - val_acc: 0.4150\n",
      "Epoch 2/10\n",
      "50000/50000 [==============================] - 160s 3ms/step - loss: 1.5954 - acc: 0.4250 - val_loss: 1.5783 - val_acc: 0.4484\n",
      "Epoch 3/10\n",
      "50000/50000 [==============================] - 200s 4ms/step - loss: 1.4270 - acc: 0.4927 - val_loss: 1.3011 - val_acc: 0.5481\n",
      "Epoch 4/10\n",
      "50000/50000 [==============================] - 147s 3ms/step - loss: 1.3644 - acc: 0.5196 - val_loss: 1.2492 - val_acc: 0.5627\n",
      "Epoch 5/10\n",
      "50000/50000 [==============================] - 141s 3ms/step - loss: 1.3108 - acc: 0.5410 - val_loss: 1.4056 - val_acc: 0.5286\n",
      "Epoch 6/10\n",
      "50000/50000 [==============================] - 140s 3ms/step - loss: 1.2800 - acc: 0.5515 - val_loss: 1.3474 - val_acc: 0.5273\n",
      "Epoch 7/10\n",
      "50000/50000 [==============================] - 139s 3ms/step - loss: 1.2656 - acc: 0.5599 - val_loss: 1.3161 - val_acc: 0.5490\n",
      "Epoch 8/10\n",
      "50000/50000 [==============================] - 166s 3ms/step - loss: 1.2636 - acc: 0.5650 - val_loss: 1.1877 - val_acc: 0.5875\n",
      "Epoch 9/10\n",
      "50000/50000 [==============================] - 202s 4ms/step - loss: 1.2488 - acc: 0.5700 - val_loss: 1.1060 - val_acc: 0.6255\n",
      "Epoch 10/10\n",
      "50000/50000 [==============================] - 240s 5ms/step - loss: 1.2472 - acc: 0.5702 - val_loss: 1.3701 - val_acc: 0.5646\n",
      "Test loss: 1.3700920370101928\n",
      "Test accuracy: 0.5646\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(32, (3, 3), padding='same',\n",
    "                 input_shape=x_train.shape[1:]))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(32, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "model.add(Conv2D(64, (3, 3), padding='same'))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Conv2D(64, (3, 3)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.5))\n",
    "\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(num_classes))\n",
    "model.add(Activation('softmax'))\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(0.003),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=batch_size,\n",
    "                    epochs=epochs,\n",
    "                    verbose=1,\n",
    "                    validation_data=(x_test, y_test))\n",
    "score = model.evaluate(x_test, y_test, verbose=0)\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 同樣運算 10 個 epochs，但 CNN 在 test data 的準確率顯著優於 DNN!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 作業\n",
    "1. 請試著調整各個超參數，並說明那些超參數對於結果有明顯的影響?\n",
    "2. CNN 與 DNN 哪個模型的參數數量比較多? 造成參數的數量不同的原因在哪?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- CNN 一個kernel上有許多的權重但都是共用的，可以有效減少權重的濫用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}