dropout_from_scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 丢弃法的概念\n",
    "\n",
    "在现代神经网络中，我们所指的丢弃法，通常是对输入层或者隐含层做以下操作：\n",
    "\n",
    "* 随机选择一部分该层的输出作为丢弃元素；\n",
    "* 把丢弃元素乘以0；\n",
    "* 把非丢弃元素拉伸。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 丢弃法的实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-14T19:00:58.525980Z",
     "start_time": "2018-03-14T19:00:57.353795Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import mxnet as mx\n",
    "from mxnet import nd\n",
    "from mxnet import gluon\n",
    "from mxnet import autograd\n",
    "from utils import load_data_fashion_mnist, accuracy, evaluate_accuracy, SGD\n",
    "\n",
    "# 设置随机种子\n",
    "mx.random.seed(2)\n",
    "\n",
    "# 实现dropout\n",
    "def dropout(X, drop_probability):\n",
    "    # 计算保留数据的比例\n",
    "    keep_probability = 1 - drop_probability\n",
    "    # 确保drop_probability的输入合法\n",
    "    assert 0 <= keep_probability <= 1\n",
    "    # 丢弃所有元素\n",
    "    if keep_probability == 0:\n",
    "        return X.zeros_like()\n",
    "    # 随机生成一个相同纬度的矩阵, 根据随机值和keep_probability的对比确定是否丢弃该元素\n",
    "    mask = nd.random.uniform(0, 1.0, X.shape, ctx=X.context) < keep_probability\n",
    "    # 保证 E[dropout(X)] == X, 对剩下的数据进行缩放\n",
    "    scale = 1 / keep_probability\n",
    "    return mask * X * scale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-14T19:00:58.536105Z",
     "start_time": "2018-03-14T19:00:58.527911Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[[  0.   1.   2.   3.]\n",
       " [  4.   5.   6.   7.]\n",
       " [  8.   9.  10.  11.]\n",
       " [ 12.  13.  14.  15.]\n",
       " [ 16.  17.  18.  19.]]\n",
       "<NDArray 5x4 @cpu(0)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试dropout\n",
    "A = nd.arange(20).reshape((5,4))\n",
    "dropout(A, 0.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-14T19:00:58.543216Z",
     "start_time": "2018-03-14T19:00:58.537770Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[[ 0.  0.  0.  0.]\n",
       " [ 0.  0.  0.  0.]\n",
       " [ 0.  0.  0.  0.]\n",
       " [ 0.  0.  0.  0.]\n",
       " [ 0.  0.  0.  0.]]\n",
       "<NDArray 5x4 @cpu(0)>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dropout(A, 1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-14T19:00:58.550498Z",
     "start_time": "2018-03-14T19:00:58.544891Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\n",
       "[[  0.   2.   4.   0.]\n",
       " [  8.   0.  12.   0.]\n",
       " [ 16.  18.   0.   0.]\n",
       " [  0.   0.   0.  30.]\n",
       " [  0.  34.  36.   0.]]\n",
       "<NDArray 5x4 @cpu(0)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dropout(A, 0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-14T18:45:50.346765Z",
     "start_time": "2018-03-14T18:45:50.341096Z"
    }
   },
   "source": [
    "## 丢弃法的本质\n",
    "\n",
    "一般来说，在集成学习里，我们可以对训练数据集有放回地采样若干次并分别训练若干个不同的分类器；测试时，把这些分类器的结果集成一下作为最终分类结果。事实上，丢弃法在模拟集成学习。丢弃法实质上是对每一个这样的数据集分别训练一个原神经网络子集的分类器。与一般的集成学习不同，这里每个原神经网络子集的分类器用的是同一套参数。因此丢弃法只是在模拟集成学习。使用丢弃法的神经网络实质上是对输入层和隐含层的参数做了正则化：学到的参数使得原神经网络不同子集在训练数据上都尽可能表现良好。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据获取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-14T19:00:59.647771Z",
     "start_time": "2018-03-14T19:00:58.552220Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 批数据大小\n",
    "batch_size = 256\n",
    "\n",
    "# 加载数据\n",
    "train_data, test_data = load_data_fashion_mnist(batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 含两个隐藏层的多层感知机"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-14T19:00:59.667827Z",
     "start_time": "2018-03-14T19:00:59.650044Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 模型输入大小\n",
    "num_inputs = 28 * 28\n",
    "\n",
    "# 模型输出大小\n",
    "num_outputs = 10\n",
    "\n",
    "# 第一个隐藏层节点数量\n",
    "num_hidden1 = 256\n",
    "\n",
    "# 第二个隐藏层节点数量\n",
    "num_hidden2 = 256\n",
    "\n",
    "# 随机数据时的标准差\n",
    "weight_scale = 0.01\n",
    "\n",
    "# 第一个隐藏层权重\n",
    "W1 = nd.random_normal(shape=(num_inputs, num_hidden1), scale=weight_scale)\n",
    "# 第一个隐藏层偏置\n",
    "b1 = nd.zeros(num_hidden1)\n",
    "\n",
    "# 第二个隐藏层权重\n",
    "W2 = nd.random_normal(shape=(num_hidden1, num_hidden2), scale=weight_scale)\n",
    "# 第二个隐藏层偏置\n",
    "b2 = nd.zeros(num_hidden2)\n",
    "\n",
    "# 输出层权重\n",
    "W3 = nd.random_normal(shape=(num_hidden2, num_outputs), scale=weight_scale)\n",
    "# 输出层偏置\n",
    "b3 = nd.zeros(num_outputs)\n",
    "\n",
    "# 参数数组\n",
    "params = [W1, b1, W2, b2, W3, b3]\n",
    "\n",
    "# 需要计算梯度, 添加自动求导\n",
    "for param in params:\n",
    "    param.attach_grad()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义包含丢弃层的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-14T19:00:59.678013Z",
     "start_time": "2018-03-14T19:00:59.669321Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 第一个隐藏层的丢弃概率\n",
    "drop_prob1 = 0.2\n",
    "# 第二个隐藏层的丢弃概率\n",
    "drop_prob2 = 0.5\n",
    "\n",
    "# 定义网络\n",
    "def net(X):\n",
    "    X = X.reshape((-1, num_inputs))\n",
    "    # 第一层全连接\n",
    "    h1 = nd.relu(nd.dot(X, W1) + b1)\n",
    "    # 在第一层全连接后添加丢弃层\n",
    "    h1 = dropout(h1, drop_prob1)\n",
    "    # 第二层全连接\n",
    "    h2 = nd.relu(nd.dot(h1, W2) + b2)\n",
    "    # 在第二层全连接后添加丢弃层\n",
    "    h2 = dropout(h2, drop_prob2)\n",
    "    # 返回输出\n",
    "    return nd.dot(h2, W3) + b3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-03-14T19:01:18.009627Z",
     "start_time": "2018-03-14T19:00:59.680273Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0. Loss: 1.221062, Train acc 0.528746, Test acc 0.754006\n",
      "Epoch 1. Loss: 0.598503, Train acc 0.774890, Test acc 0.813101\n",
      "Epoch 2. Loss: 0.499490, Train acc 0.818493, Test acc 0.840244\n",
      "Epoch 3. Loss: 0.457343, Train acc 0.832699, Test acc 0.835036\n",
      "Epoch 4. Loss: 0.426575, Train acc 0.846070, Test acc 0.849159\n"
     ]
    }
   ],
   "source": [
    "# 定义交叉熵损失\n",
    "softmax_cross_entropy = gluon.loss.SoftmaxCrossEntropyLoss()\n",
    "\n",
    "# 定义学习率\n",
    "learning_rate = 0.5\n",
    "\n",
    "# 训练\n",
    "for epoch in range(5):\n",
    "    # 训练损失\n",
    "    train_loss = 0.0\n",
    "    # 训练准确率\n",
    "    train_acc = 0.0\n",
    "    # 迭代训练\n",
    "    for data, label in train_data:\n",
    "        with autograd.record():\n",
    "            # 计算输出\n",
    "            output = net(data)\n",
    "            # 计算损失\n",
    "            loss = softmax_cross_entropy(output, label)\n",
    "        # 梯度反向传播\n",
    "        loss.backward()\n",
    "        # SGD更新梯度\n",
    "        SGD(params, learning_rate / batch_size)\n",
    "        # 记录训练损失\n",
    "        train_loss += nd.mean(loss).asscalar()\n",
    "        # 记录训练准确率\n",
    "        train_acc += accuracy(output, label)\n",
    "    # 计算测试准确率\n",
    "    test_acc = evaluate_accuracy(test_data, net)\n",
    "    print(\"Epoch %d. Loss: %f, Train acc %f, Test acc %f\" % (epoch, train_loss / len(train_data), train_acc / len(train_data), test_acc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}