lab-08_4_mnist_back_prop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lab 10 MNIST and softmax\n",
    "import torch\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "# for reproducibility\n",
    "torch.manual_seed(777)\n",
    "if device == 'cuda':\n",
    "    torch.cuda.manual_seed_all(777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameters\n",
    "learning_rate = 0.5\n",
    "batch_size = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MNIST dataset\n",
    "mnist_train = dsets.MNIST(root='MNIST_data/',\n",
    "                          train=True,\n",
    "                          transform=transforms.ToTensor(),\n",
    "                          download=True)\n",
    "\n",
    "mnist_test = dsets.MNIST(root='MNIST_data/',\n",
    "                         train=False,\n",
    "                         transform=transforms.ToTensor(),\n",
    "                         download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset loader\n",
    "data_loader = torch.utils.data.DataLoader(dataset=mnist_train,\n",
    "                                          batch_size=batch_size,\n",
    "                                          shuffle=True,\n",
    "                                          drop_last=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1 = torch.nn.Parameter(torch.Tensor(784, 30)).to(device)\n",
    "b1 = torch.nn.Parameter(torch.Tensor(30)).to(device)\n",
    "w2 = torch.nn.Parameter(torch.Tensor(30, 10)).to(device)\n",
    "b2 = torch.nn.Parameter(torch.Tensor(10)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([ 0.3078, -1.9857,  1.0512,  1.5122, -1.0199, -0.7402, -1.3111,  0.6142,\n",
       "        -0.6474,  0.1758], requires_grad=True)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.nn.init.normal_(w1)\n",
    "torch.nn.init.normal_(b1)\n",
    "torch.nn.init.normal_(w2)\n",
    "torch.nn.init.normal_(b2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    #  sigmoid function\n",
    "    return 1.0 / (1.0 + torch.exp(-x))\n",
    "    # return torch.div(torch.tensor(1), torch.add(torch.tensor(1.0), torch.exp(-x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid_prime(x):\n",
    "    # derivative of the sigmoid function\n",
    "    return sigmoid(x) * (1 - sigmoid(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "736\n",
      "862\n",
      "860\n",
      "881\n",
      "874\n",
      "890\n",
      "904\n",
      "923\n",
      "916\n",
      "920\n"
     ]
    }
   ],
   "source": [
    "X_test = mnist_test.test_data.view(-1, 28 * 28).float().to(device)[:1000]\n",
    "Y_test = mnist_test.test_labels.to(device)[:1000]\n",
    "i = 0\n",
    "while not i == 10000:\n",
    "    for X, Y in data_loader:\n",
    "        i += 1\n",
    "\n",
    "        # forward\n",
    "        X = X.view(-1, 28 * 28).to(device)\n",
    "        Y = torch.zeros((batch_size, 10)).scatter_(1, Y.unsqueeze(1), 1).to(device)    # one-hot\n",
    "        l1 = torch.add(torch.matmul(X, w1), b1)\n",
    "        a1 = sigmoid(l1)\n",
    "        l2 = torch.add(torch.matmul(a1, w2), b2)\n",
    "        y_pred = sigmoid(l2)\n",
    "\n",
    "        diff = y_pred - Y\n",
    "\n",
    "        # Back prop (chain rule)\n",
    "        d_l2 = diff * sigmoid_prime(l2)\n",
    "        d_b2 = d_l2\n",
    "        d_w2 = torch.matmul(torch.transpose(a1, 0, 1), d_l2)\n",
    "\n",
    "        d_a1 = torch.matmul(d_l2, torch.transpose(w2, 0, 1))\n",
    "        d_l1 = d_a1 * sigmoid_prime(l1)\n",
    "        d_b1 = d_l1\n",
    "        d_w1 = torch.matmul(torch.transpose(X, 0, 1), d_l1)\n",
    "\n",
    "        w1 = w1 - learning_rate * d_w1\n",
    "        b1 = b1 - learning_rate * torch.mean(d_b1, 0)\n",
    "        w2 = w2 - learning_rate * d_w2\n",
    "        b2 = b2 - learning_rate * torch.mean(d_b2, 0)\n",
    "\n",
    "        if i % 1000 == 0:\n",
    "            l1 = torch.add(torch.matmul(X_test, w1), b1)\n",
    "            a1 = sigmoid(l1)\n",
    "            l2 = torch.add(torch.matmul(a1, w2), b2)\n",
    "            y_pred = sigmoid(l2)\n",
    "            acct_mat = torch.argmax(y_pred, 1) == Y_test\n",
    "            acct_res = acct_mat.sum()\n",
    "            print(acct_res.item())\n",
    "\n",
    "        if i == 10000:\n",
    "            break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}