Pruning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import math\n",
    "import numpy as np\n",
    "from torch.nn import Parameter\n",
    "from torch.nn.modules.module import Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n",
    "                                        download=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(trainset, batch_size=50,\n",
    "                                          shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root='./data', train=False,\n",
    "                                       download=True, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=50,\n",
    "                                         shuffle=False, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class PruningModule(Module):\n",
    "    def prune_by_percentile(self, q=5.0, **kwargs):\n",
    "        \"\"\"\n",
    "        Note:\n",
    "             The pruning percentile is based on all layer's parameters concatenated\n",
    "        Args:\n",
    "            q (float): percentile in float\n",
    "            **kwargs: may contain `cuda`\n",
    "        \"\"\"\n",
    "        # Calculate percentile value\n",
    "        alive_parameters = []\n",
    "        for name, p in self.named_parameters():\n",
    "            # We do not prune bias term\n",
    "            if 'bias' in name or 'mask' in name:\n",
    "                continue\n",
    "            tensor = p.data.cpu().numpy()\n",
    "            alive = tensor[np.nonzero(tensor)] # flattened array of nonzero values\n",
    "            alive_parameters.append(alive)\n",
    "\n",
    "        all_alives = np.concatenate(alive_parameters)\n",
    "        percentile_value = np.percentile(abs(all_alives), q)\n",
    "        print(f'Pruning with threshold : {percentile_value}')\n",
    "\n",
    "        # Prune the weights and mask\n",
    "        # Note that module here is the layer\n",
    "        # ex) fc1, fc2, fc3\n",
    "        for name, module in self.named_modules():\n",
    "            if name in ['fc1', 'fc2']:\n",
    "                module.prune(threshold=percentile_value)\n",
    "\n",
    "    def prune_by_std(self, s=0.25):\n",
    "        \"\"\"\n",
    "        Note that `s` is a quality parameter / sensitivity value according to the paper.\n",
    "        According to Song Han's previous paper (Learning both Weights and Connections for Efficient Neural Networks),\n",
    "        'The pruning threshold is chosen as a quality parameter multiplied by the standard deviation of a layer’s weights'\n",
    "\n",
    "        I tried multiple values and empirically, 0.25 matches the paper's compression rate and number of parameters.\n",
    "        Note : In the paper, the authors used different sensitivity values for different layers.\n",
    "        \"\"\"\n",
    "        for name, module in self.named_modules():\n",
    "            if name in ['fc1', 'fc2']:\n",
    "                threshold = np.std(module.weight.data.cpu().numpy()) * s\n",
    "                print(f'Pruning with threshold : {threshold} for layer {name}')\n",
    "                module.prune(threshold)\n",
    "\n",
    "\n",
    "class MaskedLinear(Module):\n",
    "    \"\"\"Applies a masked linear transformation to the incoming data: :math:`y = (A * M)x + b`\n",
    "\n",
    "    Args:\n",
    "        in_features: size of each input sample\n",
    "        out_features: size of each output sample\n",
    "        bias: If set to False, the layer will not learn an additive bias.\n",
    "            Default: ``True``\n",
    "\n",
    "    Shape:\n",
    "        - Input: :math:`(N, *, in\\_features)` where `*` means any number of\n",
    "          additional dimensions\n",
    "        - Output: :math:`(N, *, out\\_features)` where all but the last dimension\n",
    "          are the same shape as the input.\n",
    "\n",
    "    Attributes:\n",
    "        weight: the learnable weights of the module of shape\n",
    "            (out_features x in_features)\n",
    "        bias:   the learnable bias of the module of shape (out_features)\n",
    "        mask: the unlearnable mask for the weight.\n",
    "            It has the same shape as weight (out_features x in_features)\n",
    "\n",
    "    \"\"\"\n",
    "    def __init__(self, in_features, out_features, bias=True):\n",
    "        super(MaskedLinear, self).__init__()\n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.weight = Parameter(torch.Tensor(out_features, in_features))\n",
    "        # Initialize the mask with 1\n",
    "        self.mask = Parameter(torch.ones([out_features, in_features]), requires_grad=False)\n",
    "        if bias:\n",
    "            self.bias = Parameter(torch.Tensor(out_features))\n",
    "        else:\n",
    "            self.register_parameter('bias', None)\n",
    "        self.reset_parameters()\n",
    "\n",
    "    def reset_parameters(self):\n",
    "        stdv = 1. / math.sqrt(self.weight.size(1))\n",
    "        self.weight.data.uniform_(-stdv, stdv)\n",
    "        if self.bias is not None:\n",
    "            self.bias.data.uniform_(-stdv, stdv)\n",
    "\n",
    "    def forward(self, input):\n",
    "        return F.linear(input, self.weight * self.mask, self.bias)\n",
    "\n",
    "    def __repr__(self):\n",
    "        return self.__class__.__name__ + '(' \\\n",
    "            + 'in_features=' + str(self.in_features) \\\n",
    "            + ', out_features=' + str(self.out_features) \\\n",
    "            + ', bias=' + str(self.bias is not None) + ')'\n",
    "\n",
    "    def prune(self, threshold):\n",
    "        weight_dev = self.weight.device\n",
    "        mask_dev = self.mask.device\n",
    "        # Convert Tensors to numpy and calculate\n",
    "        tensor = self.weight.data.cpu().numpy()\n",
    "        mask = self.mask.data.cpu().numpy()\n",
    "        new_mask = np.where(abs(tensor) < threshold, 0.0, mask)\n",
    "        # Apply new weight and mask\n",
    "        self.weight.data = torch.from_numpy(tensor * new_mask).to(weight_dev)\n",
    "        self.mask.data = torch.from_numpy(new_mask).to(mask_dev)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the network:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net(\n",
      "  (conv1): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (fc1): MaskedLinear(in_features=400, out_features=120, bias=True)\n",
      "  (fc2): MaskedLinear(in_features=120, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "class Net(PruningModule):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        linear = MaskedLinear\n",
    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = linear(16 * 5 * 5, 120, bias=True)\n",
    "        self.fc2 = linear(120, 10)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = x.view(-1, 16 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return x\n",
    "\n",
    "net = Net()\n",
    "\n",
    "# Freeze the first layer\n",
    "#for param in net.fc1.parameters():\n",
    "#    param.requires_grad = False\n",
    "\n",
    "# Initialize the first layer\n",
    "#def weights_init(m):\n",
    "#    if isinstance(m, nn.Linear):\n",
    "#        m.weight.data.normal_(0, 0.01)\n",
    "    \n",
    "#net.apply(weights_init)\n",
    "\n",
    "#net.prune_by_std()\n",
    "\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "params[4] are weights in the first layer, params[5] are the masks, etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "Parameter containing:\n",
      "tensor([[-0.0027, -0.0318, -0.0415,  ..., -0.0132,  0.0069, -0.0323],\n",
      "        [ 0.0305,  0.0495, -0.0110,  ..., -0.0285,  0.0276,  0.0133],\n",
      "        [ 0.0079,  0.0090,  0.0116,  ..., -0.0286,  0.0186, -0.0427],\n",
      "        ...,\n",
      "        [-0.0296,  0.0463,  0.0181,  ..., -0.0282,  0.0156, -0.0391],\n",
      "        [ 0.0451, -0.0375,  0.0385,  ..., -0.0035,  0.0463,  0.0251],\n",
      "        [ 0.0463,  0.0081,  0.0022,  ..., -0.0094,  0.0477,  0.0154]],\n",
      "       requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "params = list(net.parameters())\n",
    "print(len(params))\n",
    "print(params[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "criterion = nn.CrossEntropyLoss() # Softmax is built in it so you do not need add that on the last layer\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train():\n",
    "    for epoch in range(150):  # loop over the dataset multiple times\n",
    "\n",
    "        running_loss = 0.0\n",
    "        for i, data in enumerate(trainloader, 0):\n",
    "            # get the inputs; data is a list of [inputs, labels]\n",
    "            inputs, labels = data\n",
    "    \n",
    "            # zero the parameter gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # forward + backward + optimize\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # print statistics\n",
    "            running_loss += loss.item()\n",
    "            if i % 1000 == 999:    # print every 1000 mini-batches\n",
    "                print('[%d, %5d] loss: %.3f' %\n",
    "                      (epoch + 1, i + 1, running_loss / 1000))\n",
    "                running_loss = 0.0\n",
    "                \n",
    "    print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            images, labels = data\n",
    "            outputs = net(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "    \n",
    "    print('Accuracy of the network on the 10000 test images: %d %%' % (\n",
    "        100 * correct / total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,  1000] loss: 2.175\n",
      "[2,  1000] loss: 1.767\n",
      "[3,  1000] loss: 1.590\n",
      "[4,  1000] loss: 1.498\n",
      "[5,  1000] loss: 1.428\n",
      "[6,  1000] loss: 1.361\n",
      "[7,  1000] loss: 1.303\n",
      "[8,  1000] loss: 1.253\n",
      "[9,  1000] loss: 1.208\n",
      "[10,  1000] loss: 1.171\n",
      "[11,  1000] loss: 1.138\n",
      "[12,  1000] loss: 1.109\n",
      "[13,  1000] loss: 1.082\n",
      "[14,  1000] loss: 1.053\n",
      "[15,  1000] loss: 1.034\n",
      "[16,  1000] loss: 1.010\n",
      "[17,  1000] loss: 0.989\n",
      "[18,  1000] loss: 0.969\n",
      "[19,  1000] loss: 0.949\n",
      "[20,  1000] loss: 0.932\n",
      "[21,  1000] loss: 0.915\n",
      "[22,  1000] loss: 0.895\n",
      "[23,  1000] loss: 0.880\n",
      "[24,  1000] loss: 0.863\n",
      "[25,  1000] loss: 0.848\n",
      "[26,  1000] loss: 0.835\n",
      "[27,  1000] loss: 0.818\n",
      "[28,  1000] loss: 0.804\n",
      "[29,  1000] loss: 0.791\n",
      "[30,  1000] loss: 0.778\n",
      "[31,  1000] loss: 0.767\n",
      "[32,  1000] loss: 0.749\n",
      "[33,  1000] loss: 0.739\n",
      "[34,  1000] loss: 0.726\n",
      "[35,  1000] loss: 0.714\n",
      "[36,  1000] loss: 0.708\n",
      "[37,  1000] loss: 0.694\n",
      "[38,  1000] loss: 0.680\n",
      "[39,  1000] loss: 0.672\n",
      "[40,  1000] loss: 0.658\n",
      "[41,  1000] loss: 0.648\n",
      "[42,  1000] loss: 0.636\n",
      "[43,  1000] loss: 0.628\n",
      "[44,  1000] loss: 0.622\n",
      "[45,  1000] loss: 0.612\n",
      "[46,  1000] loss: 0.601\n",
      "[47,  1000] loss: 0.591\n",
      "[48,  1000] loss: 0.582\n",
      "[49,  1000] loss: 0.575\n",
      "[50,  1000] loss: 0.564\n",
      "[51,  1000] loss: 0.556\n",
      "[52,  1000] loss: 0.546\n",
      "[53,  1000] loss: 0.539\n",
      "[54,  1000] loss: 0.535\n",
      "[55,  1000] loss: 0.522\n",
      "[56,  1000] loss: 0.512\n",
      "[57,  1000] loss: 0.509\n",
      "[58,  1000] loss: 0.497\n",
      "[59,  1000] loss: 0.489\n",
      "[60,  1000] loss: 0.481\n",
      "[61,  1000] loss: 0.476\n",
      "[62,  1000] loss: 0.470\n",
      "[63,  1000] loss: 0.463\n",
      "[64,  1000] loss: 0.456\n",
      "[65,  1000] loss: 0.451\n",
      "[66,  1000] loss: 0.442\n",
      "[67,  1000] loss: 0.431\n",
      "[68,  1000] loss: 0.427\n",
      "[69,  1000] loss: 0.421\n",
      "[70,  1000] loss: 0.413\n",
      "[71,  1000] loss: 0.403\n",
      "[72,  1000] loss: 0.400\n",
      "[73,  1000] loss: 0.398\n",
      "[74,  1000] loss: 0.388\n",
      "[75,  1000] loss: 0.384\n",
      "[76,  1000] loss: 0.382\n",
      "[77,  1000] loss: 0.368\n",
      "[78,  1000] loss: 0.365\n",
      "[79,  1000] loss: 0.355\n",
      "[80,  1000] loss: 0.354\n",
      "[81,  1000] loss: 0.347\n",
      "[82,  1000] loss: 0.341\n",
      "[83,  1000] loss: 0.340\n",
      "[84,  1000] loss: 0.337\n",
      "[85,  1000] loss: 0.329\n",
      "[86,  1000] loss: 0.322\n",
      "[87,  1000] loss: 0.313\n",
      "[88,  1000] loss: 0.312\n",
      "[89,  1000] loss: 0.307\n",
      "[90,  1000] loss: 0.305\n",
      "[91,  1000] loss: 0.301\n",
      "[92,  1000] loss: 0.298\n",
      "[93,  1000] loss: 0.282\n",
      "[94,  1000] loss: 0.279\n",
      "[95,  1000] loss: 0.283\n",
      "[96,  1000] loss: 0.271\n",
      "[97,  1000] loss: 0.267\n",
      "[98,  1000] loss: 0.268\n",
      "[99,  1000] loss: 0.269\n",
      "[100,  1000] loss: 0.252\n",
      "[101,  1000] loss: 0.251\n",
      "[102,  1000] loss: 0.253\n",
      "[103,  1000] loss: 0.251\n",
      "[104,  1000] loss: 0.251\n",
      "[105,  1000] loss: 0.241\n",
      "[106,  1000] loss: 0.236\n",
      "[107,  1000] loss: 0.237\n",
      "[108,  1000] loss: 0.228\n",
      "[109,  1000] loss: 0.227\n",
      "[110,  1000] loss: 0.229\n",
      "[111,  1000] loss: 0.219\n",
      "[112,  1000] loss: 0.219\n",
      "[113,  1000] loss: 0.217\n",
      "[114,  1000] loss: 0.214\n",
      "[115,  1000] loss: 0.210\n",
      "[116,  1000] loss: 0.211\n",
      "[117,  1000] loss: 0.209\n",
      "[118,  1000] loss: 0.210\n",
      "[119,  1000] loss: 0.205\n",
      "[120,  1000] loss: 0.198\n",
      "[121,  1000] loss: 0.191\n",
      "[122,  1000] loss: 0.187\n",
      "[123,  1000] loss: 0.191\n",
      "[124,  1000] loss: 0.194\n",
      "[125,  1000] loss: 0.181\n",
      "[126,  1000] loss: 0.176\n",
      "[127,  1000] loss: 0.193\n",
      "[128,  1000] loss: 0.185\n",
      "[129,  1000] loss: 0.181\n",
      "[130,  1000] loss: 0.177\n",
      "[131,  1000] loss: 0.178\n",
      "[132,  1000] loss: 0.173\n",
      "[133,  1000] loss: 0.154\n",
      "[134,  1000] loss: 0.167\n",
      "[135,  1000] loss: 0.168\n",
      "[136,  1000] loss: 0.169\n",
      "[137,  1000] loss: 0.170\n",
      "[138,  1000] loss: 0.158\n",
      "[139,  1000] loss: 0.146\n",
      "[140,  1000] loss: 0.147\n",
      "[141,  1000] loss: 0.153\n",
      "[142,  1000] loss: 0.158\n",
      "[143,  1000] loss: 0.155\n",
      "[144,  1000] loss: 0.150\n",
      "[145,  1000] loss: 0.155\n",
      "[146,  1000] loss: 0.176\n",
      "[147,  1000] loss: 0.146\n",
      "[148,  1000] loss: 0.152\n",
      "[149,  1000] loss: 0.146\n",
      "[150,  1000] loss: 0.150\n",
      "Finished Training\n",
      "Accuracy of the network on the 10000 test images: 59 %\n"
     ]
    }
   ],
   "source": [
    "train()\n",
    "test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruning with threshold : 0.02721398137509823 for layer fc1\n",
      "Pruning with threshold : 0.07363494485616684 for layer fc2\n",
      "Accuracy of the network on the 10000 test images: 59 %\n"
     ]
    }
   ],
   "source": [
    "net.prune_by_std()\n",
    "#print(params[4])\n",
    "test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,  1000] loss: 0.163\n",
      "[2,  1000] loss: 0.129\n",
      "[3,  1000] loss: 0.096\n",
      "[4,  1000] loss: 0.081\n",
      "[5,  1000] loss: 0.068\n",
      "[6,  1000] loss: 0.062\n",
      "[7,  1000] loss: 0.056\n",
      "[8,  1000] loss: 0.051\n",
      "[9,  1000] loss: 0.042\n",
      "[10,  1000] loss: 0.039\n",
      "[11,  1000] loss: 0.039\n",
      "[12,  1000] loss: 0.031\n",
      "[13,  1000] loss: 0.024\n",
      "[14,  1000] loss: 0.021\n",
      "[15,  1000] loss: 0.017\n",
      "[16,  1000] loss: 0.015\n",
      "[17,  1000] loss: 0.013\n",
      "[18,  1000] loss: 0.012\n",
      "[19,  1000] loss: 0.011\n",
      "[20,  1000] loss: 0.010\n",
      "[21,  1000] loss: 0.009\n",
      "[22,  1000] loss: 0.009\n",
      "[23,  1000] loss: 0.008\n",
      "[24,  1000] loss: 0.008\n",
      "[25,  1000] loss: 0.007\n",
      "[26,  1000] loss: 0.007\n",
      "[27,  1000] loss: 0.007\n",
      "[28,  1000] loss: 0.006\n",
      "[29,  1000] loss: 0.006\n",
      "[30,  1000] loss: 0.006\n",
      "[31,  1000] loss: 0.006\n",
      "[32,  1000] loss: 0.005\n",
      "[33,  1000] loss: 0.005\n",
      "[34,  1000] loss: 0.005\n",
      "[35,  1000] loss: 0.005\n",
      "[36,  1000] loss: 0.005\n",
      "[37,  1000] loss: 0.005\n",
      "[38,  1000] loss: 0.004\n",
      "[39,  1000] loss: 0.004\n",
      "[40,  1000] loss: 0.004\n",
      "[41,  1000] loss: 0.004\n",
      "[42,  1000] loss: 0.004\n",
      "[43,  1000] loss: 0.004\n",
      "[44,  1000] loss: 0.004\n",
      "[45,  1000] loss: 0.004\n",
      "[46,  1000] loss: 0.003\n",
      "[47,  1000] loss: 0.003\n",
      "[48,  1000] loss: 0.003\n",
      "[49,  1000] loss: 0.003\n",
      "[50,  1000] loss: 0.003\n",
      "[51,  1000] loss: 0.003\n",
      "[52,  1000] loss: 0.003\n",
      "[53,  1000] loss: 0.003\n",
      "[54,  1000] loss: 0.003\n",
      "[55,  1000] loss: 0.003\n",
      "[56,  1000] loss: 0.003\n",
      "[57,  1000] loss: 0.003\n",
      "[58,  1000] loss: 0.003\n",
      "[59,  1000] loss: 0.003\n",
      "[60,  1000] loss: 0.002\n",
      "[61,  1000] loss: 0.002\n",
      "[62,  1000] loss: 0.002\n",
      "[63,  1000] loss: 0.002\n",
      "[64,  1000] loss: 0.002\n",
      "[65,  1000] loss: 0.002\n",
      "[66,  1000] loss: 0.002\n",
      "[67,  1000] loss: 0.002\n",
      "[68,  1000] loss: 0.002\n",
      "[69,  1000] loss: 0.002\n",
      "[70,  1000] loss: 0.002\n",
      "[71,  1000] loss: 0.002\n",
      "[72,  1000] loss: 0.002\n",
      "[73,  1000] loss: 0.002\n",
      "[74,  1000] loss: 0.002\n",
      "[75,  1000] loss: 0.002\n",
      "[76,  1000] loss: 0.002\n",
      "[77,  1000] loss: 0.002\n",
      "[78,  1000] loss: 0.002\n",
      "[79,  1000] loss: 0.002\n",
      "[80,  1000] loss: 0.002\n",
      "[81,  1000] loss: 0.002\n",
      "[82,  1000] loss: 0.002\n",
      "[83,  1000] loss: 0.002\n",
      "[84,  1000] loss: 0.002\n",
      "[85,  1000] loss: 0.002\n",
      "[86,  1000] loss: 0.002\n",
      "[87,  1000] loss: 0.002\n",
      "[88,  1000] loss: 0.002\n",
      "[89,  1000] loss: 0.002\n",
      "[90,  1000] loss: 0.002\n",
      "[91,  1000] loss: 0.002\n",
      "[92,  1000] loss: 0.001\n",
      "[93,  1000] loss: 0.001\n",
      "[94,  1000] loss: 0.001\n",
      "[95,  1000] loss: 0.001\n",
      "[96,  1000] loss: 0.001\n",
      "[97,  1000] loss: 0.001\n",
      "[98,  1000] loss: 0.001\n",
      "[99,  1000] loss: 0.001\n",
      "[100,  1000] loss: 0.001\n",
      "[101,  1000] loss: 0.001\n",
      "[102,  1000] loss: 0.001\n",
      "[103,  1000] loss: 0.001\n",
      "[104,  1000] loss: 0.001\n",
      "[105,  1000] loss: 0.001\n",
      "[106,  1000] loss: 0.001\n",
      "[107,  1000] loss: 0.001\n",
      "[108,  1000] loss: 0.001\n",
      "[109,  1000] loss: 0.001\n",
      "[110,  1000] loss: 0.001\n",
      "[111,  1000] loss: 0.001\n",
      "[112,  1000] loss: 0.001\n",
      "[113,  1000] loss: 0.001\n",
      "[114,  1000] loss: 0.001\n",
      "[115,  1000] loss: 0.001\n",
      "[116,  1000] loss: 0.001\n",
      "[117,  1000] loss: 0.001\n",
      "[118,  1000] loss: 0.001\n",
      "[119,  1000] loss: 0.001\n",
      "[120,  1000] loss: 0.001\n",
      "[121,  1000] loss: 0.001\n",
      "[122,  1000] loss: 0.001\n",
      "[123,  1000] loss: 0.001\n",
      "[124,  1000] loss: 0.001\n",
      "[125,  1000] loss: 0.001\n",
      "[126,  1000] loss: 0.001\n",
      "[127,  1000] loss: 0.001\n",
      "[128,  1000] loss: 0.001\n",
      "[129,  1000] loss: 0.001\n",
      "[130,  1000] loss: 0.001\n",
      "[131,  1000] loss: 0.001\n",
      "[132,  1000] loss: 0.001\n",
      "[133,  1000] loss: 0.001\n",
      "[134,  1000] loss: 0.001\n",
      "[135,  1000] loss: 0.001\n",
      "[136,  1000] loss: 0.001\n",
      "[137,  1000] loss: 0.001\n",
      "[138,  1000] loss: 0.001\n",
      "[139,  1000] loss: 0.001\n",
      "[140,  1000] loss: 0.001\n",
      "[141,  1000] loss: 0.001\n",
      "[142,  1000] loss: 0.001\n",
      "[143,  1000] loss: 0.001\n",
      "[144,  1000] loss: 0.001\n",
      "[145,  1000] loss: 0.001\n",
      "[146,  1000] loss: 0.001\n",
      "[147,  1000] loss: 0.001\n",
      "[148,  1000] loss: 0.001\n",
      "[149,  1000] loss: 0.001\n",
      "[150,  1000] loss: 0.001\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "train()\n",
    "#print(params[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10000 test images: 60 %\n"
     ]
    }
   ],
   "source": [
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}