LeNet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Prepare the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import math\n",
    "import numpy as np\n",
    "import torch.optim as optim\n",
    "import torchvision.datasets as datasets\n",
    "from torch.autograd import Variable\n",
    "#from torchsummary import summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CIFAR10RandomLabels(datasets.CIFAR10):\n",
    "    \"\"\"CIFAR10 dataset, with support for randomly corrupt labels.\n",
    "    Params\n",
    "    ------  \n",
    "    corrupt_prob: float\n",
    "    Default 0.0. The probability of a label being replaced with\n",
    "    random label.\n",
    "    num_classes: int\n",
    "    Default 10. The number of classes in the dataset.\n",
    "    \"\"\"\n",
    "    def __init__(self, corrupt_prob=0.0, num_classes=10, **kwargs):\n",
    "        super(CIFAR10RandomLabels, self).__init__(**kwargs)\n",
    "        self.n_classes = num_classes\n",
    "        if corrupt_prob > 0:\n",
    "            self.corrupt_labels(corrupt_prob)\n",
    "\n",
    "    def corrupt_labels(self, corrupt_prob):\n",
    "        labels = np.array(self.targets)\n",
    "        np.random.seed(12345)\n",
    "        mask = np.random.rand(len(labels)) <= corrupt_prob\n",
    "        rnd_labels = np.random.choice(self.n_classes, mask.sum())\n",
    "        labels[mask] = rnd_labels\n",
    "        # we need to explicitly cast the labels from npy.int64 to\n",
    "        # builtin int type, otherwise pytorch will fail...\n",
    "        labels = [int(x) for x in labels]\n",
    "        self.targets = torch.tensor(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "trainloader = torch.utils.data.DataLoader(\n",
    "        CIFAR10RandomLabels(root='./data', train=True, download=True, \n",
    "                            transform=transform, corrupt_prob=0.2),\n",
    "                            batch_size=128, shuffle=True, num_workers=4)\n",
    "testloader = torch.utils.data.DataLoader(\n",
    "        CIFAR10RandomLabels(root='./data', train=False, download=True,\n",
    "                            transform=transform, corrupt_prob=0.2),\n",
    "                            batch_size=100, shuffle=False, num_workers=4)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MaskedLinear(nn.Linear):\n",
    "    def __init__(self, in_features, out_features, bias=True):\n",
    "        super(MaskedLinear, self).__init__(in_features, out_features, bias)\n",
    "        self.mask_flag = False\n",
    "    \n",
    "    def set_mask(self, mask):\n",
    "        self.mask = Variable(mask, requires_grad=False)\n",
    "        self.weight.data = self.weight.data*self.mask.data\n",
    "        self.mask_flag = True\n",
    "    \n",
    "    def get_mask(self):\n",
    "        print(self.mask_flag)\n",
    "        return self.mask\n",
    "    \n",
    "    def prune(self, threshold):\n",
    "        # generate mask\n",
    "        for params in self.parameters():\n",
    "            if len(params.data.size()) != 1: # Not bias\n",
    "                mask = params.data.abs() > threshold\n",
    "                self.set_mask(mask.float()) \n",
    "    \n",
    "    def forward(self, x):\n",
    "        if self.mask_flag == True:\n",
    "            weight = self.weight*self.mask\n",
    "            return F.linear(x, weight, self.bias)\n",
    "        else:\n",
    "            return F.linear(x, self.weight, self.bias)\n",
    "        \n",
    "\n",
    "        \n",
    "class MaskedConv2d(nn.Conv2d):\n",
    "    def __init__(self, in_channels, out_channels, kernel_size, stride=1,\n",
    "                 padding=0, dilation=1, groups=1, bias=True):\n",
    "        super(MaskedConv2d, self).__init__(in_channels, out_channels, \n",
    "            kernel_size, stride, padding, dilation, groups, bias)\n",
    "        self.mask_flag = False\n",
    "    \n",
    "    def set_mask(self, mask):\n",
    "        self.mask = Variable(mask, requires_grad=False)\n",
    "        self.weight.data = self.weight.data*self.mask.data\n",
    "        self.mask_flag = True\n",
    "    \n",
    "    def get_mask(self):\n",
    "        print(self.mask_flag)\n",
    "        return self.mask\n",
    "    \n",
    "    def prune(self, threshold):\n",
    "        # generate mask\n",
    "        for params in self.parameters():\n",
    "            if len(params.data.size()) != 1: # Not bias\n",
    "                mask = params.data.abs() > threshold\n",
    "                self.set_mask(mask.float()) \n",
    "    \n",
    "    def forward(self, x):\n",
    "        if self.mask_flag == True:\n",
    "            weight = self.weight*self.mask\n",
    "            return F.conv2d(x, weight, self.bias, self.stride,\n",
    "                        self.padding, self.dilation, self.groups)\n",
    "        else:\n",
    "            return F.conv2d(x, self.weight, self.bias, self.stride,\n",
    "                        self.padding, self.dilation, self.groups)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the network:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Net(\n",
       "  (conv1): MaskedConv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "  (conv2): MaskedConv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
       "  (fc1): MaskedLinear(in_features=400, out_features=240, bias=True)\n",
       "  (fc2): MaskedLinear(in_features=240, out_features=120, bias=True)\n",
       "  (fc3): MaskedLinear(in_features=120, out_features=84, bias=True)\n",
       "  (fc4): MaskedLinear(in_features=84, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        linear = MaskedLinear\n",
    "        conv2d = MaskedConv2d\n",
    "        self.conv1 = conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = conv2d(6, 16, 5)\n",
    "        self.fc1 = linear(16 * 5 * 5, 240)\n",
    "        self.fc2 = linear(240, 120)\n",
    "        self.fc3 = linear(120, 84)\n",
    "        self.fc4 = linear(84, 10)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = x.view(-1, 16 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = F.relu(self.fc3(x))\n",
    "        x = self.fc4(x)\n",
    "        return x\n",
    "    \n",
    "    def weight_prune(self, pruning_perc):\n",
    "        all_weights = []\n",
    "        for params in self.parameters():\n",
    "            if len(params.data.size()) != 1:\n",
    "                all_weights += list(params.cpu().data.abs().numpy().flatten())\n",
    "        threshold = np.percentile(np.array(all_weights), pruning_perc)\n",
    "        print(f'Pruning with threshold : %.4f' % threshold)\n",
    "        \n",
    "        # Module here refers to layer\n",
    "        for name, module in self.named_modules():\n",
    "            if name in ['conv1','conv2','fc1','fc2','fc3','fc4']:\n",
    "                module.prune(threshold)                \n",
    "\n",
    "net = Net()\n",
    "net.to(device)\n",
    "#summary(net, (3, 32, 32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, optimizer, epochs, early_stop_step=0):\n",
    "    step_counter = 0\n",
    "    best_accuracy = 0\n",
    "    for epoch in range(epochs):  # loop over the dataset multiple times\n",
    "      \n",
    "        train_correct = 0\n",
    "        train_total = 0\n",
    "        running_loss = 0.0\n",
    "        train_accuracy = 0.0\n",
    "        \n",
    "        for i, data in enumerate(trainloader, 0):\n",
    "            # get the inputs; data is a list of [inputs, labels]\n",
    "            inputs, labels = data[0].to(device), data[1].to(device)\n",
    "    \n",
    "            # zero the parameter gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # forward + backward + optimize\n",
    "            outputs = model(inputs)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            # print statistics\n",
    "            running_loss += loss.item()\n",
    "            if i % 100 == 99:    # print every 100 mini-batches\n",
    "             #   print('[%d, %5d] loss: %.3f' %\n",
    "             #         (epoch + 1, i + 1, running_loss / 100))\n",
    "                train_loss = running_loss / 100\n",
    "                running_loss = 0.0\n",
    "\n",
    "        if epoch % 10 == 9:     \n",
    "            with torch.no_grad():\n",
    "                for data in trainloader:\n",
    "                    inputs, labels = data[0].to(device), data[1].to(device)\n",
    "                    outputs = model(inputs)\n",
    "                    _, predicted = torch.max(outputs.data, 1)\n",
    "                    train_total += labels.size(0)\n",
    "                    train_correct += (predicted == labels).sum().item()\n",
    "                    \n",
    "            train_accuracy = 100 * train_correct / train_total\n",
    "            print('[%d, %5d] loss: %.3f' %\n",
    "                 (epoch + 1, i + 1, train_loss))\n",
    "            print('Train Accuracy: %.3f %%' % train_accuracy)\n",
    "            \n",
    "        '''\n",
    "        if early_stop_step != 0:\n",
    "            step_counter += 1\n",
    "            if best_accuracy < train_accuracy:\n",
    "                best_accuracy = train_accuracy\n",
    "                step_counter = 0\n",
    "            if step_counter == early_stop_step or epoch == epochs - 1:\n",
    "                print(\"Early stop at epoch %d\" % (epoch + 1))\n",
    "                print('Train Accuracy: %.3f %%, Train Loss: %.3f' % (train_accuracy, train_loss))\n",
    "                break\n",
    "        '''\n",
    "        \n",
    "        if train_accuracy - 100.0 < 0.001 and train_loss - 0.0 < 0.001:\n",
    "            print(\"Reach 100 %% train accuracy at epoch %d\" % (epoch + 1))\n",
    "            break\n",
    "            \n",
    "    #print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model):\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    test_loss = 0\n",
    "    with torch.no_grad():\n",
    "        for data in testloader:\n",
    "            images, labels = data[0].to(device), data[1].to(device)\n",
    "            outputs = model(images)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "            test_loss += criterion(outputs, labels).item()\n",
    "    \n",
    "    #print('Test Accuracy on 10000 images: %.2f %%' % (\n",
    "    #    100 * correct / total))\n",
    "    \n",
    "    # test loss for 100 mini-batches)\n",
    "    print('Test Accuracy: %.3f %%, Test Loss: %.3f' % (100 * correct / total, test_loss / 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_size(model):\n",
    "    non_zeros = 0\n",
    "    for param in model.parameters():\n",
    "        if param.requires_grad==True:\n",
    "            non_zeros += param.nonzero().size(0)\n",
    "    params_size = non_zeros* 4. / (1024 ** 2)\n",
    "    print(\"Params size (MB): %0.3f\" % params_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Params size (MB): 0.530\n",
      "[10,   391] loss: 2.188\n",
      "Train Accuracy: 21.054 %\n",
      "[20,   391] loss: 1.871\n",
      "Train Accuracy: 35.784 %\n",
      "[30,   391] loss: 1.719\n",
      "Train Accuracy: 42.478 %\n",
      "[40,   391] loss: 1.634\n",
      "Train Accuracy: 47.622 %\n",
      "[50,   391] loss: 1.562\n",
      "Train Accuracy: 51.482 %\n",
      "[60,   391] loss: 1.473\n",
      "Train Accuracy: 54.442 %\n",
      "[70,   391] loss: 1.394\n",
      "Train Accuracy: 56.314 %\n",
      "[80,   391] loss: 1.325\n",
      "Train Accuracy: 61.142 %\n",
      "[90,   391] loss: 1.216\n",
      "Train Accuracy: 63.840 %\n",
      "[100,   391] loss: 1.095\n",
      "Train Accuracy: 67.452 %\n",
      "[110,   391] loss: 1.016\n",
      "Train Accuracy: 70.966 %\n",
      "[120,   391] loss: 0.869\n",
      "Train Accuracy: 73.162 %\n",
      "[130,   391] loss: 0.752\n",
      "Train Accuracy: 76.296 %\n",
      "[140,   391] loss: 0.637\n",
      "Train Accuracy: 81.128 %\n",
      "[150,   391] loss: 0.518\n",
      "Train Accuracy: 83.382 %\n",
      "[160,   391] loss: 0.399\n",
      "Train Accuracy: 87.438 %\n",
      "[170,   391] loss: 0.343\n",
      "Train Accuracy: 90.920 %\n",
      "[180,   391] loss: 0.312\n",
      "Train Accuracy: 90.286 %\n",
      "[190,   391] loss: 0.245\n",
      "Train Accuracy: 92.468 %\n",
      "[200,   391] loss: 0.235\n",
      "Train Accuracy: 95.312 %\n",
      "[210,   391] loss: 0.129\n",
      "Train Accuracy: 95.844 %\n",
      "[220,   391] loss: 0.085\n",
      "Train Accuracy: 96.784 %\n",
      "[230,   391] loss: 0.026\n",
      "Train Accuracy: 99.866 %\n",
      "[240,   391] loss: 0.003\n",
      "Train Accuracy: 100.000 %\n",
      "[250,   391] loss: 0.002\n",
      "Train Accuracy: 100.000 %\n",
      "[260,   391] loss: 0.002\n",
      "Train Accuracy: 100.000 %\n",
      "[270,   391] loss: 0.001\n",
      "Train Accuracy: 100.000 %\n",
      "Reach 100 % train accuracy at epoch 278\n",
      "Test Accuracy: 39.740 %, Test Loss: 9.988\n",
      "Params size (MB): 0.530\n"
     ]
    }
   ],
   "source": [
    "calculate_size(net)\n",
    "criterion = nn.CrossEntropyLoss() # Softmax is built in it so you do not need add that on the last layer\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n",
    "#initial_optimizer_state_dict = optimizer.state_dict()\n",
    "train(net, optimizer, 5000)\n",
    "model_path = \"models/LeNet_480_84_10_random100.pt\"\n",
    "torch.save(net.state_dict(), model_path)\n",
    "test(net)\n",
    "calculate_size(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "Prune the model by 56.60 %\n",
      "Pruning with threshold : 0.0587\n",
      "Params size (MB): 0.231\n",
      "[10,   391] loss: 0.366\n",
      "Train Accuracy: 88.806 %\n",
      "[20,   391] loss: 0.224\n",
      "Train Accuracy: 92.828 %\n",
      "[30,   391] loss: 0.177\n",
      "Train Accuracy: 95.464 %\n",
      "[40,   391] loss: 0.128\n",
      "Train Accuracy: 96.094 %\n",
      "[50,   391] loss: 0.133\n",
      "Train Accuracy: 96.666 %\n",
      "[60,   391] loss: 0.131\n",
      "Train Accuracy: 93.942 %\n",
      "[70,   391] loss: 0.142\n",
      "Train Accuracy: 96.138 %\n",
      "[80,   391] loss: 0.117\n",
      "Train Accuracy: 97.114 %\n",
      "[90,   391] loss: 0.008\n",
      "Train Accuracy: 99.996 %\n",
      "[100,   391] loss: 0.004\n",
      "Train Accuracy: 99.998 %\n",
      "[110,   391] loss: 0.003\n",
      "Train Accuracy: 100.000 %\n",
      "[120,   391] loss: 0.002\n",
      "Train Accuracy: 100.000 %\n",
      "[130,   391] loss: 0.002\n",
      "Train Accuracy: 100.000 %\n",
      "[140,   391] loss: 0.001\n",
      "Train Accuracy: 100.000 %\n",
      "[150,   391] loss: 0.001\n",
      "Train Accuracy: 100.000 %\n",
      "[160,   391] loss: 0.001\n",
      "Train Accuracy: 100.000 %\n",
      "[170,   391] loss: 0.001\n",
      "Train Accuracy: 100.000 %\n",
      "Reach 100 % train accuracy at epoch 171\n",
      "Test Accuracy: 38.780 %, Test Loss: 13.851\n",
      "--------------------------------------\n",
      "Prune the model by 65.28 %\n",
      "Pruning with threshold : 0.0707\n",
      "Params size (MB): 0.185\n",
      "[10,   391] loss: 0.768\n",
      "Train Accuracy: 76.968 %\n",
      "[20,   391] loss: 0.567\n",
      "Train Accuracy: 82.768 %\n",
      "[30,   391] loss: 0.464\n",
      "Train Accuracy: 86.252 %\n",
      "[40,   391] loss: 0.416\n",
      "Train Accuracy: 87.594 %\n",
      "[50,   391] loss: 0.363\n",
      "Train Accuracy: 89.392 %\n",
      "[60,   391] loss: 0.326\n",
      "Train Accuracy: 90.574 %\n",
      "[70,   391] loss: 0.322\n",
      "Train Accuracy: 91.104 %\n",
      "[80,   391] loss: 0.297\n",
      "Train Accuracy: 92.144 %\n",
      "[90,   391] loss: 0.269\n",
      "Train Accuracy: 92.404 %\n",
      "[100,   391] loss: 0.273\n",
      "Train Accuracy: 89.174 %\n",
      "[110,   391] loss: 0.231\n",
      "Train Accuracy: 92.596 %\n",
      "[120,   391] loss: 0.261\n",
      "Train Accuracy: 93.248 %\n",
      "[130,   391] loss: 0.243\n",
      "Train Accuracy: 92.286 %\n",
      "[140,   391] loss: 0.243\n",
      "Train Accuracy: 92.572 %\n",
      "[150,   391] loss: 0.214\n",
      "Train Accuracy: 93.040 %\n",
      "[160,   391] loss: 0.171\n",
      "Train Accuracy: 94.656 %\n",
      "[170,   391] loss: 0.181\n",
      "Train Accuracy: 94.556 %\n",
      "[180,   391] loss: 0.186\n",
      "Train Accuracy: 93.466 %\n",
      "[190,   391] loss: 0.162\n",
      "Train Accuracy: 94.974 %\n",
      "[200,   391] loss: 0.173\n",
      "Train Accuracy: 94.300 %\n",
      "[210,   391] loss: 0.147\n",
      "Train Accuracy: 94.708 %\n",
      "[220,   391] loss: 0.158\n",
      "Train Accuracy: 94.760 %\n",
      "[230,   391] loss: 0.135\n",
      "Train Accuracy: 94.394 %\n",
      "[240,   391] loss: 0.102\n",
      "Train Accuracy: 96.570 %\n",
      "[250,   391] loss: 0.116\n",
      "Train Accuracy: 95.408 %\n",
      "[260,   391] loss: 0.139\n",
      "Train Accuracy: 94.854 %\n",
      "[270,   391] loss: 0.111\n",
      "Train Accuracy: 95.444 %\n",
      "[280,   391] loss: 0.157\n",
      "Train Accuracy: 95.246 %\n",
      "[290,   391] loss: 0.008\n",
      "Train Accuracy: 99.994 %\n",
      "[300,   391] loss: 0.005\n",
      "Train Accuracy: 100.000 %\n",
      "[310,   391] loss: 0.004\n",
      "Train Accuracy: 100.000 %\n",
      "[320,   391] loss: 0.003\n",
      "Train Accuracy: 100.000 %\n",
      "[330,   391] loss: 0.002\n",
      "Train Accuracy: 100.000 %\n",
      "[340,   391] loss: 0.002\n",
      "Train Accuracy: 100.000 %\n",
      "[350,   391] loss: 0.002\n",
      "Train Accuracy: 100.000 %\n",
      "[360,   391] loss: 0.002\n",
      "Train Accuracy: 100.000 %\n",
      "[370,   391] loss: 0.001\n",
      "Train Accuracy: 100.000 %\n",
      "[380,   391] loss: 0.001\n",
      "Train Accuracy: 100.000 %\n",
      "[390,   391] loss: 0.001\n",
      "Train Accuracy: 100.000 %\n",
      "[400,   391] loss: 0.001\n",
      "Train Accuracy: 100.000 %\n",
      "Reach 100 % train accuracy at epoch 405\n",
      "Test Accuracy: 37.300 %, Test Loss: 17.805\n",
      "--------------------------------------\n",
      "Prune the model by 73.77 %\n",
      "Pruning with threshold : 0.0848\n",
      "Params size (MB): 0.140\n",
      "[10,   391] loss: 1.153\n",
      "Train Accuracy: 63.678 %\n",
      "[20,   391] loss: 0.987\n",
      "Train Accuracy: 69.120 %\n",
      "[30,   391] loss: 0.879\n",
      "Train Accuracy: 72.398 %\n",
      "[40,   391] loss: 0.822\n",
      "Train Accuracy: 75.052 %\n",
      "[50,   391] loss: 0.785\n",
      "Train Accuracy: 76.634 %\n",
      "[60,   391] loss: 0.734\n",
      "Train Accuracy: 78.058 %\n",
      "[70,   391] loss: 0.705\n",
      "Train Accuracy: 79.324 %\n",
      "[80,   391] loss: 0.676\n",
      "Train Accuracy: 80.232 %\n",
      "[90,   391] loss: 0.640\n",
      "Train Accuracy: 81.200 %\n",
      "[100,   391] loss: 0.643\n",
      "Train Accuracy: 81.046 %\n",
      "[110,   391] loss: 0.602\n",
      "Train Accuracy: 82.234 %\n",
      "[120,   391] loss: 0.585\n",
      "Train Accuracy: 82.984 %\n",
      "[130,   391] loss: 0.575\n",
      "Train Accuracy: 83.214 %\n",
      "[140,   391] loss: 0.573\n",
      "Train Accuracy: 84.200 %\n",
      "[150,   391] loss: 0.551\n",
      "Train Accuracy: 84.012 %\n",
      "[160,   391] loss: 0.528\n",
      "Train Accuracy: 84.270 %\n",
      "[170,   391] loss: 0.531\n",
      "Train Accuracy: 84.210 %\n",
      "[180,   391] loss: 0.515\n",
      "Train Accuracy: 85.194 %\n",
      "[190,   391] loss: 0.504\n",
      "Train Accuracy: 85.378 %\n",
      "[200,   391] loss: 0.496\n",
      "Train Accuracy: 85.574 %\n",
      "[210,   391] loss: 0.487\n",
      "Train Accuracy: 85.090 %\n",
      "[220,   391] loss: 0.491\n",
      "Train Accuracy: 86.906 %\n",
      "[230,   391] loss: 0.464\n",
      "Train Accuracy: 86.690 %\n",
      "[240,   391] loss: 0.486\n",
      "Train Accuracy: 85.660 %\n",
      "[250,   391] loss: 0.445\n",
      "Train Accuracy: 86.644 %\n",
      "[260,   391] loss: 0.462\n",
      "Train Accuracy: 86.708 %\n",
      "[270,   391] loss: 0.458\n",
      "Train Accuracy: 87.604 %\n",
      "[280,   391] loss: 0.441\n",
      "Train Accuracy: 87.152 %\n",
      "[290,   391] loss: 0.420\n",
      "Train Accuracy: 86.676 %\n",
      "[300,   391] loss: 0.445\n",
      "Train Accuracy: 87.402 %\n",
      "[310,   391] loss: 0.428\n",
      "Train Accuracy: 88.158 %\n",
      "[320,   391] loss: 0.409\n",
      "Train Accuracy: 88.198 %\n",
      "[330,   391] loss: 0.411\n",
      "Train Accuracy: 87.632 %\n",
      "[340,   391] loss: 0.405\n",
      "Train Accuracy: 88.416 %\n",
      "[350,   391] loss: 0.410\n",
      "Train Accuracy: 87.910 %\n",
      "[360,   391] loss: 0.412\n",
      "Train Accuracy: 87.906 %\n",
      "[370,   391] loss: 0.411\n",
      "Train Accuracy: 87.310 %\n",
      "[380,   391] loss: 0.401\n",
      "Train Accuracy: 88.410 %\n",
      "[390,   391] loss: 0.400\n",
      "Train Accuracy: 88.762 %\n",
      "[400,   391] loss: 0.383\n",
      "Train Accuracy: 87.830 %\n",
      "[410,   391] loss: 0.376\n",
      "Train Accuracy: 89.266 %\n",
      "[420,   391] loss: 0.374\n",
      "Train Accuracy: 86.032 %\n",
      "[430,   391] loss: 0.369\n",
      "Train Accuracy: 87.408 %\n",
      "[440,   391] loss: 0.387\n",
      "Train Accuracy: 89.466 %\n",
      "[450,   391] loss: 0.387\n",
      "Train Accuracy: 88.496 %\n",
      "[460,   391] loss: 0.377\n",
      "Train Accuracy: 90.118 %\n",
      "[470,   391] loss: 0.350\n",
      "Train Accuracy: 89.878 %\n",
      "[480,   391] loss: 0.351\n",
      "Train Accuracy: 89.712 %\n",
      "[490,   391] loss: 0.365\n",
      "Train Accuracy: 88.112 %\n",
      "[500,   391] loss: 0.379\n",
      "Train Accuracy: 89.210 %\n",
      "[510,   391] loss: 0.355\n",
      "Train Accuracy: 89.766 %\n",
      "[520,   391] loss: 0.353\n",
      "Train Accuracy: 89.838 %\n",
      "[530,   391] loss: 0.343\n",
      "Train Accuracy: 89.432 %\n",
      "[540,   391] loss: 0.355\n",
      "Train Accuracy: 90.606 %\n",
      "[550,   391] loss: 0.363\n",
      "Train Accuracy: 89.678 %\n",
      "[560,   391] loss: 0.331\n",
      "Train Accuracy: 89.684 %\n",
      "[570,   391] loss: 0.318\n",
      "Train Accuracy: 89.968 %\n",
      "[580,   391] loss: 0.335\n",
      "Train Accuracy: 90.370 %\n",
      "[590,   391] loss: 0.317\n",
      "Train Accuracy: 89.652 %\n",
      "[600,   391] loss: 0.317\n",
      "Train Accuracy: 90.992 %\n",
      "[610,   391] loss: 0.323\n",
      "Train Accuracy: 90.032 %\n",
      "[620,   391] loss: 0.312\n",
      "Train Accuracy: 90.904 %\n",
      "[630,   391] loss: 0.370\n",
      "Train Accuracy: 88.962 %\n",
      "[640,   391] loss: 0.320\n",
      "Train Accuracy: 90.706 %\n",
      "[650,   391] loss: 0.313\n",
      "Train Accuracy: 89.736 %\n",
      "[660,   391] loss: 0.310\n",
      "Train Accuracy: 90.572 %\n",
      "[670,   391] loss: 0.309\n",
      "Train Accuracy: 90.512 %\n",
      "[680,   391] loss: 0.308\n",
      "Train Accuracy: 91.660 %\n",
      "[690,   391] loss: 0.294\n",
      "Train Accuracy: 91.310 %\n",
      "[700,   391] loss: 0.320\n",
      "Train Accuracy: 91.068 %\n",
      "[710,   391] loss: 0.294\n",
      "Train Accuracy: 90.578 %\n",
      "[720,   391] loss: 0.315\n",
      "Train Accuracy: 91.128 %\n",
      "[730,   391] loss: 0.298\n",
      "Train Accuracy: 91.218 %\n",
      "[740,   391] loss: 0.308\n",
      "Train Accuracy: 91.366 %\n",
      "[750,   391] loss: 0.319\n",
      "Train Accuracy: 92.348 %\n",
      "[760,   391] loss: 0.290\n",
      "Train Accuracy: 91.270 %\n",
      "[770,   391] loss: 0.287\n",
      "Train Accuracy: 90.466 %\n",
      "[780,   391] loss: 0.291\n",
      "Train Accuracy: 89.896 %\n",
      "[790,   391] loss: 0.302\n",
      "Train Accuracy: 90.624 %\n",
      "[800,   391] loss: 0.283\n",
      "Train Accuracy: 89.856 %\n",
      "[810,   391] loss: 0.283\n",
      "Train Accuracy: 90.918 %\n",
      "[820,   391] loss: 0.290\n",
      "Train Accuracy: 89.712 %\n",
      "[830,   391] loss: 0.296\n",
      "Train Accuracy: 90.258 %\n",
      "[840,   391] loss: 0.285\n",
      "Train Accuracy: 90.968 %\n",
      "[850,   391] loss: 0.280\n",
      "Train Accuracy: 91.314 %\n",
      "[860,   391] loss: 0.277\n",
      "Train Accuracy: 92.114 %\n",
      "[870,   391] loss: 0.277\n",
      "Train Accuracy: 92.188 %\n",
      "[880,   391] loss: 0.256\n",
      "Train Accuracy: 90.682 %\n",
      "[890,   391] loss: 0.270\n",
      "Train Accuracy: 92.104 %\n",
      "[900,   391] loss: 0.277\n",
      "Train Accuracy: 91.058 %\n",
      "[910,   391] loss: 0.279\n",
      "Train Accuracy: 91.682 %\n",
      "[920,   391] loss: 0.258\n",
      "Train Accuracy: 91.118 %\n",
      "[930,   391] loss: 0.257\n",
      "Train Accuracy: 91.644 %\n",
      "[940,   391] loss: 0.284\n",
      "Train Accuracy: 90.750 %\n",
      "[950,   391] loss: 0.271\n",
      "Train Accuracy: 92.178 %\n",
      "[960,   391] loss: 0.284\n",
      "Train Accuracy: 91.248 %\n",
      "[970,   391] loss: 0.252\n",
      "Train Accuracy: 89.370 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[980,   391] loss: 0.268\n",
      "Train Accuracy: 91.480 %\n",
      "[990,   391] loss: 0.253\n",
      "Train Accuracy: 91.004 %\n",
      "[1000,   391] loss: 0.252\n",
      "Train Accuracy: 92.296 %\n",
      "[1010,   391] loss: 0.250\n",
      "Train Accuracy: 92.884 %\n",
      "[1020,   391] loss: 0.275\n",
      "Train Accuracy: 90.944 %\n",
      "[1030,   391] loss: 0.296\n",
      "Train Accuracy: 90.922 %\n",
      "[1040,   391] loss: 0.249\n",
      "Train Accuracy: 91.560 %\n",
      "[1050,   391] loss: 0.276\n",
      "Train Accuracy: 90.842 %\n",
      "[1060,   391] loss: 0.289\n",
      "Train Accuracy: 90.488 %\n",
      "[1070,   391] loss: 0.249\n",
      "Train Accuracy: 92.862 %\n",
      "[1080,   391] loss: 0.277\n",
      "Train Accuracy: 90.678 %\n",
      "[1090,   391] loss: 0.246\n",
      "Train Accuracy: 92.724 %\n",
      "[1100,   391] loss: 0.254\n",
      "Train Accuracy: 92.160 %\n",
      "[1110,   391] loss: 0.262\n",
      "Train Accuracy: 91.566 %\n",
      "[1120,   391] loss: 0.260\n",
      "Train Accuracy: 92.126 %\n",
      "[1130,   391] loss: 0.252\n",
      "Train Accuracy: 91.232 %\n",
      "[1140,   391] loss: 0.261\n",
      "Train Accuracy: 91.868 %\n",
      "[1150,   391] loss: 0.228\n",
      "Train Accuracy: 92.428 %\n",
      "[1160,   391] loss: 0.254\n",
      "Train Accuracy: 93.490 %\n",
      "[1170,   391] loss: 0.251\n",
      "Train Accuracy: 91.784 %\n",
      "[1180,   391] loss: 0.253\n",
      "Train Accuracy: 89.906 %\n",
      "[1190,   391] loss: 0.242\n",
      "Train Accuracy: 91.778 %\n",
      "[1200,   391] loss: 0.261\n",
      "Train Accuracy: 93.080 %\n",
      "[1210,   391] loss: 0.252\n",
      "Train Accuracy: 91.808 %\n",
      "[1220,   391] loss: 0.240\n",
      "Train Accuracy: 92.916 %\n",
      "[1230,   391] loss: 0.264\n",
      "Train Accuracy: 93.574 %\n",
      "[1240,   391] loss: 0.260\n",
      "Train Accuracy: 91.232 %\n",
      "[1250,   391] loss: 0.253\n",
      "Train Accuracy: 92.754 %\n",
      "[1260,   391] loss: 0.248\n",
      "Train Accuracy: 93.246 %\n",
      "[1270,   391] loss: 0.237\n",
      "Train Accuracy: 93.488 %\n",
      "[1280,   391] loss: 0.236\n",
      "Train Accuracy: 92.402 %\n",
      "[1290,   391] loss: 0.226\n",
      "Train Accuracy: 91.130 %\n",
      "[1300,   391] loss: 0.230\n",
      "Train Accuracy: 93.518 %\n",
      "[1310,   391] loss: 0.250\n",
      "Train Accuracy: 92.402 %\n",
      "[1320,   391] loss: 0.242\n",
      "Train Accuracy: 93.142 %\n",
      "[1330,   391] loss: 0.256\n",
      "Train Accuracy: 91.778 %\n",
      "[1340,   391] loss: 0.226\n",
      "Train Accuracy: 92.552 %\n",
      "[1350,   391] loss: 0.248\n",
      "Train Accuracy: 90.988 %\n",
      "[1360,   391] loss: 0.207\n",
      "Train Accuracy: 92.978 %\n",
      "[1370,   391] loss: 0.229\n",
      "Train Accuracy: 92.692 %\n",
      "[1380,   391] loss: 0.245\n",
      "Train Accuracy: 92.476 %\n",
      "[1390,   391] loss: 0.219\n",
      "Train Accuracy: 93.254 %\n",
      "[1400,   391] loss: 0.236\n",
      "Train Accuracy: 92.682 %\n",
      "[1410,   391] loss: 0.271\n",
      "Train Accuracy: 90.802 %\n",
      "[1420,   391] loss: 0.227\n",
      "Train Accuracy: 92.902 %\n",
      "[1430,   391] loss: 0.223\n",
      "Train Accuracy: 93.364 %\n",
      "[1440,   391] loss: 0.236\n",
      "Train Accuracy: 91.888 %\n",
      "[1450,   391] loss: 0.209\n",
      "Train Accuracy: 93.744 %\n",
      "[1460,   391] loss: 0.204\n",
      "Train Accuracy: 91.182 %\n",
      "[1470,   391] loss: 0.252\n",
      "Train Accuracy: 92.086 %\n",
      "[1480,   391] loss: 0.230\n",
      "Train Accuracy: 91.910 %\n",
      "[1490,   391] loss: 0.218\n",
      "Train Accuracy: 92.712 %\n",
      "[1500,   391] loss: 0.208\n",
      "Train Accuracy: 92.886 %\n",
      "[1510,   391] loss: 0.261\n",
      "Train Accuracy: 91.606 %\n",
      "[1520,   391] loss: 0.215\n",
      "Train Accuracy: 93.312 %\n",
      "[1530,   391] loss: 0.194\n",
      "Train Accuracy: 94.616 %\n",
      "[1540,   391] loss: 0.187\n",
      "Train Accuracy: 94.230 %\n",
      "[1550,   391] loss: 0.235\n",
      "Train Accuracy: 92.408 %\n",
      "[1560,   391] loss: 0.225\n",
      "Train Accuracy: 93.420 %\n",
      "[1570,   391] loss: 0.215\n",
      "Train Accuracy: 93.980 %\n",
      "[1580,   391] loss: 0.233\n",
      "Train Accuracy: 94.692 %\n",
      "[1590,   391] loss: 0.225\n",
      "Train Accuracy: 93.416 %\n",
      "[1600,   391] loss: 0.242\n",
      "Train Accuracy: 91.636 %\n",
      "[1610,   391] loss: 0.209\n",
      "Train Accuracy: 91.696 %\n",
      "[1620,   391] loss: 0.234\n",
      "Train Accuracy: 92.726 %\n",
      "[1630,   391] loss: 0.243\n",
      "Train Accuracy: 91.960 %\n",
      "[1640,   391] loss: 0.228\n",
      "Train Accuracy: 93.374 %\n",
      "[1650,   391] loss: 0.202\n",
      "Train Accuracy: 93.100 %\n",
      "[1660,   391] loss: 0.220\n",
      "Train Accuracy: 94.728 %\n",
      "[1670,   391] loss: 0.217\n",
      "Train Accuracy: 93.442 %\n",
      "[1680,   391] loss: 0.211\n",
      "Train Accuracy: 90.152 %\n",
      "[1690,   391] loss: 0.244\n",
      "Train Accuracy: 90.902 %\n",
      "[1700,   391] loss: 0.209\n",
      "Train Accuracy: 93.876 %\n",
      "[1710,   391] loss: 0.222\n",
      "Train Accuracy: 92.742 %\n",
      "[1720,   391] loss: 0.211\n",
      "Train Accuracy: 92.840 %\n",
      "[1730,   391] loss: 0.245\n",
      "Train Accuracy: 92.978 %\n",
      "[1740,   391] loss: 0.232\n",
      "Train Accuracy: 93.200 %\n",
      "[1750,   391] loss: 0.237\n",
      "Train Accuracy: 93.762 %\n",
      "[1760,   391] loss: 0.239\n",
      "Train Accuracy: 90.562 %\n",
      "[1770,   391] loss: 0.218\n",
      "Train Accuracy: 92.408 %\n",
      "[1780,   391] loss: 0.210\n",
      "Train Accuracy: 93.560 %\n",
      "[1790,   391] loss: 0.235\n",
      "Train Accuracy: 90.104 %\n",
      "[1800,   391] loss: 0.251\n",
      "Train Accuracy: 93.032 %\n",
      "[1810,   391] loss: 0.230\n",
      "Train Accuracy: 91.210 %\n",
      "[1820,   391] loss: 0.236\n",
      "Train Accuracy: 92.304 %\n",
      "[1830,   391] loss: 0.224\n",
      "Train Accuracy: 92.928 %\n",
      "[1840,   391] loss: 0.227\n",
      "Train Accuracy: 92.874 %\n",
      "[1850,   391] loss: 0.277\n",
      "Train Accuracy: 93.260 %\n",
      "[1860,   391] loss: 0.255\n",
      "Train Accuracy: 91.430 %\n",
      "[1870,   391] loss: 0.267\n",
      "Train Accuracy: 91.078 %\n",
      "[1880,   391] loss: 0.218\n",
      "Train Accuracy: 94.140 %\n",
      "[1890,   391] loss: 0.193\n",
      "Train Accuracy: 88.596 %\n",
      "[1900,   391] loss: 0.207\n",
      "Train Accuracy: 94.224 %\n",
      "[1910,   391] loss: 0.241\n",
      "Train Accuracy: 92.192 %\n",
      "[1920,   391] loss: 0.205\n",
      "Train Accuracy: 93.352 %\n",
      "[1930,   391] loss: 0.242\n",
      "Train Accuracy: 93.306 %\n",
      "[1940,   391] loss: 0.251\n",
      "Train Accuracy: 91.760 %\n",
      "[1950,   391] loss: 0.193\n",
      "Train Accuracy: 93.358 %\n",
      "[1960,   391] loss: 0.178\n",
      "Train Accuracy: 93.302 %\n",
      "[1970,   391] loss: 0.217\n",
      "Train Accuracy: 92.946 %\n",
      "[1980,   391] loss: 0.207\n",
      "Train Accuracy: 92.700 %\n",
      "[1990,   391] loss: 0.215\n",
      "Train Accuracy: 91.240 %\n",
      "[2000,   391] loss: 0.217\n",
      "Train Accuracy: 92.414 %\n",
      "[2010,   391] loss: 0.296\n",
      "Train Accuracy: 92.558 %\n",
      "[2020,   391] loss: 0.252\n",
      "Train Accuracy: 92.286 %\n",
      "[2030,   391] loss: 0.234\n",
      "Train Accuracy: 91.578 %\n",
      "[2040,   391] loss: 0.240\n",
      "Train Accuracy: 93.030 %\n",
      "[2050,   391] loss: 0.195\n",
      "Train Accuracy: 93.036 %\n",
      "[2060,   391] loss: 0.225\n",
      "Train Accuracy: 93.222 %\n",
      "[2070,   391] loss: 0.190\n",
      "Train Accuracy: 94.530 %\n",
      "[2080,   391] loss: 0.217\n",
      "Train Accuracy: 93.290 %\n",
      "[2090,   391] loss: 0.219\n",
      "Train Accuracy: 93.758 %\n",
      "[2100,   391] loss: 0.253\n",
      "Train Accuracy: 91.560 %\n",
      "[2110,   391] loss: 0.254\n",
      "Train Accuracy: 92.616 %\n",
      "[2120,   391] loss: 0.209\n",
      "Train Accuracy: 93.372 %\n",
      "[2130,   391] loss: 0.228\n",
      "Train Accuracy: 93.762 %\n",
      "[2140,   391] loss: 0.212\n",
      "Train Accuracy: 92.396 %\n",
      "[2150,   391] loss: 0.235\n",
      "Train Accuracy: 93.214 %\n",
      "[2160,   391] loss: 0.194\n",
      "Train Accuracy: 93.648 %\n",
      "[2170,   391] loss: 0.219\n",
      "Train Accuracy: 92.928 %\n",
      "[2180,   391] loss: 0.231\n",
      "Train Accuracy: 93.982 %\n",
      "[2190,   391] loss: 0.169\n",
      "Train Accuracy: 93.576 %\n",
      "[2200,   391] loss: 0.217\n",
      "Train Accuracy: 94.216 %\n",
      "[2210,   391] loss: 0.204\n",
      "Train Accuracy: 92.652 %\n",
      "[2220,   391] loss: 0.228\n",
      "Train Accuracy: 92.526 %\n",
      "[2230,   391] loss: 0.249\n",
      "Train Accuracy: 90.538 %\n",
      "[2240,   391] loss: 0.212\n",
      "Train Accuracy: 93.686 %\n",
      "[2250,   391] loss: 0.212\n",
      "Train Accuracy: 94.388 %\n",
      "[2260,   391] loss: 0.211\n",
      "Train Accuracy: 94.204 %\n",
      "[2270,   391] loss: 0.253\n",
      "Train Accuracy: 91.618 %\n",
      "[2280,   391] loss: 0.221\n",
      "Train Accuracy: 92.566 %\n",
      "[2290,   391] loss: 0.238\n",
      "Train Accuracy: 92.272 %\n",
      "[2300,   391] loss: 0.249\n",
      "Train Accuracy: 92.492 %\n",
      "[2310,   391] loss: 0.221\n",
      "Train Accuracy: 92.736 %\n",
      "[2320,   391] loss: 0.229\n",
      "Train Accuracy: 93.618 %\n",
      "[2330,   391] loss: 0.221\n",
      "Train Accuracy: 94.226 %\n",
      "[2340,   391] loss: 0.260\n",
      "Train Accuracy: 92.714 %\n",
      "[2350,   391] loss: 0.178\n",
      "Train Accuracy: 94.632 %\n",
      "[2360,   391] loss: 0.209\n",
      "Train Accuracy: 92.448 %\n",
      "[2370,   391] loss: 0.195\n",
      "Train Accuracy: 93.298 %\n",
      "[2380,   391] loss: 0.232\n",
      "Train Accuracy: 93.094 %\n",
      "[2390,   391] loss: 0.220\n",
      "Train Accuracy: 90.646 %\n",
      "[2400,   391] loss: 0.229\n",
      "Train Accuracy: 94.126 %\n",
      "[2410,   391] loss: 0.179\n",
      "Train Accuracy: 94.392 %\n",
      "[2420,   391] loss: 0.194\n",
      "Train Accuracy: 94.144 %\n",
      "[2430,   391] loss: 0.217\n",
      "Train Accuracy: 93.264 %\n",
      "[2440,   391] loss: 0.199\n",
      "Train Accuracy: 92.208 %\n",
      "[2450,   391] loss: 0.231\n",
      "Train Accuracy: 92.226 %\n",
      "[2460,   391] loss: 0.202\n",
      "Train Accuracy: 93.714 %\n",
      "[2470,   391] loss: 0.223\n",
      "Train Accuracy: 90.576 %\n",
      "[2480,   391] loss: 0.221\n",
      "Train Accuracy: 90.868 %\n",
      "[2490,   391] loss: 0.217\n",
      "Train Accuracy: 89.700 %\n",
      "[2500,   391] loss: 0.209\n",
      "Train Accuracy: 93.426 %\n",
      "[2510,   391] loss: 0.165\n",
      "Train Accuracy: 94.018 %\n",
      "[2520,   391] loss: 0.233\n",
      "Train Accuracy: 92.406 %\n",
      "[2530,   391] loss: 0.236\n",
      "Train Accuracy: 91.024 %\n",
      "[2540,   391] loss: 0.270\n",
      "Train Accuracy: 93.362 %\n",
      "[2550,   391] loss: 0.228\n",
      "Train Accuracy: 93.536 %\n",
      "[2560,   391] loss: 0.210\n",
      "Train Accuracy: 94.640 %\n",
      "[2570,   391] loss: 0.197\n",
      "Train Accuracy: 91.994 %\n",
      "[2580,   391] loss: 0.204\n",
      "Train Accuracy: 93.194 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2590,   391] loss: 0.242\n",
      "Train Accuracy: 92.128 %\n",
      "[2600,   391] loss: 0.271\n",
      "Train Accuracy: 91.244 %\n",
      "[2610,   391] loss: 0.242\n",
      "Train Accuracy: 93.082 %\n",
      "[2620,   391] loss: 0.190\n",
      "Train Accuracy: 92.358 %\n",
      "[2630,   391] loss: 0.289\n",
      "Train Accuracy: 93.116 %\n",
      "[2640,   391] loss: 0.213\n",
      "Train Accuracy: 91.782 %\n",
      "[2650,   391] loss: 0.216\n",
      "Train Accuracy: 91.896 %\n",
      "[2660,   391] loss: 0.230\n",
      "Train Accuracy: 92.920 %\n",
      "[2670,   391] loss: 0.190\n",
      "Train Accuracy: 94.580 %\n",
      "[2680,   391] loss: 0.191\n",
      "Train Accuracy: 93.966 %\n",
      "[2690,   391] loss: 0.237\n",
      "Train Accuracy: 93.188 %\n",
      "[2700,   391] loss: 0.251\n",
      "Train Accuracy: 93.532 %\n",
      "[2710,   391] loss: 0.202\n",
      "Train Accuracy: 93.528 %\n",
      "[2720,   391] loss: 0.154\n",
      "Train Accuracy: 94.088 %\n",
      "[2730,   391] loss: 0.182\n",
      "Train Accuracy: 94.872 %\n",
      "[2740,   391] loss: 0.191\n",
      "Train Accuracy: 95.418 %\n",
      "[2750,   391] loss: 0.275\n",
      "Train Accuracy: 92.428 %\n",
      "[2760,   391] loss: 0.220\n",
      "Train Accuracy: 92.094 %\n",
      "[2770,   391] loss: 0.222\n",
      "Train Accuracy: 94.446 %\n",
      "[2780,   391] loss: 0.223\n",
      "Train Accuracy: 91.564 %\n",
      "[2790,   391] loss: 0.200\n",
      "Train Accuracy: 94.320 %\n",
      "[2800,   391] loss: 0.289\n",
      "Train Accuracy: 91.932 %\n",
      "[2810,   391] loss: 0.207\n",
      "Train Accuracy: 91.786 %\n",
      "[2820,   391] loss: 0.214\n",
      "Train Accuracy: 93.434 %\n",
      "[2830,   391] loss: 0.211\n",
      "Train Accuracy: 92.646 %\n",
      "[2840,   391] loss: 0.251\n",
      "Train Accuracy: 93.722 %\n",
      "[2850,   391] loss: 0.173\n",
      "Train Accuracy: 94.528 %\n",
      "[2860,   391] loss: 0.182\n",
      "Train Accuracy: 93.094 %\n",
      "[2870,   391] loss: 0.182\n",
      "Train Accuracy: 94.062 %\n",
      "[2880,   391] loss: 0.200\n",
      "Train Accuracy: 93.732 %\n",
      "[2890,   391] loss: 0.198\n",
      "Train Accuracy: 95.138 %\n",
      "[2900,   391] loss: 0.236\n",
      "Train Accuracy: 92.306 %\n",
      "[2910,   391] loss: 0.194\n",
      "Train Accuracy: 92.986 %\n",
      "[2920,   391] loss: 0.220\n",
      "Train Accuracy: 94.082 %\n",
      "[2930,   391] loss: 0.191\n",
      "Train Accuracy: 94.340 %\n",
      "[2940,   391] loss: 0.257\n",
      "Train Accuracy: 92.024 %\n",
      "[2950,   391] loss: 0.168\n",
      "Train Accuracy: 95.460 %\n",
      "[2960,   391] loss: 0.169\n",
      "Train Accuracy: 93.486 %\n",
      "[2970,   391] loss: 0.231\n",
      "Train Accuracy: 89.222 %\n",
      "[2980,   391] loss: 0.194\n",
      "Train Accuracy: 94.716 %\n",
      "[2990,   391] loss: 0.205\n",
      "Train Accuracy: 95.164 %\n",
      "[3000,   391] loss: 0.251\n",
      "Train Accuracy: 93.240 %\n",
      "[3010,   391] loss: 0.236\n",
      "Train Accuracy: 93.282 %\n",
      "[3020,   391] loss: 0.192\n",
      "Train Accuracy: 93.826 %\n",
      "[3030,   391] loss: 0.167\n",
      "Train Accuracy: 94.874 %\n",
      "[3040,   391] loss: 0.260\n",
      "Train Accuracy: 92.786 %\n",
      "[3050,   391] loss: 0.207\n",
      "Train Accuracy: 94.608 %\n",
      "[3060,   391] loss: 0.184\n",
      "Train Accuracy: 93.860 %\n",
      "[3070,   391] loss: 0.219\n",
      "Train Accuracy: 93.168 %\n",
      "[3080,   391] loss: 0.279\n",
      "Train Accuracy: 91.846 %\n",
      "[3090,   391] loss: 0.208\n",
      "Train Accuracy: 94.244 %\n",
      "[3100,   391] loss: 0.237\n",
      "Train Accuracy: 93.256 %\n",
      "[3110,   391] loss: 0.177\n",
      "Train Accuracy: 94.766 %\n",
      "[3120,   391] loss: 0.186\n",
      "Train Accuracy: 92.886 %\n",
      "[3130,   391] loss: 0.216\n",
      "Train Accuracy: 93.986 %\n",
      "[3140,   391] loss: 0.264\n",
      "Train Accuracy: 92.822 %\n",
      "[3150,   391] loss: 0.201\n",
      "Train Accuracy: 92.986 %\n",
      "[3160,   391] loss: 0.170\n",
      "Train Accuracy: 94.746 %\n",
      "[3170,   391] loss: 0.221\n",
      "Train Accuracy: 93.180 %\n",
      "[3180,   391] loss: 0.221\n",
      "Train Accuracy: 89.970 %\n",
      "[3190,   391] loss: 0.259\n",
      "Train Accuracy: 92.508 %\n",
      "[3200,   391] loss: 0.219\n",
      "Train Accuracy: 93.952 %\n",
      "[3210,   391] loss: 0.183\n",
      "Train Accuracy: 93.826 %\n",
      "[3220,   391] loss: 0.231\n",
      "Train Accuracy: 93.568 %\n",
      "[3230,   391] loss: 0.229\n",
      "Train Accuracy: 93.120 %\n",
      "[3240,   391] loss: 0.147\n",
      "Train Accuracy: 95.320 %\n",
      "[3250,   391] loss: 0.209\n",
      "Train Accuracy: 94.304 %\n",
      "[3260,   391] loss: 0.197\n",
      "Train Accuracy: 94.740 %\n",
      "[3270,   391] loss: 0.179\n",
      "Train Accuracy: 92.916 %\n",
      "[3280,   391] loss: 0.175\n",
      "Train Accuracy: 93.872 %\n",
      "[3290,   391] loss: 0.160\n",
      "Train Accuracy: 93.648 %\n",
      "[3300,   391] loss: 0.241\n",
      "Train Accuracy: 93.972 %\n",
      "[3310,   391] loss: 0.222\n",
      "Train Accuracy: 94.424 %\n",
      "[3320,   391] loss: 0.221\n",
      "Train Accuracy: 92.990 %\n",
      "[3330,   391] loss: 0.222\n",
      "Train Accuracy: 93.200 %\n",
      "[3340,   391] loss: 0.207\n",
      "Train Accuracy: 93.672 %\n",
      "[3350,   391] loss: 0.271\n",
      "Train Accuracy: 93.554 %\n",
      "[3360,   391] loss: 0.246\n",
      "Train Accuracy: 93.968 %\n",
      "[3370,   391] loss: 0.146\n",
      "Train Accuracy: 93.568 %\n",
      "[3380,   391] loss: 0.204\n",
      "Train Accuracy: 94.562 %\n",
      "[3390,   391] loss: 0.195\n",
      "Train Accuracy: 92.040 %\n",
      "[3400,   391] loss: 0.193\n",
      "Train Accuracy: 93.460 %\n",
      "[3410,   391] loss: 0.193\n",
      "Train Accuracy: 92.406 %\n",
      "[3420,   391] loss: 0.233\n",
      "Train Accuracy: 91.944 %\n",
      "[3430,   391] loss: 0.181\n",
      "Train Accuracy: 94.418 %\n",
      "[3440,   391] loss: 0.189\n",
      "Train Accuracy: 90.990 %\n",
      "[3450,   391] loss: 0.196\n",
      "Train Accuracy: 95.146 %\n",
      "[3460,   391] loss: 0.241\n",
      "Train Accuracy: 90.252 %\n",
      "[3470,   391] loss: 0.202\n",
      "Train Accuracy: 93.824 %\n",
      "[3480,   391] loss: 0.312\n",
      "Train Accuracy: 92.444 %\n",
      "[3490,   391] loss: 0.260\n",
      "Train Accuracy: 93.282 %\n",
      "[3500,   391] loss: 0.220\n",
      "Train Accuracy: 94.522 %\n",
      "[3510,   391] loss: 0.158\n",
      "Train Accuracy: 92.218 %\n",
      "[3520,   391] loss: 0.202\n",
      "Train Accuracy: 95.392 %\n",
      "[3530,   391] loss: 0.229\n",
      "Train Accuracy: 94.098 %\n",
      "[3540,   391] loss: 0.243\n",
      "Train Accuracy: 92.356 %\n",
      "[3550,   391] loss: 0.224\n",
      "Train Accuracy: 93.522 %\n",
      "[3560,   391] loss: 0.173\n",
      "Train Accuracy: 93.426 %\n",
      "[3570,   391] loss: 0.270\n",
      "Train Accuracy: 91.482 %\n",
      "[3580,   391] loss: 0.182\n",
      "Train Accuracy: 93.826 %\n",
      "[3590,   391] loss: 0.210\n",
      "Train Accuracy: 90.548 %\n",
      "[3600,   391] loss: 0.238\n",
      "Train Accuracy: 93.934 %\n",
      "[3610,   391] loss: 0.226\n",
      "Train Accuracy: 93.322 %\n",
      "[3620,   391] loss: 0.184\n",
      "Train Accuracy: 94.076 %\n",
      "[3630,   391] loss: 0.195\n",
      "Train Accuracy: 92.198 %\n",
      "[3640,   391] loss: 0.233\n",
      "Train Accuracy: 91.808 %\n",
      "[3650,   391] loss: 0.246\n",
      "Train Accuracy: 91.704 %\n",
      "[3660,   391] loss: 0.148\n",
      "Train Accuracy: 93.536 %\n",
      "[3670,   391] loss: 0.187\n",
      "Train Accuracy: 92.786 %\n",
      "[3680,   391] loss: 0.200\n",
      "Train Accuracy: 95.110 %\n",
      "[3690,   391] loss: 0.170\n",
      "Train Accuracy: 95.380 %\n",
      "[3700,   391] loss: 0.182\n",
      "Train Accuracy: 94.666 %\n",
      "[3710,   391] loss: 0.200\n",
      "Train Accuracy: 93.710 %\n",
      "[3720,   391] loss: 0.199\n",
      "Train Accuracy: 94.626 %\n",
      "[3730,   391] loss: 0.210\n",
      "Train Accuracy: 94.544 %\n",
      "[3740,   391] loss: 0.235\n",
      "Train Accuracy: 93.044 %\n",
      "[3750,   391] loss: 0.208\n",
      "Train Accuracy: 94.846 %\n",
      "[3760,   391] loss: 0.199\n",
      "Train Accuracy: 92.660 %\n",
      "[3770,   391] loss: 0.222\n",
      "Train Accuracy: 90.670 %\n",
      "[3780,   391] loss: 0.218\n",
      "Train Accuracy: 92.164 %\n",
      "[3790,   391] loss: 0.224\n",
      "Train Accuracy: 93.126 %\n",
      "[3800,   391] loss: 0.205\n",
      "Train Accuracy: 93.594 %\n",
      "[3810,   391] loss: 0.177\n",
      "Train Accuracy: 93.970 %\n",
      "[3820,   391] loss: 0.204\n",
      "Train Accuracy: 94.486 %\n",
      "[3830,   391] loss: 0.207\n",
      "Train Accuracy: 92.692 %\n",
      "[3840,   391] loss: 0.221\n",
      "Train Accuracy: 93.676 %\n",
      "[3850,   391] loss: 0.237\n",
      "Train Accuracy: 94.422 %\n",
      "[3860,   391] loss: 0.206\n",
      "Train Accuracy: 93.852 %\n",
      "[3870,   391] loss: 0.229\n",
      "Train Accuracy: 93.710 %\n",
      "[3880,   391] loss: 0.222\n",
      "Train Accuracy: 94.268 %\n",
      "[3890,   391] loss: 0.204\n",
      "Train Accuracy: 94.568 %\n",
      "[3900,   391] loss: 0.192\n",
      "Train Accuracy: 95.254 %\n",
      "[3910,   391] loss: 0.194\n",
      "Train Accuracy: 95.064 %\n",
      "[3920,   391] loss: 0.180\n",
      "Train Accuracy: 91.112 %\n",
      "[3930,   391] loss: 0.194\n",
      "Train Accuracy: 94.278 %\n",
      "[3940,   391] loss: 0.183\n",
      "Train Accuracy: 94.306 %\n",
      "[3950,   391] loss: 0.221\n",
      "Train Accuracy: 93.644 %\n",
      "[3960,   391] loss: 0.218\n",
      "Train Accuracy: 92.616 %\n",
      "[3970,   391] loss: 0.172\n",
      "Train Accuracy: 90.218 %\n",
      "[3980,   391] loss: 0.216\n",
      "Train Accuracy: 92.790 %\n",
      "[3990,   391] loss: 0.189\n",
      "Train Accuracy: 95.066 %\n",
      "[4000,   391] loss: 0.282\n",
      "Train Accuracy: 92.714 %\n",
      "[4010,   391] loss: 0.211\n",
      "Train Accuracy: 94.710 %\n",
      "[4020,   391] loss: 0.216\n",
      "Train Accuracy: 92.624 %\n",
      "[4030,   391] loss: 0.361\n",
      "Train Accuracy: 91.180 %\n",
      "[4040,   391] loss: 0.285\n",
      "Train Accuracy: 90.814 %\n",
      "[4050,   391] loss: 0.234\n",
      "Train Accuracy: 92.898 %\n",
      "[4060,   391] loss: 0.170\n",
      "Train Accuracy: 94.504 %\n",
      "[4070,   391] loss: 0.239\n",
      "Train Accuracy: 91.072 %\n",
      "[4080,   391] loss: 0.257\n",
      "Train Accuracy: 92.012 %\n",
      "[4090,   391] loss: 0.196\n",
      "Train Accuracy: 94.618 %\n",
      "[4100,   391] loss: 0.211\n",
      "Train Accuracy: 93.426 %\n",
      "[4110,   391] loss: 0.145\n",
      "Train Accuracy: 94.756 %\n",
      "[4120,   391] loss: 0.199\n",
      "Train Accuracy: 92.716 %\n",
      "[4130,   391] loss: 0.183\n",
      "Train Accuracy: 91.114 %\n",
      "[4140,   391] loss: 0.224\n",
      "Train Accuracy: 93.748 %\n",
      "[4150,   391] loss: 0.205\n",
      "Train Accuracy: 91.290 %\n",
      "[4160,   391] loss: 0.197\n",
      "Train Accuracy: 93.984 %\n",
      "[4170,   391] loss: 0.268\n",
      "Train Accuracy: 90.456 %\n",
      "[4180,   391] loss: 0.254\n",
      "Train Accuracy: 90.878 %\n",
      "[4190,   391] loss: 0.283\n",
      "Train Accuracy: 92.684 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4200,   391] loss: 0.182\n",
      "Train Accuracy: 93.480 %\n",
      "[4210,   391] loss: 0.230\n",
      "Train Accuracy: 94.164 %\n",
      "[4220,   391] loss: 0.172\n",
      "Train Accuracy: 94.304 %\n",
      "[4230,   391] loss: 0.221\n",
      "Train Accuracy: 94.054 %\n",
      "[4240,   391] loss: 0.233\n",
      "Train Accuracy: 91.192 %\n",
      "[4250,   391] loss: 0.172\n",
      "Train Accuracy: 92.704 %\n",
      "[4260,   391] loss: 0.252\n",
      "Train Accuracy: 91.130 %\n",
      "[4270,   391] loss: 0.157\n",
      "Train Accuracy: 94.680 %\n",
      "[4280,   391] loss: 0.206\n",
      "Train Accuracy: 92.976 %\n",
      "[4290,   391] loss: 0.171\n",
      "Train Accuracy: 93.504 %\n",
      "[4300,   391] loss: 0.170\n",
      "Train Accuracy: 92.790 %\n",
      "[4310,   391] loss: 0.184\n",
      "Train Accuracy: 94.396 %\n",
      "[4320,   391] loss: 0.216\n",
      "Train Accuracy: 92.926 %\n",
      "[4330,   391] loss: 0.217\n",
      "Train Accuracy: 94.386 %\n",
      "[4340,   391] loss: 0.227\n",
      "Train Accuracy: 95.250 %\n",
      "[4350,   391] loss: 0.200\n",
      "Train Accuracy: 90.920 %\n",
      "[4360,   391] loss: 0.237\n",
      "Train Accuracy: 93.676 %\n",
      "[4370,   391] loss: 0.126\n",
      "Train Accuracy: 92.062 %\n",
      "[4380,   391] loss: 0.235\n",
      "Train Accuracy: 93.184 %\n",
      "[4390,   391] loss: 0.142\n",
      "Train Accuracy: 93.026 %\n",
      "[4400,   391] loss: 0.187\n",
      "Train Accuracy: 94.828 %\n",
      "[4410,   391] loss: 0.188\n",
      "Train Accuracy: 91.574 %\n",
      "[4420,   391] loss: 0.168\n",
      "Train Accuracy: 94.240 %\n",
      "[4430,   391] loss: 0.234\n",
      "Train Accuracy: 94.396 %\n",
      "[4440,   391] loss: 0.219\n",
      "Train Accuracy: 92.556 %\n",
      "[4450,   391] loss: 0.206\n",
      "Train Accuracy: 93.752 %\n",
      "[4460,   391] loss: 0.215\n",
      "Train Accuracy: 93.530 %\n",
      "[4470,   391] loss: 0.288\n",
      "Train Accuracy: 91.748 %\n",
      "[4480,   391] loss: 0.207\n",
      "Train Accuracy: 94.648 %\n",
      "[4490,   391] loss: 0.161\n",
      "Train Accuracy: 93.870 %\n",
      "[4500,   391] loss: 0.307\n",
      "Train Accuracy: 91.740 %\n",
      "[4510,   391] loss: 0.226\n",
      "Train Accuracy: 94.412 %\n",
      "[4520,   391] loss: 0.222\n",
      "Train Accuracy: 92.722 %\n",
      "[4530,   391] loss: 0.246\n",
      "Train Accuracy: 92.982 %\n",
      "[4540,   391] loss: 0.278\n",
      "Train Accuracy: 94.362 %\n",
      "[4550,   391] loss: 0.257\n",
      "Train Accuracy: 91.360 %\n",
      "[4560,   391] loss: 0.207\n",
      "Train Accuracy: 92.446 %\n",
      "[4570,   391] loss: 0.201\n",
      "Train Accuracy: 93.388 %\n",
      "[4580,   391] loss: 0.225\n",
      "Train Accuracy: 92.782 %\n",
      "[4590,   391] loss: 0.197\n",
      "Train Accuracy: 91.284 %\n",
      "[4600,   391] loss: 0.209\n",
      "Train Accuracy: 93.328 %\n",
      "[4610,   391] loss: 0.204\n",
      "Train Accuracy: 94.290 %\n",
      "[4620,   391] loss: 0.164\n",
      "Train Accuracy: 93.246 %\n",
      "[4630,   391] loss: 0.258\n",
      "Train Accuracy: 90.856 %\n",
      "[4640,   391] loss: 0.243\n",
      "Train Accuracy: 93.634 %\n",
      "[4650,   391] loss: 0.200\n",
      "Train Accuracy: 92.750 %\n",
      "[4660,   391] loss: 0.188\n",
      "Train Accuracy: 93.162 %\n",
      "[4670,   391] loss: 0.201\n",
      "Train Accuracy: 93.968 %\n",
      "[4680,   391] loss: 0.236\n",
      "Train Accuracy: 92.728 %\n",
      "[4690,   391] loss: 0.227\n",
      "Train Accuracy: 93.654 %\n",
      "[4700,   391] loss: 0.240\n",
      "Train Accuracy: 93.340 %\n",
      "[4710,   391] loss: 0.197\n",
      "Train Accuracy: 95.070 %\n",
      "[4720,   391] loss: 0.304\n",
      "Train Accuracy: 91.756 %\n",
      "[4730,   391] loss: 0.195\n",
      "Train Accuracy: 94.354 %\n",
      "[4740,   391] loss: 0.175\n",
      "Train Accuracy: 95.386 %\n",
      "[4750,   391] loss: 0.206\n",
      "Train Accuracy: 94.780 %\n",
      "[4760,   391] loss: 0.184\n",
      "Train Accuracy: 95.538 %\n",
      "[4770,   391] loss: 0.205\n",
      "Train Accuracy: 95.116 %\n",
      "[4780,   391] loss: 0.209\n",
      "Train Accuracy: 89.300 %\n",
      "[4790,   391] loss: 0.199\n",
      "Train Accuracy: 92.716 %\n",
      "[4800,   391] loss: 0.233\n",
      "Train Accuracy: 93.878 %\n",
      "[4810,   391] loss: 0.206\n",
      "Train Accuracy: 93.656 %\n",
      "[4820,   391] loss: 0.204\n",
      "Train Accuracy: 92.252 %\n",
      "[4830,   391] loss: 0.211\n",
      "Train Accuracy: 94.812 %\n",
      "[4840,   391] loss: 0.179\n",
      "Train Accuracy: 93.454 %\n",
      "[4850,   391] loss: 0.177\n",
      "Train Accuracy: 95.238 %\n",
      "[4860,   391] loss: 0.183\n",
      "Train Accuracy: 92.100 %\n",
      "[4870,   391] loss: 0.180\n",
      "Train Accuracy: 93.480 %\n",
      "[4880,   391] loss: 0.205\n",
      "Train Accuracy: 90.160 %\n",
      "[4890,   391] loss: 0.269\n",
      "Train Accuracy: 93.638 %\n",
      "[4900,   391] loss: 0.178\n",
      "Train Accuracy: 94.406 %\n",
      "[4910,   391] loss: 0.199\n",
      "Train Accuracy: 93.840 %\n",
      "[4920,   391] loss: 0.209\n",
      "Train Accuracy: 94.792 %\n",
      "[4930,   391] loss: 0.154\n",
      "Train Accuracy: 94.934 %\n",
      "[4940,   391] loss: 0.183\n",
      "Train Accuracy: 91.052 %\n",
      "[4950,   391] loss: 0.150\n",
      "Train Accuracy: 93.554 %\n",
      "[4960,   391] loss: 0.225\n",
      "Train Accuracy: 93.164 %\n",
      "[4970,   391] loss: 0.229\n",
      "Train Accuracy: 94.184 %\n",
      "[4980,   391] loss: 0.165\n",
      "Train Accuracy: 93.340 %\n",
      "[4990,   391] loss: 0.190\n",
      "Train Accuracy: 94.888 %\n",
      "[5000,   391] loss: 0.157\n",
      "Train Accuracy: 93.108 %\n",
      "[5010,   391] loss: 0.250\n",
      "Train Accuracy: 94.484 %\n",
      "[5020,   391] loss: 0.182\n",
      "Train Accuracy: 94.820 %\n",
      "[5030,   391] loss: 0.212\n",
      "Train Accuracy: 94.548 %\n",
      "[5040,   391] loss: 0.158\n",
      "Train Accuracy: 93.924 %\n",
      "[5050,   391] loss: 0.143\n",
      "Train Accuracy: 95.776 %\n",
      "[5060,   391] loss: 0.292\n",
      "Train Accuracy: 91.328 %\n",
      "[5070,   391] loss: 0.161\n",
      "Train Accuracy: 94.408 %\n",
      "[5080,   391] loss: 0.221\n",
      "Train Accuracy: 91.116 %\n",
      "[5090,   391] loss: 0.248\n",
      "Train Accuracy: 93.734 %\n",
      "[5100,   391] loss: 0.208\n",
      "Train Accuracy: 93.702 %\n",
      "[5110,   391] loss: 0.148\n",
      "Train Accuracy: 95.434 %\n",
      "[5120,   391] loss: 0.320\n",
      "Train Accuracy: 93.940 %\n",
      "[5130,   391] loss: 0.190\n",
      "Train Accuracy: 93.474 %\n",
      "[5140,   391] loss: 0.171\n",
      "Train Accuracy: 95.842 %\n",
      "[5150,   391] loss: 0.201\n",
      "Train Accuracy: 94.668 %\n",
      "[5160,   391] loss: 0.150\n",
      "Train Accuracy: 93.832 %\n",
      "[5170,   391] loss: 0.248\n",
      "Train Accuracy: 89.792 %\n",
      "[5180,   391] loss: 0.208\n",
      "Train Accuracy: 92.794 %\n",
      "[5190,   391] loss: 0.253\n",
      "Train Accuracy: 92.858 %\n",
      "[5200,   391] loss: 0.172\n",
      "Train Accuracy: 95.144 %\n",
      "[5210,   391] loss: 0.170\n",
      "Train Accuracy: 95.346 %\n",
      "[5220,   391] loss: 0.170\n",
      "Train Accuracy: 92.944 %\n",
      "[5230,   391] loss: 0.180\n",
      "Train Accuracy: 95.228 %\n",
      "[5240,   391] loss: 0.181\n",
      "Train Accuracy: 94.504 %\n",
      "[5250,   391] loss: 0.213\n",
      "Train Accuracy: 89.656 %\n",
      "[5260,   391] loss: 0.224\n",
      "Train Accuracy: 94.796 %\n",
      "[5270,   391] loss: 0.198\n",
      "Train Accuracy: 92.558 %\n",
      "[5280,   391] loss: 0.258\n",
      "Train Accuracy: 93.146 %\n",
      "[5290,   391] loss: 0.189\n",
      "Train Accuracy: 93.574 %\n",
      "[5300,   391] loss: 0.198\n",
      "Train Accuracy: 94.624 %\n",
      "[5310,   391] loss: 0.171\n",
      "Train Accuracy: 95.432 %\n",
      "[5320,   391] loss: 0.210\n",
      "Train Accuracy: 93.566 %\n",
      "[5330,   391] loss: 0.206\n",
      "Train Accuracy: 92.968 %\n",
      "[5340,   391] loss: 0.203\n",
      "Train Accuracy: 93.854 %\n",
      "[5350,   391] loss: 0.173\n",
      "Train Accuracy: 94.736 %\n",
      "[5360,   391] loss: 0.266\n",
      "Train Accuracy: 93.284 %\n",
      "[5370,   391] loss: 0.203\n",
      "Train Accuracy: 93.936 %\n",
      "[5380,   391] loss: 0.255\n",
      "Train Accuracy: 92.614 %\n",
      "[5390,   391] loss: 0.158\n",
      "Train Accuracy: 93.946 %\n",
      "[5400,   391] loss: 0.172\n",
      "Train Accuracy: 92.336 %\n",
      "[5410,   391] loss: 0.235\n",
      "Train Accuracy: 93.756 %\n",
      "[5420,   391] loss: 0.287\n",
      "Train Accuracy: 92.200 %\n",
      "[5430,   391] loss: 0.213\n",
      "Train Accuracy: 93.738 %\n",
      "[5440,   391] loss: 0.216\n",
      "Train Accuracy: 92.448 %\n",
      "[5450,   391] loss: 0.156\n",
      "Train Accuracy: 93.832 %\n",
      "[5460,   391] loss: 0.183\n",
      "Train Accuracy: 93.912 %\n",
      "[5470,   391] loss: 0.218\n",
      "Train Accuracy: 93.902 %\n",
      "[5480,   391] loss: 0.128\n",
      "Train Accuracy: 96.210 %\n",
      "[5490,   391] loss: 0.207\n",
      "Train Accuracy: 93.040 %\n",
      "[5500,   391] loss: 0.205\n",
      "Train Accuracy: 91.932 %\n",
      "[5510,   391] loss: 0.183\n",
      "Train Accuracy: 93.766 %\n",
      "[5520,   391] loss: 0.168\n",
      "Train Accuracy: 93.808 %\n",
      "[5530,   391] loss: 0.220\n",
      "Train Accuracy: 93.118 %\n",
      "[5540,   391] loss: 0.214\n",
      "Train Accuracy: 91.262 %\n",
      "[5550,   391] loss: 0.322\n",
      "Train Accuracy: 92.024 %\n",
      "[5560,   391] loss: 0.205\n",
      "Train Accuracy: 93.778 %\n",
      "[5570,   391] loss: 0.212\n",
      "Train Accuracy: 94.362 %\n",
      "[5580,   391] loss: 0.200\n",
      "Train Accuracy: 93.558 %\n",
      "[5590,   391] loss: 0.260\n",
      "Train Accuracy: 93.410 %\n",
      "[5600,   391] loss: 0.224\n",
      "Train Accuracy: 93.076 %\n",
      "[5610,   391] loss: 0.227\n",
      "Train Accuracy: 90.940 %\n",
      "[5620,   391] loss: 0.371\n",
      "Train Accuracy: 90.724 %\n",
      "[5630,   391] loss: 0.186\n",
      "Train Accuracy: 93.580 %\n",
      "[5640,   391] loss: 0.201\n",
      "Train Accuracy: 94.280 %\n",
      "[5650,   391] loss: 0.220\n",
      "Train Accuracy: 93.714 %\n",
      "[5660,   391] loss: 0.257\n",
      "Train Accuracy: 94.328 %\n",
      "[5670,   391] loss: 0.274\n",
      "Train Accuracy: 91.928 %\n",
      "[5680,   391] loss: 0.287\n",
      "Train Accuracy: 88.544 %\n",
      "[5690,   391] loss: 0.269\n",
      "Train Accuracy: 92.222 %\n",
      "[5700,   391] loss: 0.277\n",
      "Train Accuracy: 93.204 %\n",
      "[5710,   391] loss: 0.234\n",
      "Train Accuracy: 91.922 %\n",
      "[5720,   391] loss: 0.222\n",
      "Train Accuracy: 93.982 %\n",
      "[5730,   391] loss: 0.320\n",
      "Train Accuracy: 92.742 %\n",
      "[5740,   391] loss: 0.192\n",
      "Train Accuracy: 93.830 %\n",
      "[5750,   391] loss: 0.209\n",
      "Train Accuracy: 94.456 %\n",
      "[5760,   391] loss: 0.252\n",
      "Train Accuracy: 92.280 %\n",
      "[5770,   391] loss: 0.200\n",
      "Train Accuracy: 93.076 %\n",
      "[5780,   391] loss: 0.218\n",
      "Train Accuracy: 92.508 %\n",
      "[5790,   391] loss: 0.166\n",
      "Train Accuracy: 95.314 %\n",
      "[5800,   391] loss: 0.192\n",
      "Train Accuracy: 92.082 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5810,   391] loss: 0.240\n",
      "Train Accuracy: 94.022 %\n",
      "[5820,   391] loss: 0.204\n",
      "Train Accuracy: 93.388 %\n",
      "[5830,   391] loss: 0.215\n",
      "Train Accuracy: 93.782 %\n",
      "[5840,   391] loss: 0.212\n",
      "Train Accuracy: 91.104 %\n",
      "[5850,   391] loss: 0.205\n",
      "Train Accuracy: 94.378 %\n",
      "[5860,   391] loss: 0.187\n",
      "Train Accuracy: 93.402 %\n",
      "[5870,   391] loss: 0.183\n",
      "Train Accuracy: 95.428 %\n",
      "[5880,   391] loss: 0.165\n",
      "Train Accuracy: 94.986 %\n",
      "[5890,   391] loss: 0.207\n",
      "Train Accuracy: 95.188 %\n",
      "[5900,   391] loss: 0.153\n",
      "Train Accuracy: 92.240 %\n",
      "[5910,   391] loss: 0.215\n",
      "Train Accuracy: 92.640 %\n",
      "[5920,   391] loss: 0.270\n",
      "Train Accuracy: 89.538 %\n",
      "[5930,   391] loss: 0.253\n",
      "Train Accuracy: 93.202 %\n",
      "[5940,   391] loss: 0.185\n",
      "Train Accuracy: 93.860 %\n",
      "[5950,   391] loss: 0.238\n",
      "Train Accuracy: 94.440 %\n",
      "[5960,   391] loss: 0.253\n",
      "Train Accuracy: 94.000 %\n",
      "[5970,   391] loss: 0.140\n",
      "Train Accuracy: 93.558 %\n",
      "[5980,   391] loss: 0.287\n",
      "Train Accuracy: 93.216 %\n",
      "[5990,   391] loss: 0.205\n",
      "Train Accuracy: 91.674 %\n",
      "[6000,   391] loss: 0.179\n",
      "Train Accuracy: 96.046 %\n",
      "Test Accuracy: 34.820 %, Test Loss: 19.625\n",
      "--------------------------------------\n",
      "Prune the model by 82.45 %\n",
      "Pruning with threshold : 0.1036\n",
      "Params size (MB): 0.095\n",
      "[10,   391] loss: 1.500\n",
      "Train Accuracy: 52.524 %\n",
      "[20,   391] loss: 1.374\n",
      "Train Accuracy: 57.176 %\n",
      "[30,   391] loss: 1.306\n",
      "Train Accuracy: 59.708 %\n",
      "[40,   391] loss: 1.237\n",
      "Train Accuracy: 61.572 %\n",
      "[50,   391] loss: 1.202\n",
      "Train Accuracy: 62.816 %\n",
      "[60,   391] loss: 1.183\n",
      "Train Accuracy: 63.734 %\n",
      "[70,   391] loss: 1.152\n",
      "Train Accuracy: 64.666 %\n",
      "[80,   391] loss: 1.130\n",
      "Train Accuracy: 64.992 %\n",
      "[90,   391] loss: 1.113\n",
      "Train Accuracy: 66.104 %\n",
      "[100,   391] loss: 1.091\n",
      "Train Accuracy: 66.644 %\n",
      "[110,   391] loss: 1.074\n",
      "Train Accuracy: 66.624 %\n",
      "[120,   391] loss: 1.054\n",
      "Train Accuracy: 67.318 %\n",
      "[130,   391] loss: 1.048\n",
      "Train Accuracy: 67.926 %\n",
      "[140,   391] loss: 1.050\n",
      "Train Accuracy: 68.626 %\n",
      "[150,   391] loss: 1.010\n",
      "Train Accuracy: 68.920 %\n",
      "[160,   391] loss: 1.011\n",
      "Train Accuracy: 68.862 %\n",
      "[170,   391] loss: 0.999\n",
      "Train Accuracy: 69.750 %\n",
      "[180,   391] loss: 0.975\n",
      "Train Accuracy: 69.838 %\n",
      "[190,   391] loss: 0.974\n",
      "Train Accuracy: 69.998 %\n",
      "[200,   391] loss: 0.969\n",
      "Train Accuracy: 70.558 %\n",
      "[210,   391] loss: 0.964\n",
      "Train Accuracy: 70.612 %\n",
      "[220,   391] loss: 0.958\n",
      "Train Accuracy: 71.128 %\n",
      "[230,   391] loss: 0.943\n",
      "Train Accuracy: 71.182 %\n",
      "[240,   391] loss: 0.924\n",
      "Train Accuracy: 71.588 %\n",
      "[250,   391] loss: 0.949\n",
      "Train Accuracy: 71.726 %\n",
      "[260,   391] loss: 0.927\n",
      "Train Accuracy: 71.864 %\n",
      "[270,   391] loss: 0.920\n",
      "Train Accuracy: 72.402 %\n",
      "[280,   391] loss: 0.924\n",
      "Train Accuracy: 71.900 %\n",
      "[290,   391] loss: 0.925\n",
      "Train Accuracy: 72.224 %\n",
      "[300,   391] loss: 0.907\n",
      "Train Accuracy: 72.450 %\n",
      "[310,   391] loss: 0.890\n",
      "Train Accuracy: 72.574 %\n",
      "[320,   391] loss: 0.899\n",
      "Train Accuracy: 72.664 %\n",
      "[330,   391] loss: 0.890\n",
      "Train Accuracy: 73.020 %\n",
      "[340,   391] loss: 0.886\n",
      "Train Accuracy: 72.938 %\n",
      "[350,   391] loss: 0.881\n",
      "Train Accuracy: 73.172 %\n",
      "[360,   391] loss: 0.883\n",
      "Train Accuracy: 73.656 %\n",
      "[370,   391] loss: 0.862\n",
      "Train Accuracy: 73.782 %\n",
      "[380,   391] loss: 0.872\n",
      "Train Accuracy: 73.666 %\n",
      "[390,   391] loss: 0.871\n",
      "Train Accuracy: 73.744 %\n",
      "[400,   391] loss: 0.866\n",
      "Train Accuracy: 73.364 %\n",
      "[410,   391] loss: 0.880\n",
      "Train Accuracy: 73.900 %\n",
      "[420,   391] loss: 0.856\n",
      "Train Accuracy: 74.196 %\n",
      "[430,   391] loss: 0.852\n",
      "Train Accuracy: 74.014 %\n",
      "[440,   391] loss: 0.847\n",
      "Train Accuracy: 74.064 %\n",
      "[450,   391] loss: 0.841\n",
      "Train Accuracy: 73.620 %\n",
      "[460,   391] loss: 0.842\n",
      "Train Accuracy: 74.212 %\n",
      "[470,   391] loss: 0.837\n",
      "Train Accuracy: 74.604 %\n",
      "[480,   391] loss: 0.845\n",
      "Train Accuracy: 74.278 %\n",
      "[490,   391] loss: 0.821\n",
      "Train Accuracy: 74.620 %\n",
      "[500,   391] loss: 0.842\n",
      "Train Accuracy: 74.662 %\n",
      "[510,   391] loss: 0.834\n",
      "Train Accuracy: 74.768 %\n",
      "[520,   391] loss: 0.848\n",
      "Train Accuracy: 74.174 %\n",
      "[530,   391] loss: 0.813\n",
      "Train Accuracy: 74.994 %\n",
      "[540,   391] loss: 0.832\n",
      "Train Accuracy: 74.784 %\n",
      "[550,   391] loss: 0.815\n",
      "Train Accuracy: 75.252 %\n",
      "[560,   391] loss: 0.814\n",
      "Train Accuracy: 74.758 %\n",
      "[570,   391] loss: 0.813\n",
      "Train Accuracy: 74.990 %\n",
      "[580,   391] loss: 0.825\n",
      "Train Accuracy: 75.192 %\n",
      "[590,   391] loss: 0.804\n",
      "Train Accuracy: 75.380 %\n",
      "[600,   391] loss: 0.799\n",
      "Train Accuracy: 75.380 %\n",
      "[610,   391] loss: 0.807\n",
      "Train Accuracy: 75.628 %\n",
      "[620,   391] loss: 0.792\n",
      "Train Accuracy: 75.622 %\n",
      "[630,   391] loss: 0.819\n",
      "Train Accuracy: 75.576 %\n",
      "[640,   391] loss: 0.800\n",
      "Train Accuracy: 75.816 %\n",
      "[650,   391] loss: 0.799\n",
      "Train Accuracy: 75.592 %\n",
      "[660,   391] loss: 0.795\n",
      "Train Accuracy: 75.948 %\n",
      "[670,   391] loss: 0.809\n",
      "Train Accuracy: 75.914 %\n",
      "[680,   391] loss: 0.800\n",
      "Train Accuracy: 76.040 %\n",
      "[690,   391] loss: 0.796\n",
      "Train Accuracy: 75.828 %\n",
      "[700,   391] loss: 0.786\n",
      "Train Accuracy: 75.992 %\n",
      "[710,   391] loss: 0.791\n",
      "Train Accuracy: 76.072 %\n",
      "[720,   391] loss: 0.803\n",
      "Train Accuracy: 75.584 %\n",
      "[730,   391] loss: 0.787\n",
      "Train Accuracy: 76.176 %\n",
      "[740,   391] loss: 0.786\n",
      "Train Accuracy: 75.440 %\n",
      "[750,   391] loss: 0.776\n",
      "Train Accuracy: 76.216 %\n",
      "[760,   391] loss: 0.768\n",
      "Train Accuracy: 76.024 %\n",
      "[770,   391] loss: 0.789\n",
      "Train Accuracy: 75.988 %\n",
      "[780,   391] loss: 0.784\n",
      "Train Accuracy: 76.468 %\n",
      "[790,   391] loss: 0.791\n",
      "Train Accuracy: 76.614 %\n",
      "[800,   391] loss: 0.778\n",
      "Train Accuracy: 76.772 %\n",
      "[810,   391] loss: 0.778\n",
      "Train Accuracy: 76.662 %\n",
      "[820,   391] loss: 0.782\n",
      "Train Accuracy: 76.658 %\n",
      "[830,   391] loss: 0.768\n",
      "Train Accuracy: 76.340 %\n",
      "[840,   391] loss: 0.762\n",
      "Train Accuracy: 76.526 %\n",
      "[850,   391] loss: 0.775\n",
      "Train Accuracy: 76.328 %\n",
      "[860,   391] loss: 0.768\n",
      "Train Accuracy: 77.008 %\n",
      "[870,   391] loss: 0.767\n",
      "Train Accuracy: 76.482 %\n",
      "[880,   391] loss: 0.763\n",
      "Train Accuracy: 76.502 %\n",
      "[890,   391] loss: 0.763\n",
      "Train Accuracy: 77.010 %\n",
      "[900,   391] loss: 0.761\n",
      "Train Accuracy: 76.466 %\n",
      "[910,   391] loss: 0.770\n",
      "Train Accuracy: 76.386 %\n",
      "[920,   391] loss: 0.768\n",
      "Train Accuracy: 77.314 %\n",
      "[930,   391] loss: 0.765\n",
      "Train Accuracy: 76.914 %\n",
      "[940,   391] loss: 0.753\n",
      "Train Accuracy: 76.876 %\n",
      "[950,   391] loss: 0.765\n",
      "Train Accuracy: 76.756 %\n",
      "[960,   391] loss: 0.750\n",
      "Train Accuracy: 76.722 %\n",
      "[970,   391] loss: 0.762\n",
      "Train Accuracy: 76.902 %\n",
      "[980,   391] loss: 0.750\n",
      "Train Accuracy: 76.944 %\n",
      "[990,   391] loss: 0.764\n",
      "Train Accuracy: 76.946 %\n",
      "[1000,   391] loss: 0.754\n",
      "Train Accuracy: 76.930 %\n",
      "[1010,   391] loss: 0.768\n",
      "Train Accuracy: 77.452 %\n",
      "[1020,   391] loss: 0.752\n",
      "Train Accuracy: 77.054 %\n",
      "[1030,   391] loss: 0.750\n",
      "Train Accuracy: 77.448 %\n",
      "[1040,   391] loss: 0.741\n",
      "Train Accuracy: 77.170 %\n",
      "[1050,   391] loss: 0.756\n",
      "Train Accuracy: 77.458 %\n",
      "[1060,   391] loss: 0.741\n",
      "Train Accuracy: 77.012 %\n",
      "[1070,   391] loss: 0.735\n",
      "Train Accuracy: 77.640 %\n",
      "[1080,   391] loss: 0.747\n",
      "Train Accuracy: 77.292 %\n",
      "[1090,   391] loss: 0.747\n",
      "Train Accuracy: 77.612 %\n",
      "[1100,   391] loss: 0.755\n",
      "Train Accuracy: 77.486 %\n",
      "[1110,   391] loss: 0.750\n",
      "Train Accuracy: 77.728 %\n",
      "[1120,   391] loss: 0.747\n",
      "Train Accuracy: 77.862 %\n",
      "[1130,   391] loss: 0.742\n",
      "Train Accuracy: 77.296 %\n",
      "[1140,   391] loss: 0.753\n",
      "Train Accuracy: 77.754 %\n",
      "[1150,   391] loss: 0.754\n",
      "Train Accuracy: 77.756 %\n",
      "[1160,   391] loss: 0.745\n",
      "Train Accuracy: 77.272 %\n",
      "[1170,   391] loss: 0.748\n",
      "Train Accuracy: 77.486 %\n",
      "[1180,   391] loss: 0.727\n",
      "Train Accuracy: 76.796 %\n",
      "[1190,   391] loss: 0.732\n",
      "Train Accuracy: 77.776 %\n",
      "[1200,   391] loss: 0.736\n",
      "Train Accuracy: 77.606 %\n",
      "[1210,   391] loss: 0.744\n",
      "Train Accuracy: 77.142 %\n",
      "[1220,   391] loss: 0.736\n",
      "Train Accuracy: 77.772 %\n",
      "[1230,   391] loss: 0.735\n",
      "Train Accuracy: 77.570 %\n",
      "[1240,   391] loss: 0.740\n",
      "Train Accuracy: 77.208 %\n",
      "[1250,   391] loss: 0.726\n",
      "Train Accuracy: 78.140 %\n",
      "[1260,   391] loss: 0.738\n",
      "Train Accuracy: 77.866 %\n",
      "[1270,   391] loss: 0.730\n",
      "Train Accuracy: 77.712 %\n",
      "[1280,   391] loss: 0.733\n",
      "Train Accuracy: 78.052 %\n",
      "[1290,   391] loss: 0.730\n",
      "Train Accuracy: 77.702 %\n",
      "[1300,   391] loss: 0.723\n",
      "Train Accuracy: 78.096 %\n",
      "[1310,   391] loss: 0.735\n",
      "Train Accuracy: 78.068 %\n",
      "[1320,   391] loss: 0.737\n",
      "Train Accuracy: 77.604 %\n",
      "[1330,   391] loss: 0.735\n",
      "Train Accuracy: 77.556 %\n",
      "[1340,   391] loss: 0.719\n",
      "Train Accuracy: 77.494 %\n",
      "[1350,   391] loss: 0.732\n",
      "Train Accuracy: 77.346 %\n",
      "[1360,   391] loss: 0.719\n",
      "Train Accuracy: 77.580 %\n",
      "[1370,   391] loss: 0.730\n",
      "Train Accuracy: 78.016 %\n",
      "[1380,   391] loss: 0.728\n",
      "Train Accuracy: 78.160 %\n",
      "[1390,   391] loss: 0.716\n",
      "Train Accuracy: 77.736 %\n",
      "[1400,   391] loss: 0.718\n",
      "Train Accuracy: 78.022 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1410,   391] loss: 0.728\n",
      "Train Accuracy: 78.074 %\n",
      "[1420,   391] loss: 0.724\n",
      "Train Accuracy: 78.134 %\n",
      "[1430,   391] loss: 0.734\n",
      "Train Accuracy: 78.374 %\n",
      "[1440,   391] loss: 0.723\n",
      "Train Accuracy: 77.462 %\n",
      "[1450,   391] loss: 0.721\n",
      "Train Accuracy: 77.386 %\n",
      "[1460,   391] loss: 0.729\n",
      "Train Accuracy: 78.446 %\n",
      "[1470,   391] loss: 0.722\n",
      "Train Accuracy: 77.822 %\n",
      "[1480,   391] loss: 0.711\n",
      "Train Accuracy: 78.162 %\n",
      "[1490,   391] loss: 0.719\n",
      "Train Accuracy: 78.210 %\n",
      "[1500,   391] loss: 0.736\n",
      "Train Accuracy: 77.964 %\n",
      "[1510,   391] loss: 0.718\n",
      "Train Accuracy: 78.552 %\n",
      "[1520,   391] loss: 0.696\n",
      "Train Accuracy: 78.434 %\n",
      "[1530,   391] loss: 0.732\n",
      "Train Accuracy: 78.156 %\n",
      "[1540,   391] loss: 0.720\n",
      "Train Accuracy: 78.484 %\n",
      "[1550,   391] loss: 0.710\n",
      "Train Accuracy: 78.284 %\n",
      "[1560,   391] loss: 0.716\n",
      "Train Accuracy: 78.408 %\n",
      "[1570,   391] loss: 0.714\n",
      "Train Accuracy: 78.056 %\n",
      "[1580,   391] loss: 0.718\n",
      "Train Accuracy: 78.282 %\n",
      "[1590,   391] loss: 0.721\n",
      "Train Accuracy: 78.330 %\n",
      "[1600,   391] loss: 0.711\n",
      "Train Accuracy: 77.650 %\n",
      "[1610,   391] loss: 0.703\n",
      "Train Accuracy: 78.328 %\n",
      "[1620,   391] loss: 0.713\n",
      "Train Accuracy: 78.182 %\n",
      "[1630,   391] loss: 0.711\n",
      "Train Accuracy: 78.574 %\n",
      "[1640,   391] loss: 0.709\n",
      "Train Accuracy: 78.198 %\n",
      "[1650,   391] loss: 0.725\n",
      "Train Accuracy: 78.180 %\n",
      "[1660,   391] loss: 0.711\n",
      "Train Accuracy: 78.388 %\n",
      "[1670,   391] loss: 0.709\n",
      "Train Accuracy: 78.714 %\n",
      "[1680,   391] loss: 0.703\n",
      "Train Accuracy: 78.200 %\n",
      "[1690,   391] loss: 0.704\n",
      "Train Accuracy: 78.342 %\n",
      "[1700,   391] loss: 0.704\n",
      "Train Accuracy: 78.404 %\n",
      "[1710,   391] loss: 0.723\n",
      "Train Accuracy: 77.782 %\n",
      "[1720,   391] loss: 0.708\n",
      "Train Accuracy: 77.938 %\n",
      "[1730,   391] loss: 0.703\n",
      "Train Accuracy: 78.002 %\n",
      "[1740,   391] loss: 0.707\n",
      "Train Accuracy: 78.490 %\n",
      "[1750,   391] loss: 0.704\n",
      "Train Accuracy: 78.232 %\n",
      "[1760,   391] loss: 0.705\n",
      "Train Accuracy: 78.532 %\n",
      "[1770,   391] loss: 0.704\n",
      "Train Accuracy: 78.448 %\n",
      "[1780,   391] loss: 0.711\n",
      "Train Accuracy: 78.454 %\n",
      "[1790,   391] loss: 0.700\n",
      "Train Accuracy: 78.018 %\n",
      "[1800,   391] loss: 0.695\n",
      "Train Accuracy: 78.562 %\n",
      "[1810,   391] loss: 0.711\n",
      "Train Accuracy: 78.596 %\n",
      "[1820,   391] loss: 0.692\n",
      "Train Accuracy: 77.912 %\n",
      "[1830,   391] loss: 0.693\n",
      "Train Accuracy: 77.976 %\n",
      "[1840,   391] loss: 0.713\n",
      "Train Accuracy: 78.452 %\n",
      "[1850,   391] loss: 0.714\n",
      "Train Accuracy: 78.788 %\n",
      "[1860,   391] loss: 0.706\n",
      "Train Accuracy: 78.326 %\n",
      "[1870,   391] loss: 0.713\n",
      "Train Accuracy: 78.434 %\n",
      "[1880,   391] loss: 0.704\n",
      "Train Accuracy: 78.706 %\n",
      "[1890,   391] loss: 0.704\n",
      "Train Accuracy: 78.820 %\n",
      "[1900,   391] loss: 0.706\n",
      "Train Accuracy: 78.754 %\n",
      "[1910,   391] loss: 0.702\n",
      "Train Accuracy: 77.644 %\n",
      "[1920,   391] loss: 0.703\n",
      "Train Accuracy: 78.674 %\n",
      "[1930,   391] loss: 0.701\n",
      "Train Accuracy: 78.604 %\n",
      "[1940,   391] loss: 0.693\n",
      "Train Accuracy: 78.312 %\n",
      "[1950,   391] loss: 0.710\n",
      "Train Accuracy: 78.450 %\n",
      "[1960,   391] loss: 0.702\n",
      "Train Accuracy: 78.506 %\n",
      "[1970,   391] loss: 0.704\n",
      "Train Accuracy: 78.938 %\n",
      "[1980,   391] loss: 0.696\n",
      "Train Accuracy: 78.612 %\n",
      "[1990,   391] loss: 0.718\n",
      "Train Accuracy: 78.804 %\n",
      "[2000,   391] loss: 0.690\n",
      "Train Accuracy: 78.570 %\n",
      "[2010,   391] loss: 0.693\n",
      "Train Accuracy: 78.276 %\n",
      "[2020,   391] loss: 0.713\n",
      "Train Accuracy: 78.350 %\n",
      "[2030,   391] loss: 0.687\n",
      "Train Accuracy: 78.994 %\n",
      "[2040,   391] loss: 0.703\n",
      "Train Accuracy: 78.670 %\n",
      "[2050,   391] loss: 0.696\n",
      "Train Accuracy: 78.236 %\n",
      "[2060,   391] loss: 0.704\n",
      "Train Accuracy: 78.392 %\n",
      "[2070,   391] loss: 0.703\n",
      "Train Accuracy: 78.836 %\n",
      "[2080,   391] loss: 0.695\n",
      "Train Accuracy: 78.596 %\n",
      "[2090,   391] loss: 0.694\n",
      "Train Accuracy: 78.474 %\n",
      "[2100,   391] loss: 0.696\n",
      "Train Accuracy: 78.550 %\n",
      "[2110,   391] loss: 0.704\n",
      "Train Accuracy: 78.424 %\n",
      "[2120,   391] loss: 0.686\n",
      "Train Accuracy: 78.406 %\n",
      "[2130,   391] loss: 0.688\n",
      "Train Accuracy: 78.258 %\n",
      "[2140,   391] loss: 0.683\n",
      "Train Accuracy: 78.650 %\n",
      "[2150,   391] loss: 0.677\n",
      "Train Accuracy: 78.930 %\n",
      "[2160,   391] loss: 0.688\n",
      "Train Accuracy: 79.142 %\n",
      "[2170,   391] loss: 0.696\n",
      "Train Accuracy: 78.792 %\n",
      "[2180,   391] loss: 0.687\n",
      "Train Accuracy: 78.902 %\n",
      "[2190,   391] loss: 0.699\n",
      "Train Accuracy: 79.268 %\n",
      "[2200,   391] loss: 0.691\n",
      "Train Accuracy: 78.410 %\n",
      "[2210,   391] loss: 0.704\n",
      "Train Accuracy: 78.384 %\n",
      "[2220,   391] loss: 0.674\n",
      "Train Accuracy: 79.392 %\n",
      "[2230,   391] loss: 0.701\n",
      "Train Accuracy: 78.570 %\n",
      "[2240,   391] loss: 0.701\n",
      "Train Accuracy: 79.256 %\n",
      "[2250,   391] loss: 0.685\n",
      "Train Accuracy: 79.114 %\n",
      "[2260,   391] loss: 0.685\n",
      "Train Accuracy: 78.602 %\n",
      "[2270,   391] loss: 0.691\n",
      "Train Accuracy: 79.014 %\n",
      "[2280,   391] loss: 0.690\n",
      "Train Accuracy: 78.380 %\n",
      "[2290,   391] loss: 0.683\n",
      "Train Accuracy: 78.140 %\n",
      "[2300,   391] loss: 0.687\n",
      "Train Accuracy: 78.784 %\n",
      "[2310,   391] loss: 0.691\n",
      "Train Accuracy: 78.968 %\n",
      "[2320,   391] loss: 0.695\n",
      "Train Accuracy: 79.384 %\n",
      "[2330,   391] loss: 0.679\n",
      "Train Accuracy: 79.166 %\n",
      "[2340,   391] loss: 0.683\n",
      "Train Accuracy: 79.144 %\n",
      "[2350,   391] loss: 0.684\n",
      "Train Accuracy: 78.476 %\n",
      "[2360,   391] loss: 0.682\n",
      "Train Accuracy: 78.812 %\n",
      "[2370,   391] loss: 0.684\n",
      "Train Accuracy: 77.866 %\n",
      "[2380,   391] loss: 0.688\n",
      "Train Accuracy: 79.252 %\n",
      "[2390,   391] loss: 0.689\n",
      "Train Accuracy: 79.102 %\n",
      "[2400,   391] loss: 0.673\n",
      "Train Accuracy: 78.386 %\n",
      "[2410,   391] loss: 0.684\n",
      "Train Accuracy: 78.518 %\n",
      "[2420,   391] loss: 0.690\n",
      "Train Accuracy: 78.382 %\n",
      "[2430,   391] loss: 0.697\n",
      "Train Accuracy: 78.296 %\n",
      "[2440,   391] loss: 0.692\n",
      "Train Accuracy: 78.754 %\n",
      "[2450,   391] loss: 0.675\n",
      "Train Accuracy: 79.522 %\n",
      "[2460,   391] loss: 0.692\n",
      "Train Accuracy: 79.298 %\n",
      "[2470,   391] loss: 0.695\n",
      "Train Accuracy: 78.292 %\n",
      "[2480,   391] loss: 0.701\n",
      "Train Accuracy: 79.180 %\n",
      "[2490,   391] loss: 0.681\n",
      "Train Accuracy: 79.166 %\n",
      "[2500,   391] loss: 0.677\n",
      "Train Accuracy: 79.034 %\n",
      "[2510,   391] loss: 0.690\n",
      "Train Accuracy: 78.676 %\n",
      "[2520,   391] loss: 0.678\n",
      "Train Accuracy: 78.902 %\n",
      "[2530,   391] loss: 0.708\n",
      "Train Accuracy: 78.194 %\n",
      "[2540,   391] loss: 0.679\n",
      "Train Accuracy: 78.906 %\n",
      "[2550,   391] loss: 0.693\n",
      "Train Accuracy: 78.710 %\n",
      "[2560,   391] loss: 0.681\n",
      "Train Accuracy: 78.668 %\n",
      "[2570,   391] loss: 0.696\n",
      "Train Accuracy: 79.236 %\n",
      "[2580,   391] loss: 0.683\n",
      "Train Accuracy: 79.044 %\n",
      "[2590,   391] loss: 0.670\n",
      "Train Accuracy: 79.424 %\n",
      "[2600,   391] loss: 0.670\n",
      "Train Accuracy: 78.910 %\n",
      "[2610,   391] loss: 0.695\n",
      "Train Accuracy: 79.412 %\n",
      "[2620,   391] loss: 0.688\n",
      "Train Accuracy: 78.866 %\n",
      "[2630,   391] loss: 0.689\n",
      "Train Accuracy: 78.892 %\n",
      "[2640,   391] loss: 0.677\n",
      "Train Accuracy: 79.390 %\n",
      "[2650,   391] loss: 0.685\n",
      "Train Accuracy: 79.160 %\n",
      "[2660,   391] loss: 0.687\n",
      "Train Accuracy: 78.500 %\n",
      "[2670,   391] loss: 0.666\n",
      "Train Accuracy: 78.578 %\n",
      "[2680,   391] loss: 0.674\n",
      "Train Accuracy: 79.026 %\n",
      "[2690,   391] loss: 0.683\n",
      "Train Accuracy: 78.672 %\n",
      "[2700,   391] loss: 0.673\n",
      "Train Accuracy: 78.946 %\n",
      "[2710,   391] loss: 0.691\n",
      "Train Accuracy: 79.742 %\n",
      "[2720,   391] loss: 0.672\n",
      "Train Accuracy: 78.906 %\n",
      "[2730,   391] loss: 0.682\n",
      "Train Accuracy: 78.652 %\n",
      "[2740,   391] loss: 0.687\n",
      "Train Accuracy: 79.222 %\n",
      "[2750,   391] loss: 0.670\n",
      "Train Accuracy: 79.482 %\n",
      "[2760,   391] loss: 0.680\n",
      "Train Accuracy: 79.564 %\n",
      "[2770,   391] loss: 0.688\n",
      "Train Accuracy: 78.966 %\n",
      "[2780,   391] loss: 0.679\n",
      "Train Accuracy: 79.018 %\n",
      "[2790,   391] loss: 0.697\n",
      "Train Accuracy: 79.326 %\n",
      "[2800,   391] loss: 0.671\n",
      "Train Accuracy: 78.906 %\n",
      "[2810,   391] loss: 0.665\n",
      "Train Accuracy: 79.026 %\n",
      "[2820,   391] loss: 0.687\n",
      "Train Accuracy: 79.036 %\n",
      "[2830,   391] loss: 0.661\n",
      "Train Accuracy: 78.810 %\n",
      "[2840,   391] loss: 0.686\n",
      "Train Accuracy: 79.390 %\n",
      "[2850,   391] loss: 0.682\n",
      "Train Accuracy: 79.458 %\n",
      "[2860,   391] loss: 0.673\n",
      "Train Accuracy: 78.444 %\n",
      "[2870,   391] loss: 0.666\n",
      "Train Accuracy: 79.514 %\n",
      "[2880,   391] loss: 0.683\n",
      "Train Accuracy: 79.406 %\n",
      "[2890,   391] loss: 0.671\n",
      "Train Accuracy: 78.956 %\n",
      "[2900,   391] loss: 0.679\n",
      "Train Accuracy: 79.066 %\n",
      "[2910,   391] loss: 0.666\n",
      "Train Accuracy: 79.472 %\n",
      "[2920,   391] loss: 0.677\n",
      "Train Accuracy: 78.978 %\n",
      "[2930,   391] loss: 0.691\n",
      "Train Accuracy: 78.716 %\n",
      "[2940,   391] loss: 0.664\n",
      "Train Accuracy: 78.546 %\n",
      "[2950,   391] loss: 0.671\n",
      "Train Accuracy: 79.650 %\n",
      "[2960,   391] loss: 0.682\n",
      "Train Accuracy: 78.784 %\n",
      "[2970,   391] loss: 0.678\n",
      "Train Accuracy: 78.776 %\n",
      "[2980,   391] loss: 0.676\n",
      "Train Accuracy: 79.186 %\n",
      "[2990,   391] loss: 0.673\n",
      "Train Accuracy: 78.648 %\n",
      "[3000,   391] loss: 0.666\n",
      "Train Accuracy: 77.658 %\n",
      "[3010,   391] loss: 0.676\n",
      "Train Accuracy: 77.870 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3020,   391] loss: 0.672\n",
      "Train Accuracy: 79.486 %\n",
      "[3030,   391] loss: 0.680\n",
      "Train Accuracy: 78.790 %\n",
      "[3040,   391] loss: 0.681\n",
      "Train Accuracy: 79.534 %\n",
      "[3050,   391] loss: 0.676\n",
      "Train Accuracy: 78.850 %\n",
      "[3060,   391] loss: 0.679\n",
      "Train Accuracy: 78.976 %\n",
      "[3070,   391] loss: 0.676\n",
      "Train Accuracy: 78.810 %\n",
      "[3080,   391] loss: 0.679\n",
      "Train Accuracy: 79.236 %\n",
      "[3090,   391] loss: 0.663\n",
      "Train Accuracy: 79.504 %\n",
      "[3100,   391] loss: 0.688\n",
      "Train Accuracy: 79.166 %\n",
      "[3110,   391] loss: 0.677\n",
      "Train Accuracy: 78.044 %\n",
      "[3120,   391] loss: 0.671\n",
      "Train Accuracy: 78.188 %\n",
      "[3130,   391] loss: 0.677\n",
      "Train Accuracy: 79.596 %\n",
      "[3140,   391] loss: 0.668\n",
      "Train Accuracy: 78.282 %\n",
      "[3150,   391] loss: 0.672\n",
      "Train Accuracy: 78.248 %\n",
      "[3160,   391] loss: 0.685\n",
      "Train Accuracy: 79.536 %\n",
      "[3170,   391] loss: 0.660\n",
      "Train Accuracy: 78.062 %\n",
      "[3180,   391] loss: 0.670\n",
      "Train Accuracy: 79.352 %\n",
      "[3190,   391] loss: 0.656\n",
      "Train Accuracy: 78.944 %\n",
      "[3200,   391] loss: 0.676\n",
      "Train Accuracy: 79.128 %\n",
      "[3210,   391] loss: 0.660\n",
      "Train Accuracy: 78.858 %\n",
      "[3220,   391] loss: 0.664\n",
      "Train Accuracy: 78.614 %\n",
      "[3230,   391] loss: 0.662\n",
      "Train Accuracy: 79.072 %\n",
      "[3240,   391] loss: 0.669\n",
      "Train Accuracy: 78.742 %\n",
      "[3250,   391] loss: 0.673\n",
      "Train Accuracy: 78.938 %\n",
      "[3260,   391] loss: 0.665\n",
      "Train Accuracy: 78.970 %\n",
      "[3270,   391] loss: 0.669\n",
      "Train Accuracy: 78.640 %\n",
      "[3280,   391] loss: 0.682\n",
      "Train Accuracy: 79.500 %\n",
      "[3290,   391] loss: 0.652\n",
      "Train Accuracy: 78.166 %\n",
      "[3300,   391] loss: 0.676\n",
      "Train Accuracy: 78.608 %\n",
      "[3310,   391] loss: 0.695\n",
      "Train Accuracy: 78.494 %\n",
      "[3320,   391] loss: 0.666\n",
      "Train Accuracy: 77.710 %\n",
      "[3330,   391] loss: 0.682\n",
      "Train Accuracy: 79.264 %\n",
      "[3340,   391] loss: 0.683\n",
      "Train Accuracy: 79.166 %\n",
      "[3350,   391] loss: 0.661\n",
      "Train Accuracy: 79.442 %\n",
      "[3360,   391] loss: 0.685\n",
      "Train Accuracy: 78.790 %\n",
      "[3370,   391] loss: 0.666\n",
      "Train Accuracy: 78.792 %\n",
      "[3380,   391] loss: 0.664\n",
      "Train Accuracy: 79.614 %\n",
      "[3390,   391] loss: 0.657\n",
      "Train Accuracy: 78.476 %\n",
      "[3400,   391] loss: 0.663\n",
      "Train Accuracy: 79.288 %\n",
      "[3410,   391] loss: 0.674\n",
      "Train Accuracy: 79.200 %\n",
      "[3420,   391] loss: 0.656\n",
      "Train Accuracy: 79.596 %\n",
      "[3430,   391] loss: 0.673\n",
      "Train Accuracy: 79.404 %\n",
      "[3440,   391] loss: 0.669\n",
      "Train Accuracy: 78.408 %\n",
      "[3450,   391] loss: 0.655\n",
      "Train Accuracy: 79.204 %\n",
      "[3460,   391] loss: 0.689\n",
      "Train Accuracy: 79.420 %\n",
      "[3470,   391] loss: 0.674\n",
      "Train Accuracy: 79.300 %\n",
      "[3480,   391] loss: 0.674\n",
      "Train Accuracy: 79.004 %\n",
      "[3490,   391] loss: 0.674\n",
      "Train Accuracy: 79.346 %\n",
      "[3500,   391] loss: 0.671\n",
      "Train Accuracy: 79.756 %\n",
      "[3510,   391] loss: 0.657\n",
      "Train Accuracy: 78.786 %\n",
      "[3520,   391] loss: 0.673\n",
      "Train Accuracy: 78.402 %\n",
      "[3530,   391] loss: 0.655\n",
      "Train Accuracy: 79.128 %\n",
      "[3540,   391] loss: 0.666\n",
      "Train Accuracy: 78.820 %\n",
      "[3550,   391] loss: 0.683\n",
      "Train Accuracy: 79.802 %\n",
      "[3560,   391] loss: 0.655\n",
      "Train Accuracy: 79.148 %\n",
      "[3570,   391] loss: 0.664\n",
      "Train Accuracy: 79.540 %\n",
      "[3580,   391] loss: 0.666\n",
      "Train Accuracy: 79.176 %\n",
      "[3590,   391] loss: 0.670\n",
      "Train Accuracy: 79.104 %\n",
      "[3600,   391] loss: 0.663\n",
      "Train Accuracy: 79.648 %\n",
      "[3610,   391] loss: 0.668\n",
      "Train Accuracy: 79.604 %\n",
      "[3620,   391] loss: 0.676\n",
      "Train Accuracy: 77.954 %\n",
      "[3630,   391] loss: 0.669\n",
      "Train Accuracy: 79.260 %\n",
      "[3640,   391] loss: 0.664\n",
      "Train Accuracy: 78.856 %\n",
      "[3650,   391] loss: 0.665\n",
      "Train Accuracy: 79.340 %\n",
      "[3660,   391] loss: 0.665\n",
      "Train Accuracy: 80.056 %\n",
      "[3670,   391] loss: 0.673\n",
      "Train Accuracy: 79.596 %\n",
      "[3680,   391] loss: 0.670\n",
      "Train Accuracy: 78.712 %\n",
      "[3690,   391] loss: 0.654\n",
      "Train Accuracy: 79.738 %\n",
      "[3700,   391] loss: 0.669\n",
      "Train Accuracy: 79.616 %\n",
      "[3710,   391] loss: 0.668\n",
      "Train Accuracy: 79.362 %\n",
      "[3720,   391] loss: 0.656\n",
      "Train Accuracy: 79.352 %\n",
      "[3730,   391] loss: 0.664\n",
      "Train Accuracy: 79.888 %\n",
      "[3740,   391] loss: 0.659\n",
      "Train Accuracy: 78.948 %\n",
      "[3750,   391] loss: 0.656\n",
      "Train Accuracy: 79.394 %\n",
      "[3760,   391] loss: 0.660\n",
      "Train Accuracy: 78.750 %\n",
      "[3770,   391] loss: 0.688\n",
      "Train Accuracy: 78.996 %\n",
      "[3780,   391] loss: 0.662\n",
      "Train Accuracy: 79.984 %\n",
      "[3790,   391] loss: 0.661\n",
      "Train Accuracy: 79.368 %\n",
      "[3800,   391] loss: 0.653\n",
      "Train Accuracy: 79.480 %\n",
      "[3810,   391] loss: 0.663\n",
      "Train Accuracy: 79.704 %\n",
      "[3820,   391] loss: 0.656\n",
      "Train Accuracy: 79.620 %\n",
      "[3830,   391] loss: 0.697\n",
      "Train Accuracy: 79.438 %\n",
      "[3840,   391] loss: 0.668\n",
      "Train Accuracy: 79.434 %\n",
      "[3850,   391] loss: 0.666\n",
      "Train Accuracy: 79.152 %\n",
      "[3860,   391] loss: 0.663\n",
      "Train Accuracy: 78.512 %\n",
      "[3870,   391] loss: 0.669\n",
      "Train Accuracy: 79.326 %\n",
      "[3880,   391] loss: 0.655\n",
      "Train Accuracy: 79.286 %\n",
      "[3890,   391] loss: 0.660\n",
      "Train Accuracy: 79.672 %\n",
      "[3900,   391] loss: 0.657\n",
      "Train Accuracy: 79.512 %\n",
      "[3910,   391] loss: 0.677\n",
      "Train Accuracy: 79.844 %\n",
      "[3920,   391] loss: 0.660\n",
      "Train Accuracy: 78.692 %\n",
      "[3930,   391] loss: 0.688\n",
      "Train Accuracy: 79.248 %\n",
      "[3940,   391] loss: 0.675\n",
      "Train Accuracy: 79.758 %\n",
      "[3950,   391] loss: 0.668\n",
      "Train Accuracy: 78.824 %\n",
      "[3960,   391] loss: 0.657\n",
      "Train Accuracy: 78.998 %\n",
      "[3970,   391] loss: 0.671\n",
      "Train Accuracy: 80.034 %\n",
      "[3980,   391] loss: 0.655\n",
      "Train Accuracy: 79.316 %\n",
      "[3990,   391] loss: 0.682\n",
      "Train Accuracy: 79.130 %\n",
      "[4000,   391] loss: 0.650\n",
      "Train Accuracy: 79.722 %\n",
      "[4010,   391] loss: 0.661\n",
      "Train Accuracy: 79.388 %\n",
      "[4020,   391] loss: 0.672\n",
      "Train Accuracy: 79.072 %\n",
      "[4030,   391] loss: 0.686\n",
      "Train Accuracy: 79.958 %\n",
      "[4040,   391] loss: 0.652\n",
      "Train Accuracy: 79.160 %\n",
      "[4050,   391] loss: 0.689\n",
      "Train Accuracy: 78.422 %\n",
      "[4060,   391] loss: 0.649\n",
      "Train Accuracy: 79.468 %\n",
      "[4070,   391] loss: 0.666\n",
      "Train Accuracy: 79.524 %\n",
      "[4080,   391] loss: 0.651\n",
      "Train Accuracy: 79.266 %\n",
      "[4090,   391] loss: 0.664\n",
      "Train Accuracy: 79.942 %\n",
      "[4100,   391] loss: 0.663\n",
      "Train Accuracy: 79.568 %\n",
      "[4110,   391] loss: 0.664\n",
      "Train Accuracy: 78.746 %\n",
      "[4120,   391] loss: 0.660\n",
      "Train Accuracy: 80.000 %\n",
      "[4130,   391] loss: 0.656\n",
      "Train Accuracy: 79.248 %\n",
      "[4140,   391] loss: 0.657\n",
      "Train Accuracy: 78.936 %\n",
      "[4150,   391] loss: 0.649\n",
      "Train Accuracy: 80.060 %\n",
      "[4160,   391] loss: 0.656\n",
      "Train Accuracy: 79.524 %\n",
      "[4170,   391] loss: 0.681\n",
      "Train Accuracy: 79.290 %\n",
      "[4180,   391] loss: 0.672\n",
      "Train Accuracy: 79.188 %\n",
      "[4190,   391] loss: 0.664\n",
      "Train Accuracy: 78.916 %\n",
      "[4200,   391] loss: 0.665\n",
      "Train Accuracy: 80.076 %\n",
      "[4210,   391] loss: 0.673\n",
      "Train Accuracy: 79.120 %\n",
      "[4220,   391] loss: 0.646\n",
      "Train Accuracy: 79.806 %\n",
      "[4230,   391] loss: 0.653\n",
      "Train Accuracy: 79.208 %\n",
      "[4240,   391] loss: 0.661\n",
      "Train Accuracy: 79.944 %\n",
      "[4250,   391] loss: 0.653\n",
      "Train Accuracy: 79.258 %\n",
      "[4260,   391] loss: 0.650\n",
      "Train Accuracy: 79.188 %\n",
      "[4270,   391] loss: 0.665\n",
      "Train Accuracy: 79.572 %\n",
      "[4280,   391] loss: 0.657\n",
      "Train Accuracy: 79.002 %\n",
      "[4290,   391] loss: 0.651\n",
      "Train Accuracy: 79.718 %\n",
      "[4300,   391] loss: 0.656\n",
      "Train Accuracy: 78.490 %\n",
      "[4310,   391] loss: 0.654\n",
      "Train Accuracy: 79.214 %\n",
      "[4320,   391] loss: 0.657\n",
      "Train Accuracy: 78.974 %\n",
      "[4330,   391] loss: 0.658\n",
      "Train Accuracy: 78.884 %\n",
      "[4340,   391] loss: 0.654\n",
      "Train Accuracy: 78.810 %\n",
      "[4350,   391] loss: 0.657\n",
      "Train Accuracy: 79.054 %\n",
      "[4360,   391] loss: 0.651\n",
      "Train Accuracy: 80.140 %\n",
      "[4370,   391] loss: 0.674\n",
      "Train Accuracy: 79.982 %\n",
      "[4380,   391] loss: 0.658\n",
      "Train Accuracy: 79.300 %\n",
      "[4390,   391] loss: 0.661\n",
      "Train Accuracy: 79.340 %\n",
      "[4400,   391] loss: 0.667\n",
      "Train Accuracy: 79.828 %\n",
      "[4410,   391] loss: 0.646\n",
      "Train Accuracy: 79.902 %\n",
      "[4420,   391] loss: 0.673\n",
      "Train Accuracy: 79.562 %\n",
      "[4430,   391] loss: 0.651\n",
      "Train Accuracy: 78.926 %\n",
      "[4440,   391] loss: 0.655\n",
      "Train Accuracy: 79.294 %\n",
      "[4450,   391] loss: 0.672\n",
      "Train Accuracy: 79.648 %\n",
      "[4460,   391] loss: 0.648\n",
      "Train Accuracy: 78.736 %\n",
      "[4470,   391] loss: 0.662\n",
      "Train Accuracy: 79.238 %\n",
      "[4480,   391] loss: 0.656\n",
      "Train Accuracy: 79.588 %\n",
      "[4490,   391] loss: 0.665\n",
      "Train Accuracy: 79.418 %\n",
      "[4500,   391] loss: 0.657\n",
      "Train Accuracy: 79.166 %\n",
      "[4510,   391] loss: 0.674\n",
      "Train Accuracy: 79.668 %\n",
      "[4520,   391] loss: 0.664\n",
      "Train Accuracy: 79.312 %\n",
      "[4530,   391] loss: 0.654\n",
      "Train Accuracy: 78.634 %\n",
      "[4540,   391] loss: 0.667\n",
      "Train Accuracy: 79.044 %\n",
      "[4550,   391] loss: 0.643\n",
      "Train Accuracy: 78.636 %\n",
      "[4560,   391] loss: 0.638\n",
      "Train Accuracy: 79.908 %\n",
      "[4570,   391] loss: 0.651\n",
      "Train Accuracy: 79.452 %\n",
      "[4580,   391] loss: 0.660\n",
      "Train Accuracy: 79.624 %\n",
      "[4590,   391] loss: 0.672\n",
      "Train Accuracy: 79.740 %\n",
      "[4600,   391] loss: 0.640\n",
      "Train Accuracy: 79.874 %\n",
      "[4610,   391] loss: 0.654\n",
      "Train Accuracy: 78.552 %\n",
      "[4620,   391] loss: 0.671\n",
      "Train Accuracy: 79.240 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4630,   391] loss: 0.650\n",
      "Train Accuracy: 79.642 %\n",
      "[4640,   391] loss: 0.628\n",
      "Train Accuracy: 78.392 %\n",
      "[4650,   391] loss: 0.663\n",
      "Train Accuracy: 79.956 %\n",
      "[4660,   391] loss: 0.649\n",
      "Train Accuracy: 79.798 %\n",
      "[4670,   391] loss: 0.666\n",
      "Train Accuracy: 79.086 %\n",
      "[4680,   391] loss: 0.636\n",
      "Train Accuracy: 79.848 %\n",
      "[4690,   391] loss: 0.653\n",
      "Train Accuracy: 78.794 %\n",
      "[4700,   391] loss: 0.655\n",
      "Train Accuracy: 79.198 %\n",
      "[4710,   391] loss: 0.674\n",
      "Train Accuracy: 79.716 %\n",
      "[4720,   391] loss: 0.642\n",
      "Train Accuracy: 78.026 %\n",
      "[4730,   391] loss: 0.655\n",
      "Train Accuracy: 79.822 %\n",
      "[4740,   391] loss: 0.647\n",
      "Train Accuracy: 79.032 %\n",
      "[4750,   391] loss: 0.667\n",
      "Train Accuracy: 78.760 %\n",
      "[4760,   391] loss: 0.651\n",
      "Train Accuracy: 79.444 %\n",
      "[4770,   391] loss: 0.654\n",
      "Train Accuracy: 79.776 %\n",
      "[4780,   391] loss: 0.633\n",
      "Train Accuracy: 79.986 %\n",
      "[4790,   391] loss: 0.668\n",
      "Train Accuracy: 79.672 %\n",
      "[4800,   391] loss: 0.661\n",
      "Train Accuracy: 78.864 %\n",
      "[4810,   391] loss: 0.651\n",
      "Train Accuracy: 80.084 %\n",
      "[4820,   391] loss: 0.650\n",
      "Train Accuracy: 79.536 %\n",
      "[4830,   391] loss: 0.660\n",
      "Train Accuracy: 79.158 %\n",
      "[4840,   391] loss: 0.662\n",
      "Train Accuracy: 79.894 %\n",
      "[4850,   391] loss: 0.653\n",
      "Train Accuracy: 79.508 %\n",
      "[4860,   391] loss: 0.663\n",
      "Train Accuracy: 79.690 %\n",
      "[4870,   391] loss: 0.663\n",
      "Train Accuracy: 79.560 %\n",
      "[4880,   391] loss: 0.659\n",
      "Train Accuracy: 79.852 %\n",
      "[4890,   391] loss: 0.656\n",
      "Train Accuracy: 80.154 %\n",
      "[4900,   391] loss: 0.663\n",
      "Train Accuracy: 79.640 %\n",
      "[4910,   391] loss: 0.668\n",
      "Train Accuracy: 79.954 %\n",
      "[4920,   391] loss: 0.641\n",
      "Train Accuracy: 80.320 %\n",
      "[4930,   391] loss: 0.644\n",
      "Train Accuracy: 80.482 %\n",
      "[4940,   391] loss: 0.645\n",
      "Train Accuracy: 79.016 %\n",
      "[4950,   391] loss: 0.652\n",
      "Train Accuracy: 79.338 %\n",
      "[4960,   391] loss: 0.639\n",
      "Train Accuracy: 80.102 %\n",
      "[4970,   391] loss: 0.649\n",
      "Train Accuracy: 79.622 %\n",
      "[4980,   391] loss: 0.681\n",
      "Train Accuracy: 79.528 %\n",
      "[4990,   391] loss: 0.651\n",
      "Train Accuracy: 79.982 %\n",
      "[5000,   391] loss: 0.677\n",
      "Train Accuracy: 79.342 %\n",
      "[5010,   391] loss: 0.646\n",
      "Train Accuracy: 79.564 %\n",
      "[5020,   391] loss: 0.658\n",
      "Train Accuracy: 79.488 %\n",
      "[5030,   391] loss: 0.667\n",
      "Train Accuracy: 79.518 %\n",
      "[5040,   391] loss: 0.653\n",
      "Train Accuracy: 79.722 %\n",
      "[5050,   391] loss: 0.664\n",
      "Train Accuracy: 79.858 %\n",
      "[5060,   391] loss: 0.671\n",
      "Train Accuracy: 79.180 %\n",
      "[5070,   391] loss: 0.653\n",
      "Train Accuracy: 78.812 %\n",
      "[5080,   391] loss: 0.664\n",
      "Train Accuracy: 79.242 %\n",
      "[5090,   391] loss: 0.659\n",
      "Train Accuracy: 79.830 %\n",
      "[5100,   391] loss: 0.660\n",
      "Train Accuracy: 79.848 %\n",
      "[5110,   391] loss: 0.644\n",
      "Train Accuracy: 79.856 %\n",
      "[5120,   391] loss: 0.646\n",
      "Train Accuracy: 78.654 %\n",
      "[5130,   391] loss: 0.653\n",
      "Train Accuracy: 78.986 %\n",
      "[5140,   391] loss: 0.649\n",
      "Train Accuracy: 79.780 %\n",
      "[5150,   391] loss: 0.663\n",
      "Train Accuracy: 78.532 %\n",
      "[5160,   391] loss: 0.653\n",
      "Train Accuracy: 79.880 %\n",
      "[5170,   391] loss: 0.640\n",
      "Train Accuracy: 78.794 %\n",
      "[5180,   391] loss: 0.657\n",
      "Train Accuracy: 80.216 %\n",
      "[5190,   391] loss: 0.643\n",
      "Train Accuracy: 79.582 %\n",
      "[5200,   391] loss: 0.656\n",
      "Train Accuracy: 80.024 %\n",
      "[5210,   391] loss: 0.656\n",
      "Train Accuracy: 80.186 %\n",
      "[5220,   391] loss: 0.652\n",
      "Train Accuracy: 79.282 %\n",
      "[5230,   391] loss: 0.654\n",
      "Train Accuracy: 79.280 %\n",
      "[5240,   391] loss: 0.645\n",
      "Train Accuracy: 79.498 %\n",
      "[5250,   391] loss: 0.648\n",
      "Train Accuracy: 80.246 %\n",
      "[5260,   391] loss: 0.647\n",
      "Train Accuracy: 80.066 %\n",
      "[5270,   391] loss: 0.654\n",
      "Train Accuracy: 80.268 %\n",
      "[5280,   391] loss: 0.659\n",
      "Train Accuracy: 80.604 %\n",
      "[5290,   391] loss: 0.649\n",
      "Train Accuracy: 79.690 %\n",
      "[5300,   391] loss: 0.654\n",
      "Train Accuracy: 80.180 %\n",
      "[5310,   391] loss: 0.648\n",
      "Train Accuracy: 80.164 %\n",
      "[5320,   391] loss: 0.644\n",
      "Train Accuracy: 79.244 %\n",
      "[5330,   391] loss: 0.660\n",
      "Train Accuracy: 78.990 %\n",
      "[5340,   391] loss: 0.662\n",
      "Train Accuracy: 79.308 %\n",
      "[5350,   391] loss: 0.656\n",
      "Train Accuracy: 79.460 %\n",
      "[5360,   391] loss: 0.660\n",
      "Train Accuracy: 79.772 %\n",
      "[5370,   391] loss: 0.652\n",
      "Train Accuracy: 79.780 %\n",
      "[5380,   391] loss: 0.644\n",
      "Train Accuracy: 79.754 %\n",
      "[5390,   391] loss: 0.656\n",
      "Train Accuracy: 79.766 %\n",
      "[5400,   391] loss: 0.660\n",
      "Train Accuracy: 80.322 %\n",
      "[5410,   391] loss: 0.659\n",
      "Train Accuracy: 79.634 %\n",
      "[5420,   391] loss: 0.658\n",
      "Train Accuracy: 80.110 %\n",
      "[5430,   391] loss: 0.647\n",
      "Train Accuracy: 79.922 %\n",
      "[5440,   391] loss: 0.652\n",
      "Train Accuracy: 79.410 %\n",
      "[5450,   391] loss: 0.658\n",
      "Train Accuracy: 79.768 %\n",
      "[5460,   391] loss: 0.647\n",
      "Train Accuracy: 79.892 %\n",
      "[5470,   391] loss: 0.658\n",
      "Train Accuracy: 79.818 %\n",
      "[5480,   391] loss: 0.639\n",
      "Train Accuracy: 79.782 %\n",
      "[5490,   391] loss: 0.651\n",
      "Train Accuracy: 80.098 %\n",
      "[5500,   391] loss: 0.644\n",
      "Train Accuracy: 79.486 %\n",
      "[5510,   391] loss: 0.638\n",
      "Train Accuracy: 79.856 %\n",
      "[5520,   391] loss: 0.646\n",
      "Train Accuracy: 79.916 %\n",
      "[5530,   391] loss: 0.635\n",
      "Train Accuracy: 80.140 %\n",
      "[5540,   391] loss: 0.672\n",
      "Train Accuracy: 79.450 %\n",
      "[5550,   391] loss: 0.652\n",
      "Train Accuracy: 80.040 %\n",
      "[5560,   391] loss: 0.648\n",
      "Train Accuracy: 78.946 %\n",
      "[5570,   391] loss: 0.654\n",
      "Train Accuracy: 79.750 %\n",
      "[5580,   391] loss: 0.642\n",
      "Train Accuracy: 80.040 %\n",
      "[5590,   391] loss: 0.653\n",
      "Train Accuracy: 80.038 %\n",
      "[5600,   391] loss: 0.661\n",
      "Train Accuracy: 79.674 %\n",
      "[5610,   391] loss: 0.652\n",
      "Train Accuracy: 80.564 %\n",
      "[5620,   391] loss: 0.642\n",
      "Train Accuracy: 79.682 %\n",
      "[5630,   391] loss: 0.656\n",
      "Train Accuracy: 79.688 %\n",
      "[5640,   391] loss: 0.638\n",
      "Train Accuracy: 79.762 %\n",
      "[5650,   391] loss: 0.648\n",
      "Train Accuracy: 78.220 %\n",
      "[5660,   391] loss: 0.649\n",
      "Train Accuracy: 78.982 %\n",
      "[5670,   391] loss: 0.652\n",
      "Train Accuracy: 79.660 %\n",
      "[5680,   391] loss: 0.666\n",
      "Train Accuracy: 78.800 %\n",
      "[5690,   391] loss: 0.655\n",
      "Train Accuracy: 79.186 %\n",
      "[5700,   391] loss: 0.653\n",
      "Train Accuracy: 79.946 %\n",
      "[5710,   391] loss: 0.647\n",
      "Train Accuracy: 79.810 %\n",
      "[5720,   391] loss: 0.642\n",
      "Train Accuracy: 79.544 %\n",
      "[5730,   391] loss: 0.659\n",
      "Train Accuracy: 80.448 %\n",
      "[5740,   391] loss: 0.663\n",
      "Train Accuracy: 79.536 %\n",
      "[5750,   391] loss: 0.642\n",
      "Train Accuracy: 79.804 %\n",
      "[5760,   391] loss: 0.659\n",
      "Train Accuracy: 79.032 %\n",
      "[5770,   391] loss: 0.658\n",
      "Train Accuracy: 80.082 %\n",
      "[5780,   391] loss: 0.654\n",
      "Train Accuracy: 79.114 %\n",
      "[5790,   391] loss: 0.631\n",
      "Train Accuracy: 80.140 %\n",
      "[5800,   391] loss: 0.655\n",
      "Train Accuracy: 79.016 %\n",
      "[5810,   391] loss: 0.664\n",
      "Train Accuracy: 80.270 %\n",
      "[5820,   391] loss: 0.656\n",
      "Train Accuracy: 78.878 %\n",
      "[5830,   391] loss: 0.640\n",
      "Train Accuracy: 79.312 %\n",
      "[5840,   391] loss: 0.655\n",
      "Train Accuracy: 78.530 %\n",
      "[5850,   391] loss: 0.654\n",
      "Train Accuracy: 79.636 %\n",
      "[5860,   391] loss: 0.658\n",
      "Train Accuracy: 79.454 %\n",
      "[5870,   391] loss: 0.644\n",
      "Train Accuracy: 79.226 %\n",
      "[5880,   391] loss: 0.630\n",
      "Train Accuracy: 78.862 %\n",
      "[5890,   391] loss: 0.658\n",
      "Train Accuracy: 79.708 %\n",
      "[5900,   391] loss: 0.653\n",
      "Train Accuracy: 79.840 %\n",
      "[5910,   391] loss: 0.663\n",
      "Train Accuracy: 79.970 %\n",
      "[5920,   391] loss: 0.662\n",
      "Train Accuracy: 79.596 %\n",
      "[5930,   391] loss: 0.653\n",
      "Train Accuracy: 80.258 %\n",
      "[5940,   391] loss: 0.650\n",
      "Train Accuracy: 78.474 %\n",
      "[5950,   391] loss: 0.651\n",
      "Train Accuracy: 79.536 %\n",
      "[5960,   391] loss: 0.636\n",
      "Train Accuracy: 80.636 %\n",
      "[5970,   391] loss: 0.634\n",
      "Train Accuracy: 79.406 %\n",
      "[5980,   391] loss: 0.658\n",
      "Train Accuracy: 79.438 %\n",
      "[5990,   391] loss: 0.646\n",
      "Train Accuracy: 79.404 %\n",
      "[6000,   391] loss: 0.652\n",
      "Train Accuracy: 80.050 %\n",
      "Test Accuracy: 37.070 %, Test Loss: 6.516\n",
      "--------------------------------------\n",
      "Prune the model by 91.13 %\n",
      "Pruning with threshold : 0.1335\n",
      "Params size (MB): 0.049\n",
      "[10,   391] loss: 1.709\n",
      "Train Accuracy: 44.316 %\n",
      "[20,   391] loss: 1.635\n",
      "Train Accuracy: 47.610 %\n",
      "[30,   391] loss: 1.595\n",
      "Train Accuracy: 49.572 %\n",
      "[40,   391] loss: 1.547\n",
      "Train Accuracy: 51.022 %\n",
      "[50,   391] loss: 1.534\n",
      "Train Accuracy: 51.872 %\n",
      "[60,   391] loss: 1.508\n",
      "Train Accuracy: 52.758 %\n",
      "[70,   391] loss: 1.490\n",
      "Train Accuracy: 53.358 %\n",
      "[80,   391] loss: 1.481\n",
      "Train Accuracy: 54.004 %\n",
      "[90,   391] loss: 1.471\n",
      "Train Accuracy: 54.088 %\n",
      "[100,   391] loss: 1.445\n",
      "Train Accuracy: 54.364 %\n",
      "[110,   391] loss: 1.460\n",
      "Train Accuracy: 55.120 %\n",
      "[120,   391] loss: 1.435\n",
      "Train Accuracy: 55.416 %\n",
      "[130,   391] loss: 1.429\n",
      "Train Accuracy: 55.646 %\n",
      "[140,   391] loss: 1.407\n",
      "Train Accuracy: 56.002 %\n",
      "[150,   391] loss: 1.424\n",
      "Train Accuracy: 56.346 %\n",
      "[160,   391] loss: 1.403\n",
      "Train Accuracy: 56.606 %\n",
      "[170,   391] loss: 1.399\n",
      "Train Accuracy: 56.922 %\n",
      "[180,   391] loss: 1.384\n",
      "Train Accuracy: 57.022 %\n",
      "[190,   391] loss: 1.382\n",
      "Train Accuracy: 57.246 %\n",
      "[200,   391] loss: 1.375\n",
      "Train Accuracy: 57.440 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[210,   391] loss: 1.399\n",
      "Train Accuracy: 57.544 %\n",
      "[220,   391] loss: 1.376\n",
      "Train Accuracy: 57.822 %\n",
      "[230,   391] loss: 1.367\n",
      "Train Accuracy: 57.824 %\n",
      "[240,   391] loss: 1.349\n",
      "Train Accuracy: 57.882 %\n",
      "[250,   391] loss: 1.342\n",
      "Train Accuracy: 58.182 %\n",
      "[260,   391] loss: 1.327\n",
      "Train Accuracy: 58.388 %\n",
      "[270,   391] loss: 1.348\n",
      "Train Accuracy: 58.474 %\n",
      "[280,   391] loss: 1.338\n",
      "Train Accuracy: 58.626 %\n",
      "[290,   391] loss: 1.340\n",
      "Train Accuracy: 58.672 %\n",
      "[300,   391] loss: 1.348\n",
      "Train Accuracy: 58.588 %\n",
      "[310,   391] loss: 1.326\n",
      "Train Accuracy: 58.984 %\n",
      "[320,   391] loss: 1.343\n",
      "Train Accuracy: 58.960 %\n",
      "[330,   391] loss: 1.315\n",
      "Train Accuracy: 59.162 %\n",
      "[340,   391] loss: 1.328\n",
      "Train Accuracy: 59.278 %\n",
      "[350,   391] loss: 1.325\n",
      "Train Accuracy: 59.418 %\n",
      "[360,   391] loss: 1.312\n",
      "Train Accuracy: 59.366 %\n",
      "[370,   391] loss: 1.324\n",
      "Train Accuracy: 59.588 %\n",
      "[380,   391] loss: 1.327\n",
      "Train Accuracy: 59.638 %\n",
      "[390,   391] loss: 1.303\n",
      "Train Accuracy: 59.782 %\n",
      "[400,   391] loss: 1.310\n",
      "Train Accuracy: 59.682 %\n",
      "[410,   391] loss: 1.319\n",
      "Train Accuracy: 59.752 %\n",
      "[420,   391] loss: 1.288\n",
      "Train Accuracy: 59.758 %\n",
      "[430,   391] loss: 1.302\n",
      "Train Accuracy: 60.022 %\n",
      "[440,   391] loss: 1.308\n",
      "Train Accuracy: 59.862 %\n",
      "[450,   391] loss: 1.306\n",
      "Train Accuracy: 59.888 %\n",
      "[460,   391] loss: 1.302\n",
      "Train Accuracy: 59.998 %\n",
      "[470,   391] loss: 1.299\n",
      "Train Accuracy: 60.036 %\n",
      "[480,   391] loss: 1.290\n",
      "Train Accuracy: 60.314 %\n",
      "[490,   391] loss: 1.272\n",
      "Train Accuracy: 60.372 %\n",
      "[500,   391] loss: 1.288\n",
      "Train Accuracy: 60.260 %\n",
      "[510,   391] loss: 1.274\n",
      "Train Accuracy: 60.404 %\n",
      "[520,   391] loss: 1.278\n",
      "Train Accuracy: 60.532 %\n",
      "[530,   391] loss: 1.285\n",
      "Train Accuracy: 60.456 %\n",
      "[540,   391] loss: 1.283\n",
      "Train Accuracy: 60.794 %\n",
      "[550,   391] loss: 1.288\n",
      "Train Accuracy: 60.610 %\n",
      "[560,   391] loss: 1.280\n",
      "Train Accuracy: 60.700 %\n",
      "[570,   391] loss: 1.274\n",
      "Train Accuracy: 60.662 %\n",
      "[580,   391] loss: 1.268\n",
      "Train Accuracy: 60.794 %\n",
      "[590,   391] loss: 1.268\n",
      "Train Accuracy: 60.654 %\n",
      "[600,   391] loss: 1.286\n",
      "Train Accuracy: 60.862 %\n",
      "[610,   391] loss: 1.253\n",
      "Train Accuracy: 60.950 %\n",
      "[620,   391] loss: 1.290\n",
      "Train Accuracy: 60.980 %\n",
      "[630,   391] loss: 1.262\n",
      "Train Accuracy: 60.796 %\n",
      "[640,   391] loss: 1.261\n",
      "Train Accuracy: 61.096 %\n",
      "[650,   391] loss: 1.271\n",
      "Train Accuracy: 60.946 %\n",
      "[660,   391] loss: 1.271\n",
      "Train Accuracy: 60.964 %\n",
      "[670,   391] loss: 1.253\n",
      "Train Accuracy: 61.130 %\n",
      "[680,   391] loss: 1.274\n",
      "Train Accuracy: 61.242 %\n",
      "[690,   391] loss: 1.246\n",
      "Train Accuracy: 61.036 %\n",
      "[700,   391] loss: 1.241\n",
      "Train Accuracy: 61.120 %\n",
      "[710,   391] loss: 1.259\n",
      "Train Accuracy: 61.220 %\n",
      "[720,   391] loss: 1.249\n",
      "Train Accuracy: 61.060 %\n",
      "[730,   391] loss: 1.258\n",
      "Train Accuracy: 61.244 %\n",
      "[740,   391] loss: 1.247\n",
      "Train Accuracy: 61.108 %\n",
      "[750,   391] loss: 1.256\n",
      "Train Accuracy: 61.172 %\n",
      "[760,   391] loss: 1.253\n",
      "Train Accuracy: 60.968 %\n",
      "[770,   391] loss: 1.253\n",
      "Train Accuracy: 61.234 %\n",
      "[780,   391] loss: 1.251\n",
      "Train Accuracy: 61.362 %\n",
      "[790,   391] loss: 1.257\n",
      "Train Accuracy: 61.368 %\n",
      "[800,   391] loss: 1.246\n",
      "Train Accuracy: 61.428 %\n",
      "[810,   391] loss: 1.245\n",
      "Train Accuracy: 61.596 %\n",
      "[820,   391] loss: 1.246\n",
      "Train Accuracy: 61.384 %\n",
      "[830,   391] loss: 1.250\n",
      "Train Accuracy: 61.498 %\n",
      "[840,   391] loss: 1.236\n",
      "Train Accuracy: 61.190 %\n",
      "[850,   391] loss: 1.256\n",
      "Train Accuracy: 61.504 %\n",
      "[860,   391] loss: 1.251\n",
      "Train Accuracy: 61.514 %\n",
      "[870,   391] loss: 1.257\n",
      "Train Accuracy: 61.562 %\n",
      "[880,   391] loss: 1.257\n",
      "Train Accuracy: 61.618 %\n",
      "[890,   391] loss: 1.244\n",
      "Train Accuracy: 61.460 %\n",
      "[900,   391] loss: 1.244\n",
      "Train Accuracy: 61.596 %\n",
      "[910,   391] loss: 1.246\n",
      "Train Accuracy: 61.626 %\n",
      "[920,   391] loss: 1.247\n",
      "Train Accuracy: 61.616 %\n",
      "[930,   391] loss: 1.225\n",
      "Train Accuracy: 61.310 %\n",
      "[940,   391] loss: 1.230\n",
      "Train Accuracy: 61.848 %\n",
      "[950,   391] loss: 1.242\n",
      "Train Accuracy: 61.800 %\n",
      "[960,   391] loss: 1.239\n",
      "Train Accuracy: 61.794 %\n",
      "[970,   391] loss: 1.227\n",
      "Train Accuracy: 61.890 %\n",
      "[980,   391] loss: 1.225\n",
      "Train Accuracy: 61.800 %\n",
      "[990,   391] loss: 1.228\n",
      "Train Accuracy: 61.930 %\n",
      "[1000,   391] loss: 1.235\n",
      "Train Accuracy: 61.894 %\n",
      "[1010,   391] loss: 1.227\n",
      "Train Accuracy: 61.994 %\n",
      "[1020,   391] loss: 1.247\n",
      "Train Accuracy: 61.764 %\n",
      "[1030,   391] loss: 1.229\n",
      "Train Accuracy: 61.896 %\n",
      "[1040,   391] loss: 1.224\n",
      "Train Accuracy: 61.956 %\n",
      "[1050,   391] loss: 1.233\n",
      "Train Accuracy: 61.880 %\n",
      "[1060,   391] loss: 1.237\n",
      "Train Accuracy: 61.884 %\n",
      "[1070,   391] loss: 1.231\n",
      "Train Accuracy: 61.976 %\n",
      "[1080,   391] loss: 1.227\n",
      "Train Accuracy: 61.964 %\n",
      "[1090,   391] loss: 1.229\n",
      "Train Accuracy: 61.986 %\n",
      "[1100,   391] loss: 1.217\n",
      "Train Accuracy: 61.838 %\n",
      "[1110,   391] loss: 1.218\n",
      "Train Accuracy: 62.004 %\n",
      "[1120,   391] loss: 1.232\n",
      "Train Accuracy: 62.174 %\n",
      "[1130,   391] loss: 1.246\n",
      "Train Accuracy: 62.140 %\n",
      "[1140,   391] loss: 1.212\n",
      "Train Accuracy: 61.998 %\n",
      "[1150,   391] loss: 1.235\n",
      "Train Accuracy: 62.106 %\n",
      "[1160,   391] loss: 1.221\n",
      "Train Accuracy: 62.054 %\n",
      "[1170,   391] loss: 1.230\n",
      "Train Accuracy: 62.180 %\n",
      "[1180,   391] loss: 1.223\n",
      "Train Accuracy: 62.252 %\n",
      "[1190,   391] loss: 1.209\n",
      "Train Accuracy: 62.234 %\n",
      "[1200,   391] loss: 1.213\n",
      "Train Accuracy: 62.064 %\n",
      "[1210,   391] loss: 1.208\n",
      "Train Accuracy: 62.242 %\n",
      "[1220,   391] loss: 1.215\n",
      "Train Accuracy: 62.200 %\n",
      "[1230,   391] loss: 1.214\n",
      "Train Accuracy: 62.196 %\n",
      "[1240,   391] loss: 1.218\n",
      "Train Accuracy: 62.394 %\n",
      "[1250,   391] loss: 1.208\n",
      "Train Accuracy: 62.396 %\n",
      "[1260,   391] loss: 1.227\n",
      "Train Accuracy: 62.450 %\n",
      "[1270,   391] loss: 1.214\n",
      "Train Accuracy: 62.274 %\n",
      "[1280,   391] loss: 1.202\n",
      "Train Accuracy: 62.342 %\n",
      "[1290,   391] loss: 1.227\n",
      "Train Accuracy: 62.338 %\n",
      "[1300,   391] loss: 1.214\n",
      "Train Accuracy: 62.568 %\n",
      "[1310,   391] loss: 1.214\n",
      "Train Accuracy: 62.550 %\n",
      "[1320,   391] loss: 1.203\n",
      "Train Accuracy: 62.514 %\n",
      "[1330,   391] loss: 1.207\n",
      "Train Accuracy: 62.532 %\n",
      "[1340,   391] loss: 1.223\n",
      "Train Accuracy: 62.370 %\n",
      "[1350,   391] loss: 1.209\n",
      "Train Accuracy: 62.526 %\n",
      "[1360,   391] loss: 1.204\n",
      "Train Accuracy: 62.492 %\n",
      "[1370,   391] loss: 1.192\n",
      "Train Accuracy: 62.584 %\n",
      "[1380,   391] loss: 1.225\n",
      "Train Accuracy: 62.546 %\n",
      "[1390,   391] loss: 1.227\n",
      "Train Accuracy: 62.506 %\n",
      "[1400,   391] loss: 1.211\n",
      "Train Accuracy: 62.538 %\n",
      "[1410,   391] loss: 1.215\n",
      "Train Accuracy: 62.558 %\n",
      "[1420,   391] loss: 1.207\n",
      "Train Accuracy: 62.426 %\n",
      "[1430,   391] loss: 1.197\n",
      "Train Accuracy: 62.678 %\n",
      "[1440,   391] loss: 1.221\n",
      "Train Accuracy: 62.598 %\n",
      "[1450,   391] loss: 1.195\n",
      "Train Accuracy: 62.396 %\n",
      "[1460,   391] loss: 1.210\n",
      "Train Accuracy: 62.728 %\n",
      "[1470,   391] loss: 1.227\n",
      "Train Accuracy: 62.776 %\n",
      "[1480,   391] loss: 1.205\n",
      "Train Accuracy: 62.346 %\n",
      "[1490,   391] loss: 1.199\n",
      "Train Accuracy: 62.588 %\n",
      "[1500,   391] loss: 1.194\n",
      "Train Accuracy: 62.618 %\n",
      "[1510,   391] loss: 1.200\n",
      "Train Accuracy: 62.532 %\n",
      "[1520,   391] loss: 1.207\n",
      "Train Accuracy: 62.602 %\n",
      "[1530,   391] loss: 1.210\n",
      "Train Accuracy: 62.378 %\n",
      "[1540,   391] loss: 1.209\n",
      "Train Accuracy: 62.638 %\n",
      "[1550,   391] loss: 1.208\n",
      "Train Accuracy: 62.682 %\n",
      "[1560,   391] loss: 1.196\n",
      "Train Accuracy: 62.518 %\n",
      "[1570,   391] loss: 1.221\n",
      "Train Accuracy: 62.532 %\n",
      "[1580,   391] loss: 1.202\n",
      "Train Accuracy: 62.718 %\n",
      "[1590,   391] loss: 1.190\n",
      "Train Accuracy: 62.594 %\n",
      "[1600,   391] loss: 1.216\n",
      "Train Accuracy: 62.662 %\n",
      "[1610,   391] loss: 1.191\n",
      "Train Accuracy: 62.872 %\n",
      "[1620,   391] loss: 1.185\n",
      "Train Accuracy: 62.598 %\n",
      "[1630,   391] loss: 1.207\n",
      "Train Accuracy: 62.736 %\n",
      "[1640,   391] loss: 1.199\n",
      "Train Accuracy: 62.744 %\n",
      "[1650,   391] loss: 1.197\n",
      "Train Accuracy: 62.686 %\n",
      "[1660,   391] loss: 1.201\n",
      "Train Accuracy: 62.918 %\n",
      "[1670,   391] loss: 1.198\n",
      "Train Accuracy: 62.920 %\n",
      "[1680,   391] loss: 1.203\n",
      "Train Accuracy: 62.830 %\n",
      "[1690,   391] loss: 1.204\n",
      "Train Accuracy: 62.864 %\n",
      "[1700,   391] loss: 1.192\n",
      "Train Accuracy: 62.884 %\n",
      "[1710,   391] loss: 1.197\n",
      "Train Accuracy: 62.988 %\n",
      "[1720,   391] loss: 1.194\n",
      "Train Accuracy: 62.928 %\n",
      "[1730,   391] loss: 1.206\n",
      "Train Accuracy: 62.738 %\n",
      "[1740,   391] loss: 1.202\n",
      "Train Accuracy: 62.886 %\n",
      "[1750,   391] loss: 1.200\n",
      "Train Accuracy: 62.822 %\n",
      "[1760,   391] loss: 1.210\n",
      "Train Accuracy: 62.846 %\n",
      "[1770,   391] loss: 1.190\n",
      "Train Accuracy: 63.060 %\n",
      "[1780,   391] loss: 1.180\n",
      "Train Accuracy: 62.516 %\n",
      "[1790,   391] loss: 1.196\n",
      "Train Accuracy: 62.706 %\n",
      "[1800,   391] loss: 1.179\n",
      "Train Accuracy: 63.084 %\n",
      "[1810,   391] loss: 1.176\n",
      "Train Accuracy: 63.180 %\n",
      "[1820,   391] loss: 1.190\n",
      "Train Accuracy: 62.950 %\n",
      "[1830,   391] loss: 1.201\n",
      "Train Accuracy: 62.852 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1840,   391] loss: 1.179\n",
      "Train Accuracy: 63.164 %\n",
      "[1850,   391] loss: 1.188\n",
      "Train Accuracy: 62.996 %\n",
      "[1860,   391] loss: 1.202\n",
      "Train Accuracy: 63.132 %\n",
      "[1870,   391] loss: 1.180\n",
      "Train Accuracy: 63.076 %\n",
      "[1880,   391] loss: 1.186\n",
      "Train Accuracy: 62.966 %\n",
      "[1890,   391] loss: 1.183\n",
      "Train Accuracy: 62.992 %\n",
      "[1900,   391] loss: 1.175\n",
      "Train Accuracy: 63.240 %\n",
      "[1910,   391] loss: 1.188\n",
      "Train Accuracy: 62.996 %\n",
      "[1920,   391] loss: 1.184\n",
      "Train Accuracy: 63.146 %\n",
      "[1930,   391] loss: 1.197\n",
      "Train Accuracy: 63.126 %\n",
      "[1940,   391] loss: 1.198\n",
      "Train Accuracy: 62.942 %\n",
      "[1950,   391] loss: 1.182\n",
      "Train Accuracy: 62.980 %\n",
      "[1960,   391] loss: 1.190\n",
      "Train Accuracy: 62.984 %\n",
      "[1970,   391] loss: 1.187\n",
      "Train Accuracy: 63.212 %\n",
      "[1980,   391] loss: 1.182\n",
      "Train Accuracy: 63.140 %\n",
      "[1990,   391] loss: 1.185\n",
      "Train Accuracy: 63.192 %\n",
      "[2000,   391] loss: 1.187\n",
      "Train Accuracy: 63.096 %\n",
      "[2010,   391] loss: 1.182\n",
      "Train Accuracy: 63.222 %\n",
      "[2020,   391] loss: 1.179\n",
      "Train Accuracy: 63.316 %\n",
      "[2030,   391] loss: 1.172\n",
      "Train Accuracy: 63.010 %\n",
      "[2040,   391] loss: 1.182\n",
      "Train Accuracy: 63.218 %\n",
      "[2050,   391] loss: 1.202\n",
      "Train Accuracy: 63.176 %\n",
      "[2060,   391] loss: 1.186\n",
      "Train Accuracy: 63.322 %\n",
      "[2070,   391] loss: 1.176\n",
      "Train Accuracy: 63.246 %\n",
      "[2080,   391] loss: 1.184\n",
      "Train Accuracy: 63.148 %\n",
      "[2090,   391] loss: 1.191\n",
      "Train Accuracy: 62.772 %\n",
      "[2100,   391] loss: 1.187\n",
      "Train Accuracy: 63.358 %\n",
      "[2110,   391] loss: 1.188\n",
      "Train Accuracy: 63.322 %\n",
      "[2120,   391] loss: 1.182\n",
      "Train Accuracy: 63.294 %\n",
      "[2130,   391] loss: 1.166\n",
      "Train Accuracy: 63.206 %\n",
      "[2140,   391] loss: 1.192\n",
      "Train Accuracy: 63.152 %\n",
      "[2150,   391] loss: 1.194\n",
      "Train Accuracy: 63.494 %\n",
      "[2160,   391] loss: 1.175\n",
      "Train Accuracy: 63.400 %\n",
      "[2170,   391] loss: 1.188\n",
      "Train Accuracy: 63.248 %\n",
      "[2180,   391] loss: 1.182\n",
      "Train Accuracy: 63.234 %\n",
      "[2190,   391] loss: 1.189\n",
      "Train Accuracy: 62.900 %\n",
      "[2200,   391] loss: 1.188\n",
      "Train Accuracy: 63.322 %\n",
      "[2210,   391] loss: 1.180\n",
      "Train Accuracy: 63.408 %\n",
      "[2220,   391] loss: 1.180\n",
      "Train Accuracy: 63.248 %\n",
      "[2230,   391] loss: 1.176\n",
      "Train Accuracy: 63.400 %\n",
      "[2240,   391] loss: 1.194\n",
      "Train Accuracy: 63.278 %\n",
      "[2250,   391] loss: 1.185\n",
      "Train Accuracy: 63.166 %\n",
      "[2260,   391] loss: 1.186\n",
      "Train Accuracy: 63.360 %\n",
      "[2270,   391] loss: 1.179\n",
      "Train Accuracy: 63.134 %\n",
      "[2280,   391] loss: 1.166\n",
      "Train Accuracy: 63.358 %\n",
      "[2290,   391] loss: 1.171\n",
      "Train Accuracy: 63.348 %\n",
      "[2300,   391] loss: 1.194\n",
      "Train Accuracy: 63.150 %\n",
      "[2310,   391] loss: 1.181\n",
      "Train Accuracy: 63.364 %\n",
      "[2320,   391] loss: 1.189\n",
      "Train Accuracy: 63.396 %\n",
      "[2330,   391] loss: 1.190\n",
      "Train Accuracy: 63.514 %\n",
      "[2340,   391] loss: 1.183\n",
      "Train Accuracy: 63.076 %\n",
      "[2350,   391] loss: 1.189\n",
      "Train Accuracy: 63.200 %\n",
      "[2360,   391] loss: 1.177\n",
      "Train Accuracy: 63.572 %\n",
      "[2370,   391] loss: 1.173\n",
      "Train Accuracy: 63.518 %\n",
      "[2380,   391] loss: 1.180\n",
      "Train Accuracy: 63.370 %\n",
      "[2390,   391] loss: 1.168\n",
      "Train Accuracy: 63.406 %\n",
      "[2400,   391] loss: 1.167\n",
      "Train Accuracy: 63.418 %\n",
      "[2410,   391] loss: 1.174\n",
      "Train Accuracy: 63.486 %\n",
      "[2420,   391] loss: 1.188\n",
      "Train Accuracy: 63.556 %\n",
      "[2430,   391] loss: 1.182\n",
      "Train Accuracy: 63.404 %\n",
      "[2440,   391] loss: 1.183\n",
      "Train Accuracy: 63.312 %\n",
      "[2450,   391] loss: 1.183\n",
      "Train Accuracy: 63.574 %\n",
      "[2460,   391] loss: 1.185\n",
      "Train Accuracy: 63.416 %\n",
      "[2470,   391] loss: 1.182\n",
      "Train Accuracy: 63.500 %\n",
      "[2480,   391] loss: 1.175\n",
      "Train Accuracy: 63.554 %\n",
      "[2490,   391] loss: 1.187\n",
      "Train Accuracy: 63.606 %\n",
      "[2500,   391] loss: 1.175\n",
      "Train Accuracy: 63.326 %\n",
      "[2510,   391] loss: 1.169\n",
      "Train Accuracy: 63.308 %\n",
      "[2520,   391] loss: 1.161\n",
      "Train Accuracy: 63.552 %\n",
      "[2530,   391] loss: 1.187\n",
      "Train Accuracy: 63.644 %\n",
      "[2540,   391] loss: 1.172\n",
      "Train Accuracy: 63.538 %\n",
      "[2550,   391] loss: 1.175\n",
      "Train Accuracy: 63.630 %\n",
      "[2560,   391] loss: 1.167\n",
      "Train Accuracy: 63.568 %\n",
      "[2570,   391] loss: 1.167\n",
      "Train Accuracy: 63.528 %\n",
      "[2580,   391] loss: 1.183\n",
      "Train Accuracy: 63.618 %\n",
      "[2590,   391] loss: 1.161\n",
      "Train Accuracy: 63.274 %\n",
      "[2600,   391] loss: 1.180\n",
      "Train Accuracy: 63.852 %\n",
      "[2610,   391] loss: 1.180\n",
      "Train Accuracy: 63.410 %\n",
      "[2620,   391] loss: 1.177\n",
      "Train Accuracy: 63.826 %\n",
      "[2630,   391] loss: 1.171\n",
      "Train Accuracy: 63.528 %\n",
      "[2640,   391] loss: 1.171\n",
      "Train Accuracy: 63.792 %\n",
      "[2650,   391] loss: 1.171\n",
      "Train Accuracy: 63.840 %\n",
      "[2660,   391] loss: 1.166\n",
      "Train Accuracy: 63.628 %\n",
      "[2670,   391] loss: 1.162\n",
      "Train Accuracy: 63.334 %\n",
      "[2680,   391] loss: 1.163\n",
      "Train Accuracy: 63.520 %\n",
      "[2690,   391] loss: 1.161\n",
      "Train Accuracy: 63.536 %\n",
      "[2700,   391] loss: 1.183\n",
      "Train Accuracy: 63.366 %\n",
      "[2710,   391] loss: 1.172\n",
      "Train Accuracy: 63.670 %\n",
      "[2720,   391] loss: 1.168\n",
      "Train Accuracy: 63.688 %\n",
      "[2730,   391] loss: 1.169\n",
      "Train Accuracy: 63.646 %\n",
      "[2740,   391] loss: 1.164\n",
      "Train Accuracy: 63.720 %\n",
      "[2750,   391] loss: 1.185\n",
      "Train Accuracy: 63.664 %\n",
      "[2760,   391] loss: 1.163\n",
      "Train Accuracy: 63.588 %\n",
      "[2770,   391] loss: 1.171\n",
      "Train Accuracy: 63.564 %\n",
      "[2780,   391] loss: 1.163\n",
      "Train Accuracy: 63.608 %\n",
      "[2790,   391] loss: 1.150\n",
      "Train Accuracy: 63.576 %\n",
      "[2800,   391] loss: 1.160\n",
      "Train Accuracy: 63.638 %\n",
      "[2810,   391] loss: 1.164\n",
      "Train Accuracy: 63.578 %\n",
      "[2820,   391] loss: 1.168\n",
      "Train Accuracy: 63.624 %\n",
      "[2830,   391] loss: 1.191\n",
      "Train Accuracy: 63.772 %\n",
      "[2840,   391] loss: 1.176\n",
      "Train Accuracy: 63.732 %\n",
      "[2850,   391] loss: 1.185\n",
      "Train Accuracy: 63.770 %\n",
      "[2860,   391] loss: 1.180\n",
      "Train Accuracy: 63.740 %\n",
      "[2870,   391] loss: 1.185\n",
      "Train Accuracy: 63.524 %\n",
      "[2880,   391] loss: 1.176\n",
      "Train Accuracy: 63.828 %\n",
      "[2890,   391] loss: 1.164\n",
      "Train Accuracy: 63.512 %\n",
      "[2900,   391] loss: 1.181\n",
      "Train Accuracy: 63.352 %\n",
      "[2910,   391] loss: 1.154\n",
      "Train Accuracy: 63.910 %\n",
      "[2920,   391] loss: 1.165\n",
      "Train Accuracy: 63.330 %\n",
      "[2930,   391] loss: 1.155\n",
      "Train Accuracy: 63.676 %\n",
      "[2940,   391] loss: 1.149\n",
      "Train Accuracy: 63.450 %\n",
      "[2950,   391] loss: 1.169\n",
      "Train Accuracy: 63.426 %\n",
      "[2960,   391] loss: 1.174\n",
      "Train Accuracy: 63.810 %\n",
      "[2970,   391] loss: 1.154\n",
      "Train Accuracy: 63.674 %\n",
      "[2980,   391] loss: 1.187\n",
      "Train Accuracy: 63.594 %\n",
      "[2990,   391] loss: 1.176\n",
      "Train Accuracy: 63.710 %\n",
      "[3000,   391] loss: 1.176\n",
      "Train Accuracy: 63.610 %\n",
      "[3010,   391] loss: 1.158\n",
      "Train Accuracy: 63.408 %\n",
      "[3020,   391] loss: 1.183\n",
      "Train Accuracy: 63.866 %\n",
      "[3030,   391] loss: 1.165\n",
      "Train Accuracy: 63.698 %\n",
      "[3040,   391] loss: 1.162\n",
      "Train Accuracy: 63.660 %\n",
      "[3050,   391] loss: 1.177\n",
      "Train Accuracy: 63.568 %\n",
      "[3060,   391] loss: 1.176\n",
      "Train Accuracy: 63.764 %\n",
      "[3070,   391] loss: 1.161\n",
      "Train Accuracy: 63.188 %\n",
      "[3080,   391] loss: 1.156\n",
      "Train Accuracy: 63.820 %\n",
      "[3090,   391] loss: 1.165\n",
      "Train Accuracy: 63.776 %\n",
      "[3100,   391] loss: 1.163\n",
      "Train Accuracy: 63.748 %\n",
      "[3110,   391] loss: 1.172\n",
      "Train Accuracy: 63.768 %\n",
      "[3120,   391] loss: 1.148\n",
      "Train Accuracy: 63.684 %\n",
      "[3130,   391] loss: 1.160\n",
      "Train Accuracy: 63.684 %\n",
      "[3140,   391] loss: 1.152\n",
      "Train Accuracy: 63.752 %\n",
      "[3150,   391] loss: 1.162\n",
      "Train Accuracy: 63.688 %\n",
      "[3160,   391] loss: 1.146\n",
      "Train Accuracy: 63.708 %\n",
      "[3170,   391] loss: 1.165\n",
      "Train Accuracy: 63.900 %\n",
      "[3180,   391] loss: 1.162\n",
      "Train Accuracy: 63.906 %\n",
      "[3190,   391] loss: 1.176\n",
      "Train Accuracy: 63.880 %\n",
      "[3200,   391] loss: 1.155\n",
      "Train Accuracy: 63.626 %\n",
      "[3210,   391] loss: 1.162\n",
      "Train Accuracy: 63.736 %\n",
      "[3220,   391] loss: 1.167\n",
      "Train Accuracy: 63.890 %\n",
      "[3230,   391] loss: 1.169\n",
      "Train Accuracy: 63.828 %\n",
      "[3240,   391] loss: 1.165\n",
      "Train Accuracy: 63.770 %\n",
      "[3250,   391] loss: 1.170\n",
      "Train Accuracy: 63.422 %\n",
      "[3260,   391] loss: 1.161\n",
      "Train Accuracy: 63.632 %\n",
      "[3270,   391] loss: 1.163\n",
      "Train Accuracy: 63.956 %\n",
      "[3280,   391] loss: 1.156\n",
      "Train Accuracy: 63.724 %\n",
      "[3290,   391] loss: 1.160\n",
      "Train Accuracy: 63.858 %\n",
      "[3300,   391] loss: 1.150\n",
      "Train Accuracy: 63.818 %\n",
      "[3310,   391] loss: 1.157\n",
      "Train Accuracy: 63.902 %\n",
      "[3320,   391] loss: 1.184\n",
      "Train Accuracy: 63.956 %\n",
      "[3330,   391] loss: 1.188\n",
      "Train Accuracy: 63.482 %\n",
      "[3340,   391] loss: 1.157\n",
      "Train Accuracy: 64.018 %\n",
      "[3350,   391] loss: 1.164\n",
      "Train Accuracy: 63.674 %\n",
      "[3360,   391] loss: 1.166\n",
      "Train Accuracy: 63.732 %\n",
      "[3370,   391] loss: 1.160\n",
      "Train Accuracy: 63.784 %\n",
      "[3380,   391] loss: 1.182\n",
      "Train Accuracy: 63.742 %\n",
      "[3390,   391] loss: 1.159\n",
      "Train Accuracy: 63.620 %\n",
      "[3400,   391] loss: 1.176\n",
      "Train Accuracy: 63.870 %\n",
      "[3410,   391] loss: 1.160\n",
      "Train Accuracy: 63.552 %\n",
      "[3420,   391] loss: 1.185\n",
      "Train Accuracy: 63.908 %\n",
      "[3430,   391] loss: 1.169\n",
      "Train Accuracy: 63.888 %\n",
      "[3440,   391] loss: 1.161\n",
      "Train Accuracy: 63.816 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3450,   391] loss: 1.165\n",
      "Train Accuracy: 63.974 %\n",
      "[3460,   391] loss: 1.144\n",
      "Train Accuracy: 63.990 %\n",
      "[3470,   391] loss: 1.158\n",
      "Train Accuracy: 63.800 %\n",
      "[3480,   391] loss: 1.168\n",
      "Train Accuracy: 63.940 %\n",
      "[3490,   391] loss: 1.162\n",
      "Train Accuracy: 63.694 %\n",
      "[3500,   391] loss: 1.170\n",
      "Train Accuracy: 63.940 %\n",
      "[3510,   391] loss: 1.150\n",
      "Train Accuracy: 63.956 %\n",
      "[3520,   391] loss: 1.158\n",
      "Train Accuracy: 63.810 %\n",
      "[3530,   391] loss: 1.151\n",
      "Train Accuracy: 63.894 %\n",
      "[3540,   391] loss: 1.154\n",
      "Train Accuracy: 63.740 %\n",
      "[3550,   391] loss: 1.170\n",
      "Train Accuracy: 64.038 %\n",
      "[3560,   391] loss: 1.151\n",
      "Train Accuracy: 63.848 %\n",
      "[3570,   391] loss: 1.158\n",
      "Train Accuracy: 63.794 %\n",
      "[3580,   391] loss: 1.162\n",
      "Train Accuracy: 63.998 %\n",
      "[3590,   391] loss: 1.163\n",
      "Train Accuracy: 63.896 %\n",
      "[3600,   391] loss: 1.161\n",
      "Train Accuracy: 63.802 %\n",
      "[3610,   391] loss: 1.158\n",
      "Train Accuracy: 63.858 %\n",
      "[3620,   391] loss: 1.155\n",
      "Train Accuracy: 63.816 %\n",
      "[3630,   391] loss: 1.155\n",
      "Train Accuracy: 63.988 %\n",
      "[3640,   391] loss: 1.168\n",
      "Train Accuracy: 63.672 %\n",
      "[3650,   391] loss: 1.146\n",
      "Train Accuracy: 64.080 %\n",
      "[3660,   391] loss: 1.176\n",
      "Train Accuracy: 63.638 %\n",
      "[3670,   391] loss: 1.162\n",
      "Train Accuracy: 63.776 %\n",
      "[3680,   391] loss: 1.164\n",
      "Train Accuracy: 63.912 %\n",
      "[3690,   391] loss: 1.149\n",
      "Train Accuracy: 63.972 %\n",
      "[3700,   391] loss: 1.154\n",
      "Train Accuracy: 63.940 %\n",
      "[3710,   391] loss: 1.162\n",
      "Train Accuracy: 63.988 %\n",
      "[3720,   391] loss: 1.160\n",
      "Train Accuracy: 63.988 %\n",
      "[3730,   391] loss: 1.161\n",
      "Train Accuracy: 64.080 %\n",
      "[3740,   391] loss: 1.153\n",
      "Train Accuracy: 63.852 %\n",
      "[3750,   391] loss: 1.136\n",
      "Train Accuracy: 64.050 %\n",
      "[3760,   391] loss: 1.160\n",
      "Train Accuracy: 63.988 %\n",
      "[3770,   391] loss: 1.172\n",
      "Train Accuracy: 63.922 %\n",
      "[3780,   391] loss: 1.160\n",
      "Train Accuracy: 63.880 %\n",
      "[3790,   391] loss: 1.170\n",
      "Train Accuracy: 63.886 %\n",
      "[3800,   391] loss: 1.161\n",
      "Train Accuracy: 64.058 %\n",
      "[3810,   391] loss: 1.156\n",
      "Train Accuracy: 63.940 %\n",
      "[3820,   391] loss: 1.153\n",
      "Train Accuracy: 63.720 %\n",
      "[3830,   391] loss: 1.169\n",
      "Train Accuracy: 64.108 %\n",
      "[3840,   391] loss: 1.179\n",
      "Train Accuracy: 63.790 %\n",
      "[3850,   391] loss: 1.159\n",
      "Train Accuracy: 64.026 %\n",
      "[3860,   391] loss: 1.157\n",
      "Train Accuracy: 63.884 %\n",
      "[3870,   391] loss: 1.144\n",
      "Train Accuracy: 63.888 %\n",
      "[3880,   391] loss: 1.140\n",
      "Train Accuracy: 63.982 %\n",
      "[3890,   391] loss: 1.157\n",
      "Train Accuracy: 64.046 %\n",
      "[3900,   391] loss: 1.155\n",
      "Train Accuracy: 64.000 %\n",
      "[3910,   391] loss: 1.157\n",
      "Train Accuracy: 64.080 %\n",
      "[3920,   391] loss: 1.141\n",
      "Train Accuracy: 63.934 %\n",
      "[3930,   391] loss: 1.153\n",
      "Train Accuracy: 63.998 %\n",
      "[3940,   391] loss: 1.156\n",
      "Train Accuracy: 63.926 %\n",
      "[3950,   391] loss: 1.157\n",
      "Train Accuracy: 64.092 %\n",
      "[3960,   391] loss: 1.157\n",
      "Train Accuracy: 64.092 %\n",
      "[3970,   391] loss: 1.165\n",
      "Train Accuracy: 63.792 %\n",
      "[3980,   391] loss: 1.155\n",
      "Train Accuracy: 63.922 %\n",
      "[3990,   391] loss: 1.150\n",
      "Train Accuracy: 64.156 %\n",
      "[4000,   391] loss: 1.165\n",
      "Train Accuracy: 63.824 %\n",
      "[4010,   391] loss: 1.153\n",
      "Train Accuracy: 64.102 %\n",
      "[4020,   391] loss: 1.158\n",
      "Train Accuracy: 64.164 %\n",
      "[4030,   391] loss: 1.156\n",
      "Train Accuracy: 63.996 %\n",
      "[4040,   391] loss: 1.169\n",
      "Train Accuracy: 63.876 %\n",
      "[4050,   391] loss: 1.172\n",
      "Train Accuracy: 63.924 %\n",
      "[4060,   391] loss: 1.160\n",
      "Train Accuracy: 63.752 %\n",
      "[4070,   391] loss: 1.163\n",
      "Train Accuracy: 63.772 %\n",
      "[4080,   391] loss: 1.155\n",
      "Train Accuracy: 64.006 %\n",
      "[4090,   391] loss: 1.154\n",
      "Train Accuracy: 64.124 %\n",
      "[4100,   391] loss: 1.157\n",
      "Train Accuracy: 64.034 %\n",
      "[4110,   391] loss: 1.140\n",
      "Train Accuracy: 64.106 %\n",
      "[4120,   391] loss: 1.167\n",
      "Train Accuracy: 63.956 %\n",
      "[4130,   391] loss: 1.156\n",
      "Train Accuracy: 63.974 %\n",
      "[4140,   391] loss: 1.158\n",
      "Train Accuracy: 64.112 %\n",
      "[4150,   391] loss: 1.162\n",
      "Train Accuracy: 63.990 %\n",
      "[4160,   391] loss: 1.140\n",
      "Train Accuracy: 64.118 %\n",
      "[4170,   391] loss: 1.165\n",
      "Train Accuracy: 64.014 %\n",
      "[4180,   391] loss: 1.155\n",
      "Train Accuracy: 64.148 %\n",
      "[4190,   391] loss: 1.165\n",
      "Train Accuracy: 64.224 %\n",
      "[4200,   391] loss: 1.166\n",
      "Train Accuracy: 64.066 %\n",
      "[4210,   391] loss: 1.154\n",
      "Train Accuracy: 63.998 %\n",
      "[4220,   391] loss: 1.149\n",
      "Train Accuracy: 63.984 %\n",
      "[4230,   391] loss: 1.148\n",
      "Train Accuracy: 63.920 %\n",
      "[4240,   391] loss: 1.144\n",
      "Train Accuracy: 63.992 %\n",
      "[4250,   391] loss: 1.155\n",
      "Train Accuracy: 64.264 %\n",
      "[4260,   391] loss: 1.154\n",
      "Train Accuracy: 63.868 %\n",
      "[4270,   391] loss: 1.151\n",
      "Train Accuracy: 64.266 %\n",
      "[4280,   391] loss: 1.164\n",
      "Train Accuracy: 63.944 %\n",
      "[4290,   391] loss: 1.162\n",
      "Train Accuracy: 64.122 %\n",
      "[4300,   391] loss: 1.154\n",
      "Train Accuracy: 64.040 %\n",
      "[4310,   391] loss: 1.149\n",
      "Train Accuracy: 64.058 %\n",
      "[4320,   391] loss: 1.142\n",
      "Train Accuracy: 64.246 %\n",
      "[4330,   391] loss: 1.143\n",
      "Train Accuracy: 63.952 %\n",
      "[4340,   391] loss: 1.147\n",
      "Train Accuracy: 64.076 %\n",
      "[4350,   391] loss: 1.153\n",
      "Train Accuracy: 64.104 %\n",
      "[4360,   391] loss: 1.166\n",
      "Train Accuracy: 64.018 %\n",
      "[4370,   391] loss: 1.160\n",
      "Train Accuracy: 64.256 %\n",
      "[4380,   391] loss: 1.146\n",
      "Train Accuracy: 63.822 %\n",
      "[4390,   391] loss: 1.141\n",
      "Train Accuracy: 63.804 %\n",
      "[4400,   391] loss: 1.151\n",
      "Train Accuracy: 64.210 %\n",
      "[4410,   391] loss: 1.148\n",
      "Train Accuracy: 63.780 %\n",
      "[4420,   391] loss: 1.141\n",
      "Train Accuracy: 64.204 %\n",
      "[4430,   391] loss: 1.144\n",
      "Train Accuracy: 63.920 %\n",
      "[4440,   391] loss: 1.146\n",
      "Train Accuracy: 64.328 %\n",
      "[4450,   391] loss: 1.157\n",
      "Train Accuracy: 63.906 %\n",
      "[4460,   391] loss: 1.142\n",
      "Train Accuracy: 63.766 %\n",
      "[4470,   391] loss: 1.155\n",
      "Train Accuracy: 63.930 %\n",
      "[4480,   391] loss: 1.165\n",
      "Train Accuracy: 63.910 %\n",
      "[4490,   391] loss: 1.145\n",
      "Train Accuracy: 64.040 %\n",
      "[4500,   391] loss: 1.137\n",
      "Train Accuracy: 63.992 %\n",
      "[4510,   391] loss: 1.155\n",
      "Train Accuracy: 63.720 %\n",
      "[4520,   391] loss: 1.146\n",
      "Train Accuracy: 64.034 %\n",
      "[4530,   391] loss: 1.148\n",
      "Train Accuracy: 64.076 %\n",
      "[4540,   391] loss: 1.155\n",
      "Train Accuracy: 64.038 %\n",
      "[4550,   391] loss: 1.160\n",
      "Train Accuracy: 64.176 %\n",
      "[4560,   391] loss: 1.171\n",
      "Train Accuracy: 64.274 %\n",
      "[4570,   391] loss: 1.150\n",
      "Train Accuracy: 64.234 %\n",
      "[4580,   391] loss: 1.140\n",
      "Train Accuracy: 64.348 %\n",
      "[4590,   391] loss: 1.141\n",
      "Train Accuracy: 64.102 %\n",
      "[4600,   391] loss: 1.143\n",
      "Train Accuracy: 64.048 %\n",
      "[4610,   391] loss: 1.166\n",
      "Train Accuracy: 64.052 %\n",
      "[4620,   391] loss: 1.173\n",
      "Train Accuracy: 64.292 %\n",
      "[4630,   391] loss: 1.173\n",
      "Train Accuracy: 64.154 %\n",
      "[4640,   391] loss: 1.144\n",
      "Train Accuracy: 64.060 %\n",
      "[4650,   391] loss: 1.158\n",
      "Train Accuracy: 64.046 %\n",
      "[4660,   391] loss: 1.153\n",
      "Train Accuracy: 64.286 %\n",
      "[4670,   391] loss: 1.143\n",
      "Train Accuracy: 64.188 %\n",
      "[4680,   391] loss: 1.155\n",
      "Train Accuracy: 63.854 %\n",
      "[4690,   391] loss: 1.177\n",
      "Train Accuracy: 64.040 %\n",
      "[4700,   391] loss: 1.146\n",
      "Train Accuracy: 64.012 %\n",
      "[4710,   391] loss: 1.155\n",
      "Train Accuracy: 63.996 %\n",
      "[4720,   391] loss: 1.139\n",
      "Train Accuracy: 64.376 %\n",
      "[4730,   391] loss: 1.157\n",
      "Train Accuracy: 63.878 %\n",
      "[4740,   391] loss: 1.162\n",
      "Train Accuracy: 64.190 %\n",
      "[4750,   391] loss: 1.157\n",
      "Train Accuracy: 63.794 %\n",
      "[4760,   391] loss: 1.151\n",
      "Train Accuracy: 64.176 %\n",
      "[4770,   391] loss: 1.155\n",
      "Train Accuracy: 64.058 %\n",
      "[4780,   391] loss: 1.161\n",
      "Train Accuracy: 64.362 %\n",
      "[4790,   391] loss: 1.146\n",
      "Train Accuracy: 64.094 %\n",
      "[4800,   391] loss: 1.148\n",
      "Train Accuracy: 64.144 %\n",
      "[4810,   391] loss: 1.137\n",
      "Train Accuracy: 64.186 %\n",
      "[4820,   391] loss: 1.161\n",
      "Train Accuracy: 64.008 %\n",
      "[4830,   391] loss: 1.147\n",
      "Train Accuracy: 64.066 %\n",
      "[4840,   391] loss: 1.131\n",
      "Train Accuracy: 63.882 %\n",
      "[4850,   391] loss: 1.144\n",
      "Train Accuracy: 64.150 %\n",
      "[4860,   391] loss: 1.145\n",
      "Train Accuracy: 63.940 %\n",
      "[4870,   391] loss: 1.160\n",
      "Train Accuracy: 64.154 %\n",
      "[4880,   391] loss: 1.142\n",
      "Train Accuracy: 63.890 %\n",
      "[4890,   391] loss: 1.157\n",
      "Train Accuracy: 64.176 %\n",
      "[4900,   391] loss: 1.151\n",
      "Train Accuracy: 64.270 %\n",
      "[4910,   391] loss: 1.150\n",
      "Train Accuracy: 64.212 %\n",
      "[4920,   391] loss: 1.158\n",
      "Train Accuracy: 64.184 %\n",
      "[4930,   391] loss: 1.157\n",
      "Train Accuracy: 64.232 %\n",
      "[4940,   391] loss: 1.150\n",
      "Train Accuracy: 64.022 %\n",
      "[4950,   391] loss: 1.143\n",
      "Train Accuracy: 64.000 %\n",
      "[4960,   391] loss: 1.144\n",
      "Train Accuracy: 64.322 %\n",
      "[4970,   391] loss: 1.131\n",
      "Train Accuracy: 64.306 %\n",
      "[4980,   391] loss: 1.148\n",
      "Train Accuracy: 64.090 %\n",
      "[4990,   391] loss: 1.158\n",
      "Train Accuracy: 64.168 %\n",
      "[5000,   391] loss: 1.156\n",
      "Train Accuracy: 64.188 %\n",
      "[5010,   391] loss: 1.177\n",
      "Train Accuracy: 64.410 %\n",
      "[5020,   391] loss: 1.162\n",
      "Train Accuracy: 64.174 %\n",
      "[5030,   391] loss: 1.148\n",
      "Train Accuracy: 64.082 %\n",
      "[5040,   391] loss: 1.143\n",
      "Train Accuracy: 64.166 %\n",
      "[5050,   391] loss: 1.159\n",
      "Train Accuracy: 63.904 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5060,   391] loss: 1.143\n",
      "Train Accuracy: 63.950 %\n",
      "[5070,   391] loss: 1.150\n",
      "Train Accuracy: 64.134 %\n",
      "[5080,   391] loss: 1.142\n",
      "Train Accuracy: 64.156 %\n",
      "[5090,   391] loss: 1.155\n",
      "Train Accuracy: 64.120 %\n",
      "[5100,   391] loss: 1.148\n",
      "Train Accuracy: 64.302 %\n",
      "[5110,   391] loss: 1.153\n",
      "Train Accuracy: 64.354 %\n",
      "[5120,   391] loss: 1.177\n",
      "Train Accuracy: 64.056 %\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "model_path = \"models/LeNet_480_84_10_random100.pt\"\n",
    "parameter_size = [0.230, 0.184, 0.139, 0.093, 0.047]\n",
    "percent = [(1 - x / 0.530) * 100 for x in parameter_size]\n",
    "for prune_percent in percent:\n",
    "    net_p = Net()\n",
    "    net_p.load_state_dict(torch.load(model_path))\n",
    "    net_p.to(device)\n",
    "    net_p.eval()\n",
    "    print(\"--------------------------------------\")\n",
    "    print(\"Prune the model by %.2f %%\" % prune_percent)\n",
    "    net_p.weight_prune(prune_percent)\n",
    "    #summary(net, (3, 32, 32))\n",
    "    calculate_size(net_p)\n",
    "    optimizer = optim.SGD(net_p.parameters(), lr=0.001, momentum=0.9)\n",
    "    train(net_p, optimizer, 6000, early_stop_step=0)\n",
    "    test(net_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "    test(net_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}