continual_learning_mnist_permutation_NN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Continual learning on the MNIST permutation task\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num GPUs 1\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import os\n",
    "import torchvision.transforms as transforms\n",
    "from torch.autograd import Variable\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "from torchvision import datasets\n",
    "from torch.nn import functional as F\n",
    "#from torch import nn\n",
    "from torch import autograd\n",
    "\n",
    "GPU = True\n",
    "if GPU == True:\n",
    "    torch.backends.cudnn.enabled = True\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "    dtype = torch.cuda.FloatTensor\n",
    "    os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "    print(\"num GPUs\",torch.cuda.device_count())\n",
    "else:\n",
    "    dtype = torch.FloatTensor\n",
    "\n",
    "from include import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = datasets.MNIST(root = './data', train = True,\n",
    "                        transform = transforms.ToTensor(), download = True)\n",
    "\n",
    "test_data = datasets.MNIST(root = './data', train = False,\n",
    "                       transform = transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get training and test data loader for permuated MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PermutedMNIST(datasets.MNIST):\n",
    "\n",
    "    def __init__(self, root=\"~/.torch/data/mnist\", train=True, permute_idx=None):\n",
    "        super(PermutedMNIST, self).__init__(root, train, download=True)\n",
    "        assert len(permute_idx) == 28 * 28\n",
    "        self.data = torch.stack([img.float().view(-1)[permute_idx] / 255 for img in self.data])\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        img, target = self.data[index], self.targets[index]\n",
    "        return img, target\n",
    "\n",
    "    def get_sample(self, sample_size):\n",
    "        sample_idx = random.sample(range(len(self)), sample_size)\n",
    "        return [img for img in self.data[sample_idx]]\n",
    "    \n",
    "###\n",
    "batch_size = 1 # the size of input data took for one iteration\n",
    "\n",
    "def get_permute_mnist(num_tasks=5):\n",
    "    train_loader = {}\n",
    "    test_loader = {}\n",
    "    idx = list(range(28 * 28))\n",
    "    for i in range(num_tasks):\n",
    "        print(i)\n",
    "        #train_loader[i] = torch.utils.data.DataLoader(PermutedMNIST(train=True, permute_idx=idx),\n",
    "        #                                              batch_size=batch_size,\n",
    "        #                                              num_workers=4)\n",
    "        train_loader[i] = torch.utils.data.DataLoader(PermutedMNIST(train=True, permute_idx=idx),\n",
    "                                                      batch_size=batch_size)\n",
    "        \n",
    "        test_loader[i] = torch.utils.data.DataLoader(PermutedMNIST(train=False, permute_idx=idx),\n",
    "                                                     batch_size=batch_size)\n",
    "        random.shuffle(idx)\n",
    "    return train_loader, test_loader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save a few example images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAGrUlEQVR4nO3dv2sUeRzH4WRzCBaKFgE1aBQEQQsLQSuxsDCSxsoupaBt7AQrwc78A5aCWNoEFRULKwUFiwhi4c+gkCKiWAhmc+UdXPa7587u7Hs2z1Peh5181Xu54IeZGV9fXx8D8rSGfQBgY+KEUOKEUOKEUOKEUH+VhuPj4/4pd5NZWFjoOJufn6/xJPVaWVkpzicnJzvO1tbWip+dmJgoztfX18c3+u++OSGUOCGUOCGUOCGUOCGUOCGUOCHUeOmuFHtOGDx7TmgYcUIocUIocUIocUIocUIocUKo4v2cw9Rqlf/eaLfbxfmHDx86zqanp3s6E7nevXtXnB84cKCmk/SPb04IJU4IJU4IJU4IJU4IJU4IVVylnDhxovjhZ8+e9fUw/9ZtVdLNZl2XzMzMFOdPnjwpzn/9+tXP49Rmy5YtA73+nj17Os6Wl5eLnx0f3/COsK58c0IocUIocUIocUIocUIocUIocUKo4p7z06dPdZ2DPnn06FFx/vv374H97KWlpeL85cuXxfnc3FzPP3tqaqrnz/4fBw8e7DjrdY/ZjW9OCCVOCCVOCCVOCCVOCCVOCCVOCOUVgBtYWFgozufn52s6yZ9bW1srzh8+fFicd7sfdJBKZ5+YmKh07W73B3d7FOsgeQUgNIw4IZQ4IZQ4IZQ4IZQ4IZQ4IVSlVwAm745Kuu0Cq+7UhsnZN1ba56fKrAcQJ6QSJ4QSJ4QSJ4QSJ4SqtEpJXZWMjQ329qOqLl261HF2586d4mdXV1f7fZwYr1696jg7evRopWsP88/81KlTPX0uty7Y5MQJocQJocQJocQJocQJocQJoTwacwOjfEsZ9ev2/1Or1fJoTGgScUIocUIocUIocUIocUIocUKoTbnntMfszdmzZ4vze/fu1XSS0eIVgNAw4oRQ4oRQ4oRQ4oRQ4oRQ4oRQI7vnHORza48dO1acv3jxojj/+fNnx9nWrVuLn+32rODkHW7y2YbJnhMaRpwQSpwQSpwQSpwQSpwQSpwQqtL7OZNV2ZkNeh+3uLjYcXb+/PlK107eFSafbXZ2tjgv/ZkNim9OCCVOCCVOCCVOCCVOCCVOCDWyt4wN09evX4vzXbt21XSS/hrVX9ewuWUMGkacEEqcEEqcEEqcEEqcEEqcEGpk95x3797tODt37lxt5xglVW+lG+ajMa9du1acX716dWA/uxt7TmgYcUIocUIocUIocUIocUIocUKokd1zVnH9+vXi/MqVKzWd5L/27dtXnH/8+LE4P378eHH+/PnzPz5THS5fvlyc37hxo6aT9J89JzSMOCGUOCGUOCGUOCGUOCGUOCGUPecGHjx4UJyfOXOmppP039OnT4vzkydP1nSS/nr8+HFxfvr06ZpO8ufsOaFhxAmhxAmhxAmhxAmhxAmhxAmhinvOdrtdac85yOeQsrkM85m3Y2NjY69fv+44O3z4cKVr23NCw4gTQokTQokTQokTQokTQrllrGG2b99enH///r2mk4yWYa5qrFKgYcQJocQJocQJocQJocQJocQJof4a5MVv3brVcTY3NzfIHx2ttIu8fft28bMXL16s9LNXV1eL8507d1a6/rC02+3ivNUqfw8l3t7omxNCiRNCiRNCiRNCiRNCiRNCiRNCuZ+zB7Ozs8X54uJiTSehX27evFmcX7hwoedrv3//vjifnp52Pyc0iTghlDghlDghlDghlDghlDghVKU95+TkZPHiKysrvZ2Kjob9KrxRNcjf127XbrVa9pzQJOKEUOKEUOKEUOKEUOKEUG4Z64F1Bv3kFYDQMOKEUOKEUOKEUOKEUOKEUOKEUAN9BWBTDXOP2e1Vdm/fvi3ODx061M/jbBrdft/evHlT00n+4ZsTQokTQokTQokTQokTQokTQokTQhXv59y7d2/xfs7Pnz/3/UAwCN1ew7d///5azrER93NCw4gTQokTQokTQokTQokTQokTQnluLf+b5/Vu7MuXL8X57t27i3N7TmgYcUIocUIocUIocUIocUIocUKokd1z/vjxo+Ns27Ztla595MiR4nxpaak4L+0Lu+0Kq+4av337Vpzv2LGjOKf/7DmhYcQJocQJocQJocQJocQJoSqtUro9TrDb4whT3b9/vzifmZmp6ST9N8jbvtxS1hurFGgYcUIocUIocUIocUIocUIocUKogd4ytry83HE2NTVV5dKMoNnZ2Y6zxcXFGk9SL3tOaBhxQihxQihxQihxQihxQihxQqjinhMYHt+cEEqcEEqcEEqcEEqcEEqcEOpvJJuWtrxD5SYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_loader, test_loader = get_permute_mnist(3)\n",
    "\n",
    "batch_size = 1 # the size of input data took for one iteration\n",
    "\n",
    "def find_examples_imgs(train_gen,label=0,numex=8,prefix = \"\"):\n",
    "    ex = []\n",
    "    for i ,(images,labels) in enumerate(train_gen):\n",
    "        if labels[0] == label:\n",
    "            ex += [ images[0].reshape((28,28)) ]\n",
    "            if len(ex) >= numex:\n",
    "                break\n",
    "    for i,img in enumerate(ex):\n",
    "        plt.imshow(img,cmap='gray')\n",
    "        plt.axis('off')\n",
    "        plt.savefig( prefix + str(i) +\".png\" ,format='png',)\n",
    "            \n",
    "find_examples_imgs(train_loader[0], 0, prefix = \"example_imgs/zeros_orig\")\n",
    "find_examples_imgs(train_loader[0], 1, prefix = \"example_imgs/ones_orig\")\n",
    "find_examples_imgs(train_loader[1], 0, prefix = \"example_imgs/ones_shuf\")\n",
    "find_examples_imgs(train_loader[1], 1, prefix = \"example_imgs/ones_shuf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Code to evaluate the EWC variants, and training on all the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model,train_gen,ewc,num_epochs=20):\n",
    "    optimizer = torch.optim.Adam( model.parameters(), lr=lr)\n",
    "    #optimizer = torch.optim.SGD( model.parameters(), lr=lr)    \n",
    "    for epoch in range(num_epochs):\n",
    "        for i ,(images,labels) in enumerate(train_gen):\n",
    "            images = Variable(images).cuda()\n",
    "            labels = Variable(labels).cuda()\n",
    "    \n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(images)\n",
    "            \n",
    "            loss_function = nn.MSELoss()\n",
    "            labels = torch.nn.functional.one_hot(labels).to(torch.float32)\n",
    "            loss = loss_function(outputs, labels)\n",
    "            #if i==0 and epoch==0:\n",
    "            #    print(\"std:\",loss.data)\n",
    "            loss += ewc.loss(model)\n",
    "            #if i==0 and epoch==0:\n",
    "            #    print(\"ewc:\",loss.data)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "    \n",
    "            if i == 0: #and epoch == num_epochs - 1:\n",
    "                print('Epoch [%d/%d], Loss: %.4f' %(epoch+1, num_epochs, loss.data.item()))\n",
    "\n",
    "# train over multiple data loaders, used to evaluate performance for training on all sets\n",
    "def train_model_over_sets(model,train_gens,num_epochs=20,shuffle=False):\n",
    "    optimizer = torch.optim.Adam( model.parameters(), lr=lr)\n",
    "    #optimizer = torch.optim.SGD( model.parameters(), lr=lr)\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        \n",
    "        if shuffle:\n",
    "            perm = np.random.permutation(len(train_gens))\n",
    "        else:\n",
    "            perm = [i for i in range( len(train_gens) )]\n",
    "    \n",
    "        for j in perm:\n",
    "            train_gen = train_gens[j]\n",
    "            for i ,(images,labels) in enumerate(train_gen):\n",
    "                images = Variable(images).cuda()\n",
    "                labels = Variable(labels).cuda()\n",
    "    \n",
    "                optimizer.zero_grad()\n",
    "                outputs = model(images)\n",
    "                \n",
    "                loss_function = nn.MSELoss()\n",
    "                labels = torch.nn.functional.one_hot(labels).to(torch.float32)\n",
    "                loss = loss_function(outputs, labels)\n",
    "                \n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "    \n",
    "                if i == 0:\n",
    "                    print('Epoch [%d/%d], Loss: %.4f' %(epoch+1, num_epochs, loss.data.item()))\n",
    "                 \n",
    "def test_model(model,test_gen):\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images,labels in test_gen:\n",
    "        images = Variable(images).cuda()\n",
    "        labels = labels.cuda()\n",
    "  \n",
    "        output = model(images)\n",
    "        _, predicted = torch.max(output,1)\n",
    "        correct += (predicted == labels).sum()\n",
    "        total += labels.size(0)\n",
    "    #return ((100.0*correct)/(total+1))\n",
    "    return ((100.0*correct)/(total))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_simulation(net,train_loader,test_loader,ewc,num_epochs=20):\n",
    "    #net = Net(input_size, hidden_size, num_classes).cuda()\n",
    "    num_tasks = len(train_loader)\n",
    "    res = np.zeros(num_tasks)\n",
    "    for k in range(num_tasks): \n",
    "        \n",
    "        # train model\n",
    "        train_model(net,train_loader[k],ewc,num_epochs=num_epochs)\n",
    "        # compute Hessian before updating\n",
    "        if ewc.lam>0 and k < num_tasks-1:\n",
    "            #ewc.compute_data( net,train_loader[k] )\n",
    "            ewc.update( net,train_loader[k] )\n",
    "\n",
    "        inderrors = []\n",
    "        for i in range(k+1):\n",
    "            erri = test_model(net,test_loader[i])\n",
    "            inderrors += [erri]\n",
    "            res[k] += erri / (k+1)\n",
    "        print(\"test performance : \", res)\n",
    "        print(\"individual errors: \", inderrors)\n",
    "    return res\n",
    "\n",
    "def train_on_all(net,train_loader,test_loader,num_epochs=20,shuffle=False):\n",
    "    #net = Net(input_size, hidden_size, num_classes).cuda()\n",
    "    num_tasks = len(train_loader)\n",
    "    res = np.zeros(num_tasks)\n",
    "    for k in range(num_tasks): \n",
    "        train_model_over_sets(net,[train_loader[j] for j in range(k+1)],num_epochs=20,shuffle=shuffle)\n",
    "        for i in range(k+1):\n",
    "            res[k] += test_model(net,test_loader[i]) / (k+1)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Two layer neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, input_size=784, hidden_size=500, num_classes=10):\n",
    "        super(Net,self).__init__()\n",
    "        self.fc1 = nn.Linear(input_size, hidden_size)\n",
    "        self.relu = nn.ReLU()\n",
    "        self.fc2 = nn.Linear(hidden_size, num_classes)\n",
    "  \n",
    "    def forward(self,x):\n",
    "        out = self.fc1(x)\n",
    "        out = self.relu(out)\n",
    "        out = self.fc2(out)\n",
    "        return out\n",
    "\n",
    "def get_random_feature_model(input_size = 784,hidden_size = 6*784,num_classes = 10):\n",
    "    net = Net(input_size, hidden_size, num_classes).cuda()\n",
    "    for param in net.fc1.parameters():\n",
    "        param.requires_grad = False\n",
    "    return net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_tasks = 10\n",
    "#num_classes = 10 # number of output classes discrete range [0,9]\n",
    "num_epochs = 20 # number of times which the entire dataset is passed throughout the model\n",
    "batch_size = 100 # the size of input data took for one iteration\n",
    "lr = 2e-4 # size of step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n"
     ]
    }
   ],
   "source": [
    "train_loader, test_loader = get_permute_mnist(num_tasks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "respp = run_simulation( Net().cuda(),train_loader,test_loader,EWCplusplus(lam=2e-4,s=100),num_epochs=num_epochs )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EWC++  400 0.0002\n",
      "Epoch [1/20], Loss: 0.1097\n",
      "Epoch [2/20], Loss: 0.0142\n",
      "Epoch [3/20], Loss: 0.0105\n",
      "Epoch [4/20], Loss: 0.0087\n",
      "Epoch [5/20], Loss: 0.0077\n",
      "Epoch [6/20], Loss: 0.0070\n",
      "Epoch [7/20], Loss: 0.0065\n",
      "Epoch [8/20], Loss: 0.0061\n",
      "Epoch [9/20], Loss: 0.0058\n",
      "Epoch [10/20], Loss: 0.0056\n",
      "Epoch [11/20], Loss: 0.0054\n",
      "Epoch [12/20], Loss: 0.0052\n",
      "Epoch [13/20], Loss: 0.0051\n",
      "Epoch [14/20], Loss: 0.0050\n",
      "Epoch [15/20], Loss: 0.0048\n",
      "Epoch [16/20], Loss: 0.0047\n",
      "Epoch [17/20], Loss: 0.0046\n",
      "Epoch [18/20], Loss: 0.0045\n",
      "Epoch [19/20], Loss: 0.0044\n",
      "Epoch [20/20], Loss: 0.0043\n",
      "generate task data..\n",
      "task data norm and number entries: tensor(4675.7065, device='cuda:0') torch.Size([400, 397510])\n",
      "..done\n",
      "test performance :  [98.20999908  0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(98.2100, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.1001\n",
      "Epoch [2/20], Loss: 0.0173\n",
      "Epoch [3/20], Loss: 0.0134\n",
      "Epoch [4/20], Loss: 0.0114\n",
      "Epoch [5/20], Loss: 0.0102\n",
      "Epoch [6/20], Loss: 0.0094\n",
      "Epoch [7/20], Loss: 0.0088\n",
      "Epoch [8/20], Loss: 0.0083\n",
      "Epoch [9/20], Loss: 0.0079\n",
      "Epoch [10/20], Loss: 0.0076\n",
      "Epoch [11/20], Loss: 0.0074\n",
      "Epoch [12/20], Loss: 0.0074\n",
      "Epoch [13/20], Loss: 0.0070\n",
      "Epoch [14/20], Loss: 0.0069\n",
      "Epoch [15/20], Loss: 0.0065\n",
      "Epoch [16/20], Loss: 0.0064\n",
      "Epoch [17/20], Loss: 0.0062\n",
      "Epoch [18/20], Loss: 0.0061\n",
      "Epoch [19/20], Loss: 0.0059\n",
      "Epoch [20/20], Loss: 0.0059\n",
      "update data..\n",
      "task data norm and number entries: tensor(4176.4185, device='cuda:0') torch.Size([400, 397510])\n",
      "..done\n",
      "test performance :  [98.20999908 97.93499756  0.          0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(98.0200, device='cuda:0'), tensor(97.8500, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0986\n",
      "Epoch [2/20], Loss: 0.0205\n",
      "Epoch [3/20], Loss: 0.0148\n",
      "Epoch [4/20], Loss: 0.0130\n",
      "Epoch [5/20], Loss: 0.0127\n",
      "Epoch [6/20], Loss: 0.0108\n",
      "Epoch [7/20], Loss: 0.0102\n",
      "Epoch [8/20], Loss: 0.0097\n",
      "Epoch [9/20], Loss: 0.0095\n",
      "Epoch [10/20], Loss: 0.0096\n",
      "Epoch [11/20], Loss: 0.0093\n",
      "Epoch [12/20], Loss: 0.0096\n",
      "Epoch [13/20], Loss: 0.0087\n",
      "Epoch [14/20], Loss: 0.0079\n",
      "Epoch [15/20], Loss: 0.0089\n",
      "Epoch [16/20], Loss: 0.0086\n",
      "Epoch [17/20], Loss: 0.0072\n",
      "Epoch [18/20], Loss: 0.0076\n",
      "Epoch [19/20], Loss: 0.0070\n",
      "Epoch [20/20], Loss: 0.0069\n",
      "update data..\n",
      "task data norm and number entries: tensor(3990.0916, device='cuda:0') torch.Size([400, 397510])\n",
      "..done\n",
      "test performance :  [98.20999908 97.93499756 97.70999908  0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(97.7500, device='cuda:0'), tensor(97.6900, device='cuda:0'), tensor(97.6900, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0956\n",
      "Epoch [2/20], Loss: 0.0217\n",
      "Epoch [3/20], Loss: 0.0154\n",
      "Epoch [4/20], Loss: 0.0133\n",
      "Epoch [5/20], Loss: 0.0121\n",
      "Epoch [6/20], Loss: 0.0104\n",
      "Epoch [7/20], Loss: 0.0094\n",
      "Epoch [8/20], Loss: 0.0089\n",
      "Epoch [9/20], Loss: 0.0092\n",
      "Epoch [10/20], Loss: 0.0083\n",
      "Epoch [11/20], Loss: 0.0078\n",
      "Epoch [12/20], Loss: 0.0077\n",
      "Epoch [13/20], Loss: 0.0076\n",
      "Epoch [14/20], Loss: 0.0071\n",
      "Epoch [15/20], Loss: 0.0072\n",
      "Epoch [16/20], Loss: 0.0069\n",
      "Epoch [17/20], Loss: 0.0080\n",
      "Epoch [18/20], Loss: 0.0069\n",
      "Epoch [19/20], Loss: 0.0075\n",
      "Epoch [20/20], Loss: 0.0063\n",
      "update data..\n",
      "task data norm and number entries: tensor(4007.9536, device='cuda:0') torch.Size([400, 397510])\n",
      "..done\n",
      "test performance :  [98.20999908 97.93499756 97.70999908 97.46749878  0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(97.3900, device='cuda:0'), tensor(97.4900, device='cuda:0'), tensor(97.5500, device='cuda:0'), tensor(97.4400, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0959\n",
      "Epoch [2/20], Loss: 0.0214\n",
      "Epoch [3/20], Loss: 0.0154\n",
      "Epoch [4/20], Loss: 0.0138\n",
      "Epoch [5/20], Loss: 0.0120\n",
      "Epoch [6/20], Loss: 0.0104\n",
      "Epoch [7/20], Loss: 0.0099\n",
      "Epoch [8/20], Loss: 0.0104\n",
      "Epoch [9/20], Loss: 0.0094\n",
      "Epoch [10/20], Loss: 0.0093\n",
      "Epoch [11/20], Loss: 0.0093\n",
      "Epoch [12/20], Loss: 0.0081\n",
      "Epoch [13/20], Loss: 0.0079\n",
      "Epoch [14/20], Loss: 0.0077\n",
      "Epoch [15/20], Loss: 0.0072\n",
      "Epoch [16/20], Loss: 0.0073\n",
      "Epoch [17/20], Loss: 0.0069\n",
      "Epoch [18/20], Loss: 0.0067\n",
      "Epoch [19/20], Loss: 0.0067\n",
      "Epoch [20/20], Loss: 0.0073\n",
      "update data..\n",
      "task data norm and number entries: tensor(3988.9426, device='cuda:0') torch.Size([400, 397510])\n",
      "..done\n",
      "test performance :  [98.20999908 97.93499756 97.70999908 97.46749878 97.12599182  0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(97.1100, device='cuda:0'), tensor(96.9300, device='cuda:0'), tensor(97.1000, device='cuda:0'), tensor(97.2500, device='cuda:0'), tensor(97.2400, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0953\n",
      "Epoch [2/20], Loss: 0.0198\n",
      "Epoch [3/20], Loss: 0.0151\n",
      "Epoch [4/20], Loss: 0.0145\n",
      "Epoch [5/20], Loss: 0.0124\n",
      "Epoch [6/20], Loss: 0.0114\n",
      "Epoch [7/20], Loss: 0.0122\n",
      "Epoch [8/20], Loss: 0.0109\n",
      "Epoch [9/20], Loss: 0.0097\n",
      "Epoch [10/20], Loss: 0.0091\n",
      "Epoch [11/20], Loss: 0.0097\n",
      "Epoch [12/20], Loss: 0.0089\n",
      "Epoch [13/20], Loss: 0.0091\n",
      "Epoch [14/20], Loss: 0.0078\n",
      "Epoch [15/20], Loss: 0.0081\n",
      "Epoch [16/20], Loss: 0.0076\n",
      "Epoch [17/20], Loss: 0.0073\n",
      "Epoch [18/20], Loss: 0.0083\n",
      "Epoch [19/20], Loss: 0.0069\n",
      "Epoch [20/20], Loss: 0.0075\n",
      "update data..\n",
      "task data norm and number entries: tensor(3770.7595, device='cuda:0') torch.Size([400, 397510])\n",
      "..done\n",
      "test performance :  [98.20999908 97.93499756 97.70999908 97.46749878 97.12599182 96.87667084\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(96.6400, device='cuda:0'), tensor(96.6900, device='cuda:0'), tensor(96.7400, device='cuda:0'), tensor(97.0200, device='cuda:0'), tensor(97.1400, device='cuda:0'), tensor(97.0300, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0944\n",
      "Epoch [2/20], Loss: 0.0236\n",
      "Epoch [3/20], Loss: 0.0170\n",
      "Epoch [4/20], Loss: 0.0158\n",
      "Epoch [5/20], Loss: 0.0135\n",
      "Epoch [6/20], Loss: 0.0127\n",
      "Epoch [7/20], Loss: 0.0130\n",
      "Epoch [8/20], Loss: 0.0113\n",
      "Epoch [9/20], Loss: 0.0151\n",
      "Epoch [10/20], Loss: 0.0106\n",
      "Epoch [11/20], Loss: 0.0107\n",
      "Epoch [12/20], Loss: 0.0099\n",
      "Epoch [13/20], Loss: 0.0097\n",
      "Epoch [14/20], Loss: 0.0095\n",
      "Epoch [15/20], Loss: 0.0091\n",
      "Epoch [16/20], Loss: 0.0090\n",
      "Epoch [17/20], Loss: 0.0087\n",
      "Epoch [18/20], Loss: 0.0090\n",
      "Epoch [19/20], Loss: 0.0085\n",
      "Epoch [20/20], Loss: 0.0096\n",
      "update data..\n",
      "task data norm and number entries: tensor(3873.8289, device='cuda:0') torch.Size([400, 397510])\n",
      "..done\n",
      "test performance :  [98.20999908 97.93499756 97.70999908 97.46749878 97.12599182 96.87667084\n",
      " 96.60285187  0.          0.          0.        ]\n",
      "individual errors:  [tensor(96., device='cuda:0'), tensor(95.9500, device='cuda:0'), tensor(96.4900, device='cuda:0'), tensor(96.6900, device='cuda:0'), tensor(97.0100, device='cuda:0'), tensor(96.9200, device='cuda:0'), tensor(97.1600, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0982\n",
      "Epoch [2/20], Loss: 0.0226\n",
      "Epoch [3/20], Loss: 0.0168\n",
      "Epoch [4/20], Loss: 0.0145\n",
      "Epoch [5/20], Loss: 0.0130\n",
      "Epoch [6/20], Loss: 0.0120\n",
      "Epoch [7/20], Loss: 0.0120\n",
      "Epoch [8/20], Loss: 0.0115\n",
      "Epoch [9/20], Loss: 0.0108\n",
      "Epoch [10/20], Loss: 0.0103\n",
      "Epoch [11/20], Loss: 0.0105\n",
      "Epoch [12/20], Loss: 0.0097\n",
      "Epoch [13/20], Loss: 0.0097\n",
      "Epoch [14/20], Loss: 0.0092\n",
      "Epoch [15/20], Loss: 0.0089\n",
      "Epoch [16/20], Loss: 0.0086\n",
      "Epoch [17/20], Loss: 0.0084\n",
      "Epoch [18/20], Loss: 0.0086\n",
      "Epoch [19/20], Loss: 0.0081\n",
      "Epoch [20/20], Loss: 0.0084\n",
      "update data..\n",
      "task data norm and number entries: tensor(3885.8789, device='cuda:0') torch.Size([400, 397510])\n",
      "..done\n",
      "test performance :  [98.20999908 97.93499756 97.70999908 97.46749878 97.12599182 96.87667084\n",
      " 96.60285187 96.37000275  0.          0.        ]\n",
      "individual errors:  [tensor(95.1200, device='cuda:0'), tensor(95.7500, device='cuda:0'), tensor(95.9400, device='cuda:0'), tensor(96.5700, device='cuda:0'), tensor(96.8800, device='cuda:0'), tensor(96.6900, device='cuda:0'), tensor(97.0700, device='cuda:0'), tensor(96.9400, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.1050\n",
      "Epoch [2/20], Loss: 0.0228\n",
      "Epoch [3/20], Loss: 0.0169\n",
      "Epoch [4/20], Loss: 0.0162\n",
      "Epoch [5/20], Loss: 0.0139\n",
      "Epoch [6/20], Loss: 0.0126\n",
      "Epoch [7/20], Loss: 0.0153\n",
      "Epoch [8/20], Loss: 0.0114\n",
      "Epoch [9/20], Loss: 0.0111\n",
      "Epoch [10/20], Loss: 0.0101\n",
      "Epoch [11/20], Loss: 0.0099\n",
      "Epoch [12/20], Loss: 0.0094\n",
      "Epoch [13/20], Loss: 0.0092\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [14/20], Loss: 0.0088\n",
      "Epoch [15/20], Loss: 0.0087\n",
      "Epoch [16/20], Loss: 0.0084\n",
      "Epoch [17/20], Loss: 0.0083\n",
      "Epoch [18/20], Loss: 0.0085\n",
      "Epoch [19/20], Loss: 0.0087\n",
      "Epoch [20/20], Loss: 0.0076\n",
      "update data..\n",
      "task data norm and number entries: tensor(3908.0073, device='cuda:0') torch.Size([400, 397510])\n",
      "..done\n",
      "test performance :  [98.20999908 97.93499756 97.70999908 97.46749878 97.12599182 96.87667084\n",
      " 96.60285187 96.37000275 96.21777344  0.        ]\n",
      "individual errors:  [tensor(94.8300, device='cuda:0'), tensor(95.5300, device='cuda:0'), tensor(95.7200, device='cuda:0'), tensor(96.2100, device='cuda:0'), tensor(96.6800, device='cuda:0'), tensor(96.5900, device='cuda:0'), tensor(96.8500, device='cuda:0'), tensor(96.7000, device='cuda:0'), tensor(96.8500, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0971\n",
      "Epoch [2/20], Loss: 0.0234\n",
      "Epoch [3/20], Loss: 0.0164\n",
      "Epoch [4/20], Loss: 0.0155\n",
      "Epoch [5/20], Loss: 0.0138\n",
      "Epoch [6/20], Loss: 0.0129\n",
      "Epoch [7/20], Loss: 0.0115\n",
      "Epoch [8/20], Loss: 0.0109\n",
      "Epoch [9/20], Loss: 0.0102\n",
      "Epoch [10/20], Loss: 0.0096\n",
      "Epoch [11/20], Loss: 0.0091\n",
      "Epoch [12/20], Loss: 0.0123\n",
      "Epoch [13/20], Loss: 0.0085\n",
      "Epoch [14/20], Loss: 0.0083\n",
      "Epoch [15/20], Loss: 0.0088\n",
      "Epoch [16/20], Loss: 0.0084\n",
      "Epoch [17/20], Loss: 0.0077\n",
      "Epoch [18/20], Loss: 0.0102\n",
      "Epoch [19/20], Loss: 0.0083\n",
      "Epoch [20/20], Loss: 0.0079\n",
      "test performance :  [98.20999908 97.93499756 97.70999908 97.46749878 97.12599182 96.87667084\n",
      " 96.60285187 96.37000275 96.21777344 95.94000244]\n",
      "individual errors:  [tensor(93.8900, device='cuda:0'), tensor(94.9200, device='cuda:0'), tensor(95.5700, device='cuda:0'), tensor(95.9400, device='cuda:0'), tensor(96.4500, device='cuda:0'), tensor(96.5100, device='cuda:0'), tensor(96.5300, device='cuda:0'), tensor(96.5000, device='cuda:0'), tensor(96.6000, device='cuda:0'), tensor(96.4900, device='cuda:0')]\n"
     ]
    }
   ],
   "source": [
    "resppbig = run_simulation( Net().cuda(),train_loader,test_loader,EWCplusplus(lam=2e-4,s=400),num_epochs=num_epochs )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/20], Loss: 0.1018\n",
      "Epoch [2/20], Loss: 0.0139\n",
      "Epoch [3/20], Loss: 0.0107\n",
      "Epoch [4/20], Loss: 0.0092\n",
      "Epoch [5/20], Loss: 0.0082\n",
      "Epoch [6/20], Loss: 0.0075\n",
      "Epoch [7/20], Loss: 0.0069\n",
      "Epoch [8/20], Loss: 0.0065\n",
      "Epoch [9/20], Loss: 0.0062\n",
      "Epoch [10/20], Loss: 0.0059\n",
      "Epoch [11/20], Loss: 0.0057\n",
      "Epoch [12/20], Loss: 0.0055\n",
      "Epoch [13/20], Loss: 0.0053\n",
      "Epoch [14/20], Loss: 0.0052\n",
      "Epoch [15/20], Loss: 0.0050\n",
      "Epoch [16/20], Loss: 0.0049\n",
      "Epoch [17/20], Loss: 0.0047\n",
      "Epoch [18/20], Loss: 0.0046\n",
      "Epoch [19/20], Loss: 0.0045\n",
      "Epoch [20/20], Loss: 0.0043\n",
      "Epoch [1/20], Loss: 0.0042\n",
      "Epoch [1/20], Loss: 0.1093\n",
      "Epoch [2/20], Loss: 0.0130\n",
      "Epoch [2/20], Loss: 0.0131\n",
      "Epoch [3/20], Loss: 0.0068\n",
      "Epoch [3/20], Loss: 0.0106\n",
      "Epoch [4/20], Loss: 0.0069\n",
      "Epoch [4/20], Loss: 0.0094\n",
      "Epoch [5/20], Loss: 0.0069\n",
      "Epoch [5/20], Loss: 0.0088\n",
      "Epoch [6/20], Loss: 0.0072\n",
      "Epoch [6/20], Loss: 0.0082\n",
      "Epoch [7/20], Loss: 0.0072\n",
      "Epoch [7/20], Loss: 0.0078\n",
      "Epoch [8/20], Loss: 0.0069\n",
      "Epoch [8/20], Loss: 0.0073\n",
      "Epoch [9/20], Loss: 0.0068\n",
      "Epoch [9/20], Loss: 0.0071\n",
      "Epoch [10/20], Loss: 0.0063\n",
      "Epoch [10/20], Loss: 0.0068\n",
      "Epoch [11/20], Loss: 0.0058\n",
      "Epoch [11/20], Loss: 0.0065\n",
      "Epoch [12/20], Loss: 0.0056\n",
      "Epoch [12/20], Loss: 0.0064\n",
      "Epoch [13/20], Loss: 0.0054\n",
      "Epoch [13/20], Loss: 0.0062\n",
      "Epoch [14/20], Loss: 0.0052\n",
      "Epoch [14/20], Loss: 0.0061\n",
      "Epoch [15/20], Loss: 0.0052\n",
      "Epoch [15/20], Loss: 0.0060\n",
      "Epoch [16/20], Loss: 0.0052\n",
      "Epoch [16/20], Loss: 0.0059\n",
      "Epoch [17/20], Loss: 0.0052\n",
      "Epoch [17/20], Loss: 0.0059\n",
      "Epoch [18/20], Loss: 0.0052\n",
      "Epoch [18/20], Loss: 0.0058\n",
      "Epoch [19/20], Loss: 0.0050\n",
      "Epoch [19/20], Loss: 0.0058\n",
      "Epoch [20/20], Loss: 0.0049\n",
      "Epoch [20/20], Loss: 0.0058\n",
      "Epoch [1/20], Loss: 0.0049\n",
      "Epoch [1/20], Loss: 0.0050\n",
      "Epoch [1/20], Loss: 0.0959\n",
      "Epoch [2/20], Loss: 0.0115\n",
      "Epoch [2/20], Loss: 0.0068\n",
      "Epoch [2/20], Loss: 0.0149\n",
      "Epoch [3/20], Loss: 0.0054\n",
      "Epoch [3/20], Loss: 0.0056\n",
      "Epoch [3/20], Loss: 0.0112\n",
      "Epoch [4/20], Loss: 0.0053\n",
      "Epoch [4/20], Loss: 0.0054\n",
      "Epoch [4/20], Loss: 0.0098\n",
      "Epoch [5/20], Loss: 0.0055\n",
      "Epoch [5/20], Loss: 0.0053\n",
      "Epoch [5/20], Loss: 0.0089\n",
      "Epoch [6/20], Loss: 0.0050\n",
      "Epoch [6/20], Loss: 0.0053\n",
      "Epoch [6/20], Loss: 0.0081\n",
      "Epoch [7/20], Loss: 0.0046\n",
      "Epoch [7/20], Loss: 0.0052\n",
      "Epoch [7/20], Loss: 0.0075\n",
      "Epoch [8/20], Loss: 0.0045\n",
      "Epoch [8/20], Loss: 0.0052\n",
      "Epoch [8/20], Loss: 0.0071\n",
      "Epoch [9/20], Loss: 0.0042\n",
      "Epoch [9/20], Loss: 0.0052\n",
      "Epoch [9/20], Loss: 0.0068\n",
      "Epoch [10/20], Loss: 0.0042\n",
      "Epoch [10/20], Loss: 0.0052\n",
      "Epoch [10/20], Loss: 0.0064\n",
      "Epoch [11/20], Loss: 0.0039\n",
      "Epoch [11/20], Loss: 0.0051\n",
      "Epoch [11/20], Loss: 0.0061\n",
      "Epoch [12/20], Loss: 0.0039\n",
      "Epoch [12/20], Loss: 0.0050\n",
      "Epoch [12/20], Loss: 0.0059\n",
      "Epoch [13/20], Loss: 0.0038\n",
      "Epoch [13/20], Loss: 0.0050\n",
      "Epoch [13/20], Loss: 0.0058\n",
      "Epoch [14/20], Loss: 0.0039\n",
      "Epoch [14/20], Loss: 0.0049\n",
      "Epoch [14/20], Loss: 0.0056\n",
      "Epoch [15/20], Loss: 0.0038\n",
      "Epoch [15/20], Loss: 0.0049\n",
      "Epoch [15/20], Loss: 0.0055\n",
      "Epoch [16/20], Loss: 0.0039\n",
      "Epoch [16/20], Loss: 0.0049\n",
      "Epoch [16/20], Loss: 0.0054\n",
      "Epoch [17/20], Loss: 0.0042\n",
      "Epoch [17/20], Loss: 0.0048\n",
      "Epoch [17/20], Loss: 0.0053\n",
      "Epoch [18/20], Loss: 0.0049\n",
      "Epoch [18/20], Loss: 0.0047\n",
      "Epoch [18/20], Loss: 0.0052\n",
      "Epoch [19/20], Loss: 0.0049\n",
      "Epoch [19/20], Loss: 0.0046\n",
      "Epoch [19/20], Loss: 0.0051\n",
      "Epoch [20/20], Loss: 0.0046\n",
      "Epoch [20/20], Loss: 0.0046\n",
      "Epoch [20/20], Loss: 0.0051\n",
      "Epoch [1/20], Loss: 0.0041\n",
      "Epoch [1/20], Loss: 0.0045\n",
      "Epoch [1/20], Loss: 0.0051\n",
      "Epoch [1/20], Loss: 0.0973\n",
      "Epoch [2/20], Loss: 0.0113\n",
      "Epoch [2/20], Loss: 0.0050\n",
      "Epoch [2/20], Loss: 0.0056\n",
      "Epoch [2/20], Loss: 0.0173\n",
      "Epoch [3/20], Loss: 0.0060\n",
      "Epoch [3/20], Loss: 0.0047\n",
      "Epoch [3/20], Loss: 0.0051\n",
      "Epoch [3/20], Loss: 0.0123\n",
      "Epoch [4/20], Loss: 0.0057\n",
      "Epoch [4/20], Loss: 0.0045\n",
      "Epoch [4/20], Loss: 0.0050\n",
      "Epoch [4/20], Loss: 0.0106\n",
      "Epoch [5/20], Loss: 0.0053\n",
      "Epoch [5/20], Loss: 0.0044\n",
      "Epoch [5/20], Loss: 0.0048\n",
      "Epoch [5/20], Loss: 0.0097\n",
      "Epoch [6/20], Loss: 0.0048\n",
      "Epoch [6/20], Loss: 0.0044\n",
      "Epoch [6/20], Loss: 0.0048\n",
      "Epoch [6/20], Loss: 0.0091\n",
      "Epoch [7/20], Loss: 0.0047\n",
      "Epoch [7/20], Loss: 0.0042\n",
      "Epoch [7/20], Loss: 0.0047\n",
      "Epoch [7/20], Loss: 0.0086\n",
      "Epoch [8/20], Loss: 0.0042\n",
      "Epoch [8/20], Loss: 0.0041\n",
      "Epoch [8/20], Loss: 0.0047\n",
      "Epoch [8/20], Loss: 0.0081\n",
      "Epoch [9/20], Loss: 0.0041\n",
      "Epoch [9/20], Loss: 0.0041\n",
      "Epoch [9/20], Loss: 0.0046\n",
      "Epoch [9/20], Loss: 0.0077\n",
      "Epoch [10/20], Loss: 0.0040\n",
      "Epoch [10/20], Loss: 0.0041\n",
      "Epoch [10/20], Loss: 0.0046\n",
      "Epoch [10/20], Loss: 0.0075\n",
      "Epoch [11/20], Loss: 0.0040\n",
      "Epoch [11/20], Loss: 0.0039\n",
      "Epoch [11/20], Loss: 0.0045\n",
      "Epoch [11/20], Loss: 0.0072\n",
      "Epoch [12/20], Loss: 0.0036\n",
      "Epoch [12/20], Loss: 0.0039\n",
      "Epoch [12/20], Loss: 0.0045\n",
      "Epoch [12/20], Loss: 0.0070\n",
      "Epoch [13/20], Loss: 0.0035\n",
      "Epoch [13/20], Loss: 0.0038\n",
      "Epoch [13/20], Loss: 0.0044\n",
      "Epoch [13/20], Loss: 0.0068\n",
      "Epoch [14/20], Loss: 0.0034\n",
      "Epoch [14/20], Loss: 0.0038\n",
      "Epoch [14/20], Loss: 0.0044\n",
      "Epoch [14/20], Loss: 0.0066\n",
      "Epoch [15/20], Loss: 0.0034\n",
      "Epoch [15/20], Loss: 0.0038\n",
      "Epoch [15/20], Loss: 0.0044\n",
      "Epoch [15/20], Loss: 0.0065\n",
      "Epoch [16/20], Loss: 0.0033\n",
      "Epoch [16/20], Loss: 0.0038\n",
      "Epoch [16/20], Loss: 0.0043\n",
      "Epoch [16/20], Loss: 0.0064\n",
      "Epoch [17/20], Loss: 0.0032\n",
      "Epoch [17/20], Loss: 0.0038\n",
      "Epoch [17/20], Loss: 0.0042\n",
      "Epoch [17/20], Loss: 0.0063\n",
      "Epoch [18/20], Loss: 0.0031\n",
      "Epoch [18/20], Loss: 0.0038\n",
      "Epoch [18/20], Loss: 0.0042\n",
      "Epoch [18/20], Loss: 0.0062\n",
      "Epoch [19/20], Loss: 0.0032\n",
      "Epoch [19/20], Loss: 0.0036\n",
      "Epoch [19/20], Loss: 0.0042\n",
      "Epoch [19/20], Loss: 0.0061\n",
      "Epoch [20/20], Loss: 0.0030\n",
      "Epoch [20/20], Loss: 0.0036\n",
      "Epoch [20/20], Loss: 0.0042\n",
      "Epoch [20/20], Loss: 0.0060\n",
      "Epoch [1/20], Loss: 0.0029\n",
      "Epoch [1/20], Loss: 0.0037\n",
      "Epoch [1/20], Loss: 0.0046\n",
      "Epoch [1/20], Loss: 0.0065\n",
      "Epoch [1/20], Loss: 0.0989\n",
      "Epoch [2/20], Loss: 0.0136\n",
      "Epoch [2/20], Loss: 0.0060\n",
      "Epoch [2/20], Loss: 0.0056\n",
      "Epoch [2/20], Loss: 0.0065\n",
      "Epoch [2/20], Loss: 0.0224\n",
      "Epoch [3/20], Loss: 0.0049\n",
      "Epoch [3/20], Loss: 0.0041\n",
      "Epoch [3/20], Loss: 0.0048\n",
      "Epoch [3/20], Loss: 0.0059\n",
      "Epoch [3/20], Loss: 0.0149\n",
      "Epoch [4/20], Loss: 0.0045\n",
      "Epoch [4/20], Loss: 0.0041\n",
      "Epoch [4/20], Loss: 0.0045\n",
      "Epoch [4/20], Loss: 0.0057\n",
      "Epoch [4/20], Loss: 0.0124\n",
      "Epoch [5/20], Loss: 0.0039\n",
      "Epoch [5/20], Loss: 0.0038\n",
      "Epoch [5/20], Loss: 0.0044\n",
      "Epoch [5/20], Loss: 0.0056\n",
      "Epoch [5/20], Loss: 0.0109\n",
      "Epoch [6/20], Loss: 0.0036\n",
      "Epoch [6/20], Loss: 0.0037\n",
      "Epoch [6/20], Loss: 0.0044\n",
      "Epoch [6/20], Loss: 0.0056\n",
      "Epoch [6/20], Loss: 0.0099\n",
      "Epoch [7/20], Loss: 0.0036\n",
      "Epoch [7/20], Loss: 0.0035\n",
      "Epoch [7/20], Loss: 0.0043\n",
      "Epoch [7/20], Loss: 0.0055\n",
      "Epoch [7/20], Loss: 0.0093\n",
      "Epoch [8/20], Loss: 0.0035\n",
      "Epoch [8/20], Loss: 0.0035\n",
      "Epoch [8/20], Loss: 0.0042\n",
      "Epoch [8/20], Loss: 0.0055\n",
      "Epoch [8/20], Loss: 0.0089\n",
      "Epoch [9/20], Loss: 0.0032\n",
      "Epoch [9/20], Loss: 0.0034\n",
      "Epoch [9/20], Loss: 0.0042\n",
      "Epoch [9/20], Loss: 0.0055\n",
      "Epoch [9/20], Loss: 0.0085\n",
      "Epoch [10/20], Loss: 0.0033\n",
      "Epoch [10/20], Loss: 0.0034\n",
      "Epoch [10/20], Loss: 0.0041\n",
      "Epoch [10/20], Loss: 0.0055\n",
      "Epoch [10/20], Loss: 0.0081\n",
      "Epoch [11/20], Loss: 0.0031\n",
      "Epoch [11/20], Loss: 0.0033\n",
      "Epoch [11/20], Loss: 0.0041\n",
      "Epoch [11/20], Loss: 0.0055\n",
      "Epoch [11/20], Loss: 0.0077\n",
      "Epoch [12/20], Loss: 0.0030\n",
      "Epoch [12/20], Loss: 0.0033\n",
      "Epoch [12/20], Loss: 0.0040\n",
      "Epoch [12/20], Loss: 0.0054\n",
      "Epoch [12/20], Loss: 0.0074\n",
      "Epoch [13/20], Loss: 0.0029\n",
      "Epoch [13/20], Loss: 0.0033\n",
      "Epoch [13/20], Loss: 0.0040\n",
      "Epoch [13/20], Loss: 0.0054\n",
      "Epoch [13/20], Loss: 0.0072\n",
      "Epoch [14/20], Loss: 0.0027\n",
      "Epoch [14/20], Loss: 0.0033\n",
      "Epoch [14/20], Loss: 0.0041\n",
      "Epoch [14/20], Loss: 0.0053\n",
      "Epoch [14/20], Loss: 0.0069\n",
      "Epoch [15/20], Loss: 0.0026\n",
      "Epoch [15/20], Loss: 0.0032\n",
      "Epoch [15/20], Loss: 0.0040\n",
      "Epoch [15/20], Loss: 0.0052\n",
      "Epoch [15/20], Loss: 0.0067\n",
      "Epoch [16/20], Loss: 0.0026\n",
      "Epoch [16/20], Loss: 0.0033\n",
      "Epoch [16/20], Loss: 0.0040\n",
      "Epoch [16/20], Loss: 0.0052\n",
      "Epoch [16/20], Loss: 0.0066\n",
      "Epoch [17/20], Loss: 0.0026\n",
      "Epoch [17/20], Loss: 0.0032\n",
      "Epoch [17/20], Loss: 0.0040\n",
      "Epoch [17/20], Loss: 0.0051\n",
      "Epoch [17/20], Loss: 0.0064\n",
      "Epoch [18/20], Loss: 0.0026\n",
      "Epoch [18/20], Loss: 0.0032\n",
      "Epoch [18/20], Loss: 0.0040\n",
      "Epoch [18/20], Loss: 0.0050\n",
      "Epoch [18/20], Loss: 0.0063\n",
      "Epoch [19/20], Loss: 0.0025\n",
      "Epoch [19/20], Loss: 0.0031\n",
      "Epoch [19/20], Loss: 0.0040\n",
      "Epoch [19/20], Loss: 0.0050\n",
      "Epoch [19/20], Loss: 0.0062\n",
      "Epoch [20/20], Loss: 0.0025\n",
      "Epoch [20/20], Loss: 0.0031\n",
      "Epoch [20/20], Loss: 0.0039\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [20/20], Loss: 0.0049\n",
      "Epoch [20/20], Loss: 0.0062\n",
      "Epoch [1/20], Loss: 0.0025\n",
      "Epoch [1/20], Loss: 0.0034\n",
      "Epoch [1/20], Loss: 0.0043\n",
      "Epoch [1/20], Loss: 0.0055\n",
      "Epoch [1/20], Loss: 0.0070\n",
      "Epoch [1/20], Loss: 0.1001\n",
      "Epoch [2/20], Loss: 0.0103\n",
      "Epoch [2/20], Loss: 0.0053\n",
      "Epoch [2/20], Loss: 0.0051\n",
      "Epoch [2/20], Loss: 0.0054\n",
      "Epoch [2/20], Loss: 0.0063\n",
      "Epoch [2/20], Loss: 0.0262\n",
      "Epoch [3/20], Loss: 0.0040\n",
      "Epoch [3/20], Loss: 0.0038\n",
      "Epoch [3/20], Loss: 0.0046\n",
      "Epoch [3/20], Loss: 0.0049\n",
      "Epoch [3/20], Loss: 0.0059\n",
      "Epoch [3/20], Loss: 0.0164\n",
      "Epoch [4/20], Loss: 0.0035\n",
      "Epoch [4/20], Loss: 0.0034\n",
      "Epoch [4/20], Loss: 0.0045\n",
      "Epoch [4/20], Loss: 0.0050\n",
      "Epoch [4/20], Loss: 0.0058\n",
      "Epoch [4/20], Loss: 0.0140\n",
      "Epoch [5/20], Loss: 0.0034\n",
      "Epoch [5/20], Loss: 0.0033\n",
      "Epoch [5/20], Loss: 0.0043\n",
      "Epoch [5/20], Loss: 0.0049\n",
      "Epoch [5/20], Loss: 0.0057\n",
      "Epoch [5/20], Loss: 0.0125\n",
      "Epoch [6/20], Loss: 0.0032\n",
      "Epoch [6/20], Loss: 0.0032\n",
      "Epoch [6/20], Loss: 0.0042\n",
      "Epoch [6/20], Loss: 0.0048\n",
      "Epoch [6/20], Loss: 0.0056\n",
      "Epoch [6/20], Loss: 0.0115\n",
      "Epoch [7/20], Loss: 0.0029\n",
      "Epoch [7/20], Loss: 0.0031\n",
      "Epoch [7/20], Loss: 0.0042\n",
      "Epoch [7/20], Loss: 0.0048\n",
      "Epoch [7/20], Loss: 0.0055\n",
      "Epoch [7/20], Loss: 0.0107\n",
      "Epoch [8/20], Loss: 0.0027\n",
      "Epoch [8/20], Loss: 0.0030\n",
      "Epoch [8/20], Loss: 0.0041\n",
      "Epoch [8/20], Loss: 0.0048\n",
      "Epoch [8/20], Loss: 0.0055\n",
      "Epoch [8/20], Loss: 0.0100\n",
      "Epoch [9/20], Loss: 0.0026\n",
      "Epoch [9/20], Loss: 0.0030\n",
      "Epoch [9/20], Loss: 0.0041\n",
      "Epoch [9/20], Loss: 0.0048\n",
      "Epoch [9/20], Loss: 0.0055\n",
      "Epoch [9/20], Loss: 0.0095\n",
      "Epoch [10/20], Loss: 0.0024\n",
      "Epoch [10/20], Loss: 0.0030\n",
      "Epoch [10/20], Loss: 0.0041\n",
      "Epoch [10/20], Loss: 0.0047\n",
      "Epoch [10/20], Loss: 0.0054\n",
      "Epoch [10/20], Loss: 0.0091\n",
      "Epoch [11/20], Loss: 0.0023\n",
      "Epoch [11/20], Loss: 0.0030\n",
      "Epoch [11/20], Loss: 0.0041\n",
      "Epoch [11/20], Loss: 0.0047\n",
      "Epoch [11/20], Loss: 0.0054\n",
      "Epoch [11/20], Loss: 0.0088\n",
      "Epoch [12/20], Loss: 0.0022\n",
      "Epoch [12/20], Loss: 0.0029\n",
      "Epoch [12/20], Loss: 0.0040\n",
      "Epoch [12/20], Loss: 0.0046\n",
      "Epoch [12/20], Loss: 0.0054\n",
      "Epoch [12/20], Loss: 0.0085\n",
      "Epoch [13/20], Loss: 0.0021\n",
      "Epoch [13/20], Loss: 0.0029\n",
      "Epoch [13/20], Loss: 0.0040\n",
      "Epoch [13/20], Loss: 0.0046\n",
      "Epoch [13/20], Loss: 0.0053\n",
      "Epoch [13/20], Loss: 0.0082\n",
      "Epoch [14/20], Loss: 0.0021\n",
      "Epoch [14/20], Loss: 0.0028\n",
      "Epoch [14/20], Loss: 0.0039\n",
      "Epoch [14/20], Loss: 0.0046\n",
      "Epoch [14/20], Loss: 0.0053\n",
      "Epoch [14/20], Loss: 0.0080\n",
      "Epoch [15/20], Loss: 0.0021\n",
      "Epoch [15/20], Loss: 0.0027\n",
      "Epoch [15/20], Loss: 0.0039\n",
      "Epoch [15/20], Loss: 0.0045\n",
      "Epoch [15/20], Loss: 0.0053\n",
      "Epoch [15/20], Loss: 0.0078\n",
      "Epoch [16/20], Loss: 0.0021\n",
      "Epoch [16/20], Loss: 0.0027\n",
      "Epoch [16/20], Loss: 0.0038\n",
      "Epoch [16/20], Loss: 0.0045\n",
      "Epoch [16/20], Loss: 0.0052\n",
      "Epoch [16/20], Loss: 0.0075\n",
      "Epoch [17/20], Loss: 0.0021\n",
      "Epoch [17/20], Loss: 0.0027\n",
      "Epoch [17/20], Loss: 0.0038\n",
      "Epoch [17/20], Loss: 0.0044\n",
      "Epoch [17/20], Loss: 0.0051\n",
      "Epoch [17/20], Loss: 0.0074\n",
      "Epoch [18/20], Loss: 0.0028\n",
      "Epoch [18/20], Loss: 0.0027\n",
      "Epoch [18/20], Loss: 0.0037\n",
      "Epoch [18/20], Loss: 0.0044\n",
      "Epoch [18/20], Loss: 0.0051\n",
      "Epoch [18/20], Loss: 0.0072\n",
      "Epoch [19/20], Loss: 0.0025\n",
      "Epoch [19/20], Loss: 0.0027\n",
      "Epoch [19/20], Loss: 0.0037\n",
      "Epoch [19/20], Loss: 0.0043\n",
      "Epoch [19/20], Loss: 0.0051\n",
      "Epoch [19/20], Loss: 0.0070\n",
      "Epoch [20/20], Loss: 0.0021\n",
      "Epoch [20/20], Loss: 0.0027\n",
      "Epoch [20/20], Loss: 0.0036\n",
      "Epoch [20/20], Loss: 0.0043\n",
      "Epoch [20/20], Loss: 0.0050\n",
      "Epoch [20/20], Loss: 0.0069\n",
      "Epoch [1/20], Loss: 0.0019\n",
      "Epoch [1/20], Loss: 0.0027\n",
      "Epoch [1/20], Loss: 0.0039\n",
      "Epoch [1/20], Loss: 0.0046\n",
      "Epoch [1/20], Loss: 0.0057\n",
      "Epoch [1/20], Loss: 0.0080\n",
      "Epoch [1/20], Loss: 0.0936\n",
      "Epoch [2/20], Loss: 0.0085\n",
      "Epoch [2/20], Loss: 0.0038\n",
      "Epoch [2/20], Loss: 0.0048\n",
      "Epoch [2/20], Loss: 0.0061\n",
      "Epoch [2/20], Loss: 0.0052\n",
      "Epoch [2/20], Loss: 0.0073\n",
      "Epoch [2/20], Loss: 0.0282\n",
      "Epoch [3/20], Loss: 0.0028\n",
      "Epoch [3/20], Loss: 0.0028\n",
      "Epoch [3/20], Loss: 0.0039\n",
      "Epoch [3/20], Loss: 0.0045\n",
      "Epoch [3/20], Loss: 0.0048\n",
      "Epoch [3/20], Loss: 0.0065\n",
      "Epoch [3/20], Loss: 0.0185\n",
      "Epoch [4/20], Loss: 0.0026\n",
      "Epoch [4/20], Loss: 0.0027\n",
      "Epoch [4/20], Loss: 0.0038\n",
      "Epoch [4/20], Loss: 0.0044\n",
      "Epoch [4/20], Loss: 0.0048\n",
      "Epoch [4/20], Loss: 0.0063\n",
      "Epoch [4/20], Loss: 0.0164\n",
      "Epoch [5/20], Loss: 0.0024\n",
      "Epoch [5/20], Loss: 0.0025\n",
      "Epoch [5/20], Loss: 0.0038\n",
      "Epoch [5/20], Loss: 0.0042\n",
      "Epoch [5/20], Loss: 0.0048\n",
      "Epoch [5/20], Loss: 0.0062\n",
      "Epoch [5/20], Loss: 0.0148\n",
      "Epoch [6/20], Loss: 0.0025\n",
      "Epoch [6/20], Loss: 0.0026\n",
      "Epoch [6/20], Loss: 0.0038\n",
      "Epoch [6/20], Loss: 0.0042\n",
      "Epoch [6/20], Loss: 0.0047\n",
      "Epoch [6/20], Loss: 0.0060\n",
      "Epoch [6/20], Loss: 0.0138\n",
      "Epoch [7/20], Loss: 0.0026\n",
      "Epoch [7/20], Loss: 0.0025\n",
      "Epoch [7/20], Loss: 0.0038\n",
      "Epoch [7/20], Loss: 0.0041\n",
      "Epoch [7/20], Loss: 0.0047\n",
      "Epoch [7/20], Loss: 0.0060\n",
      "Epoch [7/20], Loss: 0.0130\n",
      "Epoch [8/20], Loss: 0.0026\n",
      "Epoch [8/20], Loss: 0.0024\n",
      "Epoch [8/20], Loss: 0.0038\n",
      "Epoch [8/20], Loss: 0.0041\n",
      "Epoch [8/20], Loss: 0.0046\n",
      "Epoch [8/20], Loss: 0.0059\n",
      "Epoch [8/20], Loss: 0.0125\n",
      "Epoch [9/20], Loss: 0.0023\n",
      "Epoch [9/20], Loss: 0.0024\n",
      "Epoch [9/20], Loss: 0.0037\n",
      "Epoch [9/20], Loss: 0.0041\n",
      "Epoch [9/20], Loss: 0.0045\n",
      "Epoch [9/20], Loss: 0.0058\n",
      "Epoch [9/20], Loss: 0.0120\n",
      "Epoch [10/20], Loss: 0.0023\n",
      "Epoch [10/20], Loss: 0.0024\n",
      "Epoch [10/20], Loss: 0.0036\n",
      "Epoch [10/20], Loss: 0.0040\n",
      "Epoch [10/20], Loss: 0.0045\n",
      "Epoch [10/20], Loss: 0.0058\n",
      "Epoch [10/20], Loss: 0.0113\n",
      "Epoch [11/20], Loss: 0.0024\n",
      "Epoch [11/20], Loss: 0.0024\n",
      "Epoch [11/20], Loss: 0.0036\n",
      "Epoch [11/20], Loss: 0.0040\n",
      "Epoch [11/20], Loss: 0.0045\n",
      "Epoch [11/20], Loss: 0.0057\n",
      "Epoch [11/20], Loss: 0.0107\n",
      "Epoch [12/20], Loss: 0.0023\n",
      "Epoch [12/20], Loss: 0.0024\n",
      "Epoch [12/20], Loss: 0.0035\n",
      "Epoch [12/20], Loss: 0.0040\n",
      "Epoch [12/20], Loss: 0.0044\n",
      "Epoch [12/20], Loss: 0.0056\n",
      "Epoch [12/20], Loss: 0.0102\n",
      "Epoch [13/20], Loss: 0.0021\n",
      "Epoch [13/20], Loss: 0.0024\n",
      "Epoch [13/20], Loss: 0.0035\n",
      "Epoch [13/20], Loss: 0.0040\n",
      "Epoch [13/20], Loss: 0.0043\n",
      "Epoch [13/20], Loss: 0.0056\n",
      "Epoch [13/20], Loss: 0.0098\n",
      "Epoch [14/20], Loss: 0.0019\n",
      "Epoch [14/20], Loss: 0.0024\n",
      "Epoch [14/20], Loss: 0.0035\n",
      "Epoch [14/20], Loss: 0.0039\n",
      "Epoch [14/20], Loss: 0.0043\n",
      "Epoch [14/20], Loss: 0.0055\n",
      "Epoch [14/20], Loss: 0.0096\n",
      "Epoch [15/20], Loss: 0.0018\n",
      "Epoch [15/20], Loss: 0.0024\n",
      "Epoch [15/20], Loss: 0.0034\n",
      "Epoch [15/20], Loss: 0.0039\n",
      "Epoch [15/20], Loss: 0.0042\n",
      "Epoch [15/20], Loss: 0.0054\n",
      "Epoch [15/20], Loss: 0.0092\n",
      "Epoch [16/20], Loss: 0.0019\n",
      "Epoch [16/20], Loss: 0.0024\n",
      "Epoch [16/20], Loss: 0.0034\n",
      "Epoch [16/20], Loss: 0.0039\n",
      "Epoch [16/20], Loss: 0.0042\n",
      "Epoch [16/20], Loss: 0.0054\n",
      "Epoch [16/20], Loss: 0.0088\n",
      "Epoch [17/20], Loss: 0.0019\n",
      "Epoch [17/20], Loss: 0.0024\n",
      "Epoch [17/20], Loss: 0.0033\n",
      "Epoch [17/20], Loss: 0.0038\n",
      "Epoch [17/20], Loss: 0.0042\n",
      "Epoch [17/20], Loss: 0.0053\n",
      "Epoch [17/20], Loss: 0.0084\n",
      "Epoch [18/20], Loss: 0.0020\n",
      "Epoch [18/20], Loss: 0.0024\n",
      "Epoch [18/20], Loss: 0.0034\n",
      "Epoch [18/20], Loss: 0.0038\n",
      "Epoch [18/20], Loss: 0.0041\n",
      "Epoch [18/20], Loss: 0.0053\n",
      "Epoch [18/20], Loss: 0.0082\n",
      "Epoch [19/20], Loss: 0.0022\n",
      "Epoch [19/20], Loss: 0.0024\n",
      "Epoch [19/20], Loss: 0.0033\n",
      "Epoch [19/20], Loss: 0.0038\n",
      "Epoch [19/20], Loss: 0.0041\n",
      "Epoch [19/20], Loss: 0.0052\n",
      "Epoch [19/20], Loss: 0.0080\n",
      "Epoch [20/20], Loss: 0.0020\n",
      "Epoch [20/20], Loss: 0.0024\n",
      "Epoch [20/20], Loss: 0.0033\n",
      "Epoch [20/20], Loss: 0.0038\n",
      "Epoch [20/20], Loss: 0.0041\n",
      "Epoch [20/20], Loss: 0.0052\n",
      "Epoch [20/20], Loss: 0.0078\n",
      "Epoch [1/20], Loss: 0.0019\n",
      "Epoch [1/20], Loss: 0.0024\n",
      "Epoch [1/20], Loss: 0.0038\n",
      "Epoch [1/20], Loss: 0.0041\n",
      "Epoch [1/20], Loss: 0.0045\n",
      "Epoch [1/20], Loss: 0.0073\n",
      "Epoch [1/20], Loss: 0.0097\n",
      "Epoch [1/20], Loss: 0.0983\n",
      "Epoch [2/20], Loss: 0.0106\n",
      "Epoch [2/20], Loss: 0.0044\n",
      "Epoch [2/20], Loss: 0.0049\n",
      "Epoch [2/20], Loss: 0.0057\n",
      "Epoch [2/20], Loss: 0.0042\n",
      "Epoch [2/20], Loss: 0.0058\n",
      "Epoch [2/20], Loss: 0.0074\n",
      "Epoch [2/20], Loss: 0.0296\n",
      "Epoch [3/20], Loss: 0.0030\n",
      "Epoch [3/20], Loss: 0.0028\n",
      "Epoch [3/20], Loss: 0.0038\n",
      "Epoch [3/20], Loss: 0.0042\n",
      "Epoch [3/20], Loss: 0.0042\n",
      "Epoch [3/20], Loss: 0.0053\n",
      "Epoch [3/20], Loss: 0.0073\n",
      "Epoch [3/20], Loss: 0.0189\n",
      "Epoch [4/20], Loss: 0.0021\n",
      "Epoch [4/20], Loss: 0.0030\n",
      "Epoch [4/20], Loss: 0.0037\n",
      "Epoch [4/20], Loss: 0.0041\n",
      "Epoch [4/20], Loss: 0.0041\n",
      "Epoch [4/20], Loss: 0.0053\n",
      "Epoch [4/20], Loss: 0.0071\n",
      "Epoch [4/20], Loss: 0.0154\n",
      "Epoch [5/20], Loss: 0.0019\n",
      "Epoch [5/20], Loss: 0.0031\n",
      "Epoch [5/20], Loss: 0.0037\n",
      "Epoch [5/20], Loss: 0.0041\n",
      "Epoch [5/20], Loss: 0.0040\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [5/20], Loss: 0.0053\n",
      "Epoch [5/20], Loss: 0.0070\n",
      "Epoch [5/20], Loss: 0.0138\n",
      "Epoch [6/20], Loss: 0.0018\n",
      "Epoch [6/20], Loss: 0.0031\n",
      "Epoch [6/20], Loss: 0.0036\n",
      "Epoch [6/20], Loss: 0.0040\n",
      "Epoch [6/20], Loss: 0.0039\n",
      "Epoch [6/20], Loss: 0.0052\n",
      "Epoch [6/20], Loss: 0.0069\n",
      "Epoch [6/20], Loss: 0.0128\n",
      "Epoch [7/20], Loss: 0.0019\n",
      "Epoch [7/20], Loss: 0.0031\n",
      "Epoch [7/20], Loss: 0.0035\n",
      "Epoch [7/20], Loss: 0.0040\n",
      "Epoch [7/20], Loss: 0.0039\n",
      "Epoch [7/20], Loss: 0.0052\n",
      "Epoch [7/20], Loss: 0.0069\n",
      "Epoch [7/20], Loss: 0.0121\n",
      "Epoch [8/20], Loss: 0.0019\n",
      "Epoch [8/20], Loss: 0.0030\n",
      "Epoch [8/20], Loss: 0.0035\n",
      "Epoch [8/20], Loss: 0.0039\n",
      "Epoch [8/20], Loss: 0.0039\n",
      "Epoch [8/20], Loss: 0.0051\n",
      "Epoch [8/20], Loss: 0.0069\n",
      "Epoch [8/20], Loss: 0.0116\n",
      "Epoch [9/20], Loss: 0.0019\n",
      "Epoch [9/20], Loss: 0.0031\n",
      "Epoch [9/20], Loss: 0.0034\n",
      "Epoch [9/20], Loss: 0.0039\n",
      "Epoch [9/20], Loss: 0.0038\n",
      "Epoch [9/20], Loss: 0.0051\n",
      "Epoch [9/20], Loss: 0.0068\n",
      "Epoch [9/20], Loss: 0.0111\n",
      "Epoch [10/20], Loss: 0.0017\n",
      "Epoch [10/20], Loss: 0.0031\n",
      "Epoch [10/20], Loss: 0.0033\n",
      "Epoch [10/20], Loss: 0.0038\n",
      "Epoch [10/20], Loss: 0.0038\n",
      "Epoch [10/20], Loss: 0.0050\n",
      "Epoch [10/20], Loss: 0.0068\n",
      "Epoch [10/20], Loss: 0.0109\n",
      "Epoch [11/20], Loss: 0.0017\n",
      "Epoch [11/20], Loss: 0.0030\n",
      "Epoch [11/20], Loss: 0.0033\n",
      "Epoch [11/20], Loss: 0.0038\n",
      "Epoch [11/20], Loss: 0.0038\n",
      "Epoch [11/20], Loss: 0.0050\n",
      "Epoch [11/20], Loss: 0.0067\n",
      "Epoch [11/20], Loss: 0.0106\n",
      "Epoch [12/20], Loss: 0.0017\n",
      "Epoch [12/20], Loss: 0.0029\n",
      "Epoch [12/20], Loss: 0.0033\n",
      "Epoch [12/20], Loss: 0.0037\n",
      "Epoch [12/20], Loss: 0.0037\n",
      "Epoch [12/20], Loss: 0.0050\n",
      "Epoch [12/20], Loss: 0.0067\n",
      "Epoch [12/20], Loss: 0.0103\n",
      "Epoch [13/20], Loss: 0.0016\n",
      "Epoch [13/20], Loss: 0.0028\n",
      "Epoch [13/20], Loss: 0.0032\n",
      "Epoch [13/20], Loss: 0.0037\n",
      "Epoch [13/20], Loss: 0.0037\n",
      "Epoch [13/20], Loss: 0.0049\n",
      "Epoch [13/20], Loss: 0.0067\n",
      "Epoch [13/20], Loss: 0.0100\n",
      "Epoch [14/20], Loss: 0.0017\n",
      "Epoch [14/20], Loss: 0.0028\n",
      "Epoch [14/20], Loss: 0.0032\n",
      "Epoch [14/20], Loss: 0.0037\n",
      "Epoch [14/20], Loss: 0.0037\n",
      "Epoch [14/20], Loss: 0.0050\n",
      "Epoch [14/20], Loss: 0.0066\n",
      "Epoch [14/20], Loss: 0.0096\n",
      "Epoch [15/20], Loss: 0.0016\n",
      "Epoch [15/20], Loss: 0.0027\n",
      "Epoch [15/20], Loss: 0.0032\n",
      "Epoch [15/20], Loss: 0.0037\n",
      "Epoch [15/20], Loss: 0.0037\n",
      "Epoch [15/20], Loss: 0.0049\n",
      "Epoch [15/20], Loss: 0.0066\n",
      "Epoch [15/20], Loss: 0.0092\n",
      "Epoch [16/20], Loss: 0.0016\n",
      "Epoch [16/20], Loss: 0.0027\n",
      "Epoch [16/20], Loss: 0.0032\n",
      "Epoch [16/20], Loss: 0.0036\n",
      "Epoch [16/20], Loss: 0.0037\n",
      "Epoch [16/20], Loss: 0.0049\n",
      "Epoch [16/20], Loss: 0.0066\n",
      "Epoch [16/20], Loss: 0.0088\n",
      "Epoch [17/20], Loss: 0.0015\n",
      "Epoch [17/20], Loss: 0.0028\n",
      "Epoch [17/20], Loss: 0.0031\n",
      "Epoch [17/20], Loss: 0.0036\n",
      "Epoch [17/20], Loss: 0.0037\n",
      "Epoch [17/20], Loss: 0.0049\n",
      "Epoch [17/20], Loss: 0.0065\n",
      "Epoch [17/20], Loss: 0.0085\n",
      "Epoch [18/20], Loss: 0.0015\n",
      "Epoch [18/20], Loss: 0.0027\n",
      "Epoch [18/20], Loss: 0.0031\n",
      "Epoch [18/20], Loss: 0.0036\n",
      "Epoch [18/20], Loss: 0.0036\n",
      "Epoch [18/20], Loss: 0.0049\n",
      "Epoch [18/20], Loss: 0.0065\n",
      "Epoch [18/20], Loss: 0.0083\n",
      "Epoch [19/20], Loss: 0.0016\n",
      "Epoch [19/20], Loss: 0.0026\n",
      "Epoch [19/20], Loss: 0.0032\n",
      "Epoch [19/20], Loss: 0.0036\n",
      "Epoch [19/20], Loss: 0.0036\n",
      "Epoch [19/20], Loss: 0.0048\n",
      "Epoch [19/20], Loss: 0.0064\n",
      "Epoch [19/20], Loss: 0.0080\n",
      "Epoch [20/20], Loss: 0.0015\n",
      "Epoch [20/20], Loss: 0.0027\n",
      "Epoch [20/20], Loss: 0.0032\n",
      "Epoch [20/20], Loss: 0.0036\n",
      "Epoch [20/20], Loss: 0.0036\n",
      "Epoch [20/20], Loss: 0.0048\n",
      "Epoch [20/20], Loss: 0.0064\n",
      "Epoch [20/20], Loss: 0.0078\n",
      "Epoch [1/20], Loss: 0.0015\n",
      "Epoch [1/20], Loss: 0.0026\n",
      "Epoch [1/20], Loss: 0.0033\n",
      "Epoch [1/20], Loss: 0.0038\n",
      "Epoch [1/20], Loss: 0.0040\n",
      "Epoch [1/20], Loss: 0.0067\n",
      "Epoch [1/20], Loss: 0.0077\n",
      "Epoch [1/20], Loss: 0.0095\n",
      "Epoch [1/20], Loss: 0.1015\n",
      "Epoch [2/20], Loss: 0.0164\n",
      "Epoch [2/20], Loss: 0.0055\n",
      "Epoch [2/20], Loss: 0.0047\n",
      "Epoch [2/20], Loss: 0.0040\n",
      "Epoch [2/20], Loss: 0.0041\n",
      "Epoch [2/20], Loss: 0.0052\n",
      "Epoch [2/20], Loss: 0.0067\n",
      "Epoch [2/20], Loss: 0.0073\n",
      "Epoch [2/20], Loss: 0.0271\n",
      "Epoch [3/20], Loss: 0.0029\n",
      "Epoch [3/20], Loss: 0.0037\n",
      "Epoch [3/20], Loss: 0.0041\n",
      "Epoch [3/20], Loss: 0.0036\n",
      "Epoch [3/20], Loss: 0.0036\n",
      "Epoch [3/20], Loss: 0.0050\n",
      "Epoch [3/20], Loss: 0.0064\n",
      "Epoch [3/20], Loss: 0.0070\n",
      "Epoch [3/20], Loss: 0.0182\n",
      "Epoch [4/20], Loss: 0.0024\n",
      "Epoch [4/20], Loss: 0.0033\n",
      "Epoch [4/20], Loss: 0.0038\n",
      "Epoch [4/20], Loss: 0.0035\n",
      "Epoch [4/20], Loss: 0.0036\n",
      "Epoch [4/20], Loss: 0.0049\n",
      "Epoch [4/20], Loss: 0.0062\n",
      "Epoch [4/20], Loss: 0.0069\n",
      "Epoch [4/20], Loss: 0.0158\n",
      "Epoch [5/20], Loss: 0.0021\n",
      "Epoch [5/20], Loss: 0.0031\n",
      "Epoch [5/20], Loss: 0.0037\n",
      "Epoch [5/20], Loss: 0.0036\n",
      "Epoch [5/20], Loss: 0.0036\n",
      "Epoch [5/20], Loss: 0.0049\n",
      "Epoch [5/20], Loss: 0.0062\n",
      "Epoch [5/20], Loss: 0.0068\n",
      "Epoch [5/20], Loss: 0.0146\n",
      "Epoch [6/20], Loss: 0.0019\n",
      "Epoch [6/20], Loss: 0.0030\n",
      "Epoch [6/20], Loss: 0.0036\n",
      "Epoch [6/20], Loss: 0.0036\n",
      "Epoch [6/20], Loss: 0.0036\n",
      "Epoch [6/20], Loss: 0.0049\n",
      "Epoch [6/20], Loss: 0.0061\n",
      "Epoch [6/20], Loss: 0.0067\n",
      "Epoch [6/20], Loss: 0.0136\n",
      "Epoch [7/20], Loss: 0.0018\n",
      "Epoch [7/20], Loss: 0.0027\n",
      "Epoch [7/20], Loss: 0.0035\n",
      "Epoch [7/20], Loss: 0.0036\n",
      "Epoch [7/20], Loss: 0.0036\n",
      "Epoch [7/20], Loss: 0.0048\n",
      "Epoch [7/20], Loss: 0.0061\n",
      "Epoch [7/20], Loss: 0.0065\n",
      "Epoch [7/20], Loss: 0.0129\n",
      "Epoch [8/20], Loss: 0.0017\n",
      "Epoch [8/20], Loss: 0.0025\n",
      "Epoch [8/20], Loss: 0.0035\n",
      "Epoch [8/20], Loss: 0.0036\n",
      "Epoch [8/20], Loss: 0.0036\n",
      "Epoch [8/20], Loss: 0.0048\n",
      "Epoch [8/20], Loss: 0.0060\n",
      "Epoch [8/20], Loss: 0.0064\n",
      "Epoch [8/20], Loss: 0.0125\n",
      "Epoch [9/20], Loss: 0.0017\n",
      "Epoch [9/20], Loss: 0.0024\n",
      "Epoch [9/20], Loss: 0.0034\n",
      "Epoch [9/20], Loss: 0.0036\n",
      "Epoch [9/20], Loss: 0.0036\n",
      "Epoch [9/20], Loss: 0.0048\n",
      "Epoch [9/20], Loss: 0.0060\n",
      "Epoch [9/20], Loss: 0.0063\n",
      "Epoch [9/20], Loss: 0.0121\n",
      "Epoch [10/20], Loss: 0.0016\n",
      "Epoch [10/20], Loss: 0.0024\n",
      "Epoch [10/20], Loss: 0.0034\n",
      "Epoch [10/20], Loss: 0.0035\n",
      "Epoch [10/20], Loss: 0.0036\n",
      "Epoch [10/20], Loss: 0.0048\n",
      "Epoch [10/20], Loss: 0.0060\n",
      "Epoch [10/20], Loss: 0.0062\n",
      "Epoch [10/20], Loss: 0.0116\n",
      "Epoch [11/20], Loss: 0.0016\n",
      "Epoch [11/20], Loss: 0.0024\n",
      "Epoch [11/20], Loss: 0.0034\n",
      "Epoch [11/20], Loss: 0.0035\n",
      "Epoch [11/20], Loss: 0.0036\n",
      "Epoch [11/20], Loss: 0.0048\n",
      "Epoch [11/20], Loss: 0.0059\n",
      "Epoch [11/20], Loss: 0.0061\n",
      "Epoch [11/20], Loss: 0.0111\n",
      "Epoch [12/20], Loss: 0.0015\n",
      "Epoch [12/20], Loss: 0.0023\n",
      "Epoch [12/20], Loss: 0.0034\n",
      "Epoch [12/20], Loss: 0.0035\n",
      "Epoch [12/20], Loss: 0.0036\n",
      "Epoch [12/20], Loss: 0.0048\n",
      "Epoch [12/20], Loss: 0.0058\n",
      "Epoch [12/20], Loss: 0.0059\n",
      "Epoch [12/20], Loss: 0.0106\n",
      "Epoch [13/20], Loss: 0.0015\n",
      "Epoch [13/20], Loss: 0.0023\n",
      "Epoch [13/20], Loss: 0.0034\n",
      "Epoch [13/20], Loss: 0.0035\n",
      "Epoch [13/20], Loss: 0.0036\n",
      "Epoch [13/20], Loss: 0.0047\n",
      "Epoch [13/20], Loss: 0.0057\n",
      "Epoch [13/20], Loss: 0.0059\n",
      "Epoch [13/20], Loss: 0.0102\n",
      "Epoch [14/20], Loss: 0.0014\n",
      "Epoch [14/20], Loss: 0.0024\n",
      "Epoch [14/20], Loss: 0.0034\n",
      "Epoch [14/20], Loss: 0.0035\n",
      "Epoch [14/20], Loss: 0.0036\n",
      "Epoch [14/20], Loss: 0.0047\n",
      "Epoch [14/20], Loss: 0.0056\n",
      "Epoch [14/20], Loss: 0.0058\n",
      "Epoch [14/20], Loss: 0.0098\n",
      "Epoch [15/20], Loss: 0.0014\n",
      "Epoch [15/20], Loss: 0.0022\n",
      "Epoch [15/20], Loss: 0.0034\n",
      "Epoch [15/20], Loss: 0.0034\n",
      "Epoch [15/20], Loss: 0.0036\n",
      "Epoch [15/20], Loss: 0.0047\n",
      "Epoch [15/20], Loss: 0.0056\n",
      "Epoch [15/20], Loss: 0.0057\n",
      "Epoch [15/20], Loss: 0.0094\n",
      "Epoch [16/20], Loss: 0.0014\n",
      "Epoch [16/20], Loss: 0.0021\n",
      "Epoch [16/20], Loss: 0.0034\n",
      "Epoch [16/20], Loss: 0.0034\n",
      "Epoch [16/20], Loss: 0.0036\n",
      "Epoch [16/20], Loss: 0.0047\n",
      "Epoch [16/20], Loss: 0.0055\n",
      "Epoch [16/20], Loss: 0.0057\n",
      "Epoch [16/20], Loss: 0.0091\n",
      "Epoch [17/20], Loss: 0.0014\n",
      "Epoch [17/20], Loss: 0.0022\n",
      "Epoch [17/20], Loss: 0.0034\n",
      "Epoch [17/20], Loss: 0.0034\n",
      "Epoch [17/20], Loss: 0.0036\n",
      "Epoch [17/20], Loss: 0.0046\n",
      "Epoch [17/20], Loss: 0.0055\n",
      "Epoch [17/20], Loss: 0.0056\n",
      "Epoch [17/20], Loss: 0.0088\n",
      "Epoch [18/20], Loss: 0.0014\n",
      "Epoch [18/20], Loss: 0.0021\n",
      "Epoch [18/20], Loss: 0.0034\n",
      "Epoch [18/20], Loss: 0.0035\n",
      "Epoch [18/20], Loss: 0.0035\n",
      "Epoch [18/20], Loss: 0.0047\n",
      "Epoch [18/20], Loss: 0.0054\n",
      "Epoch [18/20], Loss: 0.0055\n",
      "Epoch [18/20], Loss: 0.0086\n",
      "Epoch [19/20], Loss: 0.0014\n",
      "Epoch [19/20], Loss: 0.0021\n",
      "Epoch [19/20], Loss: 0.0034\n",
      "Epoch [19/20], Loss: 0.0035\n",
      "Epoch [19/20], Loss: 0.0035\n",
      "Epoch [19/20], Loss: 0.0046\n",
      "Epoch [19/20], Loss: 0.0054\n",
      "Epoch [19/20], Loss: 0.0055\n",
      "Epoch [19/20], Loss: 0.0083\n",
      "Epoch [20/20], Loss: 0.0014\n",
      "Epoch [20/20], Loss: 0.0021\n",
      "Epoch [20/20], Loss: 0.0034\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [20/20], Loss: 0.0034\n",
      "Epoch [20/20], Loss: 0.0035\n",
      "Epoch [20/20], Loss: 0.0046\n",
      "Epoch [20/20], Loss: 0.0054\n",
      "Epoch [20/20], Loss: 0.0055\n",
      "Epoch [20/20], Loss: 0.0082\n",
      "Epoch [1/20], Loss: 0.0014\n",
      "Epoch [1/20], Loss: 0.0019\n",
      "Epoch [1/20], Loss: 0.0035\n",
      "Epoch [1/20], Loss: 0.0035\n",
      "Epoch [1/20], Loss: 0.0037\n",
      "Epoch [1/20], Loss: 0.0065\n",
      "Epoch [1/20], Loss: 0.0063\n",
      "Epoch [1/20], Loss: 0.0068\n",
      "Epoch [1/20], Loss: 0.0091\n",
      "Epoch [1/20], Loss: 0.0968\n",
      "Epoch [2/20], Loss: 0.0076\n",
      "Epoch [2/20], Loss: 0.0062\n",
      "Epoch [2/20], Loss: 0.0054\n",
      "Epoch [2/20], Loss: 0.0051\n",
      "Epoch [2/20], Loss: 0.0042\n",
      "Epoch [2/20], Loss: 0.0051\n",
      "Epoch [2/20], Loss: 0.0068\n",
      "Epoch [2/20], Loss: 0.0058\n",
      "Epoch [2/20], Loss: 0.0079\n",
      "Epoch [2/20], Loss: 0.0406\n",
      "Epoch [3/20], Loss: 0.0024\n",
      "Epoch [3/20], Loss: 0.0029\n",
      "Epoch [3/20], Loss: 0.0037\n",
      "Epoch [3/20], Loss: 0.0044\n",
      "Epoch [3/20], Loss: 0.0038\n",
      "Epoch [3/20], Loss: 0.0050\n",
      "Epoch [3/20], Loss: 0.0059\n",
      "Epoch [3/20], Loss: 0.0059\n",
      "Epoch [3/20], Loss: 0.0079\n",
      "Epoch [3/20], Loss: 0.0213\n",
      "Epoch [4/20], Loss: 0.0017\n",
      "Epoch [4/20], Loss: 0.0025\n",
      "Epoch [4/20], Loss: 0.0035\n",
      "Epoch [4/20], Loss: 0.0043\n",
      "Epoch [4/20], Loss: 0.0038\n",
      "Epoch [4/20], Loss: 0.0049\n",
      "Epoch [4/20], Loss: 0.0056\n",
      "Epoch [4/20], Loss: 0.0059\n",
      "Epoch [4/20], Loss: 0.0078\n",
      "Epoch [4/20], Loss: 0.0172\n",
      "Epoch [5/20], Loss: 0.0019\n",
      "Epoch [5/20], Loss: 0.0025\n",
      "Epoch [5/20], Loss: 0.0034\n",
      "Epoch [5/20], Loss: 0.0041\n",
      "Epoch [5/20], Loss: 0.0036\n",
      "Epoch [5/20], Loss: 0.0049\n",
      "Epoch [5/20], Loss: 0.0056\n",
      "Epoch [5/20], Loss: 0.0058\n",
      "Epoch [5/20], Loss: 0.0077\n",
      "Epoch [5/20], Loss: 0.0149\n",
      "Epoch [6/20], Loss: 0.0016\n",
      "Epoch [6/20], Loss: 0.0024\n",
      "Epoch [6/20], Loss: 0.0034\n",
      "Epoch [6/20], Loss: 0.0039\n",
      "Epoch [6/20], Loss: 0.0036\n",
      "Epoch [6/20], Loss: 0.0049\n",
      "Epoch [6/20], Loss: 0.0055\n",
      "Epoch [6/20], Loss: 0.0057\n",
      "Epoch [6/20], Loss: 0.0077\n",
      "Epoch [6/20], Loss: 0.0136\n",
      "Epoch [7/20], Loss: 0.0015\n",
      "Epoch [7/20], Loss: 0.0022\n",
      "Epoch [7/20], Loss: 0.0034\n",
      "Epoch [7/20], Loss: 0.0039\n",
      "Epoch [7/20], Loss: 0.0036\n",
      "Epoch [7/20], Loss: 0.0049\n",
      "Epoch [7/20], Loss: 0.0055\n",
      "Epoch [7/20], Loss: 0.0056\n",
      "Epoch [7/20], Loss: 0.0076\n",
      "Epoch [7/20], Loss: 0.0126\n",
      "Epoch [8/20], Loss: 0.0015\n",
      "Epoch [8/20], Loss: 0.0021\n",
      "Epoch [8/20], Loss: 0.0034\n",
      "Epoch [8/20], Loss: 0.0039\n",
      "Epoch [8/20], Loss: 0.0036\n",
      "Epoch [8/20], Loss: 0.0049\n",
      "Epoch [8/20], Loss: 0.0054\n",
      "Epoch [8/20], Loss: 0.0055\n",
      "Epoch [8/20], Loss: 0.0075\n",
      "Epoch [8/20], Loss: 0.0119\n",
      "Epoch [9/20], Loss: 0.0015\n",
      "Epoch [9/20], Loss: 0.0021\n",
      "Epoch [9/20], Loss: 0.0034\n",
      "Epoch [9/20], Loss: 0.0038\n",
      "Epoch [9/20], Loss: 0.0035\n",
      "Epoch [9/20], Loss: 0.0049\n",
      "Epoch [9/20], Loss: 0.0054\n",
      "Epoch [9/20], Loss: 0.0054\n",
      "Epoch [9/20], Loss: 0.0074\n",
      "Epoch [9/20], Loss: 0.0113\n",
      "Epoch [10/20], Loss: 0.0014\n",
      "Epoch [10/20], Loss: 0.0020\n",
      "Epoch [10/20], Loss: 0.0034\n",
      "Epoch [10/20], Loss: 0.0038\n",
      "Epoch [10/20], Loss: 0.0035\n",
      "Epoch [10/20], Loss: 0.0049\n",
      "Epoch [10/20], Loss: 0.0054\n",
      "Epoch [10/20], Loss: 0.0053\n",
      "Epoch [10/20], Loss: 0.0073\n",
      "Epoch [10/20], Loss: 0.0110\n",
      "Epoch [11/20], Loss: 0.0014\n",
      "Epoch [11/20], Loss: 0.0020\n",
      "Epoch [11/20], Loss: 0.0034\n",
      "Epoch [11/20], Loss: 0.0037\n",
      "Epoch [11/20], Loss: 0.0035\n",
      "Epoch [11/20], Loss: 0.0049\n",
      "Epoch [11/20], Loss: 0.0053\n",
      "Epoch [11/20], Loss: 0.0052\n",
      "Epoch [11/20], Loss: 0.0072\n",
      "Epoch [11/20], Loss: 0.0107\n",
      "Epoch [12/20], Loss: 0.0014\n",
      "Epoch [12/20], Loss: 0.0019\n",
      "Epoch [12/20], Loss: 0.0034\n",
      "Epoch [12/20], Loss: 0.0037\n",
      "Epoch [12/20], Loss: 0.0034\n",
      "Epoch [12/20], Loss: 0.0049\n",
      "Epoch [12/20], Loss: 0.0053\n",
      "Epoch [12/20], Loss: 0.0052\n",
      "Epoch [12/20], Loss: 0.0072\n",
      "Epoch [12/20], Loss: 0.0104\n",
      "Epoch [13/20], Loss: 0.0014\n",
      "Epoch [13/20], Loss: 0.0020\n",
      "Epoch [13/20], Loss: 0.0033\n",
      "Epoch [13/20], Loss: 0.0037\n",
      "Epoch [13/20], Loss: 0.0035\n",
      "Epoch [13/20], Loss: 0.0048\n",
      "Epoch [13/20], Loss: 0.0053\n",
      "Epoch [13/20], Loss: 0.0052\n",
      "Epoch [13/20], Loss: 0.0072\n",
      "Epoch [13/20], Loss: 0.0101\n",
      "Epoch [14/20], Loss: 0.0014\n",
      "Epoch [14/20], Loss: 0.0020\n",
      "Epoch [14/20], Loss: 0.0034\n",
      "Epoch [14/20], Loss: 0.0037\n",
      "Epoch [14/20], Loss: 0.0034\n",
      "Epoch [14/20], Loss: 0.0048\n",
      "Epoch [14/20], Loss: 0.0053\n",
      "Epoch [14/20], Loss: 0.0051\n",
      "Epoch [14/20], Loss: 0.0070\n",
      "Epoch [14/20], Loss: 0.0099\n",
      "Epoch [15/20], Loss: 0.0014\n",
      "Epoch [15/20], Loss: 0.0019\n",
      "Epoch [15/20], Loss: 0.0033\n",
      "Epoch [15/20], Loss: 0.0037\n",
      "Epoch [15/20], Loss: 0.0034\n",
      "Epoch [15/20], Loss: 0.0048\n",
      "Epoch [15/20], Loss: 0.0053\n",
      "Epoch [15/20], Loss: 0.0051\n",
      "Epoch [15/20], Loss: 0.0069\n",
      "Epoch [15/20], Loss: 0.0097\n",
      "Epoch [16/20], Loss: 0.0014\n",
      "Epoch [16/20], Loss: 0.0020\n",
      "Epoch [16/20], Loss: 0.0033\n",
      "Epoch [16/20], Loss: 0.0036\n",
      "Epoch [16/20], Loss: 0.0034\n",
      "Epoch [16/20], Loss: 0.0048\n",
      "Epoch [16/20], Loss: 0.0053\n",
      "Epoch [16/20], Loss: 0.0051\n",
      "Epoch [16/20], Loss: 0.0068\n",
      "Epoch [16/20], Loss: 0.0095\n",
      "Epoch [17/20], Loss: 0.0013\n",
      "Epoch [17/20], Loss: 0.0019\n",
      "Epoch [17/20], Loss: 0.0033\n",
      "Epoch [17/20], Loss: 0.0036\n",
      "Epoch [17/20], Loss: 0.0034\n",
      "Epoch [17/20], Loss: 0.0047\n",
      "Epoch [17/20], Loss: 0.0052\n",
      "Epoch [17/20], Loss: 0.0051\n",
      "Epoch [17/20], Loss: 0.0067\n",
      "Epoch [17/20], Loss: 0.0094\n",
      "Epoch [18/20], Loss: 0.0014\n",
      "Epoch [18/20], Loss: 0.0019\n",
      "Epoch [18/20], Loss: 0.0033\n",
      "Epoch [18/20], Loss: 0.0036\n",
      "Epoch [18/20], Loss: 0.0034\n",
      "Epoch [18/20], Loss: 0.0047\n",
      "Epoch [18/20], Loss: 0.0052\n",
      "Epoch [18/20], Loss: 0.0050\n",
      "Epoch [18/20], Loss: 0.0066\n",
      "Epoch [18/20], Loss: 0.0092\n",
      "Epoch [19/20], Loss: 0.0014\n",
      "Epoch [19/20], Loss: 0.0019\n",
      "Epoch [19/20], Loss: 0.0033\n",
      "Epoch [19/20], Loss: 0.0036\n",
      "Epoch [19/20], Loss: 0.0033\n",
      "Epoch [19/20], Loss: 0.0047\n",
      "Epoch [19/20], Loss: 0.0052\n",
      "Epoch [19/20], Loss: 0.0050\n",
      "Epoch [19/20], Loss: 0.0065\n",
      "Epoch [19/20], Loss: 0.0091\n",
      "Epoch [20/20], Loss: 0.0014\n",
      "Epoch [20/20], Loss: 0.0019\n",
      "Epoch [20/20], Loss: 0.0033\n",
      "Epoch [20/20], Loss: 0.0036\n",
      "Epoch [20/20], Loss: 0.0034\n",
      "Epoch [20/20], Loss: 0.0047\n",
      "Epoch [20/20], Loss: 0.0052\n",
      "Epoch [20/20], Loss: 0.0050\n",
      "Epoch [20/20], Loss: 0.0065\n",
      "Epoch [20/20], Loss: 0.0089\n"
     ]
    }
   ],
   "source": [
    "# train on all data as baseline\n",
    "resall = train_on_all( Net().cuda(),train_loader,test_loader,num_epochs=num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/20], Loss: 0.1097\n",
      "Epoch [2/20], Loss: 0.0143\n",
      "Epoch [3/20], Loss: 0.0109\n",
      "Epoch [4/20], Loss: 0.0092\n",
      "Epoch [5/20], Loss: 0.0083\n",
      "Epoch [6/20], Loss: 0.0076\n",
      "Epoch [7/20], Loss: 0.0070\n",
      "Epoch [8/20], Loss: 0.0065\n",
      "Epoch [9/20], Loss: 0.0061\n",
      "Epoch [10/20], Loss: 0.0058\n",
      "Epoch [11/20], Loss: 0.0055\n",
      "Epoch [12/20], Loss: 0.0052\n",
      "Epoch [13/20], Loss: 0.0050\n",
      "Epoch [14/20], Loss: 0.0048\n",
      "Epoch [15/20], Loss: 0.0046\n",
      "Epoch [16/20], Loss: 0.0045\n",
      "Epoch [17/20], Loss: 0.0043\n",
      "Epoch [18/20], Loss: 0.0042\n",
      "Epoch [19/20], Loss: 0.0041\n",
      "Epoch [20/20], Loss: 0.0040\n",
      "Epoch [1/20], Loss: 0.1081\n",
      "Epoch [1/20], Loss: 0.0202\n",
      "Epoch [2/20], Loss: 0.0162\n",
      "Epoch [2/20], Loss: 0.0072\n",
      "Epoch [3/20], Loss: 0.0119\n",
      "Epoch [3/20], Loss: 0.0071\n",
      "Epoch [4/20], Loss: 0.0101\n",
      "Epoch [4/20], Loss: 0.0066\n",
      "Epoch [5/20], Loss: 0.0089\n",
      "Epoch [5/20], Loss: 0.0063\n",
      "Epoch [6/20], Loss: 0.0039\n",
      "Epoch [6/20], Loss: 0.0085\n",
      "Epoch [7/20], Loss: 0.0057\n",
      "Epoch [7/20], Loss: 0.0076\n",
      "Epoch [8/20], Loss: 0.0060\n",
      "Epoch [8/20], Loss: 0.0071\n",
      "Epoch [9/20], Loss: 0.0060\n",
      "Epoch [9/20], Loss: 0.0064\n",
      "Epoch [10/20], Loss: 0.0037\n",
      "Epoch [10/20], Loss: 0.0065\n",
      "Epoch [11/20], Loss: 0.0055\n",
      "Epoch [11/20], Loss: 0.0058\n",
      "Epoch [12/20], Loss: 0.0035\n",
      "Epoch [12/20], Loss: 0.0061\n",
      "Epoch [13/20], Loss: 0.0055\n",
      "Epoch [13/20], Loss: 0.0059\n",
      "Epoch [14/20], Loss: 0.0049\n",
      "Epoch [14/20], Loss: 0.0059\n",
      "Epoch [15/20], Loss: 0.0053\n",
      "Epoch [15/20], Loss: 0.0067\n",
      "Epoch [16/20], Loss: 0.0032\n",
      "Epoch [16/20], Loss: 0.0054\n",
      "Epoch [17/20], Loss: 0.0045\n",
      "Epoch [17/20], Loss: 0.0056\n",
      "Epoch [18/20], Loss: 0.0031\n",
      "Epoch [18/20], Loss: 0.0050\n",
      "Epoch [19/20], Loss: 0.0053\n",
      "Epoch [19/20], Loss: 0.0049\n",
      "Epoch [20/20], Loss: 0.0041\n",
      "Epoch [20/20], Loss: 0.0052\n",
      "Epoch [1/20], Loss: 0.0991\n",
      "Epoch [1/20], Loss: 0.0168\n",
      "Epoch [1/20], Loss: 0.0084\n",
      "Epoch [2/20], Loss: 0.0040\n",
      "Epoch [2/20], Loss: 0.0207\n",
      "Epoch [2/20], Loss: 0.0061\n",
      "Epoch [3/20], Loss: 0.0031\n",
      "Epoch [3/20], Loss: 0.0129\n",
      "Epoch [3/20], Loss: 0.0064\n",
      "Epoch [4/20], Loss: 0.0105\n",
      "Epoch [4/20], Loss: 0.0045\n",
      "Epoch [4/20], Loss: 0.0058\n",
      "Epoch [5/20], Loss: 0.0103\n",
      "Epoch [5/20], Loss: 0.0051\n",
      "Epoch [5/20], Loss: 0.0047\n",
      "Epoch [6/20], Loss: 0.0037\n",
      "Epoch [6/20], Loss: 0.0100\n",
      "Epoch [6/20], Loss: 0.0049\n",
      "Epoch [7/20], Loss: 0.0089\n",
      "Epoch [7/20], Loss: 0.0045\n",
      "Epoch [7/20], Loss: 0.0052\n",
      "Epoch [8/20], Loss: 0.0028\n",
      "Epoch [8/20], Loss: 0.0043\n",
      "Epoch [8/20], Loss: 0.0091\n",
      "Epoch [9/20], Loss: 0.0070\n",
      "Epoch [9/20], Loss: 0.0043\n",
      "Epoch [9/20], Loss: 0.0044\n",
      "Epoch [10/20], Loss: 0.0026\n",
      "Epoch [10/20], Loss: 0.0042\n",
      "Epoch [10/20], Loss: 0.0081\n",
      "Epoch [11/20], Loss: 0.0038\n",
      "Epoch [11/20], Loss: 0.0070\n",
      "Epoch [11/20], Loss: 0.0042\n",
      "Epoch [12/20], Loss: 0.0025\n",
      "Epoch [12/20], Loss: 0.0071\n",
      "Epoch [12/20], Loss: 0.0042\n",
      "Epoch [13/20], Loss: 0.0065\n",
      "Epoch [13/20], Loss: 0.0038\n",
      "Epoch [13/20], Loss: 0.0039\n",
      "Epoch [14/20], Loss: 0.0032\n",
      "Epoch [14/20], Loss: 0.0041\n",
      "Epoch [14/20], Loss: 0.0069\n",
      "Epoch [15/20], Loss: 0.0055\n",
      "Epoch [15/20], Loss: 0.0040\n",
      "Epoch [15/20], Loss: 0.0039\n",
      "Epoch [16/20], Loss: 0.0062\n",
      "Epoch [16/20], Loss: 0.0038\n",
      "Epoch [16/20], Loss: 0.0037\n",
      "Epoch [17/20], Loss: 0.0038\n",
      "Epoch [17/20], Loss: 0.0036\n",
      "Epoch [17/20], Loss: 0.0064\n",
      "Epoch [18/20], Loss: 0.0051\n",
      "Epoch [18/20], Loss: 0.0036\n",
      "Epoch [18/20], Loss: 0.0036\n",
      "Epoch [19/20], Loss: 0.0029\n",
      "Epoch [19/20], Loss: 0.0039\n",
      "Epoch [19/20], Loss: 0.0059\n",
      "Epoch [20/20], Loss: 0.0034\n",
      "Epoch [20/20], Loss: 0.0035\n",
      "Epoch [20/20], Loss: 0.0057\n",
      "Epoch [1/20], Loss: 0.0035\n",
      "Epoch [1/20], Loss: 0.0038\n",
      "Epoch [1/20], Loss: 0.1006\n",
      "Epoch [1/20], Loss: 0.0150\n",
      "Epoch [2/20], Loss: 0.0166\n",
      "Epoch [2/20], Loss: 0.0060\n",
      "Epoch [2/20], Loss: 0.0068\n",
      "Epoch [2/20], Loss: 0.0061\n",
      "Epoch [3/20], Loss: 0.0058\n",
      "Epoch [3/20], Loss: 0.0034\n",
      "Epoch [3/20], Loss: 0.0179\n",
      "Epoch [3/20], Loss: 0.0045\n",
      "Epoch [4/20], Loss: 0.0036\n",
      "Epoch [4/20], Loss: 0.0116\n",
      "Epoch [4/20], Loss: 0.0064\n",
      "Epoch [4/20], Loss: 0.0037\n",
      "Epoch [5/20], Loss: 0.0054\n",
      "Epoch [5/20], Loss: 0.0033\n",
      "Epoch [5/20], Loss: 0.0038\n",
      "Epoch [5/20], Loss: 0.0126\n",
      "Epoch [6/20], Loss: 0.0038\n",
      "Epoch [6/20], Loss: 0.0090\n",
      "Epoch [6/20], Loss: 0.0042\n",
      "Epoch [6/20], Loss: 0.0058\n",
      "Epoch [7/20], Loss: 0.0034\n",
      "Epoch [7/20], Loss: 0.0041\n",
      "Epoch [7/20], Loss: 0.0053\n",
      "Epoch [7/20], Loss: 0.0101\n",
      "Epoch [8/20], Loss: 0.0073\n",
      "Epoch [8/20], Loss: 0.0038\n",
      "Epoch [8/20], Loss: 0.0037\n",
      "Epoch [8/20], Loss: 0.0053\n",
      "Epoch [9/20], Loss: 0.0083\n",
      "Epoch [9/20], Loss: 0.0039\n",
      "Epoch [9/20], Loss: 0.0032\n",
      "Epoch [9/20], Loss: 0.0051\n",
      "Epoch [10/20], Loss: 0.0029\n",
      "Epoch [10/20], Loss: 0.0037\n",
      "Epoch [10/20], Loss: 0.0048\n",
      "Epoch [10/20], Loss: 0.0090\n",
      "Epoch [11/20], Loss: 0.0065\n",
      "Epoch [11/20], Loss: 0.0036\n",
      "Epoch [11/20], Loss: 0.0048\n",
      "Epoch [11/20], Loss: 0.0033\n",
      "Epoch [12/20], Loss: 0.0034\n",
      "Epoch [12/20], Loss: 0.0027\n",
      "Epoch [12/20], Loss: 0.0082\n",
      "Epoch [12/20], Loss: 0.0051\n",
      "Epoch [13/20], Loss: 0.0040\n",
      "Epoch [13/20], Loss: 0.0070\n",
      "Epoch [13/20], Loss: 0.0031\n",
      "Epoch [13/20], Loss: 0.0036\n",
      "Epoch [14/20], Loss: 0.0019\n",
      "Epoch [14/20], Loss: 0.0071\n",
      "Epoch [14/20], Loss: 0.0029\n",
      "Epoch [14/20], Loss: 0.0050\n",
      "Epoch [15/20], Loss: 0.0067\n",
      "Epoch [15/20], Loss: 0.0027\n",
      "Epoch [15/20], Loss: 0.0047\n",
      "Epoch [15/20], Loss: 0.0037\n",
      "Epoch [16/20], Loss: 0.0067\n",
      "Epoch [16/20], Loss: 0.0047\n",
      "Epoch [16/20], Loss: 0.0028\n",
      "Epoch [16/20], Loss: 0.0029\n",
      "Epoch [17/20], Loss: 0.0044\n",
      "Epoch [17/20], Loss: 0.0027\n",
      "Epoch [17/20], Loss: 0.0029\n",
      "Epoch [17/20], Loss: 0.0073\n",
      "Epoch [18/20], Loss: 0.0046\n",
      "Epoch [18/20], Loss: 0.0026\n",
      "Epoch [18/20], Loss: 0.0061\n",
      "Epoch [18/20], Loss: 0.0034\n",
      "Epoch [19/20], Loss: 0.0018\n",
      "Epoch [19/20], Loss: 0.0045\n",
      "Epoch [19/20], Loss: 0.0059\n",
      "Epoch [19/20], Loss: 0.0027\n",
      "Epoch [20/20], Loss: 0.0044\n",
      "Epoch [20/20], Loss: 0.0024\n",
      "Epoch [20/20], Loss: 0.0032\n",
      "Epoch [20/20], Loss: 0.0063\n",
      "Epoch [1/20], Loss: 0.0045\n",
      "Epoch [1/20], Loss: 0.0032\n",
      "Epoch [1/20], Loss: 0.1081\n",
      "Epoch [1/20], Loss: 0.0133\n",
      "Epoch [1/20], Loss: 0.0086\n",
      "Epoch [2/20], Loss: 0.0059\n",
      "Epoch [2/20], Loss: 0.0058\n",
      "Epoch [2/20], Loss: 0.0222\n",
      "Epoch [2/20], Loss: 0.0032\n",
      "Epoch [2/20], Loss: 0.0047\n",
      "Epoch [3/20], Loss: 0.0019\n",
      "Epoch [3/20], Loss: 0.0032\n",
      "Epoch [3/20], Loss: 0.0053\n",
      "Epoch [3/20], Loss: 0.0174\n",
      "Epoch [3/20], Loss: 0.0075\n",
      "Epoch [4/20], Loss: 0.0111\n",
      "Epoch [4/20], Loss: 0.0056\n",
      "Epoch [4/20], Loss: 0.0040\n",
      "Epoch [4/20], Loss: 0.0027\n",
      "Epoch [4/20], Loss: 0.0053\n",
      "Epoch [5/20], Loss: 0.0056\n",
      "Epoch [5/20], Loss: 0.0127\n",
      "Epoch [5/20], Loss: 0.0027\n",
      "Epoch [5/20], Loss: 0.0049\n",
      "Epoch [5/20], Loss: 0.0030\n",
      "Epoch [6/20], Loss: 0.0032\n",
      "Epoch [6/20], Loss: 0.0025\n",
      "Epoch [6/20], Loss: 0.0117\n",
      "Epoch [6/20], Loss: 0.0066\n",
      "Epoch [6/20], Loss: 0.0049\n",
      "Epoch [7/20], Loss: 0.0050\n",
      "Epoch [7/20], Loss: 0.0027\n",
      "Epoch [7/20], Loss: 0.0040\n",
      "Epoch [7/20], Loss: 0.0028\n",
      "Epoch [7/20], Loss: 0.0114\n",
      "Epoch [8/20], Loss: 0.0048\n",
      "Epoch [8/20], Loss: 0.0057\n",
      "Epoch [8/20], Loss: 0.0089\n",
      "Epoch [8/20], Loss: 0.0026\n",
      "Epoch [8/20], Loss: 0.0032\n",
      "Epoch [9/20], Loss: 0.0043\n",
      "Epoch [9/20], Loss: 0.0023\n",
      "Epoch [9/20], Loss: 0.0091\n",
      "Epoch [9/20], Loss: 0.0030\n",
      "Epoch [9/20], Loss: 0.0061\n",
      "Epoch [10/20], Loss: 0.0044\n",
      "Epoch [10/20], Loss: 0.0045\n",
      "Epoch [10/20], Loss: 0.0093\n",
      "Epoch [10/20], Loss: 0.0029\n",
      "Epoch [10/20], Loss: 0.0026\n",
      "Epoch [11/20], Loss: 0.0045\n",
      "Epoch [11/20], Loss: 0.0053\n",
      "Epoch [11/20], Loss: 0.0026\n",
      "Epoch [11/20], Loss: 0.0089\n",
      "Epoch [11/20], Loss: 0.0025\n",
      "Epoch [12/20], Loss: 0.0018\n",
      "Epoch [12/20], Loss: 0.0054\n",
      "Epoch [12/20], Loss: 0.0082\n",
      "Epoch [12/20], Loss: 0.0031\n",
      "Epoch [12/20], Loss: 0.0050\n",
      "Epoch [13/20], Loss: 0.0024\n",
      "Epoch [13/20], Loss: 0.0038\n",
      "Epoch [13/20], Loss: 0.0021\n",
      "Epoch [13/20], Loss: 0.0052\n",
      "Epoch [13/20], Loss: 0.0087\n",
      "Epoch [14/20], Loss: 0.0025\n",
      "Epoch [14/20], Loss: 0.0045\n",
      "Epoch [14/20], Loss: 0.0050\n",
      "Epoch [14/20], Loss: 0.0078\n",
      "Epoch [14/20], Loss: 0.0028\n",
      "Epoch [15/20], Loss: 0.0027\n",
      "Epoch [15/20], Loss: 0.0043\n",
      "Epoch [15/20], Loss: 0.0075\n",
      "Epoch [15/20], Loss: 0.0053\n",
      "Epoch [15/20], Loss: 0.0022\n",
      "Epoch [16/20], Loss: 0.0040\n",
      "Epoch [16/20], Loss: 0.0046\n",
      "Epoch [16/20], Loss: 0.0075\n",
      "Epoch [16/20], Loss: 0.0021\n",
      "Epoch [16/20], Loss: 0.0031\n",
      "Epoch [17/20], Loss: 0.0070\n",
      "Epoch [17/20], Loss: 0.0052\n",
      "Epoch [17/20], Loss: 0.0022\n",
      "Epoch [17/20], Loss: 0.0047\n",
      "Epoch [17/20], Loss: 0.0026\n",
      "Epoch [18/20], Loss: 0.0036\n",
      "Epoch [18/20], Loss: 0.0017\n",
      "Epoch [18/20], Loss: 0.0078\n",
      "Epoch [18/20], Loss: 0.0028\n",
      "Epoch [18/20], Loss: 0.0052\n",
      "Epoch [19/20], Loss: 0.0023\n",
      "Epoch [19/20], Loss: 0.0043\n",
      "Epoch [19/20], Loss: 0.0044\n",
      "Epoch [19/20], Loss: 0.0076\n",
      "Epoch [19/20], Loss: 0.0028\n",
      "Epoch [20/20], Loss: 0.0025\n",
      "Epoch [20/20], Loss: 0.0046\n",
      "Epoch [20/20], Loss: 0.0021\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [20/20], Loss: 0.0069\n",
      "Epoch [20/20], Loss: 0.0050\n",
      "Epoch [1/20], Loss: 0.0025\n",
      "Epoch [1/20], Loss: 0.0022\n",
      "Epoch [1/20], Loss: 0.0039\n",
      "Epoch [1/20], Loss: 0.0053\n",
      "Epoch [1/20], Loss: 0.0080\n",
      "Epoch [1/20], Loss: 0.0976\n",
      "Epoch [2/20], Loss: 0.0106\n",
      "Epoch [2/20], Loss: 0.0045\n",
      "Epoch [2/20], Loss: 0.0073\n",
      "Epoch [2/20], Loss: 0.0045\n",
      "Epoch [2/20], Loss: 0.0034\n",
      "Epoch [2/20], Loss: 0.0287\n",
      "Epoch [3/20], Loss: 0.0076\n",
      "Epoch [3/20], Loss: 0.0030\n",
      "Epoch [3/20], Loss: 0.0044\n",
      "Epoch [3/20], Loss: 0.0034\n",
      "Epoch [3/20], Loss: 0.0178\n",
      "Epoch [3/20], Loss: 0.0080\n",
      "Epoch [4/20], Loss: 0.0139\n",
      "Epoch [4/20], Loss: 0.0052\n",
      "Epoch [4/20], Loss: 0.0031\n",
      "Epoch [4/20], Loss: 0.0048\n",
      "Epoch [4/20], Loss: 0.0072\n",
      "Epoch [4/20], Loss: 0.0028\n",
      "Epoch [5/20], Loss: 0.0059\n",
      "Epoch [5/20], Loss: 0.0049\n",
      "Epoch [5/20], Loss: 0.0022\n",
      "Epoch [5/20], Loss: 0.0046\n",
      "Epoch [5/20], Loss: 0.0026\n",
      "Epoch [5/20], Loss: 0.0189\n",
      "Epoch [6/20], Loss: 0.0054\n",
      "Epoch [6/20], Loss: 0.0029\n",
      "Epoch [6/20], Loss: 0.0022\n",
      "Epoch [6/20], Loss: 0.0133\n",
      "Epoch [6/20], Loss: 0.0052\n",
      "Epoch [6/20], Loss: 0.0074\n",
      "Epoch [7/20], Loss: 0.0116\n",
      "Epoch [7/20], Loss: 0.0027\n",
      "Epoch [7/20], Loss: 0.0057\n",
      "Epoch [7/20], Loss: 0.0058\n",
      "Epoch [7/20], Loss: 0.0045\n",
      "Epoch [7/20], Loss: 0.0033\n",
      "Epoch [8/20], Loss: 0.0019\n",
      "Epoch [8/20], Loss: 0.0135\n",
      "Epoch [8/20], Loss: 0.0056\n",
      "Epoch [8/20], Loss: 0.0063\n",
      "Epoch [8/20], Loss: 0.0029\n",
      "Epoch [8/20], Loss: 0.0041\n",
      "Epoch [9/20], Loss: 0.0056\n",
      "Epoch [9/20], Loss: 0.0118\n",
      "Epoch [9/20], Loss: 0.0041\n",
      "Epoch [9/20], Loss: 0.0049\n",
      "Epoch [9/20], Loss: 0.0028\n",
      "Epoch [9/20], Loss: 0.0021\n",
      "Epoch [10/20], Loss: 0.0035\n",
      "Epoch [10/20], Loss: 0.0117\n",
      "Epoch [10/20], Loss: 0.0028\n",
      "Epoch [10/20], Loss: 0.0017\n",
      "Epoch [10/20], Loss: 0.0047\n",
      "Epoch [10/20], Loss: 0.0067\n",
      "Epoch [11/20], Loss: 0.0052\n",
      "Epoch [11/20], Loss: 0.0113\n",
      "Epoch [11/20], Loss: 0.0049\n",
      "Epoch [11/20], Loss: 0.0045\n",
      "Epoch [11/20], Loss: 0.0026\n",
      "Epoch [11/20], Loss: 0.0021\n",
      "Epoch [12/20], Loss: 0.0102\n",
      "Epoch [12/20], Loss: 0.0026\n",
      "Epoch [12/20], Loss: 0.0037\n",
      "Epoch [12/20], Loss: 0.0018\n",
      "Epoch [12/20], Loss: 0.0064\n",
      "Epoch [12/20], Loss: 0.0049\n",
      "Epoch [13/20], Loss: 0.0036\n",
      "Epoch [13/20], Loss: 0.0106\n",
      "Epoch [13/20], Loss: 0.0027\n",
      "Epoch [13/20], Loss: 0.0037\n",
      "Epoch [13/20], Loss: 0.0057\n",
      "Epoch [13/20], Loss: 0.0024\n",
      "Epoch [14/20], Loss: 0.0050\n",
      "Epoch [14/20], Loss: 0.0019\n",
      "Epoch [14/20], Loss: 0.0037\n",
      "Epoch [14/20], Loss: 0.0027\n",
      "Epoch [14/20], Loss: 0.0045\n",
      "Epoch [14/20], Loss: 0.0108\n",
      "Epoch [15/20], Loss: 0.0082\n",
      "Epoch [15/20], Loss: 0.0030\n",
      "Epoch [15/20], Loss: 0.0035\n",
      "Epoch [15/20], Loss: 0.0024\n",
      "Epoch [15/20], Loss: 0.0063\n",
      "Epoch [15/20], Loss: 0.0046\n",
      "Epoch [16/20], Loss: 0.0093\n",
      "Epoch [16/20], Loss: 0.0042\n",
      "Epoch [16/20], Loss: 0.0037\n",
      "Epoch [16/20], Loss: 0.0023\n",
      "Epoch [16/20], Loss: 0.0021\n",
      "Epoch [16/20], Loss: 0.0053\n",
      "Epoch [17/20], Loss: 0.0036\n",
      "Epoch [17/20], Loss: 0.0020\n",
      "Epoch [17/20], Loss: 0.0041\n",
      "Epoch [17/20], Loss: 0.0049\n",
      "Epoch [17/20], Loss: 0.0103\n",
      "Epoch [17/20], Loss: 0.0028\n",
      "Epoch [18/20], Loss: 0.0044\n",
      "Epoch [18/20], Loss: 0.0024\n",
      "Epoch [18/20], Loss: 0.0051\n",
      "Epoch [18/20], Loss: 0.0023\n",
      "Epoch [18/20], Loss: 0.0085\n",
      "Epoch [18/20], Loss: 0.0041\n",
      "Epoch [19/20], Loss: 0.0042\n",
      "Epoch [19/20], Loss: 0.0021\n",
      "Epoch [19/20], Loss: 0.0078\n",
      "Epoch [19/20], Loss: 0.0032\n",
      "Epoch [19/20], Loss: 0.0025\n",
      "Epoch [19/20], Loss: 0.0058\n",
      "Epoch [20/20], Loss: 0.0024\n",
      "Epoch [20/20], Loss: 0.0047\n",
      "Epoch [20/20], Loss: 0.0037\n",
      "Epoch [20/20], Loss: 0.0042\n",
      "Epoch [20/20], Loss: 0.0022\n",
      "Epoch [20/20], Loss: 0.0086\n",
      "Epoch [1/20], Loss: 0.0032\n",
      "Epoch [1/20], Loss: 0.0076\n",
      "Epoch [1/20], Loss: 0.0999\n",
      "Epoch [1/20], Loss: 0.0135\n",
      "Epoch [1/20], Loss: 0.0196\n",
      "Epoch [1/20], Loss: 0.0082\n",
      "Epoch [1/20], Loss: 0.0084\n",
      "Epoch [2/20], Loss: 0.0048\n",
      "Epoch [2/20], Loss: 0.0039\n",
      "Epoch [2/20], Loss: 0.0083\n",
      "Epoch [2/20], Loss: 0.0028\n",
      "Epoch [2/20], Loss: 0.0427\n",
      "Epoch [2/20], Loss: 0.0093\n",
      "Epoch [2/20], Loss: 0.0030\n",
      "Epoch [3/20], Loss: 0.0052\n",
      "Epoch [3/20], Loss: 0.0074\n",
      "Epoch [3/20], Loss: 0.0068\n",
      "Epoch [3/20], Loss: 0.0024\n",
      "Epoch [3/20], Loss: 0.0026\n",
      "Epoch [3/20], Loss: 0.0033\n",
      "Epoch [3/20], Loss: 0.0280\n",
      "Epoch [4/20], Loss: 0.0066\n",
      "Epoch [4/20], Loss: 0.0076\n",
      "Epoch [4/20], Loss: 0.0040\n",
      "Epoch [4/20], Loss: 0.0028\n",
      "Epoch [4/20], Loss: 0.0047\n",
      "Epoch [4/20], Loss: 0.0210\n",
      "Epoch [4/20], Loss: 0.0029\n",
      "Epoch [5/20], Loss: 0.0025\n",
      "Epoch [5/20], Loss: 0.0020\n",
      "Epoch [5/20], Loss: 0.0056\n",
      "Epoch [5/20], Loss: 0.0039\n",
      "Epoch [5/20], Loss: 0.0040\n",
      "Epoch [5/20], Loss: 0.0197\n",
      "Epoch [5/20], Loss: 0.0085\n",
      "Epoch [6/20], Loss: 0.0027\n",
      "Epoch [6/20], Loss: 0.0125\n",
      "Epoch [6/20], Loss: 0.0054\n",
      "Epoch [6/20], Loss: 0.0028\n",
      "Epoch [6/20], Loss: 0.0033\n",
      "Epoch [6/20], Loss: 0.0070\n",
      "Epoch [6/20], Loss: 0.0060\n",
      "Epoch [7/20], Loss: 0.0042\n",
      "Epoch [7/20], Loss: 0.0025\n",
      "Epoch [7/20], Loss: 0.0065\n",
      "Epoch [7/20], Loss: 0.0034\n",
      "Epoch [7/20], Loss: 0.0188\n",
      "Epoch [7/20], Loss: 0.0029\n",
      "Epoch [7/20], Loss: 0.0051\n",
      "Epoch [8/20], Loss: 0.0041\n",
      "Epoch [8/20], Loss: 0.0022\n",
      "Epoch [8/20], Loss: 0.0044\n",
      "Epoch [8/20], Loss: 0.0153\n",
      "Epoch [8/20], Loss: 0.0026\n",
      "Epoch [8/20], Loss: 0.0040\n",
      "Epoch [8/20], Loss: 0.0075\n",
      "Epoch [9/20], Loss: 0.0026\n",
      "Epoch [9/20], Loss: 0.0018\n",
      "Epoch [9/20], Loss: 0.0133\n",
      "Epoch [9/20], Loss: 0.0062\n",
      "Epoch [9/20], Loss: 0.0036\n",
      "Epoch [9/20], Loss: 0.0059\n",
      "Epoch [9/20], Loss: 0.0043\n",
      "Epoch [10/20], Loss: 0.0065\n",
      "Epoch [10/20], Loss: 0.0024\n",
      "Epoch [10/20], Loss: 0.0022\n",
      "Epoch [10/20], Loss: 0.0148\n",
      "Epoch [10/20], Loss: 0.0050\n",
      "Epoch [10/20], Loss: 0.0042\n",
      "Epoch [10/20], Loss: 0.0036\n",
      "Epoch [11/20], Loss: 0.0033\n",
      "Epoch [11/20], Loss: 0.0020\n",
      "Epoch [11/20], Loss: 0.0020\n",
      "Epoch [11/20], Loss: 0.0044\n",
      "Epoch [11/20], Loss: 0.0147\n",
      "Epoch [11/20], Loss: 0.0076\n",
      "Epoch [11/20], Loss: 0.0037\n",
      "Epoch [12/20], Loss: 0.0026\n",
      "Epoch [12/20], Loss: 0.0062\n",
      "Epoch [12/20], Loss: 0.0046\n",
      "Epoch [12/20], Loss: 0.0022\n",
      "Epoch [12/20], Loss: 0.0048\n",
      "Epoch [12/20], Loss: 0.0021\n",
      "Epoch [12/20], Loss: 0.0137\n",
      "Epoch [13/20], Loss: 0.0019\n",
      "Epoch [13/20], Loss: 0.0036\n",
      "Epoch [13/20], Loss: 0.0038\n",
      "Epoch [13/20], Loss: 0.0044\n",
      "Epoch [13/20], Loss: 0.0112\n",
      "Epoch [13/20], Loss: 0.0024\n",
      "Epoch [13/20], Loss: 0.0073\n",
      "Epoch [14/20], Loss: 0.0033\n",
      "Epoch [14/20], Loss: 0.0036\n",
      "Epoch [14/20], Loss: 0.0021\n",
      "Epoch [14/20], Loss: 0.0061\n",
      "Epoch [14/20], Loss: 0.0049\n",
      "Epoch [14/20], Loss: 0.0020\n",
      "Epoch [14/20], Loss: 0.0126\n",
      "Epoch [15/20], Loss: 0.0061\n",
      "Epoch [15/20], Loss: 0.0016\n",
      "Epoch [15/20], Loss: 0.0045\n",
      "Epoch [15/20], Loss: 0.0037\n",
      "Epoch [15/20], Loss: 0.0021\n",
      "Epoch [15/20], Loss: 0.0038\n",
      "Epoch [15/20], Loss: 0.0113\n",
      "Epoch [16/20], Loss: 0.0030\n",
      "Epoch [16/20], Loss: 0.0046\n",
      "Epoch [16/20], Loss: 0.0020\n",
      "Epoch [16/20], Loss: 0.0019\n",
      "Epoch [16/20], Loss: 0.0101\n",
      "Epoch [16/20], Loss: 0.0072\n",
      "Epoch [16/20], Loss: 0.0043\n",
      "Epoch [17/20], Loss: 0.0058\n",
      "Epoch [17/20], Loss: 0.0021\n",
      "Epoch [17/20], Loss: 0.0020\n",
      "Epoch [17/20], Loss: 0.0048\n",
      "Epoch [17/20], Loss: 0.0036\n",
      "Epoch [17/20], Loss: 0.0036\n",
      "Epoch [17/20], Loss: 0.0116\n",
      "Epoch [18/20], Loss: 0.0084\n",
      "Epoch [18/20], Loss: 0.0025\n",
      "Epoch [18/20], Loss: 0.0066\n",
      "Epoch [18/20], Loss: 0.0031\n",
      "Epoch [18/20], Loss: 0.0035\n",
      "Epoch [18/20], Loss: 0.0023\n",
      "Epoch [18/20], Loss: 0.0047\n",
      "Epoch [19/20], Loss: 0.0038\n",
      "Epoch [19/20], Loss: 0.0063\n",
      "Epoch [19/20], Loss: 0.0038\n",
      "Epoch [19/20], Loss: 0.0032\n",
      "Epoch [19/20], Loss: 0.0017\n",
      "Epoch [19/20], Loss: 0.0019\n",
      "Epoch [19/20], Loss: 0.0120\n",
      "Epoch [20/20], Loss: 0.0059\n",
      "Epoch [20/20], Loss: 0.0032\n",
      "Epoch [20/20], Loss: 0.0035\n",
      "Epoch [20/20], Loss: 0.0020\n",
      "Epoch [20/20], Loss: 0.0097\n",
      "Epoch [20/20], Loss: 0.0051\n",
      "Epoch [20/20], Loss: 0.0020\n",
      "Epoch [1/20], Loss: 0.0035\n",
      "Epoch [1/20], Loss: 0.0040\n",
      "Epoch [1/20], Loss: 0.0070\n",
      "Epoch [1/20], Loss: 0.0026\n",
      "Epoch [1/20], Loss: 0.0964\n",
      "Epoch [1/20], Loss: 0.0139\n",
      "Epoch [1/20], Loss: 0.0090\n",
      "Epoch [1/20], Loss: 0.0175\n",
      "Epoch [2/20], Loss: 0.0081\n",
      "Epoch [2/20], Loss: 0.0282\n",
      "Epoch [2/20], Loss: 0.0088\n",
      "Epoch [2/20], Loss: 0.0035\n",
      "Epoch [2/20], Loss: 0.0043\n",
      "Epoch [2/20], Loss: 0.0023\n",
      "Epoch [2/20], Loss: 0.0051\n",
      "Epoch [2/20], Loss: 0.0072\n",
      "Epoch [3/20], Loss: 0.0213\n",
      "Epoch [3/20], Loss: 0.0025\n",
      "Epoch [3/20], Loss: 0.0055\n",
      "Epoch [3/20], Loss: 0.0038\n",
      "Epoch [3/20], Loss: 0.0021\n",
      "Epoch [3/20], Loss: 0.0037\n",
      "Epoch [3/20], Loss: 0.0119\n",
      "Epoch [3/20], Loss: 0.0052\n",
      "Epoch [4/20], Loss: 0.0035\n",
      "Epoch [4/20], Loss: 0.0220\n",
      "Epoch [4/20], Loss: 0.0026\n",
      "Epoch [4/20], Loss: 0.0020\n",
      "Epoch [4/20], Loss: 0.0038\n",
      "Epoch [4/20], Loss: 0.0099\n",
      "Epoch [4/20], Loss: 0.0063\n",
      "Epoch [4/20], Loss: 0.0048\n",
      "Epoch [5/20], Loss: 0.0023\n",
      "Epoch [5/20], Loss: 0.0087\n",
      "Epoch [5/20], Loss: 0.0038\n",
      "Epoch [5/20], Loss: 0.0040\n",
      "Epoch [5/20], Loss: 0.0035\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [5/20], Loss: 0.0224\n",
      "Epoch [5/20], Loss: 0.0065\n",
      "Epoch [5/20], Loss: 0.0025\n",
      "Epoch [6/20], Loss: 0.0013\n",
      "Epoch [6/20], Loss: 0.0021\n",
      "Epoch [6/20], Loss: 0.0032\n",
      "Epoch [6/20], Loss: 0.0034\n",
      "Epoch [6/20], Loss: 0.0099\n",
      "Epoch [6/20], Loss: 0.0165\n",
      "Epoch [6/20], Loss: 0.0063\n",
      "Epoch [6/20], Loss: 0.0055\n",
      "Epoch [7/20], Loss: 0.0037\n",
      "Epoch [7/20], Loss: 0.0087\n",
      "Epoch [7/20], Loss: 0.0125\n",
      "Epoch [7/20], Loss: 0.0041\n",
      "Epoch [7/20], Loss: 0.0029\n",
      "Epoch [7/20], Loss: 0.0036\n",
      "Epoch [7/20], Loss: 0.0025\n",
      "Epoch [7/20], Loss: 0.0057\n",
      "Epoch [8/20], Loss: 0.0047\n",
      "Epoch [8/20], Loss: 0.0123\n",
      "Epoch [8/20], Loss: 0.0022\n",
      "Epoch [8/20], Loss: 0.0055\n",
      "Epoch [8/20], Loss: 0.0034\n",
      "Epoch [8/20], Loss: 0.0099\n",
      "Epoch [8/20], Loss: 0.0016\n",
      "Epoch [8/20], Loss: 0.0035\n",
      "Epoch [9/20], Loss: 0.0125\n",
      "Epoch [9/20], Loss: 0.0031\n",
      "Epoch [9/20], Loss: 0.0048\n",
      "Epoch [9/20], Loss: 0.0018\n",
      "Epoch [9/20], Loss: 0.0032\n",
      "Epoch [9/20], Loss: 0.0057\n",
      "Epoch [9/20], Loss: 0.0090\n",
      "Epoch [9/20], Loss: 0.0023\n",
      "Epoch [10/20], Loss: 0.0030\n",
      "Epoch [10/20], Loss: 0.0016\n",
      "Epoch [10/20], Loss: 0.0051\n",
      "Epoch [10/20], Loss: 0.0021\n",
      "Epoch [10/20], Loss: 0.0050\n",
      "Epoch [10/20], Loss: 0.0147\n",
      "Epoch [10/20], Loss: 0.0088\n",
      "Epoch [10/20], Loss: 0.0037\n",
      "Epoch [11/20], Loss: 0.0072\n",
      "Epoch [11/20], Loss: 0.0098\n",
      "Epoch [11/20], Loss: 0.0020\n",
      "Epoch [11/20], Loss: 0.0046\n",
      "Epoch [11/20], Loss: 0.0060\n",
      "Epoch [11/20], Loss: 0.0021\n",
      "Epoch [11/20], Loss: 0.0036\n",
      "Epoch [11/20], Loss: 0.0035\n",
      "Epoch [12/20], Loss: 0.0021\n",
      "Epoch [12/20], Loss: 0.0106\n",
      "Epoch [12/20], Loss: 0.0087\n",
      "Epoch [12/20], Loss: 0.0030\n",
      "Epoch [12/20], Loss: 0.0020\n",
      "Epoch [12/20], Loss: 0.0045\n",
      "Epoch [12/20], Loss: 0.0033\n",
      "Epoch [12/20], Loss: 0.0058\n",
      "Epoch [13/20], Loss: 0.0050\n",
      "Epoch [13/20], Loss: 0.0027\n",
      "Epoch [13/20], Loss: 0.0021\n",
      "Epoch [13/20], Loss: 0.0034\n",
      "Epoch [13/20], Loss: 0.0041\n",
      "Epoch [13/20], Loss: 0.0018\n",
      "Epoch [13/20], Loss: 0.0092\n",
      "Epoch [13/20], Loss: 0.0128\n",
      "Epoch [14/20], Loss: 0.0022\n",
      "Epoch [14/20], Loss: 0.0044\n",
      "Epoch [14/20], Loss: 0.0083\n",
      "Epoch [14/20], Loss: 0.0038\n",
      "Epoch [14/20], Loss: 0.0015\n",
      "Epoch [14/20], Loss: 0.0077\n",
      "Epoch [14/20], Loss: 0.0060\n",
      "Epoch [14/20], Loss: 0.0036\n",
      "Epoch [15/20], Loss: 0.0027\n",
      "Epoch [15/20], Loss: 0.0094\n",
      "Epoch [15/20], Loss: 0.0053\n",
      "Epoch [15/20], Loss: 0.0016\n",
      "Epoch [15/20], Loss: 0.0033\n",
      "Epoch [15/20], Loss: 0.0082\n",
      "Epoch [15/20], Loss: 0.0046\n",
      "Epoch [15/20], Loss: 0.0024\n",
      "Epoch [16/20], Loss: 0.0037\n",
      "Epoch [16/20], Loss: 0.0033\n",
      "Epoch [16/20], Loss: 0.0020\n",
      "Epoch [16/20], Loss: 0.0098\n",
      "Epoch [16/20], Loss: 0.0077\n",
      "Epoch [16/20], Loss: 0.0017\n",
      "Epoch [16/20], Loss: 0.0059\n",
      "Epoch [16/20], Loss: 0.0035\n",
      "Epoch [17/20], Loss: 0.0042\n",
      "Epoch [17/20], Loss: 0.0029\n",
      "Epoch [17/20], Loss: 0.0084\n",
      "Epoch [17/20], Loss: 0.0034\n",
      "Epoch [17/20], Loss: 0.0077\n",
      "Epoch [17/20], Loss: 0.0053\n",
      "Epoch [17/20], Loss: 0.0025\n",
      "Epoch [17/20], Loss: 0.0016\n",
      "Epoch [18/20], Loss: 0.0069\n",
      "Epoch [18/20], Loss: 0.0050\n",
      "Epoch [18/20], Loss: 0.0014\n",
      "Epoch [18/20], Loss: 0.0036\n",
      "Epoch [18/20], Loss: 0.0021\n",
      "Epoch [18/20], Loss: 0.0097\n",
      "Epoch [18/20], Loss: 0.0030\n",
      "Epoch [18/20], Loss: 0.0051\n",
      "Epoch [19/20], Loss: 0.0027\n",
      "Epoch [19/20], Loss: 0.0018\n",
      "Epoch [19/20], Loss: 0.0031\n",
      "Epoch [19/20], Loss: 0.0054\n",
      "Epoch [19/20], Loss: 0.0082\n",
      "Epoch [19/20], Loss: 0.0082\n",
      "Epoch [19/20], Loss: 0.0023\n",
      "Epoch [19/20], Loss: 0.0044\n",
      "Epoch [20/20], Loss: 0.0036\n",
      "Epoch [20/20], Loss: 0.0022\n",
      "Epoch [20/20], Loss: 0.0071\n",
      "Epoch [20/20], Loss: 0.0019\n",
      "Epoch [20/20], Loss: 0.0053\n",
      "Epoch [20/20], Loss: 0.0036\n",
      "Epoch [20/20], Loss: 0.0029\n",
      "Epoch [20/20], Loss: 0.0088\n",
      "Epoch [1/20], Loss: 0.0045\n",
      "Epoch [1/20], Loss: 0.0033\n",
      "Epoch [1/20], Loss: 0.0020\n",
      "Epoch [1/20], Loss: 0.0031\n",
      "Epoch [1/20], Loss: 0.0025\n",
      "Epoch [1/20], Loss: 0.0058\n",
      "Epoch [1/20], Loss: 0.0085\n",
      "Epoch [1/20], Loss: 0.0956\n",
      "Epoch [1/20], Loss: 0.0156\n",
      "Epoch [2/20], Loss: 0.0113\n",
      "Epoch [2/20], Loss: 0.0246\n",
      "Epoch [2/20], Loss: 0.0101\n",
      "Epoch [2/20], Loss: 0.0061\n",
      "Epoch [2/20], Loss: 0.0040\n",
      "Epoch [2/20], Loss: 0.0060\n",
      "Epoch [2/20], Loss: 0.0082\n",
      "Epoch [2/20], Loss: 0.0084\n",
      "Epoch [2/20], Loss: 0.0044\n",
      "Epoch [3/20], Loss: 0.0285\n",
      "Epoch [3/20], Loss: 0.0046\n",
      "Epoch [3/20], Loss: 0.0057\n",
      "Epoch [3/20], Loss: 0.0074\n",
      "Epoch [3/20], Loss: 0.0037\n",
      "Epoch [3/20], Loss: 0.0029\n",
      "Epoch [3/20], Loss: 0.0074\n",
      "Epoch [3/20], Loss: 0.0020\n",
      "Epoch [3/20], Loss: 0.0057\n",
      "Epoch [4/20], Loss: 0.0038\n",
      "Epoch [4/20], Loss: 0.0059\n",
      "Epoch [4/20], Loss: 0.0079\n",
      "Epoch [4/20], Loss: 0.0238\n",
      "Epoch [4/20], Loss: 0.0025\n",
      "Epoch [4/20], Loss: 0.0036\n",
      "Epoch [4/20], Loss: 0.0076\n",
      "Epoch [4/20], Loss: 0.0041\n",
      "Epoch [4/20], Loss: 0.0023\n",
      "Epoch [5/20], Loss: 0.0051\n",
      "Epoch [5/20], Loss: 0.0067\n",
      "Epoch [5/20], Loss: 0.0021\n",
      "Epoch [5/20], Loss: 0.0031\n",
      "Epoch [5/20], Loss: 0.0019\n",
      "Epoch [5/20], Loss: 0.0053\n",
      "Epoch [5/20], Loss: 0.0036\n",
      "Epoch [5/20], Loss: 0.0215\n",
      "Epoch [5/20], Loss: 0.0096\n",
      "Epoch [6/20], Loss: 0.0057\n",
      "Epoch [6/20], Loss: 0.0019\n",
      "Epoch [6/20], Loss: 0.0032\n",
      "Epoch [6/20], Loss: 0.0053\n",
      "Epoch [6/20], Loss: 0.0149\n",
      "Epoch [6/20], Loss: 0.0045\n",
      "Epoch [6/20], Loss: 0.0079\n",
      "Epoch [6/20], Loss: 0.0080\n",
      "Epoch [6/20], Loss: 0.0036\n",
      "Epoch [7/20], Loss: 0.0062\n",
      "Epoch [7/20], Loss: 0.0050\n",
      "Epoch [7/20], Loss: 0.0066\n",
      "Epoch [7/20], Loss: 0.0030\n",
      "Epoch [7/20], Loss: 0.0022\n",
      "Epoch [7/20], Loss: 0.0032\n",
      "Epoch [7/20], Loss: 0.0166\n",
      "Epoch [7/20], Loss: 0.0062\n",
      "Epoch [7/20], Loss: 0.0030\n",
      "Epoch [8/20], Loss: 0.0048\n",
      "Epoch [8/20], Loss: 0.0030\n",
      "Epoch [8/20], Loss: 0.0073\n",
      "Epoch [8/20], Loss: 0.0051\n",
      "Epoch [8/20], Loss: 0.0023\n",
      "Epoch [8/20], Loss: 0.0137\n",
      "Epoch [8/20], Loss: 0.0054\n",
      "Epoch [8/20], Loss: 0.0025\n",
      "Epoch [8/20], Loss: 0.0083\n",
      "Epoch [9/20], Loss: 0.0050\n",
      "Epoch [9/20], Loss: 0.0069\n",
      "Epoch [9/20], Loss: 0.0057\n",
      "Epoch [9/20], Loss: 0.0131\n",
      "Epoch [9/20], Loss: 0.0069\n",
      "Epoch [9/20], Loss: 0.0035\n",
      "Epoch [9/20], Loss: 0.0020\n",
      "Epoch [9/20], Loss: 0.0034\n",
      "Epoch [9/20], Loss: 0.0024\n",
      "Epoch [10/20], Loss: 0.0048\n",
      "Epoch [10/20], Loss: 0.0071\n",
      "Epoch [10/20], Loss: 0.0055\n",
      "Epoch [10/20], Loss: 0.0017\n",
      "Epoch [10/20], Loss: 0.0071\n",
      "Epoch [10/20], Loss: 0.0034\n",
      "Epoch [10/20], Loss: 0.0030\n",
      "Epoch [10/20], Loss: 0.0021\n",
      "Epoch [10/20], Loss: 0.0144\n",
      "Epoch [11/20], Loss: 0.0029\n",
      "Epoch [11/20], Loss: 0.0018\n",
      "Epoch [11/20], Loss: 0.0021\n",
      "Epoch [11/20], Loss: 0.0031\n",
      "Epoch [11/20], Loss: 0.0073\n",
      "Epoch [11/20], Loss: 0.0112\n",
      "Epoch [11/20], Loss: 0.0076\n",
      "Epoch [11/20], Loss: 0.0066\n",
      "Epoch [11/20], Loss: 0.0055\n",
      "Epoch [12/20], Loss: 0.0064\n",
      "Epoch [12/20], Loss: 0.0019\n",
      "Epoch [12/20], Loss: 0.0062\n",
      "Epoch [12/20], Loss: 0.0041\n",
      "Epoch [12/20], Loss: 0.0117\n",
      "Epoch [12/20], Loss: 0.0055\n",
      "Epoch [12/20], Loss: 0.0036\n",
      "Epoch [12/20], Loss: 0.0029\n",
      "Epoch [12/20], Loss: 0.0033\n",
      "Epoch [13/20], Loss: 0.0070\n",
      "Epoch [13/20], Loss: 0.0027\n",
      "Epoch [13/20], Loss: 0.0049\n",
      "Epoch [13/20], Loss: 0.0068\n",
      "Epoch [13/20], Loss: 0.0111\n",
      "Epoch [13/20], Loss: 0.0052\n",
      "Epoch [13/20], Loss: 0.0023\n",
      "Epoch [13/20], Loss: 0.0022\n",
      "Epoch [13/20], Loss: 0.0030\n",
      "Epoch [14/20], Loss: 0.0062\n",
      "Epoch [14/20], Loss: 0.0093\n",
      "Epoch [14/20], Loss: 0.0025\n",
      "Epoch [14/20], Loss: 0.0045\n",
      "Epoch [14/20], Loss: 0.0057\n",
      "Epoch [14/20], Loss: 0.0029\n",
      "Epoch [14/20], Loss: 0.0018\n",
      "Epoch [14/20], Loss: 0.0037\n",
      "Epoch [14/20], Loss: 0.0074\n",
      "Epoch [15/20], Loss: 0.0015\n",
      "Epoch [15/20], Loss: 0.0058\n",
      "Epoch [15/20], Loss: 0.0028\n",
      "Epoch [15/20], Loss: 0.0049\n",
      "Epoch [15/20], Loss: 0.0111\n",
      "Epoch [15/20], Loss: 0.0072\n",
      "Epoch [15/20], Loss: 0.0032\n",
      "Epoch [15/20], Loss: 0.0025\n",
      "Epoch [15/20], Loss: 0.0049\n",
      "Epoch [16/20], Loss: 0.0058\n",
      "Epoch [16/20], Loss: 0.0021\n",
      "Epoch [16/20], Loss: 0.0033\n",
      "Epoch [16/20], Loss: 0.0095\n",
      "Epoch [16/20], Loss: 0.0047\n",
      "Epoch [16/20], Loss: 0.0023\n",
      "Epoch [16/20], Loss: 0.0053\n",
      "Epoch [16/20], Loss: 0.0030\n",
      "Epoch [16/20], Loss: 0.0072\n",
      "Epoch [17/20], Loss: 0.0020\n",
      "Epoch [17/20], Loss: 0.0058\n",
      "Epoch [17/20], Loss: 0.0031\n",
      "Epoch [17/20], Loss: 0.0069\n",
      "Epoch [17/20], Loss: 0.0095\n",
      "Epoch [17/20], Loss: 0.0030\n",
      "Epoch [17/20], Loss: 0.0049\n",
      "Epoch [17/20], Loss: 0.0019\n",
      "Epoch [17/20], Loss: 0.0051\n",
      "Epoch [18/20], Loss: 0.0021\n",
      "Epoch [18/20], Loss: 0.0027\n",
      "Epoch [18/20], Loss: 0.0085\n",
      "Epoch [18/20], Loss: 0.0064\n",
      "Epoch [18/20], Loss: 0.0016\n",
      "Epoch [18/20], Loss: 0.0036\n",
      "Epoch [18/20], Loss: 0.0046\n",
      "Epoch [18/20], Loss: 0.0071\n",
      "Epoch [18/20], Loss: 0.0053\n",
      "Epoch [19/20], Loss: 0.0014\n",
      "Epoch [19/20], Loss: 0.0029\n",
      "Epoch [19/20], Loss: 0.0038\n",
      "Epoch [19/20], Loss: 0.0093\n",
      "Epoch [19/20], Loss: 0.0050\n",
      "Epoch [19/20], Loss: 0.0027\n",
      "Epoch [19/20], Loss: 0.0033\n",
      "Epoch [19/20], Loss: 0.0066\n",
      "Epoch [19/20], Loss: 0.0069\n",
      "Epoch [20/20], Loss: 0.0085\n",
      "Epoch [20/20], Loss: 0.0059\n",
      "Epoch [20/20], Loss: 0.0019\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [20/20], Loss: 0.0031\n",
      "Epoch [20/20], Loss: 0.0059\n",
      "Epoch [20/20], Loss: 0.0055\n",
      "Epoch [20/20], Loss: 0.0024\n",
      "Epoch [20/20], Loss: 0.0047\n",
      "Epoch [20/20], Loss: 0.0032\n",
      "Epoch [1/20], Loss: 0.0056\n",
      "Epoch [1/20], Loss: 0.0055\n",
      "Epoch [1/20], Loss: 0.0031\n",
      "Epoch [1/20], Loss: 0.0023\n",
      "Epoch [1/20], Loss: 0.0986\n",
      "Epoch [1/20], Loss: 0.0274\n",
      "Epoch [1/20], Loss: 0.0135\n",
      "Epoch [1/20], Loss: 0.0085\n",
      "Epoch [1/20], Loss: 0.0061\n",
      "Epoch [1/20], Loss: 0.0069\n",
      "Epoch [2/20], Loss: 0.0032\n",
      "Epoch [2/20], Loss: 0.0039\n",
      "Epoch [2/20], Loss: 0.0357\n",
      "Epoch [2/20], Loss: 0.0074\n",
      "Epoch [2/20], Loss: 0.0048\n",
      "Epoch [2/20], Loss: 0.0078\n",
      "Epoch [2/20], Loss: 0.0090\n",
      "Epoch [2/20], Loss: 0.0080\n",
      "Epoch [2/20], Loss: 0.0029\n",
      "Epoch [2/20], Loss: 0.0049\n",
      "Epoch [3/20], Loss: 0.0021\n",
      "Epoch [3/20], Loss: 0.0077\n",
      "Epoch [3/20], Loss: 0.0051\n",
      "Epoch [3/20], Loss: 0.0239\n",
      "Epoch [3/20], Loss: 0.0053\n",
      "Epoch [3/20], Loss: 0.0026\n",
      "Epoch [3/20], Loss: 0.0068\n",
      "Epoch [3/20], Loss: 0.0056\n",
      "Epoch [3/20], Loss: 0.0070\n",
      "Epoch [3/20], Loss: 0.0044\n",
      "Epoch [4/20], Loss: 0.0048\n",
      "Epoch [4/20], Loss: 0.0061\n",
      "Epoch [4/20], Loss: 0.0035\n",
      "Epoch [4/20], Loss: 0.0043\n",
      "Epoch [4/20], Loss: 0.0062\n",
      "Epoch [4/20], Loss: 0.0087\n",
      "Epoch [4/20], Loss: 0.0031\n",
      "Epoch [4/20], Loss: 0.0203\n",
      "Epoch [4/20], Loss: 0.0026\n",
      "Epoch [4/20], Loss: 0.0030\n",
      "Epoch [5/20], Loss: 0.0040\n",
      "Epoch [5/20], Loss: 0.0155\n",
      "Epoch [5/20], Loss: 0.0036\n",
      "Epoch [5/20], Loss: 0.0020\n",
      "Epoch [5/20], Loss: 0.0014\n",
      "Epoch [5/20], Loss: 0.0077\n",
      "Epoch [5/20], Loss: 0.0070\n",
      "Epoch [5/20], Loss: 0.0051\n",
      "Epoch [5/20], Loss: 0.0080\n",
      "Epoch [5/20], Loss: 0.0062\n",
      "Epoch [6/20], Loss: 0.0028\n",
      "Epoch [6/20], Loss: 0.0067\n",
      "Epoch [6/20], Loss: 0.0161\n",
      "Epoch [6/20], Loss: 0.0050\n",
      "Epoch [6/20], Loss: 0.0037\n",
      "Epoch [6/20], Loss: 0.0067\n",
      "Epoch [6/20], Loss: 0.0038\n",
      "Epoch [6/20], Loss: 0.0047\n",
      "Epoch [6/20], Loss: 0.0064\n",
      "Epoch [6/20], Loss: 0.0025\n",
      "Epoch [7/20], Loss: 0.0077\n",
      "Epoch [7/20], Loss: 0.0048\n",
      "Epoch [7/20], Loss: 0.0053\n",
      "Epoch [7/20], Loss: 0.0016\n",
      "Epoch [7/20], Loss: 0.0025\n",
      "Epoch [7/20], Loss: 0.0028\n",
      "Epoch [7/20], Loss: 0.0151\n",
      "Epoch [7/20], Loss: 0.0069\n",
      "Epoch [7/20], Loss: 0.0049\n",
      "Epoch [7/20], Loss: 0.0040\n",
      "Epoch [8/20], Loss: 0.0022\n",
      "Epoch [8/20], Loss: 0.0058\n",
      "Epoch [8/20], Loss: 0.0026\n",
      "Epoch [8/20], Loss: 0.0132\n",
      "Epoch [8/20], Loss: 0.0021\n",
      "Epoch [8/20], Loss: 0.0076\n",
      "Epoch [8/20], Loss: 0.0047\n",
      "Epoch [8/20], Loss: 0.0057\n",
      "Epoch [8/20], Loss: 0.0084\n",
      "Epoch [8/20], Loss: 0.0044\n",
      "Epoch [9/20], Loss: 0.0125\n",
      "Epoch [9/20], Loss: 0.0056\n",
      "Epoch [9/20], Loss: 0.0051\n",
      "Epoch [9/20], Loss: 0.0064\n",
      "Epoch [9/20], Loss: 0.0027\n",
      "Epoch [9/20], Loss: 0.0032\n",
      "Epoch [9/20], Loss: 0.0020\n",
      "Epoch [9/20], Loss: 0.0070\n",
      "Epoch [9/20], Loss: 0.0051\n",
      "Epoch [9/20], Loss: 0.0038\n",
      "Epoch [10/20], Loss: 0.0029\n",
      "Epoch [10/20], Loss: 0.0019\n",
      "Epoch [10/20], Loss: 0.0061\n",
      "Epoch [10/20], Loss: 0.0028\n",
      "Epoch [10/20], Loss: 0.0042\n",
      "Epoch [10/20], Loss: 0.0053\n",
      "Epoch [10/20], Loss: 0.0076\n",
      "Epoch [10/20], Loss: 0.0066\n",
      "Epoch [10/20], Loss: 0.0023\n",
      "Epoch [10/20], Loss: 0.0135\n",
      "Epoch [11/20], Loss: 0.0055\n",
      "Epoch [11/20], Loss: 0.0030\n",
      "Epoch [11/20], Loss: 0.0063\n",
      "Epoch [11/20], Loss: 0.0072\n",
      "Epoch [11/20], Loss: 0.0020\n",
      "Epoch [11/20], Loss: 0.0109\n",
      "Epoch [11/20], Loss: 0.0064\n",
      "Epoch [11/20], Loss: 0.0049\n",
      "Epoch [11/20], Loss: 0.0042\n",
      "Epoch [11/20], Loss: 0.0023\n",
      "Epoch [12/20], Loss: 0.0103\n",
      "Epoch [12/20], Loss: 0.0030\n",
      "Epoch [12/20], Loss: 0.0014\n",
      "Epoch [12/20], Loss: 0.0062\n",
      "Epoch [12/20], Loss: 0.0051\n",
      "Epoch [12/20], Loss: 0.0038\n",
      "Epoch [12/20], Loss: 0.0074\n",
      "Epoch [12/20], Loss: 0.0062\n",
      "Epoch [12/20], Loss: 0.0023\n",
      "Epoch [12/20], Loss: 0.0043\n",
      "Epoch [13/20], Loss: 0.0055\n",
      "Epoch [13/20], Loss: 0.0033\n",
      "Epoch [13/20], Loss: 0.0038\n",
      "Epoch [13/20], Loss: 0.0104\n",
      "Epoch [13/20], Loss: 0.0020\n",
      "Epoch [13/20], Loss: 0.0070\n",
      "Epoch [13/20], Loss: 0.0019\n",
      "Epoch [13/20], Loss: 0.0049\n",
      "Epoch [13/20], Loss: 0.0071\n",
      "Epoch [13/20], Loss: 0.0035\n",
      "Epoch [14/20], Loss: 0.0046\n",
      "Epoch [14/20], Loss: 0.0014\n",
      "Epoch [14/20], Loss: 0.0062\n",
      "Epoch [14/20], Loss: 0.0064\n",
      "Epoch [14/20], Loss: 0.0020\n",
      "Epoch [14/20], Loss: 0.0026\n",
      "Epoch [14/20], Loss: 0.0107\n",
      "Epoch [14/20], Loss: 0.0039\n",
      "Epoch [14/20], Loss: 0.0050\n",
      "Epoch [14/20], Loss: 0.0064\n",
      "Epoch [15/20], Loss: 0.0047\n",
      "Epoch [15/20], Loss: 0.0073\n",
      "Epoch [15/20], Loss: 0.0030\n",
      "Epoch [15/20], Loss: 0.0048\n",
      "Epoch [15/20], Loss: 0.0094\n",
      "Epoch [15/20], Loss: 0.0038\n",
      "Epoch [15/20], Loss: 0.0028\n",
      "Epoch [15/20], Loss: 0.0062\n",
      "Epoch [15/20], Loss: 0.0048\n",
      "Epoch [15/20], Loss: 0.0023\n",
      "Epoch [16/20], Loss: 0.0034\n",
      "Epoch [16/20], Loss: 0.0028\n",
      "Epoch [16/20], Loss: 0.0070\n",
      "Epoch [16/20], Loss: 0.0062\n",
      "Epoch [16/20], Loss: 0.0057\n",
      "Epoch [16/20], Loss: 0.0042\n",
      "Epoch [16/20], Loss: 0.0017\n",
      "Epoch [16/20], Loss: 0.0016\n",
      "Epoch [16/20], Loss: 0.0099\n",
      "Epoch [16/20], Loss: 0.0053\n",
      "Epoch [17/20], Loss: 0.0083\n",
      "Epoch [17/20], Loss: 0.0043\n",
      "Epoch [17/20], Loss: 0.0044\n",
      "Epoch [17/20], Loss: 0.0019\n",
      "Epoch [17/20], Loss: 0.0061\n",
      "Epoch [17/20], Loss: 0.0047\n",
      "Epoch [17/20], Loss: 0.0065\n",
      "Epoch [17/20], Loss: 0.0015\n",
      "Epoch [17/20], Loss: 0.0073\n",
      "Epoch [17/20], Loss: 0.0033\n",
      "Epoch [18/20], Loss: 0.0048\n",
      "Epoch [18/20], Loss: 0.0021\n",
      "Epoch [18/20], Loss: 0.0023\n",
      "Epoch [18/20], Loss: 0.0012\n",
      "Epoch [18/20], Loss: 0.0058\n",
      "Epoch [18/20], Loss: 0.0063\n",
      "Epoch [18/20], Loss: 0.0038\n",
      "Epoch [18/20], Loss: 0.0053\n",
      "Epoch [18/20], Loss: 0.0060\n",
      "Epoch [18/20], Loss: 0.0106\n",
      "Epoch [19/20], Loss: 0.0058\n",
      "Epoch [19/20], Loss: 0.0040\n",
      "Epoch [19/20], Loss: 0.0069\n",
      "Epoch [19/20], Loss: 0.0056\n",
      "Epoch [19/20], Loss: 0.0035\n",
      "Epoch [19/20], Loss: 0.0032\n",
      "Epoch [19/20], Loss: 0.0022\n",
      "Epoch [19/20], Loss: 0.0022\n",
      "Epoch [19/20], Loss: 0.0053\n",
      "Epoch [19/20], Loss: 0.0088\n",
      "Epoch [20/20], Loss: 0.0045\n",
      "Epoch [20/20], Loss: 0.0061\n",
      "Epoch [20/20], Loss: 0.0073\n",
      "Epoch [20/20], Loss: 0.0049\n",
      "Epoch [20/20], Loss: 0.0078\n",
      "Epoch [20/20], Loss: 0.0035\n",
      "Epoch [20/20], Loss: 0.0062\n",
      "Epoch [20/20], Loss: 0.0017\n",
      "Epoch [20/20], Loss: 0.0036\n",
      "Epoch [20/20], Loss: 0.0018\n"
     ]
    }
   ],
   "source": [
    "# train on all but shuffle\n",
    "resall_shuffle = train_on_all( Net().cuda(),train_loader,test_loader,num_epochs=num_epochs,shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EWC 1e-06\n",
      "Epoch [1/20], Loss: 0.1047\n",
      "Epoch [2/20], Loss: 0.0142\n",
      "Epoch [3/20], Loss: 0.0110\n",
      "Epoch [4/20], Loss: 0.0094\n",
      "Epoch [5/20], Loss: 0.0084\n",
      "Epoch [6/20], Loss: 0.0076\n",
      "Epoch [7/20], Loss: 0.0070\n",
      "Epoch [8/20], Loss: 0.0066\n",
      "Epoch [9/20], Loss: 0.0062\n",
      "Epoch [10/20], Loss: 0.0059\n",
      "Epoch [11/20], Loss: 0.0056\n",
      "Epoch [12/20], Loss: 0.0053\n",
      "Epoch [13/20], Loss: 0.0051\n",
      "Epoch [14/20], Loss: 0.0049\n",
      "Epoch [15/20], Loss: 0.0048\n",
      "Epoch [16/20], Loss: 0.0046\n",
      "Epoch [17/20], Loss: 0.0045\n",
      "Epoch [18/20], Loss: 0.0043\n",
      "Epoch [19/20], Loss: 0.0042\n",
      "Epoch [20/20], Loss: 0.0041\n",
      "generate task data..\n",
      "task data norm and number entries: tensor(297162., device='cuda:0') torch.Size([397510])\n",
      "..done\n",
      "test performance :  [98.26999664  0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(98.2700, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.1014\n",
      "Epoch [2/20], Loss: 0.0151\n",
      "Epoch [3/20], Loss: 0.0115\n",
      "Epoch [4/20], Loss: 0.0100\n",
      "Epoch [5/20], Loss: 0.0091\n",
      "Epoch [6/20], Loss: 0.0085\n",
      "Epoch [7/20], Loss: 0.0080\n",
      "Epoch [8/20], Loss: 0.0077\n",
      "Epoch [9/20], Loss: 0.0074\n",
      "Epoch [10/20], Loss: 0.0072\n",
      "Epoch [11/20], Loss: 0.0070\n",
      "Epoch [12/20], Loss: 0.0069\n",
      "Epoch [13/20], Loss: 0.0067\n",
      "Epoch [14/20], Loss: 0.0066\n",
      "Epoch [15/20], Loss: 0.0065\n",
      "Epoch [16/20], Loss: 0.0064\n",
      "Epoch [17/20], Loss: 0.0064\n",
      "Epoch [18/20], Loss: 0.0064\n",
      "Epoch [19/20], Loss: 0.0063\n",
      "Epoch [20/20], Loss: 0.0063\n",
      "update data..\n",
      "task data norm and number entries: tensor(332695.2500, device='cuda:0') torch.Size([397510])\n",
      "..done\n",
      "test performance :  [98.26999664 97.98500061  0.          0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(98.2200, device='cuda:0'), tensor(97.7500, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0951\n",
      "Epoch [2/20], Loss: 0.0174\n",
      "Epoch [3/20], Loss: 0.0140\n",
      "Epoch [4/20], Loss: 0.0123\n",
      "Epoch [5/20], Loss: 0.0113\n",
      "Epoch [6/20], Loss: 0.0105\n",
      "Epoch [7/20], Loss: 0.0100\n",
      "Epoch [8/20], Loss: 0.0095\n",
      "Epoch [9/20], Loss: 0.0091\n",
      "Epoch [10/20], Loss: 0.0088\n",
      "Epoch [11/20], Loss: 0.0086\n",
      "Epoch [12/20], Loss: 0.0084\n",
      "Epoch [13/20], Loss: 0.0082\n",
      "Epoch [14/20], Loss: 0.0081\n",
      "Epoch [15/20], Loss: 0.0079\n",
      "Epoch [16/20], Loss: 0.0078\n",
      "Epoch [17/20], Loss: 0.0077\n",
      "Epoch [18/20], Loss: 0.0076\n",
      "Epoch [19/20], Loss: 0.0076\n",
      "Epoch [20/20], Loss: 0.0075\n",
      "update data..\n",
      "task data norm and number entries: tensor(370599.2188, device='cuda:0') torch.Size([397510])\n",
      "..done\n",
      "test performance :  [98.26999664 97.98500061 97.70666504  0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(97.8000, device='cuda:0'), tensor(97.7300, device='cuda:0'), tensor(97.5900, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0956\n",
      "Epoch [2/20], Loss: 0.0169\n",
      "Epoch [3/20], Loss: 0.0137\n",
      "Epoch [4/20], Loss: 0.0122\n",
      "Epoch [5/20], Loss: 0.0116\n",
      "Epoch [6/20], Loss: 0.0113\n",
      "Epoch [7/20], Loss: 0.0110\n",
      "Epoch [8/20], Loss: 0.0107\n",
      "Epoch [9/20], Loss: 0.0105\n",
      "Epoch [10/20], Loss: 0.0103\n",
      "Epoch [11/20], Loss: 0.0101\n",
      "Epoch [12/20], Loss: 0.0100\n",
      "Epoch [13/20], Loss: 0.0099\n",
      "Epoch [14/20], Loss: 0.0098\n",
      "Epoch [15/20], Loss: 0.0098\n",
      "Epoch [16/20], Loss: 0.0097\n",
      "Epoch [17/20], Loss: 0.0096\n",
      "Epoch [18/20], Loss: 0.0095\n",
      "Epoch [19/20], Loss: 0.0094\n",
      "Epoch [20/20], Loss: 0.0094\n",
      "update data..\n",
      "task data norm and number entries: tensor(388491.5000, device='cuda:0') torch.Size([397510])\n",
      "..done\n",
      "test performance :  [98.26999664 97.98500061 97.70666504 97.38999939  0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(97.0500, device='cuda:0'), tensor(97.4900, device='cuda:0'), tensor(97.5800, device='cuda:0'), tensor(97.4400, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0949\n",
      "Epoch [2/20], Loss: 0.0186\n",
      "Epoch [3/20], Loss: 0.0155\n",
      "Epoch [4/20], Loss: 0.0140\n",
      "Epoch [5/20], Loss: 0.0132\n",
      "Epoch [6/20], Loss: 0.0126\n",
      "Epoch [7/20], Loss: 0.0122\n",
      "Epoch [8/20], Loss: 0.0119\n",
      "Epoch [9/20], Loss: 0.0116\n",
      "Epoch [10/20], Loss: 0.0113\n",
      "Epoch [11/20], Loss: 0.0111\n",
      "Epoch [12/20], Loss: 0.0109\n",
      "Epoch [13/20], Loss: 0.0108\n",
      "Epoch [14/20], Loss: 0.0107\n",
      "Epoch [15/20], Loss: 0.0106\n",
      "Epoch [16/20], Loss: 0.0105\n",
      "Epoch [17/20], Loss: 0.0104\n",
      "Epoch [18/20], Loss: 0.0103\n",
      "Epoch [19/20], Loss: 0.0102\n",
      "Epoch [20/20], Loss: 0.0101\n",
      "update data..\n",
      "task data norm and number entries: tensor(392454.2812, device='cuda:0') torch.Size([397510])\n",
      "..done\n",
      "test performance :  [98.26999664 97.98500061 97.70666504 97.38999939 97.22000122  0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(96.7000, device='cuda:0'), tensor(97.3600, device='cuda:0'), tensor(97.3100, device='cuda:0'), tensor(97.4300, device='cuda:0'), tensor(97.3000, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0943\n",
      "Epoch [2/20], Loss: 0.0189\n",
      "Epoch [3/20], Loss: 0.0159\n",
      "Epoch [4/20], Loss: 0.0148\n",
      "Epoch [5/20], Loss: 0.0141\n",
      "Epoch [6/20], Loss: 0.0136\n",
      "Epoch [7/20], Loss: 0.0132\n",
      "Epoch [8/20], Loss: 0.0128\n",
      "Epoch [9/20], Loss: 0.0125\n",
      "Epoch [10/20], Loss: 0.0122\n",
      "Epoch [11/20], Loss: 0.0120\n",
      "Epoch [12/20], Loss: 0.0117\n",
      "Epoch [13/20], Loss: 0.0115\n",
      "Epoch [14/20], Loss: 0.0112\n",
      "Epoch [15/20], Loss: 0.0110\n",
      "Epoch [16/20], Loss: 0.0108\n",
      "Epoch [17/20], Loss: 0.0106\n",
      "Epoch [18/20], Loss: 0.0105\n",
      "Epoch [19/20], Loss: 0.0104\n",
      "Epoch [20/20], Loss: 0.0103\n",
      "update data..\n",
      "task data norm and number entries: tensor(351190.9688, device='cuda:0') torch.Size([397510])\n",
      "..done\n",
      "test performance :  [98.26999664 97.98500061 97.70666504 97.38999939 97.22000122 96.92333984\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(96.0500, device='cuda:0'), tensor(96.9100, device='cuda:0'), tensor(96.9500, device='cuda:0'), tensor(97.2100, device='cuda:0'), tensor(97.2800, device='cuda:0'), tensor(97.1400, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0972\n",
      "Epoch [2/20], Loss: 0.0187\n",
      "Epoch [3/20], Loss: 0.0163\n",
      "Epoch [4/20], Loss: 0.0152\n",
      "Epoch [5/20], Loss: 0.0144\n",
      "Epoch [6/20], Loss: 0.0138\n",
      "Epoch [7/20], Loss: 0.0133\n",
      "Epoch [8/20], Loss: 0.0130\n",
      "Epoch [9/20], Loss: 0.0127\n",
      "Epoch [10/20], Loss: 0.0123\n",
      "Epoch [11/20], Loss: 0.0121\n",
      "Epoch [12/20], Loss: 0.0119\n",
      "Epoch [13/20], Loss: 0.0117\n",
      "Epoch [14/20], Loss: 0.0116\n",
      "Epoch [15/20], Loss: 0.0114\n",
      "Epoch [16/20], Loss: 0.0113\n",
      "Epoch [17/20], Loss: 0.0112\n",
      "Epoch [18/20], Loss: 0.0111\n",
      "Epoch [19/20], Loss: 0.0110\n",
      "Epoch [20/20], Loss: 0.0109\n",
      "update data..\n",
      "task data norm and number entries: tensor(335459.7500, device='cuda:0') torch.Size([397510])\n",
      "..done\n",
      "test performance :  [98.26999664 97.98500061 97.70666504 97.38999939 97.22000122 96.92333984\n",
      " 96.74000549  0.          0.          0.        ]\n",
      "individual errors:  [tensor(95.5500, device='cuda:0'), tensor(96.6700, device='cuda:0'), tensor(97.0200, device='cuda:0'), tensor(96.8900, device='cuda:0'), tensor(97.0800, device='cuda:0'), tensor(97., device='cuda:0'), tensor(96.9700, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0964\n",
      "Epoch [2/20], Loss: 0.0199\n",
      "Epoch [3/20], Loss: 0.0172\n",
      "Epoch [4/20], Loss: 0.0160\n",
      "Epoch [5/20], Loss: 0.0153\n",
      "Epoch [6/20], Loss: 0.0148\n",
      "Epoch [7/20], Loss: 0.0144\n",
      "Epoch [8/20], Loss: 0.0140\n",
      "Epoch [9/20], Loss: 0.0136\n",
      "Epoch [10/20], Loss: 0.0134\n",
      "Epoch [11/20], Loss: 0.0131\n",
      "Epoch [12/20], Loss: 0.0130\n",
      "Epoch [13/20], Loss: 0.0128\n",
      "Epoch [14/20], Loss: 0.0126\n",
      "Epoch [15/20], Loss: 0.0124\n",
      "Epoch [16/20], Loss: 0.0122\n",
      "Epoch [17/20], Loss: 0.0120\n",
      "Epoch [18/20], Loss: 0.0119\n",
      "Epoch [19/20], Loss: 0.0118\n",
      "Epoch [20/20], Loss: 0.0117\n",
      "update data..\n",
      "task data norm and number entries: tensor(406612.0312, device='cuda:0') torch.Size([397510])\n",
      "..done\n",
      "test performance :  [98.26999664 97.98500061 97.70666504 97.38999939 97.22000122 96.92333984\n",
      " 96.74000549 96.60500336  0.          0.        ]\n",
      "individual errors:  [tensor(95.5300, device='cuda:0'), tensor(96.5300, device='cuda:0'), tensor(96.4700, device='cuda:0'), tensor(96.7900, device='cuda:0'), tensor(96.8400, device='cuda:0'), tensor(96.8700, device='cuda:0'), tensor(96.8700, device='cuda:0'), tensor(96.9400, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0977\n",
      "Epoch [2/20], Loss: 0.0203\n",
      "Epoch [3/20], Loss: 0.0182\n",
      "Epoch [4/20], Loss: 0.0169\n",
      "Epoch [5/20], Loss: 0.0159\n",
      "Epoch [6/20], Loss: 0.0154\n",
      "Epoch [7/20], Loss: 0.0151\n",
      "Epoch [8/20], Loss: 0.0147\n",
      "Epoch [9/20], Loss: 0.0145\n",
      "Epoch [10/20], Loss: 0.0143\n",
      "Epoch [11/20], Loss: 0.0141\n",
      "Epoch [12/20], Loss: 0.0140\n",
      "Epoch [13/20], Loss: 0.0138\n",
      "Epoch [14/20], Loss: 0.0137\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [15/20], Loss: 0.0136\n",
      "Epoch [16/20], Loss: 0.0135\n",
      "Epoch [17/20], Loss: 0.0135\n",
      "Epoch [18/20], Loss: 0.0134\n",
      "Epoch [19/20], Loss: 0.0133\n",
      "Epoch [20/20], Loss: 0.0133\n",
      "update data..\n",
      "task data norm and number entries: tensor(347286.4062, device='cuda:0') torch.Size([397510])\n",
      "..done\n",
      "test performance :  [98.26999664 97.98500061 97.70666504 97.38999939 97.22000122 96.92333984\n",
      " 96.74000549 96.60500336 96.3188858   0.        ]\n",
      "individual errors:  [tensor(95.0200, device='cuda:0'), tensor(96.0200, device='cuda:0'), tensor(96.0700, device='cuda:0'), tensor(96.4700, device='cuda:0'), tensor(96.6000, device='cuda:0'), tensor(96.5200, device='cuda:0'), tensor(96.6500, device='cuda:0'), tensor(96.7500, device='cuda:0'), tensor(96.7700, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0965\n",
      "Epoch [2/20], Loss: 0.0204\n",
      "Epoch [3/20], Loss: 0.0176\n",
      "Epoch [4/20], Loss: 0.0164\n",
      "Epoch [5/20], Loss: 0.0157\n",
      "Epoch [6/20], Loss: 0.0152\n",
      "Epoch [7/20], Loss: 0.0149\n",
      "Epoch [8/20], Loss: 0.0146\n",
      "Epoch [9/20], Loss: 0.0145\n",
      "Epoch [10/20], Loss: 0.0142\n",
      "Epoch [11/20], Loss: 0.0140\n",
      "Epoch [12/20], Loss: 0.0138\n",
      "Epoch [13/20], Loss: 0.0136\n",
      "Epoch [14/20], Loss: 0.0135\n",
      "Epoch [15/20], Loss: 0.0134\n",
      "Epoch [16/20], Loss: 0.0132\n",
      "Epoch [17/20], Loss: 0.0131\n",
      "Epoch [18/20], Loss: 0.0131\n",
      "Epoch [19/20], Loss: 0.0129\n",
      "Epoch [20/20], Loss: 0.0128\n",
      "test performance :  [98.26999664 97.98500061 97.70666504 97.38999939 97.22000122 96.92333984\n",
      " 96.74000549 96.60500336 96.3188858  96.04100037]\n",
      "individual errors:  [tensor(93.9400, device='cuda:0'), tensor(95.5300, device='cuda:0'), tensor(95.9200, device='cuda:0'), tensor(96.2200, device='cuda:0'), tensor(96.3000, device='cuda:0'), tensor(96.1200, device='cuda:0'), tensor(96.5900, device='cuda:0'), tensor(96.4400, device='cuda:0'), tensor(96.5900, device='cuda:0'), tensor(96.7600, device='cuda:0')]\n"
     ]
    }
   ],
   "source": [
    "lam = 1e-6\n",
    "resewc = run_simulation( Net().cuda(),train_loader,test_loader,EWC(lam=lam),num_epochs=num_epochs )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 1e-06\n",
      "Epoch [1/20], Loss: 0.1016\n",
      "Epoch [2/20], Loss: 0.0144\n",
      "Epoch [3/20], Loss: 0.0107\n",
      "Epoch [4/20], Loss: 0.0090\n",
      "Epoch [5/20], Loss: 0.0080\n",
      "Epoch [6/20], Loss: 0.0073\n",
      "Epoch [7/20], Loss: 0.0067\n",
      "Epoch [8/20], Loss: 0.0064\n",
      "Epoch [9/20], Loss: 0.0060\n",
      "Epoch [10/20], Loss: 0.0058\n",
      "Epoch [11/20], Loss: 0.0055\n",
      "Epoch [12/20], Loss: 0.0053\n",
      "Epoch [13/20], Loss: 0.0051\n",
      "Epoch [14/20], Loss: 0.0049\n",
      "Epoch [15/20], Loss: 0.0048\n",
      "Epoch [16/20], Loss: 0.0046\n",
      "Epoch [17/20], Loss: 0.0045\n",
      "Epoch [18/20], Loss: 0.0044\n",
      "Epoch [19/20], Loss: 0.0042\n",
      "Epoch [20/20], Loss: 0.0041\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAVcElEQVR4nO3df6xf9X3f8edrdqFZugQTPMuzYXYSNxNBqwMWcdSmYqEJhlUxmVhmaypuhuJEAS3ZKjVm/YMsDRJsS1mQEloSvJgqxVBIipU4pZ6LGk0ahEtB/KZcHFJsGewCga10pE7e++P7ucmXm3s/Nvde32t8nw/p6J7zPp/POZ9z/LVfnB/fS6oKSZIm8w/megCSpGObQSFJ6jIoJEldBoUkqcugkCR1LZzrAcy0U045pVasWDHXw5Ck15V77733b6pq8UTrjrugWLFiBSMjI3M9DEl6XUny/cnWeetJktRlUEiSugwKSVKXQSFJ6jpsUCTZmuRAkoeGajcnub9NTyW5v9VXJPm7oXW/P9TnrCQPJhlNcm2StPrJSXYleaL9XNTqae1GkzyQ5MwZP3pJ0mEdyRXFV4F1w4Wq+jdVtbqqVgO3AV8fWv3k2Lqq+vhQ/Trgo8CqNo1tcwuwu6pWAbvbMsD5Q203t/6SpFl22KCoqu8Az0+0rl0VfBi4qbeNJEuBN1XVXTX4dbU3Ahe21euBbW1+27j6jTVwF3BS244kaRZN9xnFe4Fnq+qJodrKJPcl+Ysk7221ZcDeoTZ7Ww1gSVXtb/PPAEuG+jw9SR9J0iyZ7hfuNvLqq4n9wGlV9VySs4A/SfLOI91YVVWS1/w/yEiymcHtKU477bTX2l2S1DHlK4okC4F/Bdw8VquqV6rquTZ/L/Ak8IvAPmD5UPflrQbw7NgtpfbzQKvvA06dpM+rVNX1VbWmqtYsXjzhN9CPyIot35pyX0k6Xk3n1tOvAY9V1U9uKSVZnGRBm38rgwfRe9qtpZeSrG3PNS4Gbm/ddgCb2vymcfWL29tPa4EXh25RSZJmyZG8HnsT8L+BdyTZm+SStmoDP/sQ+1eBB9rrsrcCH6+qsQfhnwC+AowyuNL4dqtfBbw/yRMMwueqVt8J7Gntv9z6S5Jm2WGfUVTVxknqvzlB7TYGr8tO1H4EOGOC+nPAuRPUC7j0cOOTJB1dfjNbktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV2HDYokW5McSPLQUO0zSfYlub9NFwytuzzJaJLHk5w3VF/XaqNJtgzVVya5u9VvTnJCq5/Ylkfb+hUzdtSSpCN2JFcUXwXWTVC/pqpWt2knQJLTgQ3AO1ufLyVZkGQB8EXgfOB0YGNrC3B129bbgReAS1r9EuCFVr+mtZMkzbLDBkVVfQd4/gi3tx7YXlWvVNX3gFHg7DaNVtWeqvohsB1YnyTA+4BbW/9twIVD29rW5m8Fzm3tJUmzaDrPKC5L8kC7NbWo1ZYBTw+12dtqk9XfAvygqg6Nq79qW239i639z0iyOclIkpGDBw9O45AkSeNNNSiuA94GrAb2A5+fqQFNRVVdX1VrqmrN4sWL53IoknTcmVJQVNWzVfWjqvox8GUGt5YA9gGnDjVd3mqT1Z8DTkqycFz9Vdtq69/c2kuSZtGUgiLJ0qHFDwFjb0TtADa0N5ZWAquA7wL3AKvaG04nMHjgvaOqCrgTuKj13wTcPrStTW3+IuDPW3tJ0ixaeLgGSW4CzgFOSbIXuAI4J8lqoICngI8BVNXDSW4BHgEOAZdW1Y/adi4D7gAWAFur6uG2i08D25N8DrgPuKHVbwD+MMkog4fpG6Z7sJKk1+6wQVFVGyco3zBBbaz9lcCVE9R3AjsnqO/hp7euhuv/D/jXhxufJOno8pvZkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeo6bFAk2ZrkQJKHhmr/NcljSR5I8o0kJ7X6iiR/l+T+Nv3+UJ+zkjyYZDTJtUnS6icn2ZXkifZzUauntRtt+zlzxo9eknRYR3JF8VVg3bjaLuCMqvrnwF8Blw+te7KqVrfp40P164CPAqvaNLbNLcDuqloF7G7LAOcPtd3c+kuSZtlhg6KqvgM8P672Z1V1qC3eBSzvbSPJUuBNVXVXVRVwI3BhW70e2Nbmt42r31gDdwEnte1IkmbRTDyj+HfAt4eWVya5L8lfJHlvqy0D9g612dtqAEuqan+bfwZYMtTn6Un6vEqSzUlGkowcPHhwGociSRpvWkGR5HeAQ8DXWmk/cFpVvQv4j8AfJXnTkW6vXW3Uax1HVV1fVWuqas3ixYtfa3dJUsfCqXZM8pvArwPntn/gqapXgFfa/L1JngR+EdjHq29PLW81gGeTLK2q/e3W0oFW3wecOkkfSdIsmdIVRZJ1wG8DH6yql4fqi5MsaPNvZfAgek+7tfRSkrXtbaeLgdtbtx3Apja/aVz94vb201rgxaFbVJKkWXLYK4okNwHnAKck2QtcweAtpxOBXe0t17vaG06/Cnw2yd8DPwY+XlVjD8I/weANqjcweKYx9lzjKuCWJJcA3wc+3Oo7gQuAUeBl4CPTOVBJ0tQcNiiqauME5RsmaXsbcNsk60aAMyaoPwecO0G9gEsPNz5J0tHlN7MlSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1HVEQZFka5IDSR4aqp2cZFeSJ9rPRa2eJNcmGU3yQJIzh/psau2fSLJpqH5Wkgdbn2uTpLcPSdLsOdIriq8C68bVtgC7q2oVsLstA5wPrGrTZuA6GPyjD1wBvBs4G7hi6B/+64CPDvVbd5h9SJJmyREFRVV9B3h+XHk9sK3NbwMuHKrfWAN3ASclWQqcB+yqquer6gVgF7CurXtTVd1VVQXcOG5bE+1DkjRLpvOMYklV7W/zzwBL2vwy4OmhdntbrVffO0G9t49XSbI5yUiSkYMHD07xcCRJE5mRh9ntSqBmYltT2UdVXV9Va6pqzeLFi4/mMCRp3plOUDzbbhvRfh5o9X3AqUPtlrdar758gnpvH5KkWTKdoNgBjL25tAm4fah+cXv7aS3wYrt9dAfwgSSL2kPsDwB3tHUvJVnb3na6eNy2JtqHJGmWLDySRkluAs4BTkmyl8HbS1cBtyS5BPg+8OHWfCdwATAKvAx8BKCqnk/yu8A9rd1nq2rsAfknGLxZ9Qbg222isw9J0iw5oqCoqo2TrDp3grYFXDrJdrYCWyeojwBnTFB/bqJ9SJJmj9/MliR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSuqYcFEnekeT+oemlJJ9K8pkk+4bqFwz1uTzJaJLHk5w3VF/XaqNJtgzVVya5u9VvTnLC1A9VkjQVUw6Kqnq8qlZX1WrgLOBl4Btt9TVj66pqJ0CS04ENwDuBdcCXkixIsgD4InA+cDqwsbUFuLpt6+3AC8AlUx2vJGlqZurW07nAk1X1/U6b9cD2qnqlqr4HjAJnt2m0qvZU1Q+B7cD6JAHeB9za+m8DLpyh8UqSjtBMBcUG4Kah5cuSPJBka5JFrbYMeHqozd5Wm6z+FuAHVXVoXF2SNIumHRTtucEHgT9upeuAtwGrgf3A56e7jyMYw+YkI0lGDh48eLR3J0nzykxcUZwP/GVVPQtQVc9W1Y+q6sfAlxncWgLYB5w61G95q01Wfw44KcnCcfWfUVXXV9WaqlqzePHiGTgkSdKYmQiKjQzddkqydGjdh4CH2vwOYEOSE5OsBFYB3wXuAVa1N5xOYHAba0dVFXAncFHrvwm4fQbGK0l6DRYevsnkkrwReD/wsaHyf0myGijgqbF1VfVwkluAR4BDwKVV9aO2ncuAO4AFwNaqerht69PA9iSfA+4DbpjOeCVJr920gqKq/pbBQ+fh2m902l8JXDlBfSewc4L6Hn5660qSNAf8ZrYkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSuqYdFEmeSvJgkvuTjLTayUl2JXmi/VzU6klybZLRJA8kOXNoO5ta+yeSbBqqn9W2P9r6ZrpjliQduZm6ovgXVbW6qta05S3A7qpaBexuywDnA6vatBm4DgbBAlwBvBs4G7hiLFxam48O9Vs3Q2OWJB2Bo3XraT2wrc1vAy4cqt9YA3cBJyVZCpwH7Kqq56vqBWAXsK6te1NV3VVVBdw4tC1J0iyYiaAo4M+S3Jtkc6stqar9bf4ZYEmbXwY8PdR3b6v16nsnqL9Kks1JRpKMHDx4cLrHI0kasnAGtvErVbUvyT8GdiV5bHhlVVWSmoH9TKqqrgeuB1izZs1R3ZckzTfTvqKoqn3t5wHgGwyeMTzbbhvRfh5ozfcBpw51X95qvfryCeqSpFkyraBI8sYk/2hsHvgA8BCwAxh7c2kTcHub3wFc3N5+Wgu82G5R3QF8IMmi9hD7A8Adbd1LSda2t50uHtqWJGkWTPfW0xLgG+2N1YXAH1XVnya5B7glySXA94EPt/Y7gQuAUeBl4CMAVfV8kt8F7mntPltVz7f5TwBfBd4AfLtNkqRZMq2gqKo9wC9NUH8OOHeCegGXTrKtrcDWCeojwBnTGackaer8ZrYkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSuqYcFElOTXJnkkeSPJzkk63+mST7ktzfpguG+lyeZDTJ40nOG6qva7XRJFuG6iuT3N3qNyc5YarjlSRNzXSuKA4Bv1VVpwNrgUuTnN7WXVNVq9u0E6Ct2wC8E1gHfCnJgiQLgC8C5wOnAxuHtnN129bbgReAS6YxXknSFEw5KKpqf1X9ZZv/P8CjwLJOl/XA9qp6paq+B4wCZ7dptKr2VNUPge3A+iQB3gfc2vpvAy6c6nglSVMzI88okqwA3gXc3UqXJXkgydYki1ptGfD0ULe9rTZZ/S3AD6rq0Lj6RPvfnGQkycjBgwdn4pAkSc20gyLJLwC3AZ+qqpeA64C3AauB/cDnp7uPw6mq66tqTVWtWbx48dHenSTNKwun0znJzzEIia9V1dcBqurZofVfBr7ZFvcBpw51X95qTFJ/DjgpycJ2VTHcXpI0S6bz1lOAG4BHq+r3hupLh5p9CHioze8ANiQ5MclKYBXwXeAeYFV7w+kEBg+8d1RVAXcCF7X+m4DbpzpeSdLUTOeK4peB3wAeTHJ/q/0nBm8trQYKeAr4GEBVPZzkFuARBm9MXVpVPwJIchlwB7AA2FpVD7ftfRrYnuRzwH0MgkmSNIumHBRV9b+ATLBqZ6fPlcCVE9R3TtSvqvYweCtKkjRH/Ga2JKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQTHOii3fmushSNIxxaCQJHUZFJKkLoNCktRlUEzA5xSS9FMGhSSpy6CYxIot3/LKQpKAhXM9gMNJsg74ArAA+EpVXTWb+x8fFk9d9S9nc/eSNOeO6aBIsgD4IvB+YC9wT5IdVfXIXI1pqlcZBoyk16tjOiiAs4HRqtoDkGQ7sB6Ys6CYqtm+jWUwSZopx3pQLAOeHlreC7x7fKMkm4HNbfH/Jnl8ivs7BfibKfY9puTqaXU/bs7DNHkeBjwPA8f7efink6041oPiiFTV9cD1091OkpGqWjMDQ3pd8zwMeB4GPA8D8/k8HOtvPe0DTh1aXt5qkqRZcqwHxT3AqiQrk5wAbAB2zPGYJGleOaZvPVXVoSSXAXcweD12a1U9fBR3Oe3bV8cJz8OA52HA8zAwb89DqmquxyBJOoYd67eeJElzzKCQJHUZFE2SdUkeTzKaZMtcj+doSnJqkjuTPJLk4SSfbPWTk+xK8kT7uajVk+Tadm4eSHLm3B7BzEmyIMl9Sb7Zllcmubsd683tJQqSnNiWR9v6FXM68BmU5KQktyZ5LMmjSd4zTz8L/6H9fXgoyU1Jfn4+fh4mYlDwql8Vcj5wOrAxyelzO6qj6hDwW1V1OrAWuLQd7xZgd1WtAna3ZRicl1Vt2gxcN/tDPmo+CTw6tHw1cE1VvR14Abik1S8BXmj1a1q748UXgD+tqn8G/BKD8zGvPgtJlgH/HlhTVWcweHlmA/Pz8/CzqmreT8B7gDuGli8HLp/rcc3i8d/O4PdpPQ4sbbWlwONt/g+AjUPtf9Lu9Twx+F7ObuB9wDeBMPjm7cLxnwsGb969p80vbO0y18cwA+fgzcD3xh/LPPwsjP0WiJPbn+83gfPm2+dhsskrioGJflXIsjkay6xql8zvAu4GllTV/rbqGWBJmz9ez89/B34b+HFbfgvwg6o61JaHj/Mn56Ctf7G1f71bCRwE/ke7BfeVJG9knn0Wqmof8N+Avwb2M/jzvZf593mYkEExjyX5BeA24FNV9dLwuhr8p9Jx++50kl8HDlTVvXM9ljm2EDgTuK6q3gX8LT+9zQQc/58FgPYMZj2D4PwnwBuBdXM6qGOIQTEw735VSJKfYxASX6uqr7fys0mWtvVLgQOtfjyen18GPpjkKWA7g9tPXwBOSjL2RdTh4/zJOWjr3ww8N5sDPkr2Anur6u62fCuD4JhPnwWAXwO+V1UHq+rvga8z+IzMt8/DhAyKgXn1q0KSBLgBeLSqfm9o1Q5gU5vfxODZxVj94vbGy1rgxaHbEq9LVXV5VS2vqhUM/rz/vKr+LXAncFFrNv4cjJ2bi1r71/1/ZVfVM8DTSd7RSucy+DX+8+az0Pw1sDbJP2x/P8bOw7z6PExqrh+SHCsTcAHwV8CTwO/M9XiO8rH+CoNbCQ8A97fpAgb3WHcDTwD/Ezi5tQ+Dt8KeBB5k8GbInB/HDJ6Pc4Bvtvm3At8FRoE/Bk5s9Z9vy6Nt/VvnetwzePyrgZH2efgTYNF8/CwA/xl4DHgI+EPgxPn4eZho8ld4SJK6vPUkSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6/j/D0LF3cr5ZsAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQaklEQVR4nO3df6xkZX3H8fenIGLRurtyu9my2rtGguEfgd5QiMa0IIhigCaEQEy7WppN+itam9il/mXSP5a28UeTRt2AdtOggghdAqmWrpimSbN6EURgWXfBRZfsslcr/kyq6Ld/zLnLcJndmbv3zp37yPuVTOac55zZ+c5z737uM885ZyZVhSSpPb826QIkSSfGAJekRhngktQoA1ySGmWAS1KjTl7JJzv99NNrenp6JZ9Skpp3//33f7eqpha2r2iAT09PMzs7u5JPKUnNS/LkoHanUCSpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEreiVm66a33nPc7Qe2Xb5ClUiSI3BJatbQAE9yVpIH+24/TPLeJOuS3JtkX3e/diUKliT1DA3wqtpbVedU1TnA7wA/Be4EtgK7qupMYFe3LklaIYudQrkYeLyqngSuBHZ07TuAq5axLknSEIsN8GuBz3TL66vqULd8GFg/6AFJtiSZTTI7Nzd3gmVKkhYaOcCTnAJcAXxu4baqKqAGPa6qtlfVTFXNTE294PPIJUknaDEj8LcBX6uqp7v1p5NsAOjujyx3cZKkY1vMeeDX8dz0CcBdwGZgW3e/cxnr+pU07Dxy8FxySaMbaQSe5DTgEuCOvuZtwCVJ9gFv6dYlSStkpBF4Vf0EeNWCtu/ROytFkjQBXokpSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGjfqt9GuS3J7ksSR7klyYZF2Se5Ps6+7XjrtYSdJzRh2BfxT4QlW9HngDsAfYCuyqqjOBXd26JGmFDA3wJK8E3gzcDFBVP6uqZ4ArgR3dbjuAq8ZToiRpkFFG4JuAOeBTSR5IclOS04D1VXWo2+cwsH7Qg5NsSTKbZHZubm55qpYkjRTgJwPnAR+rqnOBn7BguqSqCqhBD66q7VU1U1UzU1NTS61XktQZJcAPAgerane3fju9QH86yQaA7v7IeEqUJA1y8rAdqupwku8kOauq9gIXA492t83Atu5+51grbcD01nsmXYKkF5GhAd75S+CWJKcATwDvpjd6vy3J9cCTwDXjKVGSNMhIAV5VDwIzAzZdvKzVSJJGNuoIXCtk2DTMgW2Xr1AlklY7L6WXpEYZ4JLUKANckhplgEtSowxwSWqUZ6H08UIcSS1xBC5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqMMcElqlAEuSY0ywCWpUQa4JDXKAJekRhngktSokT6NMMkB4EfAL4Bnq2omyTrgVmAaOABcU1XfH0+ZkqSFFjMC//2qOqeq5r+dfiuwq6rOBHZ165KkFbKUKZQrgR3d8g7gqiVXI0ka2agBXsB/JLk/yZaubX1VHeqWDwPrBz0wyZYks0lm5+bmlliuJGneqN/I86aqeirJbwL3Jnmsf2NVVZIa9MCq2g5sB5iZmRm4jyRp8UYagVfVU939EeBO4Hzg6SQbALr7I+MqUpL0QkMDPMlpSV4xvwxcCjwM3AVs7nbbDOwcV5GSpBcaZQplPXBnkvn9P11VX0jyVeC2JNcDTwLXjK9MSdJCQwO8qp4A3jCg/XvAxeMoSpI0nFdiSlKjDHBJapQBLkmNGvU8cK0S01vvOe72A9suX6FKJE2aI3BJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNcoAl6RGGeCS1CgDXJIaZYBLUqNGDvAkJyV5IMnd3fqmJLuT7E9ya5JTxlemJGmhxYzA3wPs6Vu/EfhwVb0O+D5w/XIWJkk6vpECPMlG4HLgpm49wEXA7d0uO4CrxlCfJOkYRh2BfwR4P/DLbv1VwDNV9Wy3fhA4Y9ADk2xJMptkdm5ubim1SpL6DA3wJO8AjlTV/SfyBFW1vapmqmpmamrqRP4JSdIAo3wr/RuBK5K8HTgV+A3go8CaJCd3o/CNwFPjK1OStNDQEXhV3VBVG6tqGrgW+FJVvRO4D7i6220zsHNsVUqSXmAp54H/DfC+JPvpzYnfvDwlSZJGMcoUylFV9WXgy93yE8D5y1+SJGkUXokpSY0ywCWpUQa4JDXKAJekRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhq1qI+T1eo3vfWe424/sO3yFapE0rg5ApekRhngktQoA1ySGmWAS1KjhgZ4klOTfCXJ15M8kuSDXfumJLuT7E9ya5JTxl+uJGneKCPw/wMuqqo3AOcAlyW5ALgR+HBVvQ74PnD92KqUJL3A0ACvnh93qy/pbgVcBNzete8ArhpHgZKkwUaaA09yUpIHgSPAvcDjwDNV9Wy3y0HgjLFUKEkaaKQAr6pfVNU5wEbgfOD1oz5Bki1JZpPMzs3NnViVkqQXWNRZKFX1DHAfcCGwJsn8lZwbgaeO8ZjtVTVTVTNTU1NLqVWS1GeUs1Cmkqzpll8GXALsoRfkV3e7bQZ2jqlGSdIAo3wWygZgR5KT6AX+bVV1d5JHgc8m+TvgAeDmMdYpSVpgaIBX1UPAuQPan6A3Hy5JmgCvxJSkRhngktQoA1ySGmWAS1KjDHBJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRo3yhw6+E6a33TLoESVpWjsAlqVEGuCQ16kUzhaKeUaaSDmy7fAUqkbRUjsAlqVFDAzzJq5Pcl+TRJI8keU/Xvi7JvUn2dfdrx1+uJGneKCPwZ4G/rqqzgQuAP09yNrAV2FVVZwK7unVJ0goZGuBVdaiqvtYt/wjYA5wBXAns6HbbAVw1pholSQMsag48yTRwLrAbWF9Vh7pNh4H1x3jMliSzSWbn5uaWUqskqc/IAZ7k5cDngfdW1Q/7t1VVATXocVW1vapmqmpmampqScVKkp4zUoAneQm98L6lqu7omp9OsqHbvgE4Mp4SJUmDjHIWSoCbgT1V9aG+TXcBm7vlzcDO5S9PknQso1zI80bgD4FvJHmwa/tbYBtwW5LrgSeBa8ZSoSRpoKEBXlX/DeQYmy9e3nIkSaPySkxJapQBLkmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjTLAJalRBrgkNWqUTyOUnmd66z3H3X5g2+UrVIn04uYIXJIaZYBLUqMMcElqlAEuSY3yIKZeYNhBSkmrgyNwSWqUAS5JjRoa4Ek+meRIkof72tYluTfJvu5+7XjLlCQtNMoI/F+Ayxa0bQV2VdWZwK5uXZK0goYGeFX9F/C/C5qvBHZ0yzuAq5a3LEnSMCd6Fsr6qjrULR8G1h9rxyRbgC0Ar3nNa07w6YbzzAlJLzZLPohZVQXUcbZvr6qZqpqZmppa6tNJkjonGuBPJ9kA0N0fWb6SJEmjONEAvwvY3C1vBnYuTzmSpFGNchrhZ4D/Ac5KcjDJ9cA24JIk+4C3dOuSpBU09CBmVV13jE0XL3MtkqRF8LNQtOz8wgdpZXgpvSQ1ygCXpEYZ4JLUKANckhplgEtSowxwSWqUAS5JjfI8cK04zxOXlocjcElqlAEuSY0ywCWpUQa4JDXKAJekRjVzForfeSlJz+cIXJIaZYBLUqOamULRi8dyTJcNuxjIi4n0q8ARuCQ1ygCXpEYtaQolyWXAR4GTgJuqym+n16rgWUsrY5R+fjFMR01qSu6ER+BJTgL+GXgbcDZwXZKzl6swSdLxLWUK5Xxgf1U9UVU/Az4LXLk8ZUmShlnKFMoZwHf61g8Cv7twpyRbgC3d6o+T7O3bfDrw3SXUMGnWPzljrT03jutfPupF0/cr0JeLteJ9vwx98NuDGsd+GmFVbQe2D9qWZLaqZsZdw7hY/+S0XDu0XX/LtUP79fdbyhTKU8Cr+9Y3dm2SpBWwlAD/KnBmkk1JTgGuBe5anrIkScOc8BRKVT2b5C+AL9I7jfCTVfXIIv+ZgVMrDbH+yWm5dmi7/pZrh/brPypVNekaJEknwCsxJalRBrgkNWpiAZ7ksiR7k+xPsnVSdRxLklcnuS/Jo0keSfKern1dknuT7Ovu13btSfJP3et5KMl5k30FPUlOSvJAkru79U1Jdnd13todgCbJS7v1/d326YkW3qtpTZLbkzyWZE+SC1vp/yR/1f3ePJzkM0lOXc19n+STSY4kebivbdF9nWRzt/++JJsnWPs/dL83DyW5M8mavm03dLXvTfLWvvZVnUkDVdWK3+gd9HwceC1wCvB14OxJ1HKcGjcA53XLrwC+Se8jA/4e2Nq1bwVu7JbfDvw7EOACYPekX0NX1/uATwN3d+u3Add2yx8H/rRb/jPg493ytcCtq6D2HcCfdMunAGta6H96F7l9C3hZX5+/azX3PfBm4Dzg4b62RfU1sA54ortf2y2vnVDtlwInd8s39tV+dpc3LwU2dTl0UguZNPC1T+RJ4ULgi33rNwA3TLozhtS8E7gE2Ats6No2AHu75U8A1/Xtf3S/Cda8EdgFXATc3f2H+27fL/bRnwO9s4ku7JZP7vbLBGt/ZReCWdC+6vuf565SXtf15d3AW1d73wPTC0JwUX0NXAd8oq/9efutZO0Ltv0BcEu3/Lysme/7FjOpqiY2hTLoMvwzJlTLUN1b2nOB3cD6qjrUbToMrO+WV+Nr+gjwfuCX3fqrgGeq6tluvb/Go/V323/Q7T8pm4A54FPdFNBNSU6jgf6vqqeAfwS+DRyi15f3007fz1tsX6+an8ECf0zvHQO0V/txeRBziCQvBz4PvLeqfti/rXp/qlfleZhJ3gEcqar7J13LCTqZ3tvij1XVucBP6L2NP2q19n83V3wlvT9CvwWcBlw20aKWaLX29TBJPgA8C9wy6VrGYVIB3sRl+EleQi+8b6mqO7rmp5Ns6LZvAI507avtNb0RuCLJAXqfFHkRvc9uX5Nk/gKu/hqP1t9tfyXwvZUseIGDwMGq2t2t304v0Fvo/7cA36qquar6OXAHvZ9HK30/b7F9vZp+BiR5F/AO4J3dHyBopPZRTSrAV/1l+EkC3AzsqaoP9W26C5g/ur6Z3tz4fPsfdUfoLwB+0Pf2c8VV1Q1VtbGqpun175eq6p3AfcDV3W4L659/XVd3+09sxFVVh4HvJDmra7oYeJQ2+v/bwAVJfr37PZqvvYm+77PYvv4icGmStd27kEu7thWX3pfNvB+4oqp+2rfpLuDa7syfTcCZwFdoIJMGmtTkO70j2d+kd+T3A5M+GDCgvjfRe8v4EPBgd3s7vbnJXcA+4D+Bdd3+ofcFF48D3wBmJv0a+l7L7/HcWSivpfcLux/4HPDSrv3Ubn1/t/21q6Duc4DZ7mfwb/TObGii/4EPAo8BDwP/Su+sh1Xb98Bn6M3X/5zeu5/rT6Sv6c037+9u755g7fvpzWnP/9/9eN/+H+hq3wu8ra99VWfSoJuX0ktSozyIKUmNMsAlqVEGuCQ1ygCXpEYZ4JLUKANckhplgEtSo/4fcnkeW2FxuMkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAATBklEQVR4nO3df6zd9X3f8edrONCJktjArWX5R01XL1X+CbhXCVWzaAtqyo82ZlqKiKriUk+eKlIlyqbWXf5YJ+0PsmnNgjYReSGbqdIQQouwGtbGc+iqSYPUEIdACOVCQbZlsEP40ZX1B917f5yP12PnXt9z7z3n/vjwfEhH5/P9fD/fc97fL4eXv+dzvufcVBWSpL78nZUuQJI0foa7JHXIcJekDhnuktQhw12SOrRupQsAuPzyy2v79u0rXYYkrSmPPvrod6tqarZ1qyLct2/fzpEjR1a6DElaU5K8MNc6p2UkqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtSh+YN9yTvTHJ06PZ6ko8nuTTJoSTPtPsNbXyS3JFkJsnjSXZOfjckScPmDfeqerqqrqyqK4EfB94A7gf2AYeragdwuC0DXAfsaLe9wJ0TqFuSdB4LnZa5Bni2ql4AdgEHWv8B4MbW3gXcXQMPA+uTbBpHsZKk0Sz0G6o3A19s7Y1VdbK1XwQ2tvZm4NjQNsdb38mhPpLsZXBmz7Zt2xZYxty27/vKWcvP337D2B5bktaKkc/ck1wIfAj48rnravDnnBb0J52qan9VTVfV9NTUrD+NIElapIVMy1wHPFZVL7Xll85Mt7T7U63/BLB1aLstrU+StEwWEu4f4W+nZAAOArtbezfwwFD/Le2qmauB14ambyRJy2CkOfckFwM/Bfyzoe7bgXuT7AFeAG5q/Q8C1wMzDK6suXVs1UqSRjJSuFfVnwOXndP3MoOrZ84dW8BtY6lOkrQofkNVkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdGinck6xPcl+S7yR5KslPJLk0yaEkz7T7DW1sktyRZCbJ40l2TnYXJEnnGvXM/TPA71fVjwHvBp4C9gGHq2oHcLgtA1wH7Gi3vcCdY61YkjSvecM9yTuA9wN3AVTVX1XVq8Au4EAbdgC4sbV3AXfXwMPA+iSbxly3JOk8RjlzvwI4DfyXJN9I8rkkFwMbq+pkG/MisLG1NwPHhrY/3vrOkmRvkiNJjpw+fXrxeyBJ+j6jhPs6YCdwZ1VdBfw5fzsFA0BVFVALeeKq2l9V01U1PTU1tZBNJUnzGCXcjwPHq+qRtnwfg7B/6cx0S7s/1dafALYObb+l9UmSlsm84V5VLwLHkryzdV0DfBs4COxufbuBB1r7IHBLu2rmauC1oekbSdIyWDfiuF8BvpDkQuA54FYG/zDcm2QP8AJwUxv7IHA9MAO80cZKkpbRSOFeVUeB6VlWXTPL2AJuW1pZkqSl8BuqktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0aKdyTPJ/kW0mOJjnS+i5NcijJM+1+Q+tPkjuSzCR5PMnOSe6AJOn7LeTM/R9V1ZVVNd2W9wGHq2oHcLgtA1wH7Gi3vcCd4ypWkjSapUzL7AIOtPYB4Mah/rtr4GFgfZJNS3geSdICjRruBXw1yaNJ9ra+jVV1srVfBDa29mbg2NC2x1vfWZLsTXIkyZHTp08vonRJ0lzWjTjufVV1IskPAYeSfGd4ZVVVklrIE1fVfmA/wPT09IK2lSSd30hn7lV1ot2fAu4H3gO8dGa6pd2fasNPAFuHNt/S+iRJy2TecE9ycZJLzrSBDwJPAAeB3W3YbuCB1j4I3NKumrkaeG1o+kaStAxGmZbZCNyf5Mz4366q30/yx8C9SfYALwA3tfEPAtcDM8AbwK1jr1qSdF7zhntVPQe8e5b+l4FrZukv4LaxVCdJWhS/oSpJHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0a9Vch31K27/vKWcvP337DClUiSYvjmbskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ2+5LzGd+wUlSeqRZ+6S1KGRwz3JBUm+keT32vIVSR5JMpPkS0kubP0XteWZtn77hGqXJM1hIWfuHwOeGlr+FPDpqvpR4BVgT+vfA7zS+j/dxkmSltFI4Z5kC3AD8Lm2HOADwH1tyAHgxtbe1ZZp669p4yVJy2TUD1T/A/CrwCVt+TLg1ap6sy0fBza39mbgGEBVvZnktTb+u8MPmGQvsBdg27Ztiyx/fn6AKumtaN4z9yQ/A5yqqkfH+cRVtb+qpqtqempqapwPLUlveaOcuf8k8KEk1wM/ALwd+AywPsm6dva+BTjRxp8AtgLHk6wD3gG8PPbKJUlzmvfMvap+vaq2VNV24Gbga1X188BDwIfbsN3AA619sC3T1n+tqmqsVUuSzmsp17n/GvCJJDMM5tTvav13AZe1/k8A+5ZWoiRpoRb0DdWq+kPgD1v7OeA9s4z5C+DnxlCbJGmR1vzPD3g1jCR9P39+QJI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHZo33JP8QJKvJ/lmkieT/OvWf0WSR5LMJPlSkgtb/0Vteaat3z7hfZAknWOUM/e/BD5QVe8GrgSuTXI18Cng01X1o8ArwJ42fg/wSuv/dBsnSVpG84Z7Dfzvtvi2divgA8B9rf8AcGNr72rLtPXXJMm4CpYkzW+kOfckFyQ5CpwCDgHPAq9W1ZttyHFgc2tvBo4BtPWvAZfN8ph7kxxJcuT06dNL2glJ0tlGCveq+puquhLYArwH+LGlPnFV7a+q6aqanpqaWurDSZKGLOhqmap6FXgI+AlgfZJ1bdUW4ERrnwC2ArT17wBeHkexkqTRrJtvQJIp4K+r6tUkfxf4KQYfkj4EfBi4B9gNPNA2OdiW/1db/7WqqgnUvmy27/vKWcvP337DClUiSaOZN9yBTcCBJBcwONO/t6p+L8m3gXuS/BvgG8BdbfxdwG8lmQG+B9w8gbolSecxb7hX1ePAVbP0P8dg/v3c/r8Afm4s1UmSFsVvqEpShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUPzhnuSrUkeSvLtJE8m+VjrvzTJoSTPtPsNrT9J7kgyk+TxJDsnvROSpLOtG2HMm8A/r6rHklwCPJrkEPCLwOGquj3JPmAf8GvAdcCOdnsvcGe779b2fV85a/n5229YoUokaWDeM/eqOllVj7X2nwFPAZuBXcCBNuwAcGNr7wLuroGHgfVJNo27cEnS3BY0555kO3AV8AiwsapOtlUvAhtbezNwbGiz461PkrRMRpmWASDJDwK/A3y8ql5P8v/XVVUlqYU8cZK9wF6Abdu2LWTTFXfuNIwkrTYjnbkneRuDYP9CVf1u637pzHRLuz/V+k8AW4c239L6zlJV+6tquqqmp6amFlu/JGkWo1wtE+Au4Kmq+s2hVQeB3a29G3hgqP+WdtXM1cBrQ9M3kqRlMMq0zE8CvwB8K8nR1vcvgduBe5PsAV4AbmrrHgSuB2aAN4Bbx1mwJGl+84Z7Vf1PIHOsvmaW8QXctsS6JElL4DdUJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQyP/KqQWzz/mIWm5eeYuSR0y3CWpQ4a7JHXIcJekDhnuktQhr5aZgEn/jVWvvpE0H8/cJalDhrskdchwl6QOGe6S1CHDXZI6NG+4J/l8klNJnhjquzTJoSTPtPsNrT9J7kgyk+TxJDsnWbwkaXajnLn/V+Dac/r2AYeragdwuC0DXAfsaLe9wJ3jKVOStBDzhntV/RHwvXO6dwEHWvsAcONQ/9018DCwPsmmMdUqSRrRYr/EtLGqTrb2i8DG1t4MHBsad7z1neQcSfYyOLtn27ZtiyxjbZrvS0hL/RKUX3KStOQPVKuqgFrEdvurarqqpqemppZahiRpyGLD/aUz0y3t/lTrPwFsHRq3pfVJkpbRYqdlDgK7gdvb/QND/R9Ncg/wXuC1oekbzWHSv0Uj6a1n3nBP8kXgHwKXJzkO/CsGoX5vkj3AC8BNbfiDwPXADPAGcOsEapYkzWPecK+qj8yx6ppZxhZw21KLkiQtjT/5+xYwiatnvCJHWt38+QFJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalD/rZMB5bjJ4MX+hz+9oy0sjxzl6QOGe6S1CGnZbQq/hKU0zjSeHnmLkkdMtwlqUNOy7wFTWIaZr7HXOq0y6S3X8zjr7appNVWj1aWZ+6S1CHDXZI6NJFpmSTXAp8BLgA+V1W3T+J5tHYtdBpnqY+3ULM93qSnOVZ6WmWln38tWOj03ihjJnWcx37mnuQC4D8B1wHvAj6S5F3jfh5J0twmMS3zHmCmqp6rqr8C7gF2TeB5JElzSFWN9wGTDwPXVtU/bcu/ALy3qj56zri9wN62+E7g6UU+5eXAdxe57Upai3WvxZrBupebdS+fH66qqdlWrNilkFW1H9i/1MdJcqSqpsdQ0rJai3WvxZrBupebda8Ok5iWOQFsHVre0vokSctkEuH+x8COJFckuRC4GTg4geeRJM1h7NMyVfVmko8Cf8DgUsjPV9WT436eIUue2lkha7HutVgzWPdys+5VYOwfqEqSVp7fUJWkDhnuktShNRvuSa5N8nSSmST7VkE9W5M8lOTbSZ5M8rHW/xtJTiQ52m7XD23z663+p5P89FD/su5bkueTfKvVd6T1XZrkUJJn2v2G1p8kd7TaHk+yc+hxdrfxzyTZPeGa3zl0TI8meT3Jx1fj8U7y+SSnkjwx1De245vkx9t/v5m2bSZU879L8p1W1/1J1rf+7Un+z9Ax/+x8tc21/xOqe2yviQwuFHmk9X8pg4tGVqeqWnM3Bh/UPgv8CHAh8E3gXStc0yZgZ2tfAvwJg59f+A3gX8wy/l2t7ouAK9r+XLAS+wY8D1x+Tt+/Bfa19j7gU619PfDfgABXA4+0/kuB59r9htbesIyvhxeBH16Nxxt4P7ATeGISxxf4ehubtu11E6r5g8C61v7UUM3bh8ed8ziz1jbX/k+o7rG9JoB7gZtb+7PALy/Ha3wxt7V65r7qfuKgqk5W1WOt/WfAU8Dm82yyC7inqv6yqv4UmGGwX6tl33YBB1r7AHDjUP/dNfAwsD7JJuCngUNV9b2qegU4BFy7TLVeAzxbVS+cZ8yKHe+q+iPge7PUs+Tj29a9vaoerkHi3D30WGOtuaq+WlVvtsWHGXyHZU7z1DbX/o+97vNY0Guivev4AHDfuOuehLUa7puBY0PLxzl/kC6rJNuBq4BHWtdH21vZzw+9/ZxrH1Zi3wr4apJHM/hZCICNVXWytV8ENrb2aqr7jJuBLw4tr/bjDeM7vptb+9z+SfslBmfiZ1yR5BtJ/keSf9D6zlfbXPs/KeN4TVwGvDr0D9yqyp1zrdVwX7WS/CDwO8DHq+p14E7g7wFXAieBf79y1c3pfVW1k8Eved6W5P3DK9tZ16q8ZrbNeX4I+HLrWgvH+yyr+fjOJskngTeBL7Suk8C2qroK+ATw20nePurjLcP+r7nXxDis1XBflT9xkORtDIL9C1X1uwBV9VJV/U1V/V/gPzN4ywdz78Oy71tVnWj3p4D7W40vtbfVZ95en1ptdTfXAY9V1UuwNo53M67je4Kzp0cmWn+SXwR+Bvj5Fsq0aY2XW/tRBvPVf3+e2uba/7Eb42viZQbTZOvO6V+V1mq4r7qfOGjzcXcBT1XVbw71bxoa9o+BM5/iHwRuTnJRkiuAHQw+fFrWfUtycZJLzrQZfGj2RHvOM1dk7AYeGKr7lnZVx9XAa+3t9R8AH0yyob3t/WDrm7SPMDQls9qP95CxHN+27vUkV7fX4C1DjzVWGfwRnl8FPlRVbwz1T2XwdxxI8iMMju1z89Q21/5Pou6xvCbaP2YPAR9ejrqXbKU/0V3sjcFVBX/C4Czhk6ugnvcxeGv5OHC03a4Hfgv4Vus/CGwa2uaTrf6nGbrCYTn3jcEVAd9styfPPB+D+cXDwDPAfwcubf1h8MdYnm37NT30WL/E4EOpGeDWZTjmFzM4m3rHUN+qO94M/vE5Cfw1g3naPeM8vsA0g8B6FviPtG+eT6DmGQZz0Wde359tY/9Je+0cBR4Dfna+2uba/wnVPbbXRPv/5evtWHwZuGjSr/PF3vz5AUnq0FqdlpEknYfhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjr0/wAvmyk6qtXJywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAEFCAYAAAD69rxNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAMpklEQVR4nO3dfYyld1mH8evLLkVbSmnZsdS+ODUWEiRIccqLRBBaTLWmJbHREqtb07gGYq2g0Rr+IJF/qihiojFuSrUqAXGt0lgV6kIlJnRl+hJtu2ArIiy0dKqmikRq09s/5mwyHWfnnD3nOWd62+uTbPa8PHPO/eym1zx99jy/SVUhSernWTs9gCRpOgZckpoy4JLUlAGXpKYMuCQ1tXuRb7Znz55aXl5e5FtKUnt33nnno1W1tPnxhQZ8eXmZ1dXVRb6lJLWX5F+2etxTKJLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJampswJPcmOSRJPdueOy0JLcleWD0+6nzHVOStNkkR+C/B1y86bHrgINVdR5wcHRfkrRAYwNeVZ8E/m3Tw5cBN41u3wS8edixJEnjTHsl5ulV9dDo9sPA6cfaMMk+YB/AOeecM+XbwfJ1t079tZK0kz5//SVzed2Z/xGz1n+kzzF/rE9V7a+qlapaWVr6P5fyS5KmNG3Av5LkDIDR748MN5IkaRLTBvwWYO/o9l7gI8OMI0ma1CQfI/wg8CngxUmOJLkauB54U5IHgItG9yVJCzT2HzGr6i3HeOrCgWeRJB0Hr8SUpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktTUTAFP8vYk9yW5N8kHk3zDUINJkrY3dcCTnAn8NLBSVS8FdgFXDDWYJGl7s55C2Q18Y5LdwInAl2cfSZI0iakDXlVfAn4V+ALwEPBYVX1s83ZJ9iVZTbK6trY2/aSSpKeY5RTKqcBlwLnANwMnJbly83ZVtb+qVqpqZWlpafpJJUlPMcsplIuAf66qtar6H+Bm4LuGGUuSNM4sAf8C8OokJyYJcCFweJixJEnjzHIO/BBwALgL+IfRa+0faC5J0hi7Z/niqnoX8K6BZpEkHQevxJSkpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1NRMAU/y/CQHknwmyeEkrxlqMEnS9nbP+PW/AfxVVV2e5ATgxAFmkiRNYOqAJzkFeB1wFUBVPQ48PsxYkqRxZjmFci6wBvxukruT3JDkpIHmkiSNMUvAdwOvAH67qs4H/gu4bvNGSfYlWU2yura2NsPbSZI2miXgR4AjVXVodP8A60F/iqraX1UrVbWytLQ0w9tJkjaaOuBV9TDwxSQvHj10IXD/IFNJksaa9VMo1wAfGH0C5XPAj88+kiRpEjMFvKruAVaGGUWSdDy8ElOSmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKamjngSXYluTvJnw8xkCRpMkMcgV8LHB7gdSRJx2GmgCc5C7gEuGGYcSRJk5r1CPx9wM8DTx5rgyT7kqwmWV1bW5vx7SRJR00d8CQ/ADxSVXdut11V7a+qlapaWVpamvbtJEmbzHIE/lrg0iSfBz4EvDHJHw4ylSRprKkDXlW/WFVnVdUycAXw8aq6crDJJEnb8nPgktTU7iFepKpuB24f4rUkSZPxCFySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlNTBzzJ2Uk+keT+JPcluXbIwSRJ29s9w9c+AfxsVd2V5GTgziS3VdX9A80mSdrG1EfgVfVQVd01uv2fwGHgzKEGkyRtb5Bz4EmWgfOBQ1s8ty/JapLVtbW1Id5OksQAAU/yXOBPgJ+pqv/Y/HxV7a+qlapaWVpamvXtJEkjMwU8ybNZj/cHqurmYUaSJE1ilk+hBHg/cLiq3jvcSJKkScxyBP5a4EeBNya5Z/Tr+weaS5I0xtQfI6yqvwUy4CySpOPglZiS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLUlAGXpKYMuCQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktSUAZekpgy4JDVlwCWpKQMuSU0ZcElqyoBLUlMGXJKaMuCS1JQBl6SmDLgkNWXAJakpAy5JTRlwSWrKgEtSUwZckpoy4JLU1EwBT3Jxks8meTDJdUMNJUkab+qAJ9kF/BbwfcBLgLckeclQg0mStjfLEfgrgQer6nNV9TjwIeCyYcaSJI2ze4avPRP44ob7R4BXbd4oyT5g3+juV5N8dob33Al7gEd3eogFc5+fGdznBckvz/wS37LVg7MEfCJVtR/YP+/3mZckq1W1stNzLJL7/MzgPvc3yymULwFnb7h/1ugxSdICzBLwTwPnJTk3yQnAFcAtw4wlSRpn6lMoVfVEkp8CPgrsAm6sqvsGm+zpo+3pnxm4z88M7nNzqaqdnkGSNAWvxJSkpgy4JDVlwLeQ5LQktyV5YPT7qdts+7wkR5L85iJnHNIk+5vk5Uk+leS+JH+f5Id3YtZZjVv+IclzkvzR6PlDSZZ3YMxBTbDP70hy/+jv9WCSLT9z3Mmky3wk+cEklaTlRwsN+NauAw5W1XnAwdH9Y3k38MmFTDU/k+zv14Afq6pvBy4G3pfk+YsbcXYTLv9wNfDvVfVtwK8Ds1+CsYMm3Oe7gZWqehlwAPiVxU45rEmX+UhyMnAtcGixEw7HgG/tMuCm0e2bgDdvtVGS7wROBz62mLHmZuz+VtU/VtUDo9tfBh4BlhY14EAmWf5h45/FAeDCJFngjEMbu89V9Ymq+tro7h2sX9PR2aTLfLyb9W/Q/73I4YZkwLd2elU9NLr9MOuRfookzwJ+Dfi5RQ42J2P3d6MkrwROAP5p3oMNbKvlH8481jZV9QTwGPCChUw3H5Ps80ZXA38514nmb+w+J3kFcHZV3brIwYY290vpn66S/DXwwi2eeufGO1VVSbb6rOXbgL+oqiMdDtAG2N+jr3MG8AfA3qp6ctgptZOSXAmsAK/f6VnmaXTw9V7gqh0eZWbP2IBX1UXHei7JV5KcUVUPjYL1yBabvQb47iRvA54LnJDkq1X1tFwXfYD9JcnzgFuBd1bVHXMadZ4mWf7h6DZHkuwGTgH+dTHjzcVES14kuYj1b+avr6qvL2i2eRm3zycDLwVuHx18vRC4JcmlVbW6sCkH4CmUrd0C7B3d3gt8ZPMGVfUjVXVOVS2zfhrl95+u8Z7A2P0dLZfwp6zv54EFzjakSZZ/2PhncTnw8ep9tdvYfU5yPvA7wKVVteU372a23eeqeqyq9lTV8ui/3ztY3/dW8QYDfizXA29K8gBw0eg+SVaS3LCjk83HJPv7Q8DrgKuS3DP69fIdmXZKo3PaR5d/OAx8uKruS/JLSS4dbfZ+4AVJHgTewfafQHram3Cf38P6/0X+8ejvtfWaRhPu8/8LXkovSU15BC5JTRlwSWrKgEtSUwZckpoy4JI0J0muSfKZ0SJwE60xczwL5D1jL+SRpKEk+R7gqqq6asNjb2B9DZbvqKqvJ/mmCV9u4gXyPAKXpPl4K3D90Stbj14klWRXkvck+fRoCd+fPPoFx7tAngGXpPl4EevLbRxK8jdJLhg9fjXwWFVdAFwA/MToqtHjXiDPUyiSNKUkh4DnsH4l62lJ7hk99Qus9/U04NWsh/rDSb4V+F7gZUkuH217CnAecAnHuUCeAZekKVXVq+CY58DfDtw8Wkvn75I8CewBAlxTVR/d+FpJ9nKcC+R5CkWS5uPPgDcAJHkR62voP8r6Gi1vTfLso88lOWmaBfI8Apek+bgRuDHJvcDjrK+hX6MF4paBu0Y/7WmNY/zUr3FczEqSmvIUiiQ1ZcAlqSkDLklNGXBJasqAS1JTBlySmjLgktTU/wKuYxvO5jQ6rQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "task data norm and number entries: tensor(234290.7969, device='cuda:0') torch.Size([397510])\n",
      "test performance :  [98.26999664  0.          0.          0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(98.2700, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0996\n",
      "Epoch [2/20], Loss: 0.0154\n",
      "Epoch [3/20], Loss: 0.0131\n",
      "Epoch [4/20], Loss: 0.0122\n",
      "Epoch [5/20], Loss: 0.0117\n",
      "Epoch [6/20], Loss: 0.0113\n",
      "Epoch [7/20], Loss: 0.0110\n",
      "Epoch [8/20], Loss: 0.0107\n",
      "Epoch [9/20], Loss: 0.0105\n",
      "Epoch [10/20], Loss: 0.0104\n",
      "Epoch [11/20], Loss: 0.0102\n",
      "Epoch [12/20], Loss: 0.0101\n",
      "Epoch [13/20], Loss: 0.0100\n",
      "Epoch [14/20], Loss: 0.0099\n",
      "Epoch [15/20], Loss: 0.0099\n",
      "Epoch [16/20], Loss: 0.0098\n",
      "Epoch [17/20], Loss: 0.0098\n",
      "Epoch [18/20], Loss: 0.0097\n",
      "Epoch [19/20], Loss: 0.0097\n",
      "Epoch [20/20], Loss: 0.0096\n",
      "test performance :  [98.26999664 97.68499756  0.          0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(97.7500, device='cuda:0'), tensor(97.6200, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0986\n",
      "Epoch [2/20], Loss: 0.0164\n",
      "Epoch [3/20], Loss: 0.0139\n",
      "Epoch [4/20], Loss: 0.0129\n",
      "Epoch [5/20], Loss: 0.0125\n",
      "Epoch [6/20], Loss: 0.0122\n",
      "Epoch [7/20], Loss: 0.0120\n",
      "Epoch [8/20], Loss: 0.0119\n",
      "Epoch [9/20], Loss: 0.0118\n",
      "Epoch [10/20], Loss: 0.0117\n",
      "Epoch [11/20], Loss: 0.0116\n",
      "Epoch [12/20], Loss: 0.0116\n",
      "Epoch [13/20], Loss: 0.0115\n",
      "Epoch [14/20], Loss: 0.0114\n",
      "Epoch [15/20], Loss: 0.0114\n",
      "Epoch [16/20], Loss: 0.0114\n",
      "Epoch [17/20], Loss: 0.0114\n",
      "Epoch [18/20], Loss: 0.0113\n",
      "Epoch [19/20], Loss: 0.0113\n",
      "Epoch [20/20], Loss: 0.0113\n",
      "test performance :  [98.26999664 97.68499756 96.80332947  0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(96.2000, device='cuda:0'), tensor(97.1500, device='cuda:0'), tensor(97.0600, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0945\n",
      "Epoch [2/20], Loss: 0.0164\n",
      "Epoch [3/20], Loss: 0.0143\n",
      "Epoch [4/20], Loss: 0.0138\n",
      "Epoch [5/20], Loss: 0.0136\n",
      "Epoch [6/20], Loss: 0.0134\n",
      "Epoch [7/20], Loss: 0.0133\n",
      "Epoch [8/20], Loss: 0.0132\n",
      "Epoch [9/20], Loss: 0.0132\n",
      "Epoch [10/20], Loss: 0.0132\n",
      "Epoch [11/20], Loss: 0.0132\n",
      "Epoch [12/20], Loss: 0.0132\n",
      "Epoch [13/20], Loss: 0.0133\n",
      "Epoch [14/20], Loss: 0.0132\n",
      "Epoch [15/20], Loss: 0.0132\n",
      "Epoch [16/20], Loss: 0.0132\n",
      "Epoch [17/20], Loss: 0.0132\n",
      "Epoch [18/20], Loss: 0.0132\n",
      "Epoch [19/20], Loss: 0.0131\n",
      "Epoch [20/20], Loss: 0.0131\n",
      "test performance :  [98.26999664 97.68499756 96.80332947 96.22750092  0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(94.5300, device='cuda:0'), tensor(96.6200, device='cuda:0'), tensor(96.7900, device='cuda:0'), tensor(96.9700, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0970\n",
      "Epoch [2/20], Loss: 0.0173\n",
      "Epoch [3/20], Loss: 0.0154\n",
      "Epoch [4/20], Loss: 0.0147\n",
      "Epoch [5/20], Loss: 0.0144\n",
      "Epoch [6/20], Loss: 0.0143\n",
      "Epoch [7/20], Loss: 0.0142\n",
      "Epoch [8/20], Loss: 0.0141\n",
      "Epoch [9/20], Loss: 0.0140\n",
      "Epoch [10/20], Loss: 0.0140\n",
      "Epoch [11/20], Loss: 0.0140\n",
      "Epoch [12/20], Loss: 0.0139\n",
      "Epoch [13/20], Loss: 0.0139\n",
      "Epoch [14/20], Loss: 0.0138\n",
      "Epoch [15/20], Loss: 0.0138\n",
      "Epoch [16/20], Loss: 0.0137\n",
      "Epoch [17/20], Loss: 0.0137\n",
      "Epoch [18/20], Loss: 0.0137\n",
      "Epoch [19/20], Loss: 0.0136\n",
      "Epoch [20/20], Loss: 0.0135\n",
      "test performance :  [98.26999664 97.68499756 96.80332947 96.22750092 96.20000458  0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(95.1700, device='cuda:0'), tensor(96.1800, device='cuda:0'), tensor(96.3700, device='cuda:0'), tensor(96.5600, device='cuda:0'), tensor(96.7200, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0984\n",
      "Epoch [2/20], Loss: 0.0182\n",
      "Epoch [3/20], Loss: 0.0162\n",
      "Epoch [4/20], Loss: 0.0156\n",
      "Epoch [5/20], Loss: 0.0153\n",
      "Epoch [6/20], Loss: 0.0150\n",
      "Epoch [7/20], Loss: 0.0149\n",
      "Epoch [8/20], Loss: 0.0148\n",
      "Epoch [9/20], Loss: 0.0148\n",
      "Epoch [10/20], Loss: 0.0146\n",
      "Epoch [11/20], Loss: 0.0145\n",
      "Epoch [12/20], Loss: 0.0144\n",
      "Epoch [13/20], Loss: 0.0144\n",
      "Epoch [14/20], Loss: 0.0144\n",
      "Epoch [15/20], Loss: 0.0144\n",
      "Epoch [16/20], Loss: 0.0144\n",
      "Epoch [17/20], Loss: 0.0144\n",
      "Epoch [18/20], Loss: 0.0144\n",
      "Epoch [19/20], Loss: 0.0144\n",
      "Epoch [20/20], Loss: 0.0144\n",
      "test performance :  [98.26999664 97.68499756 96.80332947 96.22750092 96.20000458 95.73000336\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(93.2300, device='cuda:0'), tensor(95.7500, device='cuda:0'), tensor(95.9800, device='cuda:0'), tensor(96.1600, device='cuda:0'), tensor(96.5400, device='cuda:0'), tensor(96.7200, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0996\n",
      "Epoch [2/20], Loss: 0.0179\n",
      "Epoch [3/20], Loss: 0.0162\n",
      "Epoch [4/20], Loss: 0.0156\n",
      "Epoch [5/20], Loss: 0.0154\n",
      "Epoch [6/20], Loss: 0.0153\n",
      "Epoch [7/20], Loss: 0.0152\n",
      "Epoch [8/20], Loss: 0.0150\n",
      "Epoch [9/20], Loss: 0.0149\n",
      "Epoch [10/20], Loss: 0.0149\n",
      "Epoch [11/20], Loss: 0.0149\n",
      "Epoch [12/20], Loss: 0.0149\n",
      "Epoch [13/20], Loss: 0.0149\n",
      "Epoch [14/20], Loss: 0.0148\n",
      "Epoch [15/20], Loss: 0.0148\n",
      "Epoch [16/20], Loss: 0.0149\n",
      "Epoch [17/20], Loss: 0.0149\n",
      "Epoch [18/20], Loss: 0.0149\n",
      "Epoch [19/20], Loss: 0.0149\n",
      "Epoch [20/20], Loss: 0.0150\n",
      "test performance :  [98.26999664 97.68499756 96.80332947 96.22750092 96.20000458 95.73000336\n",
      " 95.20999908  0.          0.          0.        ]\n",
      "individual errors:  [tensor(92.4200, device='cuda:0'), tensor(94.8400, device='cuda:0'), tensor(95.4000, device='cuda:0'), tensor(94.8100, device='cuda:0'), tensor(96.1700, device='cuda:0'), tensor(96.2800, device='cuda:0'), tensor(96.5500, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0934\n",
      "Epoch [2/20], Loss: 0.0180\n",
      "Epoch [3/20], Loss: 0.0168\n",
      "Epoch [4/20], Loss: 0.0166\n",
      "Epoch [5/20], Loss: 0.0163\n",
      "Epoch [6/20], Loss: 0.0162\n",
      "Epoch [7/20], Loss: 0.0160\n",
      "Epoch [8/20], Loss: 0.0159\n",
      "Epoch [9/20], Loss: 0.0157\n",
      "Epoch [10/20], Loss: 0.0157\n",
      "Epoch [11/20], Loss: 0.0158\n",
      "Epoch [12/20], Loss: 0.0158\n",
      "Epoch [13/20], Loss: 0.0159\n",
      "Epoch [14/20], Loss: 0.0158\n",
      "Epoch [15/20], Loss: 0.0158\n",
      "Epoch [16/20], Loss: 0.0158\n",
      "Epoch [17/20], Loss: 0.0158\n",
      "Epoch [18/20], Loss: 0.0158\n",
      "Epoch [19/20], Loss: 0.0158\n",
      "Epoch [20/20], Loss: 0.0158\n",
      "test performance :  [98.26999664 97.68499756 96.80332947 96.22750092 96.20000458 95.73000336\n",
      " 95.20999908 94.86249542  0.          0.        ]\n",
      "individual errors:  [tensor(91.5100, device='cuda:0'), tensor(93.5300, device='cuda:0'), tensor(94.6900, device='cuda:0'), tensor(95.1700, device='cuda:0'), tensor(95.7000, device='cuda:0'), tensor(95.7000, device='cuda:0'), tensor(96.2300, device='cuda:0'), tensor(96.3700, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0990\n",
      "Epoch [2/20], Loss: 0.0187\n",
      "Epoch [3/20], Loss: 0.0176\n",
      "Epoch [4/20], Loss: 0.0170\n",
      "Epoch [5/20], Loss: 0.0168\n",
      "Epoch [6/20], Loss: 0.0167\n",
      "Epoch [7/20], Loss: 0.0166\n",
      "Epoch [8/20], Loss: 0.0166\n",
      "Epoch [9/20], Loss: 0.0165\n",
      "Epoch [10/20], Loss: 0.0164\n",
      "Epoch [11/20], Loss: 0.0164\n",
      "Epoch [12/20], Loss: 0.0164\n",
      "Epoch [13/20], Loss: 0.0163\n",
      "Epoch [14/20], Loss: 0.0163\n",
      "Epoch [15/20], Loss: 0.0163\n",
      "Epoch [16/20], Loss: 0.0163\n",
      "Epoch [17/20], Loss: 0.0163\n",
      "Epoch [18/20], Loss: 0.0163\n",
      "Epoch [19/20], Loss: 0.0163\n",
      "Epoch [20/20], Loss: 0.0162\n",
      "test performance :  [98.26999664 97.68499756 96.80332947 96.22750092 96.20000458 95.73000336\n",
      " 95.20999908 94.86249542 94.01555634  0.        ]\n",
      "individual errors:  [tensor(88.5600, device='cuda:0'), tensor(92.9200, device='cuda:0'), tensor(93.4700, device='cuda:0'), tensor(94.4800, device='cuda:0'), tensor(95.1700, device='cuda:0'), tensor(94.3000, device='cuda:0'), tensor(94.9200, device='cuda:0'), tensor(96.0800, device='cuda:0'), tensor(96.2400, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.1012\n",
      "Epoch [2/20], Loss: 0.0200\n",
      "Epoch [3/20], Loss: 0.0180\n",
      "Epoch [4/20], Loss: 0.0173\n",
      "Epoch [5/20], Loss: 0.0170\n",
      "Epoch [6/20], Loss: 0.0169\n",
      "Epoch [7/20], Loss: 0.0168\n",
      "Epoch [8/20], Loss: 0.0168\n",
      "Epoch [9/20], Loss: 0.0168\n",
      "Epoch [10/20], Loss: 0.0168\n",
      "Epoch [11/20], Loss: 0.0168\n",
      "Epoch [12/20], Loss: 0.0168\n",
      "Epoch [13/20], Loss: 0.0168\n",
      "Epoch [14/20], Loss: 0.0168\n",
      "Epoch [15/20], Loss: 0.0168\n",
      "Epoch [16/20], Loss: 0.0168\n",
      "Epoch [17/20], Loss: 0.0169\n",
      "Epoch [18/20], Loss: 0.0169\n",
      "Epoch [19/20], Loss: 0.0168\n",
      "Epoch [20/20], Loss: 0.0168\n",
      "test performance :  [98.26999664 97.68499756 96.80332947 96.22750092 96.20000458 95.73000336\n",
      " 95.20999908 94.86249542 94.01555634 93.79000092]\n",
      "individual errors:  [tensor(87.9000, device='cuda:0'), tensor(93.0400, device='cuda:0'), tensor(92.7800, device='cuda:0'), tensor(93.9200, device='cuda:0'), tensor(94.4300, device='cuda:0'), tensor(94.1400, device='cuda:0'), tensor(94.5600, device='cuda:0'), tensor(95.2700, device='cuda:0'), tensor(95.7700, device='cuda:0'), tensor(96.0900, device='cuda:0')]\n"
     ]
    }
   ],
   "source": [
    "# with the best lambda = 1e-6\n",
    "l2new = run_simulation( Net().cuda(),train_loader,test_loader,L2(lam=1e-6),num_epochs=num_epochs )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "L2 0\n",
      "Epoch [1/20], Loss: 0.1094\n",
      "Epoch [2/20], Loss: 0.0141\n",
      "Epoch [3/20], Loss: 0.0106\n",
      "Epoch [4/20], Loss: 0.0090\n",
      "Epoch [5/20], Loss: 0.0080\n",
      "Epoch [6/20], Loss: 0.0073\n",
      "Epoch [7/20], Loss: 0.0069\n",
      "Epoch [8/20], Loss: 0.0065\n",
      "Epoch [9/20], Loss: 0.0062\n",
      "Epoch [10/20], Loss: 0.0059\n",
      "Epoch [11/20], Loss: 0.0057\n",
      "Epoch [12/20], Loss: 0.0054\n",
      "Epoch [13/20], Loss: 0.0053\n",
      "Epoch [14/20], Loss: 0.0051\n",
      "Epoch [15/20], Loss: 0.0048\n",
      "Epoch [16/20], Loss: 0.0047\n",
      "Epoch [17/20], Loss: 0.0045\n",
      "Epoch [18/20], Loss: 0.0044\n",
      "Epoch [19/20], Loss: 0.0043\n",
      "Epoch [20/20], Loss: 0.0042\n",
      "test performance :  [98.1499939  0.         0.         0.         0.         0.\n",
      "  0.         0.         0.         0.       ]\n",
      "individual errors:  [tensor(98.1500, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.1015\n",
      "Epoch [2/20], Loss: 0.0137\n",
      "Epoch [3/20], Loss: 0.0105\n",
      "Epoch [4/20], Loss: 0.0089\n",
      "Epoch [5/20], Loss: 0.0078\n",
      "Epoch [6/20], Loss: 0.0070\n",
      "Epoch [7/20], Loss: 0.0065\n",
      "Epoch [8/20], Loss: 0.0060\n",
      "Epoch [9/20], Loss: 0.0057\n",
      "Epoch [10/20], Loss: 0.0054\n",
      "Epoch [11/20], Loss: 0.0052\n",
      "Epoch [12/20], Loss: 0.0050\n",
      "Epoch [13/20], Loss: 0.0049\n",
      "Epoch [14/20], Loss: 0.0047\n",
      "Epoch [15/20], Loss: 0.0046\n",
      "Epoch [16/20], Loss: 0.0045\n",
      "Epoch [17/20], Loss: 0.0044\n",
      "Epoch [18/20], Loss: 0.0043\n",
      "Epoch [19/20], Loss: 0.0042\n",
      "Epoch [20/20], Loss: 0.0041\n",
      "test performance :  [98.1499939  83.08999634  0.          0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(67.9400, device='cuda:0'), tensor(98.2400, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0963\n",
      "Epoch [2/20], Loss: 0.0152\n",
      "Epoch [3/20], Loss: 0.0112\n",
      "Epoch [4/20], Loss: 0.0092\n",
      "Epoch [5/20], Loss: 0.0081\n",
      "Epoch [6/20], Loss: 0.0075\n",
      "Epoch [7/20], Loss: 0.0070\n",
      "Epoch [8/20], Loss: 0.0066\n",
      "Epoch [9/20], Loss: 0.0062\n",
      "Epoch [10/20], Loss: 0.0059\n",
      "Epoch [11/20], Loss: 0.0057\n",
      "Epoch [12/20], Loss: 0.0055\n",
      "Epoch [13/20], Loss: 0.0054\n",
      "Epoch [14/20], Loss: 0.0053\n",
      "Epoch [15/20], Loss: 0.0051\n",
      "Epoch [16/20], Loss: 0.0051\n",
      "Epoch [17/20], Loss: 0.0049\n",
      "Epoch [18/20], Loss: 0.0048\n",
      "Epoch [19/20], Loss: 0.0047\n",
      "Epoch [20/20], Loss: 0.0046\n",
      "test performance :  [98.1499939  83.08999634 77.29666901  0.          0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(50.0900, device='cuda:0'), tensor(83.8500, device='cuda:0'), tensor(97.9500, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0991\n",
      "Epoch [2/20], Loss: 0.0173\n",
      "Epoch [3/20], Loss: 0.0131\n",
      "Epoch [4/20], Loss: 0.0110\n",
      "Epoch [5/20], Loss: 0.0099\n",
      "Epoch [6/20], Loss: 0.0089\n",
      "Epoch [7/20], Loss: 0.0083\n",
      "Epoch [8/20], Loss: 0.0078\n",
      "Epoch [9/20], Loss: 0.0074\n",
      "Epoch [10/20], Loss: 0.0070\n",
      "Epoch [11/20], Loss: 0.0067\n",
      "Epoch [12/20], Loss: 0.0065\n",
      "Epoch [13/20], Loss: 0.0063\n",
      "Epoch [14/20], Loss: 0.0060\n",
      "Epoch [15/20], Loss: 0.0058\n",
      "Epoch [16/20], Loss: 0.0057\n",
      "Epoch [17/20], Loss: 0.0056\n",
      "Epoch [18/20], Loss: 0.0055\n",
      "Epoch [19/20], Loss: 0.0054\n",
      "Epoch [20/20], Loss: 0.0053\n",
      "test performance :  [98.1499939  83.08999634 77.29666901 68.36000061  0.          0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(32.4800, device='cuda:0'), tensor(62., device='cuda:0'), tensor(81.1800, device='cuda:0'), tensor(97.7800, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0968\n",
      "Epoch [2/20], Loss: 0.0171\n",
      "Epoch [3/20], Loss: 0.0128\n",
      "Epoch [4/20], Loss: 0.0112\n",
      "Epoch [5/20], Loss: 0.0101\n",
      "Epoch [6/20], Loss: 0.0093\n",
      "Epoch [7/20], Loss: 0.0086\n",
      "Epoch [8/20], Loss: 0.0080\n",
      "Epoch [9/20], Loss: 0.0075\n",
      "Epoch [10/20], Loss: 0.0070\n",
      "Epoch [11/20], Loss: 0.0066\n",
      "Epoch [12/20], Loss: 0.0063\n",
      "Epoch [13/20], Loss: 0.0061\n",
      "Epoch [14/20], Loss: 0.0059\n",
      "Epoch [15/20], Loss: 0.0057\n",
      "Epoch [16/20], Loss: 0.0056\n",
      "Epoch [17/20], Loss: 0.0054\n",
      "Epoch [18/20], Loss: 0.0053\n",
      "Epoch [19/20], Loss: 0.0052\n",
      "Epoch [20/20], Loss: 0.0052\n",
      "test performance :  [98.1499939  83.08999634 77.29666901 68.36000061 64.93999481  0.\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(23.8700, device='cuda:0'), tensor(48.9300, device='cuda:0'), tensor(68.0200, device='cuda:0'), tensor(86.3700, device='cuda:0'), tensor(97.5100, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0945\n",
      "Epoch [2/20], Loss: 0.0180\n",
      "Epoch [3/20], Loss: 0.0128\n",
      "Epoch [4/20], Loss: 0.0109\n",
      "Epoch [5/20], Loss: 0.0098\n",
      "Epoch [6/20], Loss: 0.0091\n",
      "Epoch [7/20], Loss: 0.0086\n",
      "Epoch [8/20], Loss: 0.0081\n",
      "Epoch [9/20], Loss: 0.0079\n",
      "Epoch [10/20], Loss: 0.0075\n",
      "Epoch [11/20], Loss: 0.0071\n",
      "Epoch [12/20], Loss: 0.0069\n",
      "Epoch [13/20], Loss: 0.0066\n",
      "Epoch [14/20], Loss: 0.0064\n",
      "Epoch [15/20], Loss: 0.0062\n",
      "Epoch [16/20], Loss: 0.0061\n",
      "Epoch [17/20], Loss: 0.0060\n",
      "Epoch [18/20], Loss: 0.0059\n",
      "Epoch [19/20], Loss: 0.0058\n",
      "Epoch [20/20], Loss: 0.0056\n",
      "test performance :  [98.1499939  83.08999634 77.29666901 68.36000061 64.93999481 55.65999985\n",
      "  0.          0.          0.          0.        ]\n",
      "individual errors:  [tensor(18.1500, device='cuda:0'), tensor(27.1000, device='cuda:0'), tensor(48.0600, device='cuda:0'), tensor(58.9800, device='cuda:0'), tensor(84.2800, device='cuda:0'), tensor(97.3900, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.0997\n",
      "Epoch [2/20], Loss: 0.0215\n",
      "Epoch [3/20], Loss: 0.0147\n",
      "Epoch [4/20], Loss: 0.0118\n",
      "Epoch [5/20], Loss: 0.0103\n",
      "Epoch [6/20], Loss: 0.0096\n",
      "Epoch [7/20], Loss: 0.0089\n",
      "Epoch [8/20], Loss: 0.0085\n",
      "Epoch [9/20], Loss: 0.0080\n",
      "Epoch [10/20], Loss: 0.0077\n",
      "Epoch [11/20], Loss: 0.0074\n",
      "Epoch [12/20], Loss: 0.0072\n",
      "Epoch [13/20], Loss: 0.0069\n",
      "Epoch [14/20], Loss: 0.0067\n",
      "Epoch [15/20], Loss: 0.0065\n",
      "Epoch [16/20], Loss: 0.0064\n",
      "Epoch [17/20], Loss: 0.0062\n",
      "Epoch [18/20], Loss: 0.0061\n",
      "Epoch [19/20], Loss: 0.0060\n",
      "Epoch [20/20], Loss: 0.0059\n",
      "test performance :  [98.1499939  83.08999634 77.29666901 68.36000061 64.93999481 55.65999985\n",
      " 52.50428391  0.          0.          0.        ]\n",
      "individual errors:  [tensor(20.0400, device='cuda:0'), tensor(20.2500, device='cuda:0'), tensor(39.1100, device='cuda:0'), tensor(39.5400, device='cuda:0'), tensor(67.0200, device='cuda:0'), tensor(84.3200, device='cuda:0'), tensor(97.2500, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.1045\n",
      "Epoch [2/20], Loss: 0.0219\n",
      "Epoch [3/20], Loss: 0.0146\n",
      "Epoch [4/20], Loss: 0.0114\n",
      "Epoch [5/20], Loss: 0.0100\n",
      "Epoch [6/20], Loss: 0.0091\n",
      "Epoch [7/20], Loss: 0.0084\n",
      "Epoch [8/20], Loss: 0.0080\n",
      "Epoch [9/20], Loss: 0.0077\n",
      "Epoch [10/20], Loss: 0.0074\n",
      "Epoch [11/20], Loss: 0.0073\n",
      "Epoch [12/20], Loss: 0.0072\n",
      "Epoch [13/20], Loss: 0.0070\n",
      "Epoch [14/20], Loss: 0.0069\n",
      "Epoch [15/20], Loss: 0.0067\n",
      "Epoch [16/20], Loss: 0.0067\n",
      "Epoch [17/20], Loss: 0.0066\n",
      "Epoch [18/20], Loss: 0.0065\n",
      "Epoch [19/20], Loss: 0.0065\n",
      "Epoch [20/20], Loss: 0.0065\n",
      "test performance :  [98.1499939  83.08999634 77.29666901 68.36000061 64.93999481 55.65999985\n",
      " 52.50428391 50.11000061  0.          0.        ]\n",
      "individual errors:  [tensor(15.4800, device='cuda:0'), tensor(15.5900, device='cuda:0'), tensor(28.8300, device='cuda:0'), tensor(31.5000, device='cuda:0'), tensor(52.1000, device='cuda:0'), tensor(75.7100, device='cuda:0'), tensor(84.6600, device='cuda:0'), tensor(97.0100, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.1049\n",
      "Epoch [2/20], Loss: 0.0219\n",
      "Epoch [3/20], Loss: 0.0138\n",
      "Epoch [4/20], Loss: 0.0113\n",
      "Epoch [5/20], Loss: 0.0101\n",
      "Epoch [6/20], Loss: 0.0094\n",
      "Epoch [7/20], Loss: 0.0088\n",
      "Epoch [8/20], Loss: 0.0084\n",
      "Epoch [9/20], Loss: 0.0080\n",
      "Epoch [10/20], Loss: 0.0077\n",
      "Epoch [11/20], Loss: 0.0074\n",
      "Epoch [12/20], Loss: 0.0071\n",
      "Epoch [13/20], Loss: 0.0069\n",
      "Epoch [14/20], Loss: 0.0067\n",
      "Epoch [15/20], Loss: 0.0065\n",
      "Epoch [16/20], Loss: 0.0063\n",
      "Epoch [17/20], Loss: 0.0062\n",
      "Epoch [18/20], Loss: 0.0061\n",
      "Epoch [19/20], Loss: 0.0060\n",
      "Epoch [20/20], Loss: 0.0060\n",
      "test performance :  [98.1499939  83.08999634 77.29666901 68.36000061 64.93999481 55.65999985\n",
      " 52.50428391 50.11000061 50.23666382  0.        ]\n",
      "individual errors:  [tensor(17.0800, device='cuda:0'), tensor(14.9200, device='cuda:0'), tensor(29.2700, device='cuda:0'), tensor(32.1200, device='cuda:0'), tensor(43.1200, device='cuda:0'), tensor(70.4200, device='cuda:0'), tensor(70.7200, device='cuda:0'), tensor(77.1600, device='cuda:0'), tensor(97.3200, device='cuda:0')]\n",
      "Epoch [1/20], Loss: 0.1000\n",
      "Epoch [2/20], Loss: 0.0232\n",
      "Epoch [3/20], Loss: 0.0153\n",
      "Epoch [4/20], Loss: 0.0128\n",
      "Epoch [5/20], Loss: 0.0113\n",
      "Epoch [6/20], Loss: 0.0098\n",
      "Epoch [7/20], Loss: 0.0090\n",
      "Epoch [8/20], Loss: 0.0085\n",
      "Epoch [9/20], Loss: 0.0081\n",
      "Epoch [10/20], Loss: 0.0077\n",
      "Epoch [11/20], Loss: 0.0075\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [12/20], Loss: 0.0072\n",
      "Epoch [13/20], Loss: 0.0069\n",
      "Epoch [14/20], Loss: 0.0067\n",
      "Epoch [15/20], Loss: 0.0065\n",
      "Epoch [16/20], Loss: 0.0063\n",
      "Epoch [17/20], Loss: 0.0061\n",
      "Epoch [18/20], Loss: 0.0059\n",
      "Epoch [19/20], Loss: 0.0058\n",
      "Epoch [20/20], Loss: 0.0057\n",
      "test performance :  [98.1499939  83.08999634 77.29666901 68.36000061 64.93999481 55.65999985\n",
      " 52.50428391 50.11000061 50.23666382 47.58599854]\n",
      "individual errors:  [tensor(16.1900, device='cuda:0'), tensor(13.5100, device='cuda:0'), tensor(26.5600, device='cuda:0'), tensor(25.2500, device='cuda:0'), tensor(37.5300, device='cuda:0'), tensor(51.8500, device='cuda:0'), tensor(57.4500, device='cuda:0'), tensor(65.8500, device='cuda:0'), tensor(84.5900, device='cuda:0'), tensor(97.0800, device='cuda:0')]\n"
     ]
    }
   ],
   "source": [
    "noreg = run_simulation( Net().cuda(),train_loader,test_loader,L2(lam=0),num_epochs=num_epochs )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABKIUlEQVR4nO3dd3iU15nH/e+ZkTTqXUK9gyqoIqpExxiwMcV24sSJ8yb2Js463pqyu8mbTfImm42Ttb2b6pJmx8ammGKbLkB0hARqIIR6Rb2XGc2c948RzRhMk2ZGOp/r4gKeGY1ujcXPR/dzipBSoiiKotgejaULUBRFUe6NCnBFURQbpQJcURTFRqkAVxRFsVEqwBVFUWyU3Xh+Ml9fXxkRETGen1JRFMXmnTlzpk1K6ffJ6+Ma4BEREeTl5Y3np1QURbF5QoiaT7uuWiiKoig2SgW4oiiKjVIBriiKYqNUgCuKotgoFeCKoig2SgW4oiiKjVIBriiKYqNsIsA/OlDF7/9wltbeIUuXoiiKYjXGdSHPvSo50Yxz7SA/LcplIMWDtbNCWRTnj85Oa+nSFEVRLMYmAvyfv5PJ/u2VyD119OX38oPzBejd7Xg0OYj1aSHMCPFACGHpMhVFUcaVTQR4++9+R8yhw4QtWMexmkCe6dfQGOjEu6fq+MvxGmL8XdmQHsLa1GCmuDtaulxFUZRxYRMB7hASghwxYPrfH5Lm6MGFWd9ClgfwP6mBdCV6sqWokf/6+AL/vesC86f6sT4tmIcSA3C0Vy0WRVEmLpsI8MMRrZx7IZJ1Tl8h8NglnHa+wSX7FKrkw7idq+HVld4Mr53P1sJmtuQ38OK7Z3HT2bFqRiAb0kNID/dSLRZFUSYcMZ6HGmdkZMh72Y3wt++t4Y3+SwxrNMSNwGP2/izrCaex2ItThgWYNPYkNG4ndn4Ybqsf4azjFDYVNPBxUTODBiMRPs6sSwthXVowIV7OY/CVKYqijB0hxBkpZcZN120hwGm7RHfdcT6uO8DW7lJKTQPYS8mS/gFWteloafwWrSKWkMZDxJRvwTHAFY8lc7Ff9Rh7RkLZXNDEicoOAGZHebMhPZSHkwJw0dnEDyCKokxyth3gn1DWUcbW8s3srNhJt6GXII07j9augboMvE1VpFz6HTT2AeDkZ8AjyR271Kmc1wazu8WdEz2+XLYPITspgg1pIcyO8kGjUS0WRVGs04QK8CuGjcPk1OWwtXwrxxuPE9k+gyWVX8Rea8fC+UP4FX9I96F89M09oAHXwCE8IgZwDRpCo4Um6UO5KYjLDmF4hSWSmJxBYNQMcAsA1TNXFMVK3FeACyFeBJ4FBPCalPJlIUQK8DvAERgBnpdSnrrd6zzoAL9eU18TH1R8wN7CQyQXrMR3IISh6fWs2JBOdIeG7u076Nm5k5HWVjTOOtzSwnGb5oDBvgGnnkqc5ODV1zJoXRD+07DziwXfqeAXC77TwCsS7BzGpH5FUZRbuecAF0IkAe8CmYAe2AV8HfgN8D9Syo+FECuBb0spF97utcYywK8wSRMn6k6Su/EirhWhNLqXUz3zGKuSHmJl+Arsz16ge/sOevfswTQwgF1AAB6rVjEyO428tmaqy87i3lfFVE0TCQ7NeI20XntxoTUHelAaBKWafwVMB3s191xRlLFzPwH+OLBCSvnV0b9/HxgGlgBvSik3CiE+DzwipXzqdq81HgF+vYLcSo5trEKvHWRX9Bu0edWyJGwJj019jEzPZAYOHqJ7+w76jhyBkRF0sbG4P7KahrRsNtfp2Xa2kZHBHtJd29kQNsA8zw58esugMR/6R4NdYwf+8aOBPhrs/glqpK4oygNzPwEeD2wD5gCDwH4gD/MIfDfmtooGmCulvOngTSHEc8BzAGFhYek1NZ96NueYaW/oY9cfiulqGWAorY4tLq/RY+gmyCWIx2IeY03MGvz1jvR8/DE923cweO4cCIFzZibOq1ZxJiyFTWVd5JS1YjRJpgd7sDYliMdiBN5dxdBYYP7VkA9DXeZPqtVBQNK1UXpQmrkNo1ELixRFuXv32wP/KvA80A+UYB6Ba4BDUsrNQogngOeklEtv9zrjPQK/Qj80Qs5bF7iU10JokhfaxZfZVr+FE00nAJgdOJu1U9eyOGwxor6Z7p076dm+A31NDcLBAdfFixHLVrDbJZLNhZcpaexBqxFkT/VlbVoIyxOm4Gingc7q0UDPh8az5l/6XnMR9s4QMAOC066FuncUaGxiQ0hFUSzogc1CEUL8FKgHfgZ4SimlMC9z7JZSut/uYy0V4ABSSooPNXDk/XJcPHQ89FwSRt8+tl3axgeXPqCxvxF3B3dWRa1ibcxa4rzjGCoqMt/8/OgjjB0daD08cF22lN7kTHbYhbL5QgdN3UO46ux4OCmAtWnBzI68bkqiyQTtl66N0hvzoakQRkZvmOrcITD5ulBPBc9wNQNGUZQb3O8I3F9K2SKECAP2ALOB48A3pJQHhRBLgP+WUqbf7nUsGeBXXK7qYddrRQx065m3YSrTFwYjkZxsOsnWS1vZX7MfvUlPvHc8a6euZWXkStw1zvQfP27ulx88iKmvD2Fvj9PMmbQlZrDTJYr3GiX9eiNBHo6sSQ1mXWowU6e43VyAcQTayswtlyvBfrkYjHrz407e18L8SrC7B43vm6QoilW53wDPBXwAA/BPUsr9Qoj5wCuY91MZwjyN8MztXscaAhxgqN/Avj+VUlPUTkyGP4u+GIeDo3lVZvdwNx9VfcTW8q2c7ziPg8aBJeFLWBuzllmBsxAjRgbyC+g7dIi+gwfRV1YCYBceQWtiOrvdonl32IdhtCQFu7M2NYRHk4Pwc9PduqARPbSUXNdPL4CWUpBG8+OuATeHuovvWL9NiqJYiQm5kOd+SJMkf08NJ7dV4uHvzIrnkvAJdr3hOefbz7P10lY+rPyQHn0PQS5BLI9YTvqUdFL9U/HQeaCvq6Pv4CH6Dh1i4ORJpMGAcHGhLS6FfR4xbHMIp8fZg/kxvqxLC2Z5QgBODndwM9MwCM3Fo/300WBvLQNG/3u5B5t76gHTIXCG+c+eYar9oigTkArwW2go62T3GyUYBkdY8IVY4mYH3vScYeMwB2oPsLV8K6cvn2bENAJAjGcMaf5ppE5JJd0/nSnCnf4TJ64G+khLCwCdoTEc9p7KAc+pNPqH89D0YNalBTM7ygft3SzhH+4199Cv9NKbC6HtIkiT+XFHj9FQnzEa6tPBNxa0as8XRbFlKsBvo797mD2vl9BY3kXC/CCynpyK3S32Eh8aGaK4rZj8lnzyL+dztvUs/YZ+AAJdAkn1TzWP0P1SCW7WM3A4l76Dh8zTE6VkwNWDE36xHPONoyF6Og/NimZdagixAZ/SL78ThkG4XArN50ZDvQgul1y7UarVwZSE60bryTAlERxc7u3zKYoy7lSAfwaT0cTJHVXk76rBN9SVFc8l4eH32VvPGk1GLnZeJL8lnzOXz5B/OZ/2oXYAPHQepPqlkjYljVSHaEJL2hjKPUJf7lFMvT0YNVqKfSI5OSWe9umZZC9O49GUIPzd7nNlp3HEPPuluei6YC+Ewc7RJwjwibnWernyu+qrK4pVUgF+h6oL29j3p1KkhCVfiicq1e+uPl5KSV1vHWcun6GgpYD8lnxqesyLlxy1jkz3m066TwozW90IKmxm4NBRjJUVADS4+HI6IJ7B9Nmkr17EspRQnB0eUPtDSuhpuBbmV0br3bXXnuMWdK31ciXY1bRGRbE4FeB3oadtkN2vFdNS00vy0lDmrI1Gq733BTdtg23mML9sHqWXdZZhkiY0QkOcdxzzxFQyKgVup2oQBWfRjhgY1DpQGBDLcMZsktauZHZm7N31y+/UQMfoSL3oWrC3Xbw2A0bnceON0oDp5lWlWvsHX4uiKJ9KBfhdMhpMHN1UTtGhBgKjPVj+tURcvR7MplX9hn7OtZzjTIt5lF7YWsiwcRiAGMdQlrcFEVk8iPeZStx6ugCo9g5hKH0OcWtXELdwNmIsV3AaBs3TGK8frX+yr+4fbw71qEUQsxQcb7uGS1GU+6AC/B6V510m568X0NprWP7/JBKa4P3AP4fBaKCkveTqKD2/JZ8efQ9IyYxuT+ZWeRBZ3EtYQxtaKel1dGMwfTZxTzyC38JsNLrbzDF/UExGc1/9Sqg3F5q3ChjqAq0DRGZD7ErzL/ebZ/IoinLvVIDfh87mfnb9oZiOpn7mro0hdXnYmH4+kzRR2VV57cZoSz7N/c24DkjSq+1Ju+jIjEt9uBhGMDjosJs9j+BHHsZ14QK0bvc4m+WeCjVC3Um48KH5V2eV+XpwujnI41ab2y2qh64o90UF+H0yDBvZ/+fzVOS3sOTL8cTNGd9RZlNfk7nlcrmAU82nqOusYkaNHenFHmRU9OE9NIDU2uE8exYey5bhtmQxdn53dwP2vkgJrReuhXljvvm6d9S1MA/NVDsyKso9UAH+ABhHTOz433M0lXfxyLeSCYl78O2UOyGlpLitmM3lm/mo6mOGDP3E1nqTUujB3OoOAvo6QAicUlJwW7oUt2VLcQgb258abtLTCGUfm8O86jCYDODsC9NWQNwqiFoIDp89TVNRFBXgD8zwgIEtL+XT1znMun9NwyfI9bM/aAwNGAbYVb2LTRc3UdRWhJAaAqujmVnqwtLmFqa0mKcJ6qZNuxrmurg4xHi2NYZ64NI+c5iX74XhbrBzgujF5jCftgJcfMavHkWxMSrAH6Ce9kE2//wMGjvBhu9k4OIxDjcR78DFzotsKd/C9ks76DX0IEa8ca9JYHG1Mw931OBZUQpSYh8cfDXMnVJTEdpxbGuM6KHmqDnMyz4yz00XGgidbQ7zuJXmtouiKFepAH/AWmt72fLLfLymOPPYP6Ve3c3QGgwbh9lfs5/3L24i7/JpkIKRvjjcL0/nmSEnsi6XYleQhzQY0Pr44LZ4EW5Ll+I8Zw4ah3E8Ck5KaDp3LcwvF5uv+8WbgzxuFQSmqkMvlElPBfgYqC5q46PfFBKW5MPKr09Hcx+LfcZKbU8tW8q3sPniVrr0HTDiznBXOtPkbJ53GCGxooChI7mY+vvRuLjguiAbt6VLcclegNZ1nPdL6ay+1jevOWZeTOQWCLEPm8M8IludNapMSirAx0jxoXoOvXORpAXBZH9u2vj2lu+CwWTgcP1h3i/bzLHGo0hMjPTHoO2bzbqoRXzBvgP3vKP07j+AsaMDYW+P89w55lbL4sXY+Yxzj3qgA8r3wIWdcOkAGPrBwQ2mLjOHecxScPIc35oUxUJUgI+hY5svUbC3lrnrY0hdNs6zPe5Bc38zH1z6gHfPb6Z9uBlpdMbQlUa861K+lplJ1nAzQwf207tvH4aGBtBocEpLNYf50mU4hASPb8GGIag6ZA7zso+hvxU0dhAx3zw9MfZh8AgZ35oUZRypAB9D0iTZ/XoJFfktPPRsEjHp/pYu6Y4YTUZONp3kb+ffI7fhECZGMA6Eoxuaw+fiV/P07Kl4N9XQu3cfvfv2MXzxIgC6+Hjcli7BbekyHGOnjW/RJhM05JnD/MJH0F5uvu4dDRHzICILwueBxzj/T0ZRxpAK8DE2ojey7eWztNb2suYfUwmM9rB0SXelfbCdbZe283bp+7QM1SGNOkZ6UkjzWsE35i5gXrQvI/V1V8N88OxZkBKn5GS8v/IMbkuXIuwscCO39SKU74bqo1B7DIa6zde9Im8MdM/Q8a9NUR4QFeDjYLBPz+afn2F4cIT1307H09/2FqpIKclvyeevxe9xsH4vRgwYh4JwN8zj6aTH+GJmPB7O9oy0ttLz8S463noLQ20t9kFBeH3paTw3bEDraqG58SajeSZL9VGoPmKerjjUZX7MM9zccomYbw50r3DL1Kgo90AF+DjpujzA5v8+g87ZjvXfScfJ1XZnTfToe9h+aSd/LnqP5qEKpMkeU98M5vqt5IV5y0gO9UIajfQdPEj7H//IYN4ZNC4ueD7+ON5PfxH7YAu3MUwm82HR1UehOtc8s2Www/yYR+i1MI+YD14Ras8WxWqpAB9HTRXdbPufAvzC3Fjzjym3PJ7NVkgpKe0o5fWCdzjYsIcRBjEO++Ens1gStpzsqKmkhnrhUHGBjj/9mZ5duwBwW74Mn2eewSk52cJfwSiTybxfS/URqDli/n3AfHoS7sE3Brp3lAp0xWqoAB9nl860sPu1YqLT/Hnoa4mIsTiMwQIGDANsK/+IPxW9R+PQeQCMQ4GM9MYTYJ9GZuAMZrsZSTy5G82H2zD19uKUmor3M8/gtnTJ+K76/CxSQmvZ6Oh8tO3S32p+zC3wukDPAp9oFeiKxagAt4CCPbUc23KJ1OVhzF0XY+lyHrjq7mp2Vx9gV+V+KnqKkZhgxB19bxwjffF4D0bwhc4LZJ3bh3P7ZbTBwfh++ct4rFs3/ouE7oSU0FZ+Y6D3XTY/5jrl2ug8Yj74TlOBrowbFeAWIKXk8DsXKT7cwIKnYknKnrhT27qGushtyCWnLocj9UcZNA6gxQEHQxw9LdGkXxSsvXiKxI5qhh2d6Vq0kinPPM206VPH5qi4B0FKaK+41m6pPgq9jebHXPxuDHS/OBXoyphRAW4hJqOJj35XRG1xOyufn0HE9Il/8rveqCevOY+cuhwO1R+iqb8JgcDfYSrRVYHMOtxIZk0ZAMdDUyjLXk1gZhppYV6khnni6WylN36lhI7Ka6Pz6qPQU29+zNnXfCpR0jqYuhzsrGODM2ViUAFuQfqhET74VQGdlwdY989p+IWN46k5Fial5GLnRXOY1x2iuN28YVXcsD+PnnFh+oladMPDFPtEsSU6m5OBCUT4uZEa5kVauCdpYV5Mm+JmnaN0KaGr5lqYX9pr7qE7ekDCYzDjCQibqzbjUu6bCnAL6+8eZtN/5WEySTZ8JwM37wdzQLKtaRlo4VD9IQ7VHeJE0wk0A0OsKHZgdR64dQzS7zuFE6kP8Tev6TTqzaHt4qAlOdQc5mnhnqSEeuHtYoWjdOMIVB6Eovfg/E7z/i3uITB9PUx/AgKSLF2hYqNUgFuB9oY+tvziDK7ejqz713R0TtazBa0lDBgGONF0gkP1hzhcnUN0UTuPnJJMa5CMuDgysvph6rMe51SfPQV1nZxv6sVoMn+/Rvq6kBpmDvWMCC9ip7hZ10Zi+n7zvi2F75kPs5BG8E8wj8qTNqiVocpdUQFuJeoudLDz1XMETfNk9d8no7VTP16D+SDn4rZiDtYd5NKRj0g5UMusMokUcHlODD7PfIXomasoaewjv7aTgtouCmo7aevTAxAX4MbjGaGsTQ22vtF5fxuUbDWHef0p87XweTD9cUhYA86WOZpPsR0qwK3IheNN7P/zeeLmBLD4S/HWNXK0EvW99Rw/8wHD724h4VgTTnooC7en6ZEMpq76HHND5uNk50RdxyCHy1t5P6+Oc/XdOGg1LEucwpMZocyP8UVjbb3zjioo2mRus7RdBI29+abnjMfNR8vZO1m6QsUKqQC3Mqd2VHL6w2oyH4lk5qpIS5dj1brbmyj+48s4bN6La+cgjV6wO9OeweWzmR+9hAWhCwhwCeBCcw8bT9extaCBrgEDwZ5ObEgP4fGMEEK8rGxfmiunERW+B8Wboa8ZdO4Q/6g5zCOyQGNFi54Ui1IBbmWklOz/83nKTjSz5Jl44mYHWrokqydHRujc/TGNr/0O7YVK+p007E6V7E7TEBmdzuro1SwPX46j1pW9pZfZeLqOI5faAJgf48sTGaEsT5yCzs7KgtFkhKrDUPQ+lG4Hfa95JWjSenObJTBZzTGf5FSAWyHjiIkd/3uOpktdPPJCMiFxqhd6J6SUDBYU0P7HP9G3bx8mjeBSlCMHo4coiHUgJW4Bq6NWkxWSRWuPkU1n6nk/r56GrkE8ne15LCWYJ2eGEh/obukv5WaGQbi4CwrfN59IZDKYV33OeMIc5l4Rlq5QsQAV4FZqeMDA5l/k0981zPp/Tcc7yAqXmFsxfW0tXe9vomf3bgy1tUghKA+348hUIyVJbmROX8GqqFWk+KVxvKKDjXl17C25jN5oYkaIB09khPJoShDujvaW/lJuNtABpdvMbZbaY+ZrobPMQZ64DlzG+Zg7xWJUgFuxnvZBNv/8DFo7Deu/k46Lh1rFd7eklAxfvEjv7j307N6NvqICgPIQDcenQWWqP3PS1rA6ajU+DmFsLWjgvbw6LjT34mivYeX0QJ7MCCUz0ts6byp31Y7e/HwfWkrNR8pFLzGPzGNXgoOV9fiVB+q+AlwI8SLwLCCA16SULwshNgKxo0/xBLqklCm3ex0V4LfWUtPD1l/m4xXgwmP/lIqD4+SeI36/hisr6d2zh+7du9CfNy/brwwQnIgVtM6KZs6s9ayIWEFzh46NeXXsONtI7/AIkb4uPJ4Rwoa0EPzdrXSxVXOxeRZL0SboaQAHV/PZoDMeh8iFoFXfOxPNPQe4ECIJeBfIBPTALuDrUspL1z3nl0C3lPJHt3stFeC3V13Yxke/LSQ8yYeHvzHD+qbA2Sh9XR29e/bQsesjRopKAaj1g5OxGvrnz2DOvCeYF7SYwxd62ZhXx6mqDrQawaJYP57ICGVRnD/2Wiucr28ymfdlKXrP3GoZ6gYXf4h/BGKWmGeyOFphn1+5a/cT4I8DK6SUXx39+/eBYSnlf4/+XQC1wGIpZfntXksF+GcrOljP4XcvMn1BMFmfm2adP87bMENTE71799L60Q6M54oREhq9IS/OHrloNnMWPEWQYypb8pvYdKae1t5hfF11rE8P5smMUKL8LHRc3GcZGTbf9Cx8Dy7tNy/jF1oImQnRiyF6EQSlqdG5jbqfAI8HtgFzgEFgP5AnpXxh9PFs4Fef9uKjjz8HPAcQFhaWXlNTcz9fx6RwdFM5Z/fVMW9DDClLwyxdzoQ10tpKz959NH24BQpK0JgkLR5wNsEJ+yXZzFr8NO2dwbyXV09OWQtGk2RmhBdPZISyakYgzg5WGoYjevOKz4ocqDgAjQWABJ0HRGVD1CJzqHur9Qe24n574F8Fngf6gRLMI/B/GH3st8AlKeUvP+t11Aj8zkiTZPfrxVQUtLLi2SSi0/wtXdKEN9LZSfe+PdRtfx+7/FK0Rkm7G5QmueOybAlx87/IqUod7+XVUdXWj6vOjkeSA3lyZhjJIR7W/ZPSQAdUHTKHeUUOdNeZr3tFjI7OF5vbLU6elqxSuY0HNgtFCPFToF5K+RshhB3QAKRLKes/62NVgN+5Eb2RbS8X0FrXx2P/mEpAlIelS5o0jD09tO3dRc2OjTjlncduRNLlDBXJPrgvewjn5Cf4uGSID4saGTKYiJ3ixhMzrXQflk+6ckhF5ejovOow6PtAaCA4w9xqiV4MwemgtcKplZPU/Y7A/aWULUKIMGAPMFtK2SWEWAF8T0q54E6KUAF+dwZ79Wz67zPoB0dY/+10PP3VVLHxZurvp37PDmq3b8T1zEV0ehO9TlCbHIDrshU0h65ia2EX5+q7EQJip7gxK9KbmZHeZEZ4W+9MliuMBqjPM4d5ZQ40nAFpMi/rj8i6FujqkGeLut8AzwV8AAPwT1LK/aPX/wSckFL+7k6KUAF+97ouD7D5v8+gc7Fjw7czcHRVoyJLMQ0NUb7rfep3vI9X3iWchiUDOmhOCUWTvZyLAcs43TREfm0nA3ojAOE+zsyM8CZzNNDDfZytu90y2GkelVfkQMV+8/xzAM8wc5BHLYKoBeDkZdk6Jxm1kMeGNV3qYtvLZ/EPd+PRf0jBzt7K9vKYhEaGhyj6+C0ad27B/0w1roOSIQdBc0Y4nqvX4JCyhoKGQU5WdZBX3UHngAEAPzcdmaOBPjPCm9gAKz1tCK4dIVeZYw70qsMw3GNutwSlXRudh8xU7ZYxpgLcxpXnXWbP6yXEpPuz/KuJCGv9Rz8JDQ31cfrjP9G+cxshefW4DEO3i6BhdiR+j60nLftJmrpMnKru4FRVB6erOmjsHgLAzdGOjHAvMiN9yIz0YnqwJw7Wuke8ccTcYrnSbqnPMx9U4eA62m4Zna7oE6PaLQ+YCvAJIH93Dce3VpD2UBhz1sZYuhzlUwwO9HD2gzfo3bmTgHON2BuhyVtQPy8avzXrmZe5DncHd+o7Bzg9GuinqjqoaO0HQGenISXU09xyifQmLcwLF52VTlcc6oaq3NHZLQegs8p83SMUohaab4R6R4JXJLgHqzno90EF+AQgpeTQOxcpOdzArEejSF8RrkbiVmy4s52SzW/Qt+ND/MpaACgPFtTPjSbg0fVkJ63G18kXgPa+YU5Xd5pH6NUdlDR2Y5Kg1QgSg9yZGeE9+ssLH1cr3Suno+q6dsshc8BfobEz99G9Iq+FunekeSqjVwQ4qE3cbkcF+ARhMprY96fzlJ++TPh0H5Z+OUHd2LQBww31XHz/TQY+3I17XQcjGjgXKaifG0XIw2tZNG0Fwa7BV5/fNzxCfo050E9Vd3C2rgv9iAmAGH/X0Ruj5tZLsKcVnuJjMpn3aemsNo/MO6pu/P36cAdwnXJjuHtFXPuzi++kb8moAJ9ApJQUH2rgyKZynN0cWP61JAKj1TxxWzF44QI1m/7K4Ed7cOzoY9ABTk0T1MwOJ3LJGpZGLifKM+qGjxkeMVJY3311hH6mupPe4REAgj2dmBnhdXXqYoy/q3XPdAHz4qKbwn307z2NwHW55OA6Gurhnxi9R5rbNZOgNaMCfAJqqelh92vF9HUMM/uxaFKWhqqWig2RJhMDp/No3PIOQ3tzsBsYpssFjsYLKmYFEzd3FUvDl5Hgk3BTIBtNkvNNPZyu7hjtpXfS1jcMmG+Mxk5xIzbAjbhAd+IC3Jg2xQ0PJxv5Sc0wZJ6++Gkj984aMA5fe67Qgmfop7RmRkfxOivdu+YuqQCfoIYHDOT89QIVBa1ETPdhiWqp2CTT8DB9hw7RsnUTw7nH0IwYafQW5CYKLsz0JzVlBYvDFpPmn4b2U87KlFJS1dbP6eoOCuu7KWvupay59+ooHSDIw5HYADdiA8yhHhvgRrSfq/XOevk0JhP0Nt0i3KvN89iv5+I/uqHXIvMcdp9om2zHqACfwKSUFB1s4OimcpzdHXjo2SS19N6GGbu76dmzh44PtqI/UwDAxWANuYlwPsWbmXFLWBK2hFmBs3DQ3nrpvpSSxu4hypp7uDAa6GXNvVS09mEwmv/d22kEUX4uV0P9SrAHezpZfxvm0wx23Rjq7RVQnXttQdKVGTLRi8x7p9vIqUYqwCeBG1oqa0dbKrb4j1C5ytDYSPeHH9K1fTuG8kuYNILCaC2H4k2cT3RjVmQ2S8OWMj94Ps72d7bVgn7ERGVbH2XNvTcEe0PX4NXnuOnsmDYa5nEBbsROcSMuwB0PZxv86U5Kc5hX5JhnyVQdvnYTNWDGtdF52Bywt86tD1SATxLDAwYO/OUClWdbiZjhy5Ivx+PoYoP/6JSbDJWV0bNjB107d2JsvoxBZ0derJb98QbKox2ZHTKPpeHmMPd2vPsDsnuGDFz8RKhfaO6hZ+haGybA3ZG4wOuD3Z1ofxd0dja0Otg4Ak1nrwV63Snz4dF2juYQvxLoU5JAYx3tJRXgk4iUksKceo5tvoSzhwMPfU21VCaSKzc/e3buoGfXbky9vQx5OHI8Qcvu2CEqAyDYLYQEnwQSfRJJ8k0i3iced4e7P51HSklzz9AnQr2XSy29V9swWo0gytflWqiPtmNCvGykDTPcZz7Z6Eqgt14wX3fxg8gF1wLdI/j2rzOGVIBPQperzS2V/s5h5qyLJnmJaqlMNFdufvbs2EHfwUNIg4GhAE/Kk7w4HDFIrk8rptGZSeHu4VdDPdEnkQSfhDtuu3ySwWiiqq1/NNh7rgZ7fee1NkyYtzPr00JYlxZMqLcN7aTZ0wiVB0cD/SD0mxdh4Ttt9DCMRRAxH3Ru41aSCvBJSrVUJg9jdzc9u3fTu3cfAydOIA0GhLsbQzMTqZruw/GwQQoGLtLc3wyAQBDlEUWib+LVYI/zjsPR7t77wL1DBi5e7qO0qYfdxc0crWhDSpgd5c36tBBWTg+03q0BPo2U0FJ6bXRefRRGBs0rS0NmXgv0MT6uTgX4JCalpPBAPce2XMLFQ8fyZxMJiFQtlYnM2NdP/7Gj9B3Ioe/gQYxdXWBvj8vMmYj5M6mZ4U+R3WVK2ksobiumfagdAK3QEuMZQ6Jv4tWR+lSvqbed7XI7DV2DbM2vZ3N+A1Vt/Tg7aHk4KZD16cHMjvSxvYO7R4ah7uS1QG88y9Xj6iKzRme4PPj901WAK1yuGm2pdA8zd10MMxaHqJbKJCCNRgbPnqX3wAH6cg6ir6wEQBcbi+viRbguWkRPpB+lnecpaS+hpK2EkvYSuoa7ALDX2DPNa5o50EeDPdozGjvNnY84pZTk13ay6UwDO8810js8QrCnE+vTglmfHkK4j43uhXL1uLrRQL86XTEMoheO7p++EJzv/qby9VSAKwAM9Rs48JfzVJ1rIzLZl8VfUi2VyWa4qoq+nIP0HTjAQH4+mExo/XxxW7gI18WLcJkzB6HT0djfeDXMS9pKKG0vpdfQC4BOqyPOO+6GUI9wj/jURUafNGQwsrukmc35DeSWtyIlzIzwYkO6ucXi5mij34837Z+eC8PdgIDAZHj45xA2+55eWgW4ctXVlsrmS7h46Xjoa0lMibz7GQqK7Rvp7KQ/N5feAzn05+Zi6u9HODriMncubosX4bpwIXa+5h0TTdJEXW/d1VAvbivmfMd5BkfMNy6d7ZyJ94m/2nrJDMy8utvirTR1D7K1oIHNZ+qpaO3H0V7DisQA1qeHMDfa13oPu7gTn5yuuOqX4B9/Ty+lAly5SXNVN7tfK2agW69aKgomvZ6BU6fpO3CA3oM5jDQ2gRA4zpiO26LFuC5ehG7q1Bu+R4wmI9U91VdH6cXtxZR1lDFsHEYrtMwLnsej0Y+yMHQhOu2tt8GVUnK2rovN+fVsP9tIz9AIgR6OrE01t1ii/SbGnib3SgW48qmG+g3s//N5qgtVS0W5RkrJcFmZuW9+IIeh4mIA7ENCcF28CLdFi3DOyEDY3/y9YjAZKO8sZ2/NXrZXbKdloAV3B3cejnyYNdFrSPJNuu1AYchgZN/5y2w+U8+hi62YJKSFebI+PYTVM4JsZ1OuB0gFuHJLUkrO7a/j+JYKc0vl2SSmRKiWinKN4XILfQfNffP+48eRej0aNzdcs7JwXbwY16z5aD1untlkNBk52XSSbRXb2F+7n2HjMJEekayJXsPqqNVMcZly28/b0jPEB2cb2HSmnouX+3Cw07A8YQob0kPImupn2y2Wu6ACXPlMzZXd7H7d3FKZtyGG6QtVS0W5mWlggP5jx+i9MkWxowO0WpwzMsx980WLcAgLu+njevW97Knew7aKbRS0FKARGuYEzmFNzBoWhS667fxzKSVFDd1sPlPPtnONdA0YmOKu47HUYDakhTB1yvgtqrEEFeDKHbm+pRKV6sfip+PQ2eIGRsq4kEYjg4WF9B3IoTfnAPpLFQA4xETjMms2jomJOCYmoouOQthdm3ZY01PD9ort7KjYQVN/E272bjwU+RBroteQ7Jd824HD8IiRnAstbDpTT05ZK0aTJDnEgw3pITySHISn873NWbdmKsCVOyal5Oy+Ok5srcDV29xS8Q9XLRXls+lra+nLyaE35yCDhYXIgQEAhKMjjrGxVwPdMSkRXVQU0k7L6ebTbK/Yzt6avQyODBLuHs6j0Y/ySNQjBLoG3vbztfYOs220xXKhuRcHrYalCf6sTwthwTQ/7LTWsRnV/VIBrty15srRWSo9qqWi3D1pNKKvrmaopIShkhIGS0oYLj2P6Uqo63ToYmNxTEzAKTERGRvFIfsqtlXvJO9yHgJBZmAma6LXsCRsyWfu21LS2M2mM/VsO9tIR78eX1cdj6UEsS4thIQg2x6AqABX7slQn4F9fy6lpqid6FQ/Fn0pHp2TDe1loVgVaTKhr665GupDJSUMlZZi6u8HQDg4oIuNxTgtnGLvAXbal5Ln2opO58JDEQ/xaPSjpE9Jv+1AQj9i4mCZucVy4EILIyZJXIAba1ODWZMSTICHde75fTsqwJV7Jk2Sgn21nPigEjfVUlEeMGkyoa+pYai0lKGS0muh3mte9Snt7egIcqPQp4+L/kb6I6eQPnctq+PWEuIWctvX7ujXs7OwkS35DZyt60IImBvtw9rUEFYkBeBqIxtrqQBX7ltTRTd7Xi9moFfP/A1TSVoQrFoqypiQJhOGurqrrZeh0lKGikuuhvqIBmr9oDfSnylpc0nJWodHwgw0ulsvFqps7eODgga2nm2grmMQR3sNDyUGsDY1mPkxvlbdL1cBrjwQQ30G9v2plJpic0slMsUPOwcNdvZa7Ow1aB002Dto0dqPXnPQmK/ba1TYK/dFSomhvp6hkhLaCk7SnH8M3aV6nAdNAJi0AhkZiveMDByTEnFKTEQXG4vG0fGm1zlT08mWggY+LGyie9CAr6uOR5ODWJcWTGKQu9V9r6oAVx6Y61sq0nTn3z9XAv5K2Ns5aNDaa7Ef/f1K2Jufd9310ede+x+CdvR1zH92dLXHw89pDL9ixVqZTCYKi/aTd+hdOs+dIaRRT8xlgeuAOdSxs8Nl1iw81jyK29KlaJxvvBF6ZUri1oIGDlxowWCUTPV3ZW1aMI+lBBPkaR3fVyrAlQduqN/AUJ+BEYOJEb3x6u9Gg+naNb2JEYP5MaP+uud92nOv+91oMGHQGzGN3Nn3Z0yGP/M3TMXF89Y/QisT29DIEAdqD7D90jYuXjhGRLOJrE5/UosH0bV2I5ydcV+2FPdHH8Vl9myE9sadE7sG9OwsbGJrQQNnajoRAmZFerMuNYSHpwdYdJdEFeCKTTKZ5GjIj/7P4LqAv/Lny1U9FOypRWMnmPVoFNMXhtjeQQHKA3W5/zI7K3eyo2IHlV2XiKuDhy86k14yhP2AAa2fHx6rV+Ox5lEc4+Ju+via9n4+KGhka0E91e0D6Ow0LEuYwrq0YLKm+mE/zv1yFeDKhNbVMkDuuxepLe3AN9SVhU/FqS1yFQDqeus40nCEw/WHOVt/isSyIRaVaEi+ZERrkmhjovBZuw731auxn3Lj3ixSSgrqutia38DOwkY6Bwz4uDjwSHIQa1ODmRHiMS79chXgyoQnpaQiv5Uj712kv0dPYlYws9dEqd0VlasGRwY53Xyaw/WHyS87SOSZJrKLTUxrBCnAlJZI0PrP47F8BVrXG08J0o+YOHSxla0F9ew734J+xESUnwvrRueXj+XBzSrAlUlDPzjCqR1VFObU4ehqz7z1MUybFWB1MwsUy5JSUtldSW59LoUFu/E+VMS8YiMBXTBir2Fw7gxCH/8iAQsfumEfF4DuQQMfFTWxNb+BU9UdAGRGerM2NZiV0wMf+Ja3KsCVSae1rpdDfyvjclUPwdM8yf58LN6BNnr2ojLmevW9nGg8TumhrTjuPUla8SCuQ9DnZkfn/CSCNzxF3JyVNx0bV9cxwLazDWwpaKCytR8HOw1L4/1Zm2rej8XB7v775fcV4EKIF4FnAQG8JqV8efT6C8A3ASPwoZTy27d7HRXgyniTJknp0UaOb63AMGwkZVkYGSsjsHf47LMblcnLJE2cby6k9MO30e4+QkxpF/ZGaPLTcjkrDr/HNpCZshJ3h2v3WaSUFNZ3s7WggR3nGmnv1+PlbM8jyUE8lhpMaqjnPf8UeM8BLoRIAt4FMgE9sAv4OhAK/DuwSko5LITwl1K23O61VIArljLQo+f4lktcONGMm48j2U9OI2LG7c9rVJQr2i5XU/L+a8iPc5hS0QlASZigdm4kPg+vZt60ZUR7Rl8NaIPRRG55K1vyG9hbepnhERO/fiqNVTNuv7virdxPgD8OrJBSfnX0798HhoEM4A9Syn13WoQKcMXSGss7Ofi3i3Q29ROZ7EvWk9Nw87a9zY0UyxmsqaJ845voP9qLS3M3ejvImyooTvfBZ8FSsiIWkhmYiZOdeRFQz5CBXUXNrJoRiMs97r1yPwEeD2wD5gCDwH4gD8gavb4CGAL+RUp5+lM+/jngOYCwsLD0mpqae/oCFOVBMY6YOLe/jtM7q0AjmLkqguQloWiteC8MxfpIKRkqLKRpy0b6P96NXc8APc6Co/FwfLoDPumzyArJJjskm1C30Pv6XPfbA/8q8DzQD5RgHoEvBXKAbwEzgY1AlLzNC6oRuGJNetoHyd1YTnVhG95BLix4KpagGE9Ll6XYIGkw0Jd7hM5tH9B34ADCMEKrrz37E4zkJgpcwiL5wZwfMDNg5j29/gObhSKE+ClQDzwK/FxKmTN6vQKYLaVsvdXHqgBXrFHVuVYOb7xIX8cw8XMDmbMuGifXiXcslzI+jD099O7ZQ/e27QycNjclGqLcifyP/yR67op7es1bBfgdNWSu3KAUQoQB64DZgAlYBOQIIaYBDkDbPVWnKBYUmexHSJw3eR9VcXZvHZXnWpm7Lob4OYEItSRfuUtad3c8N2zAc8MGDA0NdO/YicP27YT4xzzwz3WnLZRcwAcwAP8kpdwvhHAA3gRSMM9O+Rcp5YHbvY4agSvWrr2xj8PvXKSxvIuAKA8WPBWLb4irpctSbJyU8r4WkqmFPIpyh6SUlJ1o5ujmSwwPjDBjcQiZqyNxcLSN01uUiee+WiiKMpkIIYibE0jEDF+Of1DBuX11XMprIeuJqUSl+qkl+YrVUPOmFOUWHF3sWfSFONZ/Ox1HV3t2/aGYD39dSHfroKVLUxRABbiifKaAKA+e+F4G8x+fSmN5F+/86CR5H1VjNJgsXZoyyakWiqLcAY1WQ/KSUKLT/Dnyfjknt1dSdrKZBZ+fRkict6XLUyYpNQJXlLvg6qVjxXNJrH4hGZNJsu3ls+x9s4T+7mFLl6ZMQirAFeUehCf68PnvZ5KxKoJL+S387YcnKTpYf1eHPCvK/VIBrij3yM5By6xHovj892fhH+7G4Xcvsu2VAnra1E1OZXyoAFeU++Q5xZlHX0xh0dNxtNT08u6PT1F6pJHxXGOhTE4qwBXlARBCkDAviM99PxP/CHdy3rrAzv8rpL9L9caVsaMCXFEeIHcfJ9a8mELWk9NovNjJOz86SdnJZjUaV8aECnBFecCERjBjUQhP/kcmXgEu7PtjKbv+UMxAj97SpSkTjApwRRkjnlOcWfsvacxZF011URvv/vgkFQW3PXVQUe6KCnBFGUMajSBteThP/NtMXL0c2fX7Yva+WcJQv8HSpSkTgApwRRkHPkGurP9OOpmPRHIpr4V3f3SSmuJ2S5el2DgV4IoyTrRaDTNXRbLhuxnoXOzZ+X/nyPnrefSDI5YuTbFRKsAVZZz5hbnxxPdmkvZQOOePNfHOj09Sf6HD0mUpNkgFuKJYgNZew5y10az713Ts7LVse/kshzdexKA3Wro0xYaoAFcUCwqI8uCJf5/JjMUhFOXUs/Enp2iq6LZ0WYqNUAGuKBZm76Al64lpPPaPqZiMkq0vneHYlkuMGNRoXLk9i+8HbjAYqK+vZ2hoyNKlTGiOjo6EhIRgb29v6VKUWwiO9eJz38/k6OZLFOyppaa4nSVfjsc/3N3SpSlWyuKHGldVVeHm5oaPj486a3CMSClpb2+nt7eXyMhIS5ej3IGaknZy/nKegV4DGQ+Hk74yAq1W/cA8Wd3qUGOLf0cMDQ2p8B5jQgh8fHzUTzk2JDzRh8/9YBZTZ/pz+sNqNv1XHu0NfZYuS7EyFg9wQIX3OFDvse1xdLFn2VcSefjvptPfNcx7PztN/u4aTOrQCGWUxXvgiqLcXlSqH4ExHhz6WxnHt1ZQda6VJV9OwHOKs6VLUyzMKkbgiqLcnpObAw89l8SyrybQ2TzAxp+c4tyBOnWE2ySnRuCKYiOEEEybGUDwVC9y3rrAkffKqTrbyuIvxePu62Tp8hQLsKoA/88dJZQ29jzQ10wIcuf/fSTxts956623ePXVV9Hr9cyaNYvFixdz8uRJfvWrX/HKK6/wyiuvUFlZSWVlJU8//TRHjx7l9OnTvPjii/T396PT6di/fz9ubm4PtHZF+TQunjpWfXMG5481ceT9ct798SnmbYghYX6QutcxyVhVgFvC+fPn2bhxI0ePHsXe3p7nn3+e4eFhcnNzAcjNzcXHx4eGhgZyc3PJzs5Gr9fz5JNPsnHjRmbOnElPTw9OTmoEpIyfK0e4hcR5ceAvFzj4dhmVZ1tZ9MV4XL10li5PGSdWFeCfNVIeC/v37+fMmTPMnDkTgMHBQfz9/enr66O3t5e6ujqeeuopDh8+TG5uLuvWraOsrIzAwMCrH+PurhZaKJZx5Qi3okMNHN9yiXd/fJI5a6OJmxOI1k7d4prorCrALUFKyZe//GV+9rOf3XC9rq6OP/7xj8TGxpKVlcWbb77J8ePH+eUvf0ltba2FqlWUm105wi0swZsDfznPwbfLOL2zihmLQ0nMCkLnrFbfTlST/n/RS5YsYdOmTbS0mI+66ujooKamhqysLF566SWys7NJTU0lJycHnU6Hh4cHsbGxNDU1cfr0aQB6e3sZGVF7OiuWdeUIt0deSMYr0IXjWyv48/eOceS9cnraBy1dnjIGJv0IPCEhgZ/85CcsX74ck8mEvb09v/71r8nKyqKuro7s7Gy0Wi2hoaHExcUB4ODgwMaNG3nhhRcYHBzEycmJffv24erqauGvRpnshBCEJfoQluhDa10vZ/fVUnSwnsKD9cSk+ZGyLEztrTKBWHwvlPPnzxMfHz9uNUxm6r2enHo7hijMqacktwHDkJHgWE9SloYRnuiD0KhZK7bgVnuhTPoRuKJMdG7ejsxbH0PGyghKjzRSeKCOD39diFegCylLQ4nNDEBrP+m7qTZJBbiiTBI6JztSl4UxY3EIl/JaKNhbS85fL3BiWyUzFoWQlB2Mo4u64WlL7ijAhRAvAs8CAnhNSvmyEOKHo9daR5/2b1LKj8akSkVRHhitVkPsrACmZU6h/kInZ/fWcnJbJWc+riZ+XhDJi0Px8FPrGmzBZwa4ECIJc1BnAnpglxBi5+jD/yOlfGkM61MUZYwIIQiN9yY03pu2+j7O7aul5HADxQfriUr1J3VZGFMi1Q1Pa3YnI/B44KSUcgBACHEIWDemVSmKMq58Q1xZ8kwCs9ZEU3SwjuLDjVTktxAY40HqsjAipvuqG55W6E7uXBQDWUIIHyGEM7ASCB197O+FEIVCiDeFEF5jVqWiKOPC1UvHnLUxfPlnc5n/+FR6O4b46LdF/O0/T1KS28CIXp3TaU0+M8CllOeBnwN7gF3AWcAI/BaIBlKAJuCXn/bxQojnhBB5Qoi81tbWT3uK1YqIiKCtrQ1AzfFWJhUHRzuSl4Ty9I/nsPyridjrtBx8u4y//PsxTn9YxWCf3tIlKtzhTUwp5RvAGwBCiJ8C9VLKy1ceF0K8Buy8xcf+AfgDmOeB32/BiqKMH41Ww9SZU4jJ8KfxYhcF+2o5taOK/F01xM0JJHlJqDpYwoLudBaKv5SyRQgRhrn/PVsIESilbBp9ylrMrZb78/F3obnovl/mBgHT4eH/+synPfbYY9TV1TE0NMSLL77Ic88992DrUBQbJoQgONaL4FgvOhr7ObuvltJjjRTnNhCVbF7hGRjtYekyJ507nQe+WQjhAxiAb0opu4QQ/yuESAEkUA383diUOD7efPNNvL29GRwcZObMmaxfv97SJSmKVfIOcmHxl+KZtSaKopx6ig83UHm2lYAod1KWhRGZ7IdG3fAcF3faQsn6lGtPP/Bq7mCkPFZeffVVtm7dCph3IiwvL7dYLYpiC1w8dMx+LJq0FeFcON7Euf117Pp9Me5+TqQsCSV+biB2DlpLlzmhqZWYwMGDB9m3bx/Hjx/H2dmZhQsXMjQ0ZOmyFMUmODjaMWNRKEnZwVSebaNgTw2H371I2clm1vxDKvY6FeJjRW2AAHR3d+Pl5YWzszMXLlzgxIkTli5JUWyORqshJt2fDd/NYPnXEmmp7mHXH4oxGk2WLm3CUgEOrFixgpGREeLj4/nud7/L7NmzLV2SotgsIQRTM6aw4KlYakvayfnLBaRJTUAbC6qFAuh0Oj7++OObrldXV1/9c19f3zhWpCi2LzErmIEePad2VOHs7sDc9TGWLmnCUQGuKMqYyVgZwUCPnoK9tTh7OJCyNMzSJU0oKsAVRRkzQgiynpzGYI+eo5su4eTmQOysAEuXNWGoHriiKGNKoxEs/X8SCJ7myYE/n6e2pN3SJU0YKsAVRRlzdvZaHv7GDLyCXPj4D8Vcru6xdEkTggpwRVHGhc7JjkdeSMbJ1Z6d/3eOrssDli7J5qkAVxRl3Lh46Hj0WykIAdtfPUt/97ClS7JpKsAVRRlXnlOcWf33yQz2Gdjx6jmGB0csXZLNUgE+Rn74wx/ypz/96ZaPV1dXs3DhwnGrR1GsiX+4Ow//XRKdTf189JtCRgzqoIh7YVXTCH9+6udc6LjwQF8zzjuO72R+57bPeeutt3j11VfR6/XMmjWLxYsXc/LkSX71q1/xyiuv8Morr1BZWUllZSVPP/00R48e5fTp07z44ov09/ej0+nYv38/bm5uD7R2RZnIwhJ8WPJMPHvfLGXvm6U89GyS2sXwLk36Efj58+fZuHEjR48e5ezZs2i1WoaHh8nNzQUgNzcXHx8fGhoayM3NJTs7G71ez5NPPskrr7zCuXPn2LdvH05O6hRvRblb0zIDmLchhsqCVg6/exEp1ZL7u2FVI/DPGimPhf3793PmzBlmzpwJwODgIP7+/vT19dHb20tdXR1PPfUUhw8fJjc3l3Xr1lFWVkZgYODVj3F3N5/cXVRUxNNPm3fZbW5uxsHBgZdffvnq5/Hx8WHt2rVUVVWh1+upra0lJSUFgBdffJGvfOUr4/vFK4oVSFkaZl6tuacWFw8HZq6KtHRJNsOqAtwSpJR8+ctf5mc/+9kN1+vq6vjjH/9IbGwsWVlZvPnmmxw/fpxf/vKX1NbWfuprTZ8+nbNnzwLmHnhERATPPPPMDc+5sud4dXU1zzzzDAcPHnzQX5Ki2Jw5a6MZHN03xcnNgaTsYEuXZBMmfQtlyZIlbNq0iZaWFgA6OjqoqakhKyuLl156iezsbFJTU8nJyUGn0+Hh4UFsbCxNTU2cPn0agN7eXkZG1J10RblXQggWPh1HeJIPh98po7LAtg5At5RJH+AJCQn85Cc/Yfny5cyYMYNly5bR1NREVlYWdXV1ZGdno9VqCQ0NZf78+QA4ODiwceNGXnjhBZKTk1m2bJk6AEJR7pNWq+GhZ5Pwj3BnzxslNJZ3WrokqyfG86ZBRkaGzMvLu+Ha+fPniY+PH7caJjP1Xiu2YLBPz5Zf5DPQo2fdv6ThE+xq6ZIsTghxRkqZ8cnrk34EriiKdXFydeCRbyVj76Bh+6tn6WkbtHRJVksFuKIoVsfdx4lHvpWC0WBix/+eY7BPb+mSrJIKcEVRrJJPsCsrvzGD3vYhdv5fIYZhtVrzk1SAK4pitYKmerL8a4m01vSw6w9F6oDkT1ABriiKVYtK8Rs9ILlDHZD8CZN+IY+iKNZPHZD86dQI3Mr98Ic/5KWXXrrpemNjIxs2bLBARYpiGRkrI0haEEzB3loK9n76aujJRgX4GBrLLWWDgoLYtGnTvRWmKDboygHJ0Wl+HNt8ibKTzZYuyeKsqoXS/NOfMnz+wW4nq4uPI+Df/u22z/nkdrK/+c1v2LJlC8ePH7eKLWXPnTvHnDlzaGtr49vf/jbPPvss1dXVrF69muLiYgYGBnjmmWcoLi4mNjaWxsZGfv3rX5ORcdO8f0WxaRqNYOlXEhjqO8eBP5/HydWesEQfS5dlMZN+BP5p28m+/fbbZGVlWc2WsoWFhRw4cIDjx4/zox/9iMbGxhse/81vfoOXlxelpaX8+Mc/5syZM2NSh6JYg5sOSK6avAckW9UI/LNGymPhVtvJBgQEWM2WsmvWrMHJyQknJycWLVrEqVOnrn4MwJEjR3jxxRcBSEpKYsaMGWPxVimK1bhyQPKWX5xh56/Psf5f0/Gc4mzpssadVQW4JdxqO1mAuXPnWsWWskKI2/5dUSYjFw8dj7yQwpaXzrD9lbOs/3Y6Lp46S5c1riZ9C+VW28kCVrOl7LZt2xgaGqK9vZ2DBw9eHfVfMW/ePN577z0ASktLKSoqGpM6FMXaXD0gud/Ajv89x/CAwdIljatJH+C32k4WsJotZWfMmMGiRYuYPXs23//+9wkKCrrh8eeff57W1lYSEhL4j//4DxITE/Hw8BiTWhTF2viHu7Py76bT2dzPR78tmlQHJKvtZCcAo9GIwWDA0dGRiooKli5dSllZGQ4ODjc8T73XykR28XQze98oJSrVb8IdkHyr7WQnfQ98IhgYGGDRokUYDAaklPzmN7+5KbwVZaKbNjOAwR4DR94v5/A7ZSx4KnbC3y+6owAXQrwIPAsI4DUp5cvXPfbPwEuAn5SybSyKVG7Pzc2NT/5koyiTUfKSUAZ6hsnfXYuzh47M1RP7gOTPDHAhRBLm8M4E9MAuIcROKeUlIUQosBxQ61oVRbEKsx+LZqBbz+md5n1TJvIByXdyEzMeOCmlHJBSjgCHgHWjj/0P8G1AbQ+mKIpV+OQByTlvX6C6qA2DfuLd3LyTFkox8P8JIXyAQWAlkCeEWAM0SCnPTfQ+k6IotuXKAckH/3aBi6cuU5rbiNZeQ0isFxHTfQif7oubt6Oly7xvnxngUsrzQoifA3uAfuAsoAP+DXP75LaEEM8BzwGEhYXdT62Koih3zF6nZdlXEjEaTDSUd1JT1E51URs1xe3wzkW8g1yImO5L+HQfAiLd0Whtb1b1XU8jFEL8FLgM/DswMHo5BGgEMqWUt9wibDJOIzx48CAvvfQSO3fuvOH6F77wBfLy8rC3tyczM5Pf//732Nvbj2ktE/29VpTPIqWk6/IA1UXt1BS30VTejckk0TnbEZboQ8R0H8ISfXB0Gdt/i3frvqYRCiH8pZQtQogwzP3v2VLKV657vBrIULNQ7twXvvAF3nrrLQCeeuopXn/9db7xjW9YuCpFmdiEEHgFuOAV4ELqsjCGB0eoK+2gpqiNmpJ2yk9fRggIiPYgPMmHiOm+eAe5WO10xDudB755tAduAL4ppewai2Jy37tIW13fA31N31BXsp6YdsvHq6urefjhh5k/fz7Hjh0jODiYbdu24eTkREVFBd/85jdpbW3F2dmZ1157jalTpxITE0NlZSXd3d34+PiQk5NDdnY22dnZvPHGG0ydOvUz61q5cuXVP2dmZlJfX/9Avl5FUe6czsmOmHR/YtL9kSbJ5Zqeq62WEx9UcuKDSly9dUQkmVstIbFe2DloLV32VXcU4FLKrM94POKBVGMh5eXlvPPOO7z22ms88cQTbN68mS9+8Ys899xz/O53v2Pq1KmcPHmS559/ngMHDhAbG0tpaSlVVVWkpaWRm5vLrFmzqKuru6Pwvp7BYOCvf/0rr7zyymc/WVGUMSM0goBIDwIiPZj1aBT9XcPUFJvD/MLJZooPN2BnryE4zsvcO0/ysfiNUKtaiXm7kfJYioyMvLo9a3p6OtXV1fT19XHs2DEef/zxq88bHh4GzHukHD58mKqqKr73ve/x2muvsWDBgps2mboTzz//PNnZ2WRl3fb/kYqijDMXTx0J84NImB909UZodVG7ud1S1A6AT7AL4dN9iUjyYUqUx7gv37eqALcUne7aFpRarZbBwUFMJhOenp5Xt4a9XnZ2Nr/97W9pbGzkRz/6Eb/4xS84ePDgXYfwf/7nf9La2srvf//7+/0SFEUZQ1p7DWEJPoQl+CCfmGq+EVpovhF6dk8t+btq0LnYEZ7oQ/h08/PG40aoCvBbcHd3JzIykvfff5/HH38cKSWFhYUkJyeTmZnJ008/TVRUFI6OjqSkpPD73//+ppkmt/P666+ze/du9u/fj0Zje9OXFGWyuuFG6PIwhgcM1JZ2UFPcTk1xOxdPXbsReqXVMlY3QlWA38bbb7/NN77xDX7yk59gMBj43Oc+R3JyMjqdjtDQUGbPng2YWyrvvPMO06dP/9TX2b9/PyEhIVf//v777/P1r3+d8PBw5syZA8C6dev4wQ9+MPZflKIoD5TO2Z6pGVOYmjEFk0nSUt1ztXd+fGsFx7dW4ObtyOIvxRES5/1AP7faTnYSUe+1ooyvvs5haorNi4fmro/B0//ejn1T28kqiqKMM1cvHYlZwSRmjc2GWqr5qiiKYqOsIsDHs40zWan3WFEmHosHuKOjI+3t7SpgxpCUkvb2dhwdbX/3NUVRrrF4DzwkJIT6+npaW1stXcqE5ujoeMNMGEVRbJ/FA9ze3p7IyIl97JGiKMpYsHgLRVEURbk3KsAVRVFslApwRVEUGzWuKzGFEK1AzT1+uC+gDoy4Rr0f16j34kbq/bjRRHg/wqWUfp+8OK4Bfj+EEHmftpR0slLvxzXqvbiRej9uNJHfD9VCURRFsVEqwBVFUWyULQX4HyxdgJVR78c16r24kXo/bjRh3w+b6YEriqIoN7KlEbiiKIpyHRXgiqIoNsomAlwIsUIIUSaEuCSE+K6l67EUIUSoECJHCFEqhCgRQrxo6ZqsgRBCK4QoEELc+aGkE5QQwlMIsUkIcUEIcV4IMcfSNVmKEOIfR/+dFAsh3hFCTLjtOK0+wIUQWuDXwMNAAvB5IUSCZauymBHgn6WUCcBs4JuT+L243ovAeUsXYSVeAXZJKeOAZCbp+yKECAa+BWRIKZMALfA5y1b14Fl9gAOZwCUpZaWUUg+8C6yxcE0WIaVsklLmj/65F/M/zrE5q8lGCCFCgFXA65auxdKEEB5ANvAGgJRSL6XssmhRlmUHOAkh7ABnoNHC9TxwthDgwUDddX+vZ5KHFoAQIgJIBU5auBRLexn4NmCycB3WIBJoBf442lJ6XQjhYumiLEFK2QC8BNQCTUC3lHKPZat68GwhwJVPEEK4ApuBf5BS9li6HksRQqwGWqSUZyxdi5WwA9KA30opU4F+YFLeMxJCeGH+ST0SCAJchBBftGxVD54tBHgDEHrd30NGr01KQgh7zOH9tpRyi6XrsbB5wKNCiGrMrbXFQoi3LFuSRdUD9VLKKz+VbcIc6JPRUqBKStkqpTQAW4C5Fq7pgbOFAD8NTBVCRAohHDDfiNhu4ZosQgghMPc3z0spf2XpeixNSvk9KWWIlDIC8/fFASnlhBtl3SkpZTNQJ4SIHb20BCi1YEmWVAvMFkI4j/67WcIEvKFr8SPVPouUckQI8ffAbsx3kt+UUpZYuCxLmQc8DRQJIc6OXvs3KeVHlitJsTIvAG+PDnYqga9YuB6LkFKeFEJsAvIxz94qYAIuqVdL6RVFUWyULbRQFEVRlE+hAlxRFMVGqQBXFEWxUSrAFUVRbJQKcEVRFBulAlxRFMVGqQBXFEWxUf8/VHGFUWZsv50AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(resewc,label='ewc')\n",
    "plt.plot(resall,label='all')\n",
    "#plt.plot(resall_shuffle,label='all shuffle')\n",
    "plt.plot(respp,label='ewc++')\n",
    "plt.plot(resppbig,label='ewc++ big')\n",
    "plt.plot(l2new,label='new L2')\n",
    "#plt.plot(l2newrecomp,label='L2 - recomp')\n",
    "#plt.plot(noreg,label='noreg')\n",
    "\n",
    "\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10, 6)\n"
     ]
    }
   ],
   "source": [
    "results = [resewc,resall,respp,resppbig,l2new]\n",
    "\n",
    "def res_to_mtx(losses,filename):\n",
    "    ks = [i for i in range(len(losses[0]))]\n",
    "    A = [np.array(ks)]\n",
    "    A += losses\n",
    "    A = np.vstack( A ).T\n",
    "    print(A.shape)\n",
    "    np.savetxt(filename, A, delimiter=' ')\n",
    "\n",
    "res_to_mtx(results,\"MNIST_permutation_2layernn.dat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}