notebooks/storing_and_loading_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Storing and Loading Models\n",
    "\n",
    "https://pytorch.org/tutorials/beginner/saving_loading_models.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Init, helpers, utils, ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:17.884746Z",
     "iopub.status.busy": "2020-12-06T08:43:17.884079Z",
     "iopub.status.idle": "2020-12-06T08:43:18.121868Z",
     "shell.execute_reply": "2020-12-06T08:43:18.122171Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.124748Z",
     "iopub.status.busy": "2020-12-06T08:43:18.124440Z",
     "iopub.status.idle": "2020-12-06T08:43:18.398916Z",
     "shell.execute_reply": "2020-12-06T08:43:18.398651Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stefan/projects/pytorch_tutorial/.venv/lib/python3.8/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)\n",
      "  return torch._C._cuda_getDeviceCount() > 0\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "\n",
    "DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.401221Z",
     "iopub.status.busy": "2020-12-06T08:43:18.400913Z",
     "iopub.status.idle": "2020-12-06T08:43:18.402435Z",
     "shell.execute_reply": "2020-12-06T08:43:18.402680Z"
    }
   },
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from IPython.core.debugger import set_trace"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `state_dict()`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `nn.Module.state_dict()`\n",
    "`nn.Module` contain state dict, that maps each layer to the learnable parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.406569Z",
     "iopub.status.busy": "2020-12-06T08:43:18.406248Z",
     "iopub.status.idle": "2020-12-06T08:43:18.408255Z",
     "shell.execute_reply": "2020-12-06T08:43:18.407906Z"
    }
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = x.view(-1, 16 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.410414Z",
     "iopub.status.busy": "2020-12-06T08:43:18.410113Z",
     "iopub.status.idle": "2020-12-06T08:43:18.413017Z",
     "shell.execute_reply": "2020-12-06T08:43:18.412696Z"
    }
   },
   "outputs": [],
   "source": [
    "model = Net()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.415034Z",
     "iopub.status.busy": "2020-12-06T08:43:18.414733Z",
     "iopub.status.idle": "2020-12-06T08:43:18.437874Z",
     "shell.execute_reply": "2020-12-06T08:43:18.437548Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('conv1.weight',\n",
       "              tensor([[[[-0.0716, -0.0885, -0.0418, -0.0174,  0.0306],\n",
       "                        [-0.0987, -0.0719,  0.0415, -0.0615, -0.0813],\n",
       "                        [ 0.1020,  0.1077, -0.0484, -0.1027,  0.0212],\n",
       "                        [-0.0158,  0.0715,  0.1063,  0.0149, -0.0665],\n",
       "                        [ 0.0862,  0.0699,  0.0826,  0.0582,  0.1031]],\n",
       "              \n",
       "                       [[ 0.0377,  0.0997, -0.0155,  0.0759,  0.1136],\n",
       "                        [-0.0491,  0.0229,  0.0384, -0.0084, -0.0572],\n",
       "                        [ 0.0543, -0.0504, -0.0309,  0.0171, -0.1097],\n",
       "                        [ 0.0639, -0.0258,  0.0666, -0.0692, -0.0280],\n",
       "                        [ 0.0159, -0.0713,  0.0845,  0.0009, -0.0742]],\n",
       "              \n",
       "                       [[-0.0562, -0.0505,  0.0886, -0.0612,  0.0182],\n",
       "                        [ 0.0630, -0.0025, -0.0011, -0.1099,  0.0106],\n",
       "                        [ 0.0115,  0.0749,  0.0138, -0.0455,  0.0599],\n",
       "                        [ 0.1043, -0.0480, -0.0227,  0.0353, -0.0910],\n",
       "                        [-0.0777,  0.1007, -0.0296,  0.1094, -0.0085]]],\n",
       "              \n",
       "              \n",
       "                      [[[ 0.0198, -0.0578,  0.0236,  0.0710, -0.0087],\n",
       "                        [ 0.1036, -0.0811, -0.0718, -0.0987, -0.1026],\n",
       "                        [-0.0825,  0.1102,  0.0464,  0.0612, -0.0288],\n",
       "                        [ 0.1024, -0.0334, -0.1103,  0.0073, -0.0547],\n",
       "                        [ 0.0940,  0.0818, -0.0308, -0.0918,  0.0508]],\n",
       "              \n",
       "                       [[-0.0499,  0.0200,  0.0665,  0.0698,  0.1144],\n",
       "                        [ 0.0855,  0.1135,  0.0456, -0.1149, -0.1086],\n",
       "                        [-0.0958,  0.0302, -0.0111,  0.0448, -0.0356],\n",
       "                        [ 0.0381,  0.0402, -0.0113, -0.0277, -0.0071],\n",
       "                        [-0.0011,  0.0800,  0.0767,  0.1029, -0.0509]],\n",
       "              \n",
       "                       [[ 0.0917,  0.0492, -0.0837,  0.0730, -0.0958],\n",
       "                        [ 0.1089, -0.0793, -0.0483, -0.1123,  0.0270],\n",
       "                        [ 0.0968,  0.0599,  0.0662, -0.0793,  0.0828],\n",
       "                        [ 0.0272,  0.0595,  0.1086, -0.0873, -0.0392],\n",
       "                        [-0.0993, -0.0371, -0.0937,  0.1080,  0.1077]]],\n",
       "              \n",
       "              \n",
       "                      [[[-0.0432, -0.0701,  0.1100,  0.0519, -0.0560],\n",
       "                        [ 0.0948,  0.0728, -0.0680,  0.0375,  0.0762],\n",
       "                        [ 0.0596, -0.0470,  0.0603, -0.0621,  0.0492],\n",
       "                        [ 0.0426,  0.0337, -0.0404, -0.0255,  0.0680],\n",
       "                        [-0.0866, -0.0771, -0.0061, -0.0816,  0.0662]],\n",
       "              \n",
       "                       [[ 0.1151,  0.0176, -0.0844, -0.0093,  0.0190],\n",
       "                        [ 0.0331, -0.0661,  0.0564, -0.0361, -0.0157],\n",
       "                        [-0.0710,  0.0088, -0.0073, -0.0713,  0.0960],\n",
       "                        [ 0.0112,  0.0795,  0.0715, -0.0795, -0.0827],\n",
       "                        [ 0.0318,  0.0677, -0.0292,  0.0748,  0.0070]],\n",
       "              \n",
       "                       [[-0.0735, -0.1117, -0.0559,  0.0070,  0.0559],\n",
       "                        [-0.0869, -0.0352, -0.0974,  0.0707,  0.0813],\n",
       "                        [-0.0607, -0.0011,  0.0727,  0.0222, -0.0703],\n",
       "                        [ 0.0869, -0.0854,  0.0739, -0.0987,  0.0580],\n",
       "                        [-0.0651, -0.0990,  0.0813, -0.0579, -0.0079]]],\n",
       "              \n",
       "              \n",
       "                      [[[-0.0420,  0.0285, -0.1118,  0.0793, -0.0883],\n",
       "                        [-0.0295,  0.0365, -0.1094,  0.1103, -0.0915],\n",
       "                        [-0.0724,  0.0686, -0.0313, -0.0566,  0.0822],\n",
       "                        [-0.0090,  0.0953,  0.0089, -0.1105, -0.1013],\n",
       "                        [-0.0510, -0.0146,  0.0072, -0.0891,  0.0501]],\n",
       "              \n",
       "                       [[ 0.0983, -0.0499, -0.0352,  0.0764,  0.0805],\n",
       "                        [-0.1040, -0.0871,  0.0855,  0.0970,  0.0304],\n",
       "                        [ 0.0049, -0.0615, -0.1147,  0.0050,  0.0275],\n",
       "                        [-0.0078, -0.0647,  0.0384, -0.0644, -0.0512],\n",
       "                        [ 0.1131,  0.0124, -0.0547,  0.0221, -0.0290]],\n",
       "              \n",
       "                       [[ 0.0042,  0.0871,  0.0399, -0.0457, -0.0046],\n",
       "                        [ 0.0547,  0.1031,  0.0593,  0.0504, -0.0069],\n",
       "                        [ 0.1069, -0.0788,  0.1061,  0.0257,  0.0860],\n",
       "                        [-0.0967,  0.0251, -0.0402, -0.0505,  0.0409],\n",
       "                        [-0.0422, -0.0121, -0.0470, -0.0533, -0.0142]]],\n",
       "              \n",
       "              \n",
       "                      [[[ 0.0285,  0.0429, -0.0792, -0.0811, -0.0220],\n",
       "                        [-0.0798,  0.0693, -0.0312, -0.0013, -0.0155],\n",
       "                        [-0.0487,  0.0493,  0.0234,  0.0388, -0.0643],\n",
       "                        [ 0.0947,  0.0144,  0.0984, -0.1072, -0.0877],\n",
       "                        [ 0.0152,  0.0508, -0.0876,  0.0581, -0.0280]],\n",
       "              \n",
       "                       [[-0.1132,  0.0636,  0.0527,  0.0690,  0.0053],\n",
       "                        [ 0.0610,  0.0101, -0.0953,  0.0103,  0.0100],\n",
       "                        [-0.1001,  0.0351, -0.0535,  0.0061, -0.0029],\n",
       "                        [ 0.0315,  0.0868, -0.0971, -0.1006,  0.0201],\n",
       "                        [ 0.0906, -0.0464, -0.0743, -0.0456,  0.0192]],\n",
       "              \n",
       "                       [[-0.0767,  0.1149,  0.0373,  0.0994, -0.0110],\n",
       "                        [ 0.0065,  0.1025, -0.0064, -0.0510,  0.1111],\n",
       "                        [ 0.1132, -0.0731, -0.0512, -0.1029, -0.0098],\n",
       "                        [-0.0144,  0.0815,  0.0599, -0.0213,  0.0205],\n",
       "                        [-0.0712,  0.0017, -0.0283, -0.0282,  0.0718]]],\n",
       "              \n",
       "              \n",
       "                      [[[ 0.1060,  0.0249, -0.0688, -0.0181,  0.0535],\n",
       "                        [ 0.1132,  0.1079,  0.1001,  0.0853,  0.0082],\n",
       "                        [ 0.0819,  0.0780, -0.0565,  0.0324,  0.0743],\n",
       "                        [-0.1147,  0.0276, -0.1027, -0.0008,  0.0715],\n",
       "                        [-0.1063,  0.0352,  0.0813, -0.0536,  0.0989]],\n",
       "              \n",
       "                       [[ 0.0767, -0.1032, -0.0153, -0.0858, -0.0911],\n",
       "                        [ 0.1051, -0.0468, -0.0690,  0.0915,  0.0044],\n",
       "                        [ 0.0857,  0.1087, -0.0886,  0.0118,  0.0460],\n",
       "                        [ 0.0520,  0.0019,  0.1074,  0.1074, -0.0898],\n",
       "                        [-0.1129,  0.0108, -0.0253,  0.0584, -0.0006]],\n",
       "              \n",
       "                       [[ 0.0385,  0.0645, -0.0755,  0.1099,  0.0390],\n",
       "                        [ 0.1038, -0.0983,  0.0530, -0.0409, -0.0137],\n",
       "                        [ 0.1064,  0.0303,  0.0592, -0.0846, -0.0066],\n",
       "                        [-0.0327,  0.0012,  0.0490,  0.0860,  0.0797],\n",
       "                        [-0.0037,  0.0207,  0.0837, -0.0665, -0.0606]]]])),\n",
       "             ('conv1.bias',\n",
       "              tensor([-0.0513,  0.0638,  0.0927, -0.0586, -0.0851, -0.0095])),\n",
       "             ('conv2.weight',\n",
       "              tensor([[[[ 4.1904e-02,  2.4201e-02, -3.0167e-03, -4.6679e-02,  1.1058e-02],\n",
       "                        [-1.9261e-02,  2.5148e-02,  3.9350e-02,  3.6170e-02,  4.2323e-02],\n",
       "                        [-3.3384e-02, -3.6684e-02,  2.9364e-02,  7.7372e-02, -5.3670e-02],\n",
       "                        [ 4.9913e-02, -1.8233e-02, -8.6749e-03,  1.8934e-02, -5.8539e-02],\n",
       "                        [-2.2458e-02,  7.5278e-02, -6.5807e-02,  1.4085e-02, -5.0685e-02]],\n",
       "              \n",
       "                       [[-6.5826e-02, -3.2964e-02,  2.1917e-02, -9.5495e-03,  2.7229e-02],\n",
       "                        [ 5.8031e-02,  9.5947e-03, -5.3045e-02, -8.0154e-02, -7.6017e-03],\n",
       "                        [-4.9222e-02, -8.0538e-02, -5.0448e-02,  6.8835e-02,  3.6773e-02],\n",
       "                        [ 6.4616e-02,  4.5119e-02, -1.8120e-02,  2.7221e-02, -9.4454e-03],\n",
       "                        [-2.2578e-02,  2.0986e-02, -8.2121e-04, -7.2404e-02,  5.9041e-02]],\n",
       "              \n",
       "                       [[-2.6516e-02,  4.5631e-02, -1.1182e-02, -6.4108e-03, -3.4161e-02],\n",
       "                        [-7.5357e-02, -5.1851e-02,  1.3428e-02, -4.9648e-03, -6.9116e-02],\n",
       "                        [ 6.9323e-02, -6.9944e-02, -3.2889e-02, -6.0697e-02,  2.0140e-02],\n",
       "                        [ 6.0037e-02,  6.8279e-02, -4.1563e-03,  7.8933e-02, -5.1255e-02],\n",
       "                        [ 8.1510e-02, -7.6880e-02, -3.9604e-02,  7.4774e-02,  7.2364e-02]],\n",
       "              \n",
       "                       [[ 4.6287e-02, -3.8903e-02, -3.1258e-02,  7.2676e-02,  2.3875e-02],\n",
       "                        [ 6.7443e-02, -3.7898e-02,  3.1365e-02,  7.9612e-03,  2.4547e-02],\n",
       "                        [ 2.1904e-02,  2.4315e-02, -5.8083e-02, -5.8505e-02, -2.2194e-02],\n",
       "                        [-3.2459e-03,  5.0463e-02, -5.9157e-02, -6.8928e-02, -1.2338e-03],\n",
       "                        [-1.6254e-03,  1.4726e-02,  1.0563e-02,  3.7500e-02, -6.1539e-03]],\n",
       "              \n",
       "                       [[ 5.8620e-02,  2.4580e-02,  3.6906e-02, -1.0145e-02,  7.4356e-02],\n",
       "                        [ 7.3785e-02,  1.8596e-02, -7.9579e-02, -2.8739e-02, -1.9139e-02],\n",
       "                        [ 6.9464e-02, -2.0512e-02, -6.5394e-02,  8.0399e-02, -1.4843e-03],\n",
       "                        [ 6.8836e-02,  2.3483e-02, -7.8369e-02,  1.1197e-02, -1.1748e-03],\n",
       "                        [-6.8214e-02, -1.2998e-02, -3.5589e-02, -3.2472e-02,  6.1204e-02]],\n",
       "              \n",
       "                       [[ 1.4855e-02, -1.4002e-02,  7.6459e-02,  4.2424e-02,  5.4374e-02],\n",
       "                        [-1.2550e-02,  6.2436e-02,  5.6971e-03,  6.3653e-02, -2.1360e-02],\n",
       "                        [-5.2922e-02, -7.3173e-02, -5.4917e-02, -5.1043e-02,  1.7286e-02],\n",
       "                        [ 3.4048e-02, -6.1201e-02, -5.3640e-02, -7.8310e-02, -7.4896e-02],\n",
       "                        [ 2.7302e-02, -7.3241e-02, -7.3456e-02, -4.3926e-02, -9.5609e-03]]],\n",
       "              \n",
       "              \n",
       "                      [[[-5.4592e-02, -6.3463e-02,  6.5288e-04,  3.5952e-02, -5.5588e-02],\n",
       "                        [ 5.4672e-02, -3.1190e-02,  6.1907e-03,  6.6337e-03, -2.0895e-02],\n",
       "                        [ 2.9617e-03,  3.4418e-02,  6.7033e-02, -3.7661e-02,  1.0367e-02],\n",
       "                        [ 7.2062e-02,  3.2949e-03,  2.6317e-02, -3.1087e-02, -2.5686e-02],\n",
       "                        [-1.5577e-02,  3.2070e-02, -4.9359e-02,  4.2945e-02,  7.8953e-02]],\n",
       "              \n",
       "                       [[ 3.2304e-02, -4.8440e-02, -2.1110e-02,  1.5567e-02, -4.0562e-02],\n",
       "                        [-3.8105e-02, -3.2255e-02, -1.2886e-02,  7.2996e-02, -8.0325e-02],\n",
       "                        [-3.8957e-02, -7.8102e-02,  6.0198e-02,  7.2410e-03, -8.1450e-02],\n",
       "                        [ 3.8727e-02, -1.3728e-02, -2.9018e-02,  7.4644e-02, -4.3442e-02],\n",
       "                        [ 4.1651e-02,  1.6152e-02, -1.2693e-02,  7.6132e-02,  8.8840e-03]],\n",
       "              \n",
       "                       [[ 8.6380e-04,  2.2372e-02, -4.2537e-02,  2.5337e-02,  7.8715e-02],\n",
       "                        [ 6.6709e-02,  6.3609e-02, -2.9480e-02, -1.6856e-03, -3.9037e-02],\n",
       "                        [ 2.4898e-02, -5.0577e-02,  3.4967e-02,  2.7741e-03,  3.5984e-02],\n",
       "                        [ 2.6298e-02, -5.2694e-02, -7.5277e-03,  7.0087e-02,  7.2002e-02],\n",
       "                        [-5.2806e-02, -2.0120e-02, -3.5417e-02, -9.3975e-03, -6.7863e-03]],\n",
       "              \n",
       "                       [[ 2.5457e-02,  8.0086e-02, -4.1152e-02,  7.3821e-02, -2.3224e-02],\n",
       "                        [-6.8475e-02, -6.3089e-02, -5.8841e-02,  3.2971e-02,  1.6452e-04],\n",
       "                        [-2.8815e-02,  1.7451e-02, -7.2556e-02, -6.0914e-02,  5.4732e-02],\n",
       "                        [-6.5419e-02, -1.3789e-02, -4.9730e-02, -1.9960e-02, -6.1123e-02],\n",
       "                        [-9.8044e-03,  6.6083e-02, -4.1143e-02,  1.8845e-02, -3.8298e-02]],\n",
       "              \n",
       "                       [[-4.5579e-02, -3.8066e-02, -7.9374e-02, -7.6764e-02,  1.0537e-02],\n",
       "                        [ 3.8778e-02, -2.6967e-02,  4.2730e-02,  2.1605e-02, -2.1691e-02],\n",
       "                        [ 2.6506e-02, -7.8812e-02, -7.8687e-04,  8.0315e-02, -7.3939e-02],\n",
       "                        [ 3.5507e-02,  7.9640e-02, -4.0839e-02,  6.7439e-02, -6.0298e-02],\n",
       "                        [ 1.9328e-02,  6.0529e-02, -8.1358e-04,  3.6348e-02, -7.5758e-02]],\n",
       "              \n",
       "                       [[ 6.9500e-02,  4.6842e-02, -6.5507e-02,  1.3020e-02,  2.0093e-02],\n",
       "                        [ 5.3984e-02,  7.6515e-02, -6.6133e-02, -7.5851e-02, -1.4756e-02],\n",
       "                        [ 1.9752e-02,  4.1261e-02,  5.1411e-02, -6.8580e-02, -6.5399e-02],\n",
       "                        [ 3.4109e-02, -3.1521e-02,  2.5358e-02, -1.1479e-02,  7.6774e-02],\n",
       "                        [-2.8118e-02, -3.3794e-02,  6.0184e-02, -5.2791e-02, -1.9980e-02]]],\n",
       "              \n",
       "              \n",
       "                      [[[-3.8210e-02, -4.2161e-02, -3.9423e-02, -2.1322e-02,  7.0140e-02],\n",
       "                        [ 6.7337e-02, -2.1847e-03, -4.2958e-02, -5.1811e-02, -4.7309e-02],\n",
       "                        [ 7.2566e-02,  1.4023e-03, -7.5928e-02,  2.7988e-02, -3.7876e-02],\n",
       "                        [-2.8593e-03, -2.9854e-03, -4.2032e-02,  4.2200e-02, -4.3034e-03],\n",
       "                        [ 7.0848e-02, -4.0290e-02, -7.3230e-02, -4.6787e-02,  4.3673e-02]],\n",
       "              \n",
       "                       [[ 2.2518e-02, -7.7074e-02, -4.6926e-02,  7.8924e-02, -1.5153e-02],\n",
       "                        [ 6.2495e-02,  2.7227e-02,  2.8026e-02,  4.1027e-04, -5.3120e-03],\n",
       "                        [ 5.3819e-02,  1.4187e-02, -4.4776e-02,  6.8585e-02, -1.4363e-02],\n",
       "                        [-7.2928e-02, -3.5205e-02,  5.4156e-03, -3.7265e-02,  8.0635e-02],\n",
       "                        [ 6.9658e-02,  2.3197e-02, -1.7254e-02,  8.1399e-02,  4.2029e-03]],\n",
       "              \n",
       "                       [[ 2.7601e-02, -2.9564e-02,  8.6550e-03,  5.5561e-02,  7.7703e-02],\n",
       "                        [ 6.5621e-02,  7.1932e-02, -7.0272e-02, -4.6994e-02, -2.8362e-02],\n",
       "                        [-6.4538e-03, -4.5529e-02,  3.2642e-02, -7.9746e-02,  3.2092e-02],\n",
       "                        [-2.7084e-02, -7.7648e-02, -2.3784e-02,  2.6799e-02, -3.2669e-02],\n",
       "                        [ 2.3182e-02, -6.7762e-02,  6.3847e-02, -3.1726e-03,  2.1573e-02]],\n",
       "              \n",
       "                       [[-5.4961e-02,  1.6435e-02, -6.2172e-02,  1.8766e-02, -5.4989e-02],\n",
       "                        [-5.5064e-02,  4.8266e-02, -7.3601e-02,  7.2843e-02,  6.0190e-03],\n",
       "                        [-2.3214e-02,  4.1102e-02,  6.1786e-03,  1.3852e-03,  4.2317e-02],\n",
       "                        [-6.2143e-02,  4.6521e-02, -5.8726e-03,  5.9306e-03, -4.8584e-02],\n",
       "                        [-3.5394e-02, -3.2924e-02,  7.6548e-03, -1.5569e-02, -7.3160e-02]],\n",
       "              \n",
       "                       [[-2.8731e-04,  3.5759e-02, -3.6463e-02,  2.1846e-02,  6.5786e-03],\n",
       "                        [ 2.5538e-02,  4.6567e-02,  2.9227e-02,  6.6731e-03,  2.5543e-02],\n",
       "                        [ 3.0449e-02,  6.3248e-02,  3.4695e-02, -9.8319e-03,  6.1053e-02],\n",
       "                        [ 5.1948e-02,  6.2803e-02,  5.1109e-03,  7.9499e-02, -7.2061e-02],\n",
       "                        [ 5.6251e-02, -4.6507e-02, -4.2439e-02,  2.1365e-02, -4.0908e-02]],\n",
       "              \n",
       "                       [[-2.9924e-02,  3.5621e-02, -3.6899e-02, -4.6373e-02, -4.3850e-02],\n",
       "                        [-6.9880e-02,  5.1435e-02, -3.3334e-02,  1.2267e-02, -6.1399e-02],\n",
       "                        [ 3.6918e-02, -3.5074e-03, -7.7285e-02, -7.0741e-02,  9.2793e-03],\n",
       "                        [ 7.9256e-02,  3.7383e-02,  2.4533e-02, -3.8257e-02, -5.8194e-02],\n",
       "                        [ 7.3048e-02, -7.5073e-02, -7.0429e-02,  5.1220e-02,  1.6855e-02]]],\n",
       "              \n",
       "              \n",
       "                      ...,\n",
       "              \n",
       "              \n",
       "                      [[[-7.4364e-02,  1.4738e-02,  7.8939e-02, -3.4306e-02, -2.4407e-02],\n",
       "                        [ 7.7032e-02,  4.4698e-02,  5.8655e-03, -2.6842e-02,  7.6624e-02],\n",
       "                        [-3.1000e-02, -4.0127e-03,  7.6115e-02,  7.2105e-03, -5.5917e-02],\n",
       "                        [ 5.9270e-02, -7.8486e-02, -5.9297e-02, -2.5499e-02, -4.2825e-02],\n",
       "                        [-1.5523e-02, -7.3929e-02,  5.4838e-02,  2.6725e-02, -2.9003e-02]],\n",
       "              \n",
       "                       [[-8.2635e-03,  2.3306e-02, -6.6999e-03,  4.6684e-02, -7.2253e-02],\n",
       "                        [ 1.6843e-02,  5.3672e-02, -3.0410e-02, -2.9730e-02,  4.8967e-02],\n",
       "                        [ 1.9799e-03, -5.6135e-02, -1.9419e-02, -2.5401e-02, -1.7289e-02],\n",
       "                        [ 2.1077e-02,  2.2607e-02, -2.0468e-02, -2.0552e-03,  3.6504e-02],\n",
       "                        [ 2.5135e-02, -1.6899e-02,  5.4045e-02,  7.2151e-02, -3.8279e-02]],\n",
       "              \n",
       "                       [[ 7.1230e-02, -4.4831e-03, -6.2429e-02,  5.1374e-02, -4.5751e-02],\n",
       "                        [-6.0333e-02, -7.5107e-02, -6.5458e-02, -2.2736e-02, -6.7891e-02],\n",
       "                        [-7.0316e-02,  1.9121e-02,  4.7667e-03, -7.9081e-02,  6.1172e-03],\n",
       "                        [ 1.7986e-03,  2.4109e-02,  3.4889e-02,  7.4220e-02,  4.1777e-02],\n",
       "                        [-2.1056e-02, -2.2648e-02,  1.2820e-02, -6.1849e-02, -2.1833e-02]],\n",
       "              \n",
       "                       [[-2.3595e-02, -4.2878e-02, -1.6007e-02, -3.3592e-02,  3.5762e-02],\n",
       "                        [-4.2725e-02,  7.8801e-02,  7.3880e-04,  6.0147e-02,  8.1233e-02],\n",
       "                        [-5.4498e-02, -3.9082e-02, -3.8043e-02, -1.3878e-02, -3.5619e-02],\n",
       "                        [ 4.0309e-02,  1.2303e-02,  4.3481e-03,  2.9614e-02,  7.3735e-02],\n",
       "                        [-2.4869e-03, -2.3597e-03, -7.8644e-02,  3.4106e-02,  3.2840e-02]],\n",
       "              \n",
       "                       [[-6.5632e-03, -7.7562e-02, -4.9476e-02,  2.2011e-02,  3.3577e-02],\n",
       "                        [-1.0759e-02,  1.7593e-02, -6.6337e-02,  3.2450e-02,  3.9269e-02],\n",
       "                        [-7.3358e-04,  3.1063e-02,  2.0943e-02,  4.3634e-02,  2.3564e-02],\n",
       "                        [ 3.4677e-02,  5.7123e-02,  8.4364e-03, -5.5662e-03,  5.1776e-02],\n",
       "                        [ 5.1494e-02,  5.6414e-02, -1.9961e-03, -5.7147e-02, -8.1806e-03]],\n",
       "              \n",
       "                       [[ 3.1050e-03,  3.0832e-02,  2.8048e-02, -2.7907e-02,  5.2436e-02],\n",
       "                        [-6.6117e-02, -2.8833e-03, -6.5738e-02,  5.8514e-02, -5.9042e-02],\n",
       "                        [ 6.9849e-03, -6.5734e-02, -2.3597e-02,  1.4548e-02, -7.3027e-02],\n",
       "                        [-5.8812e-02,  7.7408e-02, -3.0233e-02, -7.6830e-02,  1.3943e-02],\n",
       "                        [-4.7348e-02, -1.7195e-02, -2.8651e-02,  6.6875e-02, -3.0221e-02]]],\n",
       "              \n",
       "              \n",
       "                      [[[-4.8120e-02,  2.4020e-03, -3.8569e-02, -4.8376e-02,  3.1411e-02],\n",
       "                        [ 4.1922e-02,  2.0483e-02, -2.5093e-05,  3.9790e-02,  5.4139e-02],\n",
       "                        [-5.8335e-02, -3.2821e-02,  6.5327e-02, -3.4343e-03,  7.6986e-02],\n",
       "                        [ 4.8058e-02, -1.8780e-02,  5.5352e-02,  3.6725e-02,  3.9409e-03],\n",
       "                        [ 4.6732e-02, -7.8514e-02, -1.8536e-02, -2.4817e-02,  5.0569e-02]],\n",
       "              \n",
       "                       [[ 6.9937e-02, -2.2496e-02,  2.8990e-02,  2.9867e-02,  8.0200e-02],\n",
       "                        [ 6.9358e-02, -7.4951e-02,  7.2986e-04, -1.4231e-02, -5.0292e-02],\n",
       "                        [ 5.2989e-02,  2.0389e-02,  1.9102e-02, -2.4804e-02, -2.4038e-02],\n",
       "                        [-4.6675e-02,  7.5506e-04, -6.0437e-02, -2.1249e-02,  2.6403e-02],\n",
       "                        [ 6.5675e-02, -4.9061e-02, -2.7744e-02,  1.7793e-02,  2.8709e-02]],\n",
       "              \n",
       "                       [[ 7.0493e-02,  1.7076e-02, -5.8276e-02, -3.5603e-02, -2.4403e-02],\n",
       "                        [ 6.9705e-02, -3.8544e-02,  4.9937e-02,  7.0241e-02, -6.8352e-02],\n",
       "                        [ 7.4987e-02, -7.1049e-02,  4.6799e-02,  5.2335e-02, -6.7187e-02],\n",
       "                        [ 1.2335e-03,  8.0402e-04, -7.1757e-02, -4.8423e-02,  6.1698e-02],\n",
       "                        [-1.5285e-02, -2.2850e-02,  7.5740e-02,  5.0680e-02, -6.1178e-02]],\n",
       "              \n",
       "                       [[-6.1988e-02,  4.1336e-02,  4.4163e-02, -4.1044e-02,  7.9419e-03],\n",
       "                        [-4.9297e-02,  4.5400e-03, -4.6461e-02,  6.6414e-02, -7.1862e-02],\n",
       "                        [ 4.0352e-02,  1.9251e-02,  3.5010e-02, -2.5858e-03, -7.7897e-02],\n",
       "                        [ 3.6210e-02,  4.8361e-02, -3.5184e-02, -3.6128e-02,  1.2134e-02],\n",
       "                        [-4.2729e-02, -5.6739e-02,  1.6021e-03,  3.7036e-03,  8.0141e-02]],\n",
       "              \n",
       "                       [[ 7.3895e-02, -7.5595e-02, -4.5798e-02, -5.6158e-02, -2.8305e-02],\n",
       "                        [ 7.8356e-02, -1.9630e-02, -7.5982e-02, -6.5198e-02, -5.6067e-02],\n",
       "                        [-6.2886e-02,  4.8927e-02, -1.5103e-02, -6.9210e-02,  5.8471e-02],\n",
       "                        [ 5.9226e-02, -9.7354e-03, -7.7876e-02,  6.8175e-02, -5.8168e-03],\n",
       "                        [ 5.2664e-02,  3.8644e-02,  3.2132e-02,  2.1694e-02,  4.0696e-02]],\n",
       "              \n",
       "                       [[ 3.3606e-02,  6.4631e-03, -6.8327e-02, -6.5683e-02,  2.5997e-03],\n",
       "                        [-7.2214e-02, -2.5156e-03, -2.9184e-02, -6.9151e-02,  8.0338e-02],\n",
       "                        [ 5.3843e-02,  1.8122e-02, -4.0601e-03, -6.1499e-02, -7.7499e-03],\n",
       "                        [ 4.8272e-03,  6.9608e-04,  1.9521e-02,  8.1506e-02,  2.3347e-02],\n",
       "                        [ 4.5207e-02, -6.8800e-02,  1.1802e-02,  6.2374e-02, -7.6632e-02]]],\n",
       "              \n",
       "              \n",
       "                      [[[-5.5812e-03,  8.1533e-03,  6.4646e-02,  6.6102e-02, -2.9010e-02],\n",
       "                        [ 1.1864e-02,  8.0969e-02,  7.2408e-02,  2.4474e-02,  6.6870e-02],\n",
       "                        [ 8.3302e-03,  6.1734e-02, -3.8681e-02,  8.0696e-02,  1.3050e-02],\n",
       "                        [-8.0821e-02, -6.1482e-02,  2.0893e-02, -8.1132e-02,  2.1363e-02],\n",
       "                        [ 5.3095e-02, -7.5962e-02,  2.4110e-02,  1.8107e-02,  5.6836e-03]],\n",
       "              \n",
       "                       [[-2.0531e-02, -5.1249e-02, -7.8541e-03,  1.9181e-02, -7.0819e-02],\n",
       "                        [-2.7591e-02, -1.7718e-02,  5.2253e-02, -7.5794e-02, -5.7678e-02],\n",
       "                        [-3.5227e-02,  1.2037e-02, -7.3083e-02,  7.9023e-02, -8.4906e-03],\n",
       "                        [ 7.1754e-02,  5.6225e-02, -6.9776e-02, -2.3101e-02, -5.7638e-02],\n",
       "                        [ 8.3107e-03,  3.3497e-02, -3.1132e-03,  3.0695e-02,  7.8346e-02]],\n",
       "              \n",
       "                       [[-7.2040e-02, -7.7623e-02,  2.6010e-02, -5.9062e-02, -1.2908e-02],\n",
       "                        [ 2.5792e-02,  6.0085e-02,  7.5948e-02,  7.7973e-02, -4.8302e-03],\n",
       "                        [-3.1436e-02, -2.0723e-02,  3.3885e-02, -3.2490e-02, -2.9639e-02],\n",
       "                        [ 6.4487e-02,  4.6235e-02,  6.7366e-02,  1.9630e-02,  2.5325e-02],\n",
       "                        [ 6.5528e-02, -1.4177e-02,  5.9971e-02,  1.4511e-02, -7.8321e-02]],\n",
       "              \n",
       "                       [[ 4.3606e-03, -5.1712e-02,  6.3008e-02,  2.3999e-02, -2.2484e-02],\n",
       "                        [ 7.1459e-04,  4.0347e-02,  8.1492e-02, -7.7402e-02, -2.2633e-02],\n",
       "                        [-9.5712e-03,  4.5553e-02, -7.4055e-02, -5.0654e-02, -5.4071e-02],\n",
       "                        [ 1.5462e-02,  7.6300e-02,  3.2892e-02,  1.7055e-02, -6.5188e-02],\n",
       "                        [ 7.6624e-02,  3.8176e-02,  2.6670e-03, -6.0762e-03,  2.3524e-02]],\n",
       "              \n",
       "                       [[-3.2062e-02, -3.3772e-02,  2.2708e-02, -3.3167e-02,  3.4549e-02],\n",
       "                        [ 3.8739e-02,  7.7802e-02,  6.8208e-02,  3.9176e-02,  7.7594e-02],\n",
       "                        [ 7.6880e-02,  3.9165e-02,  7.2050e-02,  6.3054e-02, -6.6770e-02],\n",
       "                        [ 3.3292e-02,  4.0580e-02,  1.9907e-02,  5.1099e-02,  5.8870e-02],\n",
       "                        [-1.2455e-02, -6.7003e-02, -1.2629e-02, -6.6823e-02, -2.6433e-02]],\n",
       "              \n",
       "                       [[-5.3961e-02, -3.9846e-02, -1.4579e-03,  8.1178e-02, -9.9622e-03],\n",
       "                        [ 2.9681e-02, -3.6054e-02,  7.3021e-02, -5.0761e-02,  5.7464e-02],\n",
       "                        [-6.1037e-02, -4.7454e-02, -3.2520e-02, -6.4129e-02,  1.5626e-02],\n",
       "                        [-4.1897e-02,  6.5552e-02,  7.1684e-02,  5.7596e-02,  7.6574e-02],\n",
       "                        [-2.1850e-02, -6.4364e-02, -2.4973e-02,  7.6635e-03, -6.7927e-02]]]])),\n",
       "             ('conv2.bias',\n",
       "              tensor([ 0.0037, -0.0013,  0.0057, -0.0013,  0.0164,  0.0244,  0.0122,  0.0126,\n",
       "                       0.0446,  0.0729,  0.0649, -0.0426, -0.0271,  0.0517,  0.0280,  0.0173])),\n",
       "             ('fc1.weight',\n",
       "              tensor([[-0.0333,  0.0213,  0.0018,  ...,  0.0076, -0.0279,  0.0379],\n",
       "                      [-0.0495,  0.0289,  0.0407,  ..., -0.0149,  0.0021, -0.0162],\n",
       "                      [ 0.0153,  0.0341,  0.0428,  ..., -0.0029,  0.0475, -0.0084],\n",
       "                      ...,\n",
       "                      [ 0.0048,  0.0372, -0.0241,  ...,  0.0045, -0.0193, -0.0094],\n",
       "                      [-0.0143,  0.0213,  0.0178,  ...,  0.0442,  0.0405,  0.0109],\n",
       "                      [-0.0397,  0.0148,  0.0043,  ...,  0.0469, -0.0301, -0.0093]])),\n",
       "             ('fc1.bias',\n",
       "              tensor([ 0.0454,  0.0194, -0.0198,  0.0087,  0.0335, -0.0291, -0.0422,  0.0221,\n",
       "                      -0.0467,  0.0012,  0.0172,  0.0477,  0.0062, -0.0261,  0.0123,  0.0077,\n",
       "                       0.0186, -0.0447, -0.0206,  0.0129,  0.0228, -0.0101,  0.0258, -0.0415,\n",
       "                      -0.0209,  0.0337,  0.0221,  0.0311, -0.0156,  0.0434, -0.0408,  0.0405,\n",
       "                      -0.0380, -0.0140, -0.0236,  0.0019,  0.0047, -0.0481, -0.0021,  0.0258,\n",
       "                      -0.0243,  0.0499, -0.0354, -0.0309, -0.0190,  0.0486, -0.0061,  0.0234,\n",
       "                      -0.0162, -0.0270, -0.0045, -0.0486,  0.0334, -0.0313, -0.0101, -0.0320,\n",
       "                      -0.0464, -0.0098,  0.0261,  0.0247,  0.0462, -0.0018, -0.0437, -0.0391,\n",
       "                      -0.0139, -0.0374,  0.0500, -0.0449, -0.0470, -0.0495,  0.0207, -0.0066,\n",
       "                      -0.0406, -0.0290,  0.0013, -0.0108, -0.0206, -0.0075, -0.0318, -0.0092,\n",
       "                      -0.0111,  0.0048, -0.0163, -0.0436, -0.0378, -0.0418, -0.0357,  0.0283,\n",
       "                       0.0008,  0.0485,  0.0456, -0.0315, -0.0412, -0.0364,  0.0169,  0.0397,\n",
       "                      -0.0206, -0.0006,  0.0451, -0.0349, -0.0240,  0.0414,  0.0251,  0.0439,\n",
       "                       0.0416, -0.0022, -0.0069, -0.0340,  0.0129, -0.0042,  0.0358,  0.0176,\n",
       "                       0.0237, -0.0030,  0.0003,  0.0389, -0.0397, -0.0029,  0.0449, -0.0466])),\n",
       "             ('fc2.weight',\n",
       "              tensor([[-0.0606, -0.0332,  0.0170,  ..., -0.0652, -0.0630,  0.0054],\n",
       "                      [-0.0331,  0.0373,  0.0739,  ...,  0.0812, -0.0156, -0.0423],\n",
       "                      [-0.0447, -0.0729, -0.0907,  ...,  0.0334,  0.0300,  0.0831],\n",
       "                      ...,\n",
       "                      [ 0.0048, -0.0791,  0.0009,  ...,  0.0528,  0.0295, -0.0740],\n",
       "                      [ 0.0317, -0.0342,  0.0729,  ..., -0.0170, -0.0671,  0.0278],\n",
       "                      [ 0.0551,  0.0459,  0.0389,  ..., -0.0821,  0.0892, -0.0628]])),\n",
       "             ('fc2.bias',\n",
       "              tensor([-0.0809,  0.0678,  0.0643, -0.0302, -0.0321,  0.0470,  0.0415,  0.0657,\n",
       "                       0.0752, -0.0819,  0.0136,  0.0342,  0.0215, -0.0488,  0.0377, -0.0770,\n",
       "                       0.0415,  0.0003,  0.0646,  0.0354,  0.0702,  0.0568, -0.0382,  0.0631,\n",
       "                      -0.0122, -0.0153,  0.0839,  0.0397, -0.0413,  0.0499, -0.0087,  0.0415,\n",
       "                       0.0702, -0.0180,  0.0243, -0.0495,  0.0086,  0.0209,  0.0817,  0.0509,\n",
       "                      -0.0055,  0.0447,  0.0554,  0.0863,  0.0879,  0.0266,  0.0693, -0.0027,\n",
       "                      -0.0335, -0.0540,  0.0207,  0.0668, -0.0758,  0.0781,  0.0787, -0.0201,\n",
       "                       0.0569, -0.0750, -0.0647, -0.0874, -0.0134, -0.0891,  0.0238,  0.0249,\n",
       "                       0.0457,  0.0278, -0.0394, -0.0386, -0.0602,  0.0217, -0.0029,  0.0307,\n",
       "                      -0.0884,  0.0881,  0.0385,  0.0002,  0.0410, -0.0549,  0.0040,  0.0483,\n",
       "                       0.0607, -0.0317, -0.0720, -0.0523])),\n",
       "             ('fc3.weight',\n",
       "              tensor([[ 0.0855, -0.0729,  0.0563,  0.0715,  0.0292,  0.0540, -0.0324, -0.0795,\n",
       "                        0.0007,  0.0290, -0.0898, -0.0989, -0.0069, -0.0419,  0.1005,  0.0402,\n",
       "                       -0.0731,  0.0503,  0.0886, -0.0458,  0.0628,  0.0745, -0.0245, -0.0008,\n",
       "                       -0.0946, -0.0081,  0.0374, -0.0121,  0.0108,  0.0690,  0.0760, -0.1052,\n",
       "                       -0.0758,  0.0042, -0.0048, -0.0792, -0.0867, -0.0945,  0.0998,  0.0184,\n",
       "                        0.0923,  0.1073, -0.0683, -0.0089,  0.0795, -0.0051,  0.0973, -0.0078,\n",
       "                        0.0516, -0.0828,  0.0451, -0.0367, -0.0400, -0.0916, -0.0855, -0.0970,\n",
       "                       -0.1085, -0.0035,  0.0574,  0.0687,  0.0639, -0.0488,  0.0582, -0.0115,\n",
       "                        0.0065,  0.0621,  0.0180, -0.0826,  0.0876, -0.0726,  0.0970,  0.0888,\n",
       "                        0.0702,  0.0734, -0.0507,  0.0776, -0.0753,  0.0661, -0.0077,  0.0468,\n",
       "                       -0.0150,  0.0706, -0.0332, -0.0157],\n",
       "                      [ 0.0247, -0.0516, -0.0428, -0.0751, -0.0926, -0.0712,  0.1020, -0.0034,\n",
       "                        0.0441,  0.0044,  0.0029, -0.0468,  0.0482, -0.0361, -0.0554, -0.0087,\n",
       "                       -0.1024,  0.0290,  0.0941, -0.0496, -0.0306,  0.0860,  0.0957, -0.0818,\n",
       "                        0.0481,  0.0754,  0.0836, -0.0386, -0.1042, -0.0614,  0.0295, -0.0649,\n",
       "                       -0.0117, -0.0869,  0.0477,  0.0806, -0.0875,  0.0460,  0.0222, -0.0517,\n",
       "                       -0.0319, -0.0040, -0.0727, -0.0116, -0.1033, -0.0561, -0.0962, -0.0751,\n",
       "                       -0.0214,  0.0861, -0.0336,  0.0088,  0.0293,  0.0278,  0.0699,  0.0117,\n",
       "                        0.0203,  0.0784,  0.0662, -0.0032, -0.0813, -0.0431,  0.0941,  0.0936,\n",
       "                       -0.0511,  0.0985, -0.0192, -0.0291, -0.0471,  0.0232,  0.0721, -0.0189,\n",
       "                       -0.0952,  0.0922,  0.0951,  0.0921,  0.0647,  0.0243,  0.0665,  0.0214,\n",
       "                       -0.0950, -0.0073, -0.0057,  0.0381],\n",
       "                      [ 0.0240,  0.1081,  0.0270,  0.0192,  0.0996,  0.0716,  0.0114,  0.0502,\n",
       "                        0.1006, -0.0243, -0.0051,  0.0400,  0.0679, -0.0120, -0.0234, -0.0332,\n",
       "                        0.0780, -0.0237,  0.0929,  0.0486, -0.0998,  0.0846, -0.0282,  0.0606,\n",
       "                       -0.0182,  0.0228,  0.0454,  0.0270,  0.0039, -0.0426, -0.0421,  0.1016,\n",
       "                        0.0164,  0.0258, -0.0746,  0.0041, -0.0460,  0.0510,  0.0689, -0.0632,\n",
       "                        0.0218, -0.0545, -0.0414, -0.0225,  0.0085, -0.1071,  0.0326,  0.0367,\n",
       "                        0.0542,  0.0696,  0.0585, -0.0998, -0.0291, -0.0574, -0.0364,  0.0172,\n",
       "                        0.1072,  0.0598, -0.1041,  0.0091,  0.0105,  0.0907,  0.0730,  0.0816,\n",
       "                       -0.0771, -0.0511,  0.0591,  0.0105, -0.0386,  0.0093,  0.0306, -0.1034,\n",
       "                       -0.1008,  0.0548,  0.0890,  0.0664,  0.0244,  0.0101,  0.0425, -0.0417,\n",
       "                       -0.0561,  0.1047, -0.0263,  0.0034],\n",
       "                      [-0.0520, -0.1071,  0.0383,  0.0228, -0.0594,  0.0819, -0.0562, -0.0281,\n",
       "                       -0.0921, -0.0755, -0.0115,  0.1083, -0.0697, -0.0226,  0.0808, -0.0358,\n",
       "                        0.0450, -0.0625,  0.0592, -0.0858, -0.0909, -0.0120, -0.0474, -0.0509,\n",
       "                       -0.0166,  0.0581,  0.0245,  0.0626,  0.0791,  0.0526,  0.0784, -0.0345,\n",
       "                        0.0762,  0.0961,  0.0647,  0.0387, -0.0414,  0.0607, -0.0006, -0.0849,\n",
       "                       -0.0018,  0.0982,  0.0808, -0.0140,  0.0179,  0.0227, -0.1050, -0.0745,\n",
       "                       -0.0452,  0.0150,  0.0128,  0.0118,  0.0960, -0.0598, -0.1006, -0.0303,\n",
       "                       -0.0062, -0.0016, -0.0024, -0.0633, -0.0638,  0.0061,  0.0250,  0.1014,\n",
       "                        0.0208, -0.0180,  0.1005,  0.0593, -0.0826,  0.0370,  0.1004,  0.1062,\n",
       "                        0.0886, -0.0809,  0.0377, -0.0765,  0.0586, -0.0041,  0.0479, -0.0275,\n",
       "                       -0.0401, -0.0085, -0.0633, -0.0613],\n",
       "                      [-0.0213,  0.0545, -0.0497, -0.0789,  0.0693, -0.1080,  0.0052, -0.1052,\n",
       "                       -0.1076, -0.0934, -0.1024,  0.0872,  0.0410, -0.0126,  0.0465, -0.0679,\n",
       "                       -0.0971,  0.0105,  0.0741,  0.0271, -0.1030,  0.0815,  0.0817, -0.0619,\n",
       "                        0.0437,  0.0873,  0.0027, -0.0268, -0.1016, -0.0966, -0.0283, -0.0769,\n",
       "                       -0.0777, -0.0411,  0.0911,  0.0036, -0.0343,  0.0350, -0.0576,  0.0950,\n",
       "                        0.0837, -0.0373,  0.0538,  0.0670, -0.0290, -0.0495,  0.0780, -0.0407,\n",
       "                       -0.0605, -0.0964, -0.1073, -0.0705,  0.0517,  0.0512,  0.0454,  0.0442,\n",
       "                        0.0791, -0.0865, -0.1087,  0.0558,  0.0126, -0.0172, -0.0033,  0.0736,\n",
       "                        0.0916, -0.0109,  0.0168, -0.0552,  0.0290, -0.0604,  0.0302, -0.0271,\n",
       "                        0.0653,  0.0669, -0.1026,  0.0270, -0.0996,  0.0221,  0.0026, -0.0978,\n",
       "                        0.0314, -0.0705, -0.0215,  0.1087],\n",
       "                      [-0.0615,  0.0097,  0.1049, -0.0414,  0.0766,  0.0733,  0.0719, -0.0552,\n",
       "                        0.1034, -0.0822, -0.0761, -0.0712, -0.0878,  0.0814,  0.0758, -0.1056,\n",
       "                        0.0884,  0.0680, -0.0144, -0.0279, -0.0799,  0.0048, -0.0520, -0.0731,\n",
       "                        0.0657,  0.0333,  0.0518, -0.0210,  0.0829, -0.0323, -0.0578,  0.0541,\n",
       "                       -0.0148, -0.0869,  0.0755,  0.1059, -0.0442, -0.0524, -0.0174, -0.0092,\n",
       "                       -0.1076, -0.0844,  0.0271,  0.0145,  0.1059, -0.0402,  0.0629,  0.0417,\n",
       "                        0.0085, -0.0408,  0.0742,  0.0216,  0.0785, -0.0489,  0.0625, -0.0617,\n",
       "                       -0.0067, -0.0044,  0.0891,  0.0783, -0.0262, -0.0345, -0.0777, -0.1062,\n",
       "                       -0.0016, -0.0177, -0.1051, -0.0876, -0.0435, -0.0525, -0.1024, -0.1075,\n",
       "                       -0.0882, -0.0258, -0.0045, -0.0736,  0.0992,  0.0495, -0.0798,  0.0736,\n",
       "                        0.0025, -0.1017,  0.0340,  0.0290],\n",
       "                      [ 0.1013, -0.0578,  0.0395, -0.0220,  0.0532, -0.0621, -0.0447,  0.0746,\n",
       "                        0.0405, -0.1050, -0.1061,  0.0688, -0.0049,  0.1025, -0.0166,  0.0622,\n",
       "                       -0.0311, -0.0614,  0.0897,  0.0002,  0.0784,  0.0874, -0.0621, -0.0562,\n",
       "                       -0.0729, -0.1032,  0.0117, -0.0136, -0.0887, -0.0977, -0.0764,  0.0829,\n",
       "                       -0.1011, -0.0770, -0.0947,  0.0973,  0.0744, -0.0832,  0.0099, -0.0191,\n",
       "                       -0.0758, -0.0588, -0.0134, -0.0571,  0.0823, -0.0150,  0.0970,  0.0411,\n",
       "                       -0.0980, -0.1084,  0.0447, -0.0764,  0.0117,  0.0876,  0.0418, -0.0748,\n",
       "                       -0.0057,  0.0162, -0.0840, -0.1010,  0.0251, -0.0311, -0.0123,  0.0714,\n",
       "                       -0.0832, -0.0701, -0.0714,  0.0547, -0.1083, -0.0981,  0.0829, -0.0356,\n",
       "                        0.0068, -0.0231,  0.0817, -0.0857, -0.0489, -0.1089, -0.0148,  0.0662,\n",
       "                       -0.0664, -0.0663,  0.1082,  0.1050],\n",
       "                      [ 0.0580, -0.0003,  0.0885, -0.0813,  0.1045, -0.0211, -0.0309,  0.0309,\n",
       "                       -0.0089,  0.0466,  0.0541,  0.0481, -0.0711,  0.0975,  0.0027, -0.0593,\n",
       "                       -0.0903, -0.0910, -0.0135,  0.0376, -0.0374,  0.0845,  0.0281, -0.1057,\n",
       "                        0.0327, -0.0231,  0.0650, -0.0645,  0.0940,  0.0095,  0.0443,  0.0381,\n",
       "                        0.0249,  0.0666, -0.0606, -0.0643,  0.0370, -0.0566, -0.0507,  0.0418,\n",
       "                        0.0789,  0.0099,  0.0949,  0.0546,  0.0416,  0.0766, -0.0835,  0.0854,\n",
       "                        0.0394,  0.0511, -0.1066,  0.0399,  0.0037, -0.1000,  0.0330, -0.0342,\n",
       "                       -0.0452,  0.0913,  0.0433,  0.0309,  0.1018, -0.0079,  0.0692,  0.0249,\n",
       "                        0.1016,  0.0170,  0.0586, -0.0599,  0.0575,  0.0280,  0.0679,  0.0593,\n",
       "                        0.0869,  0.0721,  0.1085, -0.1012, -0.0820, -0.0498, -0.0666, -0.0886,\n",
       "                       -0.0373,  0.0552, -0.0482, -0.0329],\n",
       "                      [ 0.0796, -0.0874, -0.0288, -0.0205, -0.0898, -0.0060,  0.0928, -0.0950,\n",
       "                        0.0012,  0.0229,  0.0462,  0.0099, -0.0371,  0.0140,  0.0251, -0.0244,\n",
       "                        0.0733,  0.0194, -0.0703,  0.0804, -0.0783, -0.0494,  0.0897,  0.0419,\n",
       "                       -0.0492,  0.0464, -0.0230,  0.0152,  0.0992,  0.0457,  0.0284, -0.0648,\n",
       "                        0.0794, -0.0827, -0.0649, -0.0985,  0.0462,  0.0714,  0.0431, -0.0597,\n",
       "                        0.0318,  0.1011,  0.0280,  0.0300, -0.0162,  0.0797,  0.0795,  0.0137,\n",
       "                        0.0974,  0.0930, -0.1080, -0.0535,  0.0872, -0.0803,  0.1076, -0.0868,\n",
       "                       -0.0622,  0.0096, -0.0292,  0.0440,  0.0109,  0.0773, -0.0910, -0.1062,\n",
       "                        0.0469,  0.0608, -0.0391, -0.0621,  0.0759,  0.0096, -0.0029, -0.0204,\n",
       "                       -0.0157, -0.1052, -0.0551,  0.0386,  0.0938,  0.0398, -0.0045,  0.0503,\n",
       "                        0.0190, -0.0441, -0.0927, -0.0488],\n",
       "                      [ 0.0135, -0.0388,  0.0274, -0.0771, -0.0386,  0.0647, -0.0692,  0.0675,\n",
       "                       -0.0650, -0.0143,  0.1058, -0.0790, -0.0182, -0.0383,  0.0285, -0.0190,\n",
       "                        0.0888, -0.1040,  0.0163, -0.0423, -0.0129, -0.0786, -0.0803,  0.0004,\n",
       "                        0.0231, -0.0046, -0.0358, -0.0167, -0.0721,  0.0447,  0.0027, -0.0497,\n",
       "                        0.0202, -0.0874, -0.0610, -0.0475, -0.0822,  0.0531,  0.0346,  0.0102,\n",
       "                       -0.1077, -0.0276,  0.0483,  0.0696,  0.0892, -0.0566,  0.0040, -0.0157,\n",
       "                        0.0585, -0.0442,  0.0828,  0.1072,  0.0521, -0.0782, -0.0155,  0.1059,\n",
       "                        0.1074,  0.0803,  0.0622, -0.0118, -0.0641, -0.0998, -0.0165,  0.0798,\n",
       "                        0.0063, -0.0591,  0.0578,  0.0509,  0.0402, -0.0664,  0.0775, -0.0444,\n",
       "                       -0.0802,  0.0575, -0.0834, -0.0548,  0.0997, -0.0528, -0.0007,  0.0019,\n",
       "                        0.0414, -0.0550,  0.0012,  0.0612]])),\n",
       "             ('fc3.bias',\n",
       "              tensor([ 0.0676, -0.0524,  0.0423,  0.0640, -0.0047, -0.1043,  0.0330,  0.0269,\n",
       "                       0.0739,  0.0788]))])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.440513Z",
     "iopub.status.busy": "2020-12-06T08:43:18.440194Z",
     "iopub.status.idle": "2020-12-06T08:43:18.441983Z",
     "shell.execute_reply": "2020-12-06T08:43:18.441628Z"
    }
   },
   "outputs": [],
   "source": [
    "def state_dict_info(obj):\n",
    "    print(f\"{'layer':25} shape\")\n",
    "    print(\"===================================================\")\n",
    "    for k,v in obj.state_dict().items():\n",
    "        try:\n",
    "            print(f\"{k:25} {v.shape}\")\n",
    "        except AttributeError:\n",
    "            print(f\"{k:25} {v}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.444064Z",
     "iopub.status.busy": "2020-12-06T08:43:18.443617Z",
     "iopub.status.idle": "2020-12-06T08:43:18.445993Z",
     "shell.execute_reply": "2020-12-06T08:43:18.446235Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer                     shape\n",
      "===================================================\n",
      "conv1.weight              torch.Size([6, 3, 5, 5])\n",
      "conv1.bias                torch.Size([6])\n",
      "conv2.weight              torch.Size([16, 6, 5, 5])\n",
      "conv2.bias                torch.Size([16])\n",
      "fc1.weight                torch.Size([120, 400])\n",
      "fc1.bias                  torch.Size([120])\n",
      "fc2.weight                torch.Size([84, 120])\n",
      "fc2.bias                  torch.Size([84])\n",
      "fc3.weight                torch.Size([10, 84])\n",
      "fc3.bias                  torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "state_dict_info(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `nn.Optimizer`\n",
    "\n",
    "Optimizers also have a a `state_dict`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.448367Z",
     "iopub.status.busy": "2020-12-06T08:43:18.448062Z",
     "iopub.status.idle": "2020-12-06T08:43:18.449787Z",
     "shell.execute_reply": "2020-12-06T08:43:18.449475Z"
    }
   },
   "outputs": [],
   "source": [
    "optimizer = optim.Adadelta(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.451843Z",
     "iopub.status.busy": "2020-12-06T08:43:18.451538Z",
     "iopub.status.idle": "2020-12-06T08:43:18.453489Z",
     "shell.execute_reply": "2020-12-06T08:43:18.453172Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "layer                     shape\n",
      "===================================================\n",
      "state                     {}\n",
      "param_groups              [{'lr': 1.0, 'rho': 0.9, 'eps': 1e-06, 'weight_decay': 0, 'params': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]\n"
     ]
    }
   ],
   "source": [
    "state_dict_info(optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.455588Z",
     "iopub.status.busy": "2020-12-06T08:43:18.455250Z",
     "iopub.status.idle": "2020-12-06T08:43:18.457095Z",
     "shell.execute_reply": "2020-12-06T08:43:18.457342Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.state_dict()[\"state\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.459785Z",
     "iopub.status.busy": "2020-12-06T08:43:18.459439Z",
     "iopub.status.idle": "2020-12-06T08:43:18.461426Z",
     "shell.execute_reply": "2020-12-06T08:43:18.461111Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'lr': 1.0,\n",
       "  'rho': 0.9,\n",
       "  'eps': 1e-06,\n",
       "  'weight_decay': 0,\n",
       "  'params': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.state_dict()[\"param_groups\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Storing and loading `state_dict`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.463424Z",
     "iopub.status.busy": "2020-12-06T08:43:18.463113Z",
     "iopub.status.idle": "2020-12-06T08:43:18.534447Z",
     "shell.execute_reply": "2020-12-06T08:43:18.535799Z"
    }
   },
   "outputs": [],
   "source": [
    "model_file = \"model_state_dict.pt\"\n",
    "torch.save(model.state_dict(), model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.548329Z",
     "iopub.status.busy": "2020-12-06T08:43:18.546634Z",
     "iopub.status.idle": "2020-12-06T08:43:18.562734Z",
     "shell.execute_reply": "2020-12-06T08:43:18.564311Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Net()\n",
    "model.load_state_dict(torch.load(model_file))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Storing and loading the full model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.575979Z",
     "iopub.status.busy": "2020-12-06T08:43:18.572975Z",
     "iopub.status.idle": "2020-12-06T08:43:18.580319Z",
     "shell.execute_reply": "2020-12-06T08:43:18.581053Z"
    }
   },
   "outputs": [],
   "source": [
    "model_file = \"model_123.pt\"\n",
    "torch.save(model, model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-06T08:43:18.585693Z",
     "iopub.status.busy": "2020-12-06T08:43:18.584925Z",
     "iopub.status.idle": "2020-12-06T08:43:18.589318Z",
     "shell.execute_reply": "2020-12-06T08:43:18.588718Z"
    }
   },
   "outputs": [],
   "source": [
    "# Only works if code for `Net` is available right now\n",
    "model = torch.load(model_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Example Checkpointing\n",
    "You can store model, optimizer and arbitrary information and reload it.\n",
    "\n",
    "Example:\n",
    "```python\n",
    "torch.save(\n",
    "    {\n",
    "        'model_state_dict': model.state_dict(),\n",
    "        'optimizer_state_dict': optimizer.state_dict(),\n",
    "        'epoch': epoch,\n",
    "        'loss': loss,\n",
    "    },\n",
    "    PATH,\n",
    ")\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise\n",
    "- Find out what is going to be in the `state` variable of the `state_dict` of an optimizer.\n",
    "- Write your own checkpoint functionality."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}