cnn2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Neural Networks II - Advanced Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import torch.utils.data as data\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "import numpy as np\n",
    "import time\n",
    "import copy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LeNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LeNet(nn.Module):\n",
    "    def __init__(self, output_dim):\n",
    "        super().__init__()\n",
    "\n",
    "        self.conv1 = nn.Conv2d(in_channels = 1, \n",
    "                               out_channels = 6, \n",
    "                               kernel_size = 5)\n",
    "        \n",
    "        self.conv2 = nn.Conv2d(in_channels = 6, \n",
    "                               out_channels = 16, \n",
    "                               kernel_size = 5)\n",
    "        \n",
    "        self.fc_1 = nn.Linear(16 * 4 * 4, 120)\n",
    "        self.fc_2 = nn.Linear(120, 84)\n",
    "        self.fc_3 = nn.Linear(84, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        #x = [batch size, 1, 28, 28]\n",
    "        \n",
    "        x = self.conv1(x)\n",
    "        \n",
    "        #x = [batch size, 6, 24, 24]\n",
    "        \n",
    "        x = F.max_pool2d(x, kernel_size = 2)\n",
    "        \n",
    "        #x = [batch size, 6, 12, 12]\n",
    "        \n",
    "        x = F.relu(x)\n",
    "        \n",
    "        x = self.conv2(x)\n",
    "        \n",
    "        #x = [batch size, 16, 8, 8]\n",
    "        \n",
    "        x = F.max_pool2d(x, kernel_size = 2)\n",
    "        \n",
    "        #x = [batch size, 16, 4, 4]\n",
    "        \n",
    "        x = F.relu(x)\n",
    "        \n",
    "        x = x.view(x.shape[0], -1)\n",
    "        \n",
    "        #x = [batch size, 16*4*4 = 256]\n",
    "        \n",
    "        h = x\n",
    "        \n",
    "        x = self.fc_1(x)\n",
    "        \n",
    "        #x = [batch size, 100]\n",
    "        \n",
    "        x = F.relu(x)\n",
    "\n",
    "        x = self.fc_2(x)\n",
    "        \n",
    "        #x = batch size, 84]\n",
    "        \n",
    "        x = F.relu(x)\n",
    "\n",
    "        x = self.fc_3(x)\n",
    "\n",
    "        #x = [batch size, output dim]\n",
    "        \n",
    "        return x, h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = LeNet(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LeNet(\n",
      "  (conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n",
      "  (fc_1): Linear(in_features=256, out_features=120, bias=True)\n",
      "  (fc_2): Linear(in_features=120, out_features=84, bias=True)\n",
      "  (fc_3): Linear(in_features=84, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "fake_num = torch.randn(64,1,28,28)\n",
    "result = net(fake_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 10])\n",
      "torch.Size([64, 256])\n"
     ]
    }
   ],
   "source": [
    "print(result[0].shape)\n",
    "print(result[1].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1            [-1, 6, 24, 24]             156\n",
      "            Conv2d-2             [-1, 16, 8, 8]           2,416\n",
      "            Linear-3                  [-1, 120]          30,840\n",
      "            Linear-4                   [-1, 84]          10,164\n",
      "            Linear-5                   [-1, 10]             850\n",
      "================================================================\n",
      "Total params: 44,426\n",
      "Trainable params: 44,426\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.00\n",
      "Forward/backward pass size (MB): 0.04\n",
      "Params size (MB): 0.17\n",
      "Estimated Total Size (MB): 0.21\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "import torchsummary\n",
    "torchsummary.summary(net,(1,28,28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(net.parameters())\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_accuracy(y_pred, y):\n",
    "    top_pred = y_pred.argmax(1, keepdim = True)\n",
    "    correct = top_pred.eq(y.view_as(top_pred)).sum()\n",
    "    acc = correct.float() / y.shape[0]\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion, device):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    for (x, y) in iterator:\n",
    "        \n",
    "        x = x.to(device)\n",
    "        y = y.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "                \n",
    "        y_pred, _ = model(x)\n",
    "        \n",
    "        loss = criterion(y_pred, y)\n",
    "        \n",
    "        acc = calculate_accuracy(y_pred, y)\n",
    "        \n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, iterator, criterion, device):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        \n",
    "        for (x, y) in iterator:\n",
    "\n",
    "            x = x.to(device)\n",
    "            y = y.to(device)\n",
    "\n",
    "            y_pred, _ = model(x)\n",
    "\n",
    "            loss = criterion(y_pred, y)\n",
    "\n",
    "            acc = calculate_accuracy(y_pred, y)\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculated mean: 0.13064366579055786\n",
      "Calculated std: 0.30810779333114624\n"
     ]
    }
   ],
   "source": [
    "ROOT = '.data'\n",
    "\n",
    "train_data = datasets.MNIST(root = ROOT, \n",
    "                            train = True, \n",
    "                            download = False)\n",
    "\n",
    "mean = train_data.data.float().mean() / 255\n",
    "std = train_data.data.float().std() / 255\n",
    "\n",
    "print(f'Calculated mean: {mean}')\n",
    "print(f'Calculated std: {std}')\n",
    "\n",
    "train_transforms = transforms.Compose([\n",
    "                            transforms.RandomRotation(10, fill=(0,)),\n",
    "                            transforms.RandomCrop(28, padding = 5),\n",
    "                            transforms.ToTensor(),\n",
    "                            transforms.Normalize(mean = [mean], std = [std])\n",
    "                                      ])\n",
    "\n",
    "test_transforms = transforms.Compose([\n",
    "                           transforms.ToTensor(),\n",
    "                           transforms.Normalize(mean = [mean], std = [std])\n",
    "                                     ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = datasets.MNIST(root = ROOT, \n",
    "                            train = True, \n",
    "                            download = True, \n",
    "                            transform = train_transforms)\n",
    "\n",
    "test_data = datasets.MNIST(root = ROOT, \n",
    "                           train = False, \n",
    "                           download = True, \n",
    "                           transform = test_transforms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "VALID_RATIO = 0.9\n",
    "\n",
    "n_train_examples = int(len(train_data) * VALID_RATIO)\n",
    "n_valid_examples = len(train_data) - n_train_examples\n",
    "\n",
    "train_data, valid_data = data.random_split(train_data, \n",
    "                                           [n_train_examples, n_valid_examples])\n",
    "\n",
    "valid_data = copy.deepcopy(valid_data)\n",
    "valid_data.dataset.transform = test_transforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "\n",
    "train_iterator = data.DataLoader(train_data, \n",
    "                                 shuffle = True, \n",
    "                                 batch_size = BATCH_SIZE)\n",
    "\n",
    "valid_iterator = data.DataLoader(valid_data, \n",
    "                                 batch_size = BATCH_SIZE)\n",
    "\n",
    "test_iterator = data.DataLoader(test_data, \n",
    "                                batch_size = BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 1m 0s\n",
      "\tTrain Loss: 0.252 | Train Acc: 92.07%\n",
      "\t Val. Loss: 0.208 |  Val. Acc: 93.78%\n",
      "Epoch: 02 | Epoch Time: 1m 0s\n",
      "\tTrain Loss: 0.196 | Train Acc: 93.83%\n",
      "\t Val. Loss: 0.194 |  Val. Acc: 94.19%\n",
      "Epoch: 03 | Epoch Time: 1m 0s\n",
      "\tTrain Loss: 0.163 | Train Acc: 94.98%\n",
      "\t Val. Loss: 0.144 |  Val. Acc: 95.61%\n",
      "Epoch: 04 | Epoch Time: 1m 0s\n",
      "\tTrain Loss: 0.147 | Train Acc: 95.48%\n",
      "\t Val. Loss: 0.131 |  Val. Acc: 95.93%\n",
      "Epoch: 05 | Epoch Time: 1m 2s\n",
      "\tTrain Loss: 0.131 | Train Acc: 95.98%\n",
      "\t Val. Loss: 0.154 |  Val. Acc: 95.22%\n",
      "Epoch: 06 | Epoch Time: 1m 0s\n",
      "\tTrain Loss: 0.123 | Train Acc: 96.17%\n",
      "\t Val. Loss: 0.113 |  Val. Acc: 96.42%\n",
      "Epoch: 07 | Epoch Time: 1m 1s\n",
      "\tTrain Loss: 0.116 | Train Acc: 96.42%\n",
      "\t Val. Loss: 0.101 |  Val. Acc: 96.84%\n",
      "Epoch: 08 | Epoch Time: 1m 1s\n",
      "\tTrain Loss: 0.103 | Train Acc: 96.81%\n",
      "\t Val. Loss: 0.110 |  Val. Acc: 96.68%\n",
      "Epoch: 09 | Epoch Time: 1m 5s\n",
      "\tTrain Loss: 0.101 | Train Acc: 96.85%\n",
      "\t Val. Loss: 0.099 |  Val. Acc: 96.49%\n",
      "Epoch: 10 | Epoch Time: 1m 1s\n",
      "\tTrain Loss: 0.098 | Train Acc: 96.94%\n",
      "\t Val. Loss: 0.099 |  Val. Acc: 97.19%\n",
      "Epoch: 11 | Epoch Time: 1m 6s\n",
      "\tTrain Loss: 0.093 | Train Acc: 97.08%\n",
      "\t Val. Loss: 0.091 |  Val. Acc: 96.95%\n",
      "Epoch: 12 | Epoch Time: 1m 2s\n",
      "\tTrain Loss: 0.091 | Train Acc: 97.22%\n",
      "\t Val. Loss: 0.109 |  Val. Acc: 96.92%\n",
      "Epoch: 13 | Epoch Time: 0m 59s\n",
      "\tTrain Loss: 0.085 | Train Acc: 97.32%\n",
      "\t Val. Loss: 0.097 |  Val. Acc: 96.97%\n",
      "Epoch: 14 | Epoch Time: 1m 0s\n",
      "\tTrain Loss: 0.086 | Train Acc: 97.25%\n",
      "\t Val. Loss: 0.085 |  Val. Acc: 97.43%\n",
      "Epoch: 15 | Epoch Time: 1m 5s\n",
      "\tTrain Loss: 0.082 | Train Acc: 97.43%\n",
      "\t Val. Loss: 0.086 |  Val. Acc: 97.50%\n",
      "Epoch: 16 | Epoch Time: 0m 56s\n",
      "\tTrain Loss: 0.080 | Train Acc: 97.47%\n",
      "\t Val. Loss: 0.076 |  Val. Acc: 97.72%\n",
      "Epoch: 17 | Epoch Time: 0m 56s\n",
      "\tTrain Loss: 0.077 | Train Acc: 97.60%\n",
      "\t Val. Loss: 0.092 |  Val. Acc: 97.31%\n",
      "Epoch: 18 | Epoch Time: 0m 57s\n",
      "\tTrain Loss: 0.074 | Train Acc: 97.73%\n",
      "\t Val. Loss: 0.091 |  Val. Acc: 97.28%\n",
      "Epoch: 19 | Epoch Time: 0m 54s\n",
      "\tTrain Loss: 0.073 | Train Acc: 97.70%\n",
      "\t Val. Loss: 0.078 |  Val. Acc: 97.62%\n",
      "Epoch: 20 | Epoch Time: 0m 53s\n",
      "\tTrain Loss: 0.071 | Train Acc: 97.84%\n",
      "\t Val. Loss: 0.076 |  Val. Acc: 97.74%\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 20\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    \n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_loss, train_acc = train(net, train_iterator, optimizer, criterion, device)\n",
    "    valid_loss, valid_acc = evaluate(net, valid_iterator, criterion, device)\n",
    "    \n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(net.state_dict(), 'lenet.pt')\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AlexNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": [
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using downloaded and verified file: .data\\cifar-10-python.tar.gz\n",
      "Extracting .data\\cifar-10-python.tar.gz to .data\n",
      "Calculated means: [0.49139968 0.48215841 0.44653091]\n",
      "Calculated stds: [0.24703223 0.24348513 0.26158784]\n"
     ]
    }
   ],
   "source": [
    "ROOT = '.data'\n",
    "\n",
    "train_data = datasets.CIFAR10(root = ROOT, \n",
    "                              train = True, \n",
    "                              download = True)\n",
    "\n",
    "means = train_data.data.mean(axis = (0,1,2)) / 255\n",
    "stds = train_data.data.std(axis = (0,1,2)) / 255\n",
    "\n",
    "print(f'Calculated means: {means}')\n",
    "print(f'Calculated stds: {stds}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AlexNet(nn.Module):\n",
    "    def __init__(self, output_dim):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            nn.Conv2d(3, 64, 3, 2, 1), #in_channels, out_channels, kernel_size, stride, padding\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(2), #kernel_size\n",
    "            nn.Conv2d(64, 192, 3, padding = 1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(2),\n",
    "            nn.Conv2d(192, 384, 3, padding = 1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(384, 256, 3, padding = 1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(256, 256, 3, padding = 1),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.MaxPool2d(2),\n",
    "        )\n",
    "        \n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(256 * 2 * 2, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(4096, 4096),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(4096, output_dim),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        h = x.view(x.shape[0], -1)\n",
    "        x = self.classifier(h)\n",
    "        return x, h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "OUTPUT_DIM = 10\n",
    "\n",
    "net_alex = AlexNet(OUTPUT_DIM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "fake_num = torch.randn(64,3,32,32)\n",
    "result = net_alex(fake_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 10])\n",
      "torch.Size([64, 1024])\n"
     ]
    }
   ],
   "source": [
    "print(result[0].shape)\n",
    "print(result[1].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 64, 16, 16]           1,792\n",
      "              ReLU-2           [-1, 64, 16, 16]               0\n",
      "         MaxPool2d-3             [-1, 64, 8, 8]               0\n",
      "            Conv2d-4            [-1, 192, 8, 8]         110,784\n",
      "              ReLU-5            [-1, 192, 8, 8]               0\n",
      "         MaxPool2d-6            [-1, 192, 4, 4]               0\n",
      "            Conv2d-7            [-1, 384, 4, 4]         663,936\n",
      "              ReLU-8            [-1, 384, 4, 4]               0\n",
      "            Conv2d-9            [-1, 256, 4, 4]         884,992\n",
      "             ReLU-10            [-1, 256, 4, 4]               0\n",
      "           Conv2d-11            [-1, 256, 4, 4]         590,080\n",
      "             ReLU-12            [-1, 256, 4, 4]               0\n",
      "        MaxPool2d-13            [-1, 256, 2, 2]               0\n",
      "          Dropout-14                 [-1, 1024]               0\n",
      "           Linear-15                 [-1, 4096]       4,198,400\n",
      "             ReLU-16                 [-1, 4096]               0\n",
      "          Dropout-17                 [-1, 4096]               0\n",
      "           Linear-18                 [-1, 4096]      16,781,312\n",
      "             ReLU-19                 [-1, 4096]               0\n",
      "           Linear-20                   [-1, 10]          40,970\n",
      "================================================================\n",
      "Total params: 23,272,266\n",
      "Trainable params: 23,272,266\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 0.88\n",
      "Params size (MB): 88.78\n",
      "Estimated Total Size (MB): 89.67\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "import torchsummary\n",
    "torchsummary.summary(net_alex,(3,32,32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VGG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VGGBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, batch_norm):\n",
    "        super().__init__()\n",
    "        \n",
    "        modules = []\n",
    "        modules.append(nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1))\n",
    "        if batch_norm:\n",
    "            modules.append(nn.BatchNorm2d(out_channels))\n",
    "        modules.append(nn.ReLU(inplace=True))\n",
    "    \n",
    "        self.block = nn.Sequential(*modules)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.block(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VGG11(nn.Module):\n",
    "    def __init__(self, output_dim, block, pool, batch_norm):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            block(3, 64, batch_norm), #in_channels, out_channels\n",
    "            pool(2, 2), #kernel_size, stride\n",
    "            block(64, 128, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(128, 256, batch_norm),\n",
    "            block(256, 256, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(256, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(512, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            pool(2, 2),\n",
    "        )\n",
    "        \n",
    "        self.classifier = nn.Linear(512, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = self.classifier(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VGG16(nn.Module):\n",
    "    def __init__(self, output_dim, block, pool, batch_norm):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            block(3, 64, batch_norm),\n",
    "            block(64, 64, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(64, 128, batch_norm),\n",
    "            block(128, 128, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(128, 256, batch_norm),\n",
    "            block(256, 256, batch_norm),\n",
    "            block(256, 256, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(256, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(512, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            pool(2, 2),\n",
    "        )\n",
    "        \n",
    "        self.classifier = nn.Linear(512, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = self.classifier(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VGG19(nn.Module):\n",
    "    def __init__(self, output_dim, block, pool, batch_norm):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.features = nn.Sequential(\n",
    "            block(3, 64, batch_norm),\n",
    "            block(64, 64, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(64, 128, batch_norm),\n",
    "            block(128, 128, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(128, 256, batch_norm),\n",
    "            block(256, 256, batch_norm),\n",
    "            block(256, 256, batch_norm),\n",
    "            block(256, 256, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(256, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            pool(2, 2),\n",
    "            block(512, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            block(512, 512, batch_norm),\n",
    "            pool(2, 2),\n",
    "        )\n",
    "        \n",
    "        self.classifier = nn.Linear(512, output_dim)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x)\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = self.classifier(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "OUTPUT_DIM = 10\n",
    "BATCH_NORM = True\n",
    "\n",
    "vgg11_model = VGG11(OUTPUT_DIM, VGGBlock, nn.MaxPool2d, BATCH_NORM) \n",
    "vgg16_model = VGG16(OUTPUT_DIM, VGGBlock, nn.MaxPool2d, BATCH_NORM) \n",
    "vgg19_model = VGG19(OUTPUT_DIM, VGGBlock, nn.MaxPool2d, BATCH_NORM) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 64, 32, 32]           1,792\n",
      "       BatchNorm2d-2           [-1, 64, 32, 32]             128\n",
      "              ReLU-3           [-1, 64, 32, 32]               0\n",
      "          VGGBlock-4           [-1, 64, 32, 32]               0\n",
      "         MaxPool2d-5           [-1, 64, 16, 16]               0\n",
      "            Conv2d-6          [-1, 128, 16, 16]          73,856\n",
      "       BatchNorm2d-7          [-1, 128, 16, 16]             256\n",
      "              ReLU-8          [-1, 128, 16, 16]               0\n",
      "          VGGBlock-9          [-1, 128, 16, 16]               0\n",
      "        MaxPool2d-10            [-1, 128, 8, 8]               0\n",
      "           Conv2d-11            [-1, 256, 8, 8]         295,168\n",
      "      BatchNorm2d-12            [-1, 256, 8, 8]             512\n",
      "             ReLU-13            [-1, 256, 8, 8]               0\n",
      "         VGGBlock-14            [-1, 256, 8, 8]               0\n",
      "           Conv2d-15            [-1, 256, 8, 8]         590,080\n",
      "      BatchNorm2d-16            [-1, 256, 8, 8]             512\n",
      "             ReLU-17            [-1, 256, 8, 8]               0\n",
      "         VGGBlock-18            [-1, 256, 8, 8]               0\n",
      "        MaxPool2d-19            [-1, 256, 4, 4]               0\n",
      "           Conv2d-20            [-1, 512, 4, 4]       1,180,160\n",
      "      BatchNorm2d-21            [-1, 512, 4, 4]           1,024\n",
      "             ReLU-22            [-1, 512, 4, 4]               0\n",
      "         VGGBlock-23            [-1, 512, 4, 4]               0\n",
      "           Conv2d-24            [-1, 512, 4, 4]       2,359,808\n",
      "      BatchNorm2d-25            [-1, 512, 4, 4]           1,024\n",
      "             ReLU-26            [-1, 512, 4, 4]               0\n",
      "         VGGBlock-27            [-1, 512, 4, 4]               0\n",
      "        MaxPool2d-28            [-1, 512, 2, 2]               0\n",
      "           Conv2d-29            [-1, 512, 2, 2]       2,359,808\n",
      "      BatchNorm2d-30            [-1, 512, 2, 2]           1,024\n",
      "             ReLU-31            [-1, 512, 2, 2]               0\n",
      "         VGGBlock-32            [-1, 512, 2, 2]               0\n",
      "           Conv2d-33            [-1, 512, 2, 2]       2,359,808\n",
      "      BatchNorm2d-34            [-1, 512, 2, 2]           1,024\n",
      "             ReLU-35            [-1, 512, 2, 2]               0\n",
      "         VGGBlock-36            [-1, 512, 2, 2]               0\n",
      "        MaxPool2d-37            [-1, 512, 1, 1]               0\n",
      "           Linear-38                   [-1, 10]           5,130\n",
      "================================================================\n",
      "Total params: 9,231,114\n",
      "Trainable params: 9,231,114\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 4.86\n",
      "Params size (MB): 35.21\n",
      "Estimated Total Size (MB): 40.09\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "import torchsummary\n",
    "torchsummary.summary(vgg11_model,(3,32,32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ResNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResNetBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        \n",
    "        self.downsample = nn.Sequential()\n",
    "        \n",
    "        if stride != 1 or in_channels != out_channels:\n",
    "            self.downsample = nn.Sequential(\n",
    "                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(out_channels)\n",
    "            )\n",
    "            \n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        out += self.downsample(x)\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "class ResNetLayer(nn.Module):\n",
    "    def __init__(self, block, n_blocks, in_channels, out_channels, stride):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.modules = []\n",
    "        \n",
    "        self.modules.append(block(in_channels, out_channels, stride))\n",
    "        \n",
    "        for _ in range(n_blocks-1):\n",
    "            self.modules.append(block(out_channels, out_channels, 1))\n",
    "            \n",
    "        self.blocks = nn.Sequential(*self.modules)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.blocks(x) \n",
    "    \n",
    "class ResNet18(nn.Module):\n",
    "    def __init__(self, layer, block):\n",
    "        super().__init__()\n",
    "        \n",
    "        n_blocks = [2, 2, 2, 2]\n",
    "        \n",
    "        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(64)\n",
    "        self.layer1 = layer(block, n_blocks[0], 64, 64, 1)\n",
    "        self.layer2 = layer(block, n_blocks[1], 64, 128, 2)\n",
    "        self.layer3 = layer(block, n_blocks[2], 128, 256, 2)\n",
    "        self.layer4 = layer(block, n_blocks[3], 256, 512, 2)\n",
    "        self.fc = nn.Linear(512, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.layer4(out)\n",
    "        out = F.avg_pool2d(out, 4)\n",
    "        out = out.view(out.shape[0], -1)\n",
    "        out = self.fc(out)\n",
    "        return out\n",
    "    \n",
    "class ResNet34(nn.Module):\n",
    "    def __init__(self, layer, block):\n",
    "        super().__init__()\n",
    "        \n",
    "        n_blocks = [3, 4, 6, 3]\n",
    "        \n",
    "        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(64)\n",
    "        self.layer1 = layer(block, n_blocks[0], 64, 64, 1)\n",
    "        self.layer2 = layer(block, n_blocks[1], 64, 128, 2)\n",
    "        self.layer3 = layer(block, n_blocks[2], 128, 256, 2)\n",
    "        self.layer4 = layer(block, n_blocks[3], 256, 512, 2)\n",
    "        self.fc = nn.Linear(512, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.layer4(out)\n",
    "        out = F.avg_pool2d(out, 4)\n",
    "        out = out.view(out.shape[0], -1)\n",
    "        out = self.fc(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ResNet18(ResNetLayer, ResNetBlock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 64, 32, 32]           1,728\n",
      "       BatchNorm2d-2           [-1, 64, 32, 32]             128\n",
      "            Conv2d-3           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-4           [-1, 64, 32, 32]             128\n",
      "            Conv2d-5           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-6           [-1, 64, 32, 32]             128\n",
      "       ResNetBlock-7           [-1, 64, 32, 32]               0\n",
      "            Conv2d-8           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-9           [-1, 64, 32, 32]             128\n",
      "           Conv2d-10           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-11           [-1, 64, 32, 32]             128\n",
      "      ResNetBlock-12           [-1, 64, 32, 32]               0\n",
      "      ResNetLayer-13           [-1, 64, 32, 32]               0\n",
      "           Conv2d-14          [-1, 128, 16, 16]          73,728\n",
      "      BatchNorm2d-15          [-1, 128, 16, 16]             256\n",
      "           Conv2d-16          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-17          [-1, 128, 16, 16]             256\n",
      "           Conv2d-18          [-1, 128, 16, 16]           8,192\n",
      "      BatchNorm2d-19          [-1, 128, 16, 16]             256\n",
      "      ResNetBlock-20          [-1, 128, 16, 16]               0\n",
      "           Conv2d-21          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-22          [-1, 128, 16, 16]             256\n",
      "           Conv2d-23          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-24          [-1, 128, 16, 16]             256\n",
      "      ResNetBlock-25          [-1, 128, 16, 16]               0\n",
      "      ResNetLayer-26          [-1, 128, 16, 16]               0\n",
      "           Conv2d-27            [-1, 256, 8, 8]         294,912\n",
      "      BatchNorm2d-28            [-1, 256, 8, 8]             512\n",
      "           Conv2d-29            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-30            [-1, 256, 8, 8]             512\n",
      "           Conv2d-31            [-1, 256, 8, 8]          32,768\n",
      "      BatchNorm2d-32            [-1, 256, 8, 8]             512\n",
      "      ResNetBlock-33            [-1, 256, 8, 8]               0\n",
      "           Conv2d-34            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-35            [-1, 256, 8, 8]             512\n",
      "           Conv2d-36            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-37            [-1, 256, 8, 8]             512\n",
      "      ResNetBlock-38            [-1, 256, 8, 8]               0\n",
      "      ResNetLayer-39            [-1, 256, 8, 8]               0\n",
      "           Conv2d-40            [-1, 512, 4, 4]       1,179,648\n",
      "      BatchNorm2d-41            [-1, 512, 4, 4]           1,024\n",
      "           Conv2d-42            [-1, 512, 4, 4]       2,359,296\n",
      "      BatchNorm2d-43            [-1, 512, 4, 4]           1,024\n",
      "           Conv2d-44            [-1, 512, 4, 4]         131,072\n",
      "      BatchNorm2d-45            [-1, 512, 4, 4]           1,024\n",
      "      ResNetBlock-46            [-1, 512, 4, 4]               0\n",
      "           Conv2d-47            [-1, 512, 4, 4]       2,359,296\n",
      "      BatchNorm2d-48            [-1, 512, 4, 4]           1,024\n",
      "           Conv2d-49            [-1, 512, 4, 4]       2,359,296\n",
      "      BatchNorm2d-50            [-1, 512, 4, 4]           1,024\n",
      "      ResNetBlock-51            [-1, 512, 4, 4]               0\n",
      "      ResNetLayer-52            [-1, 512, 4, 4]               0\n",
      "           Linear-53                   [-1, 10]           5,130\n",
      "================================================================\n",
      "Total params: 11,173,962\n",
      "Trainable params: 11,173,962\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 12.19\n",
      "Params size (MB): 42.63\n",
      "Estimated Total Size (MB): 54.82\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "import torchsummary\n",
    "torchsummary.summary(model,(3,32,32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ResNet34(ResNetLayer, ResNetBlock)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 64, 32, 32]           1,728\n",
      "       BatchNorm2d-2           [-1, 64, 32, 32]             128\n",
      "            Conv2d-3           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-4           [-1, 64, 32, 32]             128\n",
      "            Conv2d-5           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-6           [-1, 64, 32, 32]             128\n",
      "       ResNetBlock-7           [-1, 64, 32, 32]               0\n",
      "            Conv2d-8           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-9           [-1, 64, 32, 32]             128\n",
      "           Conv2d-10           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-11           [-1, 64, 32, 32]             128\n",
      "      ResNetBlock-12           [-1, 64, 32, 32]               0\n",
      "           Conv2d-13           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-14           [-1, 64, 32, 32]             128\n",
      "           Conv2d-15           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-16           [-1, 64, 32, 32]             128\n",
      "      ResNetBlock-17           [-1, 64, 32, 32]               0\n",
      "      ResNetLayer-18           [-1, 64, 32, 32]               0\n",
      "           Conv2d-19          [-1, 128, 16, 16]          73,728\n",
      "      BatchNorm2d-20          [-1, 128, 16, 16]             256\n",
      "           Conv2d-21          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-22          [-1, 128, 16, 16]             256\n",
      "           Conv2d-23          [-1, 128, 16, 16]           8,192\n",
      "      BatchNorm2d-24          [-1, 128, 16, 16]             256\n",
      "      ResNetBlock-25          [-1, 128, 16, 16]               0\n",
      "           Conv2d-26          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-27          [-1, 128, 16, 16]             256\n",
      "           Conv2d-28          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-29          [-1, 128, 16, 16]             256\n",
      "      ResNetBlock-30          [-1, 128, 16, 16]               0\n",
      "           Conv2d-31          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-32          [-1, 128, 16, 16]             256\n",
      "           Conv2d-33          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-34          [-1, 128, 16, 16]             256\n",
      "      ResNetBlock-35          [-1, 128, 16, 16]               0\n",
      "           Conv2d-36          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-37          [-1, 128, 16, 16]             256\n",
      "           Conv2d-38          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-39          [-1, 128, 16, 16]             256\n",
      "      ResNetBlock-40          [-1, 128, 16, 16]               0\n",
      "      ResNetLayer-41          [-1, 128, 16, 16]               0\n",
      "           Conv2d-42            [-1, 256, 8, 8]         294,912\n",
      "      BatchNorm2d-43            [-1, 256, 8, 8]             512\n",
      "           Conv2d-44            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-45            [-1, 256, 8, 8]             512\n",
      "           Conv2d-46            [-1, 256, 8, 8]          32,768\n",
      "      BatchNorm2d-47            [-1, 256, 8, 8]             512\n",
      "      ResNetBlock-48            [-1, 256, 8, 8]               0\n",
      "           Conv2d-49            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-50            [-1, 256, 8, 8]             512\n",
      "           Conv2d-51            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-52            [-1, 256, 8, 8]             512\n",
      "      ResNetBlock-53            [-1, 256, 8, 8]               0\n",
      "           Conv2d-54            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-55            [-1, 256, 8, 8]             512\n",
      "           Conv2d-56            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-57            [-1, 256, 8, 8]             512\n",
      "      ResNetBlock-58            [-1, 256, 8, 8]               0\n",
      "           Conv2d-59            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-60            [-1, 256, 8, 8]             512\n",
      "           Conv2d-61            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-62            [-1, 256, 8, 8]             512\n",
      "      ResNetBlock-63            [-1, 256, 8, 8]               0\n",
      "           Conv2d-64            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-65            [-1, 256, 8, 8]             512\n",
      "           Conv2d-66            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-67            [-1, 256, 8, 8]             512\n",
      "      ResNetBlock-68            [-1, 256, 8, 8]               0\n",
      "           Conv2d-69            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-70            [-1, 256, 8, 8]             512\n",
      "           Conv2d-71            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-72            [-1, 256, 8, 8]             512\n",
      "      ResNetBlock-73            [-1, 256, 8, 8]               0\n",
      "      ResNetLayer-74            [-1, 256, 8, 8]               0\n",
      "           Conv2d-75            [-1, 512, 4, 4]       1,179,648\n",
      "      BatchNorm2d-76            [-1, 512, 4, 4]           1,024\n",
      "           Conv2d-77            [-1, 512, 4, 4]       2,359,296\n",
      "      BatchNorm2d-78            [-1, 512, 4, 4]           1,024\n",
      "           Conv2d-79            [-1, 512, 4, 4]         131,072\n",
      "      BatchNorm2d-80            [-1, 512, 4, 4]           1,024\n",
      "      ResNetBlock-81            [-1, 512, 4, 4]               0\n",
      "           Conv2d-82            [-1, 512, 4, 4]       2,359,296\n",
      "      BatchNorm2d-83            [-1, 512, 4, 4]           1,024\n",
      "           Conv2d-84            [-1, 512, 4, 4]       2,359,296\n",
      "      BatchNorm2d-85            [-1, 512, 4, 4]           1,024\n",
      "      ResNetBlock-86            [-1, 512, 4, 4]               0\n",
      "           Conv2d-87            [-1, 512, 4, 4]       2,359,296\n",
      "      BatchNorm2d-88            [-1, 512, 4, 4]           1,024\n",
      "           Conv2d-89            [-1, 512, 4, 4]       2,359,296\n",
      "      BatchNorm2d-90            [-1, 512, 4, 4]           1,024\n",
      "      ResNetBlock-91            [-1, 512, 4, 4]               0\n",
      "      ResNetLayer-92            [-1, 512, 4, 4]               0\n",
      "           Linear-93                   [-1, 10]           5,130\n",
      "================================================================\n",
      "Total params: 21,282,122\n",
      "Trainable params: 21,282,122\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 20.00\n",
      "Params size (MB): 81.18\n",
      "Estimated Total Size (MB): 101.20\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "import torchsummary\n",
    "torchsummary.summary(model,(3,32,32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "py37",
   "language": "python",
   "name": "py37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}