studies_loss.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "baselines.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.5"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a6f30b36709f434886701d6b60e5709f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_73b75f48e54049479524f662636c3a19",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f0c5f9e311a745379c64c31f8a1e3c2c",
              "IPY_MODEL_79c17cb9113a4c00b2c9d6829ff7b4d7"
            ]
          }
        },
        "73b75f48e54049479524f662636c3a19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f0c5f9e311a745379c64c31f8a1e3c2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_05c00011dbac4415b59b32333730b15f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c3302adf71294e6cb5258b33a5a851b7"
          }
        },
        "79c17cb9113a4c00b2c9d6829ff7b4d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6ce6c082bc00468486e67a2c40780656",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 169009152/? [00:30&lt;00:00, 17667165.32it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fd59197435924642a9bb83d0f731a568"
          }
        },
        "05c00011dbac4415b59b32333730b15f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c3302adf71294e6cb5258b33a5a851b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6ce6c082bc00468486e67a2c40780656": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fd59197435924642a9bb83d0f731a568": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "NzqxHIh4OCdW"
      },
      "source": [
        "# Incremental learning on image classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "wBHSznCZxpNB"
      },
      "source": [
        "## Libraries and packages\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4eQ6O12jxMFf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 661
        },
        "outputId": "033c0250-3002-4dcb-bdd8-493772fb3d3e"
      },
      "source": [
        "!pip3 install 'torch==1.4.0'\n",
        "!pip3 install 'torchvision==0.5.0'\n",
        "!pip3 install 'Pillow-SIMD'"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting torch==1.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/24/19/4804aea17cd136f1705a5e98a00618cb8f6ccc375ad8bfa437408e09d058/torch-1.4.0-cp36-cp36m-manylinux1_x86_64.whl (753.4MB)\n",
            "\u001b[K     |████████████████████████████████| 753.4MB 22kB/s \n",
            "\u001b[31mERROR: torchvision 0.6.1+cu101 has requirement torch==1.5.1, but you'll have torch 1.4.0 which is incompatible.\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: torch\n",
            "  Found existing installation: torch 1.5.1+cu101\n",
            "    Uninstalling torch-1.5.1+cu101:\n",
            "      Successfully uninstalled torch-1.5.1+cu101\n",
            "Successfully installed torch-1.4.0\n",
            "Collecting torchvision==0.5.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/90/6141bf41f5655c78e24f40f710fdd4f8a8aff6c8b7c6f0328240f649bdbe/torchvision-0.5.0-cp36-cp36m-manylinux1_x86_64.whl (4.0MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0MB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision==0.5.0) (7.0.0)\n",
            "Requirement already satisfied: torch==1.4.0 in /usr/local/lib/python3.6/dist-packages (from torchvision==0.5.0) (1.4.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision==0.5.0) (1.18.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision==0.5.0) (1.15.0)\n",
            "Installing collected packages: torchvision\n",
            "  Found existing installation: torchvision 0.6.1+cu101\n",
            "    Uninstalling torchvision-0.6.1+cu101:\n",
            "      Successfully uninstalled torchvision-0.6.1+cu101\n",
            "Successfully installed torchvision-0.5.0\n",
            "Collecting Pillow-SIMD\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a0/6a/30d21c886293cca3755b8e55de34137a5068b77eba1c0644d3632080516b/Pillow-SIMD-7.0.0.post3.tar.gz (630kB)\n",
            "\u001b[K     |████████████████████████████████| 634kB 2.6MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: Pillow-SIMD\n",
            "  Building wheel for Pillow-SIMD (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for Pillow-SIMD: filename=Pillow_SIMD-7.0.0.post3-cp36-cp36m-linux_x86_64.whl size=1110316 sha256=d6b50447fc439940451f73bee09c8cf455eeaff7ef35bba3c33689e493805ae9\n",
            "  Stored in directory: /root/.cache/pip/wheels/d3/ac/4f/4cdf8febba528e5f1b09fc58d5181e1c12ed1e8655dcd583b8\n",
            "Successfully built Pillow-SIMD\n",
            "Installing collected packages: Pillow-SIMD\n",
            "Successfully installed Pillow-SIMD-7.0.0.post3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xAYXtIdpx0Yy",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import urllib\n",
        "import logging\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.init as init\n",
        "import torch.optim as optim\n",
        "from torch.utils.data import Dataset, Subset, DataLoader, ConcatDataset\n",
        "from torch.backends import cudnn\n",
        "\n",
        "import torchvision\n",
        "from torchvision import transforms\n",
        "from torchvision.models import resnet34\n",
        "\n",
        "from PIL import Image\n",
        "from copy import deepcopy\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "from sklearn.metrics import confusion_matrix"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "09iWc_oCotu2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 419
        },
        "outputId": "d4d3aa50-f1a4-493f-862b-be28d4ee26f2"
      },
      "source": [
        "# Download packages from repository\n",
        "!git clone https://github.com/manuelemacchia/incremental-learning-image-classification.git\n",
        "!mv -v incremental-learning-image-classification/* .\n",
        "!rm -rf incremental-learning-image-classification README.md\n",
        "\n",
        "from data.cifar100 import Cifar100\n",
        "from model.resnet_cifar import resnet32\n",
        "from model.manager import Manager\n",
        "from model.lwf import LWF\n",
        "from model.icarl import Exemplars\n",
        "from model.icarl import iCaRL\n",
        "from losses.losses import AsymLoss, DKHLoss\n",
        "from utils import plot"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'incremental-learning-image-classification'...\n",
            "remote: Enumerating objects: 103, done.\u001b[K\n",
            "remote: Counting objects: 100% (103/103), done.\u001b[K\n",
            "remote: Compressing objects: 100% (81/81), done.\u001b[K\n",
            "remote: Total 1216 (delta 54), reused 47 (delta 16), pack-reused 1113\u001b[K\n",
            "Receiving objects: 100% (1216/1216), 6.98 MiB | 5.28 MiB/s, done.\n",
            "Resolving deltas: 100% (694/694), done.\n",
            "renamed 'incremental-learning-image-classification/baselines.ipynb' -> './baselines.ipynb'\n",
            "mv: cannot move 'incremental-learning-image-classification/data' to './data': Directory not empty\n",
            "renamed 'incremental-learning-image-classification/distillation_targets.ipynb' -> './distillation_targets.ipynb'\n",
            "mv: cannot move 'incremental-learning-image-classification/losses' to './losses': Directory not empty\n",
            "mv: cannot move 'incremental-learning-image-classification/model' to './model': Directory not empty\n",
            "renamed 'incremental-learning-image-classification/README.md' -> './README.md'\n",
            "mv: cannot move 'incremental-learning-image-classification/report' to './report': Directory not empty\n",
            "renamed 'incremental-learning-image-classification/representation_drift.ipynb' -> './representation_drift.ipynb'\n",
            "renamed 'incremental-learning-image-classification/representation_drift_tsne.ipynb' -> './representation_drift_tsne.ipynb'\n",
            "mv: cannot move 'incremental-learning-image-classification/results' to './results': Directory not empty\n",
            "renamed 'incremental-learning-image-classification/studies_classifier.ipynb' -> './studies_classifier.ipynb'\n",
            "renamed 'incremental-learning-image-classification/studies_loss.ipynb' -> './studies_loss.ipynb'\n",
            "mv: cannot move 'incremental-learning-image-classification/utils' to './utils': Directory not empty\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "j12pgffMR6Qv"
      },
      "source": [
        "## Arguments"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JwE0x8gkSisn",
        "colab": {}
      },
      "source": [
        "# Directories\n",
        "DATA_DIR = 'data'       # Directory where the dataset will be downloaded\n",
        "\n",
        "# Settings\n",
        "DEVICE = 'cuda'\n",
        "\n",
        "# Dataset\n",
        "RANDOM_STATES = [658, 423, 422]      # For reproducibility of results                        \n",
        "                                     # Note: different random states give very different\n",
        "                                     # splits and therefore very different results.\n",
        "\n",
        "NUM_CLASSES = 100       # Total number of classes\n",
        "\n",
        "VAL_SIZE = 0.1          # Proportion of validation set with respect to training set (between 0 and 1)\n",
        "\n",
        "# Training\n",
        "BATCH_SIZE = 64         # Batch size (iCaRL sets this to 128)\n",
        "LR = 2                  # Initial learning rate\n",
        "                       \n",
        "MOMENTUM = 0.9          # Momentum for stochastic gradient descent (SGD)\n",
        "WEIGHT_DECAY = 1e-5     # Weight decay from iCaRL\n",
        "\n",
        "NUM_RUNS = 3            # Number of runs of every method\n",
        "                        # Note: this should be at least 3 to have a fair benchmark\n",
        "\n",
        "NUM_EPOCHS = 70         # Total number of training epochs\n",
        "MILESTONES = [49, 63]   # Step down policy from iCaRL (MultiStepLR)\n",
        "                        # Decrease the learning rate by gamma at each milestone\n",
        "GAMMA = 0.2             # Gamma factor from iCaRL"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7B6CcDcDlMf_",
        "colab_type": "text"
      },
      "source": [
        "## iCaRL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9VfTxDkoLd26",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class iCaRLwithLoss(iCaRL):\n",
        "    \"\"\"Implement iCaRL, a strategy for simultaneously learning classifiers and a\n",
        "    feature representation in the class-incremental setting.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, device, net, lr, criterion, momentum, weight_decay, milestones, gamma, num_epochs, batch_size, train_transform, test_transform):\n",
        "        \n",
        "        super().__init__(device, net, lr, momentum, weight_decay, milestones, gamma, num_epochs, batch_size, train_transform, test_transform)\n",
        "        # Loss function\n",
        "        self.criterion = criterion\n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rF0ypxGognNR",
        "colab_type": "text"
      },
      "source": [
        "### Data preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4mf04UjEgmPG",
        "colab": {}
      },
      "source": [
        "# Transformations for Learning Without Forgetting\n",
        "train_transform = transforms.Compose([transforms.RandomCrop(32, padding=4),\n",
        "                                      transforms.RandomHorizontalFlip(),\n",
        "                                      transforms.ToTensor(), # Turn PIL Image to torch.Tensor\n",
        "                                      transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n",
        "])\n",
        "\n",
        "test_transform = transforms.Compose([transforms.ToTensor(),\n",
        "                                     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))                                    \n",
        "])"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Y9Oq44dxgmPN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84,
          "referenced_widgets": [
            "a6f30b36709f434886701d6b60e5709f",
            "73b75f48e54049479524f662636c3a19",
            "f0c5f9e311a745379c64c31f8a1e3c2c",
            "79c17cb9113a4c00b2c9d6829ff7b4d7",
            "05c00011dbac4415b59b32333730b15f",
            "c3302adf71294e6cb5258b33a5a851b7",
            "6ce6c082bc00468486e67a2c40780656",
            "fd59197435924642a9bb83d0f731a568"
          ]
        },
        "outputId": "3ed6e3a4-4684-41d4-cd28-87c179c8239b"
      },
      "source": [
        "train_subsets = [[] for i in range(NUM_RUNS)]\n",
        "val_subsets = [[] for i in range(NUM_RUNS)]\n",
        "test_subsets = [[] for i in range(NUM_RUNS)]\n",
        "\n",
        "for run_i in range(NUM_RUNS):\n",
        "    for split_i in range(10):\n",
        "        if run_i+split_i == 0: # Download dataset only at first instantiation\n",
        "            download = True\n",
        "        else:\n",
        "            download = False\n",
        "\n",
        "        # Create CIFAR100 dataset\n",
        "        train_dataset = Cifar100(DATA_DIR, train=True, download=download, random_state=RANDOM_STATES[run_i], transform=train_transform)\n",
        "        test_dataset = Cifar100(DATA_DIR, train=False, download=False, random_state=RANDOM_STATES[run_i], transform=test_transform)\n",
        "    \n",
        "        # Subspace of CIFAR100 of 10 classes\n",
        "        train_dataset.set_classes_batch(train_dataset.batch_splits[split_i]) \n",
        "        test_dataset.set_classes_batch([test_dataset.batch_splits[i] for i in range(0, split_i+1)])\n",
        "\n",
        "        # Define train and validation indices\n",
        "        train_indices, val_indices = train_dataset.train_val_split(VAL_SIZE, RANDOM_STATES[run_i])\n",
        "\n",
        "        # Define subsets\n",
        "        train_subsets[run_i].append(Subset(train_dataset, train_indices))\n",
        "        val_subsets[run_i].append(Subset(train_dataset, val_indices))\n",
        "        test_subsets[run_i].append(test_dataset)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz to data/cifar-100-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a6f30b36709f434886701d6b60e5709f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/cifar-100-python.tar.gz to data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cmCpRMBKgvDB",
        "colab_type": "text"
      },
      "source": [
        "### Execution"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-nRU--zYjmZT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# iCaRL hyperparameters\n",
        "LR = 2\n",
        "MOMENTUM = 0.9\n",
        "WEIGHT_DECAY = 0.00001\n",
        "MILESTONES = [49, 63]\n",
        "GAMMA = 0.2\n",
        "NUM_EPOCHS = 70\n",
        "BATCH_SIZE = 64"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1mcTQUN7VLPF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "bfd19ed7-3fa6-4a01-e597-953a7659dcfe"
      },
      "source": [
        "# Define what tests to run\n",
        "TEST_ICARL = True # Run test with iCaRL (exemplars + train dataset)\n",
        "TEST_HYBRID1 = False # Run test with hybrid1\n",
        "\n",
        "# Initialize logs\n",
        "logs_icarl = [[] for _ in range(NUM_RUNS)]\n",
        "logs_hybrid1 = [[] for _ in range(NUM_RUNS)]\n",
        "\n",
        "for run_i in range(NUM_RUNS-2):\n",
        "    net = resnet32()\n",
        "    criterion = AsymLoss(nn.BCEWithLogitsLoss)\n",
        "    icarl = iCaRLwithLoss(DEVICE, net, LR, criterion, MOMENTUM, WEIGHT_DECAY, MILESTONES, GAMMA, NUM_EPOCHS, BATCH_SIZE, train_transform, test_transform)\n",
        "\n",
        "    for split_i in range(1):\n",
        "        print(f\"## Split {split_i} of run {run_i} ##\")\n",
        "        \n",
        "        train_logs = icarl.incremental_train(split_i, train_subsets[run_i][split_i], val_subsets[run_i][split_i])\n",
        "\n",
        "        if TEST_ICARL:\n",
        "            logs_icarl[run_i].append({})\n",
        "\n",
        "            acc, all_targets, all_preds = icarl.test(test_subsets[run_i][split_i], train_subsets[run_i][split_i])\n",
        "\n",
        "            logs_icarl[run_i][split_i]['accuracy'] = acc\n",
        "            logs_icarl[run_i][split_i]['conf_mat'] = confusion_matrix(all_targets.to('cpu'), all_preds.to('cpu'))\n",
        "\n",
        "            logs_icarl[run_i][split_i]['train_loss'] = train_logs[0]\n",
        "            logs_icarl[run_i][split_i]['train_accuracy'] = train_logs[1]\n",
        "            logs_icarl[run_i][split_i]['val_loss'] = train_logs[2]\n",
        "            logs_icarl[run_i][split_i]['val_accuracy'] = train_logs[3]\n",
        "\n",
        "        if TEST_HYBRID1:\n",
        "            logs_hybrid1[run_i].append({})\n",
        "\n",
        "            acc, all_targets, all_preds = icarl.test_without_classifier(test_subsets[run_i][split_i])\n",
        "\n",
        "            logs_hybrid1[run_i][split_i]['accuracy'] = acc\n",
        "            logs_hybrid1[run_i][split_i]['conf_mat'] = confusion_matrix(all_targets.to('cpu'), all_preds.to('cpu'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "## Split 0 of run 0 ##\n",
            "Length of exemplars set: 0\n",
            "Epoch: 1, LR: [2]\n",
            "Train loss: 0.20704161716359, Train accuracy: 0.14866071428571428\n",
            "Validation loss: 0.17530382105282374, Validation accuracy: 0.25223214285714285\n",
            "Epoch: 2, LR: [2]\n",
            "Train loss: 0.18046391308307647, Train accuracy: 0.22254464285714284\n",
            "Validation loss: 0.20530323897089278, Validation accuracy: 0.16964285714285715\n",
            "Epoch: 3, LR: [2]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wuDM6Ydokv05",
        "colab_type": "text"
      },
      "source": [
        "### Plots"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nEn9P6usbwYa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_loss = [[logs_icarl[run_i][i]['train_loss'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "train_accuracy = [[logs_icarl[run_i][i]['train_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "val_loss = [[logs_icarl[run_i][i]['val_loss'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "val_accuracy = [[logs_icarl[run_i][i]['val_accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "test_accuracy = [[logs_icarl[run_i][i]['accuracy'] for i in range(10)] for run_i in range(NUM_RUNS)]\n",
        "\n",
        "train_loss = np.array(train_loss)\n",
        "train_accuracy = np.array(train_accuracy)\n",
        "val_loss = np.array(val_loss)\n",
        "val_accuracy = np.array(val_accuracy)\n",
        "test_accuracy = np.array(test_accuracy)\n",
        "\n",
        "train_loss_stats = np.array([train_loss.mean(0), train_loss.std(0)]).transpose()\n",
        "train_accuracy_stats = np.array([train_accuracy.mean(0), train_accuracy.std(0)]).transpose()\n",
        "val_loss_stats = np.array([val_loss.mean(0), val_loss.std(0)]).transpose()\n",
        "val_accuracy_stats = np.array([val_accuracy.mean(0), val_accuracy.std(0)]).transpose()\n",
        "test_accuracy_stats = np.array([test_accuracy.mean(0), test_accuracy.std(0)]).transpose()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-GmiU4BfhVht",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plot.train_val_scores(train_loss_stats, train_accuracy_stats, val_loss_stats, val_accuracy_stats)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PAWPfif3j7pK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "plot.test_scores(test_accuracy_stats)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}