CIFAR_FS_maml.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CIFAR_FS-maml.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyNIzznw9fIcovyBgVTtJ/0E",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "daa77f4a821a45979c329f441b97b602": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8df01f0a56914c3d9ca60b23767431f7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_73381b1a57df4a82acac22e97a930cda",
              "IPY_MODEL_2165431a869941a9aa7201600fc6776a"
            ]
          }
        },
        "8df01f0a56914c3d9ca60b23767431f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "73381b1a57df4a82acac22e97a930cda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4cfbd0e5a91b4d8eb0405d4e0fb4b702",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10000,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10000,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_27e69f6202ab4d92bae67167881834df"
          }
        },
        "2165431a869941a9aa7201600fc6776a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f66ae91797ad48f398e0640dde1aad04",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10000/10000 [4:48:22&lt;00:00,  1.73s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7e98a57eba624a8995d859f2d3277a88"
          }
        },
        "4cfbd0e5a91b4d8eb0405d4e0fb4b702": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "27e69f6202ab4d92bae67167881834df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f66ae91797ad48f398e0640dde1aad04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7e98a57eba624a8995d859f2d3277a88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JoshStrong/MAML/blob/master/CIFAR_FS_maml.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3PuXRfv9yTBU"
      },
      "source": [
        "!pip install learn2learn"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZI5DLOMUyL91"
      },
      "source": [
        "import random\n",
        "import numpy as np\n",
        "\n",
        "import torch\n",
        "from torch import nn, optim\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from scipy.stats import norm\n",
        "import random\n",
        "\n",
        "import copy\n",
        "\n",
        "import learn2learn as l2l\n",
        "from learn2learn.data.transforms import (NWays,\n",
        "                                         KShots,\n",
        "                                         LoadData,\n",
        "                                         RemapLabels,\n",
        "                                         ConsecutiveLabels)\n",
        "from tqdm.notebook import tqdm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JohPOSPLBEwa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "48b12751-7cdf-4c5d-981b-0a1728c422da"
      },
      "source": [
        "# GPU assigned by google colab for training\n",
        "torch.cuda.get_device_name(0)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Tesla T4'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I8OsFsaiYnPr"
      },
      "source": [
        "## Acknowledgements\n",
        "\n",
        "This code makes heavy use of the meta-learning software library learn2learn [1], which makes implementing meta-learning algorithms, such as MAML, much easier. Without learn2learn, implementing MAML requires \"hacking\" away with PyTorch functionality, which I decided to avoid for the 2nd experimental study to ensure bugless implementation of MAML when applying MAML to the more complex problem of a computer vision related task.\n",
        "\n",
        "\n",
        "\n",
        "[1] Sebastien M.R. Arnold, Praateek Mahajan, Debajyoti Datta, Ian Bunner. \"learn2learn\". https://github.com/learnables/learn2learn, 2019"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tV8evpO3XEHJ"
      },
      "source": [
        "## Construct model to be trained, define helper functions "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KAn1Nu76bNBj"
      },
      "source": [
        "?l2l.vision.models.ConvBase"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vQzRS5-8yi3X"
      },
      "source": [
        "# CNN model to be used in training\n",
        "# Note how we seperate the head and the body of the network for potential ANIL training\n",
        "\n",
        "class CifarCNN(torch.nn.Module):\n",
        "    def __init__(self, output_size=5, hidden_size=32, layers=4):\n",
        "        super(CifarCNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "\n",
        "        # Instantiates 4-layer CNN using l2l library\n",
        "        # 3 rgb channels with 32 hidden size\n",
        "        # 4 layers\n",
        "        features = l2l.vision.models.ConvBase(\n",
        "            output_size=hidden_size,\n",
        "            hidden=hidden_size,\n",
        "            channels=3,\n",
        "            max_pool=False,\n",
        "            layers=layers,\n",
        "            max_pool_factor=0.5,\n",
        "        )\n",
        "        # Here we call Sequential on previous 4-layer CNN...\n",
        "        self.features = torch.nn.Sequential(\n",
        "            features,\n",
        "            l2l.nn.Lambda(lambda x: x.mean(dim=[2, 3])),\n",
        "            l2l.nn.Flatten(),\n",
        "        )\n",
        "        #... and connect with the 'head' of the model; a fully-connected layer\n",
        "        self.linear = torch.nn.Linear(self.hidden_size, output_size, bias=True)\n",
        "        l2l.vision.models.maml_init_(self.linear)\n",
        "\n",
        "    # Performs forward-pass of the network\n",
        "    def forward(self, x):\n",
        "        x = self.features(x)\n",
        "        x = self.linear(x)\n",
        "        return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nUFVcaTpaSYP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        },
        "outputId": "c1ce0b71-2e68-4385-ef3b-4ee7c4bfb9eb"
      },
      "source": [
        "model = CifarCNN()\n",
        "model"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CifarCNN(\n",
              "  (features): Sequential(\n",
              "    (0): ConvBase(\n",
              "      (0): ConvBlock(\n",
              "        (normalize): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU()\n",
              "        (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "      )\n",
              "      (1): ConvBlock(\n",
              "        (normalize): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU()\n",
              "        (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "      )\n",
              "      (2): ConvBlock(\n",
              "        (normalize): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU()\n",
              "        (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "      )\n",
              "      (3): ConvBlock(\n",
              "        (normalize): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU()\n",
              "        (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "      )\n",
              "    )\n",
              "    (1): Lambda()\n",
              "    (2): Flatten()\n",
              "  )\n",
              "  (linear): Linear(in_features=32, out_features=5, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bdNoIzWdEP2o"
      },
      "source": [
        "# Accuracy function to be used in evaluating models performance\n",
        "\n",
        "def accuracy(predictions, targets):\n",
        "    predictions = predictions.argmax(dim=1).view(targets.shape)\n",
        "    return (predictions == targets).sum().float() / targets.size(0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "q683FXUOXLpK"
      },
      "source": [
        "## Define fast_adapt function for inner loop training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UdgtOYvRyPQt"
      },
      "source": [
        "# inner loop for vision tasks\n",
        "\n",
        "def fast_adapt(batch, learner, loss, adaptation_steps, shots, ways, device):\n",
        "    \"\"\"\n",
        "    fast_adapt: computes inner loop of MAML for computer vision tasks\n",
        "\n",
        "    PARAMETERS:\n",
        "    1. batch = batch of data to train inner loop on \n",
        "    2. learner = model to perform inner loop on\n",
        "    3. loss = loss function to quantify performance\n",
        "    4. adaptation_steps = number of inner loop gradient steps to be performed\n",
        "    5. shots = number of data per class\n",
        "    6. ways = number of classes\n",
        "    7. device = gpu/cpu (default gpu)\n",
        "    \"\"\"\n",
        "    # seperate data & labels from inputted batch of data\n",
        "    data, labels = batch\n",
        "    data, labels = data.to(device), labels.to(device)\n",
        "\n",
        "    # Separate data into adaptation/evalutation sets with a 50:50 split\n",
        "    # adaptation data = D^{tr}, evaluation data = D^{test}\n",
        "    adaptation_indices = np.zeros(data.size(0), dtype=bool) # 50 Falses\n",
        "    adaptation_indices[np.arange(shots*ways) * 2] = True # Set every even to True\n",
        "    evaluation_indices = torch.from_numpy(~adaptation_indices) # Opposite of adaptation_indices, make array\n",
        "    adaptation_indices = torch.from_numpy(adaptation_indices) # Tensor -> array\n",
        "    adaptation_data, adaptation_labels = data[adaptation_indices], labels[adaptation_indices]\n",
        "    evaluation_data, evaluation_labels = data[evaluation_indices], labels[evaluation_indices]\n",
        "\n",
        "    # Perform gradient update of task-specific model copy\n",
        "    for step in range(adaptation_steps):\n",
        "        prediction = learner(adaptation_data)\n",
        "        # Compute loss over all data\n",
        "        adaptation_error = loss(prediction, adaptation_labels)\n",
        "        # Average the loss\n",
        "        adaptation_error /= len(adaptation_data)\n",
        "        # learner.adapt adjusts model parameters\n",
        "        learner.adapt(adaptation_error)\n",
        "\n",
        "    # Comput loss of adapted model to be used for outer-loop update\n",
        "    # similar to above, except we do not learner.adapt, instead\n",
        "    # accumulate gradients and perform a meta-update outside of this \n",
        "    # function\n",
        "    predictions = learner(evaluation_data)\n",
        "    evaluation_error = loss(predictions, evaluation_labels)\n",
        "    evaluation_error /= len(evaluation_data)\n",
        "    evaluation_accuracy = accuracy(predictions, evaluation_labels)\n",
        "    return evaluation_error, evaluation_accuracy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W7mxRvcFXSYq"
      },
      "source": [
        "## Use learn2learn library for easy access to CIFAR-FS dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tkgv1wLCBrnB"
      },
      "source": [
        "# Use l2l framework to get CIFAR-FS data\n",
        "# in a form useful for few-shot meta-learning\n",
        "# use 2*shots in generating training samples\n",
        "# since we need D^{tr} and D^{test}\n",
        "\n",
        "shots=5\n",
        "ways=5\n",
        "tasksets = l2l.vision.benchmarks.get_tasksets(\n",
        "        name='cifarfs',\n",
        "        train_samples=2*shots,\n",
        "        train_ways=ways,\n",
        "        test_samples=2*shots,\n",
        "        test_ways=ways,\n",
        "        root='~/data',\n",
        "    )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "maeAn_yaXds7"
      },
      "source": [
        "## Define CIFARFS_MAML function\n",
        "\n",
        "CIFARS_MAML implements the main portion of the MAML algorithm for computer-vision related tasks."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wQaARMMfye8_"
      },
      "source": [
        "def CIFARFS_MAML(ways=5, shots=5, meta_lr=0.003, inner_lr=0.5, meta_batch_size=32, adaptation_steps=1, num_iterations=10000):\n",
        "    # Instantiate cifarCNN model, send to gpu\n",
        "    model = CifarCNN()\n",
        "    model.to('cuda')\n",
        "    maml = l2l.algorithms.MAML(model, lr=inner_lr, first_order=False)\n",
        "\n",
        "    # Use Adam optimiser and cross entropy loss\n",
        "    opt = optim.Adam(maml.parameters(), meta_lr)\n",
        "    loss = nn.CrossEntropyLoss(reduction='mean')\n",
        "\n",
        "    # Meta-learning\n",
        "    # Outer-loop\n",
        "    # num_iterations = number of outer-loop updates\n",
        "    for iteration in tqdm(range(num_iterations)):\n",
        "        opt.zero_grad()\n",
        "        meta_valid_error = 0.0\n",
        "        meta_valid_accuracy = 0.0\n",
        "\n",
        "        # For each sampled task T_i \n",
        "        for task in range(meta_batch_size):\n",
        "            # Clone the meta-parameters to perform task-specific adaptation\n",
        "            learner = maml.clone()\n",
        "            batch = tasksets.train.sample()\n",
        "            # Perform the inner loop with this batch of data from sampled task\n",
        "            evaluation_error, evaluation_accuracy = fast_adapt(batch,\n",
        "                                                               learner,\n",
        "                                                               loss,\n",
        "                                                               adaptation_steps,\n",
        "                                                               shots,\n",
        "                                                               ways,\n",
        "                                                               'cuda')\n",
        "            # Do a backwards pass to compute & accumulate gradients\n",
        "            evaluation_error.backward()\n",
        "\n",
        "            # Compute meta-validation loss\n",
        "            learner = maml.clone()\n",
        "            batch = tasksets.validation.sample()\n",
        "            evaluation_error, evaluation_accuracy = fast_adapt(batch,\n",
        "                                                               learner,\n",
        "                                                               loss,\n",
        "                                                               adaptation_steps,\n",
        "                                                               shots,\n",
        "                                                               ways,\n",
        "                                                               'cuda')\n",
        "            meta_valid_error += evaluation_error.item()\n",
        "            meta_valid_accuracy += evaluation_accuracy.item()\n",
        "\n",
        "        avg_meta_valid_err = meta_valid_error/meta_batch_size\n",
        "        avg_meta_valid_acc = meta_valid_accuracy / meta_batch_size\n",
        "        if iteration % 50 == 0:\n",
        "          print('\\n')\n",
        "          print('Iteration', iteration)\n",
        "          print('Average Meta Valid Error', avg_meta_valid_err)\n",
        "          print('Average Meta Valid Accuracy', avg_meta_valid_acc)\n",
        "\n",
        "        # Average the accumulated gradients from all tasks and optimize\n",
        "        for p in maml.parameters():\n",
        "            p.grad.data.mul_(1.0 / meta_batch_size)\n",
        "        opt.step()\n",
        "\n",
        "    return model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KxbaLZwOy7G-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "daa77f4a821a45979c329f441b97b602",
            "8df01f0a56914c3d9ca60b23767431f7",
            "73381b1a57df4a82acac22e97a930cda",
            "2165431a869941a9aa7201600fc6776a",
            "4cfbd0e5a91b4d8eb0405d4e0fb4b702",
            "27e69f6202ab4d92bae67167881834df",
            "f66ae91797ad48f398e0640dde1aad04",
            "7e98a57eba624a8995d859f2d3277a88"
          ]
        },
        "outputId": "2da59568-adc6-4b5d-f4db-1568834d1358"
      },
      "source": [
        "model = CIFARFS_MAML()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "daa77f4a821a45979c329f441b97b602",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10000.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py:385: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
            "  if param.grad is not None:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "Iteration 0\n",
            "Average Meta Valid Error 0.06647501117549837\n",
            "Average Meta Valid Accuracy 0.19999998807907104\n",
            "\n",
            "\n",
            "Iteration 50\n",
            "Average Meta Valid Error 0.06198663148097694\n",
            "Average Meta Valid Accuracy 0.2687499930616468\n",
            "\n",
            "\n",
            "Iteration 100\n",
            "Average Meta Valid Error 0.06170825252775103\n",
            "Average Meta Valid Accuracy 0.28749999264255166\n",
            "\n",
            "\n",
            "Iteration 150\n",
            "Average Meta Valid Error 0.06118645064998418\n",
            "Average Meta Valid Accuracy 0.29874999169260263\n",
            "\n",
            "\n",
            "Iteration 200\n",
            "Average Meta Valid Error 0.05942488776054233\n",
            "Average Meta Valid Accuracy 0.34874999057501554\n",
            "\n",
            "\n",
            "Iteration 250\n",
            "Average Meta Valid Error 0.06233933346811682\n",
            "Average Meta Valid Accuracy 0.2974999900907278\n",
            "\n",
            "\n",
            "Iteration 300\n",
            "Average Meta Valid Error 0.06011559732723981\n",
            "Average Meta Valid Accuracy 0.3387499905657023\n",
            "\n",
            "\n",
            "Iteration 350\n",
            "Average Meta Valid Error 0.05991319171153009\n",
            "Average Meta Valid Accuracy 0.347499989438802\n",
            "\n",
            "\n",
            "Iteration 400\n",
            "Average Meta Valid Error 0.06033550656866282\n",
            "Average Meta Valid Accuracy 0.3399999898392707\n",
            "\n",
            "\n",
            "Iteration 450\n",
            "Average Meta Valid Error 0.060527289868332446\n",
            "Average Meta Valid Accuracy 0.3349999899510294\n",
            "\n",
            "\n",
            "Iteration 500\n",
            "Average Meta Valid Error 0.057721563847735524\n",
            "Average Meta Valid Accuracy 0.3762499901931733\n",
            "\n",
            "\n",
            "Iteration 550\n",
            "Average Meta Valid Error 0.05696353001985699\n",
            "Average Meta Valid Accuracy 0.38624998996965587\n",
            "\n",
            "\n",
            "Iteration 600\n",
            "Average Meta Valid Error 0.05504269676748663\n",
            "Average Meta Valid Accuracy 0.40999999083578587\n",
            "\n",
            "\n",
            "Iteration 650\n",
            "Average Meta Valid Error 0.05500340519938618\n",
            "Average Meta Valid Accuracy 0.41374998912215233\n",
            "\n",
            "\n",
            "Iteration 700\n",
            "Average Meta Valid Error 0.0564552036812529\n",
            "Average Meta Valid Accuracy 0.402499983785674\n",
            "\n",
            "\n",
            "Iteration 750\n",
            "Average Meta Valid Error 0.055750458617694676\n",
            "Average Meta Valid Accuracy 0.4362499909475446\n",
            "\n",
            "\n",
            "Iteration 800\n",
            "Average Meta Valid Error 0.05594336416106671\n",
            "Average Meta Valid Accuracy 0.39874998945742846\n",
            "\n",
            "\n",
            "Iteration 850\n",
            "Average Meta Valid Error 0.05372615728992969\n",
            "Average Meta Valid Accuracy 0.4337499891407788\n",
            "\n",
            "\n",
            "Iteration 900\n",
            "Average Meta Valid Error 0.054593393579125404\n",
            "Average Meta Valid Accuracy 0.4087499864399433\n",
            "\n",
            "\n",
            "Iteration 950\n",
            "Average Meta Valid Error 0.05523716693278402\n",
            "Average Meta Valid Accuracy 0.41624998953193426\n",
            "\n",
            "\n",
            "Iteration 1000\n",
            "Average Meta Valid Error 0.055508247460238636\n",
            "Average Meta Valid Accuracy 0.4087499906308949\n",
            "\n",
            "\n",
            "Iteration 1050\n",
            "Average Meta Valid Error 0.0543765613110736\n",
            "Average Meta Valid Accuracy 0.44124998757615685\n",
            "\n",
            "\n",
            "Iteration 1100\n",
            "Average Meta Valid Error 0.0562188004842028\n",
            "Average Meta Valid Accuracy 0.4162499909289181\n",
            "\n",
            "\n",
            "Iteration 1150\n",
            "Average Meta Valid Error 0.052977125451434404\n",
            "Average Meta Valid Accuracy 0.46374998707324266\n",
            "\n",
            "\n",
            "Iteration 1200\n",
            "Average Meta Valid Error 0.052233904018066823\n",
            "Average Meta Valid Accuracy 0.45749998837709427\n",
            "\n",
            "\n",
            "Iteration 1250\n",
            "Average Meta Valid Error 0.054343461990356445\n",
            "Average Meta Valid Accuracy 0.446249988861382\n",
            "\n",
            "\n",
            "Iteration 1300\n",
            "Average Meta Valid Error 0.05271352187264711\n",
            "Average Meta Valid Accuracy 0.4624999859370291\n",
            "\n",
            "\n",
            "Iteration 1350\n",
            "Average Meta Valid Error 0.04991939361207187\n",
            "Average Meta Valid Accuracy 0.4549999888986349\n",
            "\n",
            "\n",
            "Iteration 1400\n",
            "Average Meta Valid Error 0.053666044608689845\n",
            "Average Meta Valid Accuracy 0.4499999866820872\n",
            "\n",
            "\n",
            "Iteration 1450\n",
            "Average Meta Valid Error 0.0510841712821275\n",
            "Average Meta Valid Accuracy 0.47999998996965587\n",
            "\n",
            "\n",
            "Iteration 1500\n",
            "Average Meta Valid Error 0.051918490091338754\n",
            "Average Meta Valid Accuracy 0.4899999899789691\n",
            "\n",
            "\n",
            "Iteration 1550\n",
            "Average Meta Valid Error 0.05037115572486073\n",
            "Average Meta Valid Accuracy 0.48124998435378075\n",
            "\n",
            "\n",
            "Iteration 1600\n",
            "Average Meta Valid Error 0.04858390078879893\n",
            "Average Meta Valid Accuracy 0.5049999873153865\n",
            "\n",
            "\n",
            "Iteration 1650\n",
            "Average Meta Valid Error 0.05095182219520211\n",
            "Average Meta Valid Accuracy 0.4837499875575304\n",
            "\n",
            "\n",
            "Iteration 1700\n",
            "Average Meta Valid Error 0.05014418298378587\n",
            "Average Meta Valid Accuracy 0.5037499843165278\n",
            "\n",
            "\n",
            "Iteration 1750\n",
            "Average Meta Valid Error 0.052173397154547274\n",
            "Average Meta Valid Accuracy 0.4474999872036278\n",
            "\n",
            "\n",
            "Iteration 1800\n",
            "Average Meta Valid Error 0.050777173368260264\n",
            "Average Meta Valid Accuracy 0.5024999883025885\n",
            "\n",
            "\n",
            "Iteration 1850\n",
            "Average Meta Valid Error 0.05012382159475237\n",
            "Average Meta Valid Accuracy 0.4837499875575304\n",
            "\n",
            "\n",
            "Iteration 1900\n",
            "Average Meta Valid Error 0.04961543437093496\n",
            "Average Meta Valid Accuracy 0.4999999888241291\n",
            "\n",
            "\n",
            "Iteration 1950\n",
            "Average Meta Valid Error 0.049761858652345836\n",
            "Average Meta Valid Accuracy 0.4912499883212149\n",
            "\n",
            "\n",
            "Iteration 2000\n",
            "Average Meta Valid Error 0.049693255918100476\n",
            "Average Meta Valid Accuracy 0.48499998869374394\n",
            "\n",
            "\n",
            "Iteration 2050\n",
            "Average Meta Valid Error 0.050856865011155605\n",
            "Average Meta Valid Accuracy 0.4899999890476465\n",
            "\n",
            "\n",
            "Iteration 2100\n",
            "Average Meta Valid Error 0.05047817761078477\n",
            "Average Meta Valid Accuracy 0.46874998742714524\n",
            "\n",
            "\n",
            "Iteration 2150\n",
            "Average Meta Valid Error 0.04617562278872356\n",
            "Average Meta Valid Accuracy 0.5374999898485839\n",
            "\n",
            "\n",
            "Iteration 2200\n",
            "Average Meta Valid Error 0.04844916402362287\n",
            "Average Meta Valid Accuracy 0.49499998707324266\n",
            "\n",
            "\n",
            "Iteration 2250\n",
            "Average Meta Valid Error 0.048753086128272116\n",
            "Average Meta Valid Accuracy 0.48249998688697815\n",
            "\n",
            "\n",
            "Iteration 2300\n",
            "Average Meta Valid Error 0.047720147063955665\n",
            "Average Meta Valid Accuracy 0.5074999807402492\n",
            "\n",
            "\n",
            "Iteration 2350\n",
            "Average Meta Valid Error 0.04951968643581495\n",
            "Average Meta Valid Accuracy 0.5112499864771962\n",
            "\n",
            "\n",
            "Iteration 2400\n",
            "Average Meta Valid Error 0.04911745467688888\n",
            "Average Meta Valid Accuracy 0.49999998649582267\n",
            "\n",
            "\n",
            "Iteration 2450\n",
            "Average Meta Valid Error 0.04852453782223165\n",
            "Average Meta Valid Accuracy 0.5062499879859388\n",
            "\n",
            "\n",
            "Iteration 2500\n",
            "Average Meta Valid Error 0.046238255221396685\n",
            "Average Meta Valid Accuracy 0.5499999858438969\n",
            "\n",
            "\n",
            "Iteration 2550\n",
            "Average Meta Valid Error 0.04825756908394396\n",
            "Average Meta Valid Accuracy 0.5049999887123704\n",
            "\n",
            "\n",
            "Iteration 2600\n",
            "Average Meta Valid Error 0.0488512022420764\n",
            "Average Meta Valid Accuracy 0.48999998392537236\n",
            "\n",
            "\n",
            "Iteration 2650\n",
            "Average Meta Valid Error 0.04953045572619885\n",
            "Average Meta Valid Accuracy 0.4987499890848994\n",
            "\n",
            "\n",
            "Iteration 2700\n",
            "Average Meta Valid Error 0.0440544193261303\n",
            "Average Meta Valid Accuracy 0.5649999883025885\n",
            "\n",
            "\n",
            "Iteration 2750\n",
            "Average Meta Valid Error 0.047982640273403376\n",
            "Average Meta Valid Accuracy 0.5074999881908298\n",
            "\n",
            "\n",
            "Iteration 2800\n",
            "Average Meta Valid Error 0.046414468553848565\n",
            "Average Meta Valid Accuracy 0.5362499868497252\n",
            "\n",
            "\n",
            "Iteration 2850\n",
            "Average Meta Valid Error 0.04644558485597372\n",
            "Average Meta Valid Accuracy 0.5137499854899943\n",
            "\n",
            "\n",
            "Iteration 2900\n",
            "Average Meta Valid Error 0.050913001934532076\n",
            "Average Meta Valid Accuracy 0.4999999860301614\n",
            "\n",
            "\n",
            "Iteration 2950\n",
            "Average Meta Valid Error 0.04625596373807639\n",
            "Average Meta Valid Accuracy 0.5424999855458736\n",
            "\n",
            "\n",
            "Iteration 3000\n",
            "Average Meta Valid Error 0.048226295970380306\n",
            "Average Meta Valid Accuracy 0.5212499862536788\n",
            "\n",
            "\n",
            "Iteration 3050\n",
            "Average Meta Valid Error 0.04873975377995521\n",
            "Average Meta Valid Accuracy 0.5124999852851033\n",
            "\n",
            "\n",
            "Iteration 3100\n",
            "Average Meta Valid Error 0.047108076920267195\n",
            "Average Meta Valid Accuracy 0.5512499874457717\n",
            "\n",
            "\n",
            "Iteration 3150\n",
            "Average Meta Valid Error 0.046155751042533666\n",
            "Average Meta Valid Accuracy 0.5337499841116369\n",
            "\n",
            "\n",
            "Iteration 3200\n",
            "Average Meta Valid Error 0.04377187497448176\n",
            "Average Meta Valid Accuracy 0.5749999843537807\n",
            "\n",
            "\n",
            "Iteration 3250\n",
            "Average Meta Valid Error 0.04678020242135972\n",
            "Average Meta Valid Accuracy 0.5037499850150198\n",
            "\n",
            "\n",
            "Iteration 3300\n",
            "Average Meta Valid Error 0.046751354122534394\n",
            "Average Meta Valid Accuracy 0.5112499836832285\n",
            "\n",
            "\n",
            "Iteration 3350\n",
            "Average Meta Valid Error 0.047268144437111914\n",
            "Average Meta Valid Accuracy 0.5149999842979014\n",
            "\n",
            "\n",
            "Iteration 3400\n",
            "Average Meta Valid Error 0.04521765391109511\n",
            "Average Meta Valid Accuracy 0.5412499853409827\n",
            "\n",
            "\n",
            "Iteration 3450\n",
            "Average Meta Valid Error 0.049409833271056414\n",
            "Average Meta Valid Accuracy 0.4737499877810478\n",
            "\n",
            "\n",
            "Iteration 3500\n",
            "Average Meta Valid Error 0.04882758780149743\n",
            "Average Meta Valid Accuracy 0.4949999861419201\n",
            "\n",
            "\n",
            "Iteration 3550\n",
            "Average Meta Valid Error 0.044782094191759825\n",
            "Average Meta Valid Accuracy 0.548749984242022\n",
            "\n",
            "\n",
            "Iteration 3600\n",
            "Average Meta Valid Error 0.04516350623453036\n",
            "Average Meta Valid Accuracy 0.5524999853223562\n",
            "\n",
            "\n",
            "Iteration 3650\n",
            "Average Meta Valid Error 0.04775387130212039\n",
            "Average Meta Valid Accuracy 0.49249998992308974\n",
            "\n",
            "\n",
            "Iteration 3700\n",
            "Average Meta Valid Error 0.04661850607953966\n",
            "Average Meta Valid Accuracy 0.512499988079071\n",
            "\n",
            "\n",
            "Iteration 3750\n",
            "Average Meta Valid Error 0.04519434424582869\n",
            "Average Meta Valid Accuracy 0.5462499912828207\n",
            "\n",
            "\n",
            "Iteration 3800\n",
            "Average Meta Valid Error 0.04682058945763856\n",
            "Average Meta Valid Accuracy 0.541249985806644\n",
            "\n",
            "\n",
            "Iteration 3850\n",
            "Average Meta Valid Error 0.04886965820332989\n",
            "Average Meta Valid Accuracy 0.5024999887682498\n",
            "\n",
            "\n",
            "Iteration 3900\n",
            "Average Meta Valid Error 0.04694171220762655\n",
            "Average Meta Valid Accuracy 0.5287499893456697\n",
            "\n",
            "\n",
            "Iteration 3950\n",
            "Average Meta Valid Error 0.04702266398817301\n",
            "Average Meta Valid Accuracy 0.5099999867379665\n",
            "\n",
            "\n",
            "Iteration 4000\n",
            "Average Meta Valid Error 0.04607185069471598\n",
            "Average Meta Valid Accuracy 0.5299999853596091\n",
            "\n",
            "\n",
            "Iteration 4050\n",
            "Average Meta Valid Error 0.043523570406250656\n",
            "Average Meta Valid Accuracy 0.579999984242022\n",
            "\n",
            "\n",
            "Iteration 4100\n",
            "Average Meta Valid Error 0.0490572223206982\n",
            "Average Meta Valid Accuracy 0.526249987538904\n",
            "\n",
            "\n",
            "Iteration 4150\n",
            "Average Meta Valid Error 0.04663680342491716\n",
            "Average Meta Valid Accuracy 0.519999991171062\n",
            "\n",
            "\n",
            "Iteration 4200\n",
            "Average Meta Valid Error 0.045942321652546525\n",
            "Average Meta Valid Accuracy 0.5424999902024865\n",
            "\n",
            "\n",
            "Iteration 4250\n",
            "Average Meta Valid Error 0.047091758460737765\n",
            "Average Meta Valid Accuracy 0.5212499871850014\n",
            "\n",
            "\n",
            "Iteration 4300\n",
            "Average Meta Valid Error 0.043100156530272216\n",
            "Average Meta Valid Accuracy 0.5687499903142452\n",
            "\n",
            "\n",
            "Iteration 4350\n",
            "Average Meta Valid Error 0.043400254915468395\n",
            "Average Meta Valid Accuracy 0.5674999859184027\n",
            "\n",
            "\n",
            "Iteration 4400\n",
            "Average Meta Valid Error 0.047103181947022676\n",
            "Average Meta Valid Accuracy 0.5374999865889549\n",
            "\n",
            "\n",
            "Iteration 4450\n",
            "Average Meta Valid Error 0.04528987826779485\n",
            "Average Meta Valid Accuracy 0.5337499864399433\n",
            "\n",
            "\n",
            "Iteration 4500\n",
            "Average Meta Valid Error 0.04429668973898515\n",
            "Average Meta Valid Accuracy 0.5612499872222543\n",
            "\n",
            "\n",
            "Iteration 4550\n",
            "Average Meta Valid Error 0.0436829422833398\n",
            "Average Meta Valid Accuracy 0.5637499876320362\n",
            "\n",
            "\n",
            "Iteration 4600\n",
            "Average Meta Valid Error 0.04520797618897632\n",
            "Average Meta Valid Accuracy 0.5449999878183007\n",
            "\n",
            "\n",
            "Iteration 4650\n",
            "Average Meta Valid Error 0.0461238072020933\n",
            "Average Meta Valid Accuracy 0.5574999852105975\n",
            "\n",
            "\n",
            "Iteration 4700\n",
            "Average Meta Valid Error 0.04470253619365394\n",
            "Average Meta Valid Accuracy 0.5562499864026904\n",
            "\n",
            "\n",
            "Iteration 4750\n",
            "Average Meta Valid Error 0.04433562414487824\n",
            "Average Meta Valid Accuracy 0.5587499849498272\n",
            "\n",
            "\n",
            "Iteration 4800\n",
            "Average Meta Valid Error 0.044496387999970466\n",
            "Average Meta Valid Accuracy 0.5687499903142452\n",
            "\n",
            "\n",
            "Iteration 4850\n",
            "Average Meta Valid Error 0.0436825206852518\n",
            "Average Meta Valid Accuracy 0.5612499853596091\n",
            "\n",
            "\n",
            "Iteration 4900\n",
            "Average Meta Valid Error 0.04589393734931946\n",
            "Average Meta Valid Accuracy 0.5299999872222543\n",
            "\n",
            "\n",
            "Iteration 4950\n",
            "Average Meta Valid Error 0.04295335948700085\n",
            "Average Meta Valid Accuracy 0.5787499835714698\n",
            "\n",
            "\n",
            "Iteration 5000\n",
            "Average Meta Valid Error 0.04724596504820511\n",
            "Average Meta Valid Accuracy 0.5262499842792749\n",
            "\n",
            "\n",
            "Iteration 5050\n",
            "Average Meta Valid Error 0.04267705004895106\n",
            "Average Meta Valid Accuracy 0.5837499871850014\n",
            "\n",
            "\n",
            "Iteration 5100\n",
            "Average Meta Valid Error 0.045995974040124565\n",
            "Average Meta Valid Accuracy 0.5387499872595072\n",
            "\n",
            "\n",
            "Iteration 5150\n",
            "Average Meta Valid Error 0.04581009951652959\n",
            "Average Meta Valid Accuracy 0.5624999850988388\n",
            "\n",
            "\n",
            "Iteration 5200\n",
            "Average Meta Valid Error 0.044058664061594754\n",
            "Average Meta Valid Accuracy 0.5587499816901982\n",
            "\n",
            "\n",
            "Iteration 5250\n",
            "Average Meta Valid Error 0.042576236359309405\n",
            "Average Meta Valid Accuracy 0.5724999867379665\n",
            "\n",
            "\n",
            "Iteration 5300\n",
            "Average Meta Valid Error 0.04393555235583335\n",
            "Average Meta Valid Accuracy 0.5612499872222543\n",
            "\n",
            "\n",
            "Iteration 5350\n",
            "Average Meta Valid Error 0.04527974803932011\n",
            "Average Meta Valid Accuracy 0.5449999840930104\n",
            "\n",
            "\n",
            "Iteration 5400\n",
            "Average Meta Valid Error 0.04361297737341374\n",
            "Average Meta Valid Accuracy 0.5599999856203794\n",
            "\n",
            "\n",
            "Iteration 5450\n",
            "Average Meta Valid Error 0.04343045147834346\n",
            "Average Meta Valid Accuracy 0.564999982714653\n",
            "\n",
            "\n",
            "Iteration 5500\n",
            "Average Meta Valid Error 0.04528125049546361\n",
            "Average Meta Valid Accuracy 0.5287499846890569\n",
            "\n",
            "\n",
            "Iteration 5550\n",
            "Average Meta Valid Error 0.043080521922092885\n",
            "Average Meta Valid Accuracy 0.5724999881349504\n",
            "\n",
            "\n",
            "Iteration 5600\n",
            "Average Meta Valid Error 0.042979016609024256\n",
            "Average Meta Valid Accuracy 0.5937499850988388\n",
            "\n",
            "\n",
            "Iteration 5650\n",
            "Average Meta Valid Error 0.04500275861937553\n",
            "Average Meta Valid Accuracy 0.5499999886378646\n",
            "\n",
            "\n",
            "Iteration 5700\n",
            "Average Meta Valid Error 0.04482393671059981\n",
            "Average Meta Valid Accuracy 0.5537499859929085\n",
            "\n",
            "\n",
            "Iteration 5750\n",
            "Average Meta Valid Error 0.04287099320208654\n",
            "Average Meta Valid Accuracy 0.5637499885633588\n",
            "\n",
            "\n",
            "Iteration 5800\n",
            "Average Meta Valid Error 0.045908905449323356\n",
            "Average Meta Valid Accuracy 0.5312499888241291\n",
            "\n",
            "\n",
            "Iteration 5850\n",
            "Average Meta Valid Error 0.04351536475587636\n",
            "Average Meta Valid Accuracy 0.5649999845772982\n",
            "\n",
            "\n",
            "Iteration 5900\n",
            "Average Meta Valid Error 0.045708575926255435\n",
            "Average Meta Valid Accuracy 0.5424999855458736\n",
            "\n",
            "\n",
            "Iteration 5950\n",
            "Average Meta Valid Error 0.04466483503347263\n",
            "Average Meta Valid Accuracy 0.5474999928846955\n",
            "\n",
            "\n",
            "Iteration 6000\n",
            "Average Meta Valid Error 0.04543652426218614\n",
            "Average Meta Valid Accuracy 0.5624999878928065\n",
            "\n",
            "\n",
            "Iteration 6050\n",
            "Average Meta Valid Error 0.044802889227867126\n",
            "Average Meta Valid Accuracy 0.5674999849870801\n",
            "\n",
            "\n",
            "Iteration 6100\n",
            "Average Meta Valid Error 0.04267699929187074\n",
            "Average Meta Valid Accuracy 0.5462499856948853\n",
            "\n",
            "\n",
            "Iteration 6150\n",
            "Average Meta Valid Error 0.043846045969985425\n",
            "Average Meta Valid Accuracy 0.5462499866262078\n",
            "\n",
            "\n",
            "Iteration 6200\n",
            "Average Meta Valid Error 0.04253759467974305\n",
            "Average Meta Valid Accuracy 0.5687499837949872\n",
            "\n",
            "\n",
            "Iteration 6250\n",
            "Average Meta Valid Error 0.04700270766625181\n",
            "Average Meta Valid Accuracy 0.523749983869493\n",
            "\n",
            "\n",
            "Iteration 6300\n",
            "Average Meta Valid Error 0.03664265759289265\n",
            "Average Meta Valid Accuracy 0.6524999821558595\n",
            "\n",
            "\n",
            "Iteration 6350\n",
            "Average Meta Valid Error 0.04529892618302256\n",
            "Average Meta Valid Accuracy 0.5349999875761569\n",
            "\n",
            "\n",
            "Iteration 6400\n",
            "Average Meta Valid Error 0.04418653715401888\n",
            "Average Meta Valid Accuracy 0.5549999889917672\n",
            "\n",
            "\n",
            "Iteration 6450\n",
            "Average Meta Valid Error 0.041255534859374166\n",
            "Average Meta Valid Accuracy 0.5774999856948853\n",
            "\n",
            "\n",
            "Iteration 6500\n",
            "Average Meta Valid Error 0.04302105895476416\n",
            "Average Meta Valid Accuracy 0.559999986551702\n",
            "\n",
            "\n",
            "Iteration 6550\n",
            "Average Meta Valid Error 0.04330518614733592\n",
            "Average Meta Valid Accuracy 0.5524999815970659\n",
            "\n",
            "\n",
            "Iteration 6600\n",
            "Average Meta Valid Error 0.040165569982491434\n",
            "Average Meta Valid Accuracy 0.617499980609864\n",
            "\n",
            "\n",
            "Iteration 6650\n",
            "Average Meta Valid Error 0.043690180231351405\n",
            "Average Meta Valid Accuracy 0.567499989643693\n",
            "\n",
            "\n",
            "Iteration 6700\n",
            "Average Meta Valid Error 0.04236258793389425\n",
            "Average Meta Valid Accuracy 0.5837499867193401\n",
            "\n",
            "\n",
            "Iteration 6750\n",
            "Average Meta Valid Error 0.04502987471641973\n",
            "Average Meta Valid Accuracy 0.5787499849684536\n",
            "\n",
            "\n",
            "Iteration 6800\n",
            "Average Meta Valid Error 0.042358596343547106\n",
            "Average Meta Valid Accuracy 0.5837499871850014\n",
            "\n",
            "\n",
            "Iteration 6850\n",
            "Average Meta Valid Error 0.0447239056811668\n",
            "Average Meta Valid Accuracy 0.5824999855831265\n",
            "\n",
            "\n",
            "Iteration 6900\n",
            "Average Meta Valid Error 0.04361988080199808\n",
            "Average Meta Valid Accuracy 0.571249988861382\n",
            "\n",
            "\n",
            "Iteration 6950\n",
            "Average Meta Valid Error 0.044138223107438534\n",
            "Average Meta Valid Accuracy 0.5512499902397394\n",
            "\n",
            "\n",
            "Iteration 7000\n",
            "Average Meta Valid Error 0.044135125004686415\n",
            "Average Meta Valid Accuracy 0.5549999848008156\n",
            "\n",
            "\n",
            "Iteration 7050\n",
            "Average Meta Valid Error 0.04209764301776886\n",
            "Average Meta Valid Accuracy 0.5787499854341149\n",
            "\n",
            "\n",
            "Iteration 7100\n",
            "Average Meta Valid Error 0.044064694142434746\n",
            "Average Meta Valid Accuracy 0.5424999874085188\n",
            "\n",
            "\n",
            "Iteration 7150\n",
            "Average Meta Valid Error 0.043536674813367426\n",
            "Average Meta Valid Accuracy 0.5587499849498272\n",
            "\n",
            "\n",
            "Iteration 7200\n",
            "Average Meta Valid Error 0.04382967733545229\n",
            "Average Meta Valid Accuracy 0.5624999855645001\n",
            "\n",
            "\n",
            "Iteration 7250\n",
            "Average Meta Valid Error 0.04274452448589727\n",
            "Average Meta Valid Accuracy 0.5687499865889549\n",
            "\n",
            "\n",
            "Iteration 7300\n",
            "Average Meta Valid Error 0.04445266613038257\n",
            "Average Meta Valid Accuracy 0.5562499836087227\n",
            "\n",
            "\n",
            "Iteration 7350\n",
            "Average Meta Valid Error 0.0468090440845117\n",
            "Average Meta Valid Accuracy 0.528749986551702\n",
            "\n",
            "\n",
            "Iteration 7400\n",
            "Average Meta Valid Error 0.0419448409229517\n",
            "Average Meta Valid Accuracy 0.5787499826401472\n",
            "\n",
            "\n",
            "Iteration 7450\n",
            "Average Meta Valid Error 0.04396524711046368\n",
            "Average Meta Valid Accuracy 0.568749981932342\n",
            "\n",
            "\n",
            "Iteration 7500\n",
            "Average Meta Valid Error 0.044176742318086326\n",
            "Average Meta Valid Accuracy 0.559999986551702\n",
            "\n",
            "\n",
            "Iteration 7550\n",
            "Average Meta Valid Error 0.041701626032590866\n",
            "Average Meta Valid Accuracy 0.574999988079071\n",
            "\n",
            "\n",
            "Iteration 7600\n",
            "Average Meta Valid Error 0.04269569035386667\n",
            "Average Meta Valid Accuracy 0.568749981932342\n",
            "\n",
            "\n",
            "Iteration 7650\n",
            "Average Meta Valid Error 0.04374980152351782\n",
            "Average Meta Valid Accuracy 0.567499983124435\n",
            "\n",
            "\n",
            "Iteration 7700\n",
            "Average Meta Valid Error 0.0450833531212993\n",
            "Average Meta Valid Accuracy 0.5574999861419201\n",
            "\n",
            "\n",
            "Iteration 7750\n",
            "Average Meta Valid Error 0.04242754413280636\n",
            "Average Meta Valid Accuracy 0.5874999845400453\n",
            "\n",
            "\n",
            "Iteration 7800\n",
            "Average Meta Valid Error 0.042244056588970125\n",
            "Average Meta Valid Accuracy 0.5724999830126762\n",
            "\n",
            "\n",
            "Iteration 7850\n",
            "Average Meta Valid Error 0.0439729958306998\n",
            "Average Meta Valid Accuracy 0.549999987706542\n",
            "\n",
            "\n",
            "Iteration 7900\n",
            "Average Meta Valid Error 0.039931114530190825\n",
            "Average Meta Valid Accuracy 0.6199999824166298\n",
            "\n",
            "\n",
            "Iteration 7950\n",
            "Average Meta Valid Error 0.04463005549041554\n",
            "Average Meta Valid Accuracy 0.5687499810010195\n",
            "\n",
            "\n",
            "Iteration 8000\n",
            "Average Meta Valid Error 0.04097184824058786\n",
            "Average Meta Valid Accuracy 0.5687499856576324\n",
            "\n",
            "\n",
            "Iteration 8050\n",
            "Average Meta Valid Error 0.0450485396431759\n",
            "Average Meta Valid Accuracy 0.5599999874830246\n",
            "\n",
            "\n",
            "Iteration 8100\n",
            "Average Meta Valid Error 0.04482604993972927\n",
            "Average Meta Valid Accuracy 0.5649999883025885\n",
            "\n",
            "\n",
            "Iteration 8150\n",
            "Average Meta Valid Error 0.046978631522506475\n",
            "Average Meta Valid Accuracy 0.5074999849312007\n",
            "\n",
            "\n",
            "Iteration 8200\n",
            "Average Meta Valid Error 0.04481866990681738\n",
            "Average Meta Valid Accuracy 0.536249989643693\n",
            "\n",
            "\n",
            "Iteration 8250\n",
            "Average Meta Valid Error 0.04054077697219327\n",
            "Average Meta Valid Accuracy 0.581249981187284\n",
            "\n",
            "\n",
            "Iteration 8300\n",
            "Average Meta Valid Error 0.04192884574877098\n",
            "Average Meta Valid Accuracy 0.5824999827891588\n",
            "\n",
            "\n",
            "Iteration 8350\n",
            "Average Meta Valid Error 0.04451413289643824\n",
            "Average Meta Valid Accuracy 0.5737499874085188\n",
            "\n",
            "\n",
            "Iteration 8400\n",
            "Average Meta Valid Error 0.0448171267635189\n",
            "Average Meta Valid Accuracy 0.5674999877810478\n",
            "\n",
            "\n",
            "Iteration 8450\n",
            "Average Meta Valid Error 0.04436014883685857\n",
            "Average Meta Valid Accuracy 0.5412499867379665\n",
            "\n",
            "\n",
            "Iteration 8500\n",
            "Average Meta Valid Error 0.042761000455357134\n",
            "Average Meta Valid Accuracy 0.5699999900534749\n",
            "\n",
            "\n",
            "Iteration 8550\n",
            "Average Meta Valid Error 0.04424505622591823\n",
            "Average Meta Valid Accuracy 0.5574999852105975\n",
            "\n",
            "\n",
            "Iteration 8600\n",
            "Average Meta Valid Error 0.04281762783648446\n",
            "Average Meta Valid Accuracy 0.5949999843724072\n",
            "\n",
            "\n",
            "Iteration 8650\n",
            "Average Meta Valid Error 0.041725715214852244\n",
            "Average Meta Valid Accuracy 0.5812499835155904\n",
            "\n",
            "\n",
            "Iteration 8700\n",
            "Average Meta Valid Error 0.04502940707607195\n",
            "Average Meta Valid Accuracy 0.5362499877810478\n",
            "\n",
            "\n",
            "Iteration 8750\n",
            "Average Meta Valid Error 0.04378840088611469\n",
            "Average Meta Valid Accuracy 0.5599999893456697\n",
            "\n",
            "\n",
            "Iteration 8800\n",
            "Average Meta Valid Error 0.03995417896658182\n",
            "Average Meta Valid Accuracy 0.5949999857693911\n",
            "\n",
            "\n",
            "Iteration 8850\n",
            "Average Meta Valid Error 0.042813402717001736\n",
            "Average Meta Valid Accuracy 0.5912499856203794\n",
            "\n",
            "\n",
            "Iteration 8900\n",
            "Average Meta Valid Error 0.045163344766478986\n",
            "Average Meta Valid Accuracy 0.5337499873712659\n",
            "\n",
            "\n",
            "Iteration 8950\n",
            "Average Meta Valid Error 0.041439949127379805\n",
            "Average Meta Valid Accuracy 0.6049999836832285\n",
            "\n",
            "\n",
            "Iteration 9000\n",
            "Average Meta Valid Error 0.045553956064395607\n",
            "Average Meta Valid Accuracy 0.5337499845772982\n",
            "\n",
            "\n",
            "Iteration 9050\n",
            "Average Meta Valid Error 0.040609686460811645\n",
            "Average Meta Valid Accuracy 0.6012499835342169\n",
            "\n",
            "\n",
            "Iteration 9100\n",
            "Average Meta Valid Error 0.042234662105329335\n",
            "Average Meta Valid Accuracy 0.5699999844655395\n",
            "\n",
            "\n",
            "Iteration 9150\n",
            "Average Meta Valid Error 0.043403307849075645\n",
            "Average Meta Valid Accuracy 0.5712499851360917\n",
            "\n",
            "\n",
            "Iteration 9200\n",
            "Average Meta Valid Error 0.04452527826651931\n",
            "Average Meta Valid Accuracy 0.5662499889731407\n",
            "\n",
            "\n",
            "Iteration 9250\n",
            "Average Meta Valid Error 0.04412916791625321\n",
            "Average Meta Valid Accuracy 0.5549999857321382\n",
            "\n",
            "\n",
            "Iteration 9300\n",
            "Average Meta Valid Error 0.04248434747569263\n",
            "Average Meta Valid Accuracy 0.576249985024333\n",
            "\n",
            "\n",
            "Iteration 9350\n",
            "Average Meta Valid Error 0.0437461722176522\n",
            "Average Meta Valid Accuracy 0.5712499874643981\n",
            "\n",
            "\n",
            "Iteration 9400\n",
            "Average Meta Valid Error 0.04215057130204514\n",
            "Average Meta Valid Accuracy 0.5724999862723053\n",
            "\n",
            "\n",
            "Iteration 9450\n",
            "Average Meta Valid Error 0.042885160248260945\n",
            "Average Meta Valid Accuracy 0.553749986924231\n",
            "\n",
            "\n",
            "Iteration 9500\n",
            "Average Meta Valid Error 0.04205874784383923\n",
            "Average Meta Valid Accuracy 0.5899999886751175\n",
            "\n",
            "\n",
            "Iteration 9550\n",
            "Average Meta Valid Error 0.043862654303666204\n",
            "Average Meta Valid Accuracy 0.5399999860674143\n",
            "\n",
            "\n",
            "Iteration 9600\n",
            "Average Meta Valid Error 0.04291049751918763\n",
            "Average Meta Valid Accuracy 0.5537499887868762\n",
            "\n",
            "\n",
            "Iteration 9650\n",
            "Average Meta Valid Error 0.043144769384525716\n",
            "Average Meta Valid Accuracy 0.5662499815225601\n",
            "\n",
            "\n",
            "Iteration 9700\n",
            "Average Meta Valid Error 0.03874670248478651\n",
            "Average Meta Valid Accuracy 0.6099999835714698\n",
            "\n",
            "\n",
            "Iteration 9750\n",
            "Average Meta Valid Error 0.039185583882499486\n",
            "Average Meta Valid Accuracy 0.6099999882280827\n",
            "\n",
            "\n",
            "Iteration 9800\n",
            "Average Meta Valid Error 0.043026879720855504\n",
            "Average Meta Valid Accuracy 0.5687499884516001\n",
            "\n",
            "\n",
            "Iteration 9850\n",
            "Average Meta Valid Error 0.042876973981037736\n",
            "Average Meta Valid Accuracy 0.5637499848380685\n",
            "\n",
            "\n",
            "Iteration 9900\n",
            "Average Meta Valid Error 0.04054588847793639\n",
            "Average Meta Valid Accuracy 0.6012499835342169\n",
            "\n",
            "\n",
            "Iteration 9950\n",
            "Average Meta Valid Error 0.0435105393989943\n",
            "Average Meta Valid Accuracy 0.5624999888241291\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y5-U2OlIIIC4"
      },
      "source": [
        "# Save the meta-parameters to a file\n",
        "# Need to clip 'module.' from parameter keys in order to reload the model first\n",
        "maml_dict = model.state_dict()\n",
        "prefix = 'module.'\n",
        "n_clip = len(prefix)\n",
        "adapted_dict = {k[n_clip:]: v for k, v in maml_dict.items()\n",
        "                if k.startswith(prefix)}\n",
        "\n",
        "PATH = \"/content/model_CISFAR\"\n",
        "torch.save(adapted_dict, PATH)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pf1sR_diKQpu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 532
        },
        "outputId": "4921ab84-0abf-4861-9b30-fbb0548879b9"
      },
      "source": [
        "# Load MAML saved parameters\n",
        "PATH = \"/content/model_CISFAR\"\n",
        "saved_model = CifarCNN()\n",
        "saved_model.load_state_dict(torch.load(PATH), strict=False)\n",
        "saved_model.to('cuda')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "CifarCNN(\n",
              "  (features): Sequential(\n",
              "    (0): ConvBase(\n",
              "      (0): ConvBlock(\n",
              "        (normalize): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU()\n",
              "        (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "      )\n",
              "      (1): ConvBlock(\n",
              "        (normalize): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU()\n",
              "        (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "      )\n",
              "      (2): ConvBlock(\n",
              "        (normalize): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU()\n",
              "        (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "      )\n",
              "      (3): ConvBlock(\n",
              "        (normalize): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU()\n",
              "        (conv): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "      )\n",
              "    )\n",
              "    (1): Lambda()\n",
              "    (2): Flatten()\n",
              "  )\n",
              "  (linear): Linear(in_features=32, out_features=5, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k54Uchj9Xpp7"
      },
      "source": [
        "## Perform fine-tuning, evaluate performance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DV82vvYOK9i_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 641
        },
        "outputId": "4ba75704-8d79-4dc6-bff4-dd641de6b2c8"
      },
      "source": [
        "meta_test_error = 0.0\n",
        "meta_test_accuracy = 0.0\n",
        "meta_batch_size = 32\n",
        "meta_lr=0.003\n",
        "inner_lr=0.5\n",
        "adaptation_steps=1\n",
        "\n",
        "opt = optim.Adam(maml.parameters(), meta_lr)\n",
        "loss = nn.CrossEntropyLoss(reduction='mean')\n",
        "\n",
        "\n",
        "maml = l2l.algorithms.MAML(model, lr=inner_lr, first_order=False)\n",
        "for task in range(meta_batch_size):\n",
        "  # Compute meta-testing loss\n",
        "  learner = maml.clone()\n",
        "  batch = tasksets.test.sample()\n",
        "  evaluation_error, evaluation_accuracy = fast_adapt(batch,\n",
        "                                                    learner,\n",
        "                                                    loss,\n",
        "                                                    adaptation_steps,\n",
        "                                                    shots,\n",
        "                                                    ways,\n",
        "                                                    'cuda')\n",
        "  meta_test_error += evaluation_error.item()\n",
        "  meta_test_accuracy += evaluation_accuracy.item()\n",
        "  print('Avg Meta Test Accuracy', round(meta_test_accuracy / meta_batch_size,2)*100,'%')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py:385: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
            "  if param.grad is not None:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Avg Meta Test Accuracy 2.0 %\n",
            "Avg Meta Test Accuracy 4.0 %\n",
            "Avg Meta Test Accuracy 6.0 %\n",
            "Avg Meta Test Accuracy 9.0 %\n",
            "Avg Meta Test Accuracy 11.0 %\n",
            "Avg Meta Test Accuracy 13.0 %\n",
            "Avg Meta Test Accuracy 15.0 %\n",
            "Avg Meta Test Accuracy 17.0 %\n",
            "Avg Meta Test Accuracy 19.0 %\n",
            "Avg Meta Test Accuracy 21.0 %\n",
            "Avg Meta Test Accuracy 23.0 %\n",
            "Avg Meta Test Accuracy 25.0 %\n",
            "Avg Meta Test Accuracy 28.000000000000004 %\n",
            "Avg Meta Test Accuracy 30.0 %\n",
            "Avg Meta Test Accuracy 32.0 %\n",
            "Avg Meta Test Accuracy 34.0 %\n",
            "Avg Meta Test Accuracy 36.0 %\n",
            "Avg Meta Test Accuracy 38.0 %\n",
            "Avg Meta Test Accuracy 39.0 %\n",
            "Avg Meta Test Accuracy 41.0 %\n",
            "Avg Meta Test Accuracy 43.0 %\n",
            "Avg Meta Test Accuracy 44.0 %\n",
            "Avg Meta Test Accuracy 46.0 %\n",
            "Avg Meta Test Accuracy 47.0 %\n",
            "Avg Meta Test Accuracy 49.0 %\n",
            "Avg Meta Test Accuracy 51.0 %\n",
            "Avg Meta Test Accuracy 54.0 %\n",
            "Avg Meta Test Accuracy 56.00000000000001 %\n",
            "Avg Meta Test Accuracy 57.99999999999999 %\n",
            "Avg Meta Test Accuracy 60.0 %\n",
            "Avg Meta Test Accuracy 62.0 %\n",
            "Avg Meta Test Accuracy 64.0 %\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}