intent_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bhZs998drzPJ"
   },
   "source": [
    "## Intent Classification on CLINC150 dataset\n",
    "\n",
    "#### Classification of 20 in-scope intent classes performed using BERT\n",
    "\n",
    "##### HuggingFace Transformers v3.5.0 used. This project has been done using a GPU on Google Colaboratory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "25-YOPE_3ib0",
    "outputId": "57145605-8444-4f63-880d-6cf71d744010"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive/\n"
     ]
    }
   ],
   "source": [
    "#connecting and mounting Google Drive folder\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "ZqrTRDd33uXf"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('/content/drive/My Drive/research_projects/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "s3Oq1d-036py",
    "outputId": "fbf28177-499d-47bc-9729-1d88155b8ce9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers==3.5.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9c/34/fb092588df61bf33f113ade030d1cbe74fb73a0353648f8dd938a223dce7/transformers-3.5.0-py3-none-any.whl (1.3MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3MB 9.0MB/s \n",
      "\u001b[?25hCollecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 27.1MB/s \n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==3.5.0) (2.23.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==3.5.0) (2019.12.20)\n",
      "Collecting sentencepiece==0.1.91\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 40.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==3.5.0) (0.8)\n",
      "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==3.5.0) (3.0.12)\n",
      "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers==3.5.0) (3.12.4)\n",
      "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==3.5.0) (4.41.1)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==3.5.0) (20.8)\n",
      "Collecting tokenizers==0.9.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4c/34/b39eb9994bc3c999270b69c9eea40ecc6f0e97991dba28282b9fd32d44ee/tokenizers-0.9.3-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n",
      "\u001b[K     |████████████████████████████████| 2.9MB 43.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==3.5.0) (1.19.4)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3.5.0) (1.15.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3.5.0) (7.1.2)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3.5.0) (1.0.0)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.5.0) (2020.12.5)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.5.0) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.5.0) (1.24.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.5.0) (2.10)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers==3.5.0) (51.0.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==3.5.0) (2.4.7)\n",
      "Building wheels for collected packages: sacremoses\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=57fefb5cfc130b337b5b6184fcc79db59cd9e3d24eb8ae26796dacaaa5601d93\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built sacremoses\n",
      "Installing collected packages: sacremoses, sentencepiece, tokenizers, transformers\n",
      "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.9.3 transformers-3.5.0\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers==3.5.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "2ajvtaoT4Es-"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "from transformers import BertForSequenceClassification, AdamW, BertConfig, BertTokenizer\n",
    "from transformers import get_linear_schedule_with_warmup\n",
    "import numpy as np\n",
    "import random\n",
    "from random import shuffle\n",
    "import time\n",
    "import datetime\n",
    "import json\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "UxpmaUq-313M"
   },
   "outputs": [],
   "source": [
    "with open('data_full.json', 'r') as f:\n",
    "  data= json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OpI8Ho0NuwZi"
   },
   "source": [
    "#### Selecting 20 classes and preparing training, validation and testing datasets for the 20 selected in-scope classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "zufmc97hyLlg"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def loop_over(data, classes):\n",
    "  select_data, select_labels = [], []\n",
    "  shuffle(data)\n",
    "  for row in data:\n",
    "    if row[1] in classes:\n",
    "      select_data.append(row[0])\n",
    "      select_labels.append(row[1])\n",
    "  return select_data, select_labels\n",
    "\n",
    "\n",
    "def prepare_splits(data):\n",
    "\n",
    "  s = set()\n",
    "  for row in data['train']:\n",
    "    s.add(row[-1])\n",
    "  \n",
    "  classes = list(s)[:20]\n",
    "  \n",
    "\n",
    "  train_data, train_labels = loop_over(data['train'], classes)\n",
    "  val_data, val_labels = loop_over(data['val'], classes)\n",
    "  test_data, test_labels = loop_over(data['test'], classes)\n",
    "\n",
    "  return classes, [train_data, train_labels], [val_data, val_labels], [test_data, test_labels]\n",
    "\n",
    "label_set, train, val, test = prepare_splits(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uexaDr_Y-fRh",
    "outputId": "b8d3ddd5-0126-4422-e8ae-7eb35ce9859b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2000 2000\n",
      "400 400\n",
      "600 600\n"
     ]
    }
   ],
   "source": [
    "#Doing a stat check on the data in the train, validation and testing splits\n",
    "print(len(train[0]), len(train[1]))\n",
    "print(len(val[0]), len(val[1]))\n",
    "print(len(test[0]), len(test[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uJziDAF8C0Jc",
    "outputId": "938bef85-2657-4556-9f71-1ed632e582ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Printing some samples data from the train set\n",
      "do you know of any good restaurants\n",
      "restaurant_suggestion\n",
      "\n",
      "create alarm 6am\n",
      "alarm\n",
      "\n",
      "i would like a block put on my chase account asap\n",
      "freeze_account\n",
      "\n",
      "will you tell me who made the ai\n",
      "who_made_you\n",
      "\n",
      "im sorry can you repeat yourself\n",
      "repeat\n",
      "\n",
      "how much as my taxes by the way\n",
      "taxes\n",
      "\n",
      "do i need to get vaccines before my trip\n",
      "vaccines\n",
      "\n",
      "how much gas is in my gas tank\n",
      "gas\n",
      "\n",
      "could you set one alarm for 8am saturday and one for 9am sunday\n",
      "alarm\n",
      "\n",
      "start a countdown for 20 minutes\n",
      "timer\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Printing some samples data from the train set')\n",
    "for i, row in enumerate(train[0][:10]):\n",
    "  print(row)\n",
    "  print(train[1][i])\n",
    "  print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "n2VZ0LXwEhjg"
   },
   "outputs": [],
   "source": [
    "#Function to convert labels into indices\n",
    "def labels2idx_func(labels, labels2idx):\n",
    "  \n",
    "  idx_labels = []\n",
    "  for label in labels:\n",
    "    idx_labels.append(labels2idx[label])\n",
    "  return idx_labels\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SHd296mlvc4N"
   },
   "source": [
    "#### Preparing dataset for training and testing. Converting labels to indices using labels2idx_func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "ZePvs6oz6I7j"
   },
   "outputs": [],
   "source": [
    "labels2idx = {label:i for i, label in enumerate(label_set) }\n",
    "\n",
    "train_data, train_labels = train[0], labels2idx_func(train[1], labels2idx)\n",
    "val_data, val_labels = val[0], labels2idx_func(val[1], labels2idx)\n",
    "test_data, test_labels = test[0], labels2idx_func(test[1], labels2idx)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "h2ge1k--vqT4"
   },
   "source": [
    "#### Labels with their corresponding indices and printing some sample data from the train set with their indexed label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Eo5AUpXJ6I-r",
    "outputId": "7c4dab8a-aa38-4651-99d9-0facc076bf0d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels with their corresponding indices\n",
      "{'timezone': 0, 'vaccines': 1, 'reminder_update': 2, 'credit_limit_change': 3, 'restaurant_suggestion': 4, 'redeem_rewards': 5, 'meeting_schedule': 6, 'alarm': 7, 'taxes': 8, 'who_made_you': 9, 'timer': 10, 'food_last': 11, 'fun_fact': 12, 'restaurant_reservation': 13, 'meaning_of_life': 14, 'freeze_account': 15, 'gas': 16, 'tire_pressure': 17, 'repeat': 18, 'card_declined': 19}\n",
      "\n",
      "do you know of any good restaurants 4\n",
      "create alarm 6am 7\n",
      "i would like a block put on my chase account asap 15\n",
      "will you tell me who made the ai 9\n",
      "im sorry can you repeat yourself 18\n",
      "how much as my taxes by the way 8\n",
      "do i need to get vaccines before my trip 1\n",
      "how much gas is in my gas tank 16\n",
      "could you set one alarm for 8am saturday and one for 9am sunday 7\n",
      "start a countdown for 20 minutes 10\n"
     ]
    }
   ],
   "source": [
    "print('Labels with their corresponding indices')\n",
    "print(labels2idx)\n",
    "print()\n",
    "for i, sent in enumerate(train_data[:10]):\n",
    "  print(sent, train_labels[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "ZV3Ntf6o6JB8"
   },
   "outputs": [],
   "source": [
    "# Function to encode sentences and preparing them for BERT model input. More details are in the text cell below\n",
    "def encode_sentences(data, tokenizer):\n",
    "  input_ids = []\n",
    "  attention_masks = []\n",
    "  for sent in data:\n",
    "      encoded = tokenizer.encode_plus(sent, add_special_tokens=True, max_length=64, pad_to_max_length=True, return_attention_mask=True, return_tensors='pt')\n",
    "      input_ids.append(encoded['input_ids'])\n",
    "      attention_masks.append(encoded['attention_mask'])\n",
    "  input_ids = torch.cat(input_ids, dim=0)\n",
    "  attention_masks = torch.cat(attention_masks, dim=0)\n",
    "  return input_ids, attention_masks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Da49_HA0zGet"
   },
   "source": [
    "#### Here BertTokenizer is being used. For the sentences to be used for training, validation and testing, they need to be prepared appropriately.\n",
    "\n",
    "Given the dataset provided by the authors [link](https://github.com/clinc/oos-eval) talk about preprocessing the dataset [Section 2.3] (lower case sentences, punctuation removal and maintaining train/test distribution of data, explicit preprocessing becomes easier.\n",
    "\n",
    "Preprocessing such as normalization (stemming) or stopword removal could be done, but there is a possibility that the sentences could get shorter and the stemmed tokens could get assigned new token ids rather than getting mapped onto token ids already in the BertTokenizer vocabulary. And very short sentences would mean that the intent could be classified based on specific tokens rather than the contextual meaning of the entire sentence. Hence, the current existing state of sentences currently provided by the authors are good enough to be used as input to the BERT model.\n",
    "\n",
    "Each sentence needs to be tokenized, [CLS] and [SEP] tokens to be appended to the beginning and end of the sentences and tokens then have to be mapped to appropriate token ids in the vocabulary. Then sentences need to be padded or truncation to maintain uniformity in sequence length and attention masks need to be generated for the BERT model to only consider the actual tokenized words and ignore the pad tokens. BertTokenizer provides the encode_plus function for use to perform all these tasks.\n",
    "\n",
    "Maximum sequence length has been set to 64."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 139,
     "referenced_widgets": [
      "fb32d0f3943e4315bfe2b1073a19922b",
      "254db5993a2640769ece07367074bd34",
      "81c2dec6e3854aa880b3e3f99f000936",
      "23bb8c15ffcf46d8b0cb5d95f4bf2c29",
      "9794fea3bf9446b9ac7b368baeff6416",
      "be3a278cd35947aab30f232438a5927f",
      "b784cfadb8174a87b4681ba860e12b0d",
      "652aefd7251648078b7f2822a008191f"
     ]
    },
    "id": "jL6aagT4EFCH",
    "outputId": "ed16edac-1438-4b83-af3a-84188e9e68c6"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb32d0f3943e4315bfe2b1073a19922b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2022: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "#generating input ids and attention masks for the training, validation and testing splits.\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
    "train_input_ids, train_mask = encode_sentences(train_data, tokenizer)\n",
    "val_input_ids, val_mask = encode_sentences(val_data, tokenizer)\n",
    "test_input_ids, test_mask = encode_sentences(test_data, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "mEHN_IxhEytW"
   },
   "outputs": [],
   "source": [
    "#Using TensorDataset to align input ids, attention mask and corresponding labels.\n",
    "train_dataset = TensorDataset(train_input_ids, train_mask, torch.tensor(train_labels))\n",
    "val_dataset = TensorDataset(val_input_ids, val_mask, torch.tensor(val_labels))\n",
    "test_dataset = TensorDataset(test_input_ids, test_mask, torch.tensor(test_labels))\n",
    "\n",
    "#Batch size set as 64 for training and testing and this hyperparameter could be changed as per need or GPU space\n",
    "batch_size = 64\n",
    "\n",
    "#Batching and bucketing samples. Batches for training set also shuffled while training. Not need for validation or testing set.\n",
    "train_dataloader = DataLoader(train_dataset, sampler = RandomSampler(train_dataset), batch_size=batch_size)\n",
    "validation_dataloader = DataLoader(val_dataset, sampler = SequentialSampler(val_dataset), batch_size=batch_size)\n",
    "test_dataloader = DataLoader(test_dataset, sampler = SequentialSampler(test_dataset), batch_size=batch_size)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3AbSraJ5306_"
   },
   "source": [
    "#### BertForSequenceClassification model provided by HuggingFace Transformers. \n",
    "\n",
    "It contains a linear head on top the BertModel for multi-label sequence classification. Also using pre-trained Bert base uncased weights and fine tuning them for intent classification using the training set prepared above.\n",
    "\n",
    "The Bert model is pre-trained on English corpora using masked language modeling and next sentence prediction. The BERT model used here has a hidden state of 768 length and is the base model pre-trained on uncased English sentences. It consists of 12 layers with 12 heads in total and includes 110M parameters to make up the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000,
     "referenced_widgets": [
      "e9108c6f09704ace85c3552cfe80863d",
      "68975a051abd49d1b2899eff3cf5c7e5",
      "feeaac18a9834f38a478ec2f1d8dd17a",
      "de0a3c109c6f4218a9be7719b7d2116c",
      "54d4ef037e4345118d93052df529dae5",
      "df6656440483454b87e14cec59c0f8eb",
      "0e8b01aa619b4b8fb6c2ad5e2be912a5",
      "1b89fb6d55c34b93a44916a8f84ff008",
      "4b88d5d184e045d6ae3dedfc85282771",
      "48edb33573e140f2923fe6284aa2199e",
      "6186690e38dc45048d7c313a0f75da23",
      "fe18428d31154038bb101e3be6931364",
      "e99e838776c645a28171585ef17fed3d",
      "f3b4e7ec8dbb44f497ce500371dd9552",
      "6493722112d541519e334f93c54fdd06",
      "18735c0c367646639a03f3ad7e6a4066"
     ]
    },
    "id": "Gx5TupM0VvM8",
    "outputId": "7b0c8c39-70e7-4e92-96f3-fdb923dccf7d"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9108c6f09704ace85c3552cfe80863d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b88d5d184e045d6ae3dedfc85282771",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=20, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=20, output_attentions=False, output_hidden_states=False)\n",
    "\n",
    "#loading model to gpu if available or cpu if not\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pfRQn01i4_ge"
   },
   "source": [
    "Using AdamW optimizer and scheduler for controlling the learning rate. Scheduler used to warmup the model so that the learning rate is not drastically changed during the warmup steps as loss maybe very high during warmup.\n",
    "\n",
    "Epochs 5 selected. Too many epochs may lead to model overfitting. \n",
    "\n",
    "Warm step 100. This could also be changed as needed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "3uFTj0cOB7Ok"
   },
   "outputs": [],
   "source": [
    "optimizer = AdamW(model.parameters(), lr = 2e-5, eps = 1e-8)\n",
    "epochs = 5\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = 100, num_training_steps = total_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "sexoJwXaCQHY"
   },
   "outputs": [],
   "source": [
    "#function to calculate accuracy during prediction\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "rhdl9cKvCXW3"
   },
   "outputs": [],
   "source": [
    "#function to calculate time elapsed\n",
    "def format_time(elapsed):\n",
    "    elapsed_rounded = int(round((elapsed)))\n",
    "    return str(datetime.timedelta(seconds=elapsed_rounded))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "523TqhKt6qCx"
   },
   "source": [
    "#### Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "u0TpB3PYCicQ",
    "outputId": "4dd1b588-7594-4cea-d770-db0b67baa7ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "======== Epoch 1 / 5 ========\n",
      "Training...\n",
      "  Average training loss: 3.01\n",
      "  Training epcoh took: 0:00:12\n",
      "Running Validation...\n",
      "  Accuracy: 0.13\n",
      "  Validation Loss: 2.93\n",
      "  Validation took: 0:00:01\n",
      "======== Epoch 2 / 5 ========\n",
      "Training...\n",
      "  Average training loss: 2.74\n",
      "  Training epcoh took: 0:00:12\n",
      "Running Validation...\n",
      "  Accuracy: 0.50\n",
      "  Validation Loss: 2.46\n",
      "  Validation took: 0:00:01\n",
      "======== Epoch 3 / 5 ========\n",
      "Training...\n",
      "  Average training loss: 2.21\n",
      "  Training epcoh took: 0:00:12\n",
      "Running Validation...\n",
      "  Accuracy: 0.82\n",
      "  Validation Loss: 1.77\n",
      "  Validation took: 0:00:01\n",
      "======== Epoch 4 / 5 ========\n",
      "Training...\n",
      "  Average training loss: 1.48\n",
      "  Training epcoh took: 0:00:12\n",
      "Running Validation...\n",
      "  Accuracy: 0.98\n",
      "  Validation Loss: 1.07\n",
      "  Validation took: 0:00:01\n",
      "======== Epoch 5 / 5 ========\n",
      "Training...\n",
      "  Average training loss: 1.00\n",
      "  Training epcoh took: 0:00:12\n",
      "Running Validation...\n",
      "  Accuracy: 0.98\n",
      "  Validation Loss: 0.85\n",
      "  Validation took: 0:00:01\n",
      "\n",
      "Training complete!\n",
      "Total training took 0:01:04 (h:mm:ss)\n"
     ]
    }
   ],
   "source": [
    "seed_val = 42\n",
    "\n",
    "random.seed(seed_val)\n",
    "np.random.seed(seed_val)\n",
    "torch.manual_seed(seed_val)\n",
    "\n",
    "#comment out if gpu not present\n",
    "torch.cuda.manual_seed_all(seed_val)\n",
    "\n",
    "training_stats = []\n",
    "total_t0 = time.time()\n",
    "for epoch_i in range(0, epochs):\n",
    "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
    "    print('Training...')\n",
    "    t0 = time.time()\n",
    "    total_train_loss = 0\n",
    "    model.train()\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "        if step % 40 == 0 and not step == 0:\n",
    "            elapsed = format_time(time.time() - t0)\n",
    "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
    "\n",
    "        #loading data to whichever device available\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "\n",
    "        model.zero_grad()     \n",
    "\n",
    "        #Sending data to the model and storing loss and logits generated by the model.   \n",
    "        loss, logits = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n",
    "        total_train_loss += loss.item()\n",
    "\n",
    "        loss.backward() #backpropagating after loss calculation\n",
    "\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) #gradient clipping in case of gradient explosion\n",
    "\n",
    "        #running the optimizer and the scheduler for the learning rate\n",
    "        optimizer.step()\n",
    "        scheduler.step()\n",
    "\n",
    "    avg_train_loss = total_train_loss / len(train_dataloader)            \n",
    "    \n",
    "    training_time = format_time(time.time() - t0)\n",
    "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
    "    print(\"  Training epcoh took: {:}\".format(training_time))\n",
    "    print(\"Running Validation...\")\n",
    "\n",
    "    t0 = time.time()\n",
    "\n",
    "\n",
    "    #Validation\n",
    "    model.eval()\n",
    "    total_eval_accuracy = 0\n",
    "    total_eval_loss = 0\n",
    "    nb_eval_steps = 0\n",
    "\n",
    "    for batch in validation_dataloader:\n",
    "\n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "\n",
    "        with torch.no_grad():       # no need of loss propagation \n",
    "\n",
    "            (loss, logits) = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask,labels=b_labels)\n",
    " \n",
    "        total_eval_loss += loss.item()\n",
    "\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "\n",
    "        total_eval_accuracy += flat_accuracy(logits, label_ids)\n",
    "\n",
    "    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)\n",
    "    print(\"  Accuracy: {0:.2f}\".format(avg_val_accuracy))\n",
    "    avg_val_loss = total_eval_loss / len(validation_dataloader)\n",
    "    validation_time = format_time(time.time() - t0)\n",
    "    \n",
    "    print(\"  Validation Loss: {0:.2f}\".format(avg_val_loss))\n",
    "    print(\"  Validation took: {:}\".format(validation_time))\n",
    "    training_stats.append(\n",
    "        {\n",
    "            'epoch': epoch_i + 1,\n",
    "            'Training Loss': avg_train_loss,\n",
    "            'Valid. Loss': avg_val_loss,\n",
    "            'Valid. Accur.': avg_val_accuracy,\n",
    "            'Training Time': training_time,\n",
    "            'Validation Time': validation_time\n",
    "        }\n",
    "    )\n",
    "\n",
    "print(\"\")\n",
    "print(\"Training complete!\")\n",
    "print(\"Total training took {:} (h:mm:ss)\".format(format_time(time.time()-total_t0)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "g_IbAkju7o7P"
   },
   "source": [
    "#### Testing phase. Using the model and predicting on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EBMivijTDzuo",
    "outputId": "329367d2-4c98-4bf6-8a37-6a1c4a7227a1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting labels\n"
     ]
    }
   ],
   "source": [
    "print('Predicting labels')\n",
    "\n",
    "model.eval()\n",
    "\n",
    "predictions , true_labels = [], []\n",
    "\n",
    "for batch in test_dataloader:\n",
    "  batch = tuple(t.to(device) for t in batch)\n",
    "  b_input_ids, b_input_mask, b_labels = batch\n",
    "  \n",
    "  with torch.no_grad():\n",
    "      outputs = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask)\n",
    "                      \n",
    "  logits = outputs[0]\n",
    "  logits = logits.detach().cpu().numpy()\n",
    "  label_ids = b_labels.to('cpu').numpy()\n",
    "\n",
    "  predictions.append(logits)\n",
    "  true_labels.append(label_ids)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bNwP_zsR8Yq9"
   },
   "source": [
    "The predictions provided by the model are in the form of logits. The model tries to maximize the negative log likehood score across the number of classes present. Hence passing the logits passed through a softmax function would give you a probability distribution over the number of classes you have. And the highest probability given to a particular class, is the class predicted by the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "guogDZLTgfg3",
    "outputId": "fd2b3dba-5eb1-4821-e267-4cfa6df7d4ee"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "softmax = torch.nn.Softmax()\n",
    "\n",
    "labels_flat = []\n",
    "pred_flat = []\n",
    "for i, batch in enumerate(predictions):\n",
    "  for j, logits in enumerate(batch):\n",
    "    pred_flat.append(np.argmax(softmax(torch.tensor(logits))))\n",
    "    labels_flat.append(true_labels[i][j])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DXS0y6w4gqX4",
    "outputId": "853944b2-702e-47b9-bb42-b6e47a47b797"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the model to the true labels\n",
      "0.9783333333333334\n"
     ]
    }
   ],
   "source": [
    "cnt=0\n",
    "for i, val in enumerate(pred_flat):\n",
    "  if val == labels_flat[i]:\n",
    "    cnt+=1\n",
    "print('Accuracy of the model to the true labels')\n",
    "print(cnt/len(labels_flat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pEbxrlN5jftQ",
    "outputId": "857c8967-e4f9-40bb-ed6e-527ad8b941b5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Macro averaged F1 score\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9782846298635111"
      ]
     },
     "execution_count": 34,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Macro averaged F1 score')\n",
    "sklearn.metrics.f1_score(labels_flat, pred_flat, average='macro')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_vGTRQiZ_REp"
   },
   "source": [
    "#### Saving the model to an output directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yGzh3QRDjrEh",
    "outputId": "18b145fc-7517-4c5d-fb71-3f1dec109cb9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('output_dir/tokenizer_config.json',\n",
       " 'output_dir/special_tokens_map.json',\n",
       " 'output_dir/vocab.txt',\n",
       " 'output_dir/added_tokens.json')"
      ]
     },
     "execution_count": 35,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_dir = 'output_dir'\n",
    "\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  \n",
    "model_to_save.save_pretrained(output_dir)\n",
    "tokenizer.save_pretrained(output_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "_aGJDnmio_MQ"
   },
   "outputs": [],
   "source": [
    "!pip freeze > requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PSohr1H5_1_z",
    "outputId": "857749cf-0457-45f2-8046-24bc789b4d49"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_full.json\toutput_dir  requirements.txt\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wxjcP6wJ_5n1"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "Loyal AI.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0e8b01aa619b4b8fb6c2ad5e2be912a5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "18735c0c367646639a03f3ad7e6a4066": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1b89fb6d55c34b93a44916a8f84ff008": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "23bb8c15ffcf46d8b0cb5d95f4bf2c29": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_652aefd7251648078b7f2822a008191f",
      "placeholder": "​",
      "style": "IPY_MODEL_b784cfadb8174a87b4681ba860e12b0d",
      "value": " 232k/232k [00:00&lt;00:00, 790kB/s]"
     }
    },
    "254db5993a2640769ece07367074bd34": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "48edb33573e140f2923fe6284aa2199e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "4b88d5d184e045d6ae3dedfc85282771": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_6186690e38dc45048d7c313a0f75da23",
       "IPY_MODEL_fe18428d31154038bb101e3be6931364"
      ],
      "layout": "IPY_MODEL_48edb33573e140f2923fe6284aa2199e"
     }
    },
    "54d4ef037e4345118d93052df529dae5": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "6186690e38dc45048d7c313a0f75da23": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f3b4e7ec8dbb44f497ce500371dd9552",
      "max": 440473133,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e99e838776c645a28171585ef17fed3d",
      "value": 440473133
     }
    },
    "6493722112d541519e334f93c54fdd06": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "652aefd7251648078b7f2822a008191f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "68975a051abd49d1b2899eff3cf5c7e5": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "81c2dec6e3854aa880b3e3f99f000936": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_be3a278cd35947aab30f232438a5927f",
      "max": 231508,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_9794fea3bf9446b9ac7b368baeff6416",
      "value": 231508
     }
    },
    "9794fea3bf9446b9ac7b368baeff6416": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "b784cfadb8174a87b4681ba860e12b0d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "be3a278cd35947aab30f232438a5927f": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "de0a3c109c6f4218a9be7719b7d2116c": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1b89fb6d55c34b93a44916a8f84ff008",
      "placeholder": "​",
      "style": "IPY_MODEL_0e8b01aa619b4b8fb6c2ad5e2be912a5",
      "value": " 433/433 [00:00&lt;00:00, 12.2kB/s]"
     }
    },
    "df6656440483454b87e14cec59c0f8eb": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e9108c6f09704ace85c3552cfe80863d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_feeaac18a9834f38a478ec2f1d8dd17a",
       "IPY_MODEL_de0a3c109c6f4218a9be7719b7d2116c"
      ],
      "layout": "IPY_MODEL_68975a051abd49d1b2899eff3cf5c7e5"
     }
    },
    "e99e838776c645a28171585ef17fed3d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "f3b4e7ec8dbb44f497ce500371dd9552": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "fb32d0f3943e4315bfe2b1073a19922b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_81c2dec6e3854aa880b3e3f99f000936",
       "IPY_MODEL_23bb8c15ffcf46d8b0cb5d95f4bf2c29"
      ],
      "layout": "IPY_MODEL_254db5993a2640769ece07367074bd34"
     }
    },
    "fe18428d31154038bb101e3be6931364": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_18735c0c367646639a03f3ad7e6a4066",
      "placeholder": "​",
      "style": "IPY_MODEL_6493722112d541519e334f93c54fdd06",
      "value": " 440M/440M [00:06&lt;00:00, 68.4MB/s]"
     }
    },
    "feeaac18a9834f38a478ec2f1d8dd17a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Downloading: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_df6656440483454b87e14cec59c0f8eb",
      "max": 433,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_54d4ef037e4345118d93052df529dae5",
      "value": 433
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}