BERT_LSTM_Aug_WE.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XIFwSYH_0KDq",
        "outputId": "7c09c24a-4d89-4e92-e429-be70d7c22bd3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  data.zip\n",
            "   creating: data/\n",
            "   creating: data/Development/\n",
            "  inflating: data/Development/dev_definitions.txt  \n",
            "  inflating: data/Development/dev_examples.txt  \n",
            "  inflating: data/Development/dev_hypernyms.txt  \n",
            "  inflating: data/Development/dev_labels.txt  \n",
            "  inflating: data/README.txt         \n",
            "   creating: data/Test/\n",
            "  inflating: data/Test/test_definitions.txt  \n",
            "  inflating: data/Test/test_examples.txt  \n",
            "  inflating: data/Test/test_hypernyms.txt  \n",
            "   creating: data/Training/\n",
            "  inflating: data/Training/train_definitions.txt  \n",
            "  inflating: data/Training/train_examples.txt  \n",
            "  inflating: data/Training/train_hypernyms.txt  \n",
            "  inflating: data/Training/train_labels.txt  \n"
          ]
        }
      ],
      "source": [
        "!unzip data.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3QUASp5SZJFX",
        "outputId": "2aa6f67e-94a2-461b-9f78-4f43f2c9351f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting tensorflow_text\n",
            "  Downloading tensorflow_text-2.8.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (4.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.9 MB 4.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (0.12.0)\n",
            "Requirement already satisfied: tensorflow<2.9,>=2.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (2.8.0)\n",
            "Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (13.0.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.6.3)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (0.2.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.17.3)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.14.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.1.2)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.10.0.2)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (0.24.0)\n",
            "Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.0.0)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.44.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.1.0)\n",
            "Requirement already satisfied: tensorboard<2.9,>=2.8 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (2.8.0)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.15.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.3.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (57.4.0)\n",
            "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (1.21.5)\n",
            "Collecting tf-estimator-nightly==2.8.0.dev2021122109\n",
            "  Downloading tf_estimator_nightly-2.8.0.dev2021122109-py2.py3-none-any.whl (462 kB)\n",
            "\u001b[K     |████████████████████████████████| 462 kB 48.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: gast>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (0.5.3)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (3.1.0)\n",
            "Requirement already satisfied: keras<2.9,>=2.8.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (2.8.0)\n",
            "Requirement already satisfied: flatbuffers>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.9,>=2.8.0->tensorflow_text) (2.0)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.37.1)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.5.2)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.35.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.0.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.4.6)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (2.23.0)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.8.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.3.6)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.6.1)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (4.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (4.2.4)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (4.11.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.7.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (0.4.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (2021.10.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.9,>=2.8->tensorflow<2.9,>=2.8.0->tensorflow_text) (3.2.0)\n",
            "Installing collected packages: tf-estimator-nightly, tensorflow-text\n",
            "Successfully installed tensorflow-text-2.8.1 tf-estimator-nightly-2.8.0.dev2021122109\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0 MB 4.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 45.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 6.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 46.7 MB/s \n",
            "\u001b[?25hCollecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.5 MB 31.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.5.1 pyyaml-6.0 sacremoses-0.0.49 tokenizers-0.11.6 transformers-4.18.0\n"
          ]
        }
      ],
      "source": [
        "!pip install tensorflow_text\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "ZVe2WcmSCGxy"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "import os\n",
        "import errno\n",
        "import random\n",
        "import pandas as pd\n",
        "\n",
        "from sklearn import metrics\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "\n",
        "from sklearn.utils import shuffle\n",
        "\n",
        "!rm -rf processed_data\n",
        "\n",
        "try:\n",
        "    os.makedirs(\"processed_data\")\n",
        "except OSError as e:\n",
        "    if e.errno != errno.EEXIST:\n",
        "        raise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "eRMB7CMRkvT_"
      },
      "outputs": [],
      "source": [
        "def decontracted(phrase):\n",
        "\n",
        "    phrase = re.sub(r\"won't\", \"will not\", phrase)\n",
        "    phrase = re.sub(r\"can't\", \"can not\", phrase)\n",
        "    phrase = re.sub(r\"gonna\", \"going to\", phrase)\n",
        "    phrase = re.sub(r\"wanna\", \"want to\", phrase)\n",
        "\n",
        "    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n",
        "    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n",
        "    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n",
        "    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n",
        "    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n",
        "    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n",
        "    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n",
        "    phrase = re.sub(r\"\\.+\", \" \", phrase)\n",
        "    phrase = re.sub(r\"[^A-Za-z$]\", \" \", phrase)\n",
        "    phrase = re.sub(r\" +\", \" \", phrase)\n",
        "\n",
        "    return phrase"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "KWjrzeyqkz0N"
      },
      "outputs": [],
      "source": [
        "def get_df(pref, testing=False):\n",
        "\n",
        "    dataset = []\n",
        "    with open(pref + \"_\" + \"examples.txt\") as examples:\n",
        "        for example in examples:\n",
        "            example = example[:-1].lower().split(\"\\t\")\n",
        "            if example[0] != decontracted(example[0]):\n",
        "                text = re.sub(example[0], \"\\$\", example[2])\n",
        "                text = decontracted(text)\n",
        "                text = re.sub(r\"\\$\", example[0], text)\n",
        "                example[2] = text\n",
        "            else:\n",
        "                example[2] = decontracted(example[2])\n",
        "            dataset.append(example)\n",
        "\n",
        "    with open(pref + \"_\" + \"hypernyms.txt\") as hypernyms:\n",
        "        for index, line in enumerate(hypernyms):\n",
        "            line = line[:-1].lower().split(\"\\t\")\n",
        "            line = \" ; \".join(line)\n",
        "            line = re.sub(r'_', ' ', line)\n",
        "            dataset[index].append(line)\n",
        "\n",
        "    with open(pref + \"_\" + \"definitions.txt\") as definitions:\n",
        "        for index, line in enumerate(definitions):\n",
        "            line = line[:-1].lower().split(\";\")\n",
        "            for i in range(len(line)):\n",
        "                line[i] = line[i].strip()\n",
        "                if dataset[index][0] != decontracted(dataset[index][0]):\n",
        "                    text = re.sub(dataset[index][0], \"\\$\", line[i])\n",
        "                    text = decontracted(text)\n",
        "                    text = re.sub(r\"\\$\", dataset[index][0], text)\n",
        "                    line[i] = text\n",
        "                else:\n",
        "                    line[i] = decontracted(line[i])\n",
        "            line = \" ; \".join(line)\n",
        "            dataset[index].append(line)\n",
        "\n",
        "    cols = [\"target\", \"position\", \"sentence\", \"hypernym\", \"definition\"]\n",
        "\n",
        "    if not testing:\n",
        "        cols.append(\"label\")\n",
        "        with open(pref + \"_\" + \"labels.txt\") as labels:\n",
        "            for index, line in enumerate(labels):\n",
        "                line = line[:-1]\n",
        "                dataset[index].append(line)\n",
        "\n",
        "    df = pd.DataFrame(dataset, columns=cols)\n",
        "\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "nUBIiXAck3yc"
      },
      "outputs": [],
      "source": [
        "dev_df = get_df(\"data/Development/dev\")\n",
        "train_df = get_df(\"data/Training/train\")\n",
        "test_df = get_df(\"data/Test/test\", True)\n",
        "\n",
        "dev_df.to_csv(\"processed_data/dev.csv\", index=None)\n",
        "train_df.to_csv(\"processed_data/train.csv\", index=None)\n",
        "test_df.to_csv(\"processed_data/test.csv\", index=None)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "IbUPA23KOA9d"
      },
      "outputs": [],
      "source": [
        "def augment(filename):\n",
        "\n",
        "    df = pd.read_csv(filename)\n",
        "\n",
        "    n = df.shape[0]\n",
        "\n",
        "    for i in range(n):\n",
        "        row = df.iloc[i]\n",
        "        if row['label'] == 'T':\n",
        "            if not isinstance(row['hypernym'], str):\n",
        "                continue\n",
        "            position = row['position']\n",
        "            sentence = row['sentence']\n",
        "            hypernyms = row['hypernym'].split(' ; ')\n",
        "            target = row['target']\n",
        "            definition = row['definition']\n",
        "            label = row['label']\n",
        "            for hypernym in hypernyms:\n",
        "                new_row = {'target': hypernym, 'position': position, 'sentence': sentence.replace(target, hypernym), \n",
        "                            'hypernym': target, 'definition': definition, 'label': label}\n",
        "                df = df.append(new_row, ignore_index = True)\n",
        "        \n",
        "        while True:\n",
        "            random_row = df.iloc[random.randint(0, n - 1)]\n",
        "            if random_row['target'] != row['target']:\n",
        "                position = row['position']\n",
        "                sentence = row['sentence']\n",
        "                target = row['target']\n",
        "                hypernyms = random_row['hypernym']\n",
        "                definition = random_row['definition']\n",
        "                label = 'F'\n",
        "                new_row = {'target': target, 'position': position, 'sentence': sentence, \n",
        "                                'hypernym': hypernym, 'definition': definition, 'label': label}\n",
        "                df = df.append(new_row, ignore_index = True)\n",
        "                break\n",
        "\n",
        "    df = shuffle(df)\n",
        "    df.reset_index(drop = True, inplace = True)\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "l9TI4QryODhs"
      },
      "outputs": [],
      "source": [
        "aug_train_df = augment('processed_data/train.csv')\n",
        "aug_train_df.to_csv('processed_data/train_aug.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "osPsHkWvIDE8"
      },
      "outputs": [],
      "source": [
        "import tensorflow_text\n",
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from keras import backend as K\n",
        "\n",
        "import transformers\n",
        "transformers.logging.set_verbosity_error()\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from tensorflow.keras.optimizers import Adam"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "mjKWllvTIFvZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "2e2fd4e9566c4508aea0ac48536b7570",
            "5cd57bcf5ab649c5b22404b45be76f7a",
            "ff1caf073387443cbe612467ad8350fa",
            "2193d61c9e2c47919cb7f825540794b2",
            "6e23586bdc0e4e0e9db7b6190c408b3d",
            "1234ea9ca7ba4de28985629840b47e06",
            "e8b838080aa346d58d22ffd6ebfdbb5a",
            "049d7e99c92e4f179fc4b6a4abf353a8",
            "fec1e3e0868e42e584eaae7a395dc761",
            "b8d6e5f1f895491eb6f7fefd687de48d",
            "9e46e43b296046b78f93986021c95521",
            "ac49cba76ff5490ab98ca8db04dbbe75",
            "2ae08d89625e47148e839d0eb91caf8a",
            "ed273e2ef60a48ed8a2f7b4cd19634e6",
            "8a527ec5c6014edc999303ae2df2cf27",
            "3ee126274d914047abb60a32aa5e0779",
            "a02bbe01178144139273ad7b58cabb71",
            "cd053be3c08644d3a90fc62ab7ab1046",
            "430563f8d2c04fe899b039b6c5daa854",
            "ea8567fad22446c7a494889cb2ac8abd",
            "1989f73f21f34b2fa9d7ae5190ade1d1",
            "185125e4ed254feda9cb37f4348daf71",
            "7c9f943216244a8593f858fab846aa81",
            "3befc69fce2b431a8a1ef2c0bbd8fc73",
            "9536b95f90c54e4ca30ebfb43e43c0bc",
            "d9d1618136454011a7d12b15f7752e51",
            "037cf52c3648480aa4fd1a96e8b8dfcb",
            "93e2fae0af0445129dab1557fc3ac946",
            "523170cbfab54a4896a2847df6c61b2e",
            "b4a6179d875a4933b8ca9957b441027c",
            "414b26a611e04769b159bf66272f5499",
            "5dc5b6f4a6224343aab52bef1b55d63f",
            "5c437d7974534d52a4ee8ffd6e464f32"
          ]
        },
        "outputId": "d5a1a75c-9d71-43d6-80f8-930b3c1638bf"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2e2fd4e9566c4508aea0ac48536b7570"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ac49cba76ff5490ab98ca8db04dbbe75"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7c9f943216244a8593f858fab846aa81"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "transformers.logging.set_verbosity_error()\n",
        "\n",
        "def extract_data(filename, test_data = False):\n",
        "\n",
        "    df = pd.read_csv(filename)\n",
        "    X = {'input_type_ids':[],'input_word_ids':[],'input_mask':[], 'mask1': [], 'mask2': []}\n",
        "    Y = []\n",
        "\n",
        "    for _, row in df.iterrows():\n",
        "\n",
        "        word = row['target']\n",
        "        sentence = row['sentence']\n",
        "        hypernyms = row['hypernym']\n",
        "        definitions = row['definition']\n",
        "\n",
        "        if not test_data:\n",
        "          if row['label'] == 'T':\n",
        "              label = 1.0\n",
        "          else:\n",
        "              label = 0.0\n",
        "\n",
        "        desc = [definitions]\n",
        "\n",
        "        if isinstance(hypernyms, str):\n",
        "          desc.append(hypernyms)\n",
        "        \n",
        "        desc = ' ; '.join(desc)\n",
        "\n",
        "        sentence = sentence.replace(word.lower(), '$ ' + word + ' $')\n",
        "\n",
        "        x = tokenizer(sentence, desc, max_length = 128, padding='max_length', truncation = True)\n",
        "\n",
        "        sep1 = x['input_ids'].index(102)\n",
        "        sep2 = x['input_ids'].index(102, sep1 + 1)\n",
        "\n",
        "        x['mask1'] = np.zeros((128))\n",
        "        x['mask2'] = np.zeros((128))\n",
        "\n",
        "        dol1 = x['input_ids'].index(1002)\n",
        "        dol2 = x['input_ids'].index(1002, dol1 + 1)\n",
        "\n",
        "        for i in range(1, sep1):\n",
        "            x['mask1'][i] = 1.0\n",
        "        \n",
        "        for i in range(sep1 + 1, sep2):\n",
        "            x['mask2'][i] = 1.0\n",
        "\n",
        "        for i in range(dol1, dol2 + 1):\n",
        "            x['token_type_ids'][i] = 1.0\n",
        "            x['mask2'][i] = 1.0\n",
        "\n",
        "        X['input_type_ids'].append(x['token_type_ids'])\n",
        "        X['input_word_ids'].append(x['input_ids'])\n",
        "        X['input_mask'].append(x['attention_mask'])\n",
        "        X['mask1'].append(x['mask1'])\n",
        "        X['mask2'].append(x['mask2'])\n",
        "\n",
        "        if not test_data:\n",
        "          Y.append(label)     \n",
        "\n",
        "    X['input_type_ids'] = np.array(X['input_type_ids'], dtype=np.float32)\n",
        "    X['input_word_ids'] = np.array(X['input_word_ids'], dtype=np.float32)\n",
        "    X['input_mask'] = np.array(X['input_mask'], dtype=np.float32)\n",
        "\n",
        "    X['mask1'] = np.array(X['mask1'], dtype=np.float32)\n",
        "    X['mask2'] = np.array(X['mask2'], dtype=np.float32)\n",
        "\n",
        "    Y = np.array(Y)\n",
        "\n",
        "    return X, Y\n",
        "\n",
        "X,Y = extract_data('processed_data/train_aug.csv')\n",
        "Xval, Yval = extract_data('processed_data/dev.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TmJ4l6L2YIz7",
        "outputId": "ee61b1d1-2918-4286-de00-a5c5d3b2b4eb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_word_ids (InputLayer)    [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " input_mask (InputLayer)        [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " input_type_ids (InputLayer)    [(None, 128)]        0           []                               \n",
            "                                                                                                  \n",
            " input_1 (InputLayer)           [(None, 128, 1)]     0           []                               \n",
            "                                                                                                  \n",
            " bert-encoder (KerasLayer)      [(None, 768),        109482241   ['input_word_ids[0][0]',         \n",
            "                                 (None, 128, 768)]                'input_mask[0][0]',             \n",
            "                                                                  'input_type_ids[0][0]']         \n",
            "                                                                                                  \n",
            " input_2 (InputLayer)           [(None, 128, 1)]     0           []                               \n",
            "                                                                                                  \n",
            " tf.math.multiply (TFOpLambda)  (None, 128, 768)     0           ['input_1[0][0]',                \n",
            "                                                                  'bert-encoder[1][1]']           \n",
            "                                                                                                  \n",
            " tf.math.multiply_1 (TFOpLambda  (None, 128, 768)    0           ['input_2[0][0]',                \n",
            " )                                                                'bert-encoder[1][1]']           \n",
            "                                                                                                  \n",
            " masking (Masking)              (None, 128, 768)     0           ['tf.math.multiply[1][0]']       \n",
            "                                                                                                  \n",
            " masking_1 (Masking)            (None, 128, 768)     0           ['tf.math.multiply_1[1][0]']     \n",
            "                                                                                                  \n",
            " lstm (LSTM)                    (None, 32)           102528      ['masking[1][0]']                \n",
            "                                                                                                  \n",
            " lstm_1 (LSTM)                  (None, 32)           102528      ['masking_1[1][0]']              \n",
            "                                                                                                  \n",
            " dense (Dense)                  (None, 32)           24608       ['bert-encoder[1][0]']           \n",
            "                                                                                                  \n",
            " concatenate (Concatenate)      (None, 96)           0           ['lstm[1][0]',                   \n",
            "                                                                  'lstm_1[1][0]',                 \n",
            "                                                                  'dense[1][0]']                  \n",
            "                                                                                                  \n",
            " dropout (Dropout)              (None, 96)           0           ['concatenate[1][0]']            \n",
            "                                                                                                  \n",
            " output (Dense)                 (None, 1)            97          ['dropout[1][0]']                \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 109,712,002\n",
            "Trainable params: 109,712,001\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "# Input layers -----------------------------\n",
        "\n",
        "input_type_ids = tf.keras.layers.Input(shape = (128), dtype = tf.int32, name = \"input_type_ids\")\n",
        "input_word_ids = tf.keras.layers.Input(shape = (128), dtype = tf.int32, name = \"input_word_ids\")\n",
        "input_mask = tf.keras.layers.Input(shape = (128), dtype = tf.int32, name = \"input_mask\")\n",
        "\n",
        "input_sent_mask = tf.keras.layers.Input(shape=(128), dtype = tf.float32, name = \"input_sent_mask\")\n",
        "input_desc_mask = tf.keras.layers.Input(shape=(128), dtype = tf.float32, name = \"input_desc_mask\")\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "\n",
        "# Encoder -----------------------------------\n",
        "\n",
        "encoder = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2\", name = \"bert-encoder\", trainable = True)\n",
        "pooled_output, sequence_output = encoder([input_word_ids, input_mask, input_type_ids])\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "\n",
        "# Decoder -----------------------------------\n",
        "\n",
        "input_sent_mask = tf.expand_dims(input_sent_mask, axis = 2)\n",
        "sent_output = tf.multiply(input_sent_mask, sequence_output)\n",
        "sent_output = tf.keras.layers.Masking(mask_value = 0)(sent_output)\n",
        "sent_output = tf.keras.layers.LSTM(32, activation = 'relu')(sent_output)\n",
        "\n",
        "input_desc_mask = tf.expand_dims(input_desc_mask, axis = 2)\n",
        "desc_output = tf.multiply(input_desc_mask, sequence_output)\n",
        "desc_output = tf.keras.layers.Masking(mask_value = 0)(desc_output)\n",
        "desc_output = tf.keras.layers.LSTM(32, activation = 'relu')(desc_output)\n",
        "\n",
        "pooled_output = tf.keras.layers.Dense(32, activation = 'relu')(pooled_output)\n",
        "\n",
        "concat = tf.keras.layers.Concatenate(name = \"concatenate\")([sent_output, desc_output, pooled_output])\n",
        "concat = tf.keras.layers.Dropout(0.3)(concat)\n",
        "output = tf.keras.layers.Dense(1, activation = 'sigmoid', name = \"output\")(concat)\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "# Encoder-Decoder Model ---------------------\n",
        "\n",
        "model = tf.keras.models.Model(\n",
        "      inputs = [\n",
        "        input_word_ids,\n",
        "        input_mask,\n",
        "        input_type_ids,\n",
        "        input_sent_mask,\n",
        "        input_desc_mask], \n",
        "      outputs = output)\n",
        "\n",
        "model.compile(optimizer=Adam(learning_rate = 5e-6), loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n",
        "\n",
        "# -------------------------------------------\n",
        "\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "GCEGsTN7lXpT"
      },
      "outputs": [],
      "source": [
        "def find_optimal_threshold(target, predicted):\n",
        "    fpr, tpr, thresholds = metrics.roc_curve(target, predicted)\n",
        "    fpr = fpr[1:]\n",
        "    tpr = tpr[1:]\n",
        "    thresholds = thresholds[1:]\n",
        "    x = tpr - fpr\n",
        "    idx = np.argmax(x)\n",
        "    return thresholds[idx]\n",
        "\n",
        "class roc_auc_threshold(tf.keras.callbacks.Callback):\n",
        "\n",
        "    def __init__(self, training_data, validation_data):\n",
        "        self.x = training_data[0]\n",
        "        self.y = training_data[1]\n",
        "        self.x_val = validation_data[0]\n",
        "        self.y_val = validation_data[1]\n",
        "\n",
        "    def on_train_begin(self, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_train_end(self, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_epoch_begin(self, epoch, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_epoch_end(self, epoch, logs={}):\n",
        "        model.save('model.' + str(epoch) + '.h5')\n",
        "        y_pred_val = self.model.predict(self.x_val)\n",
        "        optimal_threshold = find_optimal_threshold(self.y_val, y_pred_val)\n",
        "        y_pred_val = y_pred_val.flatten()\n",
        "        y_truth_val = self.y_val.flatten()\n",
        "        val_correct = 0\n",
        "        val_total = len(y_pred_val)\n",
        "        for i in range(len(y_pred_val)):\n",
        "          pred = 0\n",
        "          if y_pred_val[i] >= optimal_threshold:\n",
        "            pred = 1\n",
        "          if pred == y_truth_val[i]:\n",
        "            val_correct += 1\n",
        "        print(' optimal_threshold:', round(optimal_threshold, 5), '-',  \n",
        "              'val_accuracy_optimal:', round(val_correct / val_total, 5))\n",
        "        return\n",
        "\n",
        "    def on_batch_begin(self, batch, logs={}):\n",
        "        return\n",
        "\n",
        "    def on_batch_end(self, batch, logs={}):\n",
        "        return\n",
        "\n",
        "cb = roc_auc_threshold(training_data = ([X['input_word_ids'], X['input_mask'], X['input_type_ids'], X['mask1'], X['mask2']], Y),\n",
        "                  validation_data = ([Xval['input_word_ids'], Xval['input_mask'], Xval['input_type_ids'], Xval['mask1'], Xval['mask2']], Yval))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ySbknbfzYmVf",
        "outputId": "a6ef2531-26f0-4633-d4b8-6f3c143c8baa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/4\n",
            "433/433 [==============================] - ETA: 0s - loss: 0.5308 - accuracy: 0.7238 optimal_threshold: 0.74596 - val_accuracy_optimal: 0.70951\n",
            "433/433 [==============================] - 741s 2s/step - loss: 0.5308 - accuracy: 0.7238 - val_loss: 0.6563 - val_accuracy: 0.6735\n",
            "Epoch 2/4\n",
            "433/433 [==============================] - ETA: 0s - loss: 0.3161 - accuracy: 0.8682 optimal_threshold: 0.7198 - val_accuracy_optimal: 0.74293\n",
            "433/433 [==============================] - 713s 2s/step - loss: 0.3161 - accuracy: 0.8682 - val_loss: 0.7045 - val_accuracy: 0.7095\n",
            "Epoch 3/4\n",
            "433/433 [==============================] - ETA: 0s - loss: 0.1837 - accuracy: 0.9339 optimal_threshold: 0.60756 - val_accuracy_optimal: 0.73008\n",
            "433/433 [==============================] - 717s 2s/step - loss: 0.1837 - accuracy: 0.9339 - val_loss: 0.8076 - val_accuracy: 0.7121\n",
            "Epoch 4/4\n",
            "433/433 [==============================] - ETA: 0s - loss: 0.1120 - accuracy: 0.9614 optimal_threshold: 0.92431 - val_accuracy_optimal: 0.73779\n",
            "433/433 [==============================] - 719s 2s/step - loss: 0.1120 - accuracy: 0.9614 - val_loss: 1.0542 - val_accuracy: 0.7095\n"
          ]
        }
      ],
      "source": [
        "history = model.fit([X['input_word_ids'], X['input_mask'], X['input_type_ids'], X['mask1'], X['mask2']], Y, \n",
        "          epochs = 4, batch_size = 16, callbacks=[cb], verbose = 1,\n",
        "          validation_data = ([Xval['input_word_ids'], Xval['input_mask'], Xval['input_type_ids'], Xval['mask1'], Xval['mask2']], Yval))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "ckgwg9lXO2X4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ce563f94-50ad-4646-9aba-a90de9c5b278"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:tensorflow:Layer lstm_1 will not use cuDNN kernels since it doesn't meet the criteria. It will use a generic GPU kernel as fallback when running on GPU.\n"
          ]
        }
      ],
      "source": [
        "epoch = 1\n",
        "pretrained_model = tf.keras.models.load_model('model.' + str(epoch) + '.h5', custom_objects={'KerasLayer': hub.KerasLayer})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "rA6k-oGg66nU"
      },
      "outputs": [],
      "source": [
        "Xtest, Ytest = extract_data('processed_data/test.csv', True)\n",
        "y_predicted = pretrained_model.predict([Xtest['input_word_ids'], Xtest['input_mask'], Xtest['input_type_ids'], Xtest['mask1'], Xtest['mask2']])\n",
        "y_predicted = y_predicted.flatten()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "optimal_threshold = 0.7198\n",
        "\n",
        "file = open('all_output.txt', 'w')\n",
        "for val in y_predicted:\n",
        "  if val < optimal_threshold:\n",
        "    print('F',file=file)\n",
        "  else:\n",
        "    print('T',file=file)\n",
        "file.close()"
      ],
      "metadata": {
        "id": "4cVE5umM-mAg"
      },
      "execution_count": 22,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "BERT + LSTM Decoder + AUG + WE + ROC.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.7"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "2e2fd4e9566c4508aea0ac48536b7570": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5cd57bcf5ab649c5b22404b45be76f7a",
              "IPY_MODEL_ff1caf073387443cbe612467ad8350fa",
              "IPY_MODEL_2193d61c9e2c47919cb7f825540794b2"
            ],
            "layout": "IPY_MODEL_6e23586bdc0e4e0e9db7b6190c408b3d"
          }
        },
        "5cd57bcf5ab649c5b22404b45be76f7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1234ea9ca7ba4de28985629840b47e06",
            "placeholder": "​",
            "style": "IPY_MODEL_e8b838080aa346d58d22ffd6ebfdbb5a",
            "value": "Downloading: 100%"
          }
        },
        "ff1caf073387443cbe612467ad8350fa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_049d7e99c92e4f179fc4b6a4abf353a8",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fec1e3e0868e42e584eaae7a395dc761",
            "value": 231508
          }
        },
        "2193d61c9e2c47919cb7f825540794b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b8d6e5f1f895491eb6f7fefd687de48d",
            "placeholder": "​",
            "style": "IPY_MODEL_9e46e43b296046b78f93986021c95521",
            "value": " 226k/226k [00:00&lt;00:00, 327kB/s]"
          }
        },
        "6e23586bdc0e4e0e9db7b6190c408b3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1234ea9ca7ba4de28985629840b47e06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e8b838080aa346d58d22ffd6ebfdbb5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "049d7e99c92e4f179fc4b6a4abf353a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fec1e3e0868e42e584eaae7a395dc761": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b8d6e5f1f895491eb6f7fefd687de48d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e46e43b296046b78f93986021c95521": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ac49cba76ff5490ab98ca8db04dbbe75": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2ae08d89625e47148e839d0eb91caf8a",
              "IPY_MODEL_ed273e2ef60a48ed8a2f7b4cd19634e6",
              "IPY_MODEL_8a527ec5c6014edc999303ae2df2cf27"
            ],
            "layout": "IPY_MODEL_3ee126274d914047abb60a32aa5e0779"
          }
        },
        "2ae08d89625e47148e839d0eb91caf8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a02bbe01178144139273ad7b58cabb71",
            "placeholder": "​",
            "style": "IPY_MODEL_cd053be3c08644d3a90fc62ab7ab1046",
            "value": "Downloading: 100%"
          }
        },
        "ed273e2ef60a48ed8a2f7b4cd19634e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_430563f8d2c04fe899b039b6c5daa854",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ea8567fad22446c7a494889cb2ac8abd",
            "value": 28
          }
        },
        "8a527ec5c6014edc999303ae2df2cf27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1989f73f21f34b2fa9d7ae5190ade1d1",
            "placeholder": "​",
            "style": "IPY_MODEL_185125e4ed254feda9cb37f4348daf71",
            "value": " 28.0/28.0 [00:00&lt;00:00, 486B/s]"
          }
        },
        "3ee126274d914047abb60a32aa5e0779": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a02bbe01178144139273ad7b58cabb71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd053be3c08644d3a90fc62ab7ab1046": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "430563f8d2c04fe899b039b6c5daa854": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ea8567fad22446c7a494889cb2ac8abd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1989f73f21f34b2fa9d7ae5190ade1d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "185125e4ed254feda9cb37f4348daf71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7c9f943216244a8593f858fab846aa81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3befc69fce2b431a8a1ef2c0bbd8fc73",
              "IPY_MODEL_9536b95f90c54e4ca30ebfb43e43c0bc",
              "IPY_MODEL_d9d1618136454011a7d12b15f7752e51"
            ],
            "layout": "IPY_MODEL_037cf52c3648480aa4fd1a96e8b8dfcb"
          }
        },
        "3befc69fce2b431a8a1ef2c0bbd8fc73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_93e2fae0af0445129dab1557fc3ac946",
            "placeholder": "​",
            "style": "IPY_MODEL_523170cbfab54a4896a2847df6c61b2e",
            "value": "Downloading: 100%"
          }
        },
        "9536b95f90c54e4ca30ebfb43e43c0bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b4a6179d875a4933b8ca9957b441027c",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_414b26a611e04769b159bf66272f5499",
            "value": 570
          }
        },
        "d9d1618136454011a7d12b15f7752e51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5dc5b6f4a6224343aab52bef1b55d63f",
            "placeholder": "​",
            "style": "IPY_MODEL_5c437d7974534d52a4ee8ffd6e464f32",
            "value": " 570/570 [00:00&lt;00:00, 15.2kB/s]"
          }
        },
        "037cf52c3648480aa4fd1a96e8b8dfcb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93e2fae0af0445129dab1557fc3ac946": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "523170cbfab54a4896a2847df6c61b2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b4a6179d875a4933b8ca9957b441027c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "414b26a611e04769b159bf66272f5499": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5dc5b6f4a6224343aab52bef1b55d63f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c437d7974534d52a4ee8ffd6e464f32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}