fast_tokenizers.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/loganathanspr/nlp_course/blob/main/fast_tokenizers.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ptlljUd3-x8b"
      },
      "source": [
        "# Fast tokenizers' special powers (PyTorch)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FsZfShzu-x8c"
      },
      "source": [
        "Install the Transformers, Datasets, and Evaluate libraries to run this notebook."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Qba0c3Th-x8d",
        "outputId": "545e65b7-f717-4325-994e-33f3c008b909",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-2.14.5-py3-none-any.whl (519 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/519.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━\u001b[0m \u001b[32m481.3/519.6 kB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.6/519.6 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting evaluate\n",
            "  Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 kB\u001b[0m \u001b[31m12.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting transformers[sentencepiece]\n",
            "  Downloading transformers-4.34.0-py3-none-any.whl (7.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.7/7.7 MB\u001b[0m \u001b[31m76.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
            "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n",
            "Collecting xxhash (from datasets)\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<2023.9.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.8.5)\n",
            "Collecting huggingface-hub<1.0.0,>=0.14.0 (from datasets)\n",
            "  Downloading huggingface_hub-0.17.3-py3-none-any.whl (295 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.0/295.0 kB\u001b[0m \u001b[31m34.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Collecting responses<0.19 (from evaluate)\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (3.12.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (2023.6.3)\n",
            "Collecting tokenizers<0.15,>=0.14 (from transformers[sentencepiece])\n",
            "  Downloading tokenizers-0.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.8/3.8 MB\u001b[0m \u001b[31m37.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers[sentencepiece])\n",
            "  Downloading safetensors-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m39.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentencepiece!=0.1.92,>=0.1.91 (from transformers[sentencepiece])\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m35.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (3.20.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (3.3.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.6)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.7.22)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: sentencepiece, xxhash, safetensors, dill, responses, multiprocess, huggingface-hub, tokenizers, transformers, datasets, evaluate\n",
            "Successfully installed datasets-2.14.5 dill-0.3.7 evaluate-0.4.0 huggingface-hub-0.17.3 multiprocess-0.70.15 responses-0.18.0 safetensors-0.4.0 sentencepiece-0.1.99 tokenizers-0.14.1 transformers-4.34.0 xxhash-3.4.1\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets evaluate transformers[sentencepiece]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "TGJVLpIc-x8d",
        "outputId": "c8968c3e-4d85-410f-84f1-3b4b2867674d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163,
          "referenced_widgets": [
            "ef339423654947fba33161c595eb1894",
            "980041176c214b8d96fb6cfd1f5e267d",
            "1cd5e52a8314469cb3d71b89b5436395",
            "b6da43646c86413f9a660a2329c7406e",
            "5d96b454670e41a28b1a1c1e97a23843",
            "e3f5a3f1946d480084b4ba83bcc13c0e",
            "0b1e28e935544d79b355ca9374cebcd6",
            "780702ad0fe34eb3bba0a0c73d69e5ec",
            "689b60a2bcaa40f0968503e09dc3774b",
            "c9f76b1354eb4472bf05194d3a6ff9fa",
            "45fd0282950e4594acaa3a1a9e966c37",
            "0b0848a2100842f380fd3f0a15a5416b",
            "a96ff4d8ca134c11b6b6e9d12202f611",
            "456030c7b5724e3ab94103f63f4d87c2",
            "38e228d6c0d941fba03cee268271e235",
            "8b63156fd203405b835d4c93b2d123dc",
            "66d5e54e0146431381bbf424fd0ee520",
            "58fe65184f9e47e5a0d82b7877286ae4",
            "cfabfe88919f483183de5f3f208574be",
            "461386c519b848c18694dd7edb4fd78c",
            "400b479345514bb09702d14a9fd46fa4",
            "2cfd2cc255da427ab80d1eba18a90d21",
            "2dc3ec84c2db4926b30dc5881e2ba8a4",
            "3b29713d6a2c428bb824320d92ddff2c",
            "fbf7a38cdf6e4243b248737114fc4362",
            "61622b3d66d6475894a795e027fb58df",
            "ba5e72ccf62049e9b896cf6d97089240",
            "88f1c6aea3d4411c8bed136e20a9d292",
            "206dc010e34446b48fd1664c33df9a4f",
            "11b1d70923a04ce890c58d7192b05467",
            "379e75f42ade497ebbf50fb5af0f7c5d",
            "774a3055677f475a9340e4e266aee832",
            "28c84798d8f74897a6d1f61d4a70bdf1",
            "700b0a8f4f5d4daea471eabfc299d9c6",
            "e34d578312724fef85af7214b9b27c3c",
            "fdcab6d63a5f486090141c882e2dc257",
            "20d0523cca1741879ecf1502dcce0570",
            "82d7bc8303f744d2a7a8957227569f1b",
            "b98d30c67ac8463c81e283b3a10eb272",
            "2e61db326e9f43f89911f20554f41a76",
            "6dbdaf5f378a433b9ee11a6490de51de",
            "40e8975f9e294502a08d293045810ba0",
            "573b92e9cf3e41ea94bf13e1ac7c6121",
            "56adc1267b6542f0824c497da1433633"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ef339423654947fba33161c595eb1894"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0b0848a2100842f380fd3f0a15a5416b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/213k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2dc3ec84c2db4926b30dc5881e2ba8a4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/436k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "700b0a8f4f5d4daea471eabfc299d9c6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'transformers.tokenization_utils_base.BatchEncoding'>\n"
          ]
        }
      ],
      "source": [
        "from transformers import AutoTokenizer\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")\n",
        "example = \"My name is Sylvain and I work at Hugging Face in Brooklyn.\"\n",
        "encoding = tokenizer(example)\n",
        "print(type(encoding))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoding"
      ],
      "metadata": {
        "id": "v3BDk7zNEl_8",
        "outputId": "f5865689-e10c-40b7-c3be-dcfb4778a780",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [101, 1422, 1271, 1110, 156, 7777, 2497, 1394, 1105, 146, 1250, 1120, 20164, 10932, 10289, 1107, 6010, 119, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "8JZCEewx-x8e",
        "outputId": "86efff45-26f4-43b9-f267-1671f188688d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "tokenizer.is_fast"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YUTOBk_E-x8e",
        "outputId": "b6c5c305-6e30-4258-aa47-e5e857bb90d5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "encoding.is_fast"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ap9A3Vgy-x8f",
        "outputId": "7a63a775-d0ce-417e-90e2-5098403de9b9"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['[CLS]', 'My', 'name', 'is', 'S', '##yl', '##va', '##in', 'and', 'I', 'work', 'at', 'Hu', '##gging', 'Face', 'in',\n",
              " 'Brooklyn', '.', '[SEP]']"
            ]
          },
          "execution_count": null,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "encoding.tokens()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "oMA8ENgh-x8f",
        "outputId": "63f4c683-29cb-43b5-8e73-fb47ada79cb2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[None, 0, 1, 2, 3, 3, 3, 3, 4, 5, 6, 7, 8, 8, 9, 10, 11, 12, None]"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "encoding.word_ids()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Test"
      ],
      "metadata": {
        "id": "nw7rA-MfILqZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "roberta_tokenizer = AutoTokenizer.from_pretrained(\"roberta-base\")\n",
        "encoding_roberta = roberta_tokenizer(\"80s\")\n",
        "encoding_roberta"
      ],
      "metadata": {
        "id": "8eC8h8hbIKT0",
        "outputId": "06a929a2-cdce-464a-bfa6-fc86355e3a9f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [0, 2940, 29, 2], 'attention_mask': [1, 1, 1, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoding_roberta.tokens()"
      ],
      "metadata": {
        "id": "Gbp7MLupI2Zn",
        "outputId": "30e46f75-6c63-4800-c119-01bed05022e4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['<s>', '80', 's', '</s>']"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer(\"80s\").tokens()"
      ],
      "metadata": {
        "id": "ae7p3OZjI65g",
        "outputId": "a66e3f2e-8605-44c5-eccd-f7ff46bdc5b5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[CLS]', '80s', '[SEP]']"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "t6Kr-7YP-x8f",
        "outputId": "0c889697-3b24-4e40-ad4d-35f6cbf3ea8a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Sylvain'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "start, end = encoding.word_to_chars(3)\n",
        "example[start:end]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Test"
      ],
      "metadata": {
        "id": "PFdG0wsTLAlP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "texts = [\"Today is Monday\", \"Next month is November .\"]\n",
        "encoding_texts = roberta_tokenizer(texts)\n",
        "encoding_texts"
      ],
      "metadata": {
        "id": "BVSIUo57K_oA",
        "outputId": "335d04d9-0914-4af9-de79-a90ecaeaa48c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [[0, 5625, 16, 302, 2], [0, 19192, 353, 16, 759, 479, 2]], 'attention_mask': [[1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1]]}"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoding_texts.word_ids()"
      ],
      "metadata": {
        "id": "cp_z_aiiLZOY",
        "outputId": "cb77ff33-34fd-4263-b9a4-3506abc72bf4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[None, 0, 1, 2, None]"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoding_texts.word_ids(1)"
      ],
      "metadata": {
        "id": "RgCNr533MK5u",
        "outputId": "0902962d-512d-4c34-9cc4-6db03a9f116b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[None, 0, 1, 2, 3, 4, None]"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# word to chars\n",
        "start, end = encoding_texts[1].word_to_chars(1)\n",
        "texts[1][start:end]"
      ],
      "metadata": {
        "id": "RprEfB7zMSZt",
        "outputId": "21bf6205-f5b1-40ca-ffe2-9bfb6365c6a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'month'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# tokens to chars\n",
        "start, end = encoding_texts[1].token_to_chars(1)\n",
        "texts[1][start:end]"
      ],
      "metadata": {
        "id": "6DL5ZNP1PeY8",
        "outputId": "75391adc-9ea1-4c6e-a463-4c88f61320ea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Next'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# char to word\n",
        "w = encoding_texts[1].char_to_word(0)\n",
        "start, end = encoding_texts[1].word_to_chars(w)\n",
        "texts[1][start:end]"
      ],
      "metadata": {
        "id": "Ikfer793P09f",
        "outputId": "90fa77b4-ba30-4678-d8c0-595b5dd05e07",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Next'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "[enc for enc in encoding_texts.tokens()]"
      ],
      "metadata": {
        "id": "jfRiEnU7THyY",
        "outputId": "a2cb4cad-ff59-4e27-8d36-c0faeff5a65e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['<s>', 'Today', 'Ġis', 'ĠMonday', '</s>']"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "metadata": {
        "id": "Zl7c9l74-x8f",
        "outputId": "f21994b2-2928-46c8-fabf-f8cb59841940",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No model was supplied, defaulted to dbmdz/bert-large-cased-finetuned-conll03-english and revision f2482bf (https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english).\n",
            "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
            "Some weights of the model checkpoint at dbmdz/bert-large-cased-finetuned-conll03-english were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ],
      "source": [
        "from transformers import pipeline\n",
        "token_classifier = pipeline(\"token-classification\")\n",
        "output = token_classifier(\"My name is Loganathan Ramasamy. I live in New York.\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "output"
      ],
      "metadata": {
        "id": "PdwvW3SMfJQX",
        "outputId": "f2d32078-474a-4f8b-c1c3-9048bd60d514",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'entity': 'I-PER',\n",
              "  'score': 0.9973621,\n",
              "  'index': 4,\n",
              "  'word': 'Logan',\n",
              "  'start': 11,\n",
              "  'end': 16},\n",
              " {'entity': 'I-PER',\n",
              "  'score': 0.99289787,\n",
              "  'index': 5,\n",
              "  'word': '##ath',\n",
              "  'start': 16,\n",
              "  'end': 19},\n",
              " {'entity': 'I-PER',\n",
              "  'score': 0.9914608,\n",
              "  'index': 6,\n",
              "  'word': '##an',\n",
              "  'start': 19,\n",
              "  'end': 21},\n",
              " {'entity': 'I-PER',\n",
              "  'score': 0.9992663,\n",
              "  'index': 7,\n",
              "  'word': 'Rama',\n",
              "  'start': 22,\n",
              "  'end': 26},\n",
              " {'entity': 'I-PER',\n",
              "  'score': 0.9945457,\n",
              "  'index': 8,\n",
              "  'word': '##sa',\n",
              "  'start': 26,\n",
              "  'end': 28},\n",
              " {'entity': 'I-PER',\n",
              "  'score': 0.9875486,\n",
              "  'index': 9,\n",
              "  'word': '##my',\n",
              "  'start': 28,\n",
              "  'end': 30},\n",
              " {'entity': 'I-LOC',\n",
              "  'score': 0.99931014,\n",
              "  'index': 14,\n",
              "  'word': 'New',\n",
              "  'start': 42,\n",
              "  'end': 45},\n",
              " {'entity': 'I-LOC',\n",
              "  'score': 0.9988575,\n",
              "  'index': 15,\n",
              "  'word': 'York',\n",
              "  'start': 46,\n",
              "  'end': 50}]"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "metadata": {
        "id": "tdzH1ifq-x8f",
        "outputId": "69bb2276-bdcb-452e-d7bf-5878d31ed2e7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No model was supplied, defaulted to dbmdz/bert-large-cased-finetuned-conll03-english and revision f2482bf (https://huggingface.co/dbmdz/bert-large-cased-finetuned-conll03-english).\n",
            "Using a pipeline without specifying a model name and revision in production is not recommended.\n",
            "Some weights of the model checkpoint at dbmdz/bert-large-cased-finetuned-conll03-english were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'entity_group': 'PER',\n",
              "  'score': 0.9938469,\n",
              "  'word': 'Loganathan Ramasamy',\n",
              "  'start': 11,\n",
              "  'end': 30},\n",
              " {'entity_group': 'LOC',\n",
              "  'score': 0.9990838,\n",
              "  'word': 'New York',\n",
              "  'start': 42,\n",
              "  'end': 50}]"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ],
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "token_classifier = pipeline(\"token-classification\", aggregation_strategy=\"simple\")\n",
        "token_classifier(\"My name is Loganathan Ramasamy. I live in New York.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {
        "id": "cOAl-aPp-x8f",
        "outputId": "bb69e664-345b-4e75-b7ee-eb302aa85ac8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at dbmdz/bert-large-cased-finetuned-conll03-english were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ],
      "source": [
        "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
        "\n",
        "model_checkpoint = \"dbmdz/bert-large-cased-finetuned-conll03-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)\n",
        "model = AutoModelForTokenClassification.from_pretrained(model_checkpoint)\n",
        "\n",
        "example = \"My name is Loganathan Ramasamy and I live in New York.\"\n",
        "inputs = tokenizer(example, return_tensors=\"pt\")\n",
        "outputs = model(**inputs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "metadata": {
        "id": "5Vt1gxnj-x8g",
        "outputId": "ed7c59a3-7fc0-4b6e-ac30-85187d7d85dc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([1, 18])\n",
            "torch.Size([1, 18, 9])\n"
          ]
        }
      ],
      "source": [
        "print(inputs[\"input_ids\"].shape)\n",
        "print(outputs.logits.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 74,
      "metadata": {
        "id": "BVuOpIgE-x8g",
        "outputId": "961ea89d-a798-4609-a595-496e8d067900",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 8, 8, 0, 0]\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "probabilities = torch.nn.functional.softmax(outputs.logits, dim=-1)[0].tolist()\n",
        "predictions = outputs.logits.argmax(dim=-1)[0].tolist()\n",
        "print(predictions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 75,
      "metadata": {
        "id": "TBzd7Rlf-x8g",
        "outputId": "1e5153a8-34c1-42c3-d3e8-bd9e9d838a74",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'O',\n",
              " 1: 'B-MISC',\n",
              " 2: 'I-MISC',\n",
              " 3: 'B-PER',\n",
              " 4: 'I-PER',\n",
              " 5: 'B-ORG',\n",
              " 6: 'I-ORG',\n",
              " 7: 'B-LOC',\n",
              " 8: 'I-LOC'}"
            ]
          },
          "metadata": {},
          "execution_count": 75
        }
      ],
      "source": [
        "model.config.id2label"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 76,
      "metadata": {
        "id": "dI5qkVzb-x8g",
        "outputId": "54d1ffa1-3c48-4589-f1f5-e93026d8ce55",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(0, 0),\n",
              " (0, 2),\n",
              " (3, 7),\n",
              " (8, 10),\n",
              " (11, 16),\n",
              " (16, 19),\n",
              " (19, 21),\n",
              " (22, 26),\n",
              " (26, 28),\n",
              " (28, 30),\n",
              " (31, 34),\n",
              " (35, 36),\n",
              " (37, 41),\n",
              " (42, 44),\n",
              " (45, 48),\n",
              " (49, 53),\n",
              " (53, 54),\n",
              " (0, 0)]"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ],
      "source": [
        "inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)\n",
        "inputs_with_offsets[\"offset_mapping\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 77,
      "metadata": {
        "id": "o83LBj8s-x8g",
        "outputId": "126aa67e-627e-4bc5-dff1-c4cd759fa58c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Loganathan Ramasamy'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 77
        }
      ],
      "source": [
        "example[11:30]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "example[5:16]"
      ],
      "metadata": {
        "id": "D__LURVYpzvi",
        "outputId": "c182a4b7-dc8d-4e79-f3a5-b94af5ef695e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'me is Logan'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 79,
      "metadata": {
        "id": "l1qwI0QY-x8g",
        "outputId": "5da4f185-9063-4413-82ba-6de0b9372bdc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'entity': 'O', 'score': 0.9990185499191284, 'word': '[CLS]', 'start': 0, 'end': 0}, {'entity': 'O', 'score': 0.9997076392173767, 'word': 'My', 'start': 0, 'end': 2}, {'entity': 'O', 'score': 0.999670147895813, 'word': 'name', 'start': 3, 'end': 7}, {'entity': 'O', 'score': 0.9998857975006104, 'word': 'is', 'start': 8, 'end': 10}, {'entity': 'I-PER', 'score': 0.9975009560585022, 'word': 'Logan', 'start': 11, 'end': 16}, {'entity': 'I-PER', 'score': 0.9939070343971252, 'word': '##ath', 'start': 16, 'end': 19}, {'entity': 'I-PER', 'score': 0.9931873083114624, 'word': '##an', 'start': 19, 'end': 21}, {'entity': 'I-PER', 'score': 0.9993284940719604, 'word': 'Rama', 'start': 22, 'end': 26}, {'entity': 'I-PER', 'score': 0.9954164028167725, 'word': '##sa', 'start': 26, 'end': 28}, {'entity': 'I-PER', 'score': 0.9891738891601562, 'word': '##my', 'start': 28, 'end': 30}, {'entity': 'O', 'score': 0.9998189806938171, 'word': 'and', 'start': 31, 'end': 34}, {'entity': 'O', 'score': 0.9997592568397522, 'word': 'I', 'start': 35, 'end': 36}, {'entity': 'O', 'score': 0.9998352527618408, 'word': 'live', 'start': 37, 'end': 41}, {'entity': 'O', 'score': 0.9998313188552856, 'word': 'in', 'start': 42, 'end': 44}, {'entity': 'I-LOC', 'score': 0.9992538094520569, 'word': 'New', 'start': 45, 'end': 48}, {'entity': 'I-LOC', 'score': 0.9988001585006714, 'word': 'York', 'start': 49, 'end': 53}, {'entity': 'O', 'score': 0.9990185499191284, 'word': '.', 'start': 53, 'end': 54}, {'entity': 'O', 'score': 0.9990185499191284, 'word': '[SEP]', 'start': 0, 'end': 0}]\n"
          ]
        }
      ],
      "source": [
        "results = []\n",
        "inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)\n",
        "tokens = inputs_with_offsets.tokens()\n",
        "offsets = inputs_with_offsets[\"offset_mapping\"]\n",
        "\n",
        "for idx, pred in enumerate(predictions):\n",
        "    label = model.config.id2label[pred]\n",
        "    if label != \"0\":\n",
        "        start, end = offsets[idx]\n",
        "        results.append(\n",
        "            {\n",
        "                \"entity\": label,\n",
        "                \"score\": probabilities[idx][pred],\n",
        "                \"word\": tokens[idx],\n",
        "                \"start\": start,\n",
        "                \"end\": end\n",
        "            }\n",
        "        )\n",
        "print(results)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 80,
      "metadata": {
        "id": "MZEOgQSG-x8g",
        "outputId": "73e32621-0d23-4eed-f488-e29d3d0c5fb0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'d I live in '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 80
        }
      ],
      "source": [
        "example[33:45]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 81,
      "metadata": {
        "id": "QGKpDJgC-x8g",
        "outputId": "156d8415-c630-4dcb-ffe2-b42ccc75f4f8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'entity_group': 'PER', 'score': 0.9947523474693298, 'word': 'Loganathan Ramasamy', 'start': 11, 'end': 30}, {'entity_group': 'LOC', 'score': 0.9990269839763641, 'word': 'New York', 'start': 45, 'end': 53}]\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "\n",
        "results = []\n",
        "inputs_with_offsets = tokenizer(example, return_offsets_mapping=True)\n",
        "tokens = inputs_with_offsets.tokens()\n",
        "offsets = inputs_with_offsets[\"offset_mapping\"]\n",
        "\n",
        "idx = 0\n",
        "while idx < len(predictions):\n",
        "    pred = predictions[idx]\n",
        "    label = model.config.id2label[pred]\n",
        "    if label != \"O\":\n",
        "        # Remove the B- or I-\n",
        "        label = label[2:]\n",
        "        start, _ = offsets[idx]\n",
        "\n",
        "        # Grab all the tokens labeled with I-label\n",
        "        all_scores = []\n",
        "        while (\n",
        "            idx < len(predictions)\n",
        "            and model.config.id2label[predictions[idx]] == f\"I-{label}\"\n",
        "        ):\n",
        "            all_scores.append(probabilities[idx][pred])\n",
        "            _, end = offsets[idx]\n",
        "            idx += 1\n",
        "\n",
        "        # The score is the mean of all the scores of the tokens in that grouped entity\n",
        "        score = np.mean(all_scores).item()\n",
        "        word = example[start:end]\n",
        "        results.append(\n",
        "            {\n",
        "                \"entity_group\": label,\n",
        "                \"score\": score,\n",
        "                \"word\": word,\n",
        "                \"start\": start,\n",
        "                \"end\": end,\n",
        "            }\n",
        "        )\n",
        "    idx += 1\n",
        "\n",
        "print(results)"
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Bs5rFfl8s3wI"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "Fast tokenizers' special powers (PyTorch)",
      "provenance": [],
      "include_colab_link": true
    },
    "language_info": {
      "name": "python"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ef339423654947fba33161c595eb1894": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_980041176c214b8d96fb6cfd1f5e267d",
              "IPY_MODEL_1cd5e52a8314469cb3d71b89b5436395",
              "IPY_MODEL_b6da43646c86413f9a660a2329c7406e"
            ],
            "layout": "IPY_MODEL_5d96b454670e41a28b1a1c1e97a23843"
          }
        },
        "980041176c214b8d96fb6cfd1f5e267d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e3f5a3f1946d480084b4ba83bcc13c0e",
            "placeholder": "​",
            "style": "IPY_MODEL_0b1e28e935544d79b355ca9374cebcd6",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "1cd5e52a8314469cb3d71b89b5436395": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_780702ad0fe34eb3bba0a0c73d69e5ec",
            "max": 29,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_689b60a2bcaa40f0968503e09dc3774b",
            "value": 29
          }
        },
        "b6da43646c86413f9a660a2329c7406e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c9f76b1354eb4472bf05194d3a6ff9fa",
            "placeholder": "​",
            "style": "IPY_MODEL_45fd0282950e4594acaa3a1a9e966c37",
            "value": " 29.0/29.0 [00:00&lt;00:00, 1.11kB/s]"
          }
        },
        "5d96b454670e41a28b1a1c1e97a23843": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3f5a3f1946d480084b4ba83bcc13c0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b1e28e935544d79b355ca9374cebcd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "780702ad0fe34eb3bba0a0c73d69e5ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "689b60a2bcaa40f0968503e09dc3774b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c9f76b1354eb4472bf05194d3a6ff9fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45fd0282950e4594acaa3a1a9e966c37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0b0848a2100842f380fd3f0a15a5416b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a96ff4d8ca134c11b6b6e9d12202f611",
              "IPY_MODEL_456030c7b5724e3ab94103f63f4d87c2",
              "IPY_MODEL_38e228d6c0d941fba03cee268271e235"
            ],
            "layout": "IPY_MODEL_8b63156fd203405b835d4c93b2d123dc"
          }
        },
        "a96ff4d8ca134c11b6b6e9d12202f611": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66d5e54e0146431381bbf424fd0ee520",
            "placeholder": "​",
            "style": "IPY_MODEL_58fe65184f9e47e5a0d82b7877286ae4",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "456030c7b5724e3ab94103f63f4d87c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cfabfe88919f483183de5f3f208574be",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_461386c519b848c18694dd7edb4fd78c",
            "value": 570
          }
        },
        "38e228d6c0d941fba03cee268271e235": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_400b479345514bb09702d14a9fd46fa4",
            "placeholder": "​",
            "style": "IPY_MODEL_2cfd2cc255da427ab80d1eba18a90d21",
            "value": " 570/570 [00:00&lt;00:00, 37.3kB/s]"
          }
        },
        "8b63156fd203405b835d4c93b2d123dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66d5e54e0146431381bbf424fd0ee520": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58fe65184f9e47e5a0d82b7877286ae4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cfabfe88919f483183de5f3f208574be": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "461386c519b848c18694dd7edb4fd78c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "400b479345514bb09702d14a9fd46fa4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2cfd2cc255da427ab80d1eba18a90d21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2dc3ec84c2db4926b30dc5881e2ba8a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3b29713d6a2c428bb824320d92ddff2c",
              "IPY_MODEL_fbf7a38cdf6e4243b248737114fc4362",
              "IPY_MODEL_61622b3d66d6475894a795e027fb58df"
            ],
            "layout": "IPY_MODEL_ba5e72ccf62049e9b896cf6d97089240"
          }
        },
        "3b29713d6a2c428bb824320d92ddff2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_88f1c6aea3d4411c8bed136e20a9d292",
            "placeholder": "​",
            "style": "IPY_MODEL_206dc010e34446b48fd1664c33df9a4f",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "fbf7a38cdf6e4243b248737114fc4362": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_11b1d70923a04ce890c58d7192b05467",
            "max": 213450,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_379e75f42ade497ebbf50fb5af0f7c5d",
            "value": 213450
          }
        },
        "61622b3d66d6475894a795e027fb58df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_774a3055677f475a9340e4e266aee832",
            "placeholder": "​",
            "style": "IPY_MODEL_28c84798d8f74897a6d1f61d4a70bdf1",
            "value": " 213k/213k [00:00&lt;00:00, 1.82MB/s]"
          }
        },
        "ba5e72ccf62049e9b896cf6d97089240": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88f1c6aea3d4411c8bed136e20a9d292": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "206dc010e34446b48fd1664c33df9a4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "11b1d70923a04ce890c58d7192b05467": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "379e75f42ade497ebbf50fb5af0f7c5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "774a3055677f475a9340e4e266aee832": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "28c84798d8f74897a6d1f61d4a70bdf1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "700b0a8f4f5d4daea471eabfc299d9c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e34d578312724fef85af7214b9b27c3c",
              "IPY_MODEL_fdcab6d63a5f486090141c882e2dc257",
              "IPY_MODEL_20d0523cca1741879ecf1502dcce0570"
            ],
            "layout": "IPY_MODEL_82d7bc8303f744d2a7a8957227569f1b"
          }
        },
        "e34d578312724fef85af7214b9b27c3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b98d30c67ac8463c81e283b3a10eb272",
            "placeholder": "​",
            "style": "IPY_MODEL_2e61db326e9f43f89911f20554f41a76",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "fdcab6d63a5f486090141c882e2dc257": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6dbdaf5f378a433b9ee11a6490de51de",
            "max": 435797,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_40e8975f9e294502a08d293045810ba0",
            "value": 435797
          }
        },
        "20d0523cca1741879ecf1502dcce0570": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_573b92e9cf3e41ea94bf13e1ac7c6121",
            "placeholder": "​",
            "style": "IPY_MODEL_56adc1267b6542f0824c497da1433633",
            "value": " 436k/436k [00:00&lt;00:00, 3.82MB/s]"
          }
        },
        "82d7bc8303f744d2a7a8957227569f1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b98d30c67ac8463c81e283b3a10eb272": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e61db326e9f43f89911f20554f41a76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6dbdaf5f378a433b9ee11a6490de51de": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "40e8975f9e294502a08d293045810ba0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "573b92e9cf3e41ea94bf13e1ac7c6121": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "56adc1267b6542f0824c497da1433633": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}