NER_hackathon_transformer_final.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZevXbB1u6kR2",
        "outputId": "fc32e194-aa1b-4198-b827-a77bea28de7d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir(\"drive/My Drive/Hackathon\")"
      ],
      "metadata": {
        "id": "axNu4T-h6p8Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OzXVdRd7I8nD",
        "outputId": "134c5f31-004b-4a01-dcb9-095f42939fb3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver. Make sure that the latest NVIDIA driver is installed and running.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf\n",
        "tf.test.gpu_device_name()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "W90YiFW3duBg",
        "outputId": "80d4f207-f422-4622-a89b-049ca93f805c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "''"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install spacy[transformers]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aseB_OCO69UQ",
        "outputId": "3d47d6db-d372-4862-8321-0e4e97307cb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: spacy[transformers] in /usr/local/lib/python3.7/dist-packages (3.4.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (21.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (2.11.3)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (3.0.8)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (0.7.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (1.0.9)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (0.8.1)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (1.0.3)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (2.0.8)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (2.4.5)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (3.0.10)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (0.10.1)\n",
            "Requirement already satisfied: typing-extensions<4.2.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (4.1.1)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (4.64.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (57.4.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (1.21.6)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (2.23.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (2.0.7)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (1.10.2)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (8.1.5)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy[transformers]) (3.3.0)\n",
            "Collecting spacy-transformers<1.2.0,>=1.1.2\n",
            "  Downloading spacy_transformers-1.1.8-py2.py3-none-any.whl (53 kB)\n",
            "\u001b[K     |████████████████████████████████| 53 kB 1.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy[transformers]) (3.10.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy[transformers]) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy[transformers]) (5.2.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy[transformers]) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy[transformers]) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy[transformers]) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy[transformers]) (2.10)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->spacy[transformers]) (1.12.1+cu113)\n",
            "Collecting spacy-alignments<1.0.0,>=0.7.2\n",
            "  Downloading spacy_alignments-0.8.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 8.4 MB/s \n",
            "\u001b[?25hCollecting transformers<4.22.0,>=3.4.0\n",
            "  Downloading transformers-4.21.3-py3-none-any.whl (4.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.7 MB 26.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy[transformers]) (0.7.9)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy[transformers]) (0.0.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[transformers]) (6.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[transformers]) (3.8.0)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 30.6 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.11.0-py3-none-any.whl (182 kB)\n",
            "\u001b[K     |████████████████████████████████| 182 kB 59.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[transformers]) (4.13.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[transformers]) (2022.6.2)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.8.0,>=0.3.0->spacy[transformers]) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy[transformers]) (2.0.1)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers, spacy-alignments, spacy-transformers\n",
            "Successfully installed huggingface-hub-0.11.0 spacy-alignments-0.8.6 spacy-transformers-1.1.8 tokenizers-0.12.1 transformers-4.21.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m spacy download en_core_web_trf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xIEBzRaY6-Mh",
        "outputId": "0b148fef-71af-41db-8f8b-9fc7fb88c600"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2022-11-28 07:14:53.900834: E tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting en-core-web-trf==3.4.1\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.4.1/en_core_web_trf-3.4.1-py3-none-any.whl (460.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 460.3 MB 23 kB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy-transformers<1.2.0,>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from en-core-web-trf==3.4.1) (1.1.8)\n",
            "Requirement already satisfied: spacy<3.5.0,>=3.4.1 in /usr/local/lib/python3.7/dist-packages (from en-core-web-trf==3.4.1) (3.4.3)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (1.0.3)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (3.0.8)\n",
            "Requirement already satisfied: typing-extensions<4.2.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (4.1.1)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (2.0.7)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (4.64.1)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (1.21.6)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (2.0.8)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (1.10.2)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (0.10.1)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (0.7.0)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (2.4.5)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (8.1.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (57.4.0)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (2.23.0)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (0.8.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (3.0.10)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (3.3.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (2.11.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (21.3)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (1.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (3.10.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (5.2.1)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (2022.9.24)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (2.10)\n",
            "Requirement already satisfied: transformers<4.22.0,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->en-core-web-trf==3.4.1) (4.21.3)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->en-core-web-trf==3.4.1) (1.12.1+cu113)\n",
            "Requirement already satisfied: spacy-alignments<1.0.0,>=0.7.2 in /usr/local/lib/python3.7/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->en-core-web-trf==3.4.1) (0.8.6)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (0.0.3)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (0.7.9)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->en-core-web-trf==3.4.1) (2022.6.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->en-core-web-trf==3.4.1) (4.13.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->en-core-web-trf==3.4.1) (6.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->en-core-web-trf==3.4.1) (3.8.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->en-core-web-trf==3.4.1) (0.12.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->en-core-web-trf==3.4.1) (0.11.0)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.8.0,>=0.3.0->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.5.0,>=3.4.1->en-core-web-trf==3.4.1) (2.0.1)\n",
            "Installing collected packages: en-core-web-trf\n",
            "Successfully installed en-core-web-trf-3.4.1\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_trf')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "\n",
        "with open('PHI_dataset_train_bak.json', 'r') as f:\n",
        "    train_data = json.load(f)"
      ],
      "metadata": {
        "id": "i4QuJ56X7QpN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "nlp = spacy.load('en_core_web_trf')"
      ],
      "metadata": {
        "id": "OQ1mWmcN7d_E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# import pandas as pd\n",
        "# df = pd.DataFrame(train_data)\n",
        "# df.head(50)"
      ],
      "metadata": {
        "id": "NOo0BD59F887"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# from sklearn.utils import shuffle\n",
        "# df = shuffle(df)\n",
        "# df.head(100)\n",
        "# train_data[0]"
      ],
      "metadata": {
        "id": "yVI7UZaAGRMQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.utils import shuffle\n",
        "train_data_shuffle = shuffle(train_data)\n",
        "# train_data_shuffle[0]"
      ],
      "metadata": {
        "id": "OYpUjYmzG26u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "from tqdm import tqdm\n",
        "import spacy\n",
        "from spacy.tokens import DocBin\n",
        "\n",
        "# nlp = spacy.load(\"en_core_web_sm\") # load other spacy model\n",
        "\n",
        "db = DocBin() # create a DocBin object\n",
        "\n",
        "for text, annot in tqdm(train_data_shuffle): # data in previous format\n",
        "    doc = nlp.make_doc(text) # create doc object from text\n",
        "    ents = []\n",
        "    for start, end, label in annot[\"entities\"]: # add character indexes\n",
        "        # print(start, end, label)\n",
        "        span = doc.char_span(start, end, label=label, alignment_mode=\"contract\")\n",
        "        if span is None:\n",
        "            print(\"Skipping entity\")\n",
        "        else:\n",
        "            ents.append(span)\n",
        "    doc.ents = ents # label the text with the ents\n",
        "    db.add(doc)\n",
        "\n",
        "db.to_disk(\"./train_shuffle.spacy\") # save the docbin object"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GN3Fjc8E7gtP",
        "outputId": "812c69e4-0b7c-49b9-d7c6-bc9817eb74ff"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  5%|▌         | 242/4546 [00:00<00:05, 776.68it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 12%|█▏        | 523/4546 [00:00<00:04, 892.70it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 21%|██        | 957/4546 [00:01<00:03, 1039.33it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 26%|██▌       | 1166/4546 [00:01<00:03, 1016.56it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 33%|███▎      | 1510/4546 [00:01<00:02, 1103.82it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 41%|████▏     | 1878/4546 [00:01<00:02, 1185.14it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 47%|████▋     | 2124/4546 [00:02<00:02, 1193.67it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 55%|█████▌    | 2516/4546 [00:02<00:01, 1210.03it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 70%|███████   | 3188/4546 [00:02<00:01, 1274.07it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 83%|████████▎ | 3751/4546 [00:03<00:00, 1376.53it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 89%|████████▊ | 4034/4546 [00:03<00:00, 1380.55it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4546/4546 [00:03<00:00, 1187.67it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "\n",
        "with open('PHI_dataset_test.json', 'r') as f:\n",
        "    test_data = json.load(f)"
      ],
      "metadata": {
        "id": "pVAW0woA7nOQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.utils import shuffle\n",
        "test_data_shuffle = shuffle(test_data)"
      ],
      "metadata": {
        "id": "O8nmkAGBHwNO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "from tqdm import tqdm\n",
        "import spacy\n",
        "from spacy.tokens import DocBin\n",
        "\n",
        "db = DocBin() # create a DocBin object\n",
        "\n",
        "for text, annot in tqdm(test_data_shuffle): # data in previous format\n",
        "    doc = nlp.make_doc(text) # create doc object from text\n",
        "    ents = []\n",
        "    for start, end, label in annot[\"entities\"]: # add character indexes\n",
        "        # print(start, end, label)\n",
        "        span = doc.char_span(start, end, label=label, alignment_mode=\"contract\")\n",
        "        if span is None:\n",
        "            print(\"Skipping entity\")\n",
        "        else:\n",
        "            ents.append(span)\n",
        "    doc.ents = ents # label the text with the ents\n",
        "    db.add(doc)\n",
        "\n",
        "db.to_disk(\"./dev_shuffle.spacy\") # save the docbin object"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mpcZVGxb7qIL",
        "outputId": "425ae90e-da04-4f26-ed0b-e46a0ffe9beb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 95%|█████████▍| 431/454 [00:00<00:00, 1342.25it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n",
            "Skipping entity\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\r100%|██████████| 454/454 [00:00<00:00, 1296.48it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m spacy init fill-config base_config.cfg config.cfg"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lsDs0l0U7swe",
        "outputId": "2c56c477-c401-492c-8c37-39e071b4ce7c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2022-11-28 07:28:29.762472: E tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
            "\u001b[38;5;2m✔ Auto-filled config with all values\u001b[0m\n",
            "\u001b[38;5;2m✔ Saved config\u001b[0m\n",
            "config.cfg\n",
            "You can now add your data and train your pipeline:\n",
            "python -m spacy train config.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m spacy train config.cfg --output ./checkpoint_trf_final --paths.train ./train_shuffle.spacy --paths.dev ./dev_shuffle.spacy --gpu-id 0"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LA27qym47_TQ",
        "outputId": "6f32a7b9-d755-4239-c796-0eb873e2b377"
      },
      "execution_count": null,
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\u001b[38;5;2m✔ Created output directory: checkpoint_trf\u001b[0m\n",
            "\u001b[38;5;4mℹ Saving to output directory: checkpoint_trf\u001b[0m\n",
            "\u001b[38;5;4mℹ Using GPU: 0\u001b[0m\n",
            "\u001b[1m\n",
            "=========================== Initializing pipeline ===========================\u001b[0m\n",
            "[2022-11-27 18:23:53,810] [INFO] Set up nlp object from config\n",
            "INFO:spacy:Set up nlp object from config\n",
            "[2022-11-27 18:23:53,821] [INFO] Pipeline: ['transformer', 'ner']\n",
            "INFO:spacy:Pipeline: ['transformer', 'ner']\n",
            "[2022-11-27 18:23:53,825] [INFO] Created vocabulary\n",
            "INFO:spacy:Created vocabulary\n",
            "[2022-11-27 18:23:53,826] [INFO] Finished initializing nlp object\n",
            "INFO:spacy:Finished initializing nlp object\n",
            "Downloading config.json: 100% 481/481 [00:00<00:00, 485kB/s]\n",
            "Downloading vocab.json: 100% 878k/878k [00:01<00:00, 667kB/s]\n",
            "Downloading merges.txt: 100% 446k/446k [00:01<00:00, 405kB/s]\n",
            "Downloading tokenizer.json: 100% 1.29M/1.29M [00:01<00:00, 995kB/s]\n",
            "Downloading pytorch_model.bin: 100% 478M/478M [00:07<00:00, 68.4MB/s]\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaModel: ['lm_head.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.decoder.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.bias']\n",
            "- This IS expected if you are initializing RobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "[2022-11-27 18:24:41,839] [INFO] Initialized pipeline components: ['transformer', 'ner']\n",
            "INFO:spacy:Initialized pipeline components: ['transformer', 'ner']\n",
            "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
            "\u001b[1m\n",
            "============================= Training pipeline =============================\u001b[0m\n",
            "\u001b[38;5;4mℹ Pipeline: ['transformer', 'ner']\u001b[0m\n",
            "\u001b[38;5;4mℹ Initial learn rate: 0.0\u001b[0m\n",
            "E    #       LOSS TRANS...  LOSS NER  ENTS_F  ENTS_P  ENTS_R  SCORE \n",
            "---  ------  -------------  --------  ------  ------  ------  ------\n",
            "  0       0          44.14     85.70    1.15    0.83    1.87    0.01\n",
            "  2     200       89993.63  121418.87   99.03   98.90   99.17    0.99\n",
            "  4     400        2013.69   2495.04   99.44   99.42   99.46    0.99\n",
            "  7     600        1451.53   1127.66   99.58   99.51   99.64    1.00\n",
            "  9     800         584.52    549.41   99.62   99.56   99.68    1.00\n",
            " 11    1000         341.85    426.33   99.75   99.69   99.81    1.00\n",
            " 14    1200         206.99    256.65   99.64   99.59   99.69    1.00\n",
            " 16    1400         119.20    168.08   99.70   99.64   99.76    1.00\n",
            " 18    1600          87.81    125.93   99.77   99.75   99.80    1.00\n",
            " 21    1800         103.47    123.29   99.65   99.61   99.69    1.00\n",
            " 23    2000          85.28    100.19   99.64   99.58   99.69    1.00\n",
            " 26    2200          67.89     93.86   99.75   99.69   99.80    1.00\n",
            " 28    2400          43.67     62.58   99.78   99.75   99.81    1.00\n",
            " 30    2600          70.82     77.63   99.82   99.78   99.86    1.00\n",
            " 33    2800          66.17     78.09   99.75   99.69   99.81    1.00\n",
            " 35    3000          64.17     71.94   99.67   99.61   99.73    1.00\n",
            " 37    3200          45.08     61.48   99.73   99.68   99.78    1.00\n",
            " 40    3400          55.03     55.60   99.70   99.64   99.76    1.00\n",
            " 42    3600         433.22    216.64   99.81   99.76   99.86    1.00\n",
            " 45    3800         675.87    340.72   99.76   99.71   99.81    1.00\n",
            " 47    4000          91.57     95.96   99.75   99.69   99.81    1.00\n",
            " 49    4200          22.10     32.54   99.85   99.80   99.90    1.00\n",
            " 52    4400          65.04     49.51   99.70   99.64   99.76    1.00\n",
            " 54    4600          67.66     58.25   99.79   99.73   99.85    1.00\n",
            " 57    4800          43.36     30.23   99.83   99.78   99.88    1.00\n",
            " 59    5000          19.34     21.73   99.61   99.56   99.66    1.00\n",
            " 61    5200          19.47     26.09   99.80   99.75   99.85    1.00\n",
            " 64    5400         330.23    202.27   99.67   99.61   99.73    1.00\n",
            " 66    5600          83.76     61.43   99.69   99.63   99.75    1.00\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nlp1 = spacy.load(\"./checkpoint_trf_final/model-last\") #load the best model\n",
        "doc = nlp1(\"what is your name Manoj. He is added to H2122 plan\") # input sample text\n",
        "\n",
        "spacy.displacy.render(doc, style=\"ent\", jupyter=True) # display in Jupyter"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 109
        },
        "id": "WW_wSls78Ldm",
        "outputId": "dc0118ff-2553-4849-dcf1-de4545cc9683"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
            "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">what is your name \n",
              "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Manoj\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Name</span>\n",
              "</mark>\n",
              ". He is added to \n",
              "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    H2122\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PlanName</span>\n",
              "</mark>\n",
              " plan</div></span>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!sudo apt install tesseract-ocr\n",
        "!pip install pytesseract"
      ],
      "metadata": {
        "id": "jmDJStskZE__",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "0fd47123-b88b-44ae-9b7e-10b16d338546"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'sudo apt autoremove' to remove it.\n",
            "The following additional packages will be installed:\n",
            "  tesseract-ocr-eng tesseract-ocr-osd\n",
            "The following NEW packages will be installed:\n",
            "  tesseract-ocr tesseract-ocr-eng tesseract-ocr-osd\n",
            "0 upgraded, 3 newly installed, 0 to remove and 5 not upgraded.\n",
            "Need to get 4,795 kB of archives.\n",
            "After this operation, 15.8 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tesseract-ocr-eng all 4.00~git24-0e00fe6-1.2 [1,588 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tesseract-ocr-osd all 4.00~git24-0e00fe6-1.2 [2,989 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/universe amd64 tesseract-ocr amd64 4.00~git2288-10f4998a-2 [218 kB]\n",
            "Fetched 4,795 kB in 0s (13.2 MB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76, <> line 3.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package tesseract-ocr-eng.\n",
            "(Reading database ... 123991 files and directories currently installed.)\n",
            "Preparing to unpack .../tesseract-ocr-eng_4.00~git24-0e00fe6-1.2_all.deb ...\n",
            "Unpacking tesseract-ocr-eng (4.00~git24-0e00fe6-1.2) ...\n",
            "Selecting previously unselected package tesseract-ocr-osd.\n",
            "Preparing to unpack .../tesseract-ocr-osd_4.00~git24-0e00fe6-1.2_all.deb ...\n",
            "Unpacking tesseract-ocr-osd (4.00~git24-0e00fe6-1.2) ...\n",
            "Selecting previously unselected package tesseract-ocr.\n",
            "Preparing to unpack .../tesseract-ocr_4.00~git2288-10f4998a-2_amd64.deb ...\n",
            "Unpacking tesseract-ocr (4.00~git2288-10f4998a-2) ...\n",
            "Setting up tesseract-ocr-osd (4.00~git24-0e00fe6-1.2) ...\n",
            "Setting up tesseract-ocr-eng (4.00~git24-0e00fe6-1.2) ...\n",
            "Setting up tesseract-ocr (4.00~git2288-10f4998a-2) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pytesseract\n",
            "  Downloading pytesseract-0.3.10-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: packaging>=21.3 in /usr/local/lib/python3.7/dist-packages (from pytesseract) (21.3)\n",
            "Collecting Pillow>=8.0.0\n",
            "  Downloading Pillow-9.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2 MB 5.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=21.3->pytesseract) (3.0.9)\n",
            "Installing collected packages: Pillow, pytesseract\n",
            "  Attempting uninstall: Pillow\n",
            "    Found existing installation: Pillow 7.1.2\n",
            "    Uninstalling Pillow-7.1.2:\n",
            "      Successfully uninstalled Pillow-7.1.2\n",
            "Successfully installed Pillow-9.3.0 pytesseract-0.3.10\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "PIL"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pytesseract\n",
        "import shutil\n",
        "import os\n",
        "import random\n",
        "\n",
        "try:\n",
        "  from PIL import Image\n",
        "except ImportError:\n",
        "  import Image\n"
      ],
      "metadata": {
        "id": "F36fx6XHiujl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 76
        },
        "id": "yBoHzm3Liusa",
        "outputId": "c978b316-a8d9-4f3d-8ba9-16315b15add9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-11febb34-e2e7-42bc-9918-c5f3a57a2c2b\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-11febb34-e2e7-42bc-9918-c5f3a57a2c2b\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving Picture1.png to Picture1.png\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "image_path_in_colab='Picture1.png'\n",
        "extractedInformation = pytesseract.image_to_string(Image.open(image_path_in_colab))\n",
        "print(extractedInformation)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W5W6UifwjCcv",
        "outputId": "191d9237-3cb2-4338-d521-f607271c9039"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " \n",
            "\n",
            "Manoj Joshi\n",
            "\n",
            "H9978\n",
            "\n",
            "Radiology\n",
            "\n",
            " \n",
            "\n",
            "manoj@gmail.com\n",
            "\n",
            "222112222\n",
            "\n",
            " \n",
            "\n",
            " \n",
            "\n",
            " \n",
            "\n",
            " \n",
            "\n",
            " \n",
            "\n",
            " \n",
            "\n",
            " \n",
            "\n",
            " \n",
            "\f\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "doc = nlp1('Many former inmates leaving prison are broke until state settlements arrive. Tiding them over with high-interest cash advances has become lucrative. Manoj Joshi H9978 Radiology This is an auto-generated partial config. manoj@gmail.com 222112222 The easiest way to get started is to clone.') # input sample text\n",
        "\n",
        "spacy.displacy.render(doc, style=\"ent\", jupyter=True) # display in Jupyter"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "id": "KDV0rNk8MdVQ",
        "outputId": "7371bb27-66ad-4bce-9d1a-edf1a5301f88"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Many former inmates leaving prison are broke until state settlements arrive. Tiding them over with high-interest cash advances has become lucrative. \n",
              "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Manoj\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Name</span>\n",
              "</mark>\n",
              " \n",
              "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Joshi\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Name</span>\n",
              "</mark>\n",
              " \n",
              "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    H9978\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PlanName</span>\n",
              "</mark>\n",
              " \n",
              "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Radiology\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Speciality</span>\n",
              "</mark>\n",
              " This is an auto-generated partial config\n",
              "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    .\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Email</span>\n",
              "</mark>\n",
              " \n",
              "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    manoj@gmail.com\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">Email</span>\n",
              "</mark>\n",
              " \n",
              "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    222112222\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">MemberID</span>\n",
              "</mark>\n",
              " The easiest way to get started is to clone.</div></span>"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}