NLPmodel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "from nltk.tokenize import sent_tokenize\n",
    "import pandas as pd\n",
    "import ast"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import the training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Summary</th>\n",
       "      <th>Annotations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>In continuation to our letter dated 26th Octob...</td>\n",
       "      <td>{'entities': [(350, 358, 'Div'), (145, 153, 'D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Balkrishna Industries Ltd has informed BSE tha...</td>\n",
       "      <td>{'entities': [(156, 167, 'Dec')]}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Vaibhav Global Ltd has informed BSE that the B...</td>\n",
       "      <td>{'entities': [(208, 216, 'Div'), (220, 225, 'O...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>K.Z.LEASING &amp;amp; FINANCE LTD.has informed BSE...</td>\n",
       "      <td>{'entities': [(121, 131, 'SDate'), (108, 120, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>KALYANI INVESTMENT COMPANY LTD.has informed BS...</td>\n",
       "      <td>{'entities': [(122, 132, 'SDate'), (109, 121, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>295</td>\n",
       "      <td>IVP LTD.has informed BSE that the meeting of t...</td>\n",
       "      <td>{'entities': [(99, 109, 'SDate'), (86, 98, 'In...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>296</td>\n",
       "      <td>LADDERUP FINANCE LTD.has informed BSE that the...</td>\n",
       "      <td>{'entities': [(112, 122, 'SDate'), (99, 111, '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>297</td>\n",
       "      <td>RSWM LTD.has informed BSE that the meeting of ...</td>\n",
       "      <td>{'entities': [(100, 110, 'SDate'), (87, 99, 'I...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>298</td>\n",
       "      <td>AJMERA REALTY &amp;amp; INFRA INDIA LTD.has inform...</td>\n",
       "      <td>{'entities': [(127, 137, 'SDate'), (114, 126, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>299</td>\n",
       "      <td>Butterfly Gandhimathi Appliances Ltdhas inform...</td>\n",
       "      <td>{'entities': [(127, 137, 'SDate'), (114, 126, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>300 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0                                            Summary  \\\n",
       "0             0  In continuation to our letter dated 26th Octob...   \n",
       "1             1  Balkrishna Industries Ltd has informed BSE tha...   \n",
       "2             2  Vaibhav Global Ltd has informed BSE that the B...   \n",
       "3             3  K.Z.LEASING &amp; FINANCE LTD.has informed BSE...   \n",
       "4             4  KALYANI INVESTMENT COMPANY LTD.has informed BS...   \n",
       "..          ...                                                ...   \n",
       "295         295  IVP LTD.has informed BSE that the meeting of t...   \n",
       "296         296  LADDERUP FINANCE LTD.has informed BSE that the...   \n",
       "297         297  RSWM LTD.has informed BSE that the meeting of ...   \n",
       "298         298  AJMERA REALTY &amp; INFRA INDIA LTD.has inform...   \n",
       "299         299  Butterfly Gandhimathi Appliances Ltdhas inform...   \n",
       "\n",
       "                                           Annotations  \n",
       "0    {'entities': [(350, 358, 'Div'), (145, 153, 'D...  \n",
       "1                    {'entities': [(156, 167, 'Dec')]}  \n",
       "2    {'entities': [(208, 216, 'Div'), (220, 225, 'O...  \n",
       "3    {'entities': [(121, 131, 'SDate'), (108, 120, ...  \n",
       "4    {'entities': [(122, 132, 'SDate'), (109, 121, ...  \n",
       "..                                                 ...  \n",
       "295  {'entities': [(99, 109, 'SDate'), (86, 98, 'In...  \n",
       "296  {'entities': [(112, 122, 'SDate'), (99, 111, '...  \n",
       "297  {'entities': [(100, 110, 'SDate'), (87, 99, 'I...  \n",
       "298  {'entities': [(127, 137, 'SDate'), (114, 126, ...  \n",
       "299  {'entities': [(127, 137, 'SDate'), (114, 126, ...  \n",
       "\n",
       "[300 rows x 3 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('data/training_data.csv')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "convert the csv data into specific format of the training data\n",
    "TRAIN_DATA = [\n",
    "    (\"Who is Shaka Khan?\", {\"entities\": [(7, 17, \"PERSON\")]}),\n",
    "    (\"I like London and Berlin.\", {\"entities\": [(7, 13, \"LOC\"), (18, 24, \"LOC\")]}),\n",
    "]\n",
    "'''\n",
    "\n",
    "train_data = []\n",
    "temp_set = ()\n",
    "temp_dict = {}\n",
    "\n",
    "for i in range(len(data)):\n",
    "    temp_dict['entities'] = ast.literal_eval(data['Annotations'][0][13:-1])\n",
    "    temp_set = (data['Summary'][0] , temp_dict)\n",
    "    train_data.append(temp_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nprint to check train_data\\n'"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "print to check train_data\n",
    "'''\n",
    "#train_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load NER (Name Entity Recognition)spacy model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.blank('en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "creating and adding \"ner\" pipeline if not exist\n",
    "'''\n",
    "\n",
    "if \"ner\" not in nlp.pipe_names:\n",
    "    ner = nlp.create_pipe(\"ner\")\n",
    "    nlp.add_pipe(ner, last=True)\n",
    "else:\n",
    "    ner = nlp.get_pipe(\"ner\")\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "fixing number of iterations model has to do in training\n",
    "At each iteration, the training data is shuffled to ensure the model doesn’t make any generalizations based on the order of examples.\n",
    "'''\n",
    "\n",
    "n_iter=100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add the above made labels in the ner (name entity recognition) pipe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "for _, annotations in train_data:\n",
    "    for ent in annotations.get('entities'):\n",
    "        ner.add_label(ent[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "get names of other pipes to disable them during training\n",
    "'''\n",
    "other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Vanshika Sharma\\Anaconda3\\lib\\site-packages\\spacy\\language.py:639: UserWarning: [W033] Training a new parser or NER using a model with no lexeme normalization table. This may degrade the performance of the model to some degree. If this is intentional or the language you're using doesn't have a normalization table, please ignore this warning. If this is surprising, make sure you have the spacy-lookups-data package installed. The languages with lexeme normalization tables are currently: da, de, el, en, id, lb, pt, ru, sr, ta, th.\n",
      "  **kwargs\n",
      "100%|██████████| 300/300 [00:23<00:00, 12.80it/s]\n",
      "  0%|          | 1/300 [00:00<00:30,  9.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 1215.8859523992728}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:34<00:00,  8.78it/s]\n",
      "  0%|          | 1/300 [00:00<00:33,  8.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 7.981945408799414e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:30<00:00,  9.78it/s]\n",
      "  0%|          | 1/300 [00:00<00:32,  9.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 4.279735676733278e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:29<00:00, 10.03it/s]\n",
      "  1%|          | 2/300 [00:00<00:25, 11.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 8.324015926857244e-07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:32<00:00,  9.16it/s]\n",
      "  0%|          | 1/300 [00:00<00:46,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 8.629334604848203e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:44<00:00,  6.70it/s]\n",
      "  0%|          | 1/300 [00:00<00:46,  6.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 3.0261564833907983e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:49<00:00,  6.12it/s]\n",
      "  0%|          | 1/300 [00:00<00:44,  6.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 9.014956076167244e-08}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:46<00:00,  6.41it/s]\n",
      "  0%|          | 1/300 [00:00<00:42,  7.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 1.0034206652543012e-07}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:46<00:00,  6.40it/s]\n",
      "  0%|          | 1/300 [00:00<00:46,  6.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 2.7754662299534823e-08}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:47<00:00,  6.28it/s]\n",
      "  0%|          | 1/300 [00:00<00:41,  7.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 1.8242138048316337e-08}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:46<00:00,  6.39it/s]\n",
      "  0%|          | 1/300 [00:00<00:43,  6.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 5.429243313677803e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [00:51<00:00,  5.85it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 10.618253563628093}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:02<00:00,  4.81it/s]\n",
      "  0%|          | 1/300 [00:00<00:57,  5.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 2.7805534152095515}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:00<00:00,  4.92it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 0.00024359136205635986}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:02<00:00,  4.82it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 2.0227634089544272e-23}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:01<00:00,  4.84it/s]\n",
      "  0%|          | 1/300 [00:00<00:56,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 1.666480078982984e-24}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:00<00:00,  4.93it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 5.395793123246451e-14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:00<00:00,  4.95it/s]\n",
      "  0%|          | 1/300 [00:00<00:56,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 1.974362897536095e-10}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:02<00:00,  4.83it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 5.841880899742984e-19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:05<00:00,  4.57it/s]\n",
      "  0%|          | 1/300 [00:00<00:56,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 1.164124772123504e-20}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:06<00:00,  4.53it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 4.945826870148887e-21}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:03<00:00,  4.72it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 6.779368256825402e-21}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:06<00:00,  4.53it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 3.2609232114350744e-17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:07<00:00,  4.47it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 2.258284984807895e-19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:03<00:00,  4.69it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 1.755336516844742e-19}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:02<00:00,  4.77it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 3.3627720344441613e-18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:04<00:00,  4.66it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 2.8511201449935273e-18}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:04<00:00,  4.69it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 2.0425136118016664e-22}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:03<00:00,  4.70it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 1.2621807424407188e-17}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:05<00:00,  4.55it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 2.3019988499859058e-20}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 300/300 [01:04<00:00,  4.62it/s]\n",
      "  0%|          | 0/300 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 3.283635014126568e-13}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 54/300 [00:11<00:52,  4.71it/s]"
     ]
    }
   ],
   "source": [
    "count=0\n",
    "exception_summary=[]\n",
    "exception_annotations=[]\n",
    "with nlp.disable_pipes(*other_pipes):  # only train NER pipeline\n",
    "    optimizer = nlp.begin_training()\n",
    "    for itn in range(n_iter):\n",
    "        random.shuffle(train_data)\n",
    "        losses = {}\n",
    "        try:\n",
    "            for text, annotations in tqdm(train_data):\n",
    "                nlp.update(\n",
    "                    [text],  # batch of texts\n",
    "                    [annotations],  # batch of annotations\n",
    "                    drop=0.35,  # dropout makes it harder for the model to memorize the training data.\n",
    "                    sgd=optimizer,  # callable to update weights\n",
    "                    losses=losses)\n",
    "        except:\n",
    "            count+=1\n",
    "            exception_summary.append(text)\n",
    "            exception_annotations.append(annotations)   \n",
    "        print(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Save/Download the model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp.to_disk(r'ner_spacy_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Load the model for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp2 = spacy.load(r'ner_spacy_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Test model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = input()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "tokenize the input text\n",
    "'''\n",
    "sentences = sent_tokenize(text)\n",
    "print(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "test the model\n",
    "'''\n",
    "for sentence in sentences:\n",
    "    doc2 = nlp2(sentence)\n",
    "    #print(doc2)\n",
    "    for ent in doc2.ents:\n",
    "        print(ent.label_, \"->\", ent.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}