POS-Host-Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'pytorch_pretrained_bert'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-b6cd67cfaa85>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;31m# from sklearn.model_selection import train_test_split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;31m# from sklearn.metrics import classification_report\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mpytorch_pretrained_bert\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBertTokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBertConfig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpytorch_pretrained_bert\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mBertForTokenClassification\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBertAdam\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pytorch_pretrained_bert'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import csv\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm, trange\n",
    "import bclm\n",
    "\n",
    "import torch\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.metrics import classification_report\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertConfig\n",
    "from pytorch_pretrained_bert import BertForTokenClassification, BertAdam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(3)\n",
    "np.random.seed(3)\n",
    "torch.cuda.manual_seed_all(3)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = bclm.read_dataframe('spmrl', subset='train')\n",
    "train_df = bclm.get_token_df(train, ['upostag'])\n",
    "train_df['token_str'] = train_df['token_str'].str.replace('”','\"')\n",
    "\n",
    "dev = bclm.read_dataframe('spmrl', subset='dev')\n",
    "dev_df = bclm.get_token_df(dev, ['upostag'])\n",
    "dev_df['token_str'] = dev_df['token_str'].str.replace('”','\"')\n",
    "\n",
    "test = bclm.read_dataframe('spmrl', subset='test')\n",
    "test_df = bclm.get_token_df(test, ['upostag'])\n",
    "test_df['token_str'] = test_df['token_str'].str.replace('”','\"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_df['prefix'] = ''\n",
    "train_df['host'] = ''\n",
    "\n",
    "dev_df['prefix'] = ''\n",
    "dev_df['host'] = ''\n",
    "\n",
    "test_df['prefix'] = ''\n",
    "test_df['host'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dev_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prefix_corretion(prefix):\n",
    "    not_prefixes = ['VB^POS','VB^DUMMY_AT','VB^AT', 'IN^DUMMY_AT', 'NN', 'IN', 'P',\n",
    "                   'POS', 'RB', 'AT']\n",
    "    first_prefix = ['DEF^BN^AT', 'CONJ^VB^AT','CONJ^BN^AT', 'CONJ^BN^AT', 'REL^VB^AT',\n",
    "                   'PREPOSITION^RB', 'DEF^yyQUOT', 'IN^yyQUOT', 'CONJ^yyQUOT^DEF',\n",
    "                   'PREPOSITION^yyQUOT^PREPOSITION^DEF', 'REL^yyQUOT', 'REL^IN', 'CONJ^IN',\n",
    "                   'PREPOSITION^yyQUOT', 'PREPOSITION^POS', 'PREPOSITION^IN', 'PREPOSITION^yyQUOT^DEF',\n",
    "                   'REL^yyQUOT^PREPOSITION', 'CONJ^yyQUOT', 'REL^AT']\n",
    "    \n",
    "    two_first_prefixes = ['CONJ^PREPOSITION^yyQUOT', 'PREPOSITION^DEF^yyQUOT', 'CONJ^DEF^yyQUOT']\n",
    "    \n",
    "    if prefix in not_prefixes:\n",
    "        host = prefix\n",
    "        prefix = '-'\n",
    "    \n",
    "    elif prefix in first_prefix:\n",
    "        tag_list = prefix.split('^')\n",
    "        prefix = tag_list[0]\n",
    "        host = '^'.join(tag_list[1:])\n",
    "        \n",
    "    elif prefix in two_first_prefixes:\n",
    "        tag_list = prefix.split('^')\n",
    "        prefix = '^'.join(tag_list[0:2])\n",
    "        host = '^'.join(tag_list[2:])\n",
    "        \n",
    "    else:\n",
    "        host = \"\"\n",
    "        prefix = prefix\n",
    "        \n",
    "    return prefix, host\n",
    "        \n",
    "\n",
    "def full_tag_to_prefix_host(df):\n",
    "    for index in df.index:\n",
    "        full_tag = df.at[index, 'upostag']\n",
    "        full_tag_list = full_tag.split('^')\n",
    "        if len(full_tag_list) > 1:\n",
    "            prefix = '^'.join(full_tag_list[:-1])\n",
    "            prefix, host = prefix_corretion(prefix)\n",
    "            df.at[index, 'prefix'] = prefix\n",
    "            if len(host) > 0:\n",
    "                host += '^'\n",
    "            host += full_tag_list[-1]\n",
    "            df.at[index, 'host'] = host\n",
    "        else:\n",
    "            df.at[index, 'prefix'] = '-'\n",
    "            df.at[index, 'host'] = full_tag_list[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "full_tag_to_prefix_host(train_df)\n",
    "full_tag_to_prefix_host(dev_df)\n",
    "full_tag_to_prefix_host(test_df)\n",
    "\n",
    "train_df.tail(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train2 = train[train['xpostag'] != train['upostag']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train2['xpostag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train2[train2['xpostag'] == 'S_PRN']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev2 = dev[dev['xpostag'] != dev['upostag']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dev2['xpostag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(dev2)/len(dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([train_df, dev_df, test_df])\n",
    "data.head()\n",
    "tag_vals = list(set(data[\"host\"].values))\n",
    "tags = ['PAD'] + tag_vals\n",
    "tag2idx = {tag:idx for idx, tag in enumerate(tags)}\n",
    "idx2tag = {idx:tag for idx, tag in enumerate(tags)}\n",
    "\n",
    "print(tag2idx)\n",
    "# print(idx2tag)\n",
    "print(len(tags))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# for label in tag_vals:\n",
    "#     print(data[data['prefix'] == 'PREPOSITIONIN'])\n",
    "    \n",
    "data[data['host'] == 'IN^DUMMY_AT^PRP']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class sentenceGetter(object):\n",
    "    def __init__(self, data, max_sent=None):\n",
    "        self.index = 0\n",
    "        self.max_sent = max_sent\n",
    "        self.tokens = data['token_str']\n",
    "        self.labels = data['host']\n",
    "        #for evaluating by word-accuracy\n",
    "        self.correspondingToken = data['token_id']\n",
    "        self.orig_sent_id = data['sent_id']\n",
    "    \n",
    "    def sentences(self):\n",
    "        sent = []\n",
    "        counter = 0\n",
    "        \n",
    "        for token,label, corres_tok, sent_id in zip(self.tokens, self.labels, self.correspondingToken, self.orig_sent_id):\n",
    "            sent.append((token, label, corres_tok, sent_id))\n",
    "            if token.strip() == \".\":\n",
    "                yield sent\n",
    "                sent = []\n",
    "                counter += 1\n",
    "            if self.max_sent is not None and counter >= self.max_sent:\n",
    "                return\n",
    "\n",
    "train_getter = sentenceGetter(train_df)\n",
    "dev_getter = sentenceGetter(dev_df)\n",
    "test_getter = sentenceGetter(test_df)\n",
    "\n",
    "train_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "train_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in train_getter.sentences()]\n",
    "\n",
    "# dev_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "# dev_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "# dev_corresTokens = [[corres_tok for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "# dev_sent_ids = [[sent_id for token, label, corres_tok, sent_id in sent] for sent in dev_getter.sentences()]\n",
    "\n",
    "# test_sentences = [[token for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "# test_labels = [[label for token, label, corres_tok, sent_id in sent] for sent in test_getter.sentences()]\n",
    "\n",
    "print(train_sentences[10])\n",
    "print(train_labels[10])\n",
    "\n",
    "# print(len(dev_sentences))\n",
    "\n",
    "# del dev_sentences[296]\n",
    "# del dev_labels[296]\n",
    "# del dev_corresTokens[296]\n",
    "# del dev_sent_ids[296]\n",
    "\n",
    "\n",
    "# del dev_sentences[226]\n",
    "# del dev_labels[226]\n",
    "# del dev_corresTokens[226]\n",
    "# del dev_sent_ids[226]\n",
    "\n",
    "\n",
    "# del dev_sentences[57]\n",
    "# del dev_labels[57]\n",
    "# del dev_corresTokens[57]\n",
    "# del dev_sent_ids[57]\n",
    "\n",
    "\n",
    "# del dev_sentences[49]\n",
    "# del dev_labels[49]\n",
    "# del dev_corresTokens[49]\n",
    "# del dev_sent_ids[49]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "torch.cuda.set_device(2)\n",
    "\n",
    "print(\"Device: \" + str(device))\n",
    "print(\"Number of gpus: \" + str(n_gpu))\n",
    "print(\"Name of gpu: \" + torch.cuda.get_device_name(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_LEN = 150\n",
    "bs = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased')\n",
    "def tokenize(sentences, orig_labels):\n",
    "    tokenized_texts = []\n",
    "    labels = []\n",
    "    for sent, sent_labels in zip(sentences, orig_labels):\n",
    "        bert_tokens = []\n",
    "        bert_labels = []\n",
    "        for orig_token, orig_label in zip(sent, sent_labels):\n",
    "            b_tokens = tokenizer.tokenize(orig_token)\n",
    "            bert_tokens.extend(b_tokens)\n",
    "            for b_token in b_tokens:\n",
    "                bert_labels.append(orig_label)\n",
    "        tokenized_texts.append(bert_tokens)\n",
    "        labels.append(bert_labels)\n",
    "        assert len(bert_tokens) == len(bert_labels)\n",
    "    return tokenized_texts, labels\n",
    "\n",
    "train_tokenized_texts, train_tokenized_labels = tokenize(train_sentences, train_labels)\n",
    "print(train_tokenized_texts[10])\n",
    "print(train_tokenized_labels[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentences_and_labels(tokenized_texts, labels):\n",
    "    input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],\n",
    "                              maxlen = MAX_LEN, dtype = \"float32\", truncating = \"post\", padding = \"post\", value = tag2idx['PAD'])\n",
    "    tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in labels], \n",
    "                         maxlen = MAX_LEN, value = tag2idx['PAD'], padding = \"post\",\n",
    "                        dtype = \"float32\", truncating = \"post\")\n",
    "    attention_masks = [[float(i>0) for i in ii] for ii in input_ids]\n",
    "    return input_ids, tags, attention_masks\n",
    "\n",
    "input_ids, tags, attention_masks = pad_sentences_and_labels(train_tokenized_texts, train_tokenized_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tr_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "tr_tags = torch.tensor(tags, dtype=torch.long)\n",
    "tr_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)\n",
    "# train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, batch_size=bs, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = BertForTokenClassification.from_pretrained('bert-base-multilingual-cased', num_labels=len(tag2idx))\n",
    "model.cuda()\n",
    "FULL_FINETUNING = True\n",
    "if FULL_FINETUNING:\n",
    "    param_optimizer = list(model.named_parameters())\n",
    "    no_decay = ['bias', 'gamma', 'beta']\n",
    "    optimizer_grouped_parameters = [\n",
    "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.01},\n",
    "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
    "         'weight_decay_rate': 0.0}\n",
    "    ]\n",
    "else:\n",
    "    param_optimizer = list(model.classifier.named_parameters())\n",
    "    optimizer_grouped_parameters = [{\"params\": [p for n, p in param_optimizer]}]\n",
    "\n",
    "optimizer = Adam(optimizer_grouped_parameters, lr=3e-5)\n",
    "\n",
    "from seqeval.metrics import f1_score\n",
    "\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "#     print (pred_flat, labels_flat)\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "epochs = 15\n",
    "max_grad_norm = 1.0\n",
    "\n",
    "for _ in trange(epochs, desc=\"Epoch\"):\n",
    "    # TRAIN loop\n",
    "    model.train()\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        # add batch to gpu\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        # forward pass\n",
    "        loss = model(b_input_ids, token_type_ids=None,\n",
    "                     attention_mask=b_input_mask, labels=b_labels)\n",
    "        # backward pass\n",
    "        loss.backward()\n",
    "        # track train loss\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += b_input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "        # gradient clipping\n",
    "        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=max_grad_norm)\n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        model.zero_grad()\n",
    "    # print train loss per epoch\n",
    "    print(\"Train loss: {}\".format(tr_loss / nb_tr_steps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function receives a sentence with its labels, and the tokenized sentence and labels\n",
    "def aggr_toks_labels_tags(orig_words, orig_labels, tok_wordps, tok_labels, predicted_tags):\n",
    "    \n",
    "    joint_tokens = []\n",
    "    joint_labels = []\n",
    "    joint_predicted = []\n",
    "#     joint_test = []\n",
    "    \n",
    "    for word in orig_words:\n",
    "        aggregated_tokenized = \"\"\n",
    "        aggregated_label = \"\"\n",
    "        aggregated_predicted = \"\"\n",
    "        aggregated_test = \"\"\n",
    "        \n",
    "        while aggregated_tokenized != word:\n",
    "#             print(len(tok_sent))\n",
    "            tmpTok = tok_wordps.pop(0)\n",
    "#             print(tmpTok)\n",
    "#             print(joint_tokens)\n",
    "            if tmpTok.startswith(\"##\"):\n",
    "                tmpTok = tmpTok[2:]\n",
    "                \n",
    "            tmpLab = tok_labels.pop(0)\n",
    "#             if aggregated_label == \"\":\n",
    "            aggregated_label += '^'\n",
    "            aggregated_label += tmpLab\n",
    "\n",
    "                \n",
    "            tmpPred = predicted_tags.pop(0)\n",
    "#             print(tmpPred)\n",
    "\n",
    "            aggregated_predicted += '^'\n",
    "            aggregated_predicted += tmpPred\n",
    "#             if aggregated_predicted == \"\":\n",
    "#                 aggregated_predicted = tmpPred\n",
    "                \n",
    "#             tmpTest = test_tags.pop(0)\n",
    "#             if aggregated_test == \"\":\n",
    "#                 aggregated_test = tmpTest\n",
    "                \n",
    "            aggregated_tokenized += tmpTok\n",
    "#             print(aggregated_tokenized)\n",
    "            \n",
    "        joint_tokens.append(aggregated_tokenized)\n",
    "        joint_labels.append(aggregated_label)\n",
    "        joint_predicted.append(aggregated_predicted)\n",
    "#         joint_test.append(aggregated_test)\n",
    "        \n",
    "    assert len(joint_tokens) == len(orig_words)\n",
    "    assert len(joint_tokens) == len(joint_predicted)\n",
    "    return joint_tokens, joint_labels, joint_predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=2).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)\n",
    "\n",
    "def delete_pads_from_preds(predicted_tags, test_tags):\n",
    "    clean_predicted = []\n",
    "    clean_test = []\n",
    "    \n",
    "    for ix in range(0, len(test_tags)):\n",
    "        if test_tags[ix] != 'PAD':\n",
    "            clean_predicted.append(predicted_tags[ix])\n",
    "            clean_test.append(test_tags[ix])\n",
    "            \n",
    "    return clean_predicted, clean_test\n",
    "    \n",
    "def calculate_accuracy(df):\n",
    "    numOfCorrectPredictions = 0\n",
    "    for index in df.index:\n",
    "        orig_pos = df.at[index, 'test_tag']\n",
    "        pred_pos = df.at[index, 'predicted_tag']\n",
    "        if orig_pos == pred_pos:\n",
    "            numOfCorrectPredictions += 1\n",
    "    return numOfCorrectPredictions/len(df)\n",
    "                \n",
    "def test_model(sentence, labels, tok_sent, tok_labels, corres_tokens, sent_id):\n",
    "    input_ids, tags, attention_masks = pad_sentences_and_labels([tok_sent], [tok_labels])\n",
    "\n",
    "    val_inputs = torch.tensor(input_ids, dtype=torch.long)\n",
    "    val_tags = torch.tensor(tags, dtype=torch.long)\n",
    "    val_masks = torch.tensor(attention_masks, dtype=torch.long)\n",
    "\n",
    "    test_data = TensorDataset(val_inputs, val_masks, val_tags)\n",
    "    test_sampler = SequentialSampler(test_data)\n",
    "    test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=bs)\n",
    "\n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    predictions, true_labels = [], []\n",
    "    counter = 0\n",
    "    for batch in test_dataloader:\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "\n",
    "        with torch.no_grad():\n",
    "            tmp_eval_loss = model(b_input_ids, token_type_ids=None,\n",
    "                                attention_mask=b_input_mask, labels=b_labels)\n",
    "            logits = model(b_input_ids, token_type_ids=None,\n",
    "                         attention_mask=b_input_mask)\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        predictions.append([list(p) for p in np.argmax(logits, axis=2)])\n",
    "        \n",
    "        true_labels.append(label_ids)\n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "\n",
    "        eval_loss += tmp_eval_loss.mean().item()\n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "        nb_eval_examples += b_input_ids.size(0)\n",
    "        nb_eval_steps += 1\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    \n",
    "    pred_tags = [idx2tag[p_ii] for p in predictions for p_i in p for p_ii in p_i]\n",
    "    test_tags = [idx2tag[l_ii] for l in true_labels for l_i in l for l_ii in l_i]\n",
    "#     print(list(zip(pred_tags, test_tags)))\n",
    "    # -----------------------------------------------------------------------\n",
    "    clean_predicted, clean_test = delete_pads_from_preds(pred_tags, test_tags)\n",
    "    joint_tokenized, joint_labels, preds, tests = aggr_toks_labels_tags(sentence, labels, tok_sent, tok_labels, \n",
    "                                                                        clean_predicted, clean_test)\n",
    "    \n",
    "    tmp = {'word': sentence, 'orig_label': labels, 'predicted_tag': preds, 'test_tag': tests, \n",
    "           'corresToken': corres_tokens, 'sent_id': sent_id}\n",
    "    tmp_df = pd.DataFrame(data=tmp)\n",
    "    # -----------------------------------------------------------------------\n",
    "    \n",
    "#     y_true = pd.Series(test_tags)\n",
    "#     y_pred = pd.Series(pred_tags)\n",
    "#     cross_tab = pd.crosstab(y_true, y_pred, rownames=['Real Label'], colnames=['Prediction'], margins=True)\n",
    "#     report = classification_report(y_true, y_pred)\n",
    "#     print(report)\n",
    "#     print(tmp_df)\n",
    "    return tmp_df\n",
    "\n",
    "full_df = pd.DataFrame()\n",
    "dev_tokenized_texts, dev_tokenized_labels = tokenize(dev_sentences, dev_labels)\n",
    "for sent, label, tok_sent, tok_label, corresTokens, sent_id in zip(dev_sentences, dev_labels, dev_tokenized_texts, \n",
    "                                                                   dev_tokenized_labels, dev_corresTokens, \n",
    "                                                                   dev_sent_ids):\n",
    "    test_df = test_model(sent, label, tok_sent, tok_label, corresTokens, sent_id)\n",
    "    full_df = full_df.append(test_df, ignore_index=True, sort=False)\n",
    "\n",
    "# full_df\n",
    "f1_accuracy = calculate_accuracy(full_df)\n",
    "print(\"Accuracy (F1): = {}\".format(f1_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "full_df.iloc[0:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rename_prefix_output(prefix_df):\n",
    "    prefix_df.rename(columns={\"orig_label\": \"orig_prefix\", \"predicted_tag\": \"predicted_prefix\"})\n",
    "    \n",
    "def rename_host_output(host_df):\n",
    "    host_df.rename(columns={\"orig_label\": \"orig_host\", \"predicted_tag\": \"predicted_host\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rename_prefix_output(full_df)\n",
    "full_df.to_csv('host-5-setting1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "host_df = pd.read_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def join_prefix_host(prefix_df, host_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from more_itertools import unique_everseen\n",
    "\n",
    "def unique_vals_to_list(df):\n",
    "    for index in df.index:\n",
    "        joint_pred = df.at[index, 'predicted_tag']\n",
    "        joint_orig = df.at[index, 'orig_label']\n",
    "        \n",
    "        predicted_tag_list = joint_pred.split('^')\n",
    "        predicted_tag_list_no_empty = list(filter(None, predicted_tag_list))\n",
    "        original_tag_list = joint_orig.split('^')\n",
    "        original_tag_list_no_empty = list(filter(None, original_tag_list))\n",
    "\n",
    "        \n",
    "        df.at[index, 'predicted_tag'] = list(unique_everseen(predicted_tag_list_no_empty))\n",
    "        df.at[index, 'orig_label'] = list(unique_everseen(original_tag_list_no_empty))\n",
    "        \n",
    "        \n",
    "unique_vals_to_list(full_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exact_match_accuracy(df):\n",
    "    exact_matches = 0\n",
    "    for index in df.index:\n",
    "        if df.at[index, 'orig_label'] == df.at[index, 'predicted_tag']:\n",
    "            exact_matches += 1\n",
    "            \n",
    "    return exact_matches\n",
    "\n",
    "print(\"Exact Match Accuracy = {0:.2f}%\".format(exact_match_accuracy(full_df)/len(full_df) * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def existence_accuracy(df):\n",
    "    # correct tag = appeared in predicted and in gold\n",
    "    total_orig_num_of_labels = 0\n",
    "    total_predicted_num_of_labels = 0\n",
    "    total_num_of_correct_tags = 0\n",
    "    \n",
    "    for index in df.index:\n",
    "        orig_list = df.at[index, 'orig_label']\n",
    "        predicted_list = df.at[index, 'predicted_tag']\n",
    "        total_orig_num_of_labels += len(orig_list)\n",
    "        total_predicted_num_of_labels += len(predicted_list)\n",
    "        total_num_of_correct_tags += len(set(orig_list).intersection(set(predicted_list)))\n",
    "        \n",
    "    precision = total_num_of_correct_tags / total_predicted_num_of_labels * 100\n",
    "    recall = total_num_of_correct_tags / total_orig_num_of_labels * 100\n",
    "    f1 = 2*precision*recall/(precision+recall)\n",
    "    \n",
    "    print(\"Precision: {0:.2f}%\".format(precision))\n",
    "    print(\"Recall: {0:.2f}%\".format(recall))\n",
    "    print(\"F1: {0:.2f}%\".format(f1))\n",
    "    \n",
    "existence_accuracy(full_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating standard df for multi-label pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class StdDf(object):\n",
    "    def __init__(self):\n",
    "        self.data = pd.concat([train, dev, test])\n",
    "        self.tag_vals = list(set(data['upostag'].values))\n",
    "        self.df = pd.DataFrame(columns = tag_vals)\n",
    "        self.create_multilabel_df()\n",
    "        \n",
    "    def create_multilabel_df(self):        \n",
    "        self.df['sent_id'] = '0'\n",
    "        self.df['token_id'] = '0'\n",
    "        self.df['token'] = ''\n",
    "        cols = self.df.columns.tolist()\n",
    "        cols = cols[-3:] + cols[:-3]\n",
    "        self.df = self.df[cols]\n",
    "        self.df[self.df.columns[3:]] = 0\n",
    "\n",
    "\n",
    "std_df = StdDf()\n",
    "std_df.df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def raw_to_multilabel_df(raw_df):\n",
    "    multi_df = StdDf()\n",
    "    for index in raw_df.index:\n",
    "        multi_df.df.at[index, 'sent_id'] = raw_df.at[index, 'sent_id']\n",
    "        multi_df.df.at[index, 'token_id'] = raw_df.at[index, 'token_id']\n",
    "        multi_df.df.at[index, 'token'] = raw_df.at[index, 'token_str']\n",
    "        \n",
    "        l_pos_tags = raw_df.at[index, 'upostag']\n",
    "        multi_df.df.at[index, l_pos_tags] = 1\n",
    "        \n",
    "    return multi_df.df\n",
    "        \n",
    "    \n",
    "multi_dev_df = raw_to_multilabel_df(dev_df)\n",
    "multi_dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "multi_dev_df[multi_dev_df.columns[3:]] = 0\n",
    "\n",
    "for index in dev_df.index:\n",
    "    l_pos_tags = dev_df.at[index, 'upostag']\n",
    "    multi_dev_df.at[index, l_pos_tags] = 1\n",
    "    \n",
    "multi_dev_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}