ner_2_2021.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "ner_2_2021.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/StefanFischer/SAKI-Project2/blob/main/ner_2_2021.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7yjL03CKOoEk"
      },
      "source": [
        "Getting started with Spacy<br>\n",
        "Import data.<br>\n",
        "We repeat the preprocessing from the previous homework."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vOxAAu3cOm9D",
        "outputId": "727aaa49-7996-4342-e036-badfb20f8b2d"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HOc5LrSCPSEC"
      },
      "source": [
        "import os"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jd_X8KyOPWlJ"
      },
      "source": [
        "os.chdir( \"/content/gdrive/MyDrive/flair\" ) "
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NBNxx2f8Pcmq"
      },
      "source": [
        "path_to_data = os.getcwd() + '/Entity Recognition in Resumes.json'"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dUKwMlfkaR2n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "76123614-8611-42cb-fcf3-d7e73f6167b1"
      },
      "source": [
        "myfile = open( path_to_data, \"r\", encoding = \"utf-8\" )\n",
        "\n",
        "imported_data = []\n",
        "\n",
        "for datum in myfile:\n",
        "    \n",
        "    # TODO process data\n",
        "    imported_data.append(datum)\n",
        "\n",
        "myfile.close()\n",
        "\n",
        "# TODO print first line\n",
        "print(imported_data[0])\n",
        "\n",
        "# TODO print how many resumees were read in\n",
        "print(\"amount resumees : \"+ str(len(imported_data)))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{\"content\": \"Afreen Jamadar\\nActive member of IIIT Committee in Third year\\n\\nSangli, Maharashtra - Email me on Indeed: indeed.com/r/Afreen-Jamadar/8baf379b705e37c6\\n\\nI wish to use my knowledge, skills and conceptual understanding to create excellent team\\nenvironments and work consistently achieving organization objectives believes in taking initiative\\nand work to excellence in my work.\\n\\nWORK EXPERIENCE\\n\\nActive member of IIIT Committee in Third year\\n\\nCisco Networking -  Kanpur, Uttar Pradesh\\n\\norganized by Techkriti IIT Kanpur and Azure Skynet.\\nPERSONALLITY TRAITS:\\n• Quick learning ability\\n• hard working\\n\\nEDUCATION\\n\\nPG-DAC\\n\\nCDAC ACTS\\n\\n2017\\n\\nBachelor of Engg in Information Technology\\n\\nShivaji University Kolhapur -  Kolhapur, Maharashtra\\n\\n2016\\n\\nSKILLS\\n\\nDatabase (Less than 1 year), HTML (Less than 1 year), Linux. (Less than 1 year), MICROSOFT\\nACCESS (Less than 1 year), MICROSOFT WINDOWS (Less than 1 year)\\n\\nADDITIONAL INFORMATION\\n\\nTECHNICAL SKILLS:\\n\\n• Programming Languages: C, C++, Java, .net, php.\\n• Web Designing: HTML, XML\\n• Operating Systems: Windows […] Windows Server 2003, Linux.\\n• Database: MS Access, MS SQL Server 2008, Oracle 10g, MySql.\\n\\nhttps://www.indeed.com/r/Afreen-Jamadar/8baf379b705e37c6?isid=rex-download&ikw=download-top&co=IN\",\"annotation\":[{\"label\":[\"Email Address\"],\"points\":[{\"start\":1155,\"end\":1198,\"text\":\"indeed.com/r/Afreen-Jamadar/8baf379b705e37c6\"}]},{\"label\":[\"Links\"],\"points\":[{\"start\":1143,\"end\":1239,\"text\":\"https://www.indeed.com/r/Afreen-Jamadar/8baf379b705e37c6?isid=rex-download&ikw=download-top&co=IN\"}]},{\"label\":[\"Skills\"],\"points\":[{\"start\":743,\"end\":1140,\"text\":\"Database (Less than 1 year), HTML (Less than 1 year), Linux. (Less than 1 year), MICROSOFT\\nACCESS (Less than 1 year), MICROSOFT WINDOWS (Less than 1 year)\\n\\nADDITIONAL INFORMATION\\n\\nTECHNICAL SKILLS:\\n\\n• Programming Languages: C, C++, Java, .net, php.\\n• Web Designing: HTML, XML\\n• Operating Systems: Windows […] Windows Server 2003, Linux.\\n• Database: MS Access, MS SQL Server 2008, Oracle 10g, MySql.\"}]},{\"label\":[\"Graduation Year\"],\"points\":[{\"start\":729,\"end\":732,\"text\":\"2016\"}]},{\"label\":[\"College Name\"],\"points\":[{\"start\":675,\"end\":702,\"text\":\"Shivaji University Kolhapur \"}]},{\"label\":[\"Degree\"],\"points\":[{\"start\":631,\"end\":672,\"text\":\"Bachelor of Engg in Information Technology\"}]},{\"label\":[\"Graduation Year\"],\"points\":[{\"start\":625,\"end\":629,\"text\":\"2017\\n\"}]},{\"label\":[\"College Name\"],\"points\":[{\"start\":614,\"end\":622,\"text\":\"CDAC ACTS\"}]},{\"label\":[\"Degree\"],\"points\":[{\"start\":606,\"end\":611,\"text\":\"PG-DAC\"}]},{\"label\":[\"Companies worked at\"],\"points\":[{\"start\":438,\"end\":453,\"text\":\"Cisco Networking\"}]},{\"label\":[\"Email Address\"],\"points\":[{\"start\":104,\"end\":147,\"text\":\"indeed.com/r/Afreen-Jamadar/8baf379b705e37c6\"}]},{\"label\":[\"Location\"],\"points\":[{\"start\":62,\"end\":67,\"text\":\"Sangli\"}]},{\"label\":[\"Name\"],\"points\":[{\"start\":0,\"end\":13,\"text\":\"Afreen Jamadar\"}]}],\"extras\":null,\"metadata\":{\"first_done_at\":1527844872000,\"last_updated_at\":1537724086000,\"sec_taken\":0,\"last_updated_by\":\"BIQNZm4INNfvByMqkaVwVt6OZTv2\",\"status\":\"done\",\"evaluation\":\"CORRECT\"}}\n",
            "\n",
            "amount resumees : 701\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SxQojbZyPrkh"
      },
      "source": [
        "import json"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Axwa389baXDN"
      },
      "source": [
        "mapped_data = [ json.loads( datum ) for datum in imported_data  ]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d2FaAX-xabje",
        "outputId": "d62a0a7f-4a73-4b94-abb4-0da7c2cb98c7"
      },
      "source": [
        "## data conversion method\n",
        "def convert_data(data):\n",
        "    \"\"\"\n",
        "    Creates NER training data in Spacy format from JSON dataset\n",
        "    Outputs the Spacy training data which can be used for Spacy training.\n",
        "    \"\"\"\n",
        "    text = data['content']\n",
        "    entities = []\n",
        "    if data['annotation'] is not None:\n",
        "        for annotation in data['annotation']:\n",
        "            # only a single point in text annotation.\n",
        "            point = annotation['points'][0]\n",
        "            labels = annotation['label']\n",
        "            # handle both list of labels or a single label.\n",
        "            if not isinstance(labels, list):\n",
        "                labels = [labels]\n",
        "            for label in labels:\n",
        "                # dataturks indices are both inclusive [start, end] but spacy is not [start, end)\n",
        "                entities.append((point['start'], point['end'] + 1, label))\n",
        "    return (text, {\"entities\": entities})\n",
        "   \n",
        "## TODO using a loop or list comprehension, convert each resume in mapped_data using the convert function above, \n",
        "## storing the result\n",
        "converted_resumes = [convert_data(entry) for entry in mapped_data]\n",
        "## TODO print the number of resumes in converted resumes \n",
        "print(len(converted_resumes))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "701\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GapDFuttaqpk"
      },
      "source": [
        "# TODO filter out the resumees whose entities have no entries.\n",
        "def empty_entity_list(data):\n",
        "  entityList = data[1][\"entities\"]\n",
        "  if len(entityList) == 0:\n",
        "    return True\n",
        "  return False\n",
        "\n",
        "converted_complete_resumees = [data for data in converted_resumes if not empty_entity_list(data)]"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1-UWPUvfbo0l"
      },
      "source": [
        "Up until now, you could reuse the code from the previous notebook, now, something new comes."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CYhNshfBbl8d",
        "outputId": "080f6007-4a31-4b38-fcb1-0dd155f11712"
      },
      "source": [
        "import spacy\n",
        "spacy.prefer_gpu()\n",
        "nlp = spacy.load('en')\n",
        "print(nlp)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<spacy.lang.en.English object at 0x7f0ed9113d90>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9AqzhVdHb-xO"
      },
      "source": [
        "__nlp__ is Spacy's English language model. For this model, a pretrained NER-model exists."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TDHFtX6Lb-CY",
        "outputId": "0ee50bc7-3289-43c1-c913-eba963beb3c8"
      },
      "source": [
        "ner = nlp.get_pipe('ner')\n",
        "labels = ner.labels\n",
        "print(labels)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "('CARDINAL', 'DATE', 'EVENT', 'FAC', 'GPE', 'LANGUAGE', 'LAW', 'LOC', 'MONEY', 'NORP', 'ORDINAL', 'ORG', 'PERCENT', 'PERSON', 'PRODUCT', 'QUANTITY', 'TIME', 'WORK_OF_ART')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sBy9Uo20cZH2"
      },
      "source": [
        "__Question 1__: Explain the labels __GPE__, __FAC__, __NORP__.<br>\n",
        "Which of these labels from __ner__ do you think will Spacy recognize in the resumees?<br>\n",
        "From Spacy Website:\n",
        "\n",
        "GPE: Geopolitical entity, i.e. countries, cities, states.\n",
        "\n",
        "NORP: Nationalities or religious or political groups\n",
        "\n",
        "FAC: buildings, airports etc.\n",
        "\n",
        "__Task 1__: choose a resumee."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4yXayIFffE7F",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3a83211f-c006-4bf8-84ce-d68556645396"
      },
      "source": [
        "# TODO get a single resume text and print it out.\n",
        "restxt = converted_complete_resumees[0][0]\n",
        "## print it out, removing extraneous spaces\n",
        "print(\"\\n\".join(restxt.split('\\n\\n')))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Afreen Jamadar\n",
            "Active member of IIIT Committee in Third year\n",
            "Sangli, Maharashtra - Email me on Indeed: indeed.com/r/Afreen-Jamadar/8baf379b705e37c6\n",
            "I wish to use my knowledge, skills and conceptual understanding to create excellent team\n",
            "environments and work consistently achieving organization objectives believes in taking initiative\n",
            "and work to excellence in my work.\n",
            "WORK EXPERIENCE\n",
            "Active member of IIIT Committee in Third year\n",
            "Cisco Networking -  Kanpur, Uttar Pradesh\n",
            "organized by Techkriti IIT Kanpur and Azure Skynet.\n",
            "PERSONALLITY TRAITS:\n",
            "• Quick learning ability\n",
            "• hard working\n",
            "EDUCATION\n",
            "PG-DAC\n",
            "CDAC ACTS\n",
            "2017\n",
            "Bachelor of Engg in Information Technology\n",
            "Shivaji University Kolhapur -  Kolhapur, Maharashtra\n",
            "2016\n",
            "SKILLS\n",
            "Database (Less than 1 year), HTML (Less than 1 year), Linux. (Less than 1 year), MICROSOFT\n",
            "ACCESS (Less than 1 year), MICROSOFT WINDOWS (Less than 1 year)\n",
            "ADDITIONAL INFORMATION\n",
            "TECHNICAL SKILLS:\n",
            "• Programming Languages: C, C++, Java, .net, php.\n",
            "• Web Designing: HTML, XML\n",
            "• Operating Systems: Windows […] Windows Server 2003, Linux.\n",
            "• Database: MS Access, MS SQL Server 2008, Oracle 10g, MySql.\n",
            "https://www.indeed.com/r/Afreen-Jamadar/8baf379b705e37c6?isid=rex-download&ikw=download-top&co=IN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W15EXUkiNZWS"
      },
      "source": [
        "Next, we let __nlp__ process that single resumee.<br>\n",
        "__Task 2__: print the results in __doc__. For each result, print the underlying text and the label."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cp961WAxfW8N",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4a70e49d-74a3-41e5-c0f1-ce7382507e0d"
      },
      "source": [
        "doc = nlp(restxt)\n",
        "# TODO  Print the results in doc. For each result, print the text and the label.\n",
        "for ent in doc.ents:\n",
        "    print(ent.text, ent.start_char, ent.end_char, ent.label_)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Afreen Jamadar 0 14 PERSON\n",
            "IIIT Committee 32 46 ORG\n",
            "Third year 50 60 DATE\n",
            "Sangli 62 68 ORG\n",
            "Maharashtra - Email 70 89 ORG\n",
            "IIIT Committee 408 422 ORG\n",
            "Third year 426 436 DATE\n",
            "Cisco Networking -   438 458 ORG\n",
            "Kanpur 458 464 GPE\n",
            "Uttar 466 471 GPE\n",
            "Techkriti IIT Kanpur 494 514 ORG\n",
            "Azure Skynet 519 531 WORK_OF_ART\n",
            "2017 625 629 DATE\n",
            "Bachelor of Engg in Information Technology 631 673 ORG\n",
            "Shivaji University 675 693 ORG\n",
            "Kolhapur 694 702 PERSON\n",
            "Kolhapur 706 714 GPE\n",
            "Maharashtra 716 727 ORG\n",
            "2016 729 733 DATE\n",
            "SKILLS 735 741 ORG\n",
            "Less than 1 year 753 769 DATE\n",
            "Less than 1 year 778 794 DATE\n",
            "Linux 797 802 PERSON\n",
            "Less than 1 year 805 821 DATE\n",
            "MICROSOFT 824 833 ORG\n",
            "ACCESS 834 840 ORG\n",
            "Less than 1 year 842 858 DATE\n",
            "MICROSOFT 861 870 ORG\n",
            "Less than 1 year 880 896 DATE\n",
            "C++ 970 973 CARDINAL\n",
            "Java 975 979 PERSON\n",
            "XML 1015 1018 ORG\n",
            "Operating Systems 1021 1038 ORG\n",
            "2003 1067 1071 DATE\n",
            "Linux 1073 1078 PERSON\n",
            "MS Access 1092 1101 ORG\n",
            "SQL 1106 1109 ORG\n",
            "2008 1117 1121 DATE\n",
            "10 1130 1132 CARDINAL\n",
            "MySql 1135 1140 PERSON\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xok9PDMplMNe"
      },
      "source": [
        "__Question 2__: How well did Spacy perform at recognizing the labels for this text?<br>\n",
        "The more common entities like date were detected quite good, but for entities skills and companies worked at the classification was bad. Often companies and universities were classified as organizations.\n",
        "\n",
        "When Spacy predicted the labels for this resumee, a pretrained model was used.<br>\n",
        "__Task 3__: print for this resumee the original labels and their corresponding text."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V8eIm9lXlznU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "57018b27-51d3-40b4-ab19-5d62d8e198d8"
      },
      "source": [
        "# TODO print for that resumee the original labels and their corresponding text.\n",
        "\n",
        "labeled_ents = converted_complete_resumees[0][1][\"entities\"]\n",
        "text = converted_complete_resumees[0][0]\n",
        "\n",
        "for ent in labeled_ents:\n",
        "    print(text[ent[0]:ent[1]] +\" \"+ str(ent[0]) +\" \"+ str(ent[1]) +\" \"+ str(ent[2]))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "indeed.com/r/Afreen-Jamadar/8baf379b705e37c6 1155 1199 Email Address\n",
            "https://www.indeed.com/r/Afreen-Jamadar/8baf379b705e37c6?isid=rex-download&ikw=download-top&co=IN 1143 1240 Links\n",
            "Database (Less than 1 year), HTML (Less than 1 year), Linux. (Less than 1 year), MICROSOFT\n",
            "ACCESS (Less than 1 year), MICROSOFT WINDOWS (Less than 1 year)\n",
            "\n",
            "ADDITIONAL INFORMATION\n",
            "\n",
            "TECHNICAL SKILLS:\n",
            "\n",
            "• Programming Languages: C, C++, Java, .net, php.\n",
            "• Web Designing: HTML, XML\n",
            "• Operating Systems: Windows […] Windows Server 2003, Linux.\n",
            "• Database: MS Access, MS SQL Server 2008, Oracle 10g, MySql. 743 1141 Skills\n",
            "2016 729 733 Graduation Year\n",
            "Shivaji University Kolhapur  675 703 College Name\n",
            "Bachelor of Engg in Information Technology 631 673 Degree\n",
            "2017\n",
            " 625 630 Graduation Year\n",
            "CDAC ACTS 614 623 College Name\n",
            "PG-DAC 606 612 Degree\n",
            "Cisco Networking 438 454 Companies worked at\n",
            "indeed.com/r/Afreen-Jamadar/8baf379b705e37c6 104 148 Email Address\n",
            "Sangli 62 68 Location\n",
            "Afreen Jamadar 0 14 Name\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aBHplvL2KHXO"
      },
      "source": [
        "__Question 3__: Compare the performance of the pretrained model __nlp__ and the true labels. Did Spacy perform well? If not, try to explain why.<br>\n",
        "\n",
        "Spacy did not perform very well, as there is additional expert knowledge for computer science skills needed to detect those. The pretrained nlp is not trained for this very specialized goal.\n",
        "\n",
        "__Task 4__: Remember last homework? You chose three labels. Select all the resumees, in which all three labels appear."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wELEGAoEKDwf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b679ec97-f3a2-458c-ac2d-f0c42f092ebc"
      },
      "source": [
        "# TODO fill in your chosen labels in chosen_entity_labels\n",
        "chosen_entity_labels = [ \"Skills\", \"Companies worked at\", \"Degree\" ]\n",
        "\n",
        "## this method gathers all resumes which have all of the chosen entites above.\n",
        "def gather_candidates(dataset,entity_labels):\n",
        "    candidates = list()\n",
        "    for resume in dataset:\n",
        "        res_ent_labels = list(zip(*resume[1][\"entities\"]))[2]\n",
        "        if set(entity_labels).issubset(res_ent_labels):\n",
        "            candidates.append(resume)\n",
        "    return candidates\n",
        "\n",
        "training_data = gather_candidates( converted_complete_resumees, chosen_entity_labels )\n",
        "print(\"Gathered {} training examples\".format(len(training_data)))"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Gathered 437 training examples\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xSDI7dDDLBbW"
      },
      "source": [
        "__Task 5__: Next, we want to remove all other entities, since we only want to train NER for the three entities in __chosen_entity_labels__."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1DBr_VzxLNqe"
      },
      "source": [
        "## filter all annotation based on filter list\n",
        "def filter_ents(ents, filter):\n",
        "    filtered = [ent for ent in ents if ent[2] in filter]\n",
        "    return filtered\n",
        "\n",
        "## TODO use method above to remove all but relevant (chosen) entity annotations and store in X variable. X shall contain all\n",
        "## the resumees from training_data, but their entity annotations shall be filtered using the function from above.\n",
        "X = list()\n",
        "#filter_ents(converted_complete_resumees[0][1][\"entities\"], chosen_entity_labels)\n",
        "for res in training_data:\n",
        "  ent = res[1][\"entities\"]\n",
        "  filtered_ent = filter_ents(ent, chosen_entity_labels)\n",
        "  res[1][\"entities\"] = filtered_ent\n",
        "  X.append(res)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IZpyxtX6NteV"
      },
      "source": [
        "__Task 6__: Some resumees cause trouble. We filter these out with the following lines of code.<br>\n",
        "First, use __add_label__ to add your chosen labels to the __ner__ model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RqwcgW_xX18e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2b19fd55-cc9f-40b8-df74-88374f613b38"
      },
      "source": [
        "from spacy.gold import GoldParse \n",
        "\n",
        "# TODO add your labels \n",
        "ner.add_label( \"Skills\" )\n",
        "\n",
        "ner.add_label( \"Companies worked at\" )\n",
        "\n",
        "ner.add_label( \"Degree\"  )\n",
        "\n",
        "nlp.begin_training()\n",
        "\n",
        "\n",
        "good = []\n",
        "\n",
        "for item in X:\n",
        "  \n",
        "  text = nlp.make_doc( item[ 0 ] )\n",
        "\n",
        "  try:\n",
        "    \n",
        "    gold = GoldParse( text, entities = item[ 1 ][ \"entities\" ] )\n",
        "\n",
        "  except:\n",
        "\n",
        "    continue\n",
        "  \n",
        "  try:\n",
        "    \n",
        "    nlp.update( [ text ], [ gold ], drop = 0.3 )\n",
        "\n",
        "  except:\n",
        "\n",
        "    pass\n",
        "\n",
        "  else:\n",
        "\n",
        "    good.append( item )\n",
        "\n",
        "print( \"Number of good samples: \" + str( len( good ) ) )\n",
        "\n",
        "print( \"\" )\n",
        "\n",
        "print( \"\" )\n",
        "\n",
        "print( \"Number of bad sampples: \" + str( len( X ) - len( good ) ) )"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of good samples: 351\n",
            "\n",
            "\n",
            "Number of bad sampples: 86\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "obZ93SLzyFIe"
      },
      "source": [
        "For a machine learning model, it is essential to be able to generalize. Only a model, that can generalize well is able to process new data in a meaningful way. Therefore, one usually separates the data set into two sets: the training set and the test set. The training set is used to train the model. The test set is used to evaluate the performance of the model on unseen data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k0XVZPTyyh4T"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split( [ item[ 0 ] for item in good ], [ item[ 1 ] for item in good ], test_size = 0.3 )"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3AcgIty-1otR"
      },
      "source": [
        "__Task 7__: Complete the following code. Shuffle __new_index__. Create the data sets __x_shuffled__ and __y_shuffled__. Use these to create minibatches, iterate over these minibatches, preprocess the data in a given minibatch using __nlp.make_doc__ and __GoldParse__. Employ __nlp.update__ to update the model using these preprocessed data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGLUALLr1rEp"
      },
      "source": [
        "import numpy as np\n",
        "from sklearn.utils import shuffle\n",
        "from spacy.util import minibatch, compounding\n",
        "\n",
        "nlp.begin_training()\n",
        "\n",
        "new_index = np.arange( len( x_train ) )\n",
        "\n",
        "x_data = np.array( x_train )\n",
        "\n",
        "y_data = np.array( y_train )\n",
        "\n",
        "for i in range( 25 ):\n",
        "\n",
        "  # TODO shuffle new_index\n",
        "  new_index = shuffle(new_index, random_state=0)\n",
        "\n",
        "  #TODO create x_shuffled from x_data by using the shuffled new_index\n",
        "  #TODO create y_shuffled from y_data by using the shuffled new_index\n",
        "  x_shuffled = list()\n",
        "  y_shuffled = list()\n",
        "  for ind in new_index:\n",
        "    x_shuffled.append(x_data[ind])\n",
        "    y_shuffled.append(y_data[ind][\"entities\"])\n",
        "  \n",
        "  # TODO\n",
        "  # divide the data in x_shuffled and y_shuffled into minibatches of identical size\n",
        "  # iterate over these minibatches\n",
        "  # preprocess the data in a minibatch using nlp.make_doc and GoldParse\n",
        "  # use these preprocessed data and nlp.update to train the model\n",
        "\n",
        "  batches = minibatch(zip(x_shuffled, y_shuffled), size=25)\n",
        "  for batch in batches:\n",
        "    texts = list()\n",
        "    golds = list()\n",
        "    for sample in batch:\n",
        "      text = nlp.make_doc(str(sample[0]))\n",
        "      gold = GoldParse(text, entities=sample[1])\n",
        "      texts.append(text)\n",
        "      golds.append(gold)\n",
        "    \n",
        "    nlp.update(texts, golds, drop=0.2)\n"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PQSe2-BOYlWN"
      },
      "source": [
        "__Question 4__: Why did we shuffle the data?<br> \n",
        "To minimize correlation between the different samples, as network could learn the ordering of the labels\n",
        "\n",
        "Why did we employ mini batches?<br>\n",
        "\n",
        "With mini-batches the training is stabilized. Furthermore, the computation efficiency is better.\n",
        "\n",
        "Reasearch the term __epoch__ in machine learning. How many epochs of training do we employ?<br>\n",
        "\n",
        "Epoch is amount of training episodes. In each episode we take the whole training set to train the network. Here, we have a epoch number of 20, so we process the whole training set 20 times.\n",
        "\n",
        "__Task 8__: Next, we choose one resumee and print it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SFGKoHVXY1pL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "104b0638-7f32-4471-c8bf-2c55424fe153"
      },
      "source": [
        "sample = 6\n",
        "resume = x_test[sample]\n",
        "\n",
        "print( resume )"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Rakesh Tikoo\n",
            "Rich mix of sales development with14 years of experience\n",
            "\n",
            "Mumbai, Maharashtra - Email me on Indeed: indeed.com/r/Rakesh-Tikoo/6f55b7d67d4510af\n",
            "\n",
            "Willing to relocate to: Mumbai, Maharashtra - Pune, Maharashtra - Indore, Madhya Pradesh\n",
            "\n",
            "WORK EXPERIENCE\n",
            "\n",
            "Sales manager\n",
            "\n",
            "Agc networks ltd -  Mumbai, Maharashtra -\n",
            "\n",
            "October 2014 to Present\n",
            "\n",
            "I have been working as sales manager, primaraily selling solutions across UC, Data centre and\n",
            "edge IT, Hyper converged infra, cyber security, and services sales\n",
            "\n",
            "Account manager\n",
            "\n",
            "Airtel business\n",
            "\n",
            "Worked as an account manager handling sales of VPN, Data centre colocation, Teleport, DMX\n",
            "platform, mobility\n",
            "\n",
            "EDUCATION\n",
            "\n",
            "BA in Economics\n",
            "\n",
            "Jammu university -  Jammu, Jammu and Kashmir\n",
            "\n",
            "SKILLS\n",
            "\n",
            "Sales, Account Management, Business Management, Sales Management, Sales Manager,\n",
            "Business Development\n",
            "\n",
            "ADDITIONAL INFORMATION\n",
            "\n",
            "14 years of experience in sales... Solution selling across IT and Telecom domain\n",
            "\n",
            "https://www.indeed.com/r/Rakesh-Tikoo/6f55b7d67d4510af?isid=rex-download&ikw=download-top&co=IN\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lKe5YQlHNZWX"
      },
      "source": [
        "__Task 9__: we process this resumee using __nlp__. Print for all items in __doc.ents__ the predicted label and the corresponding text. Then print the correct labels and their corresponding text for that resumee with data from __y_test__."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "arAyrdyfbnQR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "02268cc3-1608-4c3d-e01f-0ee37d85931b"
      },
      "source": [
        "doc = nlp( resume )\n",
        "# TODO\n",
        "# print for all the items in doc.ents the predicted label and the corresponding text\n",
        "\n",
        "print(\"Predicted Values:\")\n",
        "for item in doc.ents:\n",
        "  print(item.text, item.label_)\n",
        "# TODO\n",
        "# print the correct labels and their corresponding text for that resumee with data from y_test\n",
        "\n",
        "print()\n",
        "print(\"True Values:\")\n",
        "for item in y_test[sample][\"entities\"]:\n",
        "  print(item[2] +\" : \" + x_test[sample][item[0]:item[1]])\n",
        "\n"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predicted Values:\n",
            "Agc networks ltd Companies worked at\n",
            "BA in Economics Degree\n",
            "Sales, Account Management, Business Management, Sales Management, Sales Manager,\n",
            "Business Development\n",
            "\n",
            "ADDITIONAL INFORMATION\n",
            "\n",
            "14 years of experience in sales... Solution selling across IT and Telecom domain\n",
            "\n",
            "https://www.indeed.com/r/Rakesh-Tikoo/6f55b7d67d4510af?isid=rex-download&ikw=download-top&co=IN Skills\n",
            "\n",
            "True Values:\n",
            "Skills : Sales, Account Management, Business Management, Sales Management, Sales Manager,\n",
            "Business Development\n",
            "Degree : BA in Economics\n",
            "Companies worked at : Airtel business\n",
            "Companies worked at : Agc networks ltd \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DTjZw4RVeB87"
      },
      "source": [
        "__Question 5__: What labels did the model predict correctly?<br> Worked well for Companies worked at and Degree\n",
        "\n",
        "Where appeared problems?<br> Network has problems with Skills\n",
        "\n",
        "How can you explain the problems?<br> Skills is a very difficult entity, as there are products of companies as microsoft, which could be also interpreted as company worked at. Furthermore, it is difficult to set the correct borders between those entities\n",
        "\n",
        "\n",
        "__Question 6__: We can evaluate the performance of the model using 4 metrics: the __Accuracy__, the __Precision__, the __Recall__ and __F1__.<br>\n",
        "Inform yourself on these metrics. How are they defined? Explain the concept of __True Positive__, __True Negative__, __False Positive__ and __False Negative__. Use these to define  the __Accuracy__, the __Precision__, the __Recall__ and __F1__, and also give the formula for each of these.<br>\n",
        "\n",
        "Those were already described in the first assignment. \n",
        "\n",
        "__Task 10__: Complete the following code. Call __make_bilou_df__ with a resume from the test set and store result in __bilou_df__ variable."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W5M-ArsDtzFd",
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3bb54cef-aa1e-45f5-d4e6-7a59ed363c26"
      },
      "source": [
        "from spacy.gold import biluo_tags_from_offsets\n",
        "import pandas as pd\n",
        "from IPython.display import display, HTML\n",
        "\n",
        "## returns a pandas dataframe with tokens, prediction, and true (Gold Standard) annotations of tokens\n",
        "def make_bilou_df(nlp,resume):\n",
        "    \"\"\"\n",
        "    param nlp - a trained spacy model\n",
        "    param resume - a resume from our train or test set\n",
        "    \"\"\"\n",
        "    doc = nlp(resume[0])\n",
        "    bilou_ents_predicted = biluo_tags_from_offsets(doc, [(ent.start_char,ent.end_char,ent.label_)for ent in doc.ents])\n",
        "    bilou_ents_true = biluo_tags_from_offsets(doc, [(ent[0], ent[1], ent[2]) for ent in resume[1][\"entities\"]])\n",
        "\n",
        "    \n",
        "    doc_tokens = [tok.text for tok in doc]\n",
        "    bilou_df = pd.DataFrame()\n",
        "    bilou_df[\"Tokens\"] =doc_tokens\n",
        "    bilou_df[\"Tokens\"] = bilou_df[\"Tokens\"].str.replace(\"\\\\s+\",\"\") \n",
        "    bilou_df[\"Predicted\"] = bilou_ents_predicted\n",
        "    bilou_df[\"True\"] = bilou_ents_true\n",
        "    return bilou_df\n",
        "\n",
        "## TODO call method above with a resume from test set and store result in bilou_df variable.\n",
        "\n",
        "\n",
        "bilou_df = make_bilou_df( nlp, [x_test[sample], y_test[sample]])\n",
        "display(bilou_df)\n",
        "display(bilou_df[bilou_df[\"Predicted\"] != bilou_df[\"True\"]])  \n",
        "display(bilou_df[bilou_df[\"Predicted\"] == bilou_df[\"True\"]])  "
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tokens</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>True</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Rakesh</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Tikoo</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Rich</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>mix</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173</th>\n",
              "      <td>and</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>174</th>\n",
              "      <td>Telecom</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>175</th>\n",
              "      <td>domain</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>176</th>\n",
              "      <td></td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>177</th>\n",
              "      <td>https://www.indeed.com/r/Rakesh-Tikoo/6f55b7d6...</td>\n",
              "      <td>L-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>178 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                Tokens Predicted True\n",
              "0                                               Rakesh         O    O\n",
              "1                                                Tikoo         O    O\n",
              "2                                                              O    O\n",
              "3                                                 Rich         O    O\n",
              "4                                                  mix         O    O\n",
              "..                                                 ...       ...  ...\n",
              "173                                                and  I-Skills    O\n",
              "174                                            Telecom  I-Skills    O\n",
              "175                                             domain  I-Skills    O\n",
              "176                                                     I-Skills    O\n",
              "177  https://www.indeed.com/r/Rakesh-Tikoo/6f55b7d6...  L-Skills    O\n",
              "\n",
              "[178 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tokens</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>True</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>Agc</td>\n",
              "      <td>B-Companies worked at</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>networks</td>\n",
              "      <td>I-Companies worked at</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>ltd</td>\n",
              "      <td>L-Companies worked at</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>Airtel</td>\n",
              "      <td>O</td>\n",
              "      <td>B-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>business</td>\n",
              "      <td>O</td>\n",
              "      <td>L-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>157</th>\n",
              "      <td>Development</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>L-Skills</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>158</th>\n",
              "      <td></td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159</th>\n",
              "      <td>ADDITIONAL</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>160</th>\n",
              "      <td>INFORMATION</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>161</th>\n",
              "      <td></td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>162</th>\n",
              "      <td>14</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>163</th>\n",
              "      <td>years</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>164</th>\n",
              "      <td>of</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>165</th>\n",
              "      <td>experience</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>166</th>\n",
              "      <td>in</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>167</th>\n",
              "      <td>sales</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>168</th>\n",
              "      <td>...</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>169</th>\n",
              "      <td>Solution</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>170</th>\n",
              "      <td>selling</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>171</th>\n",
              "      <td>across</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>172</th>\n",
              "      <td>IT</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173</th>\n",
              "      <td>and</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>174</th>\n",
              "      <td>Telecom</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>175</th>\n",
              "      <td>domain</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>176</th>\n",
              "      <td></td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>177</th>\n",
              "      <td>https://www.indeed.com/r/Rakesh-Tikoo/6f55b7d6...</td>\n",
              "      <td>L-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                Tokens  ...                   True\n",
              "48                                                 Agc  ...                      -\n",
              "49                                            networks  ...                      -\n",
              "50                                                 ltd  ...                      -\n",
              "98                                              Airtel  ...  B-Companies worked at\n",
              "99                                            business  ...  L-Companies worked at\n",
              "157                                        Development  ...               L-Skills\n",
              "158                                                     ...                      O\n",
              "159                                         ADDITIONAL  ...                      O\n",
              "160                                        INFORMATION  ...                      O\n",
              "161                                                     ...                      O\n",
              "162                                                 14  ...                      O\n",
              "163                                              years  ...                      O\n",
              "164                                                 of  ...                      O\n",
              "165                                         experience  ...                      O\n",
              "166                                                 in  ...                      O\n",
              "167                                              sales  ...                      O\n",
              "168                                                ...  ...                      O\n",
              "169                                           Solution  ...                      O\n",
              "170                                            selling  ...                      O\n",
              "171                                             across  ...                      O\n",
              "172                                                 IT  ...                      O\n",
              "173                                                and  ...                      O\n",
              "174                                            Telecom  ...                      O\n",
              "175                                             domain  ...                      O\n",
              "176                                                     ...                      O\n",
              "177  https://www.indeed.com/r/Rakesh-Tikoo/6f55b7d6...  ...                      O\n",
              "\n",
              "[26 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tokens</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>True</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Rakesh</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Tikoo</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td></td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Rich</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>mix</td>\n",
              "      <td>O</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>152</th>\n",
              "      <td>Sales</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>I-Skills</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>153</th>\n",
              "      <td>Manager</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>I-Skills</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>154</th>\n",
              "      <td>,</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>I-Skills</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>155</th>\n",
              "      <td></td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>I-Skills</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>156</th>\n",
              "      <td>Business</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>I-Skills</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>152 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       Tokens Predicted      True\n",
              "0      Rakesh         O         O\n",
              "1       Tikoo         O         O\n",
              "2                     O         O\n",
              "3        Rich         O         O\n",
              "4         mix         O         O\n",
              "..        ...       ...       ...\n",
              "152     Sales  I-Skills  I-Skills\n",
              "153   Manager  I-Skills  I-Skills\n",
              "154         ,  I-Skills  I-Skills\n",
              "155            I-Skills  I-Skills\n",
              "156  Business  I-Skills  I-Skills\n",
              "\n",
              "[152 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-xgI8srVNZWY"
      },
      "source": [
        "Inform yourself on the [BILUO](https://spacy.io/usage/linguistic-features#accessing-ner) scheme.<br>\n",
        "__Question 7__: Why do you think is it better to tag entities using this scheme (consider names of humans, descriptions of items in a shop)?<br>\n",
        "Names consist mostly of 2 words, but companies, degrees and skills can consists of multiple words. Furthermore, there is a difference of a single word entity and multiple words entity.\n",
        "\n",
        "__Task 11__: employ pandas dataframe api to get a subset where predicted and true labels are the same. Compute the accuracy using the formula you researched above."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HWRI3IfluPD7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d63c1a4e-837c-4179-fa8e-3c9046581706"
      },
      "source": [
        "## TODO bilou_df is a pandas dataframe. Use pandas dataframe api to get a subset where predicted and true are the same. \n",
        "same_df = bilou_df[bilou_df[\"Predicted\"] == bilou_df[\"True\"]]\n",
        "## TODO compute the accuracy \n",
        "accuracy = same_df.shape[0] / bilou_df.shape[0]\n",
        "\n",
        "print(\"Accuracy on one resume: \",accuracy)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy on one resume:  0.8539325842696629\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1P4EgefioE-Q"
      },
      "source": [
        "The __accuracy__ is not 100%. Therefore, we want to have a look at those tokens, where the predicted and the true value differ.<br>\n",
        "__Task 12__: create a dataframe diff_df where the predicted values and the true values differ"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DdHFX1cMn-r6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 865
        },
        "outputId": "c6fc4817-5996-473a-cc67-b51a89ec301c"
      },
      "source": [
        "# TODO create a dataframe diff_df where the predicted values and the true values differ\n",
        "diff_df = bilou_df[bilou_df[\"Predicted\"] != bilou_df[\"True\"]]\n",
        "display(diff_df)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tokens</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>True</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>Agc</td>\n",
              "      <td>B-Companies worked at</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>networks</td>\n",
              "      <td>I-Companies worked at</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50</th>\n",
              "      <td>ltd</td>\n",
              "      <td>L-Companies worked at</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>98</th>\n",
              "      <td>Airtel</td>\n",
              "      <td>O</td>\n",
              "      <td>B-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>99</th>\n",
              "      <td>business</td>\n",
              "      <td>O</td>\n",
              "      <td>L-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>157</th>\n",
              "      <td>Development</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>L-Skills</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>158</th>\n",
              "      <td></td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>159</th>\n",
              "      <td>ADDITIONAL</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>160</th>\n",
              "      <td>INFORMATION</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>161</th>\n",
              "      <td></td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>162</th>\n",
              "      <td>14</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>163</th>\n",
              "      <td>years</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>164</th>\n",
              "      <td>of</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>165</th>\n",
              "      <td>experience</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>166</th>\n",
              "      <td>in</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>167</th>\n",
              "      <td>sales</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>168</th>\n",
              "      <td>...</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>169</th>\n",
              "      <td>Solution</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>170</th>\n",
              "      <td>selling</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>171</th>\n",
              "      <td>across</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>172</th>\n",
              "      <td>IT</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>173</th>\n",
              "      <td>and</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>174</th>\n",
              "      <td>Telecom</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>175</th>\n",
              "      <td>domain</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>176</th>\n",
              "      <td></td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>177</th>\n",
              "      <td>https://www.indeed.com/r/Rakesh-Tikoo/6f55b7d6...</td>\n",
              "      <td>L-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                Tokens  ...                   True\n",
              "48                                                 Agc  ...                      -\n",
              "49                                            networks  ...                      -\n",
              "50                                                 ltd  ...                      -\n",
              "98                                              Airtel  ...  B-Companies worked at\n",
              "99                                            business  ...  L-Companies worked at\n",
              "157                                        Development  ...               L-Skills\n",
              "158                                                     ...                      O\n",
              "159                                         ADDITIONAL  ...                      O\n",
              "160                                        INFORMATION  ...                      O\n",
              "161                                                     ...                      O\n",
              "162                                                 14  ...                      O\n",
              "163                                              years  ...                      O\n",
              "164                                                 of  ...                      O\n",
              "165                                         experience  ...                      O\n",
              "166                                                 in  ...                      O\n",
              "167                                              sales  ...                      O\n",
              "168                                                ...  ...                      O\n",
              "169                                           Solution  ...                      O\n",
              "170                                            selling  ...                      O\n",
              "171                                             across  ...                      O\n",
              "172                                                 IT  ...                      O\n",
              "173                                                and  ...                      O\n",
              "174                                            Telecom  ...                      O\n",
              "175                                             domain  ...                      O\n",
              "176                                                     ...                      O\n",
              "177  https://www.indeed.com/r/Rakesh-Tikoo/6f55b7d6...  ...                      O\n",
              "\n",
              "[26 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OWvyiCokonUI"
      },
      "source": [
        "Since we only considered one resumee, we now make this comparison for the whole test set.<br>\n",
        "__Task 13__: Complete the following code for the computation of the overall accuracy."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bidqT9GjovAg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "12605900-db8b-4f06-dcf3-4aec6a407aa1"
      },
      "source": [
        "doc_accuracy = []\n",
        "\n",
        "for i in range( len( x_test ) ):\n",
        "\n",
        "  resume = [x_test[i], y_test[i]]\n",
        "\n",
        "\n",
        "  bilou_df = make_bilou_df(nlp,resume)\n",
        "\n",
        "  same_df = bilou_df[bilou_df[\"Predicted\"] == bilou_df[\"True\"]]\n",
        "\n",
        "  doc_accuracy.append( same_df.shape[0] / bilou_df.shape[0] )\n",
        "\n",
        "total_acc = np.mean( doc_accuracy )\n",
        "print(\"Accuracy: \",total_acc)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy:  0.8484560920944053\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aJTL_qi085MU",
        "outputId": "79b86a86-c5f2-4a50-febd-8bc17ff6c277"
      },
      "source": [
        "text = x_test[0]\n",
        "ent = y_test[0]\n",
        "start = ent[\"entities\"][0][0]\n",
        "print(start)\n",
        "stop = ent[\"entities\"][0][1]\n",
        "print(stop)\n",
        "print(text[start:stop])"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "3321\n",
            "3376\n",
            "LS\n",
            "\n",
            "Automation, Testing, Mobile Testing, SDET, Scrum Ma\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cL7GE3RQpzbg"
      },
      "source": [
        "So we got an __accuracy__ of about 90% on average. This is quite good considering, that we only considered about 300 cases for training.<br>\n",
        "__Task 14__: Next, we want to find out, what the model did, when it went wrong. We only consider 5 resumees.<br>\n",
        "Complete the following code."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y02Pkxriq8GM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "43ecd1a5-4d83-412a-b5f0-91db6395ecb7"
      },
      "source": [
        "for i in range( 5 ):\n",
        "\n",
        "  resume = [x_test[i], y_test[i]]\n",
        "\n",
        "  text = resume[0]\n",
        "  ent = resume[1]\n",
        "  start = ent[\"entities\"][0][0]\n",
        "  stop = ent[\"entities\"][0][1]\n",
        "\n",
        "  bilou_df = make_bilou_df(nlp,resume)\n",
        "\n",
        "  difference_df = bilou_df[bilou_df[\"Predicted\"] != bilou_df[\"True\"]]\n",
        "\n",
        "  # TODO: print, where the labels from Spacy and the annotation differ. Print the text, the predicted and the true labels.\n",
        "  display(difference_df)\n"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tokens</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>True</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>548</th>\n",
              "      <td>EDUCATION</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>549</th>\n",
              "      <td></td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>550</th>\n",
              "      <td>Software</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>551</th>\n",
              "      <td>Product</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>552</th>\n",
              "      <td>Management</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>561</th>\n",
              "      <td>NY</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>562</th>\n",
              "      <td></td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>563</th>\n",
              "      <td>Bachelor</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>564</th>\n",
              "      <td>of</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>565</th>\n",
              "      <td>Computer</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>566</th>\n",
              "      <td>Science</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>567</th>\n",
              "      <td>in</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>568</th>\n",
              "      <td>Computer</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>569</th>\n",
              "      <td>Science</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>570</th>\n",
              "      <td>Engineering</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>575</th>\n",
              "      <td>SKILLS</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>576</th>\n",
              "      <td></td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>577</th>\n",
              "      <td>Automation</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>578</th>\n",
              "      <td>,</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>579</th>\n",
              "      <td>Testing</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>580</th>\n",
              "      <td>,</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>581</th>\n",
              "      <td>Mobile</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>582</th>\n",
              "      <td>Testing</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>583</th>\n",
              "      <td>,</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>584</th>\n",
              "      <td>SDET</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>585</th>\n",
              "      <td>,</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>586</th>\n",
              "      <td>Scrum</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>587</th>\n",
              "      <td>Master</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          Tokens Predicted True\n",
              "548    EDUCATION         O    -\n",
              "549                      O    -\n",
              "550     Software         O    -\n",
              "551      Product         O    -\n",
              "552   Management         O    -\n",
              "561           NY         O    -\n",
              "562                      O    -\n",
              "563     Bachelor         O    -\n",
              "564           of         O    -\n",
              "565     Computer         O    -\n",
              "566      Science         O    -\n",
              "567           in         O    -\n",
              "568     Computer         O    -\n",
              "569      Science         O    -\n",
              "570  Engineering         O    -\n",
              "575       SKILLS         O    -\n",
              "576                      O    -\n",
              "577   Automation         O    -\n",
              "578            ,         O    -\n",
              "579      Testing         O    -\n",
              "580            ,         O    -\n",
              "581       Mobile         O    -\n",
              "582      Testing         O    -\n",
              "583            ,         O    -\n",
              "584         SDET         O    -\n",
              "585            ,         O    -\n",
              "586        Scrum         O    -\n",
              "587       Master         O    -"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tokens</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>True</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Site</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>at</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Shendra</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>MIDC</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Aurangabad</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>649</th>\n",
              "      <td>Marathi</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>650</th>\n",
              "      <td>,</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>651</th>\n",
              "      <td>&amp;</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>652</th>\n",
              "      <td>amp</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>653</th>\n",
              "      <td>;</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>130 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         Tokens Predicted True\n",
              "8          Site         O    -\n",
              "9            at         O    -\n",
              "10      Shendra         O    -\n",
              "11         MIDC         O    -\n",
              "12   Aurangabad         O    -\n",
              "..          ...       ...  ...\n",
              "649     Marathi         O    -\n",
              "650           ,         O    -\n",
              "651           &         O    -\n",
              "652         amp         O    -\n",
              "653           ;         O    -\n",
              "\n",
              "[130 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tokens</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>True</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>943</th>\n",
              "      <td>B</td>\n",
              "      <td>O</td>\n",
              "      <td>B-Degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>944</th>\n",
              "      <td>Com</td>\n",
              "      <td>O</td>\n",
              "      <td>L-Degree</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>951</th>\n",
              "      <td>EXCEL</td>\n",
              "      <td>O</td>\n",
              "      <td>U-Skills</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>957</th>\n",
              "      <td>WORD</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Tokens Predicted      True\n",
              "943      B         O  B-Degree\n",
              "944    Com         O  L-Degree\n",
              "951  EXCEL         O  U-Skills\n",
              "957   WORD         O         -"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tokens</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>True</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>Microsoft</td>\n",
              "      <td>B-Companies worked at</td>\n",
              "      <td>U-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>India</td>\n",
              "      <td>L-Companies worked at</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>208</th>\n",
              "      <td>EDUCATION</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>209</th>\n",
              "      <td></td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>210</th>\n",
              "      <td>PGDBM</td>\n",
              "      <td>B-Degree</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>286</th>\n",
              "      <td></td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>287</th>\n",
              "      <td>Microsoft</td>\n",
              "      <td>U-Companies worked at</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>298</th>\n",
              "      <td>https://www.linkedin.com/in/karthik-g-v-7a25462/</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>299</th>\n",
              "      <td></td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>300</th>\n",
              "      <td>Microsoft</td>\n",
              "      <td>O</td>\n",
              "      <td>-</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>69 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               Tokens  ...                   True\n",
              "64                                          Microsoft  ...  U-Companies worked at\n",
              "65                                              India  ...                      O\n",
              "208                                         EDUCATION  ...                      -\n",
              "209                                                    ...                      -\n",
              "210                                             PGDBM  ...                      -\n",
              "..                                                ...  ...                    ...\n",
              "286                                                    ...                      -\n",
              "287                                         Microsoft  ...                      -\n",
              "298  https://www.linkedin.com/in/karthik-g-v-7a25462/  ...                      -\n",
              "299                                                    ...                      -\n",
              "300                                         Microsoft  ...                      -\n",
              "\n",
              "[69 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Tokens</th>\n",
              "      <th>Predicted</th>\n",
              "      <th>True</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>SUDAYSHEELA</td>\n",
              "      <td>O</td>\n",
              "      <td>B-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td></td>\n",
              "      <td>O</td>\n",
              "      <td>I-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>CONSULTANCY</td>\n",
              "      <td>O</td>\n",
              "      <td>I-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>AND</td>\n",
              "      <td>O</td>\n",
              "      <td>I-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>SERVICES</td>\n",
              "      <td>O</td>\n",
              "      <td>I-Companies worked at</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1829</th>\n",
              "      <td>Outlook</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1830</th>\n",
              "      <td>,</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1831</th>\n",
              "      <td>Software</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1832</th>\n",
              "      <td>testing</td>\n",
              "      <td>I-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1833</th>\n",
              "      <td>.</td>\n",
              "      <td>L-Skills</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>83 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "           Tokens Predicted                   True\n",
              "10    SUDAYSHEELA         O  B-Companies worked at\n",
              "11                        O  I-Companies worked at\n",
              "12    CONSULTANCY         O  I-Companies worked at\n",
              "13            AND         O  I-Companies worked at\n",
              "14       SERVICES         O  I-Companies worked at\n",
              "...           ...       ...                    ...\n",
              "1829      Outlook  I-Skills                      O\n",
              "1830            ,  I-Skills                      O\n",
              "1831     Software  I-Skills                      O\n",
              "1832      testing  I-Skills                      O\n",
              "1833            .  L-Skills                      O\n",
              "\n",
              "[83 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CHyPOpGnL4jM",
        "outputId": "10cf857c-53f0-4c40-d103-960829125bfc"
      },
      "source": [
        "print(difference_df[difference_df[\"Predicted\"] == \"O\"].shape[0])"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "17\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0r21TdjPrt7P"
      },
      "source": [
        "__Question 8__: What was predicted, when the prediction differed from the true label?<br> Mostly, predictions were made on text, which was not annotated. Another occuring problem was that there were the wrong BILOU tag but the correct entity label\n",
        "\n",
        "What do you think is necessary for computing the accuracy on token level?<br> Averaging the prediction scores over the different enitity labels\n",
        "\n",
        "What is the advantage of computing the accuracy on token level?<br> It ignores the text which is not labeled and has a \"none\" prediction. Those none-labels occur very often and therefore improve the prediction scores even if the real labels are not predicted well\n",
        "\n",
        "__Task 15__: Complete the following code."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GZGOVg2U20V1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "edf87a4a-7c42-4066-85e7-1c88417ae8af"
      },
      "source": [
        "## TODO cycle through chosen_entity_labels and calculate metrics for each entity using test data\n",
        "data = []\n",
        "for label in chosen_entity_labels:\n",
        "    ## variables to store results for all resumes for one entity type\n",
        "    true_positives = 0\n",
        "    false_positives = 0\n",
        "    false_negatives = 0\n",
        "    for i in range( len( x_test ) ):\n",
        "        ## use make_bilou_df on each resume in our test set, and calculate for each entity true and false positives,\n",
        "        ## and false negatives. \n",
        "\n",
        "        resume = [x_test[i], y_test[i]]\n",
        "        \n",
        "        tres_df = make_bilou_df(nlp,resume)\n",
        "        difference_df = tres_df[tres_df[\"Predicted\"] != tres_df[\"True\"]]\n",
        "        same_df = tres_df[tres_df[\"Predicted\"] == tres_df[\"True\"]]\n",
        "\n",
        "        ## calculate true false positives and false negatives for each resume\n",
        "        \n",
        "        tp = same_df[same_df[\"Predicted\"].str.contains(label)].shape[0]\n",
        "        \n",
        "        fp = difference_df[difference_df[\"Predicted\"].str.contains(label)].shape[0] \n",
        "        \n",
        "        fn = difference_df[difference_df[\"True\"].str.contains(label)].shape[0] \n",
        "\n",
        "        ## aggregate result for each resume to totals\n",
        "        true_positives = true_positives + tp\n",
        "        false_positives = false_positives + fp\n",
        "        false_negatives = false_negatives + fn\n",
        "    \n",
        "    print(\"For label '{}' tp: {} fp: {} fn: {}\".format(label,true_positives,false_positives,false_negatives))\n",
        "    \n",
        "    ## TODO Use the formulas you learned to calculate metrics and print them out\n",
        "    ## also: prevent division by zero without raising errors. Explain your choice\n",
        "    precision = 0\n",
        "    recall = 0\n",
        "    f1 = 0\n",
        "    if true_positives + false_positives != 0 and true_positives + false_negatives != 0: \n",
        "      precision = (true_positives / (true_positives + false_positives))\n",
        "      recall = (true_positives / (true_positives + false_negatives))\n",
        "      if precision + recall != 0:\n",
        "        f1 = 2* (precision*recall / (precision + recall))\n",
        "      else:\n",
        "        print(\"Division by Zero\")\n",
        "    else: \n",
        "      print(\"Division by Zero\")\n",
        "\n",
        "    row = [label,precision,recall,f1]\n",
        "    data.append(row)\n",
        "\n",
        "## make pandas dataframe with metrics data. Use the chosen entity labels as an index, and the metric names as columns. \n",
        "metric_df = pd.DataFrame( data, columns = [ \"Label\", \"Precision\", \"Recall\", \"F1\" ] )\n",
        "display(metric_df)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "For label 'Skills' tp: 1838 fp: 5941 fn: 1510\n",
            "For label 'Companies worked at' tp: 592 fp: 164 fn: 520\n",
            "For label 'Degree' tp: 293 fp: 260 fn: 203\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Label</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>F1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Skills</td>\n",
              "      <td>0.236277</td>\n",
              "      <td>0.548984</td>\n",
              "      <td>0.330368</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Companies worked at</td>\n",
              "      <td>0.783069</td>\n",
              "      <td>0.532374</td>\n",
              "      <td>0.633833</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Degree</td>\n",
              "      <td>0.529837</td>\n",
              "      <td>0.590726</td>\n",
              "      <td>0.558627</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                 Label  Precision    Recall        F1\n",
              "0               Skills   0.236277  0.548984  0.330368\n",
              "1  Companies worked at   0.783069  0.532374  0.633833\n",
              "2               Degree   0.529837  0.590726  0.558627"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rMKMlJswAqUZ"
      },
      "source": [
        "__Question 9__: Explain from these statistics how well __nlp__ performs.<br> The network works quite okay (70% F1-score) for the labels \"companies worked at\" and \"degree\". But not so good for \"skills\". For \"skills\" the precision was very bad. \n",
        "\n",
        "\n",
        "__Task 16__: Compute for each metric (Precision, Recall, F1) the mean values."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5_gEZTQy5KTr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fd44fb8a-22cd-4858-dcab-06c7347dd78c"
      },
      "source": [
        "#for label in [ \"Precision\", \"Recall\", \"F1\" ]:\n",
        "    \n",
        "    # Compute mean and print\n",
        "print(\"Mean Precision : \" + str(metric_df[\"Precision\"].mean()))\n",
        "print(\"Mean Recall : \" + str(metric_df[\"Recall\"].mean()))\n",
        "print(\"Mean F1 : \" + str(metric_df[\"F1\"].mean()))"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mean Precision : 0.5163943969572885\n",
            "Mean Recall : 0.5573614585034482\n",
            "Mean F1 : 0.5076092716250198\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2Q1rvvhfBoN9"
      },
      "source": [
        "__Question 10__: What do you learn, when you compare the performance of the model on the token level with the performance of the model on the global level from above?<br> That the performance computed with the \"none\" labels hide the real performance and give a false impression. Accuracy on global level was 83% and F1-score on token level was 57%.\n",
        "Next, we prepare data for flair."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7vkzBYIlDbLe"
      },
      "source": [
        "train = [ [ x_train[ i ], y_train[ i ] ] for i in range( len( x_train ) ) ]\n",
        "\n",
        "test = [ [ x_test[ i ], y_test[ i ] ] for i in range( len( x_test ) ) ]"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AbtEjt9zNZWc"
      },
      "source": [
        "__Task 17__: Complete the following code."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NBfdFr1qNKBv"
      },
      "source": [
        "# prepare data\n",
        "training_data_as_bilou = [make_bilou_df(nlp,res) for res in train]\n",
        "\n",
        "test_data_as_bilou = [make_bilou_df(nlp,res) for res in test]\n",
        "\n",
        "\n",
        "# set up paths\n",
        "path_to_training_file = os.getcwd() + \"/training_data.csv\"\n",
        "\n",
        "path_to_test_file = os.getcwd() + \"/test_data.csv\"\n",
        "\n",
        "\n",
        "\n",
        "# make sure, that if the corresponding files exist, they are emptied\n",
        "if os.path.isfile( path_to_training_file ):\n",
        "\n",
        "  open( path_to_training_file, \"w\" ).close()\n",
        "\n",
        "if os.path.isfile( path_to_test_file ):\n",
        "\n",
        "  open( path_to_test_file, \"w\" ).close()\n",
        "\n",
        "\n",
        "# open empty files\n",
        "training_file = open( path_to_training_file, \"a\", encoding = \"utf-8\" )\n",
        "    \n",
        "test_file = open( path_to_test_file, \"a\", encoding = \"utf-8\" )\n",
        "\n",
        "\n",
        "for item in training_data_as_bilou:\n",
        "\n",
        "  # TODO remove all tokens like \"\", \" \", \"\\n\" by ignoring them\n",
        "  # for all other tokens do the following:\n",
        "  # create a string s: s = token + \" \" + label + \"\\n\"\n",
        "  # if the label is \"-\", then write s = token + \" O\\n\"\n",
        "  #\n",
        "  # write this newly created string to file\n",
        "  # if this newly created string contains \".\", then also write a \n",
        "  # newline to file that only contains \"\\n\"\n",
        "  #\n",
        "  # Using this scheme, each line in the resulting files corresponds either to an empty line or a token.\n",
        "  # Flair assembles a block of nonempty lines into a sentence. Therefore, the empty line\n",
        "  # is a signal for Flair that the current sentence is finished. Therefore, we extracted\n",
        "  # the whitespaces above.\n",
        "  item.drop(item[item[\"Tokens\"].str.contains(\"\")].index)\n",
        "  item.drop(item[item[\"Tokens\"].str.contains(\" \")].index)\n",
        "  item.drop(item[item[\"Tokens\"].str.contains(\"\\n\")].index)\n",
        "\n",
        "  for i, row in item.iterrows():\n",
        "    token = row[\"Tokens\"]\n",
        "    label = row[\"True\"]\n",
        "    s = \"\"\n",
        "    if label == \"-\":\n",
        "      s = token + \" O\\n\"\n",
        "    else: \n",
        "      s = token + \" \" + label + \"\\n\"\n",
        "    \n",
        "    training_file.write(s)\n",
        "    if \".\" in s:\n",
        "      training_file.write(\"\\n\")\n",
        "    \n",
        "\n",
        "\n",
        "for item in test_data_as_bilou:\n",
        "    \n",
        "  # TODO the same as above.\n",
        "  item.drop(item[item[\"Tokens\"].str.contains(\"\")].index)\n",
        "  item.drop(item[item[\"Tokens\"].str.contains(\" \")].index)\n",
        "  item.drop(item[item[\"Tokens\"].str.contains(\"\\n\")].index)\n",
        "\n",
        "  for i, row in item.iterrows():\n",
        "    token = row[\"Tokens\"]\n",
        "    label = row[\"True\"]\n",
        "    s = \"\"\n",
        "    if label == \"-\":\n",
        "      s = token + \" O\\n\"\n",
        "    else: \n",
        "      s = token + \" \" + label + \"\\n\"\n",
        "    \n",
        "    test_file.write(s)\n",
        "    if \".\" in s:\n",
        "      test_file.write(\"\\n\")\n",
        "\n",
        "\n",
        "training_file.close()\n",
        "\n",
        "test_file.close()"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-V5trnr4gXa9"
      },
      "source": [
        "Start Flair"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "630pwinWhXWo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2f3d475a-c52f-4d04-883a-492c4eb7dc62"
      },
      "source": [
        "pip install flair"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: flair in /usr/local/lib/python3.7/dist-packages (0.8.0.post1)\n",
            "Requirement already satisfied: gdown==3.12.2 in /usr/local/lib/python3.7/dist-packages (from flair) (3.12.2)\n",
            "Requirement already satisfied: bpemb>=0.3.2 in /usr/local/lib/python3.7/dist-packages (from flair) (0.3.3)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from flair) (2.8.1)\n",
            "Requirement already satisfied: langdetect in /usr/local/lib/python3.7/dist-packages (from flair) (1.0.9)\n",
            "Requirement already satisfied: konoha<5.0.0,>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from flair) (4.6.5)\n",
            "Requirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from flair) (0.1.2)\n",
            "Requirement already satisfied: gensim<=3.8.3,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from flair) (3.6.0)\n",
            "Requirement already satisfied: sqlitedict>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from flair) (1.7.0)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from flair) (0.22.2.post1)\n",
            "Requirement already satisfied: deprecated>=1.2.4 in /usr/local/lib/python3.7/dist-packages (from flair) (1.2.12)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.7/dist-packages (from flair) (0.0.9)\n",
            "Requirement already satisfied: segtok>=1.5.7 in /usr/local/lib/python3.7/dist-packages (from flair) (1.5.10)\n",
            "Requirement already satisfied: numpy<1.20.0 in /usr/local/lib/python3.7/dist-packages (from flair) (1.19.5)\n",
            "Requirement already satisfied: torch<=1.7.1,>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from flair) (1.7.1)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from flair) (0.8.9)\n",
            "Requirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.7/dist-packages (from flair) (3.2.2)\n",
            "Requirement already satisfied: sentencepiece==0.1.95 in /usr/local/lib/python3.7/dist-packages (from flair) (0.1.95)\n",
            "Requirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.7/dist-packages (from flair) (4.41.1)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (from flair) (4.2.6)\n",
            "Requirement already satisfied: transformers>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from flair) (4.6.1)\n",
            "Requirement already satisfied: ftfy in /usr/local/lib/python3.7/dist-packages (from flair) (6.0.3)\n",
            "Requirement already satisfied: mpld3==0.3 in /usr/local/lib/python3.7/dist-packages (from flair) (0.3)\n",
            "Requirement already satisfied: janome in /usr/local/lib/python3.7/dist-packages (from flair) (0.4.1)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from flair) (2019.12.20)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from gdown==3.12.2->flair) (1.15.0)\n",
            "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.7/dist-packages (from gdown==3.12.2->flair) (2.25.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from gdown==3.12.2->flair) (3.0.12)\n",
            "Requirement already satisfied: overrides<4.0.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from konoha<5.0.0,>=4.0.0->flair) (3.1.0)\n",
            "Requirement already satisfied: importlib-metadata<4.0.0,>=3.7.0 in /usr/local/lib/python3.7/dist-packages (from konoha<5.0.0,>=4.0.0->flair) (3.10.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (1.4.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (2.5.1)\n",
            "Requirement already satisfied: pymongo in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (3.11.4)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from hyperopt>=0.1.1->flair) (0.16.0)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.7/dist-packages (from gensim<=3.8.3,>=3.4.0->flair) (5.0.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->flair) (1.0.1)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.7/dist-packages (from deprecated>=1.2.4->flair) (1.12.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub->flair) (3.7.4.3)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2.3->flair) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2.3->flair) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=2.2.3->flair) (2.4.7)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->flair) (0.0.45)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->flair) (20.9)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers>=4.0.0->flair) (0.10.3)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from ftfy->flair) (0.2.5)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown==3.12.2->flair) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown==3.12.2->flair) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown==3.12.2->flair) (2020.12.5)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown==3.12.2->flair) (1.24.3)\n",
            "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6; extra == \"socks\" in /usr/local/lib/python3.7/dist-packages (from requests[socks]->gdown==3.12.2->flair) (1.7.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata<4.0.0,>=3.7.0->konoha<5.0.0,>=4.0.0->flair) (3.4.1)\n",
            "Requirement already satisfied: decorator<5,>=4.3 in /usr/local/lib/python3.7/dist-packages (from networkx->hyperopt>=0.1.1->flair) (4.4.2)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers>=4.0.0->flair) (7.1.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UKoPUsQGgaEE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e347bb9d-5f85-478a-cf3d-09020b181f1b"
      },
      "source": [
        "from flair.data import Corpus\n",
        "from flair.data_fetcher import NLPTaskDataFetcher\n",
        "\n",
        "\n",
        "# your training file name\n",
        "data_folder = os.getcwd() \n",
        "\n",
        "train_file = \"training_data.csv\"\n",
        "\n",
        "# your training file name\n",
        "test_file = \"test_data.csv\"\n",
        "\n",
        "# when we wrote the data files, each row was either empty to signal the end\n",
        "# of a sentence to Flair, or the line contained a token, a white space and a label.\n",
        "# In the next line, we assign, that the token is the \"text\", and that the label is \n",
        "# \"ner\" label\n",
        "columns =  {0: 'text', 1: 'ner'}\n",
        "\n",
        "## Now load our csv into flair corpus\n",
        "corpus = NLPTaskDataFetcher.load_column_corpus(data_folder,column_format=columns,\n",
        "                                               train_file=train_file,\n",
        "                                               test_file=test_file)\n",
        "print(corpus)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-05-26 11:15:55,026 Reading data from /content/gdrive/My Drive/flair\n",
            "2021-05-26 11:15:55,027 Train: /content/gdrive/My Drive/flair/training_data.csv\n",
            "2021-05-26 11:15:55,032 Dev: None\n",
            "2021-05-26 11:15:55,038 Test: /content/gdrive/My Drive/flair/test_data.csv\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:22: DeprecationWarning: Call to deprecated function (or staticmethod) load_column_corpus. (Use 'flair.datasets' instead.) -- Deprecated since version 0.4.1.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Corpus: 6428 train + 714 dev + 3167 test sentences\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}