Team_Nova_Final_Eval_Code.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GZcyKrdBBNc1"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pylab as plt \n",
        "import nltk"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2GCd5QgEBkxh"
      },
      "source": [
        "# Reading the Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sePRqKwRBdjT",
        "outputId": "f17d78d6-2334-4a32-feff-e418d154c8f7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VV9nrz44Bf-u"
      },
      "outputs": [],
      "source": [
        "train = pd.read_csv(\"/content/gdrive/MyDrive/Data/train_df.csv\")\n",
        "test = pd.read_csv(\"/content/gdrive/MyDrive/Data/test_df.csv\")\n",
        "\n",
        "# train = pd.read_csv(\"train_df.csv\")\n",
        "# test = pd.read_csv(\"test_df.csv\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223
        },
        "id": "lNhuukEmBqLO",
        "outputId": "c13b483c-ae75-497e-e0cf-86e365ca138c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(1000000, 3)"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-3c357515-5caf-4d90-bded-868022eaf61a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>question_text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>dda0b0efc8ba86e81ec4</td>\n",
              "      <td>What are interesting facts about Microsoft his...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>dc708b74a108d0fc0ad9</td>\n",
              "      <td>What are those things which are not gonna happ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>06a27ec5d82dacd8bfe0</td>\n",
              "      <td>What should I know to avoid being \"upsold\" whe...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>00cbb6b17e3ceb7c5358</td>\n",
              "      <td>How I add any account with payment bank?</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7c304888973a701585a0</td>\n",
              "      <td>Which Multi level marketing products are actua...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3c357515-5caf-4d90-bded-868022eaf61a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3c357515-5caf-4d90-bded-868022eaf61a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3c357515-5caf-4d90-bded-868022eaf61a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                    qid                                      question_text  \\\n",
              "0  dda0b0efc8ba86e81ec4  What are interesting facts about Microsoft his...   \n",
              "1  dc708b74a108d0fc0ad9  What are those things which are not gonna happ...   \n",
              "2  06a27ec5d82dacd8bfe0  What should I know to avoid being \"upsold\" whe...   \n",
              "3  00cbb6b17e3ceb7c5358           How I add any account with payment bank?   \n",
              "4  7c304888973a701585a0  Which Multi level marketing products are actua...   \n",
              "\n",
              "   target  \n",
              "0       0  \n",
              "1       0  \n",
              "2       0  \n",
              "3       0  \n",
              "4       0  "
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "display(train.shape, train.head())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IxO-5VYBcPo7",
        "outputId": "c91e14de-bd07-439d-f02c-6ec60aa55490"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "61870"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train[\"target\"].sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PO3NwfzTBtgc"
      },
      "outputs": [],
      "source": [
        "train_len, test_len= len(train.question_text),len(test.question_text)\n",
        "print(train_len,test_len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BU31xdO-Buds"
      },
      "outputs": [],
      "source": [
        "misses_tr=train.isna().sum(axis=0)\n",
        "misses_tr "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OXhQk61bBxFD"
      },
      "outputs": [],
      "source": [
        "misses_te=test.isna().sum(axis=0)\n",
        "misses_te"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3Y3QaqqzB2xV"
      },
      "outputs": [],
      "source": [
        "posts = pd.DataFrame()\n",
        "posts[\"qid\"] = test[\"qid\"].append(train[\"qid\"],ignore_index = True)\n",
        "posts[\"qt\"] = test[\"question_text\"].append(train[\"question_text\"],ignore_index = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EFh8DYOebau6"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gcbhpjryB5RK"
      },
      "outputs": [],
      "source": [
        "posts"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cvkDmSkMB8Kt"
      },
      "source": [
        "# Data Cleaning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uiTaXhQHB9Sl"
      },
      "source": [
        "## Lower Casing Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v4Y_VpPdCAQX"
      },
      "source": [
        "Here we are converting the whole data in the question_text column to lower case, by doing so we can project the whole data into same feature space. However, this can cause few problems, like in cases of acronyms. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9Tsu8CLCCDXd"
      },
      "outputs": [],
      "source": [
        "# posts[\"qtf\"] = posts[\"qt\"].apply(lambda x: x.lower())\n",
        "# display(posts.head())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uifhsTPqCIVz"
      },
      "source": [
        "## Expanding the contractions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lddAPUdACZQq"
      },
      "source": [
        "In this section we expand contractions, by the use of contraction's package."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dDa1Pp4mCF52"
      },
      "outputs": [],
      "source": [
        "# !pip install contractions\n",
        "# import contractions\n",
        "# posts[\"qtf\"] = posts[\"qtf\"].apply(lambda x: contractions.fix(x))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UF6rPK10Ceg2"
      },
      "source": [
        "## Unwanted Text Removal"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PpeCZXyvCnfk"
      },
      "source": [
        "In this section, we are going to remove unwanted texts such as URLs, HTML tags, non-ASCII characters, or other special characters (symbols, emojis, and other graphic characters)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ab215AERMXyQ"
      },
      "outputs": [],
      "source": [
        "# import re\n",
        "# def removeURL(text):\n",
        "#     return re.sub(r\"https?://\\S+|www\\.\\S+\", \"\", text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "diMvJrZYNQSg"
      },
      "outputs": [],
      "source": [
        "# posts[\"qtf\"] = posts[\"qtf\"].apply(lambda x: removeURL(x)) #Removing URL's from the posts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "goROezToNgYG"
      },
      "outputs": [],
      "source": [
        "# def removeHTML(text):\n",
        "#     html = re.compile(r\"<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});\")\n",
        "#     return re.sub(html, \"\", text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yH48iHQAT6J7"
      },
      "outputs": [],
      "source": [
        "# posts[\"qtf\"] = posts[\"qtf\"].apply(lambda x: removeHTML(x)) #Removing HTML tags from the posts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B2dR1umdUHQT"
      },
      "outputs": [],
      "source": [
        "# def removenonASCII(text):\n",
        "#     return re.sub(r'[^\\x00-\\x7f]',r'', text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iJEegfGZUbfL"
      },
      "outputs": [],
      "source": [
        "# posts[\"qtf\"] = posts[\"qtf\"].apply(lambda x: removenonASCII(x)) #Removing non ASCII characters from the posts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P2KOXdIqUi6r"
      },
      "outputs": [],
      "source": [
        "# def removeSpecialCharacters(text):\n",
        "#     emoji_pattern = re.compile(\n",
        "#         '['\n",
        "#         u'\\U0001F600-\\U0001F64F'  # emoticons\n",
        "#         u'\\U0001F300-\\U0001F5FF'  # symbols & pictographs\n",
        "#         u'\\U0001F680-\\U0001F6FF'  # transport & map symbols\n",
        "#         u'\\U0001F1E0-\\U0001F1FF'  # flags (iOS)\n",
        "#         u'\\U00002702-\\U000027B0'\n",
        "#         u'\\U000024C2-\\U0001F251'\n",
        "#         ']+',\n",
        "#         flags=re.UNICODE)\n",
        "#     return emoji_pattern.sub(r'', text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Alzjt4jzU3dj"
      },
      "outputs": [],
      "source": [
        "# posts[\"qtf\"] = posts[\"qtf\"].apply(lambda x: removeSpecialCharacters(x)) #Removing special characters from the posts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DGx9y5a9U-0z"
      },
      "outputs": [],
      "source": [
        "# import string\n",
        "# def removePunctuations(text):\n",
        "#     return text.translate(str.maketrans('', '', string.punctuation))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "piBhzimnVY4r"
      },
      "outputs": [],
      "source": [
        "# posts[\"qtf\"] = posts[\"qtf\"].apply(lambda x: removePunctuations(x)) #Removing Punctuations from the posts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X_f2Kj6TVi6K"
      },
      "outputs": [],
      "source": [
        "# def other_clean(text):\n",
        "#         # Typos, slang and other\n",
        "#         sample_typos_slang = {\n",
        "#                                 \"w/e\": \"whatever\",\n",
        "#                                 \"usagov\": \"usa government\",\n",
        "#                                 \"recentlu\": \"recently\",\n",
        "#                                 \"ph0tos\": \"photos\",\n",
        "#                                 \"amirite\": \"am i right\",\n",
        "#                                 \"exp0sed\": \"exposed\",\n",
        "#                                 \"<3\": \"love\",\n",
        "#                                 \"luv\": \"love\",\n",
        "#                                 \"amageddon\": \"armageddon\",\n",
        "#                                 \"trfc\": \"traffic\",\n",
        "#                                 \"16yr\": \"16 year\"\n",
        "#                                 }\n",
        "\n",
        "#         # Acronyms\n",
        "#         sample_acronyms =  { \n",
        "#                             \"mh370\": \"malaysia airlines flight 370\",\n",
        "#                             \"okwx\": \"oklahoma city weather\",\n",
        "#                             \"arwx\": \"arkansas weather\",    \n",
        "#                             \"gawx\": \"georgia weather\",  \n",
        "#                             \"scwx\": \"south carolina weather\",  \n",
        "#                             \"cawx\": \"california weather\",\n",
        "#                             \"tnwx\": \"tennessee weather\",\n",
        "#                             \"azwx\": \"arizona weather\",  \n",
        "#                             \"alwx\": \"alabama weather\",\n",
        "#                             \"usnwsgov\": \"united states national weather service\",\n",
        "#                             \"2mw\": \"tomorrow\"\n",
        "#                             }\n",
        "\n",
        "        \n",
        "#         # Some common abbreviations \n",
        "#         sample_abbr = {\n",
        "#                         \"$\" : \" dollar \",\n",
        "#                         \"€\" : \" euro \",\n",
        "#                         \"4ao\" : \"for adults only\",\n",
        "#                         \"a.m\" : \"before midday\",\n",
        "#                         \"a3\" : \"anytime anywhere anyplace\",\n",
        "#                         \"aamof\" : \"as a matter of fact\",\n",
        "#                         \"acct\" : \"account\",\n",
        "#                         \"adih\" : \"another day in hell\",\n",
        "#                         \"afaic\" : \"as far as i am concerned\",\n",
        "#                         \"afaict\" : \"as far as i can tell\",\n",
        "#                         \"afaik\" : \"as far as i know\",\n",
        "#                         \"afair\" : \"as far as i remember\",\n",
        "#                         \"afk\" : \"away from keyboard\",\n",
        "#                         \"app\" : \"application\",\n",
        "#                         \"approx\" : \"approximately\",\n",
        "#                         \"apps\" : \"applications\",\n",
        "#                         \"asap\" : \"as soon as possible\",\n",
        "#                         \"asl\" : \"age, sex, location\",\n",
        "#                         \"atk\" : \"at the keyboard\",\n",
        "#                         \"ave.\" : \"avenue\",\n",
        "#                         \"aymm\" : \"are you my mother\",\n",
        "#                         \"ayor\" : \"at your own risk\", \n",
        "#                         \"b&b\" : \"bed and breakfast\",\n",
        "#                         \"b+b\" : \"bed and breakfast\",\n",
        "#                         \"b.c\" : \"before christ\",\n",
        "#                         \"b2b\" : \"business to business\",\n",
        "#                         \"b2c\" : \"business to customer\",\n",
        "#                         \"b4\" : \"before\",\n",
        "#                         \"b4n\" : \"bye for now\",\n",
        "#                         \"b@u\" : \"back at you\",\n",
        "#                         \"bae\" : \"before anyone else\",\n",
        "#                         \"bak\" : \"back at keyboard\",\n",
        "#                         \"bbbg\" : \"bye bye be good\",\n",
        "#                         \"bbc\" : \"british broadcasting corporation\",\n",
        "#                         \"bbias\" : \"be back in a second\",\n",
        "#                         \"bbl\" : \"be back later\",\n",
        "#                         \"bbs\" : \"be back soon\",\n",
        "#                         \"be4\" : \"before\",\n",
        "#                         \"bfn\" : \"bye for now\",\n",
        "#                         \"blvd\" : \"boulevard\",\n",
        "#                         \"bout\" : \"about\",\n",
        "#                         \"brb\" : \"be right back\",\n",
        "#                         \"bros\" : \"brothers\",\n",
        "#                         \"brt\" : \"be right there\",\n",
        "#                         \"bsaaw\" : \"big smile and a wink\",\n",
        "#                         \"btw\" : \"by the way\",\n",
        "#                         \"bwl\" : \"bursting with laughter\",\n",
        "#                         \"c/o\" : \"care of\",\n",
        "#                         \"cet\" : \"central european time\",\n",
        "#                         \"cf\" : \"compare\",\n",
        "#                         \"cia\" : \"central intelligence agency\",\n",
        "#                         \"csl\" : \"can not stop laughing\",\n",
        "#                         \"cu\" : \"see you\",\n",
        "#                         \"cul8r\" : \"see you later\",\n",
        "#                         \"cv\" : \"curriculum vitae\",\n",
        "#                         \"cwot\" : \"complete waste of time\",\n",
        "#                         \"cya\" : \"see you\",\n",
        "#                         \"cyt\" : \"see you tomorrow\",\n",
        "#                         \"dae\" : \"does anyone else\",\n",
        "#                         \"dbmib\" : \"do not bother me i am busy\",\n",
        "#                         \"diy\" : \"do it yourself\",\n",
        "#                         \"dm\" : \"direct message\",\n",
        "#                         \"dwh\" : \"during work hours\",\n",
        "#                         \"e123\" : \"easy as one two three\",\n",
        "#                         \"eet\" : \"eastern european time\",\n",
        "#                         \"eg\" : \"example\",\n",
        "#                         \"embm\" : \"early morning business meeting\",\n",
        "#                         \"encl\" : \"enclosed\",\n",
        "#                         \"encl.\" : \"enclosed\",\n",
        "#                         \"etc\" : \"and so on\",\n",
        "#                         \"faq\" : \"frequently asked questions\",\n",
        "#                         \"fawc\" : \"for anyone who cares\",\n",
        "#                         \"fb\" : \"facebook\",\n",
        "#                         \"fc\" : \"fingers crossed\",\n",
        "#                         \"fig\" : \"figure\",\n",
        "#                         \"fimh\" : \"forever in my heart\", \n",
        "#                         \"ft.\" : \"feet\",\n",
        "#                         \"ft\" : \"featuring\",\n",
        "#                         \"ftl\" : \"for the loss\",\n",
        "#                         \"ftw\" : \"for the win\",\n",
        "#                         \"fwiw\" : \"for what it is worth\",\n",
        "#                         \"fyi\" : \"for your information\",\n",
        "#                         \"g9\" : \"genius\",\n",
        "#                         \"gahoy\" : \"get a hold of yourself\",\n",
        "#                         \"gal\" : \"get a life\",\n",
        "#                         \"gcse\" : \"general certificate of secondary education\",\n",
        "#                         \"gfn\" : \"gone for now\",\n",
        "#                         \"gg\" : \"good game\",\n",
        "#                         \"gl\" : \"good luck\",\n",
        "#                         \"glhf\" : \"good luck have fun\",\n",
        "#                         \"gmt\" : \"greenwich mean time\",\n",
        "#                         \"gmta\" : \"great minds think alike\",\n",
        "#                         \"gn\" : \"good night\",\n",
        "#                         \"g.o.a.t\" : \"greatest of all time\",\n",
        "#                         \"goat\" : \"greatest of all time\",\n",
        "#                         \"goi\" : \"get over it\",\n",
        "#                         \"gps\" : \"global positioning system\",\n",
        "#                         \"gr8\" : \"great\",\n",
        "#                         \"gratz\" : \"congratulations\",\n",
        "#                         \"gyal\" : \"girl\",\n",
        "#                         \"h&c\" : \"hot and cold\",\n",
        "#                         \"hp\" : \"horsepower\",\n",
        "#                         \"hr\" : \"hour\",\n",
        "#                         \"hrh\" : \"his royal highness\",\n",
        "#                         \"ht\" : \"height\",\n",
        "#                         \"ibrb\" : \"i will be right back\",\n",
        "#                         \"ic\" : \"i see\",\n",
        "#                         \"icq\" : \"i seek you\",\n",
        "#                         \"icymi\" : \"in case you missed it\",\n",
        "#                         \"idc\" : \"i do not care\",\n",
        "#                         \"idgadf\" : \"i do not give a damn fuck\",\n",
        "#                         \"idgaf\" : \"i do not give a fuck\",\n",
        "#                         \"idk\" : \"i do not know\",\n",
        "#                         \"ie\" : \"that is\",\n",
        "#                         \"i.e\" : \"that is\",\n",
        "#                         \"ifyp\" : \"i feel your pain\",\n",
        "#                         \"IG\" : \"instagram\",\n",
        "#                         \"iirc\" : \"if i remember correctly\",\n",
        "#                         \"ilu\" : \"i love you\",\n",
        "#                         \"ily\" : \"i love you\",\n",
        "#                         \"imho\" : \"in my humble opinion\",\n",
        "#                         \"imo\" : \"in my opinion\",\n",
        "#                         \"imu\" : \"i miss you\",\n",
        "#                         \"iow\" : \"in other words\",\n",
        "#                         \"irl\" : \"in real life\",\n",
        "#                         \"j4f\" : \"just for fun\",\n",
        "#                         \"jic\" : \"just in case\",\n",
        "#                         \"jk\" : \"just kidding\",\n",
        "#                         \"jsyk\" : \"just so you know\",\n",
        "#                         \"l8r\" : \"later\",\n",
        "#                         \"lb\" : \"pound\",\n",
        "#                         \"lbs\" : \"pounds\",\n",
        "#                         \"ldr\" : \"long distance relationship\",\n",
        "#                         \"lmao\" : \"laugh my ass off\",\n",
        "#                         \"lmfao\" : \"laugh my fucking ass off\",\n",
        "#                         \"lol\" : \"laughing out loud\",\n",
        "#                         \"ltd\" : \"limited\",\n",
        "#                         \"ltns\" : \"long time no see\",\n",
        "#                         \"m8\" : \"mate\",\n",
        "#                         \"mf\" : \"motherfucker\",\n",
        "#                         \"mfs\" : \"motherfuckers\",\n",
        "#                         \"mfw\" : \"my face when\",\n",
        "#                         \"mofo\" : \"motherfucker\",\n",
        "#                         \"mph\" : \"miles per hour\",\n",
        "#                         \"mr\" : \"mister\",\n",
        "#                         \"mrw\" : \"my reaction when\",\n",
        "#                         \"ms\" : \"miss\",\n",
        "#                         \"mte\" : \"my thoughts exactly\",\n",
        "#                         \"nagi\" : \"not a good idea\",\n",
        "#                         \"nbc\" : \"national broadcasting company\",\n",
        "#                         \"nbd\" : \"not big deal\",\n",
        "#                         \"nfs\" : \"not for sale\",\n",
        "#                         \"ngl\" : \"not going to lie\",\n",
        "#                         \"nhs\" : \"national health service\",\n",
        "#                         \"nrn\" : \"no reply necessary\",\n",
        "#                         \"nsfl\" : \"not safe for life\",\n",
        "#                         \"nsfw\" : \"not safe for work\",\n",
        "#                         \"nth\" : \"nice to have\",\n",
        "#                         \"nvr\" : \"never\",\n",
        "#                         \"nyc\" : \"new york city\",\n",
        "#                         \"oc\" : \"original content\",\n",
        "#                         \"og\" : \"original\",\n",
        "#                         \"ohp\" : \"overhead projector\",\n",
        "#                         \"oic\" : \"oh i see\",\n",
        "#                         \"omdb\" : \"over my dead body\",\n",
        "#                         \"omg\" : \"oh my god\",\n",
        "#                         \"omw\" : \"on my way\",\n",
        "#                         \"p.a\" : \"per annum\",\n",
        "#                         \"p.m\" : \"after midday\",\n",
        "#                         \"pm\" : \"prime minister\",\n",
        "#                         \"poc\" : \"people of color\",\n",
        "#                         \"pov\" : \"point of view\",\n",
        "#                         \"pp\" : \"pages\",\n",
        "#                         \"ppl\" : \"people\",\n",
        "#                         \"prw\" : \"parents are watching\",\n",
        "#                         \"ps\" : \"postscript\",\n",
        "#                         \"pt\" : \"point\",\n",
        "#                         \"ptb\" : \"please text back\",\n",
        "#                         \"pto\" : \"please turn over\",\n",
        "#                         \"qpsa\" : \"what happens\", #\"que pasa\",\n",
        "#                         \"ratchet\" : \"rude\",\n",
        "#                         \"rbtl\" : \"read between the lines\",\n",
        "#                         \"rlrt\" : \"real life retweet\", \n",
        "#                         \"rofl\" : \"rolling on the floor laughing\",\n",
        "#                         \"roflol\" : \"rolling on the floor laughing out loud\",\n",
        "#                         \"rotflmao\" : \"rolling on the floor laughing my ass off\",\n",
        "#                         \"rt\" : \"retweet\",\n",
        "#                         \"ruok\" : \"are you ok\",\n",
        "#                         \"sfw\" : \"safe for work\",\n",
        "#                         \"sk8\" : \"skate\",\n",
        "#                         \"smh\" : \"shake my head\",\n",
        "#                         \"sq\" : \"square\",\n",
        "#                         \"srsly\" : \"seriously\", \n",
        "#                         \"ssdd\" : \"same stuff different day\",\n",
        "#                         \"tbh\" : \"to be honest\",\n",
        "#                         \"tbs\" : \"tablespooful\",\n",
        "#                         \"tbsp\" : \"tablespooful\",\n",
        "#                         \"tfw\" : \"that feeling when\",\n",
        "#                         \"thks\" : \"thank you\",\n",
        "#                         \"tho\" : \"though\",\n",
        "#                         \"thx\" : \"thank you\",\n",
        "#                         \"tia\" : \"thanks in advance\",\n",
        "#                         \"til\" : \"today i learned\",\n",
        "#                         \"tl;dr\" : \"too long i did not read\",\n",
        "#                         \"tldr\" : \"too long i did not read\",\n",
        "#                         \"tmb\" : \"tweet me back\",\n",
        "#                         \"tntl\" : \"trying not to laugh\",\n",
        "#                         \"ttyl\" : \"talk to you later\",\n",
        "#                         \"u\" : \"you\",\n",
        "#                         \"u2\" : \"you too\",\n",
        "#                         \"u4e\" : \"yours for ever\",\n",
        "#                         \"utc\" : \"coordinated universal time\",\n",
        "#                         \"w/\" : \"with\",\n",
        "#                         \"w/o\" : \"without\",\n",
        "#                         \"w8\" : \"wait\",\n",
        "#                         \"wassup\" : \"what is up\",\n",
        "#                         \"wb\" : \"welcome back\",\n",
        "#                         \"wtf\" : \"what the fuck\",\n",
        "#                         \"wtg\" : \"way to go\",\n",
        "#                         \"wtpa\" : \"where the party at\",\n",
        "#                         \"wuf\" : \"where are you from\",\n",
        "#                         \"wuzup\" : \"what is up\",\n",
        "#                         \"wywh\" : \"wish you were here\",\n",
        "#                         \"yd\" : \"yard\",\n",
        "#                         \"ygtr\" : \"you got that right\",\n",
        "#                         \"ynk\" : \"you never know\",\n",
        "#                         \"zzz\" : \"sleeping bored and tired\"\n",
        "#                         }\n",
        "            \n",
        "#         sample_typos_slang_pattern = re.compile(r'(?<!\\w)(' + '|'.join(re.escape(key) for key in sample_typos_slang.keys()) + r')(?!\\w)')\n",
        "#         sample_acronyms_pattern = re.compile(r'(?<!\\w)(' + '|'.join(re.escape(key) for key in sample_acronyms.keys()) + r')(?!\\w)')\n",
        "#         sample_abbr_pattern = re.compile(r'(?<!\\w)(' + '|'.join(re.escape(key) for key in sample_abbr.keys()) + r')(?!\\w)')\n",
        "        \n",
        "#         text = sample_typos_slang_pattern.sub(lambda x: sample_typos_slang[x.group()], text)\n",
        "#         text = sample_acronyms_pattern.sub(lambda x: sample_acronyms[x.group()], text)\n",
        "#         text = sample_abbr_pattern.sub(lambda x: sample_abbr[x.group()], text)\n",
        "        \n",
        "#         return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d8Xa4NTRWiyI"
      },
      "outputs": [],
      "source": [
        "# posts[\"qtf\"] = posts[\"qtf\"].apply(lambda x: other_clean(x)) #Manual Cleaning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9gxvVoZFXY-H"
      },
      "source": [
        "# Data Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FCskP3PvTSOD"
      },
      "source": [
        "## Under-Sampling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DnYe9CsnTRhr"
      },
      "outputs": [],
      "source": [
        "# from imblearn.under_sampling import RandomUnderSampler  \n",
        "\n",
        "# nm = RandomUnderSampler()\n",
        "# X_resCV,y_resCV=nm.fit_resample(trainPostCV,train[\"target\"])\n",
        "# X_resTF,y_resTF=nm.fit_resample(trainPostTF,train[\"target\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kwehj7ZCTXKa"
      },
      "source": [
        "## Over-Sampling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MDQbR3bTTbah"
      },
      "outputs": [],
      "source": [
        "# from imblearn.under_sampling import RandomOverSampler  \n",
        "\n",
        "# nm = RandomOverSampler()\n",
        "# X_resCV,y_resCV=nm.fit_resample(trainPostCV,train[\"target\"])\n",
        "# X_resTF,y_resTF=nm.fit_resample(trainPostTF,train[\"target\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "96v12COs2hrv"
      },
      "source": [
        "## Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RYudXxOD28tJ",
        "outputId": "e3099861-d6ca-4e2e-a262-a1e5375a7f15"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# import nltk\n",
        "# nltk.download('punkt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        },
        "id": "N-ymnxB9z3RY",
        "outputId": "398f187f-e86a-48d2-9cf9-85eaa3ca0df3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-d68ddc65-0019-4fe2-adf3-b99934e640d3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>qt</th>\n",
              "      <th>qtf</th>\n",
              "      <th>tokenized</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a4f3da3a3df9dd881edd</td>\n",
              "      <td>My period is due on my wedding day. How can I ...</td>\n",
              "      <td>my period is due on my wedding day. how can i ...</td>\n",
              "      <td>[my, period, is, due, on, my, wedding, day, .,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9914c62ed3f69684d549</td>\n",
              "      <td>How many numbers higher than a million can be ...</td>\n",
              "      <td>how many numbers higher than a million can be ...</td>\n",
              "      <td>[how, many, numbers, higher, than, a, million,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8138ae48649e37091a91</td>\n",
              "      <td>How come I feel nothing for my family, but sti...</td>\n",
              "      <td>how come i feel nothing for my family, but sti...</td>\n",
              "      <td>[how, come, i, feel, nothing, for, my, family,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>981b4753d17ef14d09f7</td>\n",
              "      <td>In case of collapse of the Democratic party, w...</td>\n",
              "      <td>in case of collapse of the democratic party, w...</td>\n",
              "      <td>[in, case, of, collapse, of, the, democratic, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>452e2c705276ba16b7b7</td>\n",
              "      <td>Who is Émile Naoumoff?</td>\n",
              "      <td>who is émile naoumoff?</td>\n",
              "      <td>[who, is, émile, naoumoff, ?]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d68ddc65-0019-4fe2-adf3-b99934e640d3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d68ddc65-0019-4fe2-adf3-b99934e640d3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d68ddc65-0019-4fe2-adf3-b99934e640d3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                    qid                                                 qt  \\\n",
              "0  a4f3da3a3df9dd881edd  My period is due on my wedding day. How can I ...   \n",
              "1  9914c62ed3f69684d549  How many numbers higher than a million can be ...   \n",
              "2  8138ae48649e37091a91  How come I feel nothing for my family, but sti...   \n",
              "3  981b4753d17ef14d09f7  In case of collapse of the Democratic party, w...   \n",
              "4  452e2c705276ba16b7b7                             Who is Émile Naoumoff?   \n",
              "\n",
              "                                                 qtf  \\\n",
              "0  my period is due on my wedding day. how can i ...   \n",
              "1  how many numbers higher than a million can be ...   \n",
              "2  how come i feel nothing for my family, but sti...   \n",
              "3  in case of collapse of the democratic party, w...   \n",
              "4                             who is émile naoumoff?   \n",
              "\n",
              "                                           tokenized  \n",
              "0  [my, period, is, due, on, my, wedding, day, .,...  \n",
              "1  [how, many, numbers, higher, than, a, million,...  \n",
              "2  [how, come, i, feel, nothing, for, my, family,...  \n",
              "3  [in, case, of, collapse, of, the, democratic, ...  \n",
              "4                      [who, is, émile, naoumoff, ?]  "
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# from nltk.tokenize import word_tokenize\n",
        "\n",
        "# posts['tokenized'] = posts['qtf'].apply(word_tokenize)\n",
        "# posts.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hJboIgCw3veu"
      },
      "source": [
        "## Stop Words Removal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 354
        },
        "id": "tiMMzRwj22fu",
        "outputId": "40864452-95d3-4789-d2a7-88e395b0e645"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-cd58875d-9b9e-4fa6-aa47-399c9e8e4df5\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>qt</th>\n",
              "      <th>qtf</th>\n",
              "      <th>tokenized</th>\n",
              "      <th>stopwords_removed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a4f3da3a3df9dd881edd</td>\n",
              "      <td>My period is due on my wedding day. How can I ...</td>\n",
              "      <td>my period is due on my wedding day. how can i ...</td>\n",
              "      <td>[my, period, is, due, on, my, wedding, day, .,...</td>\n",
              "      <td>[period, due, wedding, day, ., stop, ?, pill, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9914c62ed3f69684d549</td>\n",
              "      <td>How many numbers higher than a million can be ...</td>\n",
              "      <td>how many numbers higher than a million can be ...</td>\n",
              "      <td>[how, many, numbers, higher, than, a, million,...</td>\n",
              "      <td>[many, numbers, higher, million, formed, digit...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8138ae48649e37091a91</td>\n",
              "      <td>How come I feel nothing for my family, but sti...</td>\n",
              "      <td>how come i feel nothing for my family, but sti...</td>\n",
              "      <td>[how, come, i, feel, nothing, for, my, family,...</td>\n",
              "      <td>[come, feel, nothing, family, ,, still, love, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>981b4753d17ef14d09f7</td>\n",
              "      <td>In case of collapse of the Democratic party, w...</td>\n",
              "      <td>in case of collapse of the democratic party, w...</td>\n",
              "      <td>[in, case, of, collapse, of, the, democratic, ...</td>\n",
              "      <td>[case, collapse, democratic, party, ,, republi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>452e2c705276ba16b7b7</td>\n",
              "      <td>Who is Émile Naoumoff?</td>\n",
              "      <td>who is émile naoumoff?</td>\n",
              "      <td>[who, is, émile, naoumoff, ?]</td>\n",
              "      <td>[émile, naoumoff, ?]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cd58875d-9b9e-4fa6-aa47-399c9e8e4df5')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-cd58875d-9b9e-4fa6-aa47-399c9e8e4df5 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-cd58875d-9b9e-4fa6-aa47-399c9e8e4df5');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                    qid                                                 qt  \\\n",
              "0  a4f3da3a3df9dd881edd  My period is due on my wedding day. How can I ...   \n",
              "1  9914c62ed3f69684d549  How many numbers higher than a million can be ...   \n",
              "2  8138ae48649e37091a91  How come I feel nothing for my family, but sti...   \n",
              "3  981b4753d17ef14d09f7  In case of collapse of the Democratic party, w...   \n",
              "4  452e2c705276ba16b7b7                             Who is Émile Naoumoff?   \n",
              "\n",
              "                                                 qtf  \\\n",
              "0  my period is due on my wedding day. how can i ...   \n",
              "1  how many numbers higher than a million can be ...   \n",
              "2  how come i feel nothing for my family, but sti...   \n",
              "3  in case of collapse of the democratic party, w...   \n",
              "4                             who is émile naoumoff?   \n",
              "\n",
              "                                           tokenized  \\\n",
              "0  [my, period, is, due, on, my, wedding, day, .,...   \n",
              "1  [how, many, numbers, higher, than, a, million,...   \n",
              "2  [how, come, i, feel, nothing, for, my, family,...   \n",
              "3  [in, case, of, collapse, of, the, democratic, ...   \n",
              "4                      [who, is, émile, naoumoff, ?]   \n",
              "\n",
              "                                   stopwords_removed  \n",
              "0  [period, due, wedding, day, ., stop, ?, pill, ...  \n",
              "1  [many, numbers, higher, million, formed, digit...  \n",
              "2  [come, feel, nothing, family, ,, still, love, ...  \n",
              "3  [case, collapse, democratic, party, ,, republi...  \n",
              "4                               [émile, naoumoff, ?]  "
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# nltk.download(\"stopwords\")\n",
        "# from nltk.corpus import stopwords\n",
        "\n",
        "# stpwrd = set(stopwords.words('english'))\n",
        "\n",
        "# posts['stopwords_removed'] = posts['tokenized'].apply(lambda x: [word for word in x if word not in stpwrd])\n",
        "# posts.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QLhpBjod5H-O"
      },
      "source": [
        "## Stemming"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-tSDjMUp3sJa"
      },
      "outputs": [],
      "source": [
        "# from nltk.stem import LancasterStemmer\n",
        "\n",
        "# def lancaster_stemmer(text):\n",
        "#     stemmer = nltk.LancasterStemmer()\n",
        "#     stems = [stemmer.stem(i) for i in text]\n",
        "#     return stems"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 320
        },
        "id": "fZGGgrb75L3N",
        "outputId": "8793addd-d8dc-4817-b994-62962381a7f5"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-94f7f980-5cda-46b8-8011-fc592636edab\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>qt</th>\n",
              "      <th>qtf</th>\n",
              "      <th>tokenized</th>\n",
              "      <th>stopwords_removed</th>\n",
              "      <th>lancaster_stemmer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a4f3da3a3df9dd881edd</td>\n",
              "      <td>My period is due on my wedding day. How can I ...</td>\n",
              "      <td>my period is due on my wedding day how can i s...</td>\n",
              "      <td>[my, period, is, due, on, my, wedding, day, ho...</td>\n",
              "      <td>[period, due, wedding, day, stop, pill, option]</td>\n",
              "      <td>[period, due, wed, day, stop, pil, opt]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9914c62ed3f69684d549</td>\n",
              "      <td>How many numbers higher than a million can be ...</td>\n",
              "      <td>how many numbers higher than a million can be ...</td>\n",
              "      <td>[how, many, numbers, higher, than, a, million,...</td>\n",
              "      <td>[many, numbers, higher, million, formed, digit...</td>\n",
              "      <td>[many, numb, high, mil, form, digit, 0445553]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8138ae48649e37091a91</td>\n",
              "      <td>How come I feel nothing for my family, but sti...</td>\n",
              "      <td>how come i feel nothing for my family but stil...</td>\n",
              "      <td>[how, come, i, feel, nothing, for, my, family,...</td>\n",
              "      <td>[come, feel, nothing, family, still, love, pet...</td>\n",
              "      <td>[com, feel, noth, famy, stil, lov, pet, friend...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>981b4753d17ef14d09f7</td>\n",
              "      <td>In case of collapse of the Democratic party, w...</td>\n",
              "      <td>in case of collapse of the democratic party wi...</td>\n",
              "      <td>[in, case, of, collapse, of, the, democratic, ...</td>\n",
              "      <td>[case, collapse, democratic, party, republican...</td>\n",
              "      <td>[cas, collaps, democr, party, republ, party, s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>452e2c705276ba16b7b7</td>\n",
              "      <td>Who is Émile Naoumoff?</td>\n",
              "      <td>who is mile naoumoff</td>\n",
              "      <td>[who, is, mile, naoumoff]</td>\n",
              "      <td>[mile, naoumoff]</td>\n",
              "      <td>[mil, naoumoff]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-94f7f980-5cda-46b8-8011-fc592636edab')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-94f7f980-5cda-46b8-8011-fc592636edab button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-94f7f980-5cda-46b8-8011-fc592636edab');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                    qid                                                 qt  \\\n",
              "0  a4f3da3a3df9dd881edd  My period is due on my wedding day. How can I ...   \n",
              "1  9914c62ed3f69684d549  How many numbers higher than a million can be ...   \n",
              "2  8138ae48649e37091a91  How come I feel nothing for my family, but sti...   \n",
              "3  981b4753d17ef14d09f7  In case of collapse of the Democratic party, w...   \n",
              "4  452e2c705276ba16b7b7                             Who is Émile Naoumoff?   \n",
              "\n",
              "                                                 qtf  \\\n",
              "0  my period is due on my wedding day how can i s...   \n",
              "1  how many numbers higher than a million can be ...   \n",
              "2  how come i feel nothing for my family but stil...   \n",
              "3  in case of collapse of the democratic party wi...   \n",
              "4                               who is mile naoumoff   \n",
              "\n",
              "                                           tokenized  \\\n",
              "0  [my, period, is, due, on, my, wedding, day, ho...   \n",
              "1  [how, many, numbers, higher, than, a, million,...   \n",
              "2  [how, come, i, feel, nothing, for, my, family,...   \n",
              "3  [in, case, of, collapse, of, the, democratic, ...   \n",
              "4                          [who, is, mile, naoumoff]   \n",
              "\n",
              "                                   stopwords_removed  \\\n",
              "0    [period, due, wedding, day, stop, pill, option]   \n",
              "1  [many, numbers, higher, million, formed, digit...   \n",
              "2  [come, feel, nothing, family, still, love, pet...   \n",
              "3  [case, collapse, democratic, party, republican...   \n",
              "4                                   [mile, naoumoff]   \n",
              "\n",
              "                                   lancaster_stemmer  \n",
              "0            [period, due, wed, day, stop, pil, opt]  \n",
              "1      [many, numb, high, mil, form, digit, 0445553]  \n",
              "2  [com, feel, noth, famy, stil, lov, pet, friend...  \n",
              "3  [cas, collaps, democr, party, republ, party, s...  \n",
              "4                                    [mil, naoumoff]  "
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "posts['lancaster_stemmer'] = posts['stopwords_removed'].apply(lambda x: lancaster_stemmer(x)) #Stemming the posts\n",
        "posts.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-2gf0bWN7Yis"
      },
      "source": [
        "## Part of Speech Tagging"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u9Gmtj_r8Npa"
      },
      "source": [
        "The part of speech (noun, verb, adjective, etc.) of each word in the text is identified using part of speech (POS) tagging. Since we may infer a word's contextual meaning by determining its POS, this is a crucial stage for many NLP applications."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-c3a6qv17ohm",
        "outputId": "d5c18336-3cba-46b6-df70-723d55fbe0a4"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]   Package brown is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "# nltk.download('brown')\n",
        "# from nltk.corpus import wordnet\n",
        "# from nltk.corpus import brown\n",
        "\n",
        "# wordnet_map = {\"N\":wordnet.NOUN, \n",
        "#                \"V\":wordnet.VERB, \n",
        "#                \"J\":wordnet.ADJ, \n",
        "#                \"R\":wordnet.ADV\n",
        "#               }\n",
        "    \n",
        "# train_sents = brown.tagged_sents(categories='news')\n",
        "# t0 = nltk.DefaultTagger('NN')\n",
        "# t1 = nltk.UnigramTagger(train_sents, backoff=t0)\n",
        "# t2 = nltk.BigramTagger(train_sents, backoff=t1)\n",
        "\n",
        "# def pos_tag_wordnet(text, pos_tag_type=\"pos_tag\"):\n",
        "  \n",
        "#     pos_tagged_text = t2.tag(text)\n",
        "     \n",
        "#     pos_tagged_text = [(word, wordnet_map.get(pos_tag[0])) if pos_tag[0] in wordnet_map.keys() else (word, wordnet.NOUN) for (word, pos_tag) in pos_tagged_text ]\n",
        "#     return pos_tagged_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 406
        },
        "id": "SNfz5PFg7xwb",
        "outputId": "426ccd8c-0a70-498f-dcc6-3b0742567854"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-9c08532e-8727-4d7d-87e7-497f57e3fb3f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>qt</th>\n",
              "      <th>qtf</th>\n",
              "      <th>tokenized</th>\n",
              "      <th>stopwords_removed</th>\n",
              "      <th>lancaster_stemmer</th>\n",
              "      <th>combined_postag_wnet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a4f3da3a3df9dd881edd</td>\n",
              "      <td>My period is due on my wedding day. How can I ...</td>\n",
              "      <td>my period is due on my wedding day how can i s...</td>\n",
              "      <td>[my, period, is, due, on, my, wedding, day, ho...</td>\n",
              "      <td>[period, due, wedding, day, stop, pill, option]</td>\n",
              "      <td>[period, due, wed, day, stop, pil, opt]</td>\n",
              "      <td>[(period, n), (due, r), (wedding, n), (day, n)...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9914c62ed3f69684d549</td>\n",
              "      <td>How many numbers higher than a million can be ...</td>\n",
              "      <td>how many numbers higher than a million can be ...</td>\n",
              "      <td>[how, many, numbers, higher, than, a, million,...</td>\n",
              "      <td>[many, numbers, higher, million, formed, digit...</td>\n",
              "      <td>[many, numb, high, mil, form, digit, 0445553]</td>\n",
              "      <td>[(many, n), (numbers, n), (higher, r), (millio...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8138ae48649e37091a91</td>\n",
              "      <td>How come I feel nothing for my family, but sti...</td>\n",
              "      <td>how come i feel nothing for my family but stil...</td>\n",
              "      <td>[how, come, i, feel, nothing, for, my, family,...</td>\n",
              "      <td>[come, feel, nothing, family, still, love, pet...</td>\n",
              "      <td>[com, feel, noth, famy, stil, lov, pet, friend...</td>\n",
              "      <td>[(come, v), (feel, v), (nothing, n), (family, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>981b4753d17ef14d09f7</td>\n",
              "      <td>In case of collapse of the Democratic party, w...</td>\n",
              "      <td>in case of collapse of the democratic party wi...</td>\n",
              "      <td>[in, case, of, collapse, of, the, democratic, ...</td>\n",
              "      <td>[case, collapse, democratic, party, republican...</td>\n",
              "      <td>[cas, collaps, democr, party, republ, party, s...</td>\n",
              "      <td>[(case, n), (collapse, n), (democratic, a), (p...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>452e2c705276ba16b7b7</td>\n",
              "      <td>Who is Émile Naoumoff?</td>\n",
              "      <td>who is mile naoumoff</td>\n",
              "      <td>[who, is, mile, naoumoff]</td>\n",
              "      <td>[mile, naoumoff]</td>\n",
              "      <td>[mil, naoumoff]</td>\n",
              "      <td>[(mile, n), (naoumoff, n)]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9c08532e-8727-4d7d-87e7-497f57e3fb3f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9c08532e-8727-4d7d-87e7-497f57e3fb3f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9c08532e-8727-4d7d-87e7-497f57e3fb3f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                    qid                                                 qt  \\\n",
              "0  a4f3da3a3df9dd881edd  My period is due on my wedding day. How can I ...   \n",
              "1  9914c62ed3f69684d549  How many numbers higher than a million can be ...   \n",
              "2  8138ae48649e37091a91  How come I feel nothing for my family, but sti...   \n",
              "3  981b4753d17ef14d09f7  In case of collapse of the Democratic party, w...   \n",
              "4  452e2c705276ba16b7b7                             Who is Émile Naoumoff?   \n",
              "\n",
              "                                                 qtf  \\\n",
              "0  my period is due on my wedding day how can i s...   \n",
              "1  how many numbers higher than a million can be ...   \n",
              "2  how come i feel nothing for my family but stil...   \n",
              "3  in case of collapse of the democratic party wi...   \n",
              "4                               who is mile naoumoff   \n",
              "\n",
              "                                           tokenized  \\\n",
              "0  [my, period, is, due, on, my, wedding, day, ho...   \n",
              "1  [how, many, numbers, higher, than, a, million,...   \n",
              "2  [how, come, i, feel, nothing, for, my, family,...   \n",
              "3  [in, case, of, collapse, of, the, democratic, ...   \n",
              "4                          [who, is, mile, naoumoff]   \n",
              "\n",
              "                                   stopwords_removed  \\\n",
              "0    [period, due, wedding, day, stop, pill, option]   \n",
              "1  [many, numbers, higher, million, formed, digit...   \n",
              "2  [come, feel, nothing, family, still, love, pet...   \n",
              "3  [case, collapse, democratic, party, republican...   \n",
              "4                                   [mile, naoumoff]   \n",
              "\n",
              "                                   lancaster_stemmer  \\\n",
              "0            [period, due, wed, day, stop, pil, opt]   \n",
              "1      [many, numb, high, mil, form, digit, 0445553]   \n",
              "2  [com, feel, noth, famy, stil, lov, pet, friend...   \n",
              "3  [cas, collaps, democr, party, republ, party, s...   \n",
              "4                                    [mil, naoumoff]   \n",
              "\n",
              "                                combined_postag_wnet  \n",
              "0  [(period, n), (due, r), (wedding, n), (day, n)...  \n",
              "1  [(many, n), (numbers, n), (higher, r), (millio...  \n",
              "2  [(come, v), (feel, v), (nothing, n), (family, ...  \n",
              "3  [(case, n), (collapse, n), (democratic, a), (p...  \n",
              "4                         [(mile, n), (naoumoff, n)]  "
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# posts['combined_postag_wnet'] = posts['stopwords_removed'].apply(lambda x: pos_tag_wordnet(x))\n",
        "# posts.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LvXP1HIc9UHi"
      },
      "source": [
        "## Lemmatization"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I-5kkmQ09W_b"
      },
      "outputs": [],
      "source": [
        "# from nltk.stem import WordNetLemmatizer\n",
        "\n",
        "# def lemmatize_word(text):\n",
        "\n",
        "#     lemmatizer = WordNetLemmatizer()\n",
        "#     lemma = [lemmatizer.lemmatize(word, tag) for word, tag in text]\n",
        "#     return lemma"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 510
        },
        "id": "az3B_RS08iiL",
        "outputId": "e1996dc4-1943-42bb-82c7-e56e394ab1f7"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-8fb7adc9-1748-4998-9b3b-f92be16e4f01\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>qt</th>\n",
              "      <th>qtf</th>\n",
              "      <th>tokenized</th>\n",
              "      <th>stopwords_removed</th>\n",
              "      <th>lancaster_stemmer</th>\n",
              "      <th>combined_postag_wnet</th>\n",
              "      <th>lemmatize_word_wo_pos</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a4f3da3a3df9dd881edd</td>\n",
              "      <td>My period is due on my wedding day. How can I ...</td>\n",
              "      <td>my period is due on my wedding day how can i s...</td>\n",
              "      <td>[my, period, is, due, on, my, wedding, day, ho...</td>\n",
              "      <td>[period, due, wedding, day, stop, pill, option]</td>\n",
              "      <td>[period, due, wed, day, stop, pil, opt]</td>\n",
              "      <td>[(period, n), (due, r), (wedding, n), (day, n)...</td>\n",
              "      <td>[period, due, wedding, day, stop, pill, option]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9914c62ed3f69684d549</td>\n",
              "      <td>How many numbers higher than a million can be ...</td>\n",
              "      <td>how many numbers higher than a million can be ...</td>\n",
              "      <td>[how, many, numbers, higher, than, a, million,...</td>\n",
              "      <td>[many, numbers, higher, million, formed, digit...</td>\n",
              "      <td>[many, numb, high, mil, form, digit, 0445553]</td>\n",
              "      <td>[(many, n), (numbers, n), (higher, r), (millio...</td>\n",
              "      <td>[many, number, higher, million, formed, digit,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8138ae48649e37091a91</td>\n",
              "      <td>How come I feel nothing for my family, but sti...</td>\n",
              "      <td>how come i feel nothing for my family but stil...</td>\n",
              "      <td>[how, come, i, feel, nothing, for, my, family,...</td>\n",
              "      <td>[come, feel, nothing, family, still, love, pet...</td>\n",
              "      <td>[com, feel, noth, famy, stil, lov, pet, friend...</td>\n",
              "      <td>[(come, v), (feel, v), (nothing, n), (family, ...</td>\n",
              "      <td>[come, feel, nothing, family, still, love, pet...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>981b4753d17ef14d09f7</td>\n",
              "      <td>In case of collapse of the Democratic party, w...</td>\n",
              "      <td>in case of collapse of the democratic party wi...</td>\n",
              "      <td>[in, case, of, collapse, of, the, democratic, ...</td>\n",
              "      <td>[case, collapse, democratic, party, republican...</td>\n",
              "      <td>[cas, collaps, democr, party, republ, party, s...</td>\n",
              "      <td>[(case, n), (collapse, n), (democratic, a), (p...</td>\n",
              "      <td>[case, collapse, democratic, party, republican...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>452e2c705276ba16b7b7</td>\n",
              "      <td>Who is Émile Naoumoff?</td>\n",
              "      <td>who is mile naoumoff</td>\n",
              "      <td>[who, is, mile, naoumoff]</td>\n",
              "      <td>[mile, naoumoff]</td>\n",
              "      <td>[mil, naoumoff]</td>\n",
              "      <td>[(mile, n), (naoumoff, n)]</td>\n",
              "      <td>[mile, naoumoff]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8fb7adc9-1748-4998-9b3b-f92be16e4f01')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8fb7adc9-1748-4998-9b3b-f92be16e4f01 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8fb7adc9-1748-4998-9b3b-f92be16e4f01');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                    qid                                                 qt  \\\n",
              "0  a4f3da3a3df9dd881edd  My period is due on my wedding day. How can I ...   \n",
              "1  9914c62ed3f69684d549  How many numbers higher than a million can be ...   \n",
              "2  8138ae48649e37091a91  How come I feel nothing for my family, but sti...   \n",
              "3  981b4753d17ef14d09f7  In case of collapse of the Democratic party, w...   \n",
              "4  452e2c705276ba16b7b7                             Who is Émile Naoumoff?   \n",
              "\n",
              "                                                 qtf  \\\n",
              "0  my period is due on my wedding day how can i s...   \n",
              "1  how many numbers higher than a million can be ...   \n",
              "2  how come i feel nothing for my family but stil...   \n",
              "3  in case of collapse of the democratic party wi...   \n",
              "4                               who is mile naoumoff   \n",
              "\n",
              "                                           tokenized  \\\n",
              "0  [my, period, is, due, on, my, wedding, day, ho...   \n",
              "1  [how, many, numbers, higher, than, a, million,...   \n",
              "2  [how, come, i, feel, nothing, for, my, family,...   \n",
              "3  [in, case, of, collapse, of, the, democratic, ...   \n",
              "4                          [who, is, mile, naoumoff]   \n",
              "\n",
              "                                   stopwords_removed  \\\n",
              "0    [period, due, wedding, day, stop, pill, option]   \n",
              "1  [many, numbers, higher, million, formed, digit...   \n",
              "2  [come, feel, nothing, family, still, love, pet...   \n",
              "3  [case, collapse, democratic, party, republican...   \n",
              "4                                   [mile, naoumoff]   \n",
              "\n",
              "                                   lancaster_stemmer  \\\n",
              "0            [period, due, wed, day, stop, pil, opt]   \n",
              "1      [many, numb, high, mil, form, digit, 0445553]   \n",
              "2  [com, feel, noth, famy, stil, lov, pet, friend...   \n",
              "3  [cas, collaps, democr, party, republ, party, s...   \n",
              "4                                    [mil, naoumoff]   \n",
              "\n",
              "                                combined_postag_wnet  \\\n",
              "0  [(period, n), (due, r), (wedding, n), (day, n)...   \n",
              "1  [(many, n), (numbers, n), (higher, r), (millio...   \n",
              "2  [(come, v), (feel, v), (nothing, n), (family, ...   \n",
              "3  [(case, n), (collapse, n), (democratic, a), (p...   \n",
              "4                         [(mile, n), (naoumoff, n)]   \n",
              "\n",
              "                               lemmatize_word_wo_pos  \n",
              "0    [period, due, wedding, day, stop, pill, option]  \n",
              "1  [many, number, higher, million, formed, digit,...  \n",
              "2  [come, feel, nothing, family, still, love, pet...  \n",
              "3  [case, collapse, democratic, party, republican...  \n",
              "4                                   [mile, naoumoff]  "
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# #Lemmatization without pos tagging\n",
        "# lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "# posts['lemmatize_word_wo_pos'] = posts['stopwords_removed'].apply(lambda x: [lemmatizer.lemmatize(word) for word in x])\n",
        "# posts['lemmatize_word_wo_pos'] = posts['lemmatize_word_wo_pos'].apply(lambda x: [word for word in x if word not in stpwrd])\n",
        "# posts.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "95juEsp09l2i",
        "outputId": "ec4a340f-1ddd-4e92-db1e-d6c91253701c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-ee96a16f-d1fb-4f2c-9933-db15c139417c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>qt</th>\n",
              "      <th>qtf</th>\n",
              "      <th>tokenized</th>\n",
              "      <th>stopwords_removed</th>\n",
              "      <th>lancaster_stemmer</th>\n",
              "      <th>combined_postag_wnet</th>\n",
              "      <th>lemmatize_word_wo_pos</th>\n",
              "      <th>lemmatize_word_w_pos</th>\n",
              "      <th>lemmatize_text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a4f3da3a3df9dd881edd</td>\n",
              "      <td>My period is due on my wedding day. How can I ...</td>\n",
              "      <td>my period is due on my wedding day how can i s...</td>\n",
              "      <td>[my, period, is, due, on, my, wedding, day, ho...</td>\n",
              "      <td>[period, due, wedding, day, stop, pill, option]</td>\n",
              "      <td>[period, due, wed, day, stop, pil, opt]</td>\n",
              "      <td>[(period, n), (due, r), (wedding, n), (day, n)...</td>\n",
              "      <td>[period, due, wedding, day, stop, pill, option]</td>\n",
              "      <td>[period, due, wedding, day, stop, pill, option]</td>\n",
              "      <td>period due wedding day stop pill option</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9914c62ed3f69684d549</td>\n",
              "      <td>How many numbers higher than a million can be ...</td>\n",
              "      <td>how many numbers higher than a million can be ...</td>\n",
              "      <td>[how, many, numbers, higher, than, a, million,...</td>\n",
              "      <td>[many, numbers, higher, million, formed, digit...</td>\n",
              "      <td>[many, numb, high, mil, form, digit, 0445553]</td>\n",
              "      <td>[(many, n), (numbers, n), (higher, r), (millio...</td>\n",
              "      <td>[many, number, higher, million, formed, digit,...</td>\n",
              "      <td>[many, number, higher, million, form, digit, 0...</td>\n",
              "      <td>many number higher million form digit 0445553</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8138ae48649e37091a91</td>\n",
              "      <td>How come I feel nothing for my family, but sti...</td>\n",
              "      <td>how come i feel nothing for my family but stil...</td>\n",
              "      <td>[how, come, i, feel, nothing, for, my, family,...</td>\n",
              "      <td>[come, feel, nothing, family, still, love, pet...</td>\n",
              "      <td>[com, feel, noth, famy, stil, lov, pet, friend...</td>\n",
              "      <td>[(come, v), (feel, v), (nothing, n), (family, ...</td>\n",
              "      <td>[come, feel, nothing, family, still, love, pet...</td>\n",
              "      <td>[come, feel, nothing, family, still, love, pet...</td>\n",
              "      <td>come feel nothing family still love pet friend...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>981b4753d17ef14d09f7</td>\n",
              "      <td>In case of collapse of the Democratic party, w...</td>\n",
              "      <td>in case of collapse of the democratic party wi...</td>\n",
              "      <td>[in, case, of, collapse, of, the, democratic, ...</td>\n",
              "      <td>[case, collapse, democratic, party, republican...</td>\n",
              "      <td>[cas, collaps, democr, party, republ, party, s...</td>\n",
              "      <td>[(case, n), (collapse, n), (democratic, a), (p...</td>\n",
              "      <td>[case, collapse, democratic, party, republican...</td>\n",
              "      <td>[case, collapse, democratic, party, republican...</td>\n",
              "      <td>case collapse democratic party republican part...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>452e2c705276ba16b7b7</td>\n",
              "      <td>Who is Émile Naoumoff?</td>\n",
              "      <td>who is mile naoumoff</td>\n",
              "      <td>[who, is, mile, naoumoff]</td>\n",
              "      <td>[mile, naoumoff]</td>\n",
              "      <td>[mil, naoumoff]</td>\n",
              "      <td>[(mile, n), (naoumoff, n)]</td>\n",
              "      <td>[mile, naoumoff]</td>\n",
              "      <td>[mile, naoumoff]</td>\n",
              "      <td>mile naoumoff</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ee96a16f-d1fb-4f2c-9933-db15c139417c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ee96a16f-d1fb-4f2c-9933-db15c139417c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ee96a16f-d1fb-4f2c-9933-db15c139417c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                    qid                                                 qt  \\\n",
              "0  a4f3da3a3df9dd881edd  My period is due on my wedding day. How can I ...   \n",
              "1  9914c62ed3f69684d549  How many numbers higher than a million can be ...   \n",
              "2  8138ae48649e37091a91  How come I feel nothing for my family, but sti...   \n",
              "3  981b4753d17ef14d09f7  In case of collapse of the Democratic party, w...   \n",
              "4  452e2c705276ba16b7b7                             Who is Émile Naoumoff?   \n",
              "\n",
              "                                                 qtf  \\\n",
              "0  my period is due on my wedding day how can i s...   \n",
              "1  how many numbers higher than a million can be ...   \n",
              "2  how come i feel nothing for my family but stil...   \n",
              "3  in case of collapse of the democratic party wi...   \n",
              "4                               who is mile naoumoff   \n",
              "\n",
              "                                           tokenized  \\\n",
              "0  [my, period, is, due, on, my, wedding, day, ho...   \n",
              "1  [how, many, numbers, higher, than, a, million,...   \n",
              "2  [how, come, i, feel, nothing, for, my, family,...   \n",
              "3  [in, case, of, collapse, of, the, democratic, ...   \n",
              "4                          [who, is, mile, naoumoff]   \n",
              "\n",
              "                                   stopwords_removed  \\\n",
              "0    [period, due, wedding, day, stop, pill, option]   \n",
              "1  [many, numbers, higher, million, formed, digit...   \n",
              "2  [come, feel, nothing, family, still, love, pet...   \n",
              "3  [case, collapse, democratic, party, republican...   \n",
              "4                                   [mile, naoumoff]   \n",
              "\n",
              "                                   lancaster_stemmer  \\\n",
              "0            [period, due, wed, day, stop, pil, opt]   \n",
              "1      [many, numb, high, mil, form, digit, 0445553]   \n",
              "2  [com, feel, noth, famy, stil, lov, pet, friend...   \n",
              "3  [cas, collaps, democr, party, republ, party, s...   \n",
              "4                                    [mil, naoumoff]   \n",
              "\n",
              "                                combined_postag_wnet  \\\n",
              "0  [(period, n), (due, r), (wedding, n), (day, n)...   \n",
              "1  [(many, n), (numbers, n), (higher, r), (millio...   \n",
              "2  [(come, v), (feel, v), (nothing, n), (family, ...   \n",
              "3  [(case, n), (collapse, n), (democratic, a), (p...   \n",
              "4                         [(mile, n), (naoumoff, n)]   \n",
              "\n",
              "                               lemmatize_word_wo_pos  \\\n",
              "0    [period, due, wedding, day, stop, pill, option]   \n",
              "1  [many, number, higher, million, formed, digit,...   \n",
              "2  [come, feel, nothing, family, still, love, pet...   \n",
              "3  [case, collapse, democratic, party, republican...   \n",
              "4                                   [mile, naoumoff]   \n",
              "\n",
              "                                lemmatize_word_w_pos  \\\n",
              "0    [period, due, wedding, day, stop, pill, option]   \n",
              "1  [many, number, higher, million, form, digit, 0...   \n",
              "2  [come, feel, nothing, family, still, love, pet...   \n",
              "3  [case, collapse, democratic, party, republican...   \n",
              "4                                   [mile, naoumoff]   \n",
              "\n",
              "                                      lemmatize_text  \n",
              "0            period due wedding day stop pill option  \n",
              "1      many number higher million form digit 0445553  \n",
              "2  come feel nothing family still love pet friend...  \n",
              "3  case collapse democratic party republican part...  \n",
              "4                                      mile naoumoff  "
            ]
          },
          "execution_count": 118,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# #Lemmatization with pos tagging\n",
        "# lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "# posts['lemmatize_word_w_pos'] = posts['combined_postag_wnet'].apply(lambda x: lemmatize_word(x))\n",
        "# posts['lemmatize_word_w_pos'] = posts['lemmatize_word_w_pos'].apply(lambda x: [word for word in x if word not in stpwrd]) \n",
        "# posts['lemmatize_text'] = [' '.join(map(str, l)) for l in posts['lemmatize_word_w_pos']] # join back to text\n",
        "\n",
        "# posts.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o6SvpaYNAGCz"
      },
      "source": [
        "# Saving progress variable"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JR-R_jbV_tqC"
      },
      "outputs": [],
      "source": [
        "# import pickle\n",
        "# with open('/content/gdrive/MyDrive/Data/posts.pkl', 'wb') as file:\n",
        "     \n",
        "#     pickle.dump(posts, file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HUpwfki8IfjN"
      },
      "outputs": [],
      "source": [
        "# lemtxt = posts[\"lemmatize_text\"].tolist()\n",
        "# with open('/content/gdrive/MyDrive/Data/lemtxt.pkl', 'wb') as file:\n",
        "     \n",
        "#     pickle.dump(lemtxt, file)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fVQ9BMJmCP4X"
      },
      "source": [
        "# Loading Progress Variable"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "69_96XOHAL7a"
      },
      "outputs": [],
      "source": [
        "# import pickle\n",
        "# with open('/content/gdrive/MyDrive/Data/posts.pkl', 'rb') as file:\n",
        "#     posts = pickle.load(file)\n",
        "  \n",
        "# posts.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-JQssGmEJDMV"
      },
      "outputs": [],
      "source": [
        "# with open('/content/gdrive/MyDrive/Data/lemtxt.pkl', 'rb') as file:\n",
        "#     lemtxt = pickle.load(file)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dAZ47x2qDbO-"
      },
      "source": [
        "# Feature Extraction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fpepQlQvEFld"
      },
      "source": [
        "# Bag of words"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QcGBa2AkCiOP"
      },
      "outputs": [],
      "source": [
        "# from sklearn.feature_extraction.text import CountVectorizer\n",
        "# import numpy as np\n",
        "# def cv(data, ngram = 1, MAX_NB_WORDS = 75000):\n",
        "#     count_vectorizer = CountVectorizer(ngram_range = (ngram, ngram), max_features = MAX_NB_WORDS)\n",
        "    \n",
        "#     emb = count_vectorizer.fit_transform(data)\n",
        "#     return emb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "t0MXARdbGVNS",
        "outputId": "619f762f-3ab7-4af1-a189-e689b7e42e9c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-f606be7b-c90a-4f7b-a38e-03d392089fb4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>qid</th>\n",
              "      <th>qt</th>\n",
              "      <th>qtf</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>a4f3da3a3df9dd881edd</td>\n",
              "      <td>My period is due on my wedding day. How can I ...</td>\n",
              "      <td>my period is due on my wedding day. how can i ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>9914c62ed3f69684d549</td>\n",
              "      <td>How many numbers higher than a million can be ...</td>\n",
              "      <td>how many numbers higher than a million can be ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8138ae48649e37091a91</td>\n",
              "      <td>How come I feel nothing for my family, but sti...</td>\n",
              "      <td>how come i feel nothing for my family, but sti...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>981b4753d17ef14d09f7</td>\n",
              "      <td>In case of collapse of the Democratic party, w...</td>\n",
              "      <td>in case of collapse of the democratic party, w...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>452e2c705276ba16b7b7</td>\n",
              "      <td>Who is Émile Naoumoff?</td>\n",
              "      <td>who is émile naoumoff?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306117</th>\n",
              "      <td>4bd96088d0b5f0f2c4f4</td>\n",
              "      <td>How is CSE at VIT Chennai?</td>\n",
              "      <td>how is cse at vit chennai?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306118</th>\n",
              "      <td>e80edbfc086f7125940f</td>\n",
              "      <td>How can we prevent a holocaust by robots, AI, ...</td>\n",
              "      <td>how can we prevent a holocaust by robots, ai, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306119</th>\n",
              "      <td>1506dfad6bd340782a1f</td>\n",
              "      <td>How can I help a student remember key steps an...</td>\n",
              "      <td>how can i help a student remember key steps an...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306120</th>\n",
              "      <td>b56c60fd407f2f85553c</td>\n",
              "      <td>What is the difference between lace closure &amp; ...</td>\n",
              "      <td>what is the difference between lace closure &amp; ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1306121</th>\n",
              "      <td>a1b32d315c2782cdbcc3</td>\n",
              "      <td>What happens when you look into a broken mirror?</td>\n",
              "      <td>what happens when you look into a broken mirror?</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1306122 rows × 3 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f606be7b-c90a-4f7b-a38e-03d392089fb4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f606be7b-c90a-4f7b-a38e-03d392089fb4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f606be7b-c90a-4f7b-a38e-03d392089fb4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "                          qid  \\\n",
              "0        a4f3da3a3df9dd881edd   \n",
              "1        9914c62ed3f69684d549   \n",
              "2        8138ae48649e37091a91   \n",
              "3        981b4753d17ef14d09f7   \n",
              "4        452e2c705276ba16b7b7   \n",
              "...                       ...   \n",
              "1306117  4bd96088d0b5f0f2c4f4   \n",
              "1306118  e80edbfc086f7125940f   \n",
              "1306119  1506dfad6bd340782a1f   \n",
              "1306120  b56c60fd407f2f85553c   \n",
              "1306121  a1b32d315c2782cdbcc3   \n",
              "\n",
              "                                                        qt  \\\n",
              "0        My period is due on my wedding day. How can I ...   \n",
              "1        How many numbers higher than a million can be ...   \n",
              "2        How come I feel nothing for my family, but sti...   \n",
              "3        In case of collapse of the Democratic party, w...   \n",
              "4                                   Who is Émile Naoumoff?   \n",
              "...                                                    ...   \n",
              "1306117                         How is CSE at VIT Chennai?   \n",
              "1306118  How can we prevent a holocaust by robots, AI, ...   \n",
              "1306119  How can I help a student remember key steps an...   \n",
              "1306120  What is the difference between lace closure & ...   \n",
              "1306121   What happens when you look into a broken mirror?   \n",
              "\n",
              "                                                       qtf  \n",
              "0        my period is due on my wedding day. how can i ...  \n",
              "1        how many numbers higher than a million can be ...  \n",
              "2        how come i feel nothing for my family, but sti...  \n",
              "3        in case of collapse of the democratic party, w...  \n",
              "4                                   who is émile naoumoff?  \n",
              "...                                                    ...  \n",
              "1306117                         how is cse at vit chennai?  \n",
              "1306118  how can we prevent a holocaust by robots, ai, ...  \n",
              "1306119  how can i help a student remember key steps an...  \n",
              "1306120  what is the difference between lace closure & ...  \n",
              "1306121   what happens when you look into a broken mirror?  \n",
              "\n",
              "[1306122 rows x 3 columns]"
            ]
          },
          "execution_count": 41,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# posts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Beyx6go_HGhM"
      },
      "outputs": [],
      "source": [
        "# # posts['stopwords_removed_1'] = [' '.join(map(str, l)) for l in posts['stopwords_removed']] \n",
        "# posts_corpus = posts[\"qtf\"].tolist()\n",
        "\n",
        "# posts_1gram = cv(posts_corpus, 1)\n",
        "# # posts_2gram = cv(posts_corpus, 2)\n",
        "# # posts_3gram = cv(posts_corpus, 3)\n",
        "\n",
        "posts = pd.DataFrame()\n",
        "posts[\"qid\"] = test[\"qid\"].append(train[\"qid\"],ignore_index = True)\n",
        "posts[\"qt\"] = test[\"question_text\"].append(train[\"question_text\"],ignore_index = True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QpAvW_dxEMik"
      },
      "outputs": [],
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "import numpy as np\n",
        "def cv(data):\n",
        "    count_vectorizer = CountVectorizer(ngram_range = (1,3))\n",
        "    \n",
        "    emb = count_vectorizer.fit_transform(data)\n",
        "    return emb"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qeVacbvRERPW"
      },
      "outputs": [],
      "source": [
        "posts_corpus = posts[\"qt\"].tolist()\n",
        "\n",
        "posts_1gramCV = cv(posts_corpus)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "chvwxmrmPogw"
      },
      "source": [
        "## TF-IDF Vectorizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m3huhzkQPn8f"
      },
      "outputs": [],
      "source": [
        "# from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "# def TFIDF(data, ngram = 1, MAX_NB_WORDS = 75000):\n",
        "#     tfVectorizer = TfidfVectorizer(strip_accents = 'unicode',analyzer = 'word',stop_words = 'english',ngram_range = (1, 2),min_df = 2,max_df = 0.5,norm = 'l2',max_features = 10000,sublinear_tf=True)\n",
        "#     tfFitTransform = tfVectorizer.fit_transform(data)\n",
        "#     fres = tfFitTransform.tolil().data.tolist()\n",
        "#     sparseMatrix = tfFitTransform.tocsr()\n",
        "\n",
        "#     return sparseMatrix"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OnhHhWLYQJku"
      },
      "outputs": [],
      "source": [
        "# posts_1gram = TFIDF(posts_corpus, 1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7rwfhyPlJOOD"
      },
      "outputs": [],
      "source": [
        "# from scipy import sparse\n",
        "# from scipy import stats"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kaytG1GMKckM"
      },
      "outputs": [],
      "source": [
        "# testPostCV = posts_1gram[0:len(test)]\n",
        "# trainPostCV = posts_1gram[len(test):]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lQcW_60NUln7"
      },
      "source": [
        "# Model Fitting"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BwSflrQYEZ7j"
      },
      "outputs": [],
      "source": [
        "from scipy import sparse\n",
        "from scipy import stats"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9ykE8hwyEbWk"
      },
      "outputs": [],
      "source": [
        "testPostCV = posts_1gramCV[0:len(test)]\n",
        "trainPostCV = posts_1gramCV[len(test):]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "opTq0I2VK1hq"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_trainCV, X_testCV, Y_trainCV, Y_testCV = train_test_split(trainPostCV,train[\"target\"],test_size=0.2343752)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iyqV7g6iLbbJ"
      },
      "outputs": [],
      "source": [
        "def generateCSV(name, predictedData):\n",
        "    predictedOutput = { 'qid' : test.iloc[:, 0], 'target' : predictedData }\n",
        "\n",
        "    predictedDF = pd.DataFrame(predictedOutput)\n",
        "\n",
        "    predictedDF.to_csv(name, index=None)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DntapS6DFFkO"
      },
      "source": [
        "## Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JLP_9DXsFFJq"
      },
      "outputs": [],
      "source": [
        "# from sklearn.linear_model import LogisticRegression\n",
        "# from sklearn.metrics import classification_report\n",
        "# from sklearn.metrics import f1_score\n",
        "\n",
        "# classifier = LogisticRegression()\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i_AWyXwgEvvy"
      },
      "source": [
        "## Logistic Regression with GridSearchCV"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vSyfpHOQE0rz"
      },
      "outputs": [],
      "source": [
        "# param_grid = [\n",
        "#     {\n",
        "#         'penalty' : ['l1','l2','elasticnet','none'],\n",
        "#         'solver'  : ['lbfgs','sag','liblinear','saga','newton-cg']\n",
        "#     }\n",
        "# ]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o5binSuOE2nt"
      },
      "outputs": [],
      "source": [
        "# from sklearn.model_selection import GridSearchCV\n",
        "# wghts = {0:0.15,1:0.85}\n",
        "# classifier = GridSearchCV(LogisticRegression(class_weight = wghts),param_grid=param_grid,cv=3,verbose=True,n_jobs=-1)\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uQb5UMfiFlo3"
      },
      "source": [
        "## Naive Bayes"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "izkQRi3XLGGN"
      },
      "source": [
        "### Multinomial Naive Bayes"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Rorn4mwuLPjz"
      },
      "outputs": [],
      "source": [
        "# classifier = MultinomialNB()\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rTxTZZHILZHD"
      },
      "source": [
        "### Bernoulli Naive Baye's"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "967cn6rtLY0G"
      },
      "outputs": [],
      "source": [
        "# from sklearn.naive_bayes import BernoulliNB\n",
        "# classifier = BernoulliNB()\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H0-JZeGvMIy5"
      },
      "source": [
        "## Random Forest Regressor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "or2KWiJhNiCy"
      },
      "outputs": [],
      "source": [
        "# from sklearn.ensemble import RandomForestRegressor\n",
        "# classifier = RandomForestRegressor(max_depth=100)\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fmY-7xK9OrTH"
      },
      "source": [
        "## Suport Vector Machine"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EOeh_zX4OtsH"
      },
      "source": [
        "### Linear Kernel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sRVoQG63NjKZ"
      },
      "outputs": [],
      "source": [
        "# from sklearn import svm\n",
        "# classifier = svm.SVC(kernel='linear')\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p3ZpcvcNOw_n"
      },
      "source": [
        "### Poly kernel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yg_fbzQBOyiO"
      },
      "outputs": [],
      "source": [
        "# from sklearn import svm\n",
        "# classifier = svm.SVC(kernel='poly',degree=4)\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hVM25vX9O2oK"
      },
      "source": [
        "### RBF Kernel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AqTMUjU8O9Lz"
      },
      "outputs": [],
      "source": [
        "# from sklearn import svm\n",
        "# classifier = svm.SVC(kernel='rbf')\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qOw2NWJoO_1g"
      },
      "source": [
        "### Sigmoid Kernel"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5eXZ534AO_Yb"
      },
      "outputs": [],
      "source": [
        "# from sklearn import svm\n",
        "# classifier = svm.SVC(kernel='sigmoid')\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uEOXEl-4PbH9"
      },
      "source": [
        "## Ridge Classifier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yQpWxTQAQPPA"
      },
      "outputs": [],
      "source": [
        "# from sklearn.linear_model import Ridge\n",
        "# classifier = Ridge(alpha = 0.5, normalize = False,solver ='auto', tol = 0.001)\n",
        "# classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ntTOFk5XQhRB"
      },
      "source": [
        "## Best Fit"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "upyj18kDQjVh"
      },
      "outputs": [],
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import f1_score\n",
        "\n",
        "wghts = {0:0.15,1:0.85}\n",
        "classifier = LogisticRegression(max_iter=1000,C=0.215,class_weight=wghts,solver='liblinear')\n",
        "classifier.fit(X_trainCV,Y_trainCV)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "getKKwmuQjVh"
      },
      "outputs": [],
      "source": [
        "Y_pred = classifier.predict_proba(X_testCV)\n",
        "max=0\n",
        "num=0\n",
        "for j in range(100):\n",
        "  ftmp = [0]*len(Y_pred)\n",
        "  for i in range(len(Y_pred)):\n",
        "    if(Y_pred[i][1] > j/100):\n",
        "      ftmp[i] = 1\n",
        "  if(f1_score(ftmp,Y_testCV) > max):\n",
        "    max=f1_score(ftmp,Y_testCV)\n",
        "    num=j\n",
        "print(num,max)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1DZvwN7rQjVh"
      },
      "outputs": [],
      "source": [
        "Y_predout = classifier.predict_proba(testPostCV)\n",
        "yPredictedTest = [0]*len(Y_predout)\n",
        "for i in range(len(Y_predout)):\n",
        "  if(Y_predout[i][1] > num/100):\n",
        "    yPredictedTest[i] = 1\n",
        "generateCSV(\"FinalRes.csv\", yPredictedTest)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Note:\n",
        "All the code that has been commented in the above cells is just to show that we have tried all those methods but we were not able to obtain the desired results from them."
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "DntapS6DFFkO",
        "i_AWyXwgEvvy",
        "uQb5UMfiFlo3",
        "izkQRi3XLGGN",
        "rTxTZZHILZHD",
        "H0-JZeGvMIy5",
        "uEOXEl-4PbH9"
      ],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}