Final_SVM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f21c86ac",
   "metadata": {
    "id": "f21c86ac"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "#for text pre-processing\n",
    "import re, string\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import SnowballStemmer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "#nltk.download('punkt')\n",
    "#nltk.download('averaged_perceptron_tagger')\n",
    "#nltk.download('wordnet')\n",
    "#nltk.download('omw-1.4')\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score, confusion_matrix\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score\n",
    "from sklearn.calibration import calibration_curve, CalibrationDisplay\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "mdzRhEZxjdVa",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 11158,
     "status": "ok",
     "timestamp": 1671212375700,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "mdzRhEZxjdVa",
    "outputId": "b99d708f-dd80-45ae-e9c0-8f252d47eff2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NLTK Downloader\n",
      "---------------------------------------------------------------------------\n",
      "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
      "---------------------------------------------------------------------------\n",
      "Downloader> d\n",
      "\n",
      "Download which package (l=list; x=cancel)?\n",
      "  Identifier> stopwords\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "    Downloading package stopwords to /root/nltk_data...\n",
      "      Unzipping corpora/stopwords.zip.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------------------------------------------------------------------------\n",
      "    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n",
      "---------------------------------------------------------------------------\n",
      "Downloader> q\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72defce8",
   "metadata": {
    "id": "72defce8"
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stopwords_list = set(stopwords.words('english'))\n",
    "#stopwords_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a25d2018",
   "metadata": {
    "id": "a25d2018"
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_csv(\"train_data.csv\")\n",
    "train_results = pd.read_csv(\"train_results.csv\")\n",
    "test_data = pd.read_csv(\"test_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7689a2f4",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1671212382372,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "7689a2f4",
    "outputId": "5c3f4485-18be-47bc-d920-1dddebca3559"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1040323\n",
      "560175\n"
     ]
    }
   ],
   "source": [
    "print(len(train_data))\n",
    "print(len(test_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2071ced1",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 213,
     "status": "ok",
     "timestamp": 1671212382572,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "2071ced1",
    "outputId": "d2875fce-6758-4b66-8a6c-79d1af14ab84"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-74bc77f5-e028-4150-acba-5ee5306996a3\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Anyway Im getting of for a while</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>My red, Apache isn't feelin too well this morn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>@danyelljoy you should be  its great. friday w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>its 11:30pm and i dont wanna sleep; so i debat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Why does twitter eat my DM's?  Not happy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040318</th>\n",
       "      <td>1040318</td>\n",
       "      <td>getting ready 2 watch mental</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040319</th>\n",
       "      <td>1040319</td>\n",
       "      <td>Wristcutters and Half Nelson are on Sundance F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040320</th>\n",
       "      <td>1040320</td>\n",
       "      <td>@t_isfortammy Going out in Northbridge makes m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040321</th>\n",
       "      <td>1040321</td>\n",
       "      <td>@iModel_ lol.... Gorgeous...lol! U look sooo m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040322</th>\n",
       "      <td>1040322</td>\n",
       "      <td>@iamjonathancook why?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1040323 rows × 2 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-74bc77f5-e028-4150-acba-5ee5306996a3')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-74bc77f5-e028-4150-acba-5ee5306996a3 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-74bc77f5-e028-4150-acba-5ee5306996a3');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "              id                                               text\n",
       "0              0                  Anyway Im getting of for a while \n",
       "1              1  My red, Apache isn't feelin too well this morn...\n",
       "2              2  @danyelljoy you should be  its great. friday w...\n",
       "3              3  its 11:30pm and i dont wanna sleep; so i debat...\n",
       "4              4          Why does twitter eat my DM's?  Not happy \n",
       "...          ...                                                ...\n",
       "1040318  1040318                      getting ready 2 watch mental \n",
       "1040319  1040319  Wristcutters and Half Nelson are on Sundance F...\n",
       "1040320  1040320  @t_isfortammy Going out in Northbridge makes m...\n",
       "1040321  1040321  @iModel_ lol.... Gorgeous...lol! U look sooo m...\n",
       "1040322  1040322                             @iamjonathancook why? \n",
       "\n",
       "[1040323 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0c19f81",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 16,
     "status": "ok",
     "timestamp": 1671212382572,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "b0c19f81",
    "outputId": "4b443eb9-1c70-495e-a8c5-cb011807d736"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id      0\n",
       "text    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fba645e7",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 14,
     "status": "ok",
     "timestamp": 1671212382573,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "fba645e7",
    "outputId": "fd1e031d-1d18-4c3d-ee8f-2c527f30940f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-1746d4ff-cb7a-4d82-9ab8-37967f362765\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>so cut... missed out on all the TCE EXTREME ME...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Thank you very much Wasin  this is awesome lol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Squeezing in some internet time before Dustin ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>@fanoou; Oh i do have moral but i'm gonna be b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>@NSuissa   I just bought Sammy some joint/hip ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560170</th>\n",
       "      <td>560170</td>\n",
       "      <td>@meganintime So is mine and I've only got abou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560171</th>\n",
       "      <td>560171</td>\n",
       "      <td>@FairyForest Thanks cutie ) Wishing u a wonder...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560172</th>\n",
       "      <td>560172</td>\n",
       "      <td>ok know i just got home my family just drop me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560173</th>\n",
       "      <td>560173</td>\n",
       "      <td>long run status: complete. Now time for church...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560174</th>\n",
       "      <td>560174</td>\n",
       "      <td>@textualoffender Doestovesky yes,remaining 3 w...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>560175 rows × 2 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-1746d4ff-cb7a-4d82-9ab8-37967f362765')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-1746d4ff-cb7a-4d82-9ab8-37967f362765 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-1746d4ff-cb7a-4d82-9ab8-37967f362765');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "            id                                               text\n",
       "0            0  so cut... missed out on all the TCE EXTREME ME...\n",
       "1            1     Thank you very much Wasin  this is awesome lol\n",
       "2            2  Squeezing in some internet time before Dustin ...\n",
       "3            3  @fanoou; Oh i do have moral but i'm gonna be b...\n",
       "4            4  @NSuissa   I just bought Sammy some joint/hip ...\n",
       "...        ...                                                ...\n",
       "560170  560170  @meganintime So is mine and I've only got abou...\n",
       "560171  560171  @FairyForest Thanks cutie ) Wishing u a wonder...\n",
       "560172  560172  ok know i just got home my family just drop me...\n",
       "560173  560173  long run status: complete. Now time for church...\n",
       "560174  560174  @textualoffender Doestovesky yes,remaining 3 w...\n",
       "\n",
       "[560175 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a28d148",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 12,
     "status": "ok",
     "timestamp": 1671212382573,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "1a28d148",
    "outputId": "5b7739ea-371a-4a0d-ee29-7ac4880eb2d6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0         so cut... missed out on all the TCE EXTREME ME...\n",
       "1            Thank you very much Wasin  this is awesome lol\n",
       "2         Squeezing in some internet time before Dustin ...\n",
       "3         @fanoou; Oh i do have moral but i'm gonna be b...\n",
       "4         @NSuissa   I just bought Sammy some joint/hip ...\n",
       "                                ...                        \n",
       "560170    @meganintime So is mine and I've only got abou...\n",
       "560171    @FairyForest Thanks cutie ) Wishing u a wonder...\n",
       "560172    ok know i just got home my family just drop me...\n",
       "560173    long run status: complete. Now time for church...\n",
       "560174    @textualoffender Doestovesky yes,remaining 3 w...\n",
       "Name: text, Length: 560175, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e165619d",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 341,
     "status": "ok",
     "timestamp": 1671212382908,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "e165619d",
    "outputId": "8436dcff-f5c7-4ef2-c2e0-390c3857ba38"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-4cd2c7ee-c14a-461d-b6e1-cc35843999b9\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040318</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040319</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040320</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040321</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040322</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1040323 rows × 1 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4cd2c7ee-c14a-461d-b6e1-cc35843999b9')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-4cd2c7ee-c14a-461d-b6e1-cc35843999b9 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-4cd2c7ee-c14a-461d-b6e1-cc35843999b9');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "         target\n",
       "0             2\n",
       "1             0\n",
       "2             2\n",
       "3             2\n",
       "4             0\n",
       "...         ...\n",
       "1040318       2\n",
       "1040319       0\n",
       "1040320       0\n",
       "1040321       2\n",
       "1040322       0\n",
       "\n",
       "[1040323 rows x 1 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_train_label = train_results['target'].replace(['negative', 'neutral', 'positive'] , [0,1,2])\n",
    "train_target = pd.DataFrame(full_train_label)\n",
    "train_target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaca144f",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 214,
     "status": "ok",
     "timestamp": 1671212383115,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "eaca144f",
    "outputId": "c90fbd86-e12a-4606-cadf-b645d3372e28"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    520436\n",
       "0    519803\n",
       "1        84\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_train_label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f41823c8",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1671212383116,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "f41823c8",
    "outputId": "54c047e3-3c44-40e2-db8f-44763df18ecc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "positive    520436\n",
       "negative    519803\n",
       "neutral         84\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_results['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42b39dd8",
   "metadata": {
    "id": "42b39dd8"
   },
   "outputs": [],
   "source": [
    "#convert to lowercase, strip and remove punctuations\n",
    "def preprocess(text):\n",
    "    text = text.lower() \n",
    "    text=  text.strip()  \n",
    "    text=  re.compile('<.*?>').sub('', text) \n",
    "    text = re.compile('[%s]' % re.escape(string.punctuation)).sub(' ', text)  \n",
    "    text = re.sub('\\s+', ' ', text)  \n",
    "    text = re.sub(r'\\[[0-9]*\\]',' ',text) \n",
    "    text= re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n",
    "    text = re.sub(r'\\d',' ',text) \n",
    "    text = re.sub(r'\\s+',' ',text) \n",
    "    text = re.sub(r'https?://\\S+|www\\.\\S+', '', text) # Removes url\n",
    "    text = re.compile(r'<[^>]+>').sub('', text) #Removes HTML tags:\n",
    "    text = re.sub(r\"\\s+[a-zA-Z]\\s+\", ' ', text) # Single character removal\n",
    "    text = re.sub(r'\\s+', ' ', text) # Remove multiple spaces\n",
    "    return text\n",
    "\n",
    "# STOPWORD REMOVAL\n",
    "def stopword(string):\n",
    "    a= [i for i in string.split() if i not in stopwords.words('english')]\n",
    "    return ' '.join(a)\n",
    "\n",
    "#LEMMATIZATION\n",
    "# Initialize the lemmatizer\n",
    "wl = WordNetLemmatizer()\n",
    " \n",
    "# This is a helper function to map NTLK position tags\n",
    "def get_wordnet_pos(tag):\n",
    "    if tag.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif tag.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif tag.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif tag.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN\n",
    "    \n",
    "# Tokenize the sentence\n",
    "def lemmatizer(string):\n",
    "    word_pos_tags = nltk.pos_tag(word_tokenize(string)) # Get position tags\n",
    "    a=[wl.lemmatize(tag[0], get_wordnet_pos(tag[1])) for idx, tag in enumerate(word_pos_tags)] # Map the position tag and lemmatize the word/token\n",
    "    return \" \".join(a)\n",
    "\n",
    "def finalpreprocess(string):\n",
    "    return lemmatizer(stopword(preprocess(string)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83820909",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 34152,
     "status": "ok",
     "timestamp": 1671212417264,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "83820909",
    "outputId": "013f66c1-be4f-4cbb-e1a1-4a97506e8564"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-e6709606-327c-412b-b675-566f277c2cd0\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Anyway Im getting of for a while</td>\n",
       "      <td>anyway im getting of for while</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>My red, Apache isn't feelin too well this morn...</td>\n",
       "      <td>my red apache isn feelin too well this morning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>@danyelljoy you should be  its great. friday w...</td>\n",
       "      <td>danyelljoy you should be its great friday will...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>its 11:30pm and i dont wanna sleep; so i debat...</td>\n",
       "      <td>its pm and dont wanna sleep so debated with my...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Why does twitter eat my DM's?  Not happy</td>\n",
       "      <td>why does twitter eat my dm not happy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040318</th>\n",
       "      <td>1040318</td>\n",
       "      <td>getting ready 2 watch mental</td>\n",
       "      <td>getting ready watch mental</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040319</th>\n",
       "      <td>1040319</td>\n",
       "      <td>Wristcutters and Half Nelson are on Sundance F...</td>\n",
       "      <td>wristcutters and half nelson are on sundance f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040320</th>\n",
       "      <td>1040320</td>\n",
       "      <td>@t_isfortammy Going out in Northbridge makes m...</td>\n",
       "      <td>t isfortammy going out in northbridge makes me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040321</th>\n",
       "      <td>1040321</td>\n",
       "      <td>@iModel_ lol.... Gorgeous...lol! U look sooo m...</td>\n",
       "      <td>imodel lol gorgeous lol look sooo much like th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040322</th>\n",
       "      <td>1040322</td>\n",
       "      <td>@iamjonathancook why?</td>\n",
       "      <td>iamjonathancook why</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1040323 rows × 3 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e6709606-327c-412b-b675-566f277c2cd0')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-e6709606-327c-412b-b675-566f277c2cd0 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-e6709606-327c-412b-b675-566f277c2cd0');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "              id                                               text  \\\n",
       "0              0                  Anyway Im getting of for a while    \n",
       "1              1  My red, Apache isn't feelin too well this morn...   \n",
       "2              2  @danyelljoy you should be  its great. friday w...   \n",
       "3              3  its 11:30pm and i dont wanna sleep; so i debat...   \n",
       "4              4          Why does twitter eat my DM's?  Not happy    \n",
       "...          ...                                                ...   \n",
       "1040318  1040318                      getting ready 2 watch mental    \n",
       "1040319  1040319  Wristcutters and Half Nelson are on Sundance F...   \n",
       "1040320  1040320  @t_isfortammy Going out in Northbridge makes m...   \n",
       "1040321  1040321  @iModel_ lol.... Gorgeous...lol! U look sooo m...   \n",
       "1040322  1040322                             @iamjonathancook why?    \n",
       "\n",
       "                                                clean_text  \n",
       "0                           anyway im getting of for while  \n",
       "1        my red apache isn feelin too well this morning...  \n",
       "2        danyelljoy you should be its great friday will...  \n",
       "3        its pm and dont wanna sleep so debated with my...  \n",
       "4                     why does twitter eat my dm not happy  \n",
       "...                                                    ...  \n",
       "1040318                         getting ready watch mental  \n",
       "1040319  wristcutters and half nelson are on sundance f...  \n",
       "1040320  t isfortammy going out in northbridge makes me...  \n",
       "1040321  imodel lol gorgeous lol look sooo much like th...  \n",
       "1040322                                iamjonathancook why  \n",
       "\n",
       "[1040323 rows x 3 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data['clean_text']=train_data['text'].map(lambda s:preprocess(s)) \n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33adcc08",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 13,
     "status": "ok",
     "timestamp": 1671212417390,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "33adcc08",
    "outputId": "378a38ed-d98a-4c79-a8ae-c580ccf40d2c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-d41666cd-e00d-4577-a704-fbfad3be71ad\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>anyway im getting of for while</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>my red apache isn feelin too well this morning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>danyelljoy you should be its great friday will...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>its pm and dont wanna sleep so debated with my...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>why does twitter eat my dm not happy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040318</th>\n",
       "      <td>getting ready watch mental</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040319</th>\n",
       "      <td>wristcutters and half nelson are on sundance f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040320</th>\n",
       "      <td>t isfortammy going out in northbridge makes me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040321</th>\n",
       "      <td>imodel lol gorgeous lol look sooo much like th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040322</th>\n",
       "      <td>iamjonathancook why</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1040323 rows × 1 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d41666cd-e00d-4577-a704-fbfad3be71ad')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-d41666cd-e00d-4577-a704-fbfad3be71ad button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-d41666cd-e00d-4577-a704-fbfad3be71ad');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                                                clean_text\n",
       "0                           anyway im getting of for while\n",
       "1        my red apache isn feelin too well this morning...\n",
       "2        danyelljoy you should be its great friday will...\n",
       "3        its pm and dont wanna sleep so debated with my...\n",
       "4                     why does twitter eat my dm not happy\n",
       "...                                                    ...\n",
       "1040318                         getting ready watch mental\n",
       "1040319  wristcutters and half nelson are on sundance f...\n",
       "1040320  t isfortammy going out in northbridge makes me...\n",
       "1040321  imodel lol gorgeous lol look sooo much like th...\n",
       "1040322                                iamjonathancook why\n",
       "\n",
       "[1040323 rows x 1 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_clean = pd.DataFrame(train_data['clean_text'])\n",
    "df_train_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "967612f6",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 12,
     "status": "ok",
     "timestamp": 1671212417392,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "967612f6",
    "outputId": "42eec71a-9f5b-4b94-9560-98703dcff9af"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-e3f6fc15-527b-4794-8d5e-9112b4c60e0f\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clean_text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>anyway im getting of for while</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>my red apache isn feelin too well this morning...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>danyelljoy you should be its great friday will...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>its pm and dont wanna sleep so debated with my...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>why does twitter eat my dm not happy</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040318</th>\n",
       "      <td>getting ready watch mental</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040319</th>\n",
       "      <td>wristcutters and half nelson are on sundance f...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040320</th>\n",
       "      <td>t isfortammy going out in northbridge makes me...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040321</th>\n",
       "      <td>imodel lol gorgeous lol look sooo much like th...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1040322</th>\n",
       "      <td>iamjonathancook why</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1040323 rows × 2 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e3f6fc15-527b-4794-8d5e-9112b4c60e0f')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-e3f6fc15-527b-4794-8d5e-9112b4c60e0f button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-e3f6fc15-527b-4794-8d5e-9112b4c60e0f');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                                                clean_text  target\n",
       "0                           anyway im getting of for while       2\n",
       "1        my red apache isn feelin too well this morning...       0\n",
       "2        danyelljoy you should be its great friday will...       2\n",
       "3        its pm and dont wanna sleep so debated with my...       2\n",
       "4                     why does twitter eat my dm not happy       0\n",
       "...                                                    ...     ...\n",
       "1040318                         getting ready watch mental       2\n",
       "1040319  wristcutters and half nelson are on sundance f...       0\n",
       "1040320  t isfortammy going out in northbridge makes me...       0\n",
       "1040321  imodel lol gorgeous lol look sooo much like th...       2\n",
       "1040322                                iamjonathancook why       0\n",
       "\n",
       "[1040323 rows x 2 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_clean = pd.concat([df_train_clean, train_target], axis=1)\n",
    "df_train_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d7a5390",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 18178,
     "status": "ok",
     "timestamp": 1671212435656,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "8d7a5390",
    "outputId": "ff9b69ba-feff-4c2d-d1ed-d567d9eceba7"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-65f10b4a-9dfe-4649-bc2b-b2316ef52757\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>so cut... missed out on all the TCE EXTREME ME...</td>\n",
       "      <td>so cut missed out on all the tce extreme merch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Thank you very much Wasin  this is awesome lol</td>\n",
       "      <td>thank you very much wasin this is awesome lol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Squeezing in some internet time before Dustin ...</td>\n",
       "      <td>squeezing in some internet time before dustin ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>@fanoou; Oh i do have moral but i'm gonna be b...</td>\n",
       "      <td>fanoou oh do have moral but m gonna be bigger ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>@NSuissa   I just bought Sammy some joint/hip ...</td>\n",
       "      <td>nsuissa just bought sammy some joint hip suppl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560170</th>\n",
       "      <td>560170</td>\n",
       "      <td>@meganintime So is mine and I've only got abou...</td>\n",
       "      <td>meganintime so is mine and ve only got about w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560171</th>\n",
       "      <td>560171</td>\n",
       "      <td>@FairyForest Thanks cutie ) Wishing u a wonder...</td>\n",
       "      <td>fairyforest thanks cutie wishing a wonderful m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560172</th>\n",
       "      <td>560172</td>\n",
       "      <td>ok know i just got home my family just drop me...</td>\n",
       "      <td>ok know just got home my family just drop me o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560173</th>\n",
       "      <td>560173</td>\n",
       "      <td>long run status: complete. Now time for church...</td>\n",
       "      <td>long run status complete now time for church m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560174</th>\n",
       "      <td>560174</td>\n",
       "      <td>@textualoffender Doestovesky yes,remaining 3 w...</td>\n",
       "      <td>textualoffender doestovesky yes remaining will...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>560175 rows × 3 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-65f10b4a-9dfe-4649-bc2b-b2316ef52757')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-65f10b4a-9dfe-4649-bc2b-b2316ef52757 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-65f10b4a-9dfe-4649-bc2b-b2316ef52757');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "            id                                               text  \\\n",
       "0            0  so cut... missed out on all the TCE EXTREME ME...   \n",
       "1            1     Thank you very much Wasin  this is awesome lol   \n",
       "2            2  Squeezing in some internet time before Dustin ...   \n",
       "3            3  @fanoou; Oh i do have moral but i'm gonna be b...   \n",
       "4            4  @NSuissa   I just bought Sammy some joint/hip ...   \n",
       "...        ...                                                ...   \n",
       "560170  560170  @meganintime So is mine and I've only got abou...   \n",
       "560171  560171  @FairyForest Thanks cutie ) Wishing u a wonder...   \n",
       "560172  560172  ok know i just got home my family just drop me...   \n",
       "560173  560173  long run status: complete. Now time for church...   \n",
       "560174  560174  @textualoffender Doestovesky yes,remaining 3 w...   \n",
       "\n",
       "                                               clean_text  \n",
       "0       so cut missed out on all the tce extreme merch...  \n",
       "1           thank you very much wasin this is awesome lol  \n",
       "2       squeezing in some internet time before dustin ...  \n",
       "3       fanoou oh do have moral but m gonna be bigger ...  \n",
       "4       nsuissa just bought sammy some joint hip suppl...  \n",
       "...                                                   ...  \n",
       "560170  meganintime so is mine and ve only got about w...  \n",
       "560171  fairyforest thanks cutie wishing a wonderful m...  \n",
       "560172  ok know just got home my family just drop me o...  \n",
       "560173  long run status complete now time for church m...  \n",
       "560174  textualoffender doestovesky yes remaining will...  \n",
       "\n",
       "[560175 rows x 3 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data['clean_text']=test_data['text'].map(lambda s:preprocess(s)) \n",
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e947083b",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 424
    },
    "executionInfo": {
     "elapsed": 13,
     "status": "ok",
     "timestamp": 1671212435657,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "e947083b",
    "outputId": "0e5838db-1961-4d54-93ad-ce77864c5d0a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-21136d9e-640f-4099-8e54-31c5ea8ed8eb\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clean_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>so cut missed out on all the tce extreme merch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>thank you very much wasin this is awesome lol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>squeezing in some internet time before dustin ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fanoou oh do have moral but m gonna be bigger ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>nsuissa just bought sammy some joint hip suppl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560170</th>\n",
       "      <td>meganintime so is mine and ve only got about w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560171</th>\n",
       "      <td>fairyforest thanks cutie wishing a wonderful m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560172</th>\n",
       "      <td>ok know just got home my family just drop me o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560173</th>\n",
       "      <td>long run status complete now time for church m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560174</th>\n",
       "      <td>textualoffender doestovesky yes remaining will...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>560175 rows × 1 columns</p>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-21136d9e-640f-4099-8e54-31c5ea8ed8eb')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-21136d9e-640f-4099-8e54-31c5ea8ed8eb button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-21136d9e-640f-4099-8e54-31c5ea8ed8eb');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "                                               clean_text\n",
       "0       so cut missed out on all the tce extreme merch...\n",
       "1           thank you very much wasin this is awesome lol\n",
       "2       squeezing in some internet time before dustin ...\n",
       "3       fanoou oh do have moral but m gonna be bigger ...\n",
       "4       nsuissa just bought sammy some joint hip suppl...\n",
       "...                                                   ...\n",
       "560170  meganintime so is mine and ve only got about w...\n",
       "560171  fairyforest thanks cutie wishing a wonderful m...\n",
       "560172  ok know just got home my family just drop me o...\n",
       "560173  long run status complete now time for church m...\n",
       "560174  textualoffender doestovesky yes remaining will...\n",
       "\n",
       "[560175 rows x 1 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_clean = pd.DataFrame(test_data['clean_text'])\n",
    "df_test_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04f7ccac",
   "metadata": {
    "id": "04f7ccac"
   },
   "outputs": [],
   "source": [
    "X_train = df_train_clean['clean_text']\n",
    "X_test = df_test_clean['clean_text']\n",
    "y_train = df_train_clean['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2869aba4",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1671212435658,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "2869aba4",
    "outputId": "3d148c8f-b61f-4ead-d3ee-accff4b015a4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1040323 1040323 560175\n"
     ]
    }
   ],
   "source": [
    "print(len(X_train), len(y_train), len(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4dc6647",
   "metadata": {
    "id": "b4dc6647"
   },
   "outputs": [],
   "source": [
    "# vectorization using Bag-of-Words (with Tf-Idf ) and Word2Vec\n",
    "#Tf-Idf\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer(max_features=200000, ngram_range=(1,2))\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train) \n",
    "X_test_vectors_tfidf = tfidf_vectorizer.transform(X_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "916d26f4",
   "metadata": {
    "id": "-or2u2EuMuGv"
   },
   "source": [
    "## SVM String Kernels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "191284d4",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9436,
     "status": "ok",
     "timestamp": 1671222408256,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "191284d4",
    "outputId": "8ea48779-4c9d-4410-fb18-3daefd90822c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Collecting string-kernels\n",
      "  Downloading string_kernels-1.1.2-py3-none-any.whl (5.1 kB)\n",
      "Installing collected packages: string-kernels\n",
      "Successfully installed string-kernels-1.1.2\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "!pip install string-kernels\n",
    "from stringkernels.kernels import string_kernel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf382667",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 929953,
     "status": "ok",
     "timestamp": 1671231414994,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "cf382667",
    "outputId": "dac84bda-f889-4d85-fb62-21bf0584bf11"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(kernel=functools.partial(<function string_kernel_multithread at 0x7fc9c11cdf70>, normalize=True, n_jobs=None))"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scv_tfidf = SVC(kernel=string_kernel())\n",
    "scv_tfidf.fit(X_train[:10000], y_train[:10000])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99cd2687",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 977984,
     "status": "ok",
     "timestamp": 1671232695438,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "99cd2687",
    "outputId": "623e757b-3d49-4473-b8f9-b77c5a8ed0da"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5081"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scv_tfidf.score(X_train[:10000], y_train[:10000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fb44b6be",
   "metadata": {
    "executionInfo": {
     "elapsed": 116,
     "status": "ok",
     "timestamp": 1671384620373,
     "user": {
      "displayName": "Vijeet Gahlawat",
      "userId": "18130092959823721624"
     },
     "user_tz": 300
    },
    "id": "fb44b6be"
   },
   "outputs": [],
   "source": [
    "#SVC_predict = scv_tfidf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cxyljg2N9ln",
   "metadata": {
    "id": "6cxyljg2N9ln"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}