Twitter Project (Applying the Classifier on Testing Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Applying the Classifier on the Testing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"twitter_xytrain.csv\", usecols=(1,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10980,), (10980,))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array(data[\"text\"])\n",
    "y = np.array(data[\"airline_sentiment\"])\n",
    "x.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10980"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_array = []\n",
    "\n",
    "for tweet in x:\n",
    "    text = \"\"\n",
    "    \n",
    "    for c in tweet:\n",
    "        if c.lower() not in string.ascii_lowercase:\n",
    "            text += \" \"\n",
    "        else:\n",
    "            text += c.lower()\n",
    "            \n",
    "    text_array.append(text)\n",
    "    \n",
    "len(text_array)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "new_text_array = []\n",
    "\n",
    "for tweet in text_array:\n",
    "    words = tweet.split()\n",
    "    new_words = []\n",
    "    \n",
    "    for word in words:\n",
    "        if len(word)>1:\n",
    "            new_words.append(word)\n",
    "            \n",
    "    words_string = \" \".join(new_words)\n",
    "    new_text_array.append(words_string)\n",
    "    \n",
    "len(new_text_array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10980, 10400)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv = CountVectorizer(max_features=10400, stop_words='english', ngram_range=(1,3), max_df=0.2)\n",
    "x_train = cv.fit_transform(text_array)\n",
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cv.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = MultinomialNB()\n",
    "clf.fit(x_train, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#clf = SVC(C=10000, gamma=0.00001)\n",
    "#clf = SVC(C=7.5, gamma=0.155)\n",
    "#clf.fit(x_train, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(\"twitter_xtest.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3660,)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test = test_data[\"text\"]\n",
    "x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3660"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_array_test = []\n",
    "for tweet in x_test:\n",
    "    text = \"\"\n",
    "    for c in tweet:\n",
    "        if c.lower() not in string.ascii_lowercase:\n",
    "            text += \" \"\n",
    "        else:\n",
    "            text += c.lower()\n",
    "    text_array_test.append(text)\n",
    "    \n",
    "len(text_array_test)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "new_text_array_test = []\n",
    "\n",
    "for tweet in text_array_test:\n",
    "    words = tweet.split()\n",
    "    new_words = []\n",
    "    \n",
    "    for word in words:\n",
    "        if len(word)>1:\n",
    "            new_words.append(word)\n",
    "            \n",
    "    words_string = \" \".join(new_words)\n",
    "    new_text_array_test.append(words_string)\n",
    "    \n",
    "len(new_text_array_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3660, 10400)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test = cv.transform(text_array_test)\n",
    "x_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = clf.predict(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3660,)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(\"twitter_predictions1.csv\", y_pred, delimiter=\",\", fmt=\"%s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}