Apply+SVM+to+Amazon+reviews+data+set+bow+tfidf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting imblearn\n",
      "  Downloading https://files.pythonhosted.org/packages/81/a7/4179e6ebfd654bd0eac0b9c06125b8b4c96a9d0a8ff9e9507eb2a26d2d7e/imblearn-0.0-py2.py3-none-any.whl\n",
      "Collecting imbalanced-learn (from imblearn)\n",
      "  Downloading https://files.pythonhosted.org/packages/80/a4/900463a3c0af082aed9c5a43f4ec317a9469710c5ef80496c9abc26ed0ca/imbalanced_learn-0.3.3-py3-none-any.whl (144kB)\n",
      "\u001b[K    100% |████████████████████████████████| 153kB 3.2MB/s ta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/site-packages (from imbalanced-learn->imblearn)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/site-packages (from imbalanced-learn->imblearn)\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/site-packages (from imbalanced-learn->imblearn)\n",
      "Installing collected packages: imbalanced-learn, imblearn\n",
      "Successfully installed imbalanced-learn-0.3.3 imblearn-0.0\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 10.0.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n",
      "Collecting gensim\n",
      "  Downloading https://files.pythonhosted.org/packages/33/33/df6cb7acdcec5677ed130f4800f67509d24dbec74a03c329fcbf6b0864f0/gensim-3.4.0-cp36-cp36m-manylinux1_x86_64.whl (22.6MB)\n",
      "\u001b[K    100% |████████████████████████████████| 22.6MB 66kB/s  eta 0:00:01    58% |██████████████████▊             | 13.2MB 42.6MB/s eta 0:00:01    97% |███████████████████████████████ | 21.9MB 43.9MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/site-packages (from gensim)\n",
      "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/site-packages (from gensim)\n",
      "Collecting smart-open>=1.2.1 (from gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/4b/69/c92661a333f733510628f28b8282698b62cdead37291c8491f3271677c02/smart_open-1.5.7.tar.gz\n",
      "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/site-packages (from gensim)\n",
      "Collecting boto>=2.32 (from smart-open>=1.2.1->gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/bd/b7/a88a67002b1185ed9a8e8a6ef15266728c2361fcb4f1d02ea331e4c7741d/boto-2.48.0-py2.py3-none-any.whl (1.4MB)\n",
      "\u001b[K    100% |████████████████████████████████| 1.4MB 1.1MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting bz2file (from smart-open>=1.2.1->gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/61/39/122222b5e85cd41c391b68a99ee296584b2a2d1d233e7ee32b4532384f2d/bz2file-0.98.tar.gz\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/site-packages (from smart-open>=1.2.1->gensim)\n",
      "Collecting boto3 (from smart-open>=1.2.1->gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/25/54/288f3d87d055440e20eb7a9dce58fdd005c91c1071883dea9033c5ddb9e4/boto3-1.7.30-py2.py3-none-any.whl (128kB)\n",
      "\u001b[K    100% |████████████████████████████████| 133kB 9.3MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting s3transfer<0.2.0,>=0.1.10 (from boto3->smart-open>=1.2.1->gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/d7/14/2a0004d487464d120c9fb85313a75cd3d71a7506955be458eebfe19a6b1d/s3transfer-0.1.13-py2.py3-none-any.whl (59kB)\n",
      "\u001b[K    100% |████████████████████████████████| 61kB 10.6MB/s ta 0:00:01\n",
      "\u001b[?25hCollecting botocore<1.11.0,>=1.10.30 (from boto3->smart-open>=1.2.1->gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/da/aa/b227500e26dbbd95bd6cda78cf784f769bbad3e74b81bfc52963b55b6363/botocore-1.10.30-py2.py3-none-any.whl (4.3MB)\n",
      "\u001b[K    100% |████████████████████████████████| 4.3MB 371kB/s eta 0:00:01\n",
      "\u001b[?25hCollecting jmespath<1.0.0,>=0.7.1 (from boto3->smart-open>=1.2.1->gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/b7/31/05c8d001f7f87f0f07289a5fc0fc3832e9a57f2dbd4d3b0fee70e0d51365/jmespath-0.9.3-py2.py3-none-any.whl\n",
      "Collecting docutils>=0.10 (from botocore<1.11.0,>=1.10.30->boto3->smart-open>=1.2.1->gensim)\n",
      "  Downloading https://files.pythonhosted.org/packages/36/fa/08e9e6e0e3cbd1d362c3bbee8d01d0aedb2155c4ac112b19ef3cae8eed8d/docutils-0.14-py3-none-any.whl (543kB)\n",
      "\u001b[K    100% |████████████████████████████████| 552kB 2.6MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/site-packages (from botocore<1.11.0,>=1.10.30->boto3->smart-open>=1.2.1->gensim)\n",
      "Building wheels for collected packages: smart-open, bz2file\n",
      "  Running setup.py bdist_wheel for smart-open ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/b1/9e/7d/bb3d3b55c597e72617140a0638c06382a5f17283881eae163e\n",
      "  Running setup.py bdist_wheel for bz2file ... \u001b[?25ldone\n",
      "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/81/75/d6/e1317bf09bf1af5a30befc2a007869fa6e1f516b8f7c591cb9\n",
      "Successfully built smart-open bz2file\n",
      "Installing collected packages: boto, bz2file, jmespath, docutils, botocore, s3transfer, boto3, smart-open, gensim\n",
      "Successfully installed boto-2.48.0 boto3-1.7.30 botocore-1.10.30 bz2file-0.98 docutils-0.14 gensim-3.4.0 jmespath-0.9.3 s3transfer-0.1.13 smart-open-1.5.7\n",
      "\u001b[33mYou are using pip version 9.0.1, however version 10.0.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install imblearn\n",
    "!pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/usr/local/lib/python3.6/site-packages/sklearn/grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from scipy.stats import uniform\n",
    "from scipy.stats import norm\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "import sqlite3\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import gensim\n",
    "\n",
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "con = sqlite3.connect('final.sqlite') # this is cleaned dataset\n",
    "final = pd.read_sql_query(\"\"\"\n",
    "SELECT Score, Text_not_included\n",
    "FROM reviews\n",
    "\"\"\", con)[:2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i, seq in enumerate(final['Text_not_included']):\n",
    "  final['Text_not_included'][i]=final['Text_not_included'][i].decode('UTF-8')\n",
    "X_train, X_test, y_train , y_test = train_test_split(final['Text_not_included'], final['Score'], test_size=0.2, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Count BoW vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count_vect = CountVectorizer(ngram_range=(1,2) )\n",
    "count_vect.fit(X_train)\n",
    "bow_train=count_vect.transform(X_train)\n",
    "bow_test=count_vect.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate TF IDF vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/feature_extraction/text.py:1089: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  if hasattr(X, 'dtype') and np.issubdtype(X.dtype, np.float):\n"
     ]
    }
   ],
   "source": [
    "tf_idf_vect=TfidfVectorizer(ngram_range=(1,2), min_df=10, dtype=float)\n",
    "tf_idf_vect.fit(X_train)\n",
    "tf_idf_train=tf_idf_vect.transform(X_train)\n",
    "tf_idf_test=tf_idf_vect.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upsampling followed by standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/utils/validation.py:444: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    }
   ],
   "source": [
    "# Upsampling minority class\n",
    "over_sampler = SMOTE(ratio='minority')\n",
    "bow_train_resampled, y_train_resampled = over_sampler.fit_sample(bow_train, y_train)\n",
    "tf_idf_train_resampled, y_train_resampled = over_sampler.fit_sample(tf_idf_train, y_train)\n",
    "\n",
    "scaler_bow=StandardScaler(with_mean=False)\n",
    "scaler_tf_idf=StandardScaler(with_mean=False)\n",
    "\n",
    "scaler_bow.fit(bow_train_resampled)\n",
    "scaler_tf_idf.fit(tf_idf_train_resampled)\n",
    "\n",
    "bow_train_scaled=scaler_bow.transform(bow_train_resampled)\n",
    "tf_idf_train_scaled=scaler_tf_idf.transform(tf_idf_train_resampled)\n",
    "\n",
    "bow_test_scaled=scaler_bow.transform(bow_test)\n",
    "tf_idf_test_scaled=scaler_tf_idf.transform(tf_idf_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification using count Bow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GridSearchCV(cv=5, error_score='raise',\n",
      "       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False),\n",
      "       fit_params={}, iid=True, n_jobs=1,\n",
      "       param_grid={'C': array([10.     , 11.11111, 12.22222, 13.33333, 14.44444, 15.55556,\n",
      "       16.66667, 17.77778, 18.88889, 20.     ]), 'gamma': array([0.001, 0.112, 0.223, 0.334, 0.445, 0.556, 0.667, 0.778, 0.889,\n",
      "       1.   ])},\n",
      "       pre_dispatch='2*n_jobs', refit=True, scoring='accuracy', verbose=0)\n",
      "RandomizedSearchCV(cv=5, error_score='raise',\n",
      "          estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False),\n",
      "          fit_params={}, iid=True, n_iter=20, n_jobs=1,\n",
      "          param_distributions={'C': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f56ed3fa6d8>, 'gamma': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f56ed3fa630>},\n",
      "          pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
      "          scoring='accuracy', verbose=0)\n"
     ]
    }
   ],
   "source": [
    "tuned_parameters = {'C': np.linspace(10.0, 20, 10, dtype=float), 'gamma' : np.linspace(0.001, 1, 10, dtype=float)}\n",
    "\n",
    "#Using GridSearchCV\n",
    "gscv = GridSearchCV(SVC(), tuned_parameters, scoring = 'accuracy', cv=5)\n",
    "\n",
    "print(gscv.fit(bow_train_scaled, y_train_resampled))\n",
    "\n",
    "tuned_parameters = {'C' : uniform(10,20), 'gamma' : uniform(0,1)}\n",
    "\n",
    "#Using RandomizedSearchCV\n",
    "rscv = RandomizedSearchCV(SVC(), tuned_parameters, scoring = 'accuracy', cv=5, n_iter=20)\n",
    "\n",
    "print(rscv.fit(bow_train_scaled, y_train_resampled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.11      0.01      0.02        87\n",
      "   positive       0.78      0.97      0.87       313\n",
      "\n",
      "avg / total       0.63      0.77      0.68       400\n",
      "\n",
      "[[  1   8]\n",
      " [ 86 305]]\n",
      "TPR = 0.9744408945686901\n",
      " TNR = 0.011494252873563218\n",
      " FPR = 0.9885057471264368\n",
      " FNR = 0.025559105431309903\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.00      0.00      0.00        87\n",
      "   positive       0.78      1.00      0.88       313\n",
      "\n",
      "avg / total       0.61      0.78      0.69       400\n",
      "\n",
      "[[  0   0]\n",
      " [ 87 313]]\n",
      "TPR = 1.0\n",
      " TNR = 0.0\n",
      " FPR = 1.0\n",
      " FNR = 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "predictions = gscv.best_estimator_.predict(bow_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "\n",
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))\n",
    "\n",
    "predictions = rscv.best_estimator_.predict(bow_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "\n",
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVC(C=10.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma=0.001, kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False)\n"
     ]
    }
   ],
   "source": [
    "print(gscv.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVC(C=11.996771023901712, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma=0.14509256003855375,\n",
      "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
      "  shrinking=True, tol=0.001, verbose=False)\n"
     ]
    }
   ],
   "source": [
    "print(rscv.best_estimator_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification using TF IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GridSearchCV(cv=5, error_score='raise',\n",
      "       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False),\n",
      "       fit_params={}, iid=True, n_jobs=1,\n",
      "       param_grid={'C': array([10.     , 10.52632, 11.05263, 11.57895, 12.10526, 12.63158,\n",
      "       13.15789, 13.68421, 14.21053, 14.73684, 15.26316, 15.78947,\n",
      "       16.31579, 16.84211, 17.36842, 17.89474, 18.42105, 18.94737,\n",
      "       19.47368, 20.     ]), 'gamma': array([0.001, 0.112, 0.223, 0.334, 0.445, 0.556, 0.667, 0.778, 0.889,\n",
      "       1.   ])},\n",
      "       pre_dispatch='2*n_jobs', refit=True, scoring='accuracy', verbose=0)\n",
      "RandomizedSearchCV(cv=5, error_score='raise',\n",
      "          estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False),\n",
      "          fit_params={}, iid=True, n_iter=15, n_jobs=1,\n",
      "          param_distributions={'C': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f56ecd12cc0>, 'gamma': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f56ecd12c18>},\n",
      "          pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
      "          scoring='accuracy', verbose=0)\n"
     ]
    }
   ],
   "source": [
    "tuned_parameters = {'C': np.linspace(10.0, 20, 20, dtype=float), 'gamma' : np.linspace(0.001, 1, 10, dtype=float)}\n",
    "#Using GridSearchCV\n",
    "gscv = GridSearchCV(SVC(), tuned_parameters, scoring = 'accuracy', cv=5)\n",
    "\n",
    "print(gscv.fit(tf_idf_train_scaled, y_train_resampled))\n",
    "\n",
    "tuned_parameters = {'C' : uniform(10,20), 'gamma' : uniform(0,1)}\n",
    "\n",
    "#Using RandomizedSearchCV\n",
    "rscv = RandomizedSearchCV(SVC(), tuned_parameters, scoring = 'accuracy', cv=5, n_iter=15)\n",
    "\n",
    "print(rscv.fit(tf_idf_train_scaled, y_train_resampled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.60      0.03      0.07        87\n",
      "   positive       0.79      0.99      0.88       313\n",
      "\n",
      "avg / total       0.75      0.79      0.70       400\n",
      "\n",
      "[[  3   2]\n",
      " [ 84 311]]\n",
      "TPR = 0.9936102236421726\n",
      " TNR = 0.034482758620689655\n",
      " FPR = 0.9655172413793104\n",
      " FNR = 0.006389776357827476\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.00      0.00      0.00        87\n",
      "   positive       0.78      1.00      0.88       313\n",
      "\n",
      "avg / total       0.61      0.78      0.69       400\n",
      "\n",
      "[[  0   0]\n",
      " [ 87 313]]\n",
      "TPR = 1.0\n",
      " TNR = 0.0\n",
      " FPR = 1.0\n",
      " FNR = 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "predictions = gscv.best_estimator_.predict(tf_idf_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "\n",
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))\n",
    "\n",
    "predictions = rscv.best_estimator_.predict(tf_idf_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "\n",
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=10.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma=0.001, kernel='rbf',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gscv.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=29.836444326925047, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma=0.03469353355992577,\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rscv.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusions\n",
    "\n",
    "Both BoW and TFIDF although provide excellent TPR, are failing at TNR. Also given large amount of time taken to train, SVMs combined with such high dimensional representations are not a good choice for text classification.<br><br>\n",
    "Somewhat decent results are given by<br>\n",
    "gamma : 0.001<br>\n",
    "10 < C < 20"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}