Apply_SVM_to_Amazon_reviews_data_set_avg_w2vec_[M].ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 402
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8471,
     "status": "ok",
     "timestamp": 1527847606508,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "sG52a32YN42X",
    "outputId": "a423c5e4-4d79-4a95-ea3e-5d8b6f129a9c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: PyDrive in /usr/local/lib/python3.6/dist-packages (1.3.1)\r\n",
      "Requirement already satisfied: oauth2client>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (4.1.2)\r\n",
      "Requirement already satisfied: google-api-python-client>=1.2 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (1.6.7)\n",
      "Requirement already satisfied: PyYAML>=3.0 in /usr/local/lib/python3.6/dist-packages (from PyDrive) (3.12)\n",
      "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.4.2)\n",
      "Requirement already satisfied: httplib2>=0.9.1 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.11.3)\n",
      "Requirement already satisfied: six>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (1.11.0)\n",
      "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (3.4.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client>=4.0.0->PyDrive) (0.2.1)\n",
      "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->PyDrive) (3.0.0)\n",
      "title: Apply SVM to Amazon reviews data set [M].ipynb, id: 1ElWunFgWZPb1Iq6w4ZMmqoBSVuPt0QQ4\n",
      "title: Apply Logistic regression to Amazon reviews data set. [M].ipynb, id: 1Es1wP2edJ0vrKasA5wYJEO-zeZvrqeBI\n",
      "title: Apply Naive Bayes to Amazon reviews [M].ipynb, id: 1qPxAZeYQUM-eqaKnOSM5ubK2IPIVmdyo\n",
      "title: clean_final.sqlite, id: 1T0HyUqaVFyD8HfIQEM6WN8jF8SpEOsAo\n",
      "title: KNN on Credit Card fraud detection.ipynb, id: 1CkA-RBfXqvubKkQrpnjbYUKVsC7VHlTl\n",
      "title: creditcard.csv, id: 1VpeqlS0lPVrlzlMIqvQTzc3Pno_Cj4SV\n",
      "title: creditcard.csv, id: 1bnZktEq3N_5wjoCH85oIXHxNwXUW_jx-\n",
      "title: Untitled, id: 1K0wwkizWx3WO8d-zw-YewWIUrPdINYmp\n",
      "title: final.sqlite, id: 1OzLc3k6-T55I-XRMq47ERyCbQbVw4caF\n",
      "title: HeavyComputations.ipynb, id: 1aBORe3gqeFY-iNhzMtr-TIkzEyEvFxcG\n",
      "title: LogisticRegression.ipynb, id: 1WcVTklMZBMu9VTCIWeupOK0r2aYbHk8p\n"
     ]
    }
   ],
   "source": [
    "!pip install PyDrive\n",
    "from pydrive.auth import GoogleAuth\n",
    "from pydrive.drive import GoogleDrive\n",
    "from google.colab import auth\n",
    "from oauth2client.client import GoogleCredentials\n",
    "\n",
    "# 1. Authenticate and create the PyDrive client.\n",
    "auth.authenticate_user()\n",
    "gauth = GoogleAuth()\n",
    "gauth.credentials = GoogleCredentials.get_application_default()\n",
    "drive = GoogleDrive(gauth)\n",
    "\n",
    "file_list = drive.ListFile({'q': \"'1pbLvjcsi6UtFm3sPciCJGbCG4NK3uyuS' in parents and trashed=false\"}).GetList()\n",
    "for file1 in file_list:\n",
    "  print('title: %s, id: %s' % (file1['title'], file1['id']))\n",
    "  \n",
    "sql = drive.CreateFile({'id': '1OzLc3k6-T55I-XRMq47ERyCbQbVw4caF'})\n",
    "sql.GetContentFile('final.sqlite')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 104
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3703,
     "status": "ok",
     "timestamp": 1527847620093,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "w8RRRup_PHSL",
    "outputId": "08224ace-e621-4143-9736-17351284f2d1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: imblearn in /usr/local/lib/python3.6/dist-packages (0.0)\r\n",
      "Requirement already satisfied: imbalanced-learn in /usr/local/lib/python3.6/dist-packages (from imblearn) (0.3.3)\r\n",
      "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn) (0.19.1)\r\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn) (1.14.3)\r\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from imbalanced-learn->imblearn) (0.19.1)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install imblearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 350
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2971,
     "status": "ok",
     "timestamp": 1527847626607,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "Qq_a91HhV2CQ",
    "outputId": "e17cbe9a-0600-4b2b-916b-130693cc4e92"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gensim in /usr/local/lib/python3.6/dist-packages (3.4.0)\n",
      "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.14.3)\n",
      "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.5.7)\n",
      "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim) (1.11.0)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim) (0.19.1)\n",
      "Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.48.0)\n",
      "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (1.7.30)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (2.18.4)\n",
      "Requirement already satisfied: bz2file in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim) (0.98)\n",
      "Requirement already satisfied: botocore<1.11.0,>=1.10.30 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (1.10.30)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.9.3)\n",
      "Requirement already satisfied: s3transfer<0.2.0,>=0.1.10 in /usr/local/lib/python3.6/dist-packages (from boto3->smart-open>=1.2.1->gensim) (0.1.13)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2018.4.16)\n",
      "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (1.22)\n",
      "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (2.6)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->smart-open>=1.2.1->gensim) (3.0.4)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.11.0,>=1.10.30->boto3->smart-open>=1.2.1->gensim) (2.5.3)\n",
      "Requirement already satisfied: docutils>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.11.0,>=1.10.30->boto3->smart-open>=1.2.1->gensim) (0.14)\n"
     ]
    }
   ],
   "source": [
    "!pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "UOzIUbMIPeV0"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.grid_search import RandomizedSearchCV\n",
    "from scipy.stats import uniform\n",
    "from scipy.stats import norm\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "\n",
    "import sqlite3\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import gensim\n",
    "\n",
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "ZkTr9AVEVp74"
   },
   "outputs": [],
   "source": [
    "con = sqlite3.connect('final.sqlite') # this is cleaned dataset\n",
    "final = pd.read_sql_query(\"\"\"\n",
    "SELECT Score, Text_not_included\n",
    "FROM reviews\n",
    "\"\"\", con)[:2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "zE0vqi-1WOFu"
   },
   "outputs": [],
   "source": [
    "for i, seq in enumerate(final['Text_not_included']):\n",
    "  final['Text_not_included'][i]=final['Text_not_included'][i].decode('UTF-8')\n",
    "X_train, X_test, y_train , y_test = train_test_split(final['Text_not_included'], final['Score'], test_size=0.2, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sJZoFEWeWkx_"
   },
   "source": [
    "## Generate Count BoW vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1646,
     "status": "ok",
     "timestamp": 1527848250170,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "RHQiX0sBWhNL",
    "outputId": "0d8a397e-8e56-4972-fd07-e826c36838bc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 2), preprocessor=None, stop_words=None,\n",
       "        strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 41,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_vect = CountVectorizer(ngram_range=(1,2) )\n",
    "count_vect.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "IPvjlSpeXg2u"
   },
   "outputs": [],
   "source": [
    "bow_train=count_vect.transform(X_train)\n",
    "bow_test=count_vect.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "E7TPCNH8X3Uc"
   },
   "source": [
    "## Generate TF IDF vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "7ACXa7HnX1RF"
   },
   "outputs": [],
   "source": [
    "tf_idf_vect=TfidfVectorizer(ngram_range=(1,2), min_df=10, dtype=float)\n",
    "tf_idf_vect.fit(X_train)\n",
    "tf_idf_train=tf_idf_vect.transform(X_train)\n",
    "tf_idf_test=tf_idf_vect.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "v3rmZpYIZG3i"
   },
   "source": [
    "## Generate average Word2Vec representations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "d1vgXjLbYwTd"
   },
   "outputs": [],
   "source": [
    "sentences=[]\n",
    "for review in X_train:\n",
    "  sentence=[]\n",
    "  for word in review.split():\n",
    "    sentence.append(word)\n",
    "  sentences.append(sentence)\n",
    "\n",
    "w2vec_model=gensim.models.word2vec.Word2Vec(sentences, min_count=10)\n",
    "\n",
    "avg_w2vec_train=np.zeros(shape=(len(X_train), 100), dtype=float)\n",
    "\n",
    "for i, sentence in enumerate(sentences):\n",
    "  for word in sentence:\n",
    "    try:\n",
    "      avg_w2vec_train[i]+=w2vec_model.wv[word]\n",
    "    \n",
    "    except KeyError:\n",
    "      pass\n",
    "  \n",
    "  avg_w2vec_train[i]/=len(sentence)\n",
    "\n",
    "sentences=[]\n",
    "for review in X_test:\n",
    "  sentence=[]\n",
    "  for word in review.split():\n",
    "    sentence.append(word)\n",
    "  sentences.append(sentence)\n",
    "\n",
    "avg_w2vec_test=np.zeros(shape=(len(X_test), 100), dtype=float)\n",
    "\n",
    "for i, sentence in enumerate(sentences):\n",
    "  for word in sentence:\n",
    "    try:\n",
    "      avg_w2vec_test[i]+=w2vec_model.wv[word]\n",
    "    \n",
    "    except KeyError:\n",
    "      pass\n",
    "  \n",
    "  avg_w2vec_test[i]/=len(sentence)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "89MjDcEkvWM1"
   },
   "source": [
    "## Generate TF IDF weighted Word2Vec representations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "zZOQUIizeon1"
   },
   "outputs": [],
   "source": [
    "sentences=[]\n",
    "for review in X_train:\n",
    "  sentence=[]\n",
    "  for word in review.split():\n",
    "    sentence.append(word)\n",
    "  sentences.append(sentence)\n",
    "  \n",
    "tf_idf_w2vec_train=np.zeros((len(X_train), 100), dtype=float)\n",
    "feat=tf_idf_vect.get_feature_names()\n",
    "for i, sentence in enumerate(sentences):\n",
    "  tf_idf_sum=0\n",
    "  for word in sentence:\n",
    "    try:\n",
    "      tf_idf_w2vec_train[i]+=w2vec_model.wv[word]*tf_idf_train[i, feat.index(word)]\n",
    "      tf_idf_sum+=tf_idf_train[i, feat.index(word)]  \n",
    "    except KeyError:\n",
    "      pass\n",
    "    except ValueError:\n",
    "      pass\n",
    "  tf_idf_w2vec_train[i]/=tf_idf_sum\n",
    "  \n",
    "sentences=[]\n",
    "for review in X_test:\n",
    "  sentence=[]\n",
    "  for word in review.split():\n",
    "    sentence.append(word)\n",
    "  sentences.append(sentence)\n",
    "  \n",
    "tf_idf_w2vec_test=np.zeros((len(X_test), 100), dtype=float)\n",
    "\n",
    "for i, sentence in enumerate(sentences):\n",
    "  tf_idf_sum=0\n",
    "  for word in sentence:\n",
    "    try:\n",
    "      tf_idf_w2vec_test[i]+=w2vec_model.wv[word]*tf_idf_test[i, feat.index(word)]\n",
    "      tf_idf_sum+=tf_idf_test[i, feat.index(word)]  \n",
    "    except KeyError:\n",
    "      pass\n",
    "    except ValueError:\n",
    "      pass\n",
    "  tf_idf_w2vec_test[i]/=tf_idf_sum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p_2G_16q2fuP"
   },
   "source": [
    "## Upsampling followed by standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 72
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2961,
     "status": "ok",
     "timestamp": 1527848276721,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "JLiyqjfP0dm6",
    "outputId": "ef10ad3b-962a-4e8e-9c10-fc56ae78fa2e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    }
   ],
   "source": [
    "# Upsampling minority class\n",
    "over_sampler = SMOTE(ratio='minority')\n",
    "bow_train_resampled, y_train_resampled = over_sampler.fit_sample(bow_train, y_train)\n",
    "tf_idf_train_resampled, y_train_resampled = over_sampler.fit_sample(tf_idf_train, y_train)\n",
    "avg_w2vec_train_resampled, y_train_resampled = over_sampler.fit_sample(avg_w2vec_train, y_train)\n",
    "#tf_idf_w2vec_train_resampled, y_train_resampled = over_sampler.fit_sample(tf_idf_w2vec_train, y_train)\n",
    "\n",
    "scaler_bow=StandardScaler(with_mean=False)\n",
    "scaler_tf_idf=StandardScaler(with_mean=False)\n",
    "scaler_w2vec=StandardScaler()\n",
    "#scaler_tf_w2vec=StandardScaler()\n",
    "\n",
    "scaler_bow.fit(bow_train_resampled)\n",
    "scaler_tf_idf.fit(tf_idf_train_resampled)\n",
    "scaler_w2vec.fit(avg_w2vec_train_resampled)\n",
    "#scaler_tf_w2vec.fit(tf_idf_w2vec_train_resampled)\n",
    "\n",
    "bow_train_scaled=scaler_bow.transform(bow_train_resampled)\n",
    "tf_idf_train_scaled=scaler_tf_idf.transform(tf_idf_train_resampled)\n",
    "avg_w2vec_train_scaled=scaler_w2vec.transform(avg_w2vec_train_resampled)\n",
    "#tf_idf_w2vec_train_scaled=scaler_tf_w2vec.transform(tf_idf_w2vec_train_resampled)\n",
    "\n",
    "bow_test_scaled=scaler_bow.transform(bow_test)\n",
    "tf_idf_test_scaled=scaler_tf_idf.transform(tf_idf_test)\n",
    "avg_w2vec_test_scaled=scaler_w2vec.transform(avg_w2vec_test)\n",
    "#tf_idf_w2vec_test_scaled=scaler_tf_w2vec.transform(tf_idf_w2vec_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "z2NOaHCQh68o"
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jF4OCqzvpW63"
   },
   "source": [
    "## Classification using avg_word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 419
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2165994,
     "status": "ok",
     "timestamp": 1527857822613,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "1e1SXF4Toiht",
    "outputId": "b1567198-a18b-4208-bb69-4e9350f9af93"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GridSearchCV(cv=5, error_score='raise',\n",
      "       estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False),\n",
      "       fit_params={}, iid=True, n_jobs=1,\n",
      "       param_grid={'C': array([10.     , 10.26316, 10.52632, 10.78947, 11.05263, 11.31579,\n",
      "       11.57895, 11.84211, 12.10526, 12.36842, 12.63158, 12.89474,\n",
      "       13.15789, 13.42105, 13.68421, 13.94737, 14.21053, 14.47368,\n",
      "       14.73684, 15.     ]), 'gamma': array([0.001  , 0.05358, 0.10616, 0.15874, 0.21132, 0.26389, 0.31647,\n",
      "       0.36905, 0.42163, 0.47421, 0.52679, 0.57937, 0.63195, 0.68453,\n",
      "       0.73711, 0.78968, 0.84226, 0.89484, 0.94742, 1.     ])},\n",
      "       pre_dispatch='2*n_jobs', refit=True, scoring='accuracy', verbose=0)\n",
      "RandomizedSearchCV(cv=5, error_score='raise',\n",
      "          estimator=SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False),\n",
      "          fit_params={}, iid=True, n_iter=25, n_jobs=1,\n",
      "          param_distributions={'C': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7fe2d92a7eb8>, 'gamma': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7fe2d92a7fd0>},\n",
      "          pre_dispatch='2*n_jobs', random_state=None, refit=True,\n",
      "          scoring='accuracy', verbose=0)\n"
     ]
    }
   ],
   "source": [
    "tuned_parameters = {'C': np.linspace(10, 15, 20, dtype=float),'gamma' : np.linspace(0.001, 1, 20, dtype=float)}\n",
    "\n",
    "#Using GridSearchCV\n",
    "gscv = GridSearchCV(SVC(), tuned_parameters, scoring = 'accuracy', cv=5)\n",
    "\n",
    "print(gscv.fit(avg_w2vec_train_scaled, y_train_resampled))\n",
    "\n",
    "tuned_parameters = {'C' : uniform(10,15), 'gamma' : uniform(0, 1)}\n",
    "\n",
    "#Using RandomizedSearchCV\n",
    "rscv = RandomizedSearchCV(SVC(), tuned_parameters, scoring = 'accuracy', cv=5, n_iter=25)\n",
    "\n",
    "print(rscv.fit(avg_w2vec_train_scaled, y_train_resampled))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 469
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1115,
     "status": "ok",
     "timestamp": 1527857823879,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "jxsu6Z8CqZSn",
    "outputId": "cafc5270-91bc-48a2-eea5-915dfec0a193"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.33      0.07      0.11        87\n",
      "   positive       0.79      0.96      0.87       313\n",
      "\n",
      "avg / total       0.69      0.77      0.70       400\n",
      "\n",
      "[[  6  12]\n",
      " [ 81 301]]\n",
      "TPR = 0.9616613418530351\n",
      " TNR = 0.06896551724137931\n",
      " FPR = 0.9310344827586207\n",
      " FNR = 0.038338658146964855\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "   negative       0.45      0.11      0.18        87\n",
      "   positive       0.80      0.96      0.87       313\n",
      "\n",
      "avg / total       0.72      0.78      0.72       400\n",
      "\n",
      "[[ 10  12]\n",
      " [ 77 301]]\n",
      "TPR = 0.9616613418530351\n",
      " TNR = 0.11494252873563218\n",
      " FPR = 0.8850574712643678\n",
      " FNR = 0.038338658146964855\n"
     ]
    }
   ],
   "source": [
    "predictions = gscv.best_estimator_.predict(avg_w2vec_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "\n",
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))\n",
    "\n",
    "predictions = rscv.best_estimator_.predict(avg_w2vec_test_scaled)\n",
    "print(classification_report(y_test, predictions))\n",
    "print(confusion_matrix(y_test, predictions).T)\n",
    "tn, fp, fn, tp = confusion_matrix(y_test, predictions).ravel()\n",
    "\n",
    "print(\"TPR = {}\\n TNR = {}\\n FPR = {}\\n FNR = {}\".format(tp/(fn+tp), tn/(tn+fp), fp/(tn+fp), fn/(fn+tp)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 86
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1080,
     "status": "ok",
     "timestamp": 1527857825254,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "-M5Pnoo45Kcr",
    "outputId": "951d33e3-e393-47c9-fee2-ea97d1b9244d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=10.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma=0.9474210526315789,\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 64,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gscv.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 86
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 972,
     "status": "ok",
     "timestamp": 1527857826424,
     "user": {
      "displayName": "Vishal Singh",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104172755534144843354"
     },
     "user_tz": -330
    },
    "id": "-vBcWMiF5OdR",
    "outputId": "9785b386-ddde-4570-a23f-5e75974458f8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=23.865376755248537, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma=0.8557541623170543,\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 65,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rscv.best_estimator_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NGFJOyPRs3jB"
   },
   "source": [
    "## Conclusions\n",
    "avg_word2vec performs better than other representations especially BoW and TF IDF. TNR is somewhat improved.<br><br>\n",
    "gamma : 0.855\n",
    "<br>20 < C < 25"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "default_view": {},
   "name": "Apply SVM to Amazon reviews data set avg_w2vec [M].ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}