DBSCAN Clustering on Amazon Fine Food Reviews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import sqlite3\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import string\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import math\n",
    "from prettytable import PrettyTable\n",
    "from scipy import sparse\n",
    "import csv\n",
    "import graphviz\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import cross_val_score, cross_validate, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.preprocessing import StandardScaler, Normalizer\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, roc_curve, auc, make_scorer, silhouette_score\n",
    "from sklearn.metrics import confusion_matrix, classification_report, precision_score, roc_curve\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn.tree import DecisionTreeClassifier, export_graphviz\n",
    "from scipy.stats import randint\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from wordcloud import WordCloud\n",
    "from sklearn.cluster import KMeans, AgglomerativeClustering,DBSCAN\n",
    "\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models import KeyedVectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 270 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "data_train = pd.read_csv('./Matrices/sample_data_train.csv')\n",
    "data_test = pd.read_csv('./Matrices/sample_data_test.csv')\n",
    "data = pd.concat([data_train, data_test], ignore_index=True)\n",
    "\n",
    "del data_train, data_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_dbscan(data, vects):\n",
    "    sc = StandardScaler()\n",
    "    \n",
    "    data_np = data.drop(['Id', 'ProductId', 'UserId', 'ProfileName', 'Score', 'Summary', 'Text', 'CleanedText'], axis=1)\n",
    "    data_np = np.hstack((data_np, vects))\n",
    "    data_np = sc.fit_transform(data_np)\n",
    "    \n",
    "    scores = []; radii = [1, 10, 20, 30, 40];\n",
    "    for i in radii:\n",
    "        print(f'Starting Radius #{i}')\n",
    "        model = DBSCAN(eps=i, min_samples=200)\n",
    "        model.fit(data_np)\n",
    "        print(set(model.labels_))\n",
    "    \n",
    "    return data_np, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 22.8 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def avg_w2v(data_train):\n",
    "    train_list_of_sent=[]\n",
    "    for sent in data_train['CleanedText'].values:\n",
    "        train_list_of_sent.append(sent.split())\n",
    "\n",
    "    w2v_model = Word2Vec(train_list_of_sent, min_count=5, size=100, workers=4)\n",
    "    w2v_words = list(w2v_model.wv.vocab)\n",
    "\n",
    "    sent_vectors_train = []\n",
    "    for sent in train_list_of_sent:\n",
    "        sent_vec = np.zeros(100)\n",
    "        count_words = 0\n",
    "        for word in sent:\n",
    "            if word in w2v_words:\n",
    "                try:\n",
    "                    vec = w2v_model.wv[word]\n",
    "                except KeyError:\n",
    "                    vec = np.zeros(100)\n",
    "                sent_vec += vec\n",
    "                count_words += 1\n",
    "        if count_words != 0:\n",
    "            sent_vec /= count_words\n",
    "        sent_vectors_train.append(sent_vec)\n",
    "    \n",
    "    return sent_vectors_train\n",
    "\n",
    "vectors = avg_w2v(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Radius #1\n",
      "{-1}\n",
      "Starting Radius #10\n",
      "{0, -1}\n",
      "Starting Radius #20\n",
      "{0, -1}\n",
      "Starting Radius #30\n",
      "{0, -1}\n",
      "Starting Radius #40\n",
      "{0}\n",
      "Wall time: 10min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "data_np, model = run_dbscan(data, vectors)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}