Twitter Sentiment Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/rohit/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to /home/rohit/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /home/rohit/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "import string\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rohit/miniconda3/envs/deeplearning/lib/python3.6/site-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  import sys\n"
     ]
    }
   ],
   "source": [
    "DATASET_COLUMNS = [\"target\", \"ids\", \"date\", \"flag\", \"user\", \"text\"]\n",
    "DATASET_ENCODING = \"ISO-8859-1\"\n",
    "data = pd.read_csv('sentiment140.csv', encoding =DATASET_ENCODING , names=DATASET_COLUMNS)\n",
    "data.head()\n",
    "X = data.iloc[:,[5]]\n",
    "Y = data.iloc[:,0]\n",
    "Y[Y == 4] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>ids</th>\n",
       "      <th>date</th>\n",
       "      <th>flag</th>\n",
       "      <th>user</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810369</td>\n",
       "      <td>Mon Apr 06 22:19:45 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>_TheSpecialOne_</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810672</td>\n",
       "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>scotthamilton</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810917</td>\n",
       "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>mattycus</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811184</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>ElleCTF</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811193</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>Karoli</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   target         ids                          date      flag  \\\n",
       "0       0  1467810369  Mon Apr 06 22:19:45 PDT 2009  NO_QUERY   \n",
       "1       0  1467810672  Mon Apr 06 22:19:49 PDT 2009  NO_QUERY   \n",
       "2       0  1467810917  Mon Apr 06 22:19:53 PDT 2009  NO_QUERY   \n",
       "3       0  1467811184  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY   \n",
       "4       0  1467811193  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY   \n",
       "\n",
       "              user                                               text  \n",
       "0  _TheSpecialOne_  @switchfoot http://twitpic.com/2y1zl - Awww, t...  \n",
       "1    scotthamilton  is upset that he can't update his Facebook by ...  \n",
       "2         mattycus  @Kenichan I dived many times for the ball. Man...  \n",
       "3          ElleCTF    my whole body feels itchy and like its on fire   \n",
       "4           Karoli  @nationwideclass no, it's not behaving at all....  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of missing values: 0\n"
     ]
    }
   ],
   "source": [
    "# Text-preprocessing\n",
    "\n",
    "# Missing Values\n",
    "num_missing_desc = data.isnull().sum()[2]    # No. of values with msising descriptions\n",
    "print('Number of missing values: ' + str(num_missing_desc))\n",
    "data = data.dropna()\n",
    "\n",
    "TAG_CLEANING_RE = \"@\\S+\"\n",
    "# Remove @tags\n",
    "X['text'] = X['text'].map(lambda x: re.sub(TAG_CLEANING_RE, ' ', x))\n",
    "\n",
    "# Smart lowercase\n",
    "X['text'] = X['text'].map(lambda x: x.lower())\n",
    "\n",
    "# Remove numbers\n",
    "X['text'] = X['text'].map(lambda x: re.sub(r'\\d+', ' ', x))\n",
    "\n",
    "# Remove links\n",
    "TEXT_CLEANING_RE = \"https?:\\S+|http?:\\S|[^A-Za-z0-9]+\"\n",
    "X['text'] = X['text'].map(lambda x: re.sub(TEXT_CLEANING_RE, ' ', x))\n",
    "\n",
    "# Remove Punctuation\n",
    "X['text']  = X['text'].map(lambda x: x.translate(x.maketrans('', '', string.punctuation)))\n",
    "\n",
    "# Remove white spaces\n",
    "X['text'] = X['text'].map(lambda x: x.strip())\n",
    "\n",
    "# Tokenize into words\n",
    "X['text'] = X['text'].map(lambda x: word_tokenize(x))\n",
    " \n",
    "# Remove non alphabetic tokens\n",
    "X['text'] = X['text'].map(lambda x: [word for word in x if word.isalpha()])\n",
    "\n",
    "# Filter out stop words\n",
    "stop_words = set(stopwords.words('english'))\n",
    "X['text'] = X['text'].map(lambda x: [w for w in x if not w in stop_words])\n",
    "    \n",
    "# Word Lemmatization\n",
    "lem = WordNetLemmatizer()\n",
    "X['text'] = X['text'].map(lambda x: [lem.lemmatize(word,\"v\") for word in x])\n",
    "\n",
    "# Turn lists back to string\n",
    "X['text'] = X['text'].map(lambda x: ' '.join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>zl awww bummer shoulda get david carr third day</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>upset update facebook texting might cry result...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dive many time ball manage save rest go bound</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>whole body feel itchy like fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>behave mad see</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0    zl awww bummer shoulda get david carr third day\n",
       "1  upset update facebook texting might cry result...\n",
       "2      dive many time ball manage save rest go bound\n",
       "3                    whole body feel itchy like fire\n",
       "4                                     behave mad see"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN size: 1280000\n",
      "TEST size: 1280000\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=42)\n",
    "print(\"TRAIN size:\", len(X_train))\n",
    "print(\"TEST size:\", len(X_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Flow \n",
    "\n",
    "- Pre-processing\n",
    "- create word2vec\n",
    "- tokenize and padding\n",
    "- model creation\n",
    "- testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Word2Vec\n",
    "import gensim\n",
    "\n",
    "# WORD2VEC \n",
    "W2V_SIZE = 300\n",
    "W2V_WINDOW = 7\n",
    "W2V_EPOCH = 32\n",
    "W2V_MIN_COUNT = 10\n",
    "\n",
    "documents = [_text.split() for _text in X_train.text] \n",
    "w2v_model = gensim.models.word2vec.Word2Vec(size=W2V_SIZE, \n",
    "                                            window=W2V_WINDOW, \n",
    "                                            min_count=W2V_MIN_COUNT, \n",
    "                                            workers=8)\n",
    "w2v_model.build_vocab(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size 25276\n"
     ]
    }
   ],
   "source": [
    "words = w2v_model.wv.vocab.keys()\n",
    "vocab_size = len(words)\n",
    "print(\"Vocab size\", vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(251362385, 289225504)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train Word Embeddings\n",
    "w2v_model.train(documents, total_examples=len(documents), epochs=W2V_EPOCH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rohit/miniconda3/envs/deeplearning/lib/python3.6/site-packages/ipykernel_launcher.py:2: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('suck', 0.5349220037460327),\n",
       " ('stupid', 0.5056272149085999),\n",
       " ('dislike', 0.4653589725494385),\n",
       " ('ugh', 0.4632347524166107),\n",
       " ('hat', 0.4575125575065613),\n",
       " ('despise', 0.44876334071159363),\n",
       " ('fuck', 0.4072660505771637),\n",
       " ('annoy', 0.4054874777793884),\n",
       " ('fml', 0.3965710699558258),\n",
       " ('ughh', 0.38682791590690613)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Test word embeddings\n",
    "w2v_model.most_similar(\"hate\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 232838 unique tokens.\n",
      "Shape of data tensor: (1280000, 300)\n"
     ]
    }
   ],
   "source": [
    "# Tokenizing\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, LSTM, Dropout\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "# Max number of words in each complaint.\n",
    "MAX_SEQUENCE_LENGTH = 300\n",
    "# This is fixed.\n",
    "EMBEDDING_DIM = 300\n",
    "\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(X_train.text)\n",
    "word_index = tokenizer.word_index\n",
    "vocab_size = len(word_index)\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "# Convert the data to padded sequences\n",
    "X_train_padded = tokenizer.texts_to_sequences(X_train.text)\n",
    "X_train_padded = pad_sequences(X_train_padded, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "print('Shape of data tensor:', X_train_padded.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving\n",
    "import pickle\n",
    "with open('tokenizer.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(232839, 300)\n"
     ]
    }
   ],
   "source": [
    "# Embedding matrix for the embedding layer\n",
    "embedding_matrix = np.zeros((vocab_size+1, W2V_SIZE))\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    if word in w2v_model.wv:\n",
    "        embedding_matrix[i] = w2v_model.wv[word]\n",
    "print(embedding_matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 300, 300)          69851700  \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 300, 300)          0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 100)               160400    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 70,012,201\n",
      "Trainable params: 160,501\n",
      "Non-trainable params: 69,851,700\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Build Model\n",
    "import keras \n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(vocab_size+1, W2V_SIZE, weights=[embedding_matrix], input_length=MAX_SEQUENCE_LENGTH, trainable=False))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=\"adam\",\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1152000 samples, validate on 128000 samples\n",
      "Epoch 1/5\n",
      "  59904/1152000 [>.............................] - ETA: 14:29 - loss: 0.5648 - accuracy: 0.7047"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-21-4616d941000b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m                     \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m                     \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m                     callbacks=callbacks)\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Sentiment_LSTM_model.h5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/trainHistoryDict'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfile_pi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/deeplearning/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m   1237\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m                                         \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1239\u001b[0;31m                                         validation_freq=validation_freq)\n\u001b[0m\u001b[1;32m   1240\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1241\u001b[0m     def evaluate(self,\n",
      "\u001b[0;32m~/miniconda3/envs/deeplearning/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, fit_function, fit_inputs, out_labels, batch_size, epochs, verbose, callbacks, val_function, val_inputs, shuffle, initial_epoch, steps_per_epoch, validation_steps, validation_freq)\u001b[0m\n\u001b[1;32m    194\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfit_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    197\u001b[0m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_labels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/deeplearning/lib/python3.6/site-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3790\u001b[0m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3791\u001b[0m       \u001b[0mconverted_inputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3792\u001b[0;31m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_graph_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mconverted_inputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3793\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3794\u001b[0m     \u001b[0;31m# EagerTensor.numpy() will often make a copy to ensure memory safety.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/deeplearning/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1603\u001b[0m       \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mFor\u001b[0m \u001b[0minvalid\u001b[0m \u001b[0mpositional\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mkeyword\u001b[0m \u001b[0margument\u001b[0m \u001b[0mcombinations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1604\u001b[0m     \"\"\"\n\u001b[0;32m-> 1605\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1606\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1607\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/deeplearning/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1643\u001b[0m       raise TypeError(\"Keyword arguments {} unknown. Expected {}.\".format(\n\u001b[1;32m   1644\u001b[0m           list(kwargs.keys()), list(self._arg_keywords)))\n\u001b[0;32m-> 1645\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1646\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1647\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/deeplearning/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1744\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1745\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1746\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1747\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/deeplearning/lib/python3.6/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    596\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    597\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 598\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    599\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    600\u001b[0m           outputs = execute.execute_with_cancellation(\n",
      "\u001b[0;32m~/miniconda3/envs/deeplearning/lib/python3.6/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Training \n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "callbacks = [ ReduceLROnPlateau(monitor='val_loss', patience=5, cooldown=0),\n",
    "              EarlyStopping(monitor='val_acc', min_delta=1e-4, patience=5)]\n",
    "\n",
    "BATCH_SIZE = 1024\n",
    "history = model.fit(X_train_padded, y_train,\n",
    "                    batch_size=512,\n",
    "                    epochs=5,\n",
    "                    validation_split=0.1,\n",
    "                    verbose=1,\n",
    "                    callbacks=callbacks)\n",
    "model.save('main/Sentiment_LSTM_model.h5')\n",
    "with open('main/trainHistoryDict', 'wb') as file_pi:\n",
    "    pickle.dump(history.history, file_pi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Model\n",
    "from keras.models import load_model\n",
    "model = load_model('main/Sentiment_LSTM_model.h5')\n",
    "# loading tokenizer\n",
    "with open('main/trainHistoryDict', 'rb') as file_pi:\n",
    "    history = pickle.load(file_pi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "320000/320000 [==============================] - 540s 2ms/step\n",
      "ACCURACY: 0.784396875\n",
      "LOSS: 0.45383153524398806\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'history' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-21-d8f402d3914c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"LOSS:\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhistory\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'acc'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[0mval_acc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhistory\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'val_acc'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhistory\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'loss'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'history' is not defined"
     ]
    }
   ],
   "source": [
    "# Evaluation\n",
    "import matplotlib.pyplot as plt\n",
    "X_test_padded = tokenizer.texts_to_sequences(X_test.text)\n",
    "X_test_padded = pad_sequences(X_test_padded, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "score = model.evaluate(X_test_padded, y_test, batch_size=512)\n",
    "print(\"ACCURACY:\",score[1])\n",
    "print(\"LOSS:\",score[0])\n",
    "\n",
    "acc = history.history['acc']\n",
    "val_acc = history.history['val_acc']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    " \n",
    "epochs = range(len(acc))\n",
    " \n",
    "plt.plot(epochs, acc, 'b', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'r', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    " \n",
    "plt.figure()\n",
    " \n",
    "plt.plot(epochs, loss, 'b', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'r', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    " \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(text, include_neutral=True):\n",
    "    # Tokenize text\n",
    "    x_test = pad_sequences(tokenizer.texts_to_sequences([text]), maxlen=MAX_SEQUENCE_LENGTH)\n",
    "    # Predict\n",
    "    score = model.predict([x_test])[0]\n",
    "    if(score >=0.4 and score<=0.6):\n",
    "        label = \"Neutral\"\n",
    "    if(score <=0.4):\n",
    "        label = \"Negative\"\n",
    "    if(score >=0.6):\n",
    "        label = \"Positive\"\n",
    "\n",
    "    return {\"label\" : label,\n",
    "        \"score\": float(score)}  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'label': 'Positive', 'score': 0.7831670641899109}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict(\"God is good\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}