Spam detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "from __future__ import division\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import tarfile\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading training data\n",
      "loading test data\n"
     ]
    }
   ],
   "source": [
    "def csv_to_numpy_array(filePath, delimiter):\n",
    "    return np.genfromtxt(filePath, delimiter=delimiter, dtype='float32')\n",
    "\n",
    "def import_data():\n",
    "    if \"data\" not in os.listdir(os.getcwd()):\n",
    "        # Untar directory of data if we haven't already\n",
    "        tarObject = tarfile.open(\"data.tar.gz\")\n",
    "        tarObject.extractall()\n",
    "        tarObject.close()\n",
    "        print(\"Extracted tar to current directory\")\n",
    "    else:\n",
    "        # we've already extracted the files\n",
    "        pass\n",
    "\n",
    "    print(\"loading training data\")\n",
    "    trainX = csv_to_numpy_array(\"data/trainX.csv\", delimiter=\",\")\n",
    "    trainY = csv_to_numpy_array(\"data/trainY.csv\", delimiter=\",\")\n",
    "    print(\"loading test data\")\n",
    "    testX = csv_to_numpy_array(\"data/testX.csv\", delimiter=\",\")\n",
    "    testY = csv_to_numpy_array(\"data/testY.csv\", delimiter=\",\")\n",
    "    return trainX,trainY,testX,testY\n",
    "\n",
    "trainX,trainY,testX,testY = import_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def accuracy(predictions, labels):\n",
    "  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1))\n",
    "          / predictions.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2250, 3752)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainX.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "num_hidden_nodes1 = 2000\n",
    "num_hidden_nodes2 = 1000\n",
    "num_hidden_nodes3 = 256\n",
    "keep_prob = 0.5\n",
    "# numFeatures = the number of words extracted from each email\n",
    "numFeatures = trainX.shape[1]\n",
    "# numLabels = number of classes we are predicting (here just 2: Spam or Ham)\n",
    "numLabels = trainY.shape[1]\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "  # Input data.\n",
    "  \n",
    "  tf_train_dataset = tf.constant(trainX)\n",
    "  tf_train_labels = tf.constant(trainY)\n",
    "  tf_test_dataset = tf.constant(testX)\n",
    "\n",
    "  # Single mail input. \n",
    "  tf_mail = tf.placeholder(tf.float32, shape=(1, numFeatures))\n",
    "  \n",
    "\n",
    "  # Variables.\n",
    "  weights1 = tf.Variable(tf.truncated_normal([numFeatures, num_hidden_nodes1],\n",
    "        stddev=np.sqrt(2.0 / (numFeatures))),name=\"v1\")\n",
    "         \n",
    "  biases1 = tf.Variable(tf.zeros([num_hidden_nodes1]),name=\"v2\")\n",
    "  weights2 = tf.Variable(\n",
    "    tf.truncated_normal([num_hidden_nodes1, num_hidden_nodes2], stddev=np.sqrt(2.0 / num_hidden_nodes1)),name=\"v3\")\n",
    "  biases2 = tf.Variable(tf.zeros([num_hidden_nodes2]),name=\"v4\")\n",
    "  weights3 = tf.Variable(\n",
    "       tf.truncated_normal([num_hidden_nodes2, num_hidden_nodes3], stddev=np.sqrt(2.0 / num_hidden_nodes2)),name=\"v5\")\n",
    "  biases3 = tf.Variable(tf.zeros([num_hidden_nodes3]),name=\"v6\")\n",
    "  weights4 = tf.Variable(\n",
    "    tf.truncated_normal([num_hidden_nodes3, numLabels], stddev=np.sqrt(2.0 / num_hidden_nodes3)),name=\"v7\")\n",
    "  biases4 = tf.Variable(tf.zeros([numLabels]),name=\"v8\")\n",
    "  \n",
    "  # Add ops to save and restore all the variables.\n",
    "  saver = tf.train.Saver()\n",
    "    \n",
    "  # Training computation.\n",
    "  layer1_train = tf.nn.relu(tf.matmul(tf_train_dataset, weights1) + biases1)\n",
    "  drop1 = tf.nn.dropout(layer1_train, keep_prob)\n",
    "  layer2_train = tf.nn.relu(tf.matmul(drop1, weights2) + biases2)\n",
    "  drop2 = tf.nn.dropout(layer2_train, keep_prob)\n",
    "  layer3_train = tf.nn.relu(tf.matmul(drop2, weights3) + biases3)\n",
    "  drop3 = tf.nn.dropout(layer3_train, keep_prob)\n",
    "  logits = tf.matmul(drop3, weights4) + biases4\n",
    "  loss = tf.reduce_mean(\n",
    "     tf.nn.sigmoid_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "  \n",
    "  # Optimizer.\n",
    "  optimizer = tf.train.AdamOptimizer(learning_rate=0.1, \n",
    "                                              beta1=0.9, beta2=0.999, \n",
    "                                              epsilon=1e-08).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, test data, and single mail.\n",
    "    \n",
    "  train_prediction = tf.nn.sigmoid(logits)\n",
    "  \n",
    "  layer1_test = tf.nn.relu(tf.matmul(tf_test_dataset, weights1) + biases1)\n",
    "  layer2_test = tf.nn.relu(tf.matmul(layer1_test, weights2) + biases2)\n",
    "  layer3_test = tf.nn.relu(tf.matmul(layer2_test, weights3) + biases3)\n",
    "  test_prediction = tf.nn.sigmoid(tf.matmul(layer3_test, weights4) + biases4)\n",
    "  \n",
    "  layer1_mail = tf.nn.relu(tf.matmul(tf_mail, weights1) + biases1)\n",
    "  layer2_mail = tf.nn.relu(tf.matmul(layer1_mail, weights2) + biases2)\n",
    "  layer3_mail = tf.nn.relu(tf.matmul(layer2_mail, weights3) + biases3)\n",
    "  prediction_mail = tf.nn.sigmoid(tf.matmul(layer3_mail, weights4) + biases4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Loss at step 0: 0.694168\n",
      "Accuracy: 34.6%\n",
      "Loss at step 10: 0.571108\n",
      "Accuracy: 62.0%\n",
      "Loss at step 20: 0.241275\n",
      "Accuracy: 95.5%\n",
      "Loss at step 30: 0.128373\n",
      "Accuracy: 98.8%\n",
      "Loss at step 40: 0.025614\n",
      "Accuracy: 99.2%\n",
      "Loss at step 50: 0.030443\n",
      "Accuracy: 99.6%\n",
      "Loss at step 60: 0.007038\n",
      "Accuracy: 99.9%\n",
      "Loss at step 70: 0.011091\n",
      "Accuracy: 99.7%\n",
      "Loss at step 80: 0.003612\n",
      "Accuracy: 99.8%\n",
      "Loss at step 90: 0.005343\n",
      "Accuracy: 99.8%\n",
      "Loss at step 100: 0.002600\n",
      "Accuracy: 100.0%\n",
      "Loss at step 110: 0.002260\n",
      "Accuracy: 100.0%\n",
      "Loss at step 120: 0.003292\n",
      "Accuracy: 99.9%\n",
      "Loss at step 130: 0.007362\n",
      "Accuracy: 99.8%\n",
      "Loss at step 140: 0.002222\n",
      "Accuracy: 99.9%\n",
      "Loss at step 150: 0.006215\n",
      "Accuracy: 99.9%\n",
      "Test accuracy: 99.6%\n",
      "Model saved in file: ./model.ckpt\n",
      "time consumed in training: 983.865948 seconds\n"
     ]
    }
   ],
   "source": [
    "num_steps = 151\n",
    "start = time.time()\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.initialize_all_variables().run()\n",
    "  print(\"Initialized\")\n",
    "  for step in range(num_steps):\n",
    "    \n",
    "    \n",
    "    _, l, predictions = session.run(\n",
    "      [optimizer, loss, train_prediction])\n",
    "    acc = accuracy(predictions,trainY)\n",
    "    if (step % 10 == 0):\n",
    "      print(\"Loss at step %d: %f\" % (step, l))\n",
    "      print(\"Accuracy: %.1f%%\" % acc)\n",
    "      \n",
    "  print(\"Test accuracy: %.1f%%\" % accuracy(test_prediction.eval(), testY))\n",
    "\n",
    "  # Save the variables to disk.\n",
    "  save_path = saver.save(session, \"./model.ckpt\")\n",
    "  print(\"Model saved in file: %s\" % save_path)\n",
    "\n",
    "end = time.time()\n",
    "duration = end - start\n",
    "print(\"time consumed in training: %f seconds\" % duration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import cPickle\n",
    "from collections import Counter\n",
    "# Load Bag of words that were in training data \n",
    "BagOfWords = cPickle.load(open('BagOfWords.p', 'rb'))\n",
    "features = set(BagOfWords)\n",
    "featureDict = {feature:i for i,feature in enumerate(features)}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_feature_vector(email_text, featureDict):\n",
    "    '''\n",
    "    create feature/x vector from email text \n",
    "    row = email, cols = features\n",
    "    '''\n",
    "    featurevector = np.zeros(shape=(1,len(featureDict)),dtype=float)\n",
    "\n",
    "    \n",
    "    tokens = email_text.split()\n",
    "    UniDist = Counter(tokens)\n",
    "    for key,value in UniDist.items():\n",
    "        if key in featureDict:\n",
    "            featurevector[0,featureDict[key]] = value\n",
    "    return featurevector\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def regularize_vectors(featurevector):\n",
    "    '''\n",
    "    Input:\n",
    "      featurevector: vector, where single email is a row and features are columns\n",
    "    Returns:\n",
    "      featurevector: vector, updated by dividing each feature value by the total\n",
    "      number of features \n",
    "    '''\n",
    "    \n",
    "    totalWords = np.sum(featurevector[0,:],axis=0)\n",
    "    featurevector[0,:] = np.multiply(featurevector[0,:],(1/(totalWords + 1e-5)))\n",
    "    return featurevector\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Enter your username and password\n",
    "If ur email is a.b@gmail.com , only enter a.b \n",
    "\n",
    "And before running the cell below, u need to enable IMAP from [HERE](https://support.google.com/mail/answer/7126229?hl=en&visit_id=1-636175756291815919-2042916920&rd=1) \n",
    "\n",
    "First time Gmail rejected the loggin , and sent mail to  my inbox reviewing the block and asked me to allow loggin or not .\n",
    "\n",
    "And u just need to allow Less secure apps from [HERE](https://www.google.com/settings/security/lesssecureapps?rfn=27&rfnc=1&et=0&asae=2&anexp=ire-f3)\n",
    "\n",
    "![Gmail](picture.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter your GMail username --> mo.ab.elkasaby\n",
      "Enter your password --> ········\n",
      "OK, mo.ab.elkasaby@gmail.com  authenticated (Success)\n"
     ]
    }
   ],
   "source": [
    "import email, getpass, imaplib\n",
    "user = raw_input(\"Enter your GMail username --> \")\n",
    "pwd = getpass.getpass(\"Enter your password --> \")\n",
    "m = imaplib.IMAP4_SSL(\"imap.gmail.com\")\n",
    "m.login(user, pwd)\n",
    "print('OK, %s@gmail.com  authenticated (Success)' % user)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_text(mail,i):\n",
    "    result, data = mail.uid('search', None, \"ALL\")\n",
    "    # search and return uids instead\n",
    "    latest_email_uid = data[0].split()[i] # unique ids wrt label selected\n",
    "    result, email_data = mail.uid('fetch', latest_email_uid, '(RFC822)')\n",
    "    # fetch the email body (RFC822) for the given ID\n",
    "    raw_email = email_data[0][1]\n",
    "    raw_email_string = raw_email.decode('utf-8')\n",
    "    # converts byte literal to string removing b''\n",
    "    email_message = email.message_from_string(raw_email_string)\n",
    "    # this will loop through all the available multiparts in mail\n",
    "    for part in email_message.walk():\n",
    "       if part.get_content_type() == \"text/plain\": # ignore attachments/html\n",
    "          body = part.get_payload(decode=True)\n",
    "    \n",
    "    return str(body)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "eng.vultu\n",
      "[  1.00000000e+00   5.35245217e-23]\n",
      "Spam\n",
      "�� ����� \n",
      "[ 0.  1.]\n",
      "Not Spam\n",
      "�� ����� \n",
      "[ 0.  1.]\n",
      "Not Spam\n",
      "شاهد \n",
      "[  1.34646243e-07   9.99999762e-01]\n",
      "Not Spam\n",
      "������ ��\n",
      "[ 0.  1.]\n",
      "Not Spam\n",
      "Add Moham\n",
      "[  1.21050980e-05   9.99984980e-01]\n",
      "Not Spam\n",
      "mohamed� \n",
      "[ 0.  1.]\n",
      "Not Spam\n"
     ]
    }
   ],
   "source": [
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    m.list() # Lists all labels in GMail\n",
    "    m.select('Inbox') # Connected to 'Inbox' or '[Gmail]/Spam'\n",
    "    for i in range(0,7,1):\n",
    "        email_text = get_text(m,i)\n",
    "        print(email_text[:9])\n",
    "        email_test = get_feature_vector(email_text,featureDict)\n",
    "        email_test = regularize_vectors(email_test)\n",
    "        # Restore variables from disk.\n",
    "        saver.restore(session, \"./model.ckpt\")\n",
    "        #print(\"Model restored.\")\n",
    "        # Do some work with the model\n",
    "        feed_dict={tf_mail:email_test}\n",
    "        emailpred = session.run(prediction_mail,feed_dict=feed_dict)\n",
    "        #check on the first column of the single row \n",
    "        print(emailpred[0])\n",
    "        if emailpred[0][0] > emailpred[0][1] :\n",
    "            print(\"Spam\")\n",
    "        else:\n",
    "            print(\"Not Spam\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  1.00000000e+00   6.59820776e-09]\n",
      "Spam\n",
      "[ 1.  0.]\n",
      "Spam\n"
     ]
    }
   ],
   "source": [
    "m.list() # Lists all labels in GMail\n",
    "m.select('[Gmail]/Spam') # Connected to 'Inbox' or '[Gmail]/Spam'\n",
    "for i in range(-1,-3,-1):\n",
    "    email_text = get_text(m,i)\n",
    "    #print(email_text)\n",
    "    email_test = get_feature_vector(email_text,featureDict)\n",
    "    email_test = regularize_vectors(email_test)\n",
    "    with tf.Session(graph=graph) as session:\n",
    "        # Restore variables from disk.\n",
    "        saver.restore(session, \"./model.ckpt\")\n",
    "        #print(\"Model restored.\")\n",
    "        # Do some work with the model\n",
    "        feed_dict={tf_mail:email_test}\n",
    "        emailpred = session.run(prediction_mail,feed_dict=feed_dict)\n",
    "        #check on the first column of the single row\n",
    "        print(emailpred[0])\n",
    "        if emailpred[0][0] > emailpred[0][1] :\n",
    "            print(\"Spam\")\n",
    "        else:\n",
    "            print(\"Not Spam\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logic used\n",
    "I used One-hot encoding which means that one column is One and other is Zero.\n",
    "\n",
    "prediction consists of two columns [Spam, Not Spam] , the prediction is normalized probability if the first column is greater than the second then it is 'Spam' and else is 'Not Spam'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  0.], dtype=float32)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testY[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}