python_tutorial_part_8_collections_module.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Python's collections module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Documentation: https://docs.python.org/2/library/collections.html\n",
    "# Per documentation, \"this module implements specialized container datatypes\\\n",
    "# providing alternatives to Python’s general purpose built-in containers, dict, list, set, and tuple\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DataDoc(tag='POSITIVE', words=['i', 'love', 'pizza'])\n",
      "DataDoc(tag='POSITIVE', words=['i', 'like', 'apple'])\n",
      "DataDoc(tag='POSITIVE', words=['i', 'enjoy', 'hiking'])\n",
      "DataDoc(tag='POSITIVE', words=['i', 'am', 'passionate', 'about', 'traveling'])\n",
      "DataDoc(tag='POSITIVE', words=['we', 'had', 'fun', 'writing', 'this', 'code'])\n",
      "DataDoc(tag='NEGATIVE', words=['i', \"don't\", 'like', 'to', 'stay', 'up', 'late'])\n",
      "DataDoc(tag='NEGATIVE', words=['i', 'am', 'tired'])\n",
      "DataDoc(tag='NEGATIVE', words=['he', 'feels', 'sick'])\n"
     ]
    }
   ],
   "source": [
    "# namedtuple(): factory function for creating tuple subclasses with named fields\n",
    "# Named tuples assign a name to each position in a tuple, thus enabling accessing\n",
    "# fields by name instead of position index.\n",
    "#-----------------------------------------------\n",
    "# namedtuple(typename, field_names[, verbose=False][, rename=False])\n",
    "# Returns a new tuple subclass named typename. \n",
    "# The new subclass is used to create tuple-like objects that have fields accessible \n",
    "# by attribute lookup as well as being indexable and iterable. \n",
    "\n",
    "from collections import namedtuple\n",
    "# We create a named tuple with two fields, tags and words.\n",
    "# tags will be a string\n",
    "# words will be a list of words\n",
    "DataDoc= namedtuple('DataDoc', 'tag words')\n",
    "# we create a list and each item in the list will be a namedtuple element with the two fields \"tags\" and \"words\"\n",
    "my_data=[]\n",
    "# We have a list of document. Each document has a single sentence. \n",
    "# The first word in each sentence/document is a tag from the set {POSITIVE, NEGATIVE}, so a sentiment analysis task. \n",
    "documents = [\"POSITIVE I love pizza\", \"POSITIVE I like Apple\", \"POSITIVE I enjoy hiking\",\\\n",
    "             \"POSITIVE I am passionate about traveling\", \"POSITIVE We had fun writing this code\",\\\n",
    "            \"NEGATIVE I don't like to stay up late\", \"NEGATIVE I am tired\", \"NEGATIVE He feels sick\"]\n",
    "\n",
    "# Now we loop over the documents and populate the list of allsent, which is basically our container for the \n",
    "# instances and their labels. From each document/sentence, we get the tag and the list of words\n",
    "for line_no, doc in enumerate(documents):\n",
    "    label=doc.split()[0]\n",
    "    word_list=doc.lower().split()[1:]\n",
    "    my_data.append(DataDoc(label, word_list))\n",
    "    print my_data[line_no]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[DataDoc(tag='POSITIVE', words=['i', 'love', 'pizza']), DataDoc(tag='POSITIVE', words=['i', 'like', 'apple']), DataDoc(tag='POSITIVE', words=['i', 'enjoy', 'hiking']), DataDoc(tag='POSITIVE', words=['i', 'am', 'passionate', 'about', 'traveling']), DataDoc(tag='POSITIVE', words=['we', 'had', 'fun', 'writing', 'this', 'code']), DataDoc(tag='NEGATIVE', words=['i', \"don't\", 'like', 'to', 'stay', 'up', 'late']), DataDoc(tag='NEGATIVE', words=['i', 'am', 'tired']), DataDoc(tag='NEGATIVE', words=['he', 'feels', 'sick'])]\n"
     ]
    }
   ],
   "source": [
    "print my_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "POSITIVE\n"
     ]
    }
   ],
   "source": [
    "# Now you can access the tag of each instance\n",
    "print my_data[0].tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i', 'love', 'pizza']\n"
     ]
    }
   ],
   "source": [
    "# You can also access the instance word list itself\n",
    "print my_data[0].words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We don't need this, but provided as an example\n",
    "def get_text_list(data):\n",
    "    \"\"\"\n",
    "    input is a list of namedtuples (either train, dev, or text)\n",
    "    returns a list of lists, each inner list is just the list of words belonging to a given data point\n",
    "    Used to get train_text, dev_text, or test_text\n",
    "    \"\"\"\n",
    "    \n",
    "    text_list=[]\n",
    "    for i in range(len(data)):\n",
    "        text_list.append(data[i].words)\n",
    "    return text_list\n",
    "\n",
    "train_text= get_text_list(train_data)\n",
    "print train_text[0][:6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GT 750M\n"
     ]
    }
   ],
   "source": [
    "from gensim import corpora\n",
    "# Now let's use Gensim to get a dictionary of the words in the train data:\n",
    "# We only need that dict from the training data (Can you think why?, use since we only learn using feature from train)\n",
    "dictionary = corpora.Dictionary(train_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(113562 unique tokens: [u'fawn', u'tsukino', u'nunnery', u'gah', u\"zuniga's\"]...)\n"
     ]
    }
   ],
   "source": [
    "print dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We can do the below to get the id of each word in the dict\n",
    "#print(dictionary.token2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 1), (1, 1), (2, 1), (3, 1), (4, 5), (5, 1), (6, 6), (7, 1), (8, 1), (9, 1), (10, 1), (11, 2), (12, 1), (13, 1), (14, 1), (15, 4), (16, 1), (17, 1), (18, 1), (19, 1), (20, 1), (21, 1), (22, 3), (23, 1), (24, 1), (25, 1), (26, 1), (27, 1), (28, 1), (29, 4), (30, 1), (31, 1), (32, 1), (33, 1), (34, 1), (35, 1), (36, 1), (37, 1), (38, 1), (39, 1), (40, 4), (41, 1), (42, 27), (43, 2), (44, 1), (45, 1), (46, 3), (47, 1), (48, 1), (49, 1), (50, 4), (51, 1), (52, 1), (53, 1), (54, 1), (55, 1), (56, 1), (57, 2), (58, 1), (59, 1), (60, 1), (61, 1), (62, 2), (63, 1), (64, 1), (65, 1), (66, 1), (67, 1), (68, 4), (69, 2), (70, 4), (71, 2), (72, 2), (73, 2), (74, 1), (75, 1), (76, 1), (77, 1), (78, 1), (79, 1), (80, 1), (81, 1), (82, 1), (83, 1), (84, 1), (85, 2), (86, 1), (87, 2), (88, 1), (89, 1), (90, 4), (91, 1), (92, 1), (93, 4), (94, 1), (95, 1), (96, 9)]\n"
     ]
    }
   ],
   "source": [
    "# Now let's vectorize the training data\n",
    "train_vecs= [dictionary.doc2bow(doc) for doc in train_text]\n",
    "print train_vecs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['david', \"bryce's\", 'comments', 'nearby', 'are', 'exceptionally', 'well', 'written', 'and', 'informative']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(0, 1),\n",
       " (1, 1),\n",
       " (4, 16),\n",
       " (6, 4),\n",
       " (16, 2),\n",
       " (28, 1),\n",
       " (29, 5),\n",
       " (37, 2),\n",
       " (39, 1),\n",
       " (40, 2)]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now let's get vectors for the test data:\n",
    "# First we need the text of each data point in test, let's use the function we developed above\n",
    "test_text= get_text_list(test_data)\n",
    "print test_text[-1][:10]\n",
    "# We can now use the test_text to get test_vecs\n",
    "test_vecs= [dictionary.doc2bow(doc) for doc in test_text]\n",
    "test_vecs[-1][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 [(0, 1), (4, 9), (6, 2), (8, 1), (16, 1), (29, 5), (32, 1), (33, 1), (37, 1), (39, 2)]\n"
     ]
    }
   ],
   "source": [
    "# We should usually get tags automatically based on input data file.\n",
    "# In the input data file we have, we know that the first 12500 data points are positive/1.0 and the next 12500 are\n",
    "# negative/0.0 then the next 12500 is poitive and the fourth chunk is negative.\n",
    "# So basically the train_data has 25K (with the first half positive and the second half negative)\n",
    "# and test_data with the same setup for class label. \n",
    "# The rest of the data in the file is unknown and we don't use that part.\n",
    "# We could write code to extract label automatically and we will do this based on a standardized format we will work with\n",
    "# later, for now we will hard-code the labels.\n",
    "\n",
    "train_tags=[ 1.0 for i in range(12500)] + [ 0.0 for i in range(12500)]\n",
    "test_tags=[ 1.0 for i in range(12500)] + [ 0.0 for i in range(12500)]\n",
    "# Side note: If the first token in each line were the tag, we could get tags as follows:\n",
    "# tags= [train_data[i].tag for i in range(len(train_data))]\n",
    "print train_tags[-1], train_vecs[-1][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TAG: 1.0, Vector: [(0, 1), (1, 1), (2, 1), (3, 1), (4, 5), (5, 1), (6, 6), (7, 1), (8, 1), (9, 1)]\n",
      "TAG: 1.0, Vector: [(0, 1), (1, 1), (4, 21), (6, 20), (11, 1), (13, 1), (17, 2), (20, 5), (28, 1), (29, 5)]\n",
      "TAG: 1.0, Vector: [(4, 8), (6, 2), (17, 1), (26, 1), (29, 2), (31, 1), (37, 1), (40, 2), (42, 11), (56, 1)]\n",
      "TAG: 1.0, Vector: [(4, 9), (6, 1), (16, 1), (21, 2), (28, 1), (29, 6), (42, 7), (44, 1), (50, 2), (60, 2)]\n",
      "TAG: 1.0, Vector: [(0, 2), (4, 8), (6, 5), (16, 2), (17, 1), (29, 1), (40, 4), (42, 6), (43, 1), (44, 1)]\n"
     ]
    }
   ],
   "source": [
    "# You can loop over the data to get the tags and vectors easily now:\n",
    "for i in range(5): # len(train_tage)\n",
    "    print(\"TAG: %s, Vector: %s\" % (train_tags[i], train_vecs[i][:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DataDoc(tag='_*0', words=['bromwell', 'high', 'is', 'a', 'cartoon', 'comedy', '.', 'it', 'ran', 'at', 'the', 'same', 'time', 'as', 'some', 'other', 'programs', 'about', 'school', 'life', ',', 'such', 'as', '\"', 'teachers', '\"', '.', 'my', '35', 'years', 'in', 'the', 'teaching', 'profession', 'lead', 'me', 'to', 'believe', 'that', 'bromwell', \"high's\", 'satire', 'is', 'much', 'closer', 'to', 'reality', 'than', 'is', '\"', 'teachers', '\"', '.', 'the', 'scramble', 'to', 'survive', 'financially', ',', 'the', 'insightful', 'students', 'who', 'can', 'see', 'right', 'through', 'their', 'pathetic', \"teachers'\", 'pomp', ',', 'the', 'pettiness', 'of', 'the', 'whole', 'situation', ',', 'all', 'remind', 'me', 'of', 'the', 'schools', 'i', 'knew', 'and', 'their', 'students', '.', 'when', 'i', 'saw', 'the', 'episode', 'in', 'which', 'a', 'student', 'repeatedly', 'tried', 'to', 'burn', 'down', 'the', 'school', ',', 'i', 'immediately', 'recalled', '.', '.', '.', '.', '.', '.', '.', '.', '.', 'at', '.', '.', '.', '.', '.', '.', '.', '.', '.', '.', 'high', '.', 'a', 'classic', 'line', ':', 'inspector', ':', \"i'm\", 'here', 'to', 'sack', 'one', 'of', 'your', 'teachers', '.', 'student', ':', 'welcome', 'to', 'bromwell', 'high', '.', 'i', 'expect', 'that', 'many', 'adults', 'of', 'my', 'age', 'think', 'that', 'bromwell', 'high', 'is', 'far', 'fetched', '.', 'what', 'a', 'pity', 'that', 'it', \"isn't\", '!'])\n"
     ]
    }
   ],
   "source": [
    "print train_data[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}