python_tutorial_part_5_gensim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Gensim Tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "# Corpora and Vector Spaces: https://radimrehurek.com/gensim/tut1.html\n",
    "#----------------------------------------------------------------------\n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "from gensim import corpora, models, similarities\n",
    "# Let's create a small corpus where each document is one sentence as in the gensim tutorial from the link above.\n",
    "# Let's take text from Wikipedia article on deep learning: https://en.wikipedia.org/wiki/Deep_learning\n",
    "# We have 10 documents (each doc is a sentence).\n",
    "documents = [\n",
    "    \"Deep learning (deep structured learning, hierarchical learning or deep machine learning)\\\n",
    "    is a branch of machine learning based on a set of algorithms that attempt to model \\\n",
    "    high-level abstractions in data by using multiple processing layers with complex \\\n",
    "    structures, or otherwise composed of multiple non-linear transformations.[1][2][3][4][5][6]\",\n",
    "    \"Deep learning is part of a broader family of machine learning methods based on learning representations of data.\",\n",
    "    \"An observation (e.g., an image) can be represented in many ways such as a vector of intensity values per pixel,\\\n",
    "    or in a more abstract way as a set of edges, regions of particular shape, etc. Some representations make it \\\n",
    "    easier to learn tasks (e.g., face recognition or facial expression recognition[7]) \\\n",
    "    from examples. One of the promises of deep learning is replacing handcrafted features \\\n",
    "    with efficient algorithms for unsupervised or semi-supervised feature learning and hierarchical \\\n",
    "    feature extraction.[8]\",\n",
    "    \"Deep learning is part of a broader family of machine learning methods based on learning \\\n",
    "    representations of data.\",\n",
    "    \"An observation (e.g., an image) can be represented in many ways such as a vector of intensity\\\n",
    "    values per pixel, or in a more abstract way as a set of edges, regions of particular shape, etc.\",\n",
    "    \"Some representations make it easier to learn tasks (e.g., face recognition or facial expression recognition[7]) from examples.\",\n",
    "    \"One of the promises of deep learning is replacing handcrafted features with efficient algorithms for unsupervised or semi-supervised \\\n",
    "    feature learning and hierarchical feature extraction.[8]\",\n",
    "    \"Research in this area attempts to make better representations and create models to learn these representations\\\n",
    "    from large-scale unlabeled data.\",\n",
    "    \"Some of the representations are inspired by advances in neuroscience and are loosely based on interpretation of information processing\\\n",
    "    and communication patterns in a nervous system, such as neural coding which attempts to define a relationship between various stimuli \\\n",
    "    and associated neuronal responses in the brain.[9]\",\n",
    "    \"Various deep learning architectures such as deep neural networks, convolutional deep neural networks, \\\n",
    "    deep belief networks and recurrent neural networks have been applied to fields like computer vision, automatic\\\n",
    "    speech recognition, natural language processing, audio recognition and bioinformatics where they have been shown to produce state-of-the-art\\\n",
    "    results on various tasks.\"]\n",
    "\n",
    "print len(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'i', u'me', u'my', u'myself', u'we', u'our', u'ours', u'ourselves', u'you', u'your', u'yours', u'yourself', u'yourselves', u'he', u'him', u'his', u'himself', u'she', u'her', u'hers', u'herself', u'it', u'its', u'itself', u'they', u'them', u'their', u'theirs', u'themselves', u'what', u'which', u'who', u'whom', u'this', u'that', u'these', u'those', u'am', u'is', u'are', u'was', u'were', u'be', u'been', u'being', u'have', u'has', u'had', u'having', u'do', u'does', u'did', u'doing', u'a', u'an', u'the', u'and', u'but', u'if', u'or', u'because', u'as', u'until', u'while', u'of', u'at', u'by', u'for', u'with', u'about', u'against', u'between', u'into', u'through', u'during', u'before', u'after', u'above', u'below', u'to', u'from', u'up', u'down', u'in', u'out', u'on', u'off', u'over', u'under', u'again', u'further', u'then', u'once', u'here', u'there', u'when', u'where', u'why', u'how', u'all', u'any', u'both', u'each', u'few', u'more', u'most', u'other', u'some', u'such', u'no', u'nor', u'not', u'only', u'own', u'same', u'so', u'than', u'too', u'very', u's', u't', u'can', u'will', u'just', u'don', u'should', u'now']\n"
     ]
    }
   ],
   "source": [
    "# Let's remove common words like \"a\" \"the\", etc. in English.\n",
    "# These are called stop words and we can use nltk for a list of these in English\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "stopwords= stopwords.words('english')\n",
    "print stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['deep', 'learning', 'deep', 'structured', 'learning', 'hierarchical', 'learning', 'deep', 'machine', 'learning', 'branch', 'machine', 'learning', 'based', 'set', 'algorithms', 'attempt', 'model', 'high', 'level', 'abstractions', 'data', 'using', 'multiple', 'processing', 'layers', 'complex', 'structures', 'otherwise', 'composed', 'multiple', 'non', 'linear', 'transformations', '1', '2', '3', '4', '5', '6']]\n"
     ]
    }
   ],
   "source": [
    "#We will need to lowercase text (some times we shouldn't do that naively if we care for things like\\\n",
    "#named entities (which start with uppercase)')\n",
    "# NLTK has a number of tokenization options here: http://www.nltk.org/api/nltk.tokenize.html\n",
    "# Especially note that NLTK also supports Twitter tokenization, which will be useful for us\n",
    "# Look at this line from the link above:\n",
    "#  from nltk.tokenize import TweetTokenizer\n",
    "#-----------------------------------------\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "texts= [[w for w in tokenizer.tokenize(document.lower()) if w not in stopwords] for document in documents]\n",
    "print texts[:1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defaultdict(<type 'int'>, {'interpretation': 1, 'results': 1, 'brain': 1, 'attempts': 2, 'broader': 2, 'networks': 4, 'layers': 1, 'machine': 4, 'based': 4, 'nervous': 1, 'state': 1, 'better': 1, '4': 1, '8': 2, 'pixel': 2, 'non': 1, 'advances': 1, 'facial': 2, 'using': 1, 'like': 1, 'semi': 2, 'level': 1, 'fields': 1, 'loosely': 1, 'shape': 2, 'large': 1, 'vector': 2, 'neuronal': 1, 'automatic': 1, 'vision': 1, 'set': 3, 'art': 1, 'methods': 2, 'intensity': 2, 'computer': 1, 'examples': 2, 'recognition': 6, 'responses': 1, 'shown': 1, 'scale': 1, 'ways': 2, 'per': 2, 'research': 1, 'replacing': 2, '3': 1, 'various': 3, '7': 2, 'linear': 1, 'processing': 3, 'represented': 2, 'g': 4, 'many': 2, 'inspired': 1, 'abstractions': 1, 'etc': 2, 'produce': 1, 'supervised': 2, 'expression': 2, 'otherwise': 1, 'composed': 1, 'tasks': 3, 'features': 2, 'family': 2, 'communication': 1, 'image': 2, 'coding': 1, 'natural': 1, 'one': 2, 'learning': 16, 'neuroscience': 1, 'transformations': 1, 'area': 1, 'create': 1, 'structured': 1, 'system': 1, 'extraction': 2, '2': 1, 'way': 2, '6': 1, 'structures': 1, 'define': 1, 'convolutional': 1, 'relationship': 1, 'hierarchical': 3, 'particular': 2, 'e': 4, 'applied': 1, 'language': 1, 'neural': 4, 'easier': 2, 'regions': 2, 'values': 2, 'learn': 3, 'promises': 2, 'associated': 1, 'abstract': 2, 'speech': 1, 'deep': 11, 'high': 1, 'information': 1, 'efficient': 2, 'make': 3, 'recurrent': 1, 'feature': 4, '1': 1, 'belief': 1, 'complex': 1, '5': 1, 'branch': 1, '9': 1, 'handcrafted': 2, 'multiple': 2, 'unlabeled': 1, 'models': 1, 'edges': 2, 'architectures': 1, 'bioinformatics': 1, 'representations': 7, 'data': 4, 'attempt': 1, 'observation': 2, 'unsupervised': 2, 'stimuli': 1, 'face': 2, 'patterns': 1, 'part': 2, 'algorithms': 3, 'model': 1, 'audio': 1})\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "word_freq=defaultdict(int)\n",
    "from itertools import groupby\n",
    "for text in texts:\n",
    "    for w in text:\n",
    "        word_freq[w]+=1\n",
    "print word_freq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('1', 1), ('2', 1), ('3', 1), ('4', 1), ('5', 1), ('6', 1), ('7', 2), ('8', 2), ('9', 1), ('abstract', 2), ('abstractions', 1), ('advances', 1), ('algorithms', 3), ('applied', 1), ('architectures', 1), ('area', 1), ('art', 1), ('associated', 1), ('attempt', 1), ('attempts', 2), ('audio', 1), ('automatic', 1), ('based', 4), ('belief', 1), ('better', 1), ('bioinformatics', 1), ('brain', 1), ('branch', 1), ('broader', 2), ('coding', 1), ('communication', 1), ('complex', 1), ('composed', 1), ('computer', 1), ('convolutional', 1), ('create', 1), ('data', 4), ('deep', 11), ('define', 1), ('e', 4), ('easier', 2), ('edges', 2), ('efficient', 2), ('etc', 2), ('examples', 2), ('expression', 2), ('extraction', 2), ('face', 2), ('facial', 2), ('family', 2), ('feature', 4), ('features', 2), ('fields', 1), ('g', 4), ('handcrafted', 2), ('hierarchical', 3), ('high', 1), ('image', 2), ('information', 1), ('inspired', 1), ('intensity', 2), ('interpretation', 1), ('language', 1), ('large', 1), ('layers', 1), ('learn', 3), ('learning', 16), ('level', 1), ('like', 1), ('linear', 1), ('loosely', 1), ('machine', 4), ('make', 3), ('many', 2), ('methods', 2), ('model', 1), ('models', 1), ('multiple', 2), ('natural', 1), ('nervous', 1), ('networks', 4), ('neural', 4), ('neuronal', 1), ('neuroscience', 1), ('non', 1), ('observation', 2), ('one', 2), ('otherwise', 1), ('part', 2), ('particular', 2), ('patterns', 1), ('per', 2), ('pixel', 2), ('processing', 3), ('produce', 1), ('promises', 2), ('recognition', 6), ('recurrent', 1), ('regions', 2), ('relationship', 1), ('replacing', 2), ('representations', 7), ('represented', 2), ('research', 1), ('responses', 1), ('results', 1), ('scale', 1), ('semi', 2), ('set', 3), ('shape', 2), ('shown', 1), ('speech', 1), ('state', 1), ('stimuli', 1), ('structured', 1), ('structures', 1), ('supervised', 2), ('system', 1), ('tasks', 3), ('transformations', 1), ('unlabeled', 1), ('unsupervised', 2), ('using', 1), ('values', 2), ('various', 3), ('vector', 2), ('vision', 1), ('way', 2), ('ways', 2)])\n"
     ]
    }
   ],
   "source": [
    "# Side note: OrderedDict in Python\n",
    "# Take a look at the documentation of the Python collections module: \n",
    "# https://docs.python.org/2/library/collections.html\n",
    "from collections import OrderedDict\n",
    "# dictionary sorted by key\n",
    "print OrderedDict(sorted(word_freq.items(), key=lambda t: t[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('learning', 16), ('deep', 11), ('representations', 7), ('recognition', 6), ('networks', 4), ('machine', 4), ('based', 4), ('g', 4), ('e', 4), ('neural', 4), ('feature', 4), ('data', 4), ('set', 3), ('various', 3), ('processing', 3), ('tasks', 3), ('hierarchical', 3), ('learn', 3), ('make', 3), ('algorithms', 3), ('attempts', 2), ('broader', 2), ('8', 2), ('pixel', 2), ('facial', 2), ('semi', 2), ('shape', 2), ('vector', 2), ('methods', 2), ('intensity', 2), ('examples', 2), ('ways', 2), ('per', 2), ('replacing', 2), ('7', 2), ('represented', 2), ('many', 2), ('etc', 2), ('supervised', 2), ('expression', 2), ('features', 2), ('family', 2), ('image', 2), ('one', 2), ('extraction', 2), ('way', 2), ('particular', 2), ('easier', 2), ('regions', 2), ('values', 2), ('promises', 2), ('abstract', 2), ('efficient', 2), ('handcrafted', 2), ('multiple', 2), ('edges', 2), ('observation', 2), ('unsupervised', 2), ('face', 2), ('part', 2), ('interpretation', 1), ('results', 1), ('brain', 1), ('layers', 1), ('nervous', 1), ('state', 1), ('better', 1), ('4', 1), ('non', 1), ('advances', 1), ('using', 1), ('like', 1), ('level', 1), ('fields', 1), ('loosely', 1), ('large', 1), ('neuronal', 1), ('automatic', 1), ('vision', 1), ('art', 1), ('computer', 1), ('responses', 1), ('shown', 1), ('scale', 1), ('research', 1), ('3', 1), ('linear', 1), ('inspired', 1), ('abstractions', 1), ('produce', 1), ('otherwise', 1), ('composed', 1), ('communication', 1), ('coding', 1), ('natural', 1), ('neuroscience', 1), ('transformations', 1), ('area', 1), ('create', 1), ('structured', 1), ('system', 1), ('2', 1), ('6', 1), ('structures', 1), ('define', 1), ('convolutional', 1), ('relationship', 1), ('applied', 1), ('language', 1), ('associated', 1), ('speech', 1), ('high', 1), ('information', 1), ('recurrent', 1), ('1', 1), ('belief', 1), ('complex', 1), ('5', 1), ('branch', 1), ('9', 1), ('unlabeled', 1), ('models', 1), ('architectures', 1), ('bioinformatics', 1), ('attempt', 1), ('stimuli', 1), ('patterns', 1), ('model', 1), ('audio', 1)])\n"
     ]
    }
   ],
   "source": [
    "# dictionary sorted by value, in reverse order\n",
    "print OrderedDict(sorted(word_freq.items(), key=lambda t: t[1], reverse=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('transformations', 1), ('representations', 7), ('interpretation', 1), ('bioinformatics', 1), ('communication', 1), ('convolutional', 1), ('architectures', 1), ('abstractions', 1), ('neuroscience', 1), ('relationship', 1), ('hierarchical', 3), ('unsupervised', 2), ('recognition', 6), ('represented', 2), ('information', 1), ('handcrafted', 2), ('observation', 2), ('processing', 3), ('supervised', 2), ('expression', 2), ('structured', 1), ('extraction', 2), ('structures', 1), ('particular', 2), ('associated', 1), ('algorithms', 3), ('automatic', 1), ('intensity', 2), ('responses', 1), ('replacing', 2), ('otherwise', 1), ('efficient', 2), ('recurrent', 1), ('unlabeled', 1), ('attempts', 2), ('networks', 4), ('advances', 1), ('neuronal', 1), ('computer', 1), ('examples', 2), ('research', 1), ('inspired', 1), ('composed', 1), ('features', 2), ('learning', 16), ('language', 1), ('promises', 2), ('abstract', 2), ('multiple', 2), ('patterns', 1), ('results', 1), ('broader', 2), ('machine', 4), ('nervous', 1), ('loosely', 1), ('methods', 2), ('various', 3), ('produce', 1), ('natural', 1), ('applied', 1), ('regions', 2), ('feature', 4), ('complex', 1), ('attempt', 1), ('stimuli', 1), ('layers', 1), ('better', 1), ('facial', 2), ('fields', 1), ('vector', 2), ('vision', 1), ('linear', 1), ('family', 2), ('coding', 1), ('create', 1), ('system', 1), ('define', 1), ('neural', 4), ('easier', 2), ('values', 2), ('speech', 1), ('belief', 1), ('branch', 1), ('models', 1), ('brain', 1), ('based', 4), ('state', 1), ('pixel', 2), ('using', 1), ('level', 1), ('shape', 2), ('large', 1), ('shown', 1), ('scale', 1), ('tasks', 3), ('image', 2), ('learn', 3), ('edges', 2), ('model', 1), ('audio', 1), ('like', 1), ('semi', 2), ('ways', 2), ('many', 2), ('area', 1), ('deep', 11), ('high', 1), ('make', 3), ('data', 4), ('face', 2), ('part', 2), ('non', 1), ('set', 3), ('art', 1), ('per', 2), ('etc', 2), ('one', 2), ('way', 2), ('4', 1), ('8', 2), ('3', 1), ('7', 2), ('g', 4), ('2', 1), ('6', 1), ('e', 4), ('1', 1), ('5', 1), ('9', 1)])\n"
     ]
    }
   ],
   "source": [
    "# dictionary sorted by length of the key string, in reverse order (So you get longer keys first)\n",
    "print OrderedDict(sorted(word_freq.items(), key=lambda t: len(t[0]), reverse=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[set(['algorithms',\n",
      "      'based',\n",
      "      'data',\n",
      "      'deep',\n",
      "      'hierarchical',\n",
      "      'learning',\n",
      "      'machine',\n",
      "      'multiple',\n",
      "      'processing',\n",
      "      'set'])]\n"
     ]
    }
   ],
   "source": [
    "# Let's remove words of freq < 2 and keep only unique words, using a set\n",
    "texts = [set([w for w in text if word_freq[w] > 1]) for text in texts]\n",
    "from pprint import pprint\n",
    "pprint(texts[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(60 unique tokens: [u'promises', u'set', u'features', u'family', u'image']...)\n"
     ]
    }
   ],
   "source": [
    "# Let's represent each document as a bag-of-words, where each word is assigned a unique integer id\\\n",
    "dictionary = corpora.Dictionary(texts)\n",
    "print dictionary\n",
    "# You can save this dictionary to desk for future reference, using gensim:\n",
    "# dictionary.save('/tmp/word_freq.dict') # "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'promises': 15, u'set': 0, u'features': 17, u'family': 12, u'image': 18, u'deep': 3, u'one': 19, u'shape': 20, u'tasks': 16, u'examples': 22, u'broader': 13, u'networks': 59, u'recognition': 23, u'methods': 14, u'regions': 28, u'based': 1, u'etc': 49, u'efficient': 21, u'make': 25, u'feature': 26, u'per': 27, u'machine': 4, u'extraction': 29, u'vector': 51, u'various': 57, u'supervised': 52, u'7': 30, u'8': 32, u'abstract': 34, u'handcrafted': 35, u'attempts': 56, u'multiple': 8, u'way': 36, u'replacing': 37, u'processing': 2, u'g': 44, u'hierarchical': 5, u'facial': 39, u'particular': 40, u'represented': 41, u'representations': 10, u'data': 9, u'values': 46, u'e': 42, u'observation': 43, u'semi': 31, u'unsupervised': 45, u'many': 33, u'edges': 38, u'neural': 58, u'intensity': 47, u'face': 48, u'ways': 24, u'easier': 50, u'part': 11, u'algorithms': 6, u'learning': 7, u'learn': 53, u'expression': 54, u'pixel': 55}\n"
     ]
    }
   ],
   "source": [
    "# You can get each word and its token id:\n",
    "print(dictionary.token2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(3, 2), (7, 2)]\n"
     ]
    }
   ],
   "source": [
    "# Let's add a new document and get a sparse vector of it using gensim's \"doc2bow\" \\\n",
    "# dictionary attribute:\n",
    "new_doc= \"Deep learning? I like deep learning a lot.\"\n",
    "tokenized_and_split_doc =tokenizer.tokenize(new_doc.lower())\n",
    "new_vec = dictionary.doc2bow(tokenized_and_split_doc) \n",
    "# Only the words deep (id 3) and learning (id 7)\n",
    "# occur in our previous dictionary, and each of these occur twice in this new document\n",
    "print new_vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# What does the sparse vector [(3, 2), (7, 2)] mean?\n",
    "# Well, all it means is that it has two words, \"deep\" and \"learning\", ids 3 and 7, respectively\n",
    "# and that each of them occurs twice in this new_vec vector. This should be clear to you by now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[(0, 1), (1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1), (8, 1), (9, 1)], [(1, 1), (3, 1), (4, 1), (7, 1), (9, 1), (10, 1), (11, 1), (12, 1), (13, 1), (14, 1)], [(0, 1), (3, 1), (5, 1), (6, 1), (7, 1), (10, 1), (15, 1), (16, 1), (17, 1), (18, 1), (19, 1), (20, 1), (21, 1), (22, 1), (23, 1), (24, 1), (25, 1), (26, 1), (27, 1), (28, 1), (29, 1), (30, 1), (31, 1), (32, 1), (33, 1), (34, 1), (35, 1), (36, 1), (37, 1), (38, 1), (39, 1), (40, 1), (41, 1), (42, 1), (43, 1), (44, 1), (45, 1), (46, 1), (47, 1), (48, 1), (49, 1), (50, 1), (51, 1), (52, 1), (53, 1), (54, 1), (55, 1)], [(1, 1), (3, 1), (4, 1), (7, 1), (9, 1), (10, 1), (11, 1), (12, 1), (13, 1), (14, 1)], [(0, 1), (18, 1), (20, 1), (24, 1), (27, 1), (28, 1), (33, 1), (34, 1), (36, 1), (38, 1), (40, 1), (41, 1), (42, 1), (43, 1), (44, 1), (46, 1), (47, 1), (49, 1), (51, 1), (55, 1)], [(10, 1), (16, 1), (22, 1), (23, 1), (25, 1), (30, 1), (39, 1), (42, 1), (44, 1), (48, 1), (50, 1), (53, 1), (54, 1)], [(3, 1), (5, 1), (6, 1), (7, 1), (15, 1), (17, 1), (19, 1), (21, 1), (26, 1), (29, 1), (31, 1), (32, 1), (35, 1), (37, 1), (45, 1), (52, 1)], [(9, 1), (10, 1), (25, 1), (53, 1), (56, 1)], [(1, 1), (2, 1), (10, 1), (56, 1), (57, 1), (58, 1)], [(2, 1), (3, 1), (7, 1), (16, 1), (23, 1), (57, 1), (58, 1), (59, 1)]]\n"
     ]
    }
   ],
   "source": [
    "# Then you get a sparse vector representation for each document.\n",
    "# Remember, each word is represented as an integer and the code \n",
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "print corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This is how you save the dict to desk for later use, using gensim:\n",
    "# corpora.MmCorpus.serialize('/tmp/dictionary.mm', corpus) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# To be continued"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Memory friendly iteration over a corpus using Python's \"yield\".\n",
    "# Function from tutorial at: https://radimrehurek.com/gensim/tut1.html\n",
    "class MyCorpus(object):\n",
    "    def __iter__(self):\n",
    "        for line in open('mycorpus.txt'):\n",
    "            # assume there's one document per line, tokens separated by whitespace\n",
    "            yield dictionary.doc2bow(line.lower().split())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Corpus Streaming & Formats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Take a look at the parts under the same names from the gensim tutorial at:\n",
    "# https://radimrehurek.com/gensim/tut1.html\n",
    "# You're now pretty much up and running with gensim. Congrats!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transformations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Let's get, e.g., a tf*idf (https://en.wikipedia.org/wiki/Tf%E2%80%93idf) transformation of a document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TfidfModel(num_docs=10, num_nnz=145)\n"
     ]
    }
   ],
   "source": [
    "tfidf = models.TfidfModel(corpus) # step 1 -- initialize a model\n",
    "print tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.31348023222449883), (1, 0.23857601299713876), (2, 0.31348023222449883), (3, 0.13300444544014492), (4, 0.31348023222449883), (5, 0.31348023222449883), (6, 0.31348023222449883), (7, 0.13300444544014492), (8, 0.5995275865658466), (9, 0.23857601299713876)]\n"
     ]
    }
   ],
   "source": [
    "# Now we can apply the transformation to the whole corpus:\n",
    "corpus_tfidf = tfidf[corpus]\n",
    "for doc in corpus_tfidf[:1]: # Only printing first document transformation\n",
    "    print doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'0.262*\"family\" + 0.262*\"broader\" + 0.262*\"methods\" + 0.262*\"part\" + 0.261*\"machine\" + 0.256*\"data\" + 0.248*\"based\" + 0.187*\"attempts\" + 0.173*\"processing\" + 0.170*\"representations\"']"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=2) # initialize an LSI transformation\n",
    "corpus_lsi = lsi[corpus_tfidf] # create a double wrapper over the original corpus: bow->tfidf->fold-in-lsi\n",
    "lsi.print_topics(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.48641308269852679), (1, 0.11579771159401867)]\n",
      "[(0, 0.71628142228181579), (1, 0.59038188262621183)]\n",
      "[(0, 0.56480415960439878), (1, -0.78758599492475101)]\n",
      "[(0, 0.71628142228181579), (1, 0.59038188262621183)]\n",
      "[(0, 0.33106538645556027), (1, -0.56392050838027818)]\n",
      "[(0, 0.35299506797325275), (1, -0.42198478055514865)]\n",
      "[(0, 0.33026858639927253), (1, -0.40718497786487473)]\n",
      "[(0, 0.37419828239791864), (1, -0.025902785063798527)]\n",
      "[(0, 0.39929537547357097), (1, 0.086715418602822544)]\n",
      "[(0, 0.32548779377635029), (1, -0.037026269019385064)]\n"
     ]
    }
   ],
   "source": [
    "for doc in corpus_lsi:\n",
    "    print doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}