03_processing_raw_text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Raw Text\n",
    "\n",
    "## Accessing Text from the Web and from Disk\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import re\n",
    "import pprint\n",
    "from nltk import word_tokenize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*downloading **Crime and Punishment***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from urllib import request\n",
    "url = 'http://www.gutenberg.org/files/2554/2554.txt'\n",
    "response = request.urlopen(url)\n",
    "raw = response.read().decode('utf8')\n",
    "type(raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*number of characters:*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1176896"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The Project Gutenberg EBook of Crime and Punishment, by Fyodor Dostoevsky\\r\\n'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw[:75]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = word_tokenize(raw)\n",
    "type(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "254352"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The',\n",
       " 'Project',\n",
       " 'Gutenberg',\n",
       " 'EBook',\n",
       " 'of',\n",
       " 'Crime',\n",
       " 'and',\n",
       " 'Punishment',\n",
       " ',',\n",
       " 'by']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a **`Text`** object from `tokens`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nltk.text.Text"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = nltk.Text(tokens)\n",
    "type(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CHAPTER',\n",
       " 'I',\n",
       " 'On',\n",
       " 'an',\n",
       " 'exceptionally',\n",
       " 'hot',\n",
       " 'evening',\n",
       " 'early',\n",
       " 'in',\n",
       " 'July',\n",
       " 'a',\n",
       " 'young',\n",
       " 'man',\n",
       " 'came',\n",
       " 'out',\n",
       " 'of',\n",
       " 'the',\n",
       " 'garret',\n",
       " 'in',\n",
       " 'which',\n",
       " 'he',\n",
       " 'lodged',\n",
       " 'in',\n",
       " 'S.',\n",
       " 'Place',\n",
       " 'and',\n",
       " 'walked',\n",
       " 'slowly',\n",
       " ',',\n",
       " 'as',\n",
       " 'though',\n",
       " 'in',\n",
       " 'hesitation',\n",
       " ',',\n",
       " 'towards',\n",
       " 'K.',\n",
       " 'bridge',\n",
       " '.']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text[1024:1062]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "*find collocations (words that frequently appear together)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Katerina Ivanovna; Pyotr Petrovitch; Pulcheria Alexandrovna; Avdotya\n",
      "Romanovna; Rodion Romanovitch; Marfa Petrovna; Sofya Semyonovna; old\n",
      "woman; Project Gutenberg-tm; Porfiry Petrovitch; Amalia Ivanovna;\n",
      "great deal; Nikodim Fomitch; young man; Ilya Petrovitch; n't know;\n",
      "Project Gutenberg; Dmitri Prokofitch; Andrey Semyonovitch; Hay Market\n"
     ]
    }
   ],
   "source": [
    "text.collocations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Project Gutenberg* is a collocation for this text because it is included as a header and possibly footer for the raw text file\n",
    "\n",
    "find start and end manually using `find`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5338"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw.find('PART I')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "reverse find using `rfind`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1157746"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw.rfind(\"End of Project Gutenberg's Crime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw = raw[5338:1157746] # slightly different from NLTK Book value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw.find(\"PART I\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dealing with HTML\n",
    "\n",
    "Get an \"urban legend\" article called *Blondes to die out in 200 years* -- from the BBC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<!doctype html public \"-//W3C//DTD HTML 4.0 Transitional//EN'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = \"http://news.bbc.co.uk/2/hi/health/2284783.stm\"\n",
    "html = request.urlopen(url).read().decode('utf8')\n",
    "html[:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['BBC',\n",
       " 'NEWS',\n",
       " '|',\n",
       " 'Health',\n",
       " '|',\n",
       " 'Blondes',\n",
       " \"'to\",\n",
       " 'die',\n",
       " 'out',\n",
       " 'in',\n",
       " '200',\n",
       " \"years'\",\n",
       " 'NEWS',\n",
       " 'SPORT',\n",
       " 'WEATHER',\n",
       " 'WORLD',\n",
       " 'SERVICE',\n",
       " 'A-Z',\n",
       " 'INDEX',\n",
       " 'SEARCH',\n",
       " 'You',\n",
       " 'are',\n",
       " 'in',\n",
       " ':',\n",
       " 'Health',\n",
       " 'News',\n",
       " 'Front',\n",
       " 'Page',\n",
       " 'Africa',\n",
       " 'Americas',\n",
       " 'Asia-Pacific',\n",
       " 'Europe',\n",
       " 'Middle',\n",
       " 'East',\n",
       " 'South',\n",
       " 'Asia',\n",
       " 'UK',\n",
       " 'Business',\n",
       " 'Entertainment',\n",
       " 'Science/Nature',\n",
       " 'Technology',\n",
       " 'Health',\n",
       " 'Medical',\n",
       " 'notes',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--',\n",
       " '--']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "raw = BeautifulSoup(html, 'html.parser').get_text()\n",
    "tokens = word_tokenize(raw)\n",
    "tokens[:50]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*find start and end indices of the content (manually) and create a `Text` object*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Text: UK Blondes 'to die out in 200 years'...>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = tokens[110:390]\n",
    "text = nltk.Text(tokens)\n",
    "text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*get concordance of **gene*** -- shows occurrences of the word **gene**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 5 matches:\n",
      "hey say too few people now carry the gene for blondes to last beyond the next \n",
      "blonde hair is caused by a recessive gene . In order for a child to have blond\n",
      " have blonde hair , it must have the gene on both sides of the family in the g\n",
      "ere is a disadvantage of having that gene or by chance . They do n't disappear\n",
      "des would disappear is if having the gene was a disadvantage and I do not thin\n"
     ]
    }
   ],
   "source": [
    "text.concordance('gene')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*skipping these sections...*\n",
    "\n",
    "---\n",
    "### Processing Search Engine Results\n",
    "\n",
    "---\n",
    "### Processing RSS Feeds\n",
    "\n",
    "---\n",
    "### Reading Local Files\n",
    "\n",
    "---\n",
    "## Strings: Text Processing at the Lowest Level\n",
    "\n",
    "*skipping basic string and list operations*\n",
    "\n",
    "---\n",
    "## Text Processing with Unicode\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FileSystemPathPointer('/Users/sandip/nltk_data/corpora/unicode_samples/polish-lat2.txt')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = nltk.data.find('corpora/unicode_samples/polish-lat2.txt')\n",
    "path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruska Biblioteka Państwowa. Jej dawne zbiory znane pod nazwą\n",
      "\"Berlinka\" to skarb kultury i sztuki niemieckiej. Przewiezione przez\n",
      "Niemców pod koniec II wojny światowej na Dolny Śląsk, zostały\n",
      "odnalezione po 1945 r. na terytorium Polski. Trafiły do Biblioteki\n",
      "Jagiellońskiej w Krakowie, obejmują ponad 500 tys. zabytkowych\n",
      "archiwaliów, m.in. manuskrypty Goethego, Mozarta, Beethovena, Bacha.\n"
     ]
    }
   ],
   "source": [
    "with open(path, encoding='latin2') as f:\n",
    "    for line in f:\n",
    "        line_strip = line.strip()\n",
    "        print(line_strip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'Pruska Biblioteka Pa\\\\u0144stwowa. Jej dawne zbiory znane pod nazw\\\\u0105'\n",
      "b'\"Berlinka\" to skarb kultury i sztuki niemieckiej. Przewiezione przez'\n",
      "b'Niemc\\\\xf3w pod koniec II wojny \\\\u015bwiatowej na Dolny \\\\u015al\\\\u0105sk, zosta\\\\u0142y'\n",
      "b'odnalezione po 1945 r. na terytorium Polski. Trafi\\\\u0142y do Biblioteki'\n",
      "b'Jagiello\\\\u0144skiej w Krakowie, obejmuj\\\\u0105 ponad 500 tys. zabytkowych'\n",
      "b'archiwali\\\\xf3w, m.in. manuskrypty Goethego, Mozarta, Beethovena, Bacha.'\n"
     ]
    }
   ],
   "source": [
    "with open(path, encoding='latin2') as f:\n",
    "    for line in f:\n",
    "        line_strip = line.strip()\n",
    "        print(line_strip.encode('unicode_escape'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'Niemc\\\\xf3w pod koniec II wojny \\\\u015bwiatowej na Dolny \\\\u015al\\\\u0105sk, zosta\\\\u0142y\\\\n'\n"
     ]
    }
   ],
   "source": [
    "import unicodedata\n",
    "with open(path, encoding='latin2') as f:\n",
    "    lines = f.readlines()\n",
    "    \n",
    "line = lines[2]\n",
    "print(line.encode('unicode_escape'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'\\xc3\\xb3' U+00f3 LATIN SMALL LETTER O WITH ACUTE\n",
      "b'\\xc5\\x9b' U+015b LATIN SMALL LETTER S WITH ACUTE\n",
      "b'\\xc5\\x9a' U+015a LATIN CAPITAL LETTER S WITH ACUTE\n",
      "b'\\xc4\\x85' U+0105 LATIN SMALL LETTER A WITH OGONEK\n",
      "b'\\xc5\\x82' U+0142 LATIN SMALL LETTER L WITH STROKE\n"
     ]
    }
   ],
   "source": [
    "for c in line:\n",
    "    if ord(c) > 127:\n",
    "        print('{} U+{:04x} {}'.format(c.encode('utf8'), ord(c), unicodedata.name(c)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ó U+00f3 LATIN SMALL LETTER O WITH ACUTE\n",
      "ś U+015b LATIN SMALL LETTER S WITH ACUTE\n",
      "Ś U+015a LATIN CAPITAL LETTER S WITH ACUTE\n",
      "ą U+0105 LATIN SMALL LETTER A WITH OGONEK\n",
      "ł U+0142 LATIN SMALL LETTER L WITH STROKE\n"
     ]
    }
   ],
   "source": [
    "for c in line:\n",
    "    if ord(c) > 127:\n",
    "        print('{} U+{:04x} {}'.format(c, ord(c), unicodedata.name(c)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Using Python string methods and `re` with Unicode characters**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Niemców pod koniec II wojny światowej na Dolny Śląsk, zostały\\n'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "54"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "line.find('zosta\\u0142y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'niemców pod koniec ii wojny światowej na dolny śląsk, zostały\\n'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "line = line.lower()\n",
    "line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'niemc\\\\xf3w pod koniec ii wojny \\\\u015bwiatowej na dolny \\\\u015bl\\\\u0105sk, zosta\\\\u0142y\\\\n'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "line.encode('unicode_escape')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'światowej'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "m = re.search('\\u015b\\w*', line)\n",
    "m.group()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\\\u015bwiatowej'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m.group().encode('unicode_escape')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Can use Unicode strings with NLTK tokenizers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['niemców',\n",
       " 'pod',\n",
       " 'koniec',\n",
       " 'ii',\n",
       " 'wojny',\n",
       " 'światowej',\n",
       " 'na',\n",
       " 'dolny',\n",
       " 'śląsk',\n",
       " ',',\n",
       " 'zostały']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_tokenize(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regular Expressions for Detecting Word Patterns\n",
    "\n",
    "*skipping this section*\n",
    "\n",
    "cheatsheet:\n",
    "\n",
    "    Operator    Behavior\n",
    "    .        Wildcard, matches any character\n",
    "    ^abc     Matches some pattern abc at the start of a string\n",
    "    abc$     Matches some pattern abc at the end of a string\n",
    "    [abc]    Matches one of a set of characters\n",
    "    [A-Z0-9] Matches one of a range of characters\n",
    "    ed|ing|s Matches one of the specified strings (disjunction)\n",
    "    *        Zero or more of previous item, e.g. a*, [a-z]* (also known as Kleene Closure)\n",
    "    +        One or more of previous item, e.g. a+, [a-z]+\n",
    "    ?        Zero or one of the previous item (i.e. optional), e.g. a?, [a-z]?\n",
    "    {n}      Exactly n repeats where n is a non-negative integer\n",
    "    {n,}     At least n repeats\n",
    "    {,n}     No more than n repeats\n",
    "    {m,n}    At least m and no more than n repeats\n",
    "    a(b|c)+  Parentheses that indicate the scope of the operators\n",
    "\n",
    "---\n",
    "## Useful Applications of Regular Expressions\n",
    "\n",
    "## Extracting Word Pieces\n",
    "\n",
    "*find all vowels in a word and count them*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['u',\n",
       " 'e',\n",
       " 'a',\n",
       " 'i',\n",
       " 'a',\n",
       " 'i',\n",
       " 'i',\n",
       " 'i',\n",
       " 'e',\n",
       " 'i',\n",
       " 'a',\n",
       " 'i',\n",
       " 'o',\n",
       " 'i',\n",
       " 'o',\n",
       " 'u']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "word = 'supercalifragilisticexpialidocious'\n",
    "vowel_matches = re.findall(r'[aeiou]', word)\n",
    "vowel_matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vowel_matches)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Frequencies for sequences of 2+ vowels in the text*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12408"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wsj = sorted(set(nltk.corpus.treebank.words()))\n",
    "len(wsj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd = nltk.FreqDist(vowels for word in wsj\n",
    "                          for vowels in re.findall(r'[aeiou]{2,}', word))\n",
    "len(fd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('io', 549),\n",
       " ('ea', 476),\n",
       " ('ie', 331),\n",
       " ('ou', 329),\n",
       " ('ai', 261),\n",
       " ('ia', 253),\n",
       " ('ee', 217),\n",
       " ('oo', 174),\n",
       " ('ua', 109),\n",
       " ('au', 106),\n",
       " ('ue', 105),\n",
       " ('ui', 95)]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fd.most_common(12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Doing More with Word Pieces\n",
    "\n",
    "*Remove internal vowels from words*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "regexp = r'^[AEIOUaeiou]+|[AEIOUaeiou]+$|[^AEIOUaeiou]'\n",
    "\n",
    "def compress(word):\n",
    "    pieces = re.findall(regexp, word)\n",
    "    return ''.join(pieces)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['U', 'n', 'v', 'r', 's', 'l']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(regexp, 'Universal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unvrsl Dclrtn of Hmn Rghts Prmble Whrs rcgntn of the inhrnt dgnty and\n",
      "of the eql and inlnble rghts of all mmbrs of the hmn fmly is the fndtn\n",
      "of frdm , jstce and pce in the wrld , Whrs dsrgrd and cntmpt fr hmn\n",
      "rghts hve rsltd in brbrs acts whch hve outrgd the cnscnce of mnknd ,\n",
      "and the advnt of a wrld in whch hmn bngs shll enjy frdm of spch and\n"
     ]
    }
   ],
   "source": [
    "english_udhr = nltk.corpus.udhr.words('English-Latin1')\n",
    "print(nltk.tokenwrap(compress(w) for w in english_udhr[:75]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Extract consonant-vowel sequences from text*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ka',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'ro',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'vi',\n",
       " 'ko',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'vo',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'ko',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'si',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'ka',\n",
       " 'ko',\n",
       " 'ka']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rotokas_words = nltk.corpus.toolbox.words('rotokas.dic')\n",
    "cvs = [consonant_vowel for w in rotokas_words\n",
    "                       for consonant_vowel in re.findall(r'[ptksvr][aeiou]', w)]\n",
    "cvs[:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     a    e    i    o    u \n",
      "k  418  148   94  420  173 \n",
      "p   83   31  105   34   51 \n",
      "r  187   63   84   89   79 \n",
      "s    0    0  100    2    1 \n",
      "t   47    8    0  148   37 \n",
      "v   93   27  105   48   49 \n"
     ]
    }
   ],
   "source": [
    "cfd = nltk.ConditionalFreqDist(cvs)\n",
    "cfd.tabulate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*create an index such that:* `cv_index['su']` returns all words containing *su*\n",
    "\n",
    "Use **`nltk.Index()`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nltk.util.Index"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_word_pairs = [(cv, w) for w in rotokas_words\n",
    "                         for cv in re.findall(r'[ptksvr][aeiou]', w)]\n",
    "cv_index = nltk.Index(cv_word_pairs)\n",
    "type(cv_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['kasuari']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_index['su']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['kaapo',\n",
       " 'kaapopato',\n",
       " 'kaipori',\n",
       " 'kaiporipie',\n",
       " 'kaiporivira',\n",
       " 'kapo',\n",
       " 'kapoa',\n",
       " 'kapokao',\n",
       " 'kapokapo',\n",
       " 'kapokapo',\n",
       " 'kapokapoa',\n",
       " 'kapokapoa',\n",
       " 'kapokapora',\n",
       " 'kapokapora',\n",
       " 'kapokaporo',\n",
       " 'kapokaporo',\n",
       " 'kapokari',\n",
       " 'kapokarito',\n",
       " 'kapokoa',\n",
       " 'kapoo',\n",
       " 'kapooto',\n",
       " 'kapoovira',\n",
       " 'kapopaa',\n",
       " 'kaporo',\n",
       " 'kaporo',\n",
       " 'kaporopa',\n",
       " 'kaporoto',\n",
       " 'kapoto',\n",
       " 'karokaropo',\n",
       " 'karopo',\n",
       " 'kepo',\n",
       " 'kepoi',\n",
       " 'keposi',\n",
       " 'kepoto']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_index['po']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "### Finding Word Stems\n",
    "\n",
    "one simple approach that just removes suffixes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stem(word):\n",
    "    for suffix in ['ing', 'ly', 'ed', 'ious', 'ies', 'ive', 'es', 's', 'ment']:\n",
    "        if word.endswith(suffix):\n",
    "            return word[:-len(suffix)]\n",
    "    return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'walk'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stem('walking')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "alternative using `re` module..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stem_regexp(word):\n",
    "    regexp = r'^(.*?)(ing|ly|ed|ious|ies|ive|es|s|ment)?$'\n",
    "    stem, suffix = re.findall(regexp, word)[0]\n",
    "    return stem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DENNIS',\n",
       " ':',\n",
       " 'Listen',\n",
       " ',',\n",
       " 'strange',\n",
       " 'women',\n",
       " 'lying',\n",
       " 'in',\n",
       " 'ponds',\n",
       " 'distributing',\n",
       " 'swords',\n",
       " 'is',\n",
       " 'no',\n",
       " 'basis',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'government',\n",
       " '.',\n",
       " 'Supreme',\n",
       " 'executive',\n",
       " 'power',\n",
       " 'derives',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mandate',\n",
       " 'from',\n",
       " 'the',\n",
       " 'masses',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'some',\n",
       " 'farcical',\n",
       " 'aquatic',\n",
       " 'ceremony',\n",
       " '.']"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = \"\"\"DENNIS: Listen, strange women lying in ponds distributing swords\n",
    "is no basis for a system of government.  Supreme executive power derives from\n",
    "a mandate from the masses, not from some farcical aquatic ceremony.\"\"\"\n",
    "tokens = word_tokenize(raw)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DENNIS',\n",
       " ':',\n",
       " 'Listen',\n",
       " ',',\n",
       " 'strange',\n",
       " 'women',\n",
       " 'ly',\n",
       " 'in',\n",
       " 'pond',\n",
       " 'distribut',\n",
       " 'sword',\n",
       " 'i',\n",
       " 'no',\n",
       " 'basi',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'govern',\n",
       " '.',\n",
       " 'Supreme',\n",
       " 'execut',\n",
       " 'power',\n",
       " 'deriv',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mandate',\n",
       " 'from',\n",
       " 'the',\n",
       " 'mass',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'some',\n",
       " 'farcical',\n",
       " 'aquatic',\n",
       " 'ceremony',\n",
       " '.']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[stem_regexp(t) for t in tokens]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "### Searching Tokenized Text\n",
    "\n",
    "`\"<a> <man>\"` finds all instances of **a man** in the text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "monied; nervous; dangerous; white; white; white; pious; queer; good;\n",
      "mature; white; Cape; great; wise; wise; butterless; white; fiendish;\n",
      "pale; furious; better; certain; complete; dismasted; younger; brave;\n",
      "brave; brave; brave\n"
     ]
    }
   ],
   "source": [
    "from nltk.corpus import gutenberg, nps_chat\n",
    "moby = nltk.Text(gutenberg.words('melville-moby_dick.txt'))\n",
    "moby.findall(r\"<a> (<.*>) <man>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "you rule bro; telling you bro; u twizted bro\n"
     ]
    }
   ],
   "source": [
    "chat = nltk.Text(nps_chat.words())\n",
    "chat.findall(r\"<.*> <.*> <bro>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lol lol lol; lmao lol lol; lol lol lol; la la la la la; la la la; la\n",
      "la la; lovely lol lol love; lol lol lol.; la la la; la la la\n"
     ]
    }
   ],
   "source": [
    "chat.findall(r\"<l.*>{3,}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Normalizing Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DENNIS',\n",
       " ':',\n",
       " 'Listen',\n",
       " ',',\n",
       " 'strange',\n",
       " 'women',\n",
       " 'lying',\n",
       " 'in',\n",
       " 'ponds',\n",
       " 'distributing',\n",
       " 'swords',\n",
       " 'is',\n",
       " 'no',\n",
       " 'basis',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'government',\n",
       " '.',\n",
       " 'Supreme',\n",
       " 'executive',\n",
       " 'power',\n",
       " 'derives',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mandate',\n",
       " 'from',\n",
       " 'the',\n",
       " 'masses',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'some',\n",
       " 'farcical',\n",
       " 'aquatic',\n",
       " 'ceremony',\n",
       " '.']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = \"\"\"DENNIS: Listen, strange women lying in ponds distributing swords\n",
    "is no basis for a system of government.  Supreme executive power derives from\n",
    "a mandate from the masses, not from some farcical aquatic ceremony.\"\"\"\n",
    "tokens = word_tokenize(raw)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stemmers\n",
    "\n",
    "*\"off-the-shelf\" stemmers included in NLTK*\n",
    "* Porter\n",
    "* Lancaster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DENNI',\n",
       " ':',\n",
       " 'Listen',\n",
       " ',',\n",
       " 'strang',\n",
       " 'women',\n",
       " 'lie',\n",
       " 'in',\n",
       " 'pond',\n",
       " 'distribut',\n",
       " 'sword',\n",
       " 'is',\n",
       " 'no',\n",
       " 'basi',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'govern',\n",
       " '.',\n",
       " 'Suprem',\n",
       " 'execut',\n",
       " 'power',\n",
       " 'deriv',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mandat',\n",
       " 'from',\n",
       " 'the',\n",
       " 'mass',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'some',\n",
       " 'farcic',\n",
       " 'aquat',\n",
       " 'ceremoni',\n",
       " '.']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "porter = nltk.PorterStemmer()\n",
    "lancaster = nltk.LancasterStemmer()\n",
    "[porter.stem(t) for t in tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['den',\n",
       " ':',\n",
       " 'list',\n",
       " ',',\n",
       " 'strange',\n",
       " 'wom',\n",
       " 'lying',\n",
       " 'in',\n",
       " 'pond',\n",
       " 'distribut',\n",
       " 'sword',\n",
       " 'is',\n",
       " 'no',\n",
       " 'bas',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'govern',\n",
       " '.',\n",
       " 'suprem',\n",
       " 'execut',\n",
       " 'pow',\n",
       " 'der',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mand',\n",
       " 'from',\n",
       " 'the',\n",
       " 'mass',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'som',\n",
       " 'farc',\n",
       " 'aqu',\n",
       " 'ceremony',\n",
       " '.']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[lancaster.stem(t) for t in tokens]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Porter stemmer** correctly handled *lying* -> *lie* while **Lancaster stemmer** did not\n",
    "\n",
    "**Defining a custom `Text` class that uses the Porter Stemmer and can generate `concordance` for a text using *word stems***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class IndexedText(object):\n",
    "\n",
    "    def __init__(self, stemmer, text):\n",
    "        self._text = text\n",
    "        self._stemmer = stemmer\n",
    "        self._index = nltk.Index((self._stem(word), i)\n",
    "                                 for (i, word) in enumerate(text))\n",
    "\n",
    "    def concordance(self, word, width=40):\n",
    "        key = self._stem(word)\n",
    "        wc = int(width/4)                # words of context\n",
    "        for i in self._index[key]:\n",
    "            lcontext = ' '.join(self._text[i-wc:i])\n",
    "            rcontext = ' '.join(self._text[i:i+wc])\n",
    "            ldisplay = '{:>{width}}'.format(lcontext[-width:], width=width)\n",
    "            rdisplay = '{:{width}}'.format(rcontext[:width], width=width)\n",
    "            print(ldisplay, rdisplay)\n",
    "\n",
    "    def _stem(self, word):\n",
    "        return self._stemmer.stem(word).lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r king ! DENNIS : Listen , strange women lying in ponds distributing swords is no\n",
      " beat a very brave retreat . ROBIN : All lies ! MINSTREL : [ singing ] Bravest of\n",
      "       Nay . Nay . Come . Come . You may lie here . Oh , but you are wounded !   \n",
      "doctors immediately ! No , no , please ! Lie down . [ clap clap ] PIGLET : Well  \n",
      "ere is much danger , for beyond the cave lies the Gorge of Eternal Peril , which \n",
      "   you . Oh ... TIM : To the north there lies a cave -- the cave of Caerbannog --\n",
      "h it and lived ! Bones of full fifty men lie strewn about its lair . So , brave k\n",
      "not stop our fight ' til each one of you lies dead , and the Holy Grail returns t\n"
     ]
    }
   ],
   "source": [
    "porter = nltk.PorterStemmer()\n",
    "grail = nltk.corpus.webtext.words('grail.txt')\n",
    "text = IndexedText(porter, grail)\n",
    "text.concordance('lie')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lemmatization\n",
    "\n",
    "**WordNet lemmatizer** only removes affixes for words in its dictionary\n",
    "\n",
    "*dictionary lookup process is much slower than Porter stemmer*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DENNIS',\n",
       " ':',\n",
       " 'Listen',\n",
       " ',',\n",
       " 'strange',\n",
       " 'woman',\n",
       " 'lying',\n",
       " 'in',\n",
       " 'pond',\n",
       " 'distributing',\n",
       " 'sword',\n",
       " 'is',\n",
       " 'no',\n",
       " 'basis',\n",
       " 'for',\n",
       " 'a',\n",
       " 'system',\n",
       " 'of',\n",
       " 'government',\n",
       " '.',\n",
       " 'Supreme',\n",
       " 'executive',\n",
       " 'power',\n",
       " 'derives',\n",
       " 'from',\n",
       " 'a',\n",
       " 'mandate',\n",
       " 'from',\n",
       " 'the',\n",
       " 'mass',\n",
       " ',',\n",
       " 'not',\n",
       " 'from',\n",
       " 'some',\n",
       " 'farcical',\n",
       " 'aquatic',\n",
       " 'ceremony',\n",
       " '.']"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wnl = nltk.WordNetLemmatizer()\n",
    "[wnl.lemmatize(t) for t in tokens]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regular Expressions for Tokenizing Text\n",
    "\n",
    "### Simple Approaches to Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw = \"\"\"'When I'M a Duchess,' she said to herself, (not in a very hopeful tone\n",
    "though), 'I won't have any pepper in my kitchen AT ALL. Soup does very\n",
    "well without--Maybe it's always pepper that makes people hot-tempered,'...\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**split on whitespace**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"'When\",\n",
       " \"I'M\",\n",
       " 'a',\n",
       " \"Duchess,'\",\n",
       " 'she',\n",
       " 'said',\n",
       " 'to',\n",
       " 'herself,',\n",
       " '(not',\n",
       " 'in',\n",
       " 'a',\n",
       " 'very',\n",
       " 'hopeful',\n",
       " 'tone\\nthough),',\n",
       " \"'I\",\n",
       " \"won't\",\n",
       " 'have',\n",
       " 'any',\n",
       " 'pepper',\n",
       " 'in',\n",
       " 'my',\n",
       " 'kitchen',\n",
       " 'AT',\n",
       " 'ALL.',\n",
       " 'Soup',\n",
       " 'does',\n",
       " 'very\\nwell',\n",
       " 'without--Maybe',\n",
       " \"it's\",\n",
       " 'always',\n",
       " 'pepper',\n",
       " 'that',\n",
       " 'makes',\n",
       " 'people',\n",
       " \"hot-tempered,'...\"]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split(r' ', raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"'When\",\n",
       " \"I'M\",\n",
       " 'a',\n",
       " \"Duchess,'\",\n",
       " 'she',\n",
       " 'said',\n",
       " 'to',\n",
       " 'herself,',\n",
       " '(not',\n",
       " 'in',\n",
       " 'a',\n",
       " 'very',\n",
       " 'hopeful',\n",
       " 'tone',\n",
       " 'though),',\n",
       " \"'I\",\n",
       " \"won't\",\n",
       " 'have',\n",
       " 'any',\n",
       " 'pepper',\n",
       " 'in',\n",
       " 'my',\n",
       " 'kitchen',\n",
       " 'AT',\n",
       " 'ALL.',\n",
       " 'Soup',\n",
       " 'does',\n",
       " 'very',\n",
       " 'well',\n",
       " 'without--Maybe',\n",
       " \"it's\",\n",
       " 'always',\n",
       " 'pepper',\n",
       " 'that',\n",
       " 'makes',\n",
       " 'people',\n",
       " \"hot-tempered,'...\"]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split(r'[ \\t\\n]+', raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**`re`** offers `\\w` (word characters) and `\\W` (all characters except letters, digits, \\_ )\n",
    "\n",
    "**split on nonword characters:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " 'When',\n",
       " 'I',\n",
       " 'M',\n",
       " 'a',\n",
       " 'Duchess',\n",
       " 'she',\n",
       " 'said',\n",
       " 'to',\n",
       " 'herself',\n",
       " 'not',\n",
       " 'in',\n",
       " 'a',\n",
       " 'very',\n",
       " 'hopeful',\n",
       " 'tone',\n",
       " 'though',\n",
       " 'I',\n",
       " 'won',\n",
       " 't',\n",
       " 'have',\n",
       " 'any',\n",
       " 'pepper',\n",
       " 'in',\n",
       " 'my',\n",
       " 'kitchen',\n",
       " 'AT',\n",
       " 'ALL',\n",
       " 'Soup',\n",
       " 'does',\n",
       " 'very',\n",
       " 'well',\n",
       " 'without',\n",
       " 'Maybe',\n",
       " 'it',\n",
       " 's',\n",
       " 'always',\n",
       " 'pepper',\n",
       " 'that',\n",
       " 'makes',\n",
       " 'people',\n",
       " 'hot',\n",
       " 'tempered',\n",
       " '']"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.split(r'\\W+', raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**exclude empty strings...**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"'\",\n",
       " 'When',\n",
       " 'I',\n",
       " \"'\",\n",
       " 'M',\n",
       " 'a',\n",
       " 'Duchess',\n",
       " ',',\n",
       " \"'\",\n",
       " 'she',\n",
       " 'said',\n",
       " 'to',\n",
       " 'herself',\n",
       " ',',\n",
       " '(',\n",
       " 'not',\n",
       " 'in',\n",
       " 'a',\n",
       " 'very',\n",
       " 'hopeful',\n",
       " 'tone',\n",
       " 'though',\n",
       " ')',\n",
       " ',',\n",
       " \"'\",\n",
       " 'I',\n",
       " 'won',\n",
       " \"'\",\n",
       " 't',\n",
       " 'have',\n",
       " 'any',\n",
       " 'pepper',\n",
       " 'in',\n",
       " 'my',\n",
       " 'kitchen',\n",
       " 'AT',\n",
       " 'ALL',\n",
       " '.',\n",
       " 'Soup',\n",
       " 'does',\n",
       " 'very',\n",
       " 'well',\n",
       " 'without',\n",
       " '-',\n",
       " '-',\n",
       " 'Maybe',\n",
       " 'it',\n",
       " \"'\",\n",
       " 's',\n",
       " 'always',\n",
       " 'pepper',\n",
       " 'that',\n",
       " 'makes',\n",
       " 'people',\n",
       " 'hot',\n",
       " '-',\n",
       " 'tempered',\n",
       " ',',\n",
       " \"'\",\n",
       " '.',\n",
       " '.',\n",
       " '.']"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r'\\w+|\\Sw*', raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**allow internal hyphens and apostrophes in words**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"'\",\n",
       " 'When',\n",
       " \"I'M\",\n",
       " 'a',\n",
       " 'Duchess',\n",
       " ',',\n",
       " \"'\",\n",
       " 'she',\n",
       " 'said',\n",
       " 'to',\n",
       " 'herself',\n",
       " ',',\n",
       " '(',\n",
       " 'not',\n",
       " 'in',\n",
       " 'a',\n",
       " 'very',\n",
       " 'hopeful',\n",
       " 'tone',\n",
       " 'though',\n",
       " ')',\n",
       " ',',\n",
       " \"'\",\n",
       " 'I',\n",
       " \"won't\",\n",
       " 'have',\n",
       " 'any',\n",
       " 'pepper',\n",
       " 'in',\n",
       " 'my',\n",
       " 'kitchen',\n",
       " 'AT',\n",
       " 'ALL',\n",
       " '.',\n",
       " 'Soup',\n",
       " 'does',\n",
       " 'very',\n",
       " 'well',\n",
       " 'without',\n",
       " '--',\n",
       " 'Maybe',\n",
       " \"it's\",\n",
       " 'always',\n",
       " 'pepper',\n",
       " 'that',\n",
       " 'makes',\n",
       " 'people',\n",
       " 'hot-tempered',\n",
       " ',',\n",
       " \"'\",\n",
       " '...']"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.findall(r\"\\w+(?:[-']\\w+)*|'|[-.(\\)]+|\\S\\w*\", raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLTK's Regular Expression Tokenizer\n",
    "\n",
    "**`nltk.regexp_tokenize()`** is similar to **`re.findall()`** but more efficient -- don't need to treat parentheses as a special case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('', '', ''),\n",
       " ('A.', '', ''),\n",
       " ('', '-print', ''),\n",
       " ('', '', ''),\n",
       " ('', '', '.40'),\n",
       " ('', '', '')]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = 'That U.S.A. poster-print costs $12.40...'\n",
    "pattern = r'''(?x)    # set flag to allow verbose regexps\n",
    "    ([A-Z]\\.)+        # abbreviations, e.g. U.S.A.\n",
    "  | \\w+(-\\w+)*        # words with optional internal hyphens\n",
    "  | \\$?\\d+(\\.\\d+)?%?  # currency and percentages, e.g. $12.40, 82%\n",
    "  | \\.\\.\\.            # ellipsis\n",
    "  | [][.,;\"'?():-_`]  # these are separate tokens; includes ], [\n",
    "'''\n",
    "nltk.regexp_tokenize(text, pattern)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(?x) is a *verbose flag* -- strips out embedded whitespace and comments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Further Issues with Tokenization\n",
    "\n",
    "Important to have a \"gold standard\" for tokenization to compare performance of a custom tokenizer...\n",
    "\n",
    "NLTK Corpus includes Penn Treebank corpus, tokenized and raw text, for this purpose:\n",
    "\n",
    "**`nltk.corpus.treebank_raw.raw()`** and **`nltk.corpus.treebank.words()`**\n",
    "\n",
    "---\n",
    "\n",
    "## Segmentation\n",
    "\n",
    "**Tokenization** is a specific case of the more general **segmentation**\n",
    "\n",
    "### Sentence Segmentation\n",
    "\n",
    "**Average number of words per sentence:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20.250994070456922"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nltk.corpus.brown.words()) / len(nltk.corpus.brown.sents())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Segmenting a stream of characters into sentences: `sent_tokenize`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\"Nonsense!\"',\n",
      " 'said Gregory, who was very rational when anyone else\\nattempted paradox.',\n",
      " '\"Why do all the clerks and navvies in the\\n'\n",
      " 'railway trains look so sad and tired, so very sad and tired?',\n",
      " 'I will\\ntell you.',\n",
      " 'It is because they know that the train is going right.',\n",
      " 'It\\n'\n",
      " 'is because they know that whatever place they have taken a ticket\\n'\n",
      " 'for that place they will reach.',\n",
      " 'It is because after they have\\n'\n",
      " 'passed Sloane Square they know that the next station must be\\n'\n",
      " 'Victoria, and nothing but Victoria.',\n",
      " 'Oh, their wild rapture!',\n",
      " 'oh,\\n'\n",
      " 'their eyes like stars and their souls again in Eden, if the next\\n'\n",
      " 'station were unaccountably Baker Street!\"',\n",
      " '\"It is you who are unpoetical,\" replied the poet Syme.']\n"
     ]
    }
   ],
   "source": [
    "import pprint\n",
    "\n",
    "text = nltk.corpus.gutenberg.raw('chesterton-thursday.txt')\n",
    "sents = nltk.sent_tokenize(text)\n",
    "pprint.pprint(sents[79:89])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Formatting: From Lists to Strings\n",
    "\n",
    "*skipping basic python stuff*\n",
    "\n",
    "### From Lists to Strings\n",
    "\n",
    "*skipping -- basically just* `' '.join(iterable)`\n",
    "\n",
    "### Strings and Formats\n",
    "\n",
    "`str.format()`\n",
    "*skipping*\n",
    "\n",
    "### Writing Results to a File\n",
    "\n",
    "*skipping*\n",
    "\n",
    "### Text Wrapping\n",
    "\n",
    "*skipping*"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}