python_tutorial_part_2_nltk.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# NLTK Tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Introductory Examples for the NLTK Book ***\n",
      "Loading text1, ..., text9 and sent1, ..., sent9\n",
      "Type the name of the text or sentence to view it.\n",
      "Type: 'texts()' or 'sents()' to list the materials.\n",
      "text1: Moby Dick by Herman Melville 1851\n",
      "text2: Sense and Sensibility by Jane Austen 1811\n",
      "text3: The Book of Genesis\n",
      "text4: Inaugural Address Corpus\n",
      "text5: Chat Corpus\n",
      "text6: Monty Python and the Holy Grail\n",
      "text7: Wall Street Journal\n",
      "text8: Personals Corpus\n",
      "text9: The Man Who Was Thursday by G . K . Chesterton 1908\n"
     ]
    }
   ],
   "source": [
    "# Take a look at the preface here: http://www.nltk.org/book/ch00.html\n",
    "# This tutorial is based on Python 2.7, but it shouldn't be an issue to write the same code for Python 3 as the differences\n",
    "# are minimal so long as the tutorial is concerned\n",
    "import nltk\n",
    "from nltk.book import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 10 of 66 matches:\n",
      "r occupations may come . The Negroes are now Americans . Their ancestors came here years ago agains\n",
      "e it so or not . And yet we are not the less Americans on that account . We shall be the more Ameri\n",
      "we find them now secure ; and there comes to Americans the profound assurance that our representati\n",
      "have called me . I am certain that my fellow Americans expect that on my induction into the Preside\n",
      " and the hurricanes of disaster . In this we Americans were discovering no wholly new truth ; we we\n",
      " and that freedom is an ebbing tide . But we Americans know that this is not true . Eight years ago\n",
      "eat . We are not content to stand still . As Americans , we go forward , in the service of our coun\n",
      "uguration be simple and its words brief . We Americans of today , together with our allies , are pa\n",
      "in the discharge of this responsibility , we Americans know and we observe the difference between w\n",
      "cked bargain of trading honor for security . Americans , indeed all free men , remember that in the\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Let's look at the text4: Inaugural Address Corpus\n",
    "# NLTK can show a word in context, called a concordance (with a given text window size)\n",
    "# width: a parameter forthe window size of surrounding character context\n",
    "# lines: a parameter for the number of lines returned \n",
    "print(text4.concordance(\"Americans\", width=100, lines=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the free power opportunity fellow opinions colleges peace gangs\n",
      "judgments consent noblest ideas colors fidelity unquestionable worship\n",
      "discipline industrious just\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Other words that appear in a similar range of contexts as a given word\n",
    "print(text4.similar(\"patriotic\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "every_citizen our_citizens\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Let's look at common contexts of two words:\n",
    "print(text4.common_contexts([\"patriotic\", \"free\"])) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0669\n"
     ]
    }
   ],
   "source": [
    "# Lexical diversity shows the richness of a text's vocabulary:\n",
    "from __future__ import division # in Python 3 you don't need to do the import\n",
    "def lexical_diversity(text):\n",
    "    return len(set(text))/len(text)\n",
    "\n",
    "lex_div=lexical_diversity(text4)\n",
    "print(round(lex_div, 4))\n",
    "\n",
    "# What interesting ways can you use \"lexical_diversity\" for?\n",
    "# Can you play with some texts, say from presidential candidates and tell us what you find?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text=\"What interesting ways can you you use\".split()\n",
    "len(text)\n",
    "len(set(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'AS', u'Abandonment', u'Abhorring', u'About', u'Above', u'Abraham', u'Abroad', u'Accept', u'Across', u'Act', u'Acting', u'Action', u'Actual', u'Adams', u'Additional', u'Address', u'Administered', u'Administration', u'Administrations', u'Advance']\n"
     ]
    }
   ],
   "source": [
    "# sorted set of words\n",
    "print(sorted(set(text4))[100:120])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "['father', 'man', 'mother', 'woman']\n",
      "['father', 'man', 'mother', 'woman']\n"
     ]
    }
   ],
   "source": [
    "#P ay attenstion to the difference of these!\n",
    "tokens=[\"man\", \"woman\", \"father\", \"mother\"]\n",
    "x= tokens.sort() # Returns \"None\", but sorts the list in place\n",
    "print x\n",
    "print tokens\n",
    "print sorted(tokens) # Returns the sorted list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('man', 102)\n",
      "('woman', 3)\n",
      "('father', 4)\n",
      "('mother', 4)\n"
     ]
    }
   ],
   "source": [
    "# Counting word frequencies:\n",
    "words=[\"man\", \"woman\", \"father\", \"mother\"]\n",
    "for w in words:\n",
    "    print(w, text4.count(w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    " # We Stopped here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "****************************************************************************************************\n",
      "[(u'institutions', 76), (u'come', 75), (u'party', 75), (u'better', 75), (u'always', 74), (u'today', 74), (u'office', 73), (u'still', 73), (u'need', 73), (u'others', 73), (u'strength', 72), (u'Let', 72), (u'nor', 72), (u'itself', 72), (u'means', 70), (u'believe', 70), (u'themselves', 70), (u'place', 70), (u'land', 69), (u'could', 69), (u'then', 69), (u'.\"', 69), (u'home', 69), (u'equal', 69), (u'together', 68), (u'might', 68), (u'things', 67), (u'secure', 67), (u'Nation', 67), (u'whose', 66), (u'find', 66), (u'given', 66), (u'prosperity', 66), (u'Americans', 66), (u'old', 65), (u'am', 65), (u'full', 65), (u'give', 65), (u'here', 64), (u'Federal', 64), (u'action', 64), (u'order', 64), (u'yet', 64), (u'proper', 64), (u'found', 63), (u'up', 63), (u'important', 63), (u'responsibility', 63), (u'take', 62), (u'where', 62), (u'being', 62), (u'change', 62), (u'Executive', 62), (u'even', 62), (u'subject', 62), (u'administration', 61), (u'revenue', 61), (u'State', 61), (u'see', 60), (u'security', 60), (u'ought', 60), (u'trust', 60), (u'These', 60), (u'A', 59), (u'self', 59), (u'true', 59), (u'business', 59), (u'seek', 59), (u'character', 59), (u'honor', 59), (u'question', 59), (u'called', 59), (u'respect', 59), (u'commerce', 58), (u'cause', 58), (u'toward', 58), (u'principle', 58), (u'again', 58), (u'century', 58), (u'influence', 57), (u'become', 56), (u'protection', 56), (u'done', 56), (u'stand', 56), (u'course', 55), (u'another', 55), (u'very', 55), (u'help', 55), (u'like', 55), (u'citizen', 54), (u'authority', 54), (u'also', 53), (u'Republic', 53), (u'live', 53), (u'civil', 53), (u'past', 52), (u'sense', 52), (u'constitutional', 52), (u'meet', 52), (u'democracy', 52)]\n",
      "****************************************************************************************************\n",
      "14\n",
      "312\n",
      "****************************************************************************************************\n",
      "[u'than', u'country', u'.', u'has', u'people', u'for', u'citizens', u'time', u'so', u'nation']\n"
     ]
    }
   ],
   "source": [
    "# Frequency distribution\n",
    "freq_dist = FreqDist(text4) \n",
    "print(\"*\"*100)\n",
    "print(freq_dist.most_common(1000))[200:300]\n",
    "print(\"*\"*100)\n",
    "print(freq_dist[\"European\"])\n",
    "print(freq_dist[\"world\"])\n",
    "print(\"*\"*100)\n",
    "#------------------------------------------\n",
    "# Vocabulary\n",
    "V=set(text4)\n",
    "words=[w for w in V if freq_dist[w] > 200][:10]\n",
    "print(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l=[\"a\", \"b\"]\n",
    "l[0]\n",
    "\n",
    "d= {\"Hi\": 44, \"Hello\": 2}\n",
    "d[\"Hello\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "United States; fellow citizens; four years; years ago; Federal\n",
      "Government; General Government; American people; Vice President; Old\n",
      "World; Almighty God; Fellow citizens; Chief Magistrate; Chief Justice;\n",
      "God bless; every citizen; Indian tribes; public debt; one another;\n",
      "foreign nations; political parties\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Collocations\n",
    "print(text4.collocations())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'zodiac', u'zodiac', u'zogranda', u'zone', u'zoned', u'zoned', u'zones', u'zoology', u'zoology', u'zoroaster']\n",
      "[u'zephyr', u'zeuglodon', u'zig', u'zodiac', u'zogranda', u'zone', u'zoned', u'zones', u'zoology', u'zoroaster']\n"
     ]
    }
   ],
   "source": [
    "# Could you tell the difference?\n",
    "print(sorted(w.lower() for w in set(text1))[-10:])\n",
    "print(sorted(set(w.lower() for w in text1))[-10:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Hamlet: Entire Play\n",
      " \n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "The Tragedy of Hamlet, Prince of Denmark\n",
      "\n",
      "Shakespeare homepage \n",
      "    | Hamlet \n",
      "    | Entire play\n",
      "\n",
      "ACT I\n",
      "SCENE I. Elsinore. A platform before the castle.\n",
      "\n",
      "FRANCISCO at his post. Enter to him BERNARDO\n",
      "\n",
      "BERNARDO\n",
      "\n",
      "Who's there?\n",
      "\n",
      "FRANCISCO\n",
      "\n",
      "Nay, answer me: stand, and unfold you\n"
     ]
    }
   ],
   "source": [
    "# Fetching and cleaning a webpage:\n",
    "from urllib import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "url=\"http://shakespeare.mit.edu/hamlet/full.html\"\n",
    "page = urlopen(url)\n",
    "soup = BeautifulSoup(page.read())   \n",
    "raw = BeautifulSoup.get_text(soup)  \n",
    "print(raw[:300])\n",
    "tokens=nltk.word_tokenize(raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I', 'am', 'happy']\n"
     ]
    }
   ],
   "source": [
    "# Word tokenization with NLTK:\n",
    "import nltk\n",
    "raw=\"I am happy\"\n",
    "tokens=nltk.word_tokenize(raw)\n",
    "print tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# More on files, this time with NLTK:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<type 'list'>\n",
      "[u'The', u'Project', u'Gutenberg', u'EBook', u'of', u'Hamlet', u',', u'by', u'William', u'Shakespeare', u'This', u'eBook', u'is', u'for', u'the', u'use', u'of', u'anyone', u'anywhere', u'at']\n",
      "**************************************************\n",
      "[u'The', u'Project', u'Gutenberg', u'EBook', u'of', u'Hamlet', u',', u'by', u'William', u'Shakespeare', u'This', u'eBook', u'is', u'for', u'the', u'use', u'of', u'anyone', u'anywhere', u'at', u'no', u'cost', u'and', u'with', u'almost', u'no', u'restrictions', u'whatsoever', u'.', u'You', u'may', u'copy', u'it', u',', u'give', u'it', u'away', u'or', u're-use', u'it', u'under', u'the', u'terms', u'of', u'the', u'Project', u'Gutenberg', u'License', u'included', u'with']\n",
      "Project Gutenberg-tm; _1st Clo._; Project Gutenberg; _Crosses to_;\n",
      "Literary Archive; Gutenberg-tm electronic; Archive Foundation;\n",
      "electronic works; Gutenberg Literary; United States; _2nd Clo._;\n",
      "ROSENCRANTZ _and_; public domain; _and_ GUILDENSTERN; Dr. Johnson;\n",
      "_1st Play._; electronic work; _and_ Attendants; the_ KING; set forth\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "import codecs\n",
    "from nltk import word_tokenize, Text\n",
    "text_string=codecs.open(\"hamlet.txt\", \"r\", \"utf-8\").read() # Opens for reading and gets you the file content as a list\n",
    "tokens = word_tokenize(text_string)\n",
    "print(type(tokens))\n",
    "print(tokens[:20])\n",
    "text = Text(tokens)\n",
    "print(\"*\"*50)\n",
    "print(text[:50])\n",
    "print(text.collocations())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'[', u'The', u'Tragedie', u'of', u'Hamlet', u'by', u'William', u'Shakespeare', u'1599', u']']\n",
      "[u'Actus', u'Primus', u'.']\n",
      "[u'Scoena', u'Prima', u'.']\n",
      "[u'Enter', u'Barnardo', u'and', u'Francisco', u'two', u'Centinels', u'.']\n",
      "[u'Barnardo', u'.']\n"
     ]
    }
   ],
   "source": [
    "# Sentence splitting\n",
    "from nltk.corpus import gutenberg\n",
    "# This will return each sentence as a list of words\n",
    "hamlet_sent=gutenberg.sents('shakespeare-hamlet.txt')\n",
    "for sent in hamlet_sent[:5]:\n",
    "    print(sent)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'1789-Washington.txt', u'1793-Washington.txt', u'1797-Adams.txt', u'1801-Jefferson.txt', u'1805-Jefferson.txt']\n",
      "**************************************************\n",
      "[u'1789', u'1793', u'1797', u'1801', u'1805', u'1809', u'1813', u'1817', u'1821', u'1825', u'1829', u'1833', u'1837', u'1841', u'1845', u'1849', u'1853', u'1857', u'1861', u'1865', u'1869', u'1873', u'1877', u'1881', u'1885', u'1889', u'1893', u'1897', u'1901', u'1905', u'1909', u'1913', u'1917', u'1921', u'1925', u'1929', u'1933', u'1937', u'1941', u'1945', u'1949', u'1953', u'1957', u'1961', u'1965', u'1969', u'1973', u'1977', u'1981', u'1985', u'1989', u'1993', u'1997', u'2001', u'2005', u'2009']\n"
     ]
    }
   ],
   "source": [
    "# NLTK fileids:\n",
    "from nltk.corpus import inaugural\n",
    "print(inaugural.fileids()[:5])\n",
    "print(\"*\"*50)\n",
    "#print([fileid[:4] for fileid in inaugural.fileids()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "living creature that he said , and the land of the land of the land of the land of the land of the land of the land of the land None\n"
     ]
    }
   ],
   "source": [
    "# A function from the NLTK book: http://www.nltk.org/book/ch02.html\n",
    "def generate_model(cfdist, word, num=15):\n",
    "    for i in range(num):\n",
    "        print(word),\n",
    "        word = cfdist[word].max()\n",
    "\n",
    "text = nltk.corpus.genesis.words('english-kjv.txt')\n",
    "bigrams = nltk.bigrams(text)\n",
    "cfd = nltk.ConditionalFreqDist(bigrams)\n",
    "print(generate_model(cfd, 'living', num=30))\n",
    "#print(cfd[\"living\"].max())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WordNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Synset('nice.n.01'), Synset('nice.a.01'), Synset('decent.s.01'), Synset('nice.s.03'), Synset('dainty.s.04'), Synset('courteous.s.01')]\n",
      "**************************************************\n",
      "done with delicacy and skill\n",
      "[u'nice', u'skillful']\n",
      "**************************************************\n",
      "exhibiting courtesy and politeness\n",
      "[u'courteous', u'gracious', u'nice']\n",
      "**************************************************\n",
      "excessively fastidious and easily disgusted\n",
      "[u'dainty', u'nice', u'overnice', u'prissy', u'squeamish']\n"
     ]
    }
   ],
   "source": [
    "# WordNet is a very useful resource.\n",
    "# You should get familiar with its structure, and with ways to navigate it.\n",
    "# NLTK provides many off-the-shelf useful functions\n",
    "from nltk.corpus import wordnet as wn\n",
    "print(wn.synsets('nice'))\n",
    "print(\"*\"*50)\n",
    "print(wn.synset('nice.s.03').definition())\n",
    "print(wn.synset('nice.s.03').lemma_names())\n",
    "print(\"*\"*50)\n",
    "print(wn.synset('courteous.s.01').definition())\n",
    "print(wn.synset('courteous.s.01').lemma_names())\n",
    "print(\"*\"*50)\n",
    "print(wn.synset('dainty.s.04').definition())\n",
    "print(wn.synset('dainty.s.04').lemma_names())\n",
    "print(\"*\"*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**************************************************\n",
      "the act of drilling\n",
      "[u'drilling', u'boring']\n",
      "**************************************************\n",
      "the act of drilling a hole in the earth in the hope of producing petroleum\n",
      "[u'boring', u'drilling', u'oil_production']\n",
      "**************************************************\n",
      "cause to be bored\n",
      "[u'bore', u'tire']\n",
      "**************************************************\n",
      "make a hole, especially with a pointed power or hand tool\n",
      "[u'bore', u'drill']\n",
      "**************************************************\n",
      "so lacking in interest as to cause mental weariness\n",
      "[u'boring', u'deadening', u'dull', u'ho-hum', u'irksome', u'slow', u'tedious', u'tiresome', u'wearisome']\n"
     ]
    }
   ],
   "source": [
    "# Printing the definition and lemma names/lemmas of a given word is easily done in a \"for\" loop\n",
    "for synset in wn.synsets('boring'):\n",
    "    print(\"*\"*50)\n",
    "    print(synset.definition())\n",
    "    print(synset.lemma_names())\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Lemma('drilling.n.01.boring'), Lemma('boring.n.02.boring'), Lemma('boring.s.01.boring')]\n",
      "[Lemma('fantastic.s.02.wonderful')]\n",
      "[Lemma('dazzling.s.01.dazzling'), Lemma('blazing.s.01.dazzling')]\n"
     ]
    }
   ],
   "source": [
    "# You can access lemmas of a word directly, using the \"lemmas\" function:\n",
    "print(wn.lemmas('boring'))\n",
    "print(wn.lemmas('wonderful'))\n",
    "print(wn.lemmas('dazzling'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Lemma('dish.n.01.dish'), Lemma('dish.n.02.dish'), Lemma('dish.n.03.dish'), Lemma('smasher.n.02.dish'), Lemma('dish.n.05.dish'), Lemma('cup_of_tea.n.01.dish'), Lemma('serve.v.06.dish'), Lemma('dish.v.02.dish')]\n",
      "= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = \n",
      "**************************************************\n",
      "a piece of dishware normally used as a container for holding or serving food\n",
      "[u'dish']\n",
      "**************************************************\n",
      "a particular item of prepared food\n",
      "[u'dish']\n",
      "**************************************************\n",
      "the quantity that a dish will hold\n",
      "[u'dish', u'dishful']\n",
      "**************************************************\n",
      "a very attractive or seductive looking woman\n",
      "[u'smasher', u'stunner', u'knockout', u'beauty', u'ravisher', u'sweetheart', u'peach', u'lulu', u'looker', u'mantrap', u'dish']\n",
      "**************************************************\n",
      "directional antenna consisting of a parabolic reflector for microwave or radio frequency radiation\n",
      "[u'dish', u'dish_aerial', u'dish_antenna', u'saucer']\n",
      "**************************************************\n",
      "an activity that you like or at which you are superior\n",
      "[u'cup_of_tea', u'bag', u'dish']\n",
      "**************************************************\n",
      "provide (usually but not necessarily food)\n",
      "[u'serve', u'serve_up', u'dish_out', u'dish_up', u'dish']\n",
      "**************************************************\n",
      "make concave; shape like a dish\n",
      "[u'dish']\n"
     ]
    }
   ],
   "source": [
    "# Play with the word \"dish\"\n",
    "print(wn.lemmas('dish'))\n",
    "print(\"= \"*50)\n",
    "for synset in wn.synsets('dish'):\n",
    "    print(\"*\"*50)\n",
    "    print(synset.definition())\n",
    "    print(synset.lemma_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Happy.a.01: enjoying or showing or marked by joy or pleasure\n",
      "Felicitous.s.02: marked by good fortune\n",
      "Glad.s.02: eagerly disposed to act or to be of service\n",
      "Happy.s.04: well expressed and to the point\n",
      "Gladiolus.n.01: any of numerous plants of the genus Gladiolus native chiefly to tropical and South Africa having sword-shaped leaves and one-sided spikes of brightly colored funnel-shaped flowers; widely cultivated\n",
      "Glad.a.01: showing or causing joy and pleasure; especially made happy\n",
      "Glad.s.02: eagerly disposed to act or to be of service\n",
      "Glad.s.03: feeling happy appreciation\n",
      "Beaming.s.01: cheerful and bright\n",
      "Joyful.a.01: full of or producing joy\n",
      "Elated.s.02: full of high-spirited delight\n",
      "Joyous.a.01: full of or characterized by joy\n"
     ]
    }
   ],
   "source": [
    "# A function that prints the synsets and definitions of a given word:\n",
    "def get_definitions(word):\n",
    "    for synset in wn.synsets(word):\n",
    "        try:\n",
    "            print synset.name().capitalize() + ':', synset.definition() # capitalizing to give the feel of a dict entry\n",
    "        except:\n",
    "            continue\n",
    "            \n",
    "happy_words=[\"happy\", \"glad\", \"joyful\", \"joyous\", \"exhuberant\"]\n",
    "for w in happy_words:\n",
    "    get_definitions(w)\n",
    "\n",
    "# You can condition by a part of speech (POS), see the book!\n",
    "#for synset in wn.synsets('mint', wn.NOUN):\n",
    "#...     print(synset.name() + ':', synset.definition())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'happy']\n",
      "[u'felicitous', u'happy']\n",
      "[u'glad', u'happy']\n",
      "[u'happy', u'well-chosen']\n",
      "[u'gladiolus', u'gladiola', u'glad', u'sword_lily']\n",
      "[u'glad']\n",
      "[u'glad', u'happy']\n",
      "[u'glad']\n",
      "[u'beaming', u'glad']\n",
      "[u'joyful']\n",
      "[u'elated', u'gleeful', u'joyful', u'jubilant']\n",
      "[u'joyous']\n"
     ]
    }
   ],
   "source": [
    "# A function to print the lemma names of a passed word\n",
    "def get_lemma_names(word):\n",
    "    for synset in wn.synsets(word):\n",
    "        try:\n",
    "            print(synset.lemma_names())\n",
    "        except:\n",
    "            continue\n",
    "            \n",
    "happy_words=[\"happy\", \"glad\", \"joyful\", \"joyous\", \"exhuberant\"]\n",
    "for w in happy_words:\n",
    "    get_lemma_names(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set([u'felicitous', u'well-chosen', u'glad', u'happy'])\n",
      "set([u'gladiolus', u'beaming', u'sword_lily', u'gladiola', u'glad', u'happy'])\n",
      "set([u'elated', u'jubilant', u'joyful', u'gleeful'])\n",
      "set([u'joyous'])\n",
      "set([])\n",
      "**************************************************\n",
      "\n",
      "Here's a single unique list/set:\n",
      "\n",
      "set([u'elated', u'gladiolus', u'beaming', u'joyous', u'sword_lily', u'well-chosen', u'felicitous', u'jubilant', u'gleeful', u'gladiola', u'joyful', u'glad', u'happy'])\n"
     ]
    }
   ],
   "source": [
    "# As above, but we uniqify using a set.\n",
    "def get_unique_lemma_names(word):\n",
    "    l=[]\n",
    "    for synset in wn.synsets(word):\n",
    "        try:\n",
    "            l.extend(synset.lemma_names())\n",
    "        except:\n",
    "            continue\n",
    "    l=set(l)\n",
    "    return l\n",
    "\n",
    "happy_words=[\"happy\", \"glad\", \"joyful\", \"joyous\", \"exhuberant\"]\n",
    "for w in happy_words:\n",
    "    l=get_unique_lemma_names(w)\n",
    "    print(l)\n",
    "\n",
    "# To get a set\n",
    "print(\"*\"*50)\n",
    "print(\"\\nHere's a single unique list/set:\\n\")\n",
    "uniq_list=[]\n",
    "for w in happy_words:\n",
    "    l=get_unique_lemma_names(w)\n",
    "    uniq_list.extend(l)\n",
    "print(set(uniq_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Synset('ambulance.n.01')\n",
      "**************************************************\n",
      "[u'Model_T', u'S.U.V.', u'SUV', u'Stanley_Steamer', u'ambulance', u'beach_waggon', u'beach_wagon', u'bus', u'cab', u'compact', u'compact_car', u'convertible', u'coupe', u'cruiser', u'electric', u'electric_automobile', u'electric_car', u'estate_car', u'gas_guzzler', u'hack', u'hardtop', u'hatchback', u'heap', u'horseless_carriage', u'hot-rod', u'hot_rod', u'jalopy', u'jeep', u'landrover', u'limo', u'limousine', u'loaner', u'minicar', u'minivan', u'pace_car', u'patrol_car', u'phaeton', u'police_car', u'police_cruiser', u'prowl_car', u'race_car', u'racer', u'racing_car', u'roadster', u'runabout', u'saloon', u'secondhand_car', u'sedan', u'sport_car', u'sport_utility', u'sport_utility_vehicle', u'sports_car', u'squad_car', u'station_waggon', u'station_wagon', u'stock_car', u'subcompact', u'subcompact_car', u'taxi', u'taxicab', u'tourer', u'touring_car', u'two-seater', u'used-car', u'waggon', u'wagon']\n",
      "**************************************************\n",
      "ambulance beach_wagon station_wagon wagon estate_car beach_waggon station_waggon waggon bus jalopy heap cab hack taxi taxicab compact compact_car convertible coupe cruiser police_cruiser patrol_car police_car prowl_car squad_car electric electric_automobile electric_car gas_guzzler hardtop hatchback horseless_carriage hot_rod hot-rod jeep landrover limousine limo loaner minicar minivan Model_T pace_car racer race_car racing_car roadster runabout two-seater sedan saloon sport_utility sport_utility_vehicle S.U.V. SUV sports_car sport_car Stanley_Steamer stock_car subcompact subcompact_car touring_car phaeton tourer used-car secondhand_car\n"
     ]
    }
   ],
   "source": [
    "#Nice example from the book (http://www.nltk.org/book/ch02.html)\n",
    "motorcar = wn.synset('car.n.01')\n",
    "types_of_motorcar = motorcar.hyponyms()\n",
    "print(types_of_motorcar[0]) # prints: Synset('ambulance.n.01')\n",
    "print(\"*\"*50)\n",
    "print(sorted(lemma.name() for synset in types_of_motorcar for lemma in synset.lemmas()))\n",
    "print(\"*\"*50)\n",
    "\n",
    "# Remember, the tuple coprehension can be broken down as follows (with no sorting):\n",
    "for synset in types_of_motorcar:\n",
    "    for lemma in synset.lemmas():\n",
    "        print(lemma.name()),"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Synset('motor_vehicle.n.01')]\n",
      "2\n",
      "\n",
      "Path 1\n",
      "[u'entity.n.01', u'physical_entity.n.01', u'object.n.01', u'whole.n.02', u'artifact.n.01', u'instrumentality.n.03', u'container.n.01', u'wheeled_vehicle.n.01', u'self-propelled_vehicle.n.01', u'motor_vehicle.n.01', u'car.n.01']\n",
      "\n",
      "Path 2\n",
      "[u'entity.n.01', u'physical_entity.n.01', u'object.n.01', u'whole.n.02', u'artifact.n.01', u'instrumentality.n.03', u'conveyance.n.03', u'vehicle.n.01', u'wheeled_vehicle.n.01', u'self-propelled_vehicle.n.01', u'motor_vehicle.n.01', u'car.n.01']\n"
     ]
    }
   ],
   "source": [
    "# Another useful example, this time on hypernyms:\n",
    "motorcar = wn.synset('car.n.01')\n",
    "print(motorcar.hypernyms()) # prints: [Synset('motor_vehicle.n.01')]\n",
    "\n",
    "paths = motorcar.hypernym_paths()\n",
    "print(len(paths)) # prints 2 as there are two paths, as the book states, between car.n.01 and entity.n.01 \n",
    "                  # because wheeled_vehicle.n.01 can be classified as both a vehicle and a container.\n",
    "                  # Take a look at the output below\n",
    "\n",
    "print(\"\\nPath 1 between car.n.01 and entity.n.01\")\n",
    "print([synset.name() for synset in paths[0]])\n",
    "print(\"\\nPath 2 between car.n.01 and entity.n.01\")\n",
    "print([synset.name() for synset in paths[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Try the graphical WordNet browser from your command line:\n",
    "nltk.app.wordnet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Synset('lilac.n.01'), Synset('lavender.s.01')]\n",
      "[Synset('tulip.n.01')]\n",
      "[Synset('flower.n.01'), Synset('flower.n.02'), Synset('flower.n.03'), Synset('bloom.v.01')]\n",
      "[Synset('tree.n.01'), Synset('tree.n.02'), Synset('tree.n.03'), Synset('corner.v.02'), Synset('tree.v.02'), Synset('tree.v.03'), Synset('tree.v.04')]\n",
      "[Synset('daffodil.n.01')]\n",
      "**************************************************\n",
      "[Synset('flower.n.01')]\n",
      "[Synset('orchid.n.01')]\n",
      "[Synset('vascular_plant.n.01')]\n",
      "[Synset('vascular_plant.n.01')]\n"
     ]
    }
   ],
   "source": [
    "# Similarity\n",
    "from nltk.corpus import wordnet as wn\n",
    "print(wn.synsets('lilac'))\n",
    "print(wn.synsets('tulip'))\n",
    "print(wn.synsets('flower'))\n",
    "print(wn.synsets('tree'))\n",
    "print(wn.synsets('daffodil'))\n",
    "#--------------------------\n",
    "print(\"*\"*50)\n",
    "african = wn.synset('african_daisy.n.01')\n",
    "orchid = wn.synset('orchid.n.01')\n",
    "scarlet = wn.synset('scarlet_musk_flower.n.01')\n",
    "aster = wn.synset('white-topped_aster.n.01')\n",
    "tree = wn.synset('tree.n.01')\n",
    "daffodil = wn.synset('daffodil.n.01')\n",
    "#--------------------------\n",
    "print(\"*\"*50)\n",
    "print(african.lowest_common_hypernyms(orchid))\n",
    "print(orchid.lowest_common_hypernyms(orchid))\n",
    "print(scarlet.lowest_common_hypernyms(tree))\n",
    "print(aster.lowest_common_hypernyms(daffodil))\n",
    "#print(wn.synset('flower.n.01').hypernyms())\n",
    "#print(wn.synset('flower.n.01').hyponyms())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}