Stemming vs Lemmatization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the toolkit and the full Porter Stemmer library\n",
    "import nltk\n",
    "\n",
    "from nltk.stem.porter import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_stemmer = PorterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = ['run','runner','running','ran','runs','easily','fairly','fairness']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "run --> run\n",
      "runner --> runner\n",
      "running --> run\n",
      "ran --> ran\n",
      "runs --> run\n",
      "easily --> easili\n",
      "fairly --> fairli\n",
      "fairness --> fair\n"
     ]
    }
   ],
   "source": [
    "for word in words:\n",
    "    print(word+' --> '+p_stemmer.stem(word))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " **Better Version**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.snowball import SnowballStemmer\n",
    "\n",
    "# The Snowball Stemmer requires that you pass a language parameter\n",
    "s_stemmer = SnowballStemmer(language='english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = ['run','runner','running','ran','runs','easily','fairly','fairness']\n",
    "# words = ['generous','generation','generously','generate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "run --> run\n",
      "runner --> runner\n",
      "running --> run\n",
      "ran --> ran\n",
      "runs --> run\n",
      "easily --> easili\n",
      "fairly --> fair\n",
      "fairness --> fair\n"
     ]
    }
   ],
   "source": [
    "for word in words:\n",
    "    print(word+' --> '+s_stemmer.stem(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "words=['generous','generation','generously','generate']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generous --> generous\n",
      "generation --> generat\n",
      "generously --> generous\n",
      "generate --> generat\n"
     ]
    }
   ],
   "source": [
    "for word in words:\n",
    "    print(word+' --> '+s_stemmer.stem(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Lemmatization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform standard imports:\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I \t PRON \t 561228191312463089 \t -PRON-\n",
      "am \t AUX \t 10382539506755952630 \t be\n",
      "a \t DET \t 11901859001352538922 \t a\n",
      "runner \t NOUN \t 12640964157389618806 \t runner\n",
      "running \t VERB \t 12767647472892411841 \t run\n",
      "in \t ADP \t 3002984154512732771 \t in\n",
      "a \t DET \t 11901859001352538922 \t a\n",
      "race \t NOUN \t 8048469955494714898 \t race\n",
      "because \t SCONJ \t 16950148841647037698 \t because\n",
      "I \t PRON \t 561228191312463089 \t -PRON-\n",
      "love \t VERB \t 3702023516439754181 \t love\n",
      "to \t PART \t 3791531372978436496 \t to\n",
      "run \t VERB \t 12767647472892411841 \t run\n",
      "since \t SCONJ \t 10066841407251338481 \t since\n",
      "I \t PRON \t 561228191312463089 \t -PRON-\n",
      "ran \t VERB \t 12767647472892411841 \t run\n",
      "today \t NOUN \t 11042482332948150395 \t today\n"
     ]
    }
   ],
   "source": [
    "doc1 = nlp(u\"I am a runner running in a race because I love to run since I ran today\")\n",
    "\n",
    "for token in doc1:\n",
    "    print(token.text, '\\t', token.pos_, '\\t', token.lemma, '\\t', token.lemma_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_lemmas(text):\n",
    "    for token in text:\n",
    "        print(f'{token.text:{12}} {token.pos_:{6}} {token.lemma:<{22}} {token.lemma_}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I            PRON   561228191312463089     -PRON-\n",
      "saw          VERB   11925638236994514241   see\n",
      "eighteen     NUM    9609336664675087640    eighteen\n",
      "mice         NOUN   1384165645700560590    mouse\n",
      "today        NOUN   11042482332948150395   today\n",
      "!            PUNCT  17494803046312582752   !\n"
     ]
    }
   ],
   "source": [
    "doc2 = nlp(u\"I saw eighteen mice today!\")\n",
    "\n",
    "show_lemmas(doc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I            PRON   561228191312463089     -PRON-\n",
      "am           AUX    10382539506755952630   be\n",
      "meeting      VERB   6880656908171229526    meet\n",
      "him          PRON   561228191312463089     -PRON-\n",
      "tomorrow     NOUN   3573583789758258062    tomorrow\n",
      "at           ADP    11667289587015813222   at\n",
      "the          DET    7425985699627899538    the\n",
      "meeting      NOUN   14798207169164081740   meeting\n",
      ".            PUNCT  12646065887601541794   .\n"
     ]
    }
   ],
   "source": [
    "doc3 = nlp(u\"I am meeting him tomorrow at the meeting.\")\n",
    "\n",
    "show_lemmas(doc3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "That         DET    4380130941430378203    that\n",
      "'s           AUX    10382539506755952630   be\n",
      "an           DET    15099054000809333061   an\n",
      "enormous     ADJ    17917224542039855524   enormous\n",
      "automobile   NOUN   7211811266693931283    automobile\n"
     ]
    }
   ],
   "source": [
    "doc4 = nlp(u\"That's an enormous automobile\")\n",
    "\n",
    "show_lemmas(doc4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Stop-words**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform standard imports:\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'hereafter', 'thru', 'mine', 'either', 'how', 'some', '‘d', 'these', 'sometimes', 'anything', 'seemed', '‘m', 'forty', 'seems', 'whereupon', 'their', 'been', \"'s\", 'whatever', 'from', 'fifty', 'empty', 'nine', 'whither', 'whose', 'whereas', 'became', 'were', 'hereupon', 'now', 'front', 'meanwhile', 'very', 'moreover', 'none', 'over', 'see', 'this', 'at', 'we', 'anyone', 'just', 'that', 'more', 'anyhow', 'call', '’d', 'it', 'beyond', 'no', 'has', 'about', 'by', 'so', 'among', 'sixty', 'yet', 'everything', 'do', 'and', 'with', '‘re', 'rather', 'namely', 'does', 'third', '’m', 'several', 'few', 'whereafter', 'last', 'whole', 'yours', \"'ll\", 'during', '‘ve', \"'m\", 'me', 'is', 'everyone', 'of', 'both', 'serious', 'however', 'else', 'whence', 'themselves', 'any', 'be', 'somehow', 'per', 'twelve', 'something', '‘s', 'make', 'herein', 'part', 'take', \"'ve\", 'done', 'although', 'yourselves', 'nothing', 'latter', 'along', 'indeed', 'to', 'will', 'also', 'between', 'say', 'thereupon', '’ll', 'top', 'up', 'six', 'please', 'anyway', 'in', 'than', 'through', 'therefore', 'there', 'really', 'ten', 'less', 'latterly', 'beforehand', 'i', 'sometime', 'together', 'enough', 'if', 'noone', 'until', 'alone', 'yourself', 'unless', 'n’t', 'might', 'fifteen', 'was', 'them', 'above', 'perhaps', 'its', 'ca', 'back', 'down', '’ve', 'did', 'four', 'thus', 'side', 'each', 'whether', 'elsewhere', 'first', 'cannot', 'move', 'even', 'those', 'our', 'again', 'every', 'ever', 'somewhere', 'under', 'must', 'his', 'never', 'full', '’s', \"'d\", 'as', 'myself', 'least', 'because', 'others', 're', 'since', 'when', 'already', 'whoever', 'eleven', 'mostly', 'why', 'for', 'after', 'thence', 'various', 'though', 'otherwise', 'not', 'who', 'should', 'put', 'except', 'nobody', 'wherein', 'before', 'formerly', 'bottom', 'used', 'name', 'get', 'still', 'us', '’re', 'regarding', 'herself', 'neither', 'much', 'behind', 'below', 'all', 'former', 'always', 'next', 'quite', 'upon', 'across', 'amongst', 'may', 'keep', 'wherever', 'almost', 'or', 'what', 'my', 'hundred', 'seem', 'within', 'where', 'anywhere', 'on', 'twenty', 'via', 'could', 'himself', 'own', 'they', 'eight', 'are', 'her', 'made', \"'re\", 'she', 'most', 'show', 'he', '‘ll', 'doing', 'using', 'once', 'throughout', 'becomes', 'but', 'beside', 'onto', 'the', 'whereby', 'nor', 'off', 'itself', 'whom', 'amount', 'toward', 'thereafter', 'him', 'towards', 'afterwards', 'two', 'ours', 'have', 'am', 'becoming', 'had', 'thereby', 'can', 'nowhere', 'your', 'many', 'you', 'around', 'such', 'therein', 'other', 'ourselves', 'nevertheless', 'seeming', 'a', 'only', \"n't\", 'same', 'then', 'here', 'hence', 'being', 'due', 'someone', 'another', 'five', 'hereby', 'become', 'well', 'everywhere', 'give', 'hers', 'often', 'would', 'without', 'n‘t', 'out', 'too', 'one', 'which', 'besides', 'three', 'an', 'further', 'go', 'into', 'whenever', 'while', 'against'}\n"
     ]
    }
   ],
   "source": [
    "# Print the set of spaCy's default stop words (remember that sets are unordered):\n",
    "print(nlp.Defaults.stop_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**To check whether a word is stop word or not**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.vocab['myself'].is_stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.vocab['mystery'].is_stop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**To add a stop word**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add the word to the set of stop words. Use lowercase!\n",
    "nlp.Defaults.stop_words.add('btw')\n",
    "\n",
    "# Set the stop_word tag on the lexeme\n",
    "nlp.vocab['btw'].is_stop = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "327"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nlp.Defaults.stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.vocab['btw'].is_stop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**To remove**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove the word from the set of stop words\n",
    "nlp.Defaults.stop_words.remove('beyond')\n",
    "\n",
    "# Remove the stop_word tag from the lexeme\n",
    "nlp.vocab['beyond'].is_stop = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "326"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nlp.Defaults.stop_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.vocab['beyond'].is_stop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Vocabulary and Matching**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform standard imports\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the Matcher library\n",
    "from spacy.matcher import Matcher\n",
    "matcher = Matcher(nlp.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern1 = [{'LOWER': 'solarpower'}]\n",
    "pattern2 = [{'LOWER': 'solar'}, {'LOWER': 'power'}]\n",
    "pattern3 = [{'LOWER': 'solar'}, {'IS_PUNCT': True}, {'LOWER': 'power'}]\n",
    "\n",
    "matcher.add('SolarPower', None, pattern1, pattern2, pattern3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(u'The Solar Power industry continues to grow as demand \\\n",
    "for solarpower increases. Solar-power cars are gaining popularity.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(8656102463236116519, 1, 3), (8656102463236116519, 10, 11), (8656102463236116519, 13, 16)]\n"
     ]
    }
   ],
   "source": [
    "found_matches = matcher(doc)\n",
    "print(found_matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8656102463236116519 SolarPower 1 3 Solar Power\n",
      "8656102463236116519 SolarPower 10 11 solarpower\n",
      "8656102463236116519 SolarPower 13 16 Solar-power\n"
     ]
    }
   ],
   "source": [
    "for match_id, start, end in found_matches:\n",
    "    string_id = nlp.vocab.strings[match_id]  # get string representation\n",
    "    span = doc[start:end]                    # get the matched span\n",
    "    print(match_id, string_id, start, end, span.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}