TTR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "view-in-github",
    "colab_type": "text"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/Ramaseshanr/ANLP/blob/master/TTR.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "zbBcZocGcVU1",
    "colab_type": "code",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52.0
    },
    "outputId": "6b01935e-4478-4271-c38d-3e92f6cfe987",
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3826 7079\nNumber of tokens, Vocabulary, Type-token ratio (Bryant stories) =  15000 2796 0.1864\nNumber of tokens, Vocabulary, Type-token ratio (Jane Austen Emma) =  15000 3274 0.21826666666666666\n"
     ]
    }
   ],
   "source": [
    "#  MIT License\n",
    "#  Copyright (c) 2019.\n",
    "#  Project Name: ANLP\n",
    "#  File name: code.py\n",
    "#  Last modification date: 2/28/19 2:32 PM\n",
    "#  Author: ramaseshan\n",
    "#  Email:ramaseshanr@yahoo.com\n",
    "#  /TTR.py\n",
    "# \n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "#get the stop words for English\n",
    "stop_words = set(stopwords.words('english'))\n",
    "words_bryant = nltk.Text(nltk.corpus.gutenberg.words('bryant-stories.txt'))\n",
    "words_emma = nltk.Text(nltk.corpus.gutenberg.words('austen-emma.txt'))\n",
    "#convert to small letters\n",
    "words_bryant = [word.lower() for word in words_bryant if word.isalpha()]\n",
    "words_emma = [word.lower() for word in words_emma if word.isalpha()]\n",
    "#remove stop words\n",
    "words_bryant = [word.lower() for word in words_bryant if word not in stop_words][:15000]\n",
    "words_emma = [word.lower() for word in words_emma if word not in stop_words][:15000]\n",
    "TTR_bryant = len(set(words_bryant))/len(words_bryant)\n",
    "TTR_emma = len(set(words_emma))/len(words_emma)\n",
    "print('Number of tokens, Vocabulary, Type-token ratio (Bryant stories) = ', len(words_bryant), len(set(words_bryant)), TTR_bryant)\n",
    "print('Number of tokens, Vocabulary, Type-token ratio (Jane Austen Emma) = ', len(words_emma), len(set(words_emma)), TTR_emma)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "TTR.ipynb",
   "version": "0.3.2",
   "provenance": [],
   "include_colab_link": true
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}