TopicModeling_LSI.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TopicModeling_LSI.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Ramaseshanr/anlp/blob/master/TopicModeling_LSI.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UQPKu_7IyqAo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 693
        },
        "outputId": "6e98f226-5d79-4e39-ac76-ba3e9db41253"
      },
      "source": [
        "import os\n",
        "import gensim\n",
        "from gensim.models import LsiModel\n",
        "from gensim import models\n",
        "from gensim import corpora\n",
        "from gensim.utils import lemmatize\n",
        "import nltk\n",
        "from nltk.stem import PorterStemmer\n",
        "from nltk.corpus import stopwords\n",
        "from gensim.parsing.preprocessing import remove_stopwords, stem_text\n",
        "from gensim.parsing.preprocessing import strip_numeric, strip_short,strip_multiple_whitespaces,strip_non_alphanum,strip_punctuation,strip_tags,preprocess_string\n",
        "import pandas as pd\n",
        "from gensim import similarities\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "from gensim.models.coherencemodel import CoherenceModel\n",
        "import matplotlib.pyplot as plt\n",
        "from pprint import pprint\n",
        "\n",
        "\n",
        "\n",
        "#read the data\n",
        "corpus_dir = 'https://raw.githubusercontent.com/Ramaseshanr/anlp/master/corpus/bbc-text.csv'\n",
        "df_corpus = pd.read_csv(corpus_dir,names=['category', 'text'])\n",
        "corpus = df_corpus['text'].values.tolist()\n",
        "corpus = corpus[1:]\n",
        "my_filter = [\n",
        "    lambda x: x.lower(), strip_tags, strip_punctuation,\n",
        "    strip_multiple_whitespaces, strip_numeric,\n",
        "    remove_stopwords, strip_short, stem_text\n",
        "]\n",
        "\n",
        "\n",
        "def preprocessing(corpus):\n",
        "\n",
        "    for document in corpus:\n",
        "        doc = strip_numeric(document)\n",
        "        doc = remove_stopwords(doc)\n",
        "        doc = strip_short(doc,3)\n",
        "        #doc = stem_text(doc)\n",
        "        doc = strip_punctuation(doc)\n",
        "        strip_tags(doc)\n",
        "        yield gensim.utils.tokenize(doc, lower=True)\n",
        "\n",
        "\n",
        "texts = preprocessing(corpus)\n",
        "dictionary = corpora.Dictionary(texts)\n",
        "dictionary.filter_extremes(no_below=1, keep_n=25000)\n",
        "\n",
        "doc_term_matrix = [dictionary.doc2bow(tokens) for tokens in preprocessing(corpus)]\n",
        "tfidf = models.TfidfModel(doc_term_matrix)\n",
        "corpus_tfidf = tfidf[doc_term_matrix]\n",
        "\n",
        "lsi = models.LsiModel(corpus_tfidf, id2word=dictionary)  # initialize an LSI transformation\n",
        "pprint(lsi.print_topics(num_topics=5, num_words=25))\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[(0,\n",
            "  '0.137*\"labour\" + 0.123*\"election\" + 0.118*\"blair\" + 0.110*\"brown\" + '\n",
            "  '0.096*\"government\" + 0.092*\"party\" + 0.092*\"people\" + 0.084*\"tax\" + '\n",
            "  '0.083*\"bn\" + 0.080*\"film\" + 0.074*\"economy\" + 0.073*\"music\" + '\n",
            "  '0.072*\"howard\" + 0.069*\"new\" + 0.068*\"minister\" + 0.068*\"mobile\" + '\n",
            "  '0.068*\"best\" + 0.067*\"england\" + 0.067*\"game\" + 0.065*\"chancellor\" + '\n",
            "  '0.065*\"growth\" + 0.061*\"world\" + 0.061*\"games\" + 0.061*\"prime\" + '\n",
            "  '0.059*\"economic\"'),\n",
            " (1,\n",
            "  '-0.286*\"labour\" + -0.250*\"election\" + -0.238*\"blair\" + -0.218*\"brown\" + '\n",
            "  '-0.178*\"party\" + -0.146*\"tax\" + -0.136*\"chancellor\" + -0.128*\"howard\" + '\n",
            "  '0.120*\"film\" + -0.116*\"tory\" + -0.115*\"prime\" + -0.105*\"tories\" + '\n",
            "  '-0.104*\"minister\" + -0.101*\"government\" + 0.097*\"music\" + 0.095*\"game\" + '\n",
            "  '0.091*\"best\" + -0.090*\"lib\" + 0.089*\"mobile\" + -0.086*\"kennedy\" + '\n",
            "  '0.084*\"games\" + -0.074*\"leader\" + 0.072*\"england\" + -0.071*\"campaign\" + '\n",
            "  '-0.071*\"taxes\"'),\n",
            " (2,\n",
            "  '-0.148*\"england\" + 0.146*\"mobile\" + 0.116*\"growth\" + 0.111*\"bn\" + '\n",
            "  '-0.103*\"wales\" + -0.099*\"game\" + -0.098*\"win\" + -0.098*\"best\" + '\n",
            "  '0.095*\"economy\" + 0.093*\"sales\" + -0.092*\"ireland\" + 0.088*\"broadband\" + '\n",
            "  '0.088*\"market\" + -0.087*\"robinson\" + 0.086*\"phone\" + -0.085*\"rugby\" + '\n",
            "  '-0.083*\"cup\" + 0.082*\"digital\" + 0.081*\"china\" + -0.081*\"film\" + '\n",
            "  '0.080*\"oil\" + 0.079*\"yukos\" + -0.078*\"blair\" + 0.077*\"bank\" + '\n",
            "  '0.077*\"music\"'),\n",
            " (3,\n",
            "  '-0.314*\"film\" + -0.174*\"best\" + -0.151*\"awards\" + 0.149*\"england\" + '\n",
            "  '-0.125*\"award\" + -0.110*\"music\" + -0.109*\"oscar\" + 0.103*\"growth\" + '\n",
            "  '-0.101*\"actor\" + 0.101*\"economy\" + -0.095*\"actress\" + 0.092*\"yukos\" + '\n",
            "  '-0.090*\"aviator\" + 0.090*\"bn\" + 0.089*\"ireland\" + 0.089*\"wales\" + '\n",
            "  '-0.087*\"films\" + -0.087*\"festival\" + 0.085*\"oil\" + 0.080*\"bank\" + '\n",
            "  '0.076*\"robinson\" + -0.075*\"comedy\" + 0.074*\"economic\" + -0.074*\"nominated\" '\n",
            "  '+ -0.073*\"nominations\"'),\n",
            " (4,\n",
            "  '0.228*\"film\" + -0.185*\"mobile\" + 0.147*\"best\" + 0.132*\"bn\" + 0.131*\"dollar\" '\n",
            "  '+ 0.125*\"economy\" + 0.118*\"m\" + 0.118*\"yukos\" + 0.117*\"growth\" + '\n",
            "  '-0.114*\"phone\" + 0.111*\"oil\" + 0.109*\"awards\" + -0.099*\"phones\" + '\n",
            "  '-0.099*\"digital\" + 0.097*\"bank\" + -0.095*\"music\" + -0.094*\"microsoft\" + '\n",
            "  '-0.093*\"users\" + -0.092*\"broadband\" + 0.091*\"economic\" + -0.091*\"people\" + '\n",
            "  '0.091*\"award\" + -0.091*\"technology\" + 0.090*\"oscar\" + -0.084*\"software\"')]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}