Text_Classification_Example.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tonyw54/ml-2/blob/main/Text_Classification_Example.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Zw5GWPYDonPz"
      },
      "source": [
        "# Text Classification\n",
        "## This notebook outlines the usage of NLP Feature extraction (CountVectorizer, TfidfVectorizer) in classification of text documents"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KKjcgmvBonP1"
      },
      "source": [
        "### Import all the necessary libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_ZybotqzonP1"
      },
      "outputs": [],
      "source": [
        "from pprint import pprint\n",
        "from time import time\n",
        "import logging\n",
        "\n",
        "from sklearn.datasets import fetch_20newsgroups\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.pipeline import Pipeline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BbXryCI7onP2"
      },
      "source": [
        "### Choose a few categories fro the entire 20 categories"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X-Dr3oWJonP2"
      },
      "outputs": [],
      "source": [
        "# Load some categories from the training set\n",
        "categories = [\n",
        "    'alt.atheism',\n",
        "    'talk.religion.misc',\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ezSkoG0monP2",
        "outputId": "1bb336c7-f1f0-4527-e2f3-ff81ab7f345d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loading 20 newsgroups dataset for categories:\n",
            "['alt.atheism', 'talk.religion.misc']\n"
          ]
        }
      ],
      "source": [
        "print(\"Loading 20 newsgroups dataset for categories:\")\n",
        "print(categories)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V6dIAV3ionP3"
      },
      "source": [
        "### Fetch documents for these 2 categories"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9X7lqYBUonP3",
        "outputId": "3a92e568-048f-4dbd-993e-6b8b325fa59e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "857"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "twenty_train = fetch_20newsgroups(subset='train', categories=categories, shuffle=True, random_state=42)\n",
        "len(twenty_train.data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p3uUoDOsonP3"
      },
      "source": [
        "### Define a pipeline combining a text feature extractor with a simple classifier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KTThb2megGlO"
      },
      "outputs": [],
      "source": [
        "pipeline = Pipeline([\n",
        "    ('vect', CountVectorizer()),\n",
        "    ('tfidf', TfidfTransformer()),\n",
        "    ('clf', SGDClassifier()),\n",
        "])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FuMDir3ponP3"
      },
      "source": [
        "### Specify parameter grid\n",
        "- 'vect__max_df': (0.5, 0.75, 1.0)\n",
        "- 'vect__max_features': (None, 5000, 10000, 50000)\n",
        "- 'vect__ngram_range': ((1, 1), (1, 2))\n",
        "- 'tfidf__use_idf': (True, False)\n",
        "- 'tfidf__norm': ('l1', 'l2')\n",
        "- 'clf__max_iter': (20,)\n",
        "- 'clf__alpha': (0.00001, 0.000001)\n",
        "- 'clf__penalty': ('l2', 'elasticnet')\n",
        "- 'clf__max_iter': (10, 50, 80)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lWkMQodVonP3"
      },
      "outputs": [],
      "source": [
        "params = {\n",
        "    'vect__max_df': (0.5, 0.75, 1.0),\n",
        "    'vect__max_features': (None, 5000, 10000, 50000),\n",
        "    'vect__ngram_range': ((1, 1), (1, 2)),\n",
        "    'tfidf__use_idf': (True, False),\n",
        "    'tfidf__norm': ('l1', 'l2'),\n",
        "    'clf__max_iter': (20,),\n",
        "    'clf__alpha': (0.00001, 0.000001),\n",
        "    'clf__penalty': ('l2', 'elasticnet'),\n",
        "    'clf__max_iter': (10, 50, 80)\n",
        "}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GFlju-r0onP3"
      },
      "source": [
        "### Find the best parameters for both the feature extraction and the classifier"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PB4SZF5donP3"
      },
      "source": [
        "### Build a GridSearch with the pipeline and parameter grid"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TKDW8lGeonP3"
      },
      "outputs": [],
      "source": [
        "grid_search = GridSearchCV(pipeline, params, cv=5, n_jobs=-1, verbose=1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XfIU9h67onP3"
      },
      "source": [
        "### Start the grid search"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "5bTONuwnonP3",
        "outputId": "ac476339-5e57-486f-f997-6d0c34dd6bf4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 5 folds for each of 1152 candidates, totalling 5760 fits\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_stochastic_gradient.py:702: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=5,\n",
              "             estimator=Pipeline(steps=[(&#x27;vect&#x27;, CountVectorizer()),\n",
              "                                       (&#x27;tfidf&#x27;, TfidfTransformer()),\n",
              "                                       (&#x27;clf&#x27;, SGDClassifier())]),\n",
              "             n_jobs=-1,\n",
              "             param_grid={&#x27;clf__alpha&#x27;: (1e-05, 1e-06),\n",
              "                         &#x27;clf__max_iter&#x27;: (10, 50, 80),\n",
              "                         &#x27;clf__penalty&#x27;: (&#x27;l2&#x27;, &#x27;elasticnet&#x27;),\n",
              "                         &#x27;tfidf__norm&#x27;: (&#x27;l1&#x27;, &#x27;l2&#x27;),\n",
              "                         &#x27;tfidf__use_idf&#x27;: (True, False),\n",
              "                         &#x27;vect__max_df&#x27;: (0.5, 0.75, 1.0),\n",
              "                         &#x27;vect__max_features&#x27;: (None, 5000, 10000, 50000),\n",
              "                         &#x27;vect__ngram_range&#x27;: ((1, 1), (1, 2))},\n",
              "             verbose=1)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=5,\n",
              "             estimator=Pipeline(steps=[(&#x27;vect&#x27;, CountVectorizer()),\n",
              "                                       (&#x27;tfidf&#x27;, TfidfTransformer()),\n",
              "                                       (&#x27;clf&#x27;, SGDClassifier())]),\n",
              "             n_jobs=-1,\n",
              "             param_grid={&#x27;clf__alpha&#x27;: (1e-05, 1e-06),\n",
              "                         &#x27;clf__max_iter&#x27;: (10, 50, 80),\n",
              "                         &#x27;clf__penalty&#x27;: (&#x27;l2&#x27;, &#x27;elasticnet&#x27;),\n",
              "                         &#x27;tfidf__norm&#x27;: (&#x27;l1&#x27;, &#x27;l2&#x27;),\n",
              "                         &#x27;tfidf__use_idf&#x27;: (True, False),\n",
              "                         &#x27;vect__max_df&#x27;: (0.5, 0.75, 1.0),\n",
              "                         &#x27;vect__max_features&#x27;: (None, 5000, 10000, 50000),\n",
              "                         &#x27;vect__ngram_range&#x27;: ((1, 1), (1, 2))},\n",
              "             verbose=1)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;vect&#x27;, CountVectorizer()), (&#x27;tfidf&#x27;, TfidfTransformer()),\n",
              "                (&#x27;clf&#x27;, SGDClassifier())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CountVectorizer</label><div class=\"sk-toggleable__content\"><pre>CountVectorizer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfTransformer</label><div class=\"sk-toggleable__content\"><pre>TfidfTransformer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SGDClassifier</label><div class=\"sk-toggleable__content\"><pre>SGDClassifier()</pre></div></div></div></div></div></div></div></div></div></div></div></div>"
            ],
            "text/plain": [
              "GridSearchCV(cv=5,\n",
              "             estimator=Pipeline(steps=[('vect', CountVectorizer()),\n",
              "                                       ('tfidf', TfidfTransformer()),\n",
              "                                       ('clf', SGDClassifier())]),\n",
              "             n_jobs=-1,\n",
              "             param_grid={'clf__alpha': (1e-05, 1e-06),\n",
              "                         'clf__max_iter': (10, 50, 80),\n",
              "                         'clf__penalty': ('l2', 'elasticnet'),\n",
              "                         'tfidf__norm': ('l1', 'l2'),\n",
              "                         'tfidf__use_idf': (True, False),\n",
              "                         'vect__max_df': (0.5, 0.75, 1.0),\n",
              "                         'vect__max_features': (None, 5000, 10000, 50000),\n",
              "                         'vect__ngram_range': ((1, 1), (1, 2))},\n",
              "             verbose=1)"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "grid_search.fit(twenty_train.data, twenty_train.target)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TiRUzwGKonP3"
      },
      "source": [
        "### Best Score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OwAPR02OonP3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2e1b7ce3-2c2a-47d3-c39f-377977b6602a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best score: 0.957\n"
          ]
        }
      ],
      "source": [
        "print(\"Best score: %0.3f\" % grid_search.best_score_)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FEwJ_kl4onP3"
      },
      "source": [
        "### Best Parameter"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7u-5b54IonP3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7142719e-8112-42d1-80a2-f0e22ca57a70"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Best parameters set:\n",
            "\tclf__alpha: 1e-06\n",
            "\tclf__max_iter: 10\n",
            "\tclf__penalty: 'elasticnet'\n",
            "\ttfidf__norm: 'l1'\n",
            "\ttfidf__use_idf: True\n",
            "\tvect__max_df: 1.0\n",
            "\tvect__max_features: 10000\n",
            "\tvect__ngram_range: (1, 2)\n"
          ]
        }
      ],
      "source": [
        "print(\"Best parameters set:\")\n",
        "best_parameters = grid_search.best_estimator_.get_params()\n",
        "for param_name in sorted(params.keys()):\n",
        "    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iMH5U0UKonP3"
      },
      "source": [
        "### Choose the best model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5WBIK43AonP3"
      },
      "outputs": [],
      "source": [
        "best_model = grid_search.best_estimator_"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VNZiNKzhonP4"
      },
      "source": [
        "### Use the model to classify a piece of text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MAY6cmOBonP4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6fe9abda-b91f-489e-a9cb-ee2e5e858ea1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "'God is love' => alt.atheism\n",
            "'God is the father of all' => talk.religion.misc\n",
            "'The universe began fourteen billion years ago' => alt.atheism\n",
            "'The universe began six thousand years ago' => talk.religion.misc\n"
          ]
        }
      ],
      "source": [
        "docs_new = ['God is love', 'God is the father of all', 'The universe began fourteen billion years ago', 'The universe began six thousand years ago']\n",
        "predicted = best_model.predict(docs_new)\n",
        "\n",
        "for doc, category in zip(docs_new, predicted):\n",
        "    print('%r => %s' % (doc, twenty_train.target_names[category]))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.12.1"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}