About_Us_Classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "About Us Classifier.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/edeediong/hotel-page-classifier/blob/master/About_Us_Classifier.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "rZV8FNHgqu39",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Data Preprocessing\n"
      ]
    },
    {
      "metadata": {
        "id": "kAaC5_16KsZg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1351
        },
        "outputId": "3738ee2d-cc60-4d12-feaf-5631a8478e61"
      },
      "cell_type": "code",
      "source": [
        "import pandas as  pd\n",
        "documents = pd.read_csv('data.csv',names=['Urls','Labels'],header = None)\n",
        "documents"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Urls</th>\n",
              "      <th>Labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>http://bothot.co.za/contact-us/</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>http://bothot.co.za/about/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>http://botsesafaris.com/hunting</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>http://botsesafaris.com/about-us</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>https://www.bouldersbeachlodge.com/boulders-be...</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>http://bowenarrowmotel.com/testimonials</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>http://bridgefunctions.co.za/Pages/About-Us.asp</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>https://www.breezeguesthouse.co.za/overnight-a...</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>http://bowenarrowmotel.com/about</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>http://broomsestatelodge.co.za/about/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>http://bridgefunctions.co.za/Pages/Gallery.asp</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>http://thebroady.com.au/#our-menu</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>http://chanteclerhotel.co.za/about/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>https://www.ciaralodge.co.za/about-us</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>http://broomsestatelodge.co.za/gallery/</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>http://www.clarksinn.in/about-us/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>http://www.buitenverwagten.co.za/karoo-blog.html</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>https://clico.co.za/about/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>http://buffelshoekdirosielodge.co.za/gallery.php</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>http://bunclody.co.za/Rooms/standard.html</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>https://produvia.com/about-us.html</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>http://www.cchotel.co.za/contact/</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>http://absolutebeach.co.za/contact-2-2/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>https://www.buydomains.com/about-buydomains/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>https://www.w3.org/Consortium/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>https://www.adobe.com/africa/about-adobe.html?...</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>https://wordpress.com/about/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>https://github.com/about</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>https://statcounter.com/support/contact/</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>https://statcounter.com/careers/</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>https://www.jimdo.com/about-jimdo/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>https://www.jimdo.com/about-jimdo/press/</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>https://www.weebly.com/about</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>https://www.weebly.com/contact</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>https://www.addthis.com/about/</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>https://www.addthis.com/careers/</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>https://www.wix.com/blog/</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>https://help.dropbox.com/security/cookies</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>https://www.wix.com/about/us</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>https://www.dropbox.com/developers</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>https://www.dropbox.com/about</td>\n",
              "      <td>about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>https://www.msn.com/en-xl/africa/nigeria/us-re...</td>\n",
              "      <td>others</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 Urls  Labels\n",
              "0                     http://bothot.co.za/contact-us/  others\n",
              "1                          http://bothot.co.za/about/   about\n",
              "2                     http://botsesafaris.com/hunting  others\n",
              "3                    http://botsesafaris.com/about-us   about\n",
              "4   https://www.bouldersbeachlodge.com/boulders-be...  others\n",
              "5             http://bowenarrowmotel.com/testimonials  others\n",
              "6     http://bridgefunctions.co.za/Pages/About-Us.asp   about\n",
              "7   https://www.breezeguesthouse.co.za/overnight-a...  others\n",
              "8                    http://bowenarrowmotel.com/about   about\n",
              "9               http://broomsestatelodge.co.za/about/   about\n",
              "10     http://bridgefunctions.co.za/Pages/Gallery.asp  others\n",
              "11                  http://thebroady.com.au/#our-menu  others\n",
              "12                http://chanteclerhotel.co.za/about/   about\n",
              "13              https://www.ciaralodge.co.za/about-us   about\n",
              "14            http://broomsestatelodge.co.za/gallery/  others\n",
              "15                  http://www.clarksinn.in/about-us/   about\n",
              "16   http://www.buitenverwagten.co.za/karoo-blog.html  others\n",
              "17                         https://clico.co.za/about/   about\n",
              "18   http://buffelshoekdirosielodge.co.za/gallery.php  others\n",
              "19          http://bunclody.co.za/Rooms/standard.html  others\n",
              "20                 https://produvia.com/about-us.html   about\n",
              "21                  http://www.cchotel.co.za/contact/  others\n",
              "22            http://absolutebeach.co.za/contact-2-2/   about\n",
              "23       https://www.buydomains.com/about-buydomains/   about\n",
              "24                     https://www.w3.org/Consortium/   about\n",
              "25  https://www.adobe.com/africa/about-adobe.html?...   about\n",
              "26                       https://wordpress.com/about/   about\n",
              "27                           https://github.com/about   about\n",
              "28           https://statcounter.com/support/contact/  others\n",
              "29                   https://statcounter.com/careers/  others\n",
              "30                 https://www.jimdo.com/about-jimdo/   about\n",
              "31           https://www.jimdo.com/about-jimdo/press/  others\n",
              "32                       https://www.weebly.com/about   about\n",
              "33                     https://www.weebly.com/contact  others\n",
              "34                     https://www.addthis.com/about/   about\n",
              "35                   https://www.addthis.com/careers/  others\n",
              "36                          https://www.wix.com/blog/  others\n",
              "37          https://help.dropbox.com/security/cookies  others\n",
              "38                       https://www.wix.com/about/us   about\n",
              "39                 https://www.dropbox.com/developers  others\n",
              "40                      https://www.dropbox.com/about   about\n",
              "41  https://www.msn.com/en-xl/africa/nigeria/us-re...  others"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "metadata": {
        "id": "Qev4AnQhYz76",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import re\n",
        "ignore_words = ['http','https','www','com','xml','php','xl','asp','co','za','html','php']\n",
        "def extract_words(sentence):\n",
        "    words = re.sub(\"[^\\w]\", \" \",  sentence).split() #nltk.word_tokenize(sentence)\n",
        "    words_cleaned = [w.lower() for w in words if w not in ignore_words]\n",
        "    return words_cleaned    \n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dP0IsZqHMA70",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1351
        },
        "outputId": "cc6a4989-faf7-426b-bfae-6cab39bf517b"
      },
      "cell_type": "code",
      "source": [
        "documents['Tokens'] = documents['Urls'].apply(lambda x: extract_words(x))\n",
        "\n",
        "documents"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Urls</th>\n",
              "      <th>Labels</th>\n",
              "      <th>Tokens</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>http://bothot.co.za/contact-us/</td>\n",
              "      <td>others</td>\n",
              "      <td>[bothot, contact, us]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>http://bothot.co.za/about/</td>\n",
              "      <td>about</td>\n",
              "      <td>[bothot, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>http://botsesafaris.com/hunting</td>\n",
              "      <td>others</td>\n",
              "      <td>[botsesafaris, hunting]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>http://botsesafaris.com/about-us</td>\n",
              "      <td>about</td>\n",
              "      <td>[botsesafaris, about, us]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>https://www.bouldersbeachlodge.com/boulders-be...</td>\n",
              "      <td>others</td>\n",
              "      <td>[bouldersbeachlodge, boulders, beach, accommod...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>http://bowenarrowmotel.com/testimonials</td>\n",
              "      <td>others</td>\n",
              "      <td>[bowenarrowmotel, testimonials]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>http://bridgefunctions.co.za/Pages/About-Us.asp</td>\n",
              "      <td>about</td>\n",
              "      <td>[bridgefunctions, pages, about, us]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>https://www.breezeguesthouse.co.za/overnight-a...</td>\n",
              "      <td>others</td>\n",
              "      <td>[breezeguesthouse, overnight, accommodation, v...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>http://bowenarrowmotel.com/about</td>\n",
              "      <td>about</td>\n",
              "      <td>[bowenarrowmotel, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>http://broomsestatelodge.co.za/about/</td>\n",
              "      <td>about</td>\n",
              "      <td>[broomsestatelodge, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>http://bridgefunctions.co.za/Pages/Gallery.asp</td>\n",
              "      <td>others</td>\n",
              "      <td>[bridgefunctions, pages, gallery]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>http://thebroady.com.au/#our-menu</td>\n",
              "      <td>others</td>\n",
              "      <td>[thebroady, au, our, menu]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>http://chanteclerhotel.co.za/about/</td>\n",
              "      <td>about</td>\n",
              "      <td>[chanteclerhotel, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>https://www.ciaralodge.co.za/about-us</td>\n",
              "      <td>about</td>\n",
              "      <td>[ciaralodge, about, us]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>http://broomsestatelodge.co.za/gallery/</td>\n",
              "      <td>others</td>\n",
              "      <td>[broomsestatelodge, gallery]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>http://www.clarksinn.in/about-us/</td>\n",
              "      <td>about</td>\n",
              "      <td>[clarksinn, in, about, us]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>http://www.buitenverwagten.co.za/karoo-blog.html</td>\n",
              "      <td>others</td>\n",
              "      <td>[buitenverwagten, karoo, blog]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>https://clico.co.za/about/</td>\n",
              "      <td>about</td>\n",
              "      <td>[clico, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>http://buffelshoekdirosielodge.co.za/gallery.php</td>\n",
              "      <td>others</td>\n",
              "      <td>[buffelshoekdirosielodge, gallery]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>http://bunclody.co.za/Rooms/standard.html</td>\n",
              "      <td>others</td>\n",
              "      <td>[bunclody, rooms, standard]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>https://produvia.com/about-us.html</td>\n",
              "      <td>about</td>\n",
              "      <td>[produvia, about, us]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>http://www.cchotel.co.za/contact/</td>\n",
              "      <td>others</td>\n",
              "      <td>[cchotel, contact]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>http://absolutebeach.co.za/contact-2-2/</td>\n",
              "      <td>about</td>\n",
              "      <td>[absolutebeach, contact, 2, 2]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>https://www.buydomains.com/about-buydomains/</td>\n",
              "      <td>about</td>\n",
              "      <td>[buydomains, about, buydomains]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>https://www.w3.org/Consortium/</td>\n",
              "      <td>about</td>\n",
              "      <td>[w3, org, consortium]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>https://www.adobe.com/africa/about-adobe.html?...</td>\n",
              "      <td>about</td>\n",
              "      <td>[adobe, africa, about, adobe, promoid, 2nvqcdb...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>https://wordpress.com/about/</td>\n",
              "      <td>about</td>\n",
              "      <td>[wordpress, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>https://github.com/about</td>\n",
              "      <td>about</td>\n",
              "      <td>[github, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>https://statcounter.com/support/contact/</td>\n",
              "      <td>others</td>\n",
              "      <td>[statcounter, support, contact]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>https://statcounter.com/careers/</td>\n",
              "      <td>others</td>\n",
              "      <td>[statcounter, careers]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>https://www.jimdo.com/about-jimdo/</td>\n",
              "      <td>about</td>\n",
              "      <td>[jimdo, about, jimdo]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>https://www.jimdo.com/about-jimdo/press/</td>\n",
              "      <td>others</td>\n",
              "      <td>[jimdo, about, jimdo, press]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>https://www.weebly.com/about</td>\n",
              "      <td>about</td>\n",
              "      <td>[weebly, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>https://www.weebly.com/contact</td>\n",
              "      <td>others</td>\n",
              "      <td>[weebly, contact]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>https://www.addthis.com/about/</td>\n",
              "      <td>about</td>\n",
              "      <td>[addthis, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>https://www.addthis.com/careers/</td>\n",
              "      <td>others</td>\n",
              "      <td>[addthis, careers]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>https://www.wix.com/blog/</td>\n",
              "      <td>others</td>\n",
              "      <td>[wix, blog]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>https://help.dropbox.com/security/cookies</td>\n",
              "      <td>others</td>\n",
              "      <td>[help, dropbox, security, cookies]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>https://www.wix.com/about/us</td>\n",
              "      <td>about</td>\n",
              "      <td>[wix, about, us]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>https://www.dropbox.com/developers</td>\n",
              "      <td>others</td>\n",
              "      <td>[dropbox, developers]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>https://www.dropbox.com/about</td>\n",
              "      <td>about</td>\n",
              "      <td>[dropbox, about]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>https://www.msn.com/en-xl/africa/nigeria/us-re...</td>\n",
              "      <td>others</td>\n",
              "      <td>[msn, en, africa, nigeria, us, releases, list,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                 Urls  Labels  \\\n",
              "0                     http://bothot.co.za/contact-us/  others   \n",
              "1                          http://bothot.co.za/about/   about   \n",
              "2                     http://botsesafaris.com/hunting  others   \n",
              "3                    http://botsesafaris.com/about-us   about   \n",
              "4   https://www.bouldersbeachlodge.com/boulders-be...  others   \n",
              "5             http://bowenarrowmotel.com/testimonials  others   \n",
              "6     http://bridgefunctions.co.za/Pages/About-Us.asp   about   \n",
              "7   https://www.breezeguesthouse.co.za/overnight-a...  others   \n",
              "8                    http://bowenarrowmotel.com/about   about   \n",
              "9               http://broomsestatelodge.co.za/about/   about   \n",
              "10     http://bridgefunctions.co.za/Pages/Gallery.asp  others   \n",
              "11                  http://thebroady.com.au/#our-menu  others   \n",
              "12                http://chanteclerhotel.co.za/about/   about   \n",
              "13              https://www.ciaralodge.co.za/about-us   about   \n",
              "14            http://broomsestatelodge.co.za/gallery/  others   \n",
              "15                  http://www.clarksinn.in/about-us/   about   \n",
              "16   http://www.buitenverwagten.co.za/karoo-blog.html  others   \n",
              "17                         https://clico.co.za/about/   about   \n",
              "18   http://buffelshoekdirosielodge.co.za/gallery.php  others   \n",
              "19          http://bunclody.co.za/Rooms/standard.html  others   \n",
              "20                 https://produvia.com/about-us.html   about   \n",
              "21                  http://www.cchotel.co.za/contact/  others   \n",
              "22            http://absolutebeach.co.za/contact-2-2/   about   \n",
              "23       https://www.buydomains.com/about-buydomains/   about   \n",
              "24                     https://www.w3.org/Consortium/   about   \n",
              "25  https://www.adobe.com/africa/about-adobe.html?...   about   \n",
              "26                       https://wordpress.com/about/   about   \n",
              "27                           https://github.com/about   about   \n",
              "28           https://statcounter.com/support/contact/  others   \n",
              "29                   https://statcounter.com/careers/  others   \n",
              "30                 https://www.jimdo.com/about-jimdo/   about   \n",
              "31           https://www.jimdo.com/about-jimdo/press/  others   \n",
              "32                       https://www.weebly.com/about   about   \n",
              "33                     https://www.weebly.com/contact  others   \n",
              "34                     https://www.addthis.com/about/   about   \n",
              "35                   https://www.addthis.com/careers/  others   \n",
              "36                          https://www.wix.com/blog/  others   \n",
              "37          https://help.dropbox.com/security/cookies  others   \n",
              "38                       https://www.wix.com/about/us   about   \n",
              "39                 https://www.dropbox.com/developers  others   \n",
              "40                      https://www.dropbox.com/about   about   \n",
              "41  https://www.msn.com/en-xl/africa/nigeria/us-re...  others   \n",
              "\n",
              "                                               Tokens  \n",
              "0                               [bothot, contact, us]  \n",
              "1                                     [bothot, about]  \n",
              "2                             [botsesafaris, hunting]  \n",
              "3                           [botsesafaris, about, us]  \n",
              "4   [bouldersbeachlodge, boulders, beach, accommod...  \n",
              "5                     [bowenarrowmotel, testimonials]  \n",
              "6                 [bridgefunctions, pages, about, us]  \n",
              "7   [breezeguesthouse, overnight, accommodation, v...  \n",
              "8                            [bowenarrowmotel, about]  \n",
              "9                          [broomsestatelodge, about]  \n",
              "10                  [bridgefunctions, pages, gallery]  \n",
              "11                         [thebroady, au, our, menu]  \n",
              "12                           [chanteclerhotel, about]  \n",
              "13                            [ciaralodge, about, us]  \n",
              "14                       [broomsestatelodge, gallery]  \n",
              "15                         [clarksinn, in, about, us]  \n",
              "16                     [buitenverwagten, karoo, blog]  \n",
              "17                                     [clico, about]  \n",
              "18                 [buffelshoekdirosielodge, gallery]  \n",
              "19                        [bunclody, rooms, standard]  \n",
              "20                              [produvia, about, us]  \n",
              "21                                 [cchotel, contact]  \n",
              "22                     [absolutebeach, contact, 2, 2]  \n",
              "23                    [buydomains, about, buydomains]  \n",
              "24                              [w3, org, consortium]  \n",
              "25  [adobe, africa, about, adobe, promoid, 2nvqcdb...  \n",
              "26                                 [wordpress, about]  \n",
              "27                                    [github, about]  \n",
              "28                    [statcounter, support, contact]  \n",
              "29                             [statcounter, careers]  \n",
              "30                              [jimdo, about, jimdo]  \n",
              "31                       [jimdo, about, jimdo, press]  \n",
              "32                                    [weebly, about]  \n",
              "33                                  [weebly, contact]  \n",
              "34                                   [addthis, about]  \n",
              "35                                 [addthis, careers]  \n",
              "36                                        [wix, blog]  \n",
              "37                 [help, dropbox, security, cookies]  \n",
              "38                                   [wix, about, us]  \n",
              "39                              [dropbox, developers]  \n",
              "40                                   [dropbox, about]  \n",
              "41  [msn, en, africa, nigeria, us, releases, list,...  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "metadata": {
        "id": "w_WNahrhpNGr",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X=documents['Urls']\n",
        "y=documents['Labels']\n",
        "\n",
        "#X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "pG0fcGDC7hDg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1368
        },
        "outputId": "0245a53c-fe3e-4421-e582-8cbb2467009a"
      },
      "cell_type": "code",
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "count_vectorizer = CountVectorizer(analyzer=extract_words,stop_words=ignore_words,vocabulary=['about'])\n",
        "count_train = count_vectorizer.fit_transform(X)\n",
        "\n",
        "# Show the Model as a pandas DataFrame\n",
        "feature_names = count_vectorizer.get_feature_names()\n",
        "print(feature_names)\n",
        "X_vect = pd.DataFrame(count_train.toarray())\n",
        "X_vect"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['about']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    0\n",
              "0   0\n",
              "1   1\n",
              "2   0\n",
              "3   1\n",
              "4   0\n",
              "5   0\n",
              "6   1\n",
              "7   0\n",
              "8   1\n",
              "9   1\n",
              "10  0\n",
              "11  0\n",
              "12  1\n",
              "13  1\n",
              "14  0\n",
              "15  1\n",
              "16  0\n",
              "17  1\n",
              "18  0\n",
              "19  0\n",
              "20  1\n",
              "21  0\n",
              "22  0\n",
              "23  1\n",
              "24  0\n",
              "25  1\n",
              "26  1\n",
              "27  1\n",
              "28  0\n",
              "29  0\n",
              "30  1\n",
              "31  1\n",
              "32  1\n",
              "33  0\n",
              "34  1\n",
              "35  0\n",
              "36  0\n",
              "37  0\n",
              "38  1\n",
              "39  0\n",
              "40  1\n",
              "41  0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "metadata": {
        "id": "0PwejcnCqy4S",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "Feature Extraction"
      ]
    },
    {
      "metadata": {
        "id": "MTLh9ObUcqi7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier\n",
        "from sklearn.metrics import precision_recall_fscore_support as score\n",
        "from sklearn.metrics import accuracy_score as asc\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9owJk0qrdnZe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "b9373593-7335-4b3a-9413-170cf96c3bc6"
      },
      "cell_type": "code",
      "source": [
        "rf = RandomForestClassifier(n_estimators=150, max_depth=None, n_jobs=-1)\n",
        "rf_model = rf.fit(X_vect, y)\n",
        "#y_pred = rf_model.predict(X_test_vect)\n",
        "rf_model"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
              "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
              "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "            min_samples_leaf=1, min_samples_split=2,\n",
              "            min_weight_fraction_leaf=0.0, n_estimators=150, n_jobs=-1,\n",
              "            oob_score=False, random_state=None, verbose=0,\n",
              "            warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "3sfJDC5NvU86",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "fd5df7b1-da76-4ee9-da20-38e24537b43a"
      },
      "cell_type": "code",
      "source": [
        "rf_model.predict(count_vectorizer.fit_transform(['https://hotels.ng/about']))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['about'], dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "metadata": {
        "id": "FEhcjnViwBHb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}