Segmenting and Clustering Neighbourhoods.ipynb

{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {
                "collapsed": true
            },
            "source": "<h1> Applied Data Science Capstone Project - Segmenting and Clustering Neighborhoods in Toronto</h1>"
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": "<h2>Import Libraries</h2>"
        },
        {
            "cell_type": "code",
            "execution_count": 43,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": "Libraries imported.\n"
                }
            ],
            "source": "import numpy as np # data in a vectorized manner manipulation\nimport pandas as pd # data analsysis\nfrom pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe\nimport json # JSON files manipulation\nimport requests # HTTP library\nfrom bs4 import BeautifulSoup # scraping library\n\nfrom sklearn.cluster import KMeans # clustering algorithm\n\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\n\n# Matplotlib and associated plotting modules\nimport matplotlib.cm as cm\nimport matplotlib.colors as colors\n\n# !conda install -c conda-forge folium=0.5.0 --yes\nimport folium # map rendering library\n\nprint('Libraries imported.')"
        },
        {
            "cell_type": "code",
            "execution_count": 45,
            "metadata": {},
            "outputs": [],
            "source": "# getting data from internet\nwikipedia_link='https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'\nraw_wikipedia_page= requests.get(wikipedia_link).text\n\n# using beautiful soup to parse the HTML/XML codes.\nsoup = BeautifulSoup(raw_wikipedia_page,'xml')\n#print(soup.prettify())"
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": "<h1>Extracting raw table from webpage</h1>"
        },
        {
            "cell_type": "code",
            "execution_count": 46,
            "metadata": {},
            "outputs": [],
            "source": "# extracting the raw table inside that webpage\ntable = soup.find('table')\n\nPostcode      = []\nBorough       = []\nNeighbourhood = []\n\n# print(table)\n\n# extracting a clean form of the table\nfor tr_cell in table.find_all('tr'):\n    \n    counter = 1\n    Postcode_var      = -1\n    Borough_var       = -1\n    Neighbourhood_var = -1\n    \n    for td_cell in tr_cell.find_all('td'):\n        if counter == 1: \n            Postcode_var = td_cell.text\n        if counter == 2: \n            Borough_var = td_cell.text\n            tag_a_Borough = td_cell.find('a')\n            \n        if counter == 3: \n            Neighbourhood_var = str(td_cell.text).strip()\n            tag_a_Neighbourhood = td_cell.find('a')\n            \n        counter +=1\n        \n    if (Postcode_var == 'Not assigned' or Borough_var == 'Not assigned' or Neighbourhood_var == 'Not assigned'): \n        continue\n    try:\n        if ((tag_a_Borough is None) or (tag_a_Neighbourhood is None)):\n            continue\n    except:\n        pass\n    if(Postcode_var == -1 or Borough_var == -1 or Neighbourhood_var == -1):\n        continue\n    \n    Postcode.append(Postcode_var)\n    Borough.append(Borough_var)\n    Neighbourhood.append(Neighbourhood_var)"
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": "<h1>Integrating poastal codes with more than one neighbor</h1>"
        },
        {
            "cell_type": "code",
            "execution_count": 47,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": "num of unique Postal codes: 6\n"
                }
            ],
            "source": "unique_p = set(Postcode)\nprint('num of unique Postal codes:', len(unique_p))\nPostcode_u      = []\nBorough_u       = []\nNeighbourhood_u = []\n\n\nfor postcode_unique_element in unique_p:\n    p_var = ''; b_var = ''; n_var = ''; \n    for postcode_idx, postcode_element in enumerate(Postcode):\n        if postcode_unique_element == postcode_element:\n            p_var = postcode_element;\n            b_var = Borough[postcode_idx]\n            if n_var == '': \n                n_var = Neighbourhood[postcode_idx]\n            else:\n                n_var = n_var + ', ' + Neighbourhood[postcode_idx]\n    Postcode_u.append(p_var)\n    Borough_u.append(b_var)\n    Neighbourhood_u.append(n_var)"
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": "<h1>Creating pandas dataframe</h1>"
        },
        {
            "cell_type": "code",
            "execution_count": 48,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postcode</th>\n      <th>Borough</th>\n      <th>Neighbourhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>\\nM1HScarborough(Cedarbrae)\\n\\n</td>\n      <td>\\nM2HNorth York(Hillcrest Village)\\n\\n</td>\n      <td>M3HNorth York(Bathurst Manor / Wilson Heights ...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>\\nM1MScarborough(Cliffside / Cliffcrest / Scar...</td>\n      <td>\\nM2MNorth York(Willowdale / Newtonbrook)\\n\\n</td>\n      <td>M3MNorth York(Downsview)Central</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>\\nM1LScarborough(Golden Mile / Clairlea / Oakr...</td>\n      <td>\\nM2LNorth York(York Mills / Silver Hills)\\n\\n</td>\n      <td>M3LNorth York(Downsview)West</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>\\nM1JScarborough(Scarborough Village)\\n\\n</td>\n      <td>\\nM2JNorth York(Fairview / Henry Farm / Oriole...</td>\n      <td>M3JNorth York(Northwood Park / York University)</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>\\nM1KScarborough(Kennedy Park / Ionview / East...</td>\n      <td>\\nM2KNorth York(Bayview Village)\\n\\n</td>\n      <td>M3KNorth York(Downsview)East  (CFB Toronto)</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>\\nM1NScarborough(Birch Cliff / Cliffside West)...</td>\n      <td>\\nM2NNorth York(Willowdale)South\\n\\n</td>\n      <td>M3NNorth York(Downsview)Northwest</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
                        "text/plain": "                                            Postcode  \\\n0                    \\nM1HScarborough(Cedarbrae)\\n\\n   \n1  \\nM1MScarborough(Cliffside / Cliffcrest / Scar...   \n2  \\nM1LScarborough(Golden Mile / Clairlea / Oakr...   \n3          \\nM1JScarborough(Scarborough Village)\\n\\n   \n4  \\nM1KScarborough(Kennedy Park / Ionview / East...   \n5  \\nM1NScarborough(Birch Cliff / Cliffside West)...   \n\n                                             Borough  \\\n0             \\nM2HNorth York(Hillcrest Village)\\n\\n   \n1      \\nM2MNorth York(Willowdale / Newtonbrook)\\n\\n   \n2     \\nM2LNorth York(York Mills / Silver Hills)\\n\\n   \n3  \\nM2JNorth York(Fairview / Henry Farm / Oriole...   \n4               \\nM2KNorth York(Bayview Village)\\n\\n   \n5               \\nM2NNorth York(Willowdale)South\\n\\n   \n\n                                       Neighbourhood  \n0  M3HNorth York(Bathurst Manor / Wilson Heights ...  \n1                    M3MNorth York(Downsview)Central  \n2                       M3LNorth York(Downsview)West  \n3    M3JNorth York(Northwood Park / York University)  \n4        M3KNorth York(Downsview)East  (CFB Toronto)  \n5                  M3NNorth York(Downsview)Northwest  "
                    },
                    "execution_count": 48,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": "toronto_dict = {'Postcode':Postcode_u, 'Borough':Borough_u, 'Neighbourhood':Neighbourhood_u}\ndf_toronto = pd.DataFrame.from_dict(toronto_dict)\ndf_toronto.to_csv('toronto_part1.csv')\ndf_toronto.head(14)"
        },
        {
            "cell_type": "code",
            "execution_count": 49,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": "(6, 3)"
                    },
                    "execution_count": 49,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": "df_toronto.shape\n"
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": ""
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3.6",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.6.9"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 1
}