Segmenting and Clustering copy 1.ipynb

{
    "cells": [
        {
            "metadata": {
                "collapsed": true
            },
            "cell_type": "markdown",
            "source": "<h1>Assignment 1: Segmenting and Clustering Neighborhoods in Toronto<h1>"
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "## Part 1"
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Import"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "from bs4 import BeautifulSoup\nimport requests\nimport pandas as pd",
            "execution_count": 9,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Scrape"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "# download url data from internet\nurl = \"https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M\"\nsource = requests.get(url).text\nCanada_data = BeautifulSoup(source, 'lxml')",
            "execution_count": 10,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Convert"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "column_names = ['Postalcode','Borough','Neighborhood']\ntoronto = pd.DataFrame(columns = column_names)\n\n# loop through to find postcode, borough, neighborhood \ncontent = Canada_data.find('div', class_='mw-parser-output')\ntable = content.table.tbody\npostcode = 0\nborough = 0\nneighborhood = 0\n\nfor tr in table.find_all('tr'):\n    i = 0\n    for td in tr.find_all('td'):\n        if i == 0:\n            postcode = td.text\n            i = i + 1\n        elif i == 1:\n            borough = td.text\n            i = i + 1\n        elif i == 2: \n            neighborhood = td.text.strip('\\n').replace(']','')\n    toronto = toronto.append({'Postalcode': postcode,'Borough': borough,'Neighborhood': neighborhood},ignore_index=True)\n\n# clean dataframe \ntoronto = toronto[toronto.Borough!='Not assigned']\ntoronto = toronto[toronto.Borough!= 0]\ntoronto.reset_index(drop = True, inplace = True)\ni = 0\nfor i in range(0,toronto.shape[0]):\n    if toronto.iloc[i][2] == 'Not assigned':\n        toronto.iloc[i][2] = toronto.iloc[i][1]\n        i = i+1\n                                 \ndf = toronto.groupby(['Postalcode','Borough'])['Neighborhood'].apply(', '.join).reset_index()\ndf.head(103)",
            "execution_count": 23,
            "outputs": [
                {
                    "output_type": "execute_result",
                    "execution_count": 23,
                    "data": {
                        "text/plain": "    Postalcode           Borough  \\\n0          M1B       Scarborough   \n1          M1C       Scarborough   \n2          M1E       Scarborough   \n3          M1G       Scarborough   \n4          M1H       Scarborough   \n5          M1J       Scarborough   \n6          M1K       Scarborough   \n7          M1L       Scarborough   \n8          M1M       Scarborough   \n9          M1N       Scarborough   \n10         M1P       Scarborough   \n11         M1R       Scarborough   \n12         M1S       Scarborough   \n13         M1T       Scarborough   \n14         M1V       Scarborough   \n15         M1W       Scarborough   \n16         M1X       Scarborough   \n17         M2H        North York   \n18         M2J        North York   \n19         M2K        North York   \n20         M2L        North York   \n21         M2M        North York   \n22         M2N        North York   \n23         M2P        North York   \n24         M2R        North York   \n25         M3A        North York   \n26         M3B        North York   \n27         M3C        North York   \n28         M3H        North York   \n29         M3J        North York   \n..         ...               ...   \n73         M6C              York   \n74         M6E              York   \n75         M6G  Downtown Toronto   \n76         M6H      West Toronto   \n77         M6J      West Toronto   \n78         M6K      West Toronto   \n79         M6L        North York   \n80         M6M              York   \n81         M6N              York   \n82         M6P      West Toronto   \n83         M6R      West Toronto   \n84         M6S      West Toronto   \n85         M7A  Downtown Toronto   \n86         M7R       Mississauga   \n87         M7Y      East Toronto   \n88         M8V         Etobicoke   \n89         M8W         Etobicoke   \n90         M8X         Etobicoke   \n91         M8Y         Etobicoke   \n92         M8Z         Etobicoke   \n93         M9A      Queen's Park   \n94         M9B         Etobicoke   \n95         M9C         Etobicoke   \n96         M9L        North York   \n97         M9M        North York   \n98         M9N              York   \n99         M9P         Etobicoke   \n100        M9R         Etobicoke   \n101        M9V         Etobicoke   \n102        M9W         Etobicoke   \n\n                                          Neighborhood  \n0                                       Rouge, Malvern  \n1               Highland Creek, Rouge Hill, Port Union  \n2                    Guildwood, Morningside, West Hill  \n3                                               Woburn  \n4                                            Cedarbrae  \n5                                  Scarborough Village  \n6          East Birchmount Park, Ionview, Kennedy Park  \n7                      Clairlea, Golden Mile, Oakridge  \n8      Cliffcrest, Cliffside, Scarborough Village West  \n9                          Birch Cliff, Cliffside West  \n10   Dorset Park, Scarborough Town Centre, Wexford ...  \n11                                   Maryvale, Wexford  \n12                                           Agincourt  \n13             Clarks Corners, Sullivan, Tam O'Shanter  \n14   Agincourt North, L'Amoreaux East, Milliken, St...  \n15                                     L'Amoreaux West  \n16                                         Upper Rouge  \n17                                   Hillcrest Village  \n18                        Fairview, Henry Farm, Oriole  \n19                                     Bayview Village  \n20                            Silver Hills, York Mills  \n21                             Newtonbrook, Willowdale  \n22                                    Willowdale South  \n23                                     York Mills West  \n24                                     Willowdale West  \n25                                           Parkwoods  \n26                                     Don Mills North  \n27                    Flemingdon Park, Don Mills South  \n28     Bathurst Manor, Downsview North, Wilson Heights  \n29                     Northwood Park, York University  \n..                                                 ...  \n73                                  Humewood-Cedarvale  \n74                                 Caledonia-Fairbanks  \n75                                            Christie  \n76                        Dovercourt Village, Dufferin  \n77                            Little Portugal, Trinity  \n78        Brockton, Exhibition Place, Parkdale Village  \n79                  Downsview, North Park, Upwood Park  \n80      Del Ray, Keelesdale, Mount Dennis, Silverthorn  \n81                       The Junction North, Runnymede  \n82                       High Park, The Junction South  \n83                              Parkdale, Roncesvalles  \n84                                  Runnymede, Swansea  \n85                                        Queen's Park  \n86               Canada Post Gateway Processing Centre  \n87   Business Reply Mail Processing Centre 969 Eastern  \n88        Humber Bay Shores, Mimico South, New Toronto  \n89                              Alderwood, Long Branch  \n90       The Kingsway, Montgomery Road, Old Mill North  \n91   Humber Bay, King's Mill Park, Kingsway Park So...  \n92   Kingsway Park South West, Mimico NW, The Queen...  \n93                                        Queen's Park  \n94   Cloverdale, Islington, Martin Grove, Princess ...  \n95   Bloordale Gardens, Eringate, Markland Wood, Ol...  \n96                                       Humber Summit  \n97                                    Emery, Humberlea  \n98                                              Weston  \n99                                           Westmount  \n100  Kingsview Village, Martin Grove Gardens, Richv...  \n101  Albion Gardens, Beaumond Heights, Humbergate, ...  \n102                                          Northwest  \n\n[103 rows x 3 columns]",
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postalcode</th>\n      <th>Borough</th>\n      <th>Neighborhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M1B</td>\n      <td>Scarborough</td>\n      <td>Rouge, Malvern</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M1C</td>\n      <td>Scarborough</td>\n      <td>Highland Creek, Rouge Hill, Port Union</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M1E</td>\n      <td>Scarborough</td>\n      <td>Guildwood, Morningside, West Hill</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M1G</td>\n      <td>Scarborough</td>\n      <td>Woburn</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M1H</td>\n      <td>Scarborough</td>\n      <td>Cedarbrae</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>M1J</td>\n      <td>Scarborough</td>\n      <td>Scarborough Village</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>M1K</td>\n      <td>Scarborough</td>\n      <td>East Birchmount Park, Ionview, Kennedy Park</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>M1L</td>\n      <td>Scarborough</td>\n      <td>Clairlea, Golden Mile, Oakridge</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>M1M</td>\n      <td>Scarborough</td>\n      <td>Cliffcrest, Cliffside, Scarborough Village West</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>M1N</td>\n      <td>Scarborough</td>\n      <td>Birch Cliff, Cliffside West</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>M1P</td>\n      <td>Scarborough</td>\n      <td>Dorset Park, Scarborough Town Centre, Wexford ...</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>M1R</td>\n      <td>Scarborough</td>\n      <td>Maryvale, Wexford</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>M1S</td>\n      <td>Scarborough</td>\n      <td>Agincourt</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>M1T</td>\n      <td>Scarborough</td>\n      <td>Clarks Corners, Sullivan, Tam O'Shanter</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>M1V</td>\n      <td>Scarborough</td>\n      <td>Agincourt North, L'Amoreaux East, Milliken, St...</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>M1W</td>\n      <td>Scarborough</td>\n      <td>L'Amoreaux West</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>M1X</td>\n      <td>Scarborough</td>\n      <td>Upper Rouge</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>M2H</td>\n      <td>North York</td>\n      <td>Hillcrest Village</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>M2J</td>\n      <td>North York</td>\n      <td>Fairview, Henry Farm, Oriole</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>M2K</td>\n      <td>North York</td>\n      <td>Bayview Village</td>\n    </tr>\n    <tr>\n      <th>20</th>\n      <td>M2L</td>\n      <td>North York</td>\n      <td>Silver Hills, York Mills</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>M2M</td>\n      <td>North York</td>\n      <td>Newtonbrook, Willowdale</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>M2N</td>\n      <td>North York</td>\n      <td>Willowdale South</td>\n    </tr>\n    <tr>\n      <th>23</th>\n      <td>M2P</td>\n      <td>North York</td>\n      <td>York Mills West</td>\n    </tr>\n    <tr>\n      <th>24</th>\n      <td>M2R</td>\n      <td>North York</td>\n      <td>Willowdale West</td>\n    </tr>\n    <tr>\n      <th>25</th>\n      <td>M3A</td>\n      <td>North York</td>\n      <td>Parkwoods</td>\n    </tr>\n    <tr>\n      <th>26</th>\n      <td>M3B</td>\n      <td>North York</td>\n      <td>Don Mills North</td>\n    </tr>\n    <tr>\n      <th>27</th>\n      <td>M3C</td>\n      <td>North York</td>\n      <td>Flemingdon Park, Don Mills South</td>\n    </tr>\n    <tr>\n      <th>28</th>\n      <td>M3H</td>\n      <td>North York</td>\n      <td>Bathurst Manor, Downsview North, Wilson Heights</td>\n    </tr>\n    <tr>\n      <th>29</th>\n      <td>M3J</td>\n      <td>North York</td>\n      <td>Northwood Park, York University</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>73</th>\n      <td>M6C</td>\n      <td>York</td>\n      <td>Humewood-Cedarvale</td>\n    </tr>\n    <tr>\n      <th>74</th>\n      <td>M6E</td>\n      <td>York</td>\n      <td>Caledonia-Fairbanks</td>\n    </tr>\n    <tr>\n      <th>75</th>\n      <td>M6G</td>\n      <td>Downtown Toronto</td>\n      <td>Christie</td>\n    </tr>\n    <tr>\n      <th>76</th>\n      <td>M6H</td>\n      <td>West Toronto</td>\n      <td>Dovercourt Village, Dufferin</td>\n    </tr>\n    <tr>\n      <th>77</th>\n      <td>M6J</td>\n      <td>West Toronto</td>\n      <td>Little Portugal, Trinity</td>\n    </tr>\n    <tr>\n      <th>78</th>\n      <td>M6K</td>\n      <td>West Toronto</td>\n      <td>Brockton, Exhibition Place, Parkdale Village</td>\n    </tr>\n    <tr>\n      <th>79</th>\n      <td>M6L</td>\n      <td>North York</td>\n      <td>Downsview, North Park, Upwood Park</td>\n    </tr>\n    <tr>\n      <th>80</th>\n      <td>M6M</td>\n      <td>York</td>\n      <td>Del Ray, Keelesdale, Mount Dennis, Silverthorn</td>\n    </tr>\n    <tr>\n      <th>81</th>\n      <td>M6N</td>\n      <td>York</td>\n      <td>The Junction North, Runnymede</td>\n    </tr>\n    <tr>\n      <th>82</th>\n      <td>M6P</td>\n      <td>West Toronto</td>\n      <td>High Park, The Junction South</td>\n    </tr>\n    <tr>\n      <th>83</th>\n      <td>M6R</td>\n      <td>West Toronto</td>\n      <td>Parkdale, Roncesvalles</td>\n    </tr>\n    <tr>\n      <th>84</th>\n      <td>M6S</td>\n      <td>West Toronto</td>\n      <td>Runnymede, Swansea</td>\n    </tr>\n    <tr>\n      <th>85</th>\n      <td>M7A</td>\n      <td>Downtown Toronto</td>\n      <td>Queen's Park</td>\n    </tr>\n    <tr>\n      <th>86</th>\n      <td>M7R</td>\n      <td>Mississauga</td>\n      <td>Canada Post Gateway Processing Centre</td>\n    </tr>\n    <tr>\n      <th>87</th>\n      <td>M7Y</td>\n      <td>East Toronto</td>\n      <td>Business Reply Mail Processing Centre 969 Eastern</td>\n    </tr>\n    <tr>\n      <th>88</th>\n      <td>M8V</td>\n      <td>Etobicoke</td>\n      <td>Humber Bay Shores, Mimico South, New Toronto</td>\n    </tr>\n    <tr>\n      <th>89</th>\n      <td>M8W</td>\n      <td>Etobicoke</td>\n      <td>Alderwood, Long Branch</td>\n    </tr>\n    <tr>\n      <th>90</th>\n      <td>M8X</td>\n      <td>Etobicoke</td>\n      <td>The Kingsway, Montgomery Road, Old Mill North</td>\n    </tr>\n    <tr>\n      <th>91</th>\n      <td>M8Y</td>\n      <td>Etobicoke</td>\n      <td>Humber Bay, King's Mill Park, Kingsway Park So...</td>\n    </tr>\n    <tr>\n      <th>92</th>\n      <td>M8Z</td>\n      <td>Etobicoke</td>\n      <td>Kingsway Park South West, Mimico NW, The Queen...</td>\n    </tr>\n    <tr>\n      <th>93</th>\n      <td>M9A</td>\n      <td>Queen's Park</td>\n      <td>Queen's Park</td>\n    </tr>\n    <tr>\n      <th>94</th>\n      <td>M9B</td>\n      <td>Etobicoke</td>\n      <td>Cloverdale, Islington, Martin Grove, Princess ...</td>\n    </tr>\n    <tr>\n      <th>95</th>\n      <td>M9C</td>\n      <td>Etobicoke</td>\n      <td>Bloordale Gardens, Eringate, Markland Wood, Ol...</td>\n    </tr>\n    <tr>\n      <th>96</th>\n      <td>M9L</td>\n      <td>North York</td>\n      <td>Humber Summit</td>\n    </tr>\n    <tr>\n      <th>97</th>\n      <td>M9M</td>\n      <td>North York</td>\n      <td>Emery, Humberlea</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>M9N</td>\n      <td>York</td>\n      <td>Weston</td>\n    </tr>\n    <tr>\n      <th>99</th>\n      <td>M9P</td>\n      <td>Etobicoke</td>\n      <td>Westmount</td>\n    </tr>\n    <tr>\n      <th>100</th>\n      <td>M9R</td>\n      <td>Etobicoke</td>\n      <td>Kingsview Village, Martin Grove Gardens, Richv...</td>\n    </tr>\n    <tr>\n      <th>101</th>\n      <td>M9V</td>\n      <td>Etobicoke</td>\n      <td>Albion Gardens, Beaumond Heights, Humbergate, ...</td>\n    </tr>\n    <tr>\n      <th>102</th>\n      <td>M9W</td>\n      <td>Etobicoke</td>\n      <td>Northwest</td>\n    </tr>\n  </tbody>\n</table>\n<p>103 rows \u00d7 3 columns</p>\n</div>"
                    },
                    "metadata": {}
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Clean Data for format"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "df = df.dropna()\nempty = 'Not assigned'\ndf = df[(df.Postalcode != empty ) & (df.Borough != empty) & (df.Neighborhood != empty)]",
            "execution_count": 12,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "df.head()",
            "execution_count": 13,
            "outputs": [
                {
                    "output_type": "execute_result",
                    "execution_count": 13,
                    "data": {
                        "text/plain": "  Postalcode      Borough                            Neighborhood\n0        M1B  Scarborough                          Rouge, Malvern\n1        M1C  Scarborough  Highland Creek, Rouge Hill, Port Union\n2        M1E  Scarborough       Guildwood, Morningside, West Hill\n3        M1G  Scarborough                                  Woburn\n4        M1H  Scarborough                               Cedarbrae",
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postalcode</th>\n      <th>Borough</th>\n      <th>Neighborhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M1B</td>\n      <td>Scarborough</td>\n      <td>Rouge, Malvern</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M1C</td>\n      <td>Scarborough</td>\n      <td>Highland Creek, Rouge Hill, Port Union</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M1E</td>\n      <td>Scarborough</td>\n      <td>Guildwood, Morningside, West Hill</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M1G</td>\n      <td>Scarborough</td>\n      <td>Woburn</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M1H</td>\n      <td>Scarborough</td>\n      <td>Cedarbrae</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
                    },
                    "metadata": {}
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "def neighborhood_list(grouped):    \n    return ', '.join(sorted(grouped['Neighborhood'].tolist()))\n                    \ngrp = df.groupby(['Postalcode', 'Borough'])\ndf2 = grp.apply(neighborhood_list).reset_index(name='Neighborhood')",
            "execution_count": 14,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "print(df2.shape)\ndf2.head(10)",
            "execution_count": 16,
            "outputs": [
                {
                    "output_type": "stream",
                    "text": "(103, 3)\n",
                    "name": "stdout"
                },
                {
                    "output_type": "execute_result",
                    "execution_count": 16,
                    "data": {
                        "text/plain": "  Postalcode      Borough                                     Neighborhood\n0        M1B  Scarborough                                   Rouge, Malvern\n1        M1C  Scarborough           Highland Creek, Rouge Hill, Port Union\n2        M1E  Scarborough                Guildwood, Morningside, West Hill\n3        M1G  Scarborough                                           Woburn\n4        M1H  Scarborough                                        Cedarbrae\n5        M1J  Scarborough                              Scarborough Village\n6        M1K  Scarborough      East Birchmount Park, Ionview, Kennedy Park\n7        M1L  Scarborough                  Clairlea, Golden Mile, Oakridge\n8        M1M  Scarborough  Cliffcrest, Cliffside, Scarborough Village West\n9        M1N  Scarborough                      Birch Cliff, Cliffside West",
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postalcode</th>\n      <th>Borough</th>\n      <th>Neighborhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M1B</td>\n      <td>Scarborough</td>\n      <td>Rouge, Malvern</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M1C</td>\n      <td>Scarborough</td>\n      <td>Highland Creek, Rouge Hill, Port Union</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M1E</td>\n      <td>Scarborough</td>\n      <td>Guildwood, Morningside, West Hill</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M1G</td>\n      <td>Scarborough</td>\n      <td>Woburn</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M1H</td>\n      <td>Scarborough</td>\n      <td>Cedarbrae</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>M1J</td>\n      <td>Scarborough</td>\n      <td>Scarborough Village</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>M1K</td>\n      <td>Scarborough</td>\n      <td>East Birchmount Park, Ionview, Kennedy Park</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>M1L</td>\n      <td>Scarborough</td>\n      <td>Clairlea, Golden Mile, Oakridge</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>M1M</td>\n      <td>Scarborough</td>\n      <td>Cliffcrest, Cliffside, Scarborough Village West</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>M1N</td>\n      <td>Scarborough</td>\n      <td>Birch Cliff, Cliffside West</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
                    },
                    "metadata": {}
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "print('The DataFrame shape is', df2.shape)",
            "execution_count": 22,
            "outputs": [
                {
                    "output_type": "stream",
                    "text": "The DataFrame shape is (103, 3)\n",
                    "name": "stdout"
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "## Part 2"
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Insert New Dataframe "
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "import numpy as np # library to handle data in a vectorized manner\nimport pandas as pd # library for data analsysis\npd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', None)\nimport json # library to handle JSON files\n#!conda install -c conda-forge geopy --yes # uncomment this line if you haven't completed the Foursquare API lab\nfrom geopy.geocoders import Nominatim # convert an address into latitude and longitude values\nimport requests # library to handle requests\nfrom pandas.io.json import json_normalize # tranform JSON file into a pandas dataframe\n# Matplotlib and associated plotting modules\nimport matplotlib.cm as cm\nimport matplotlib.colors as colors\n# import k-means from clustering stage\nfrom sklearn.cluster import KMeans\nfrom sklearn.datasets.samples_generator import make_blobs\n#!conda install -c conda-forge folium=0.5.0 --yes # uncomment this line if you haven't completed the Foursquare API lab\nfrom bs4 import BeautifulSoup\nimport lxml\nprint('Libraries imported.')",
            "execution_count": 38,
            "outputs": [
                {
                    "output_type": "stream",
                    "text": "Libraries imported.\n",
                    "name": "stdout"
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "# download data and parse it:\nr = requests.get('https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M')\nsoup = BeautifulSoup(r.text, 'html.parser')\ntable=soup.find('table', attrs={'class':'wikitable sortable'})\n\n#get headers:\nheaders=table.findAll('th')\nfor i, head in enumerate(headers): headers[i]=str(headers[i]).replace(\"<th>\",\"\").replace(\"</th>\",\"\").replace(\"\\n\",\"\")\n\n#Find all items and skip first one:\nrows=table.findAll('tr')\nrows=rows[1:len(rows)]\n\n# skip all meta symbols and line feeds between rows:\nfor i, row in enumerate(rows): rows[i] = str(rows[i]).replace(\"\\n</td></tr>\",\"\").replace(\"<tr>\\n<td>\",\"\")\n\n# make dataframe, expand rows and drop the old one:\ndf=pd.DataFrame(rows)\ndf[headers] = df[0].str.split(\"</td>\\n<td>\", n = 2, expand = True) \ndf.drop(columns=[0],inplace=True)\n\n# skip not assigned boroughs:\ndf = df.drop(df[(df.Borough == \"Not assigned\")].index)\n# give \"Not assigned\" Neighborhoods same name as Borough:\ndf.Neighbourhood.replace(\"Not assigned\", df.Borough, inplace=True)\n\n# copy Borough value to Neighborhood if NaN:\ndf.Neighbourhood.fillna(df.Borough, inplace=True)\n# drop duplicate rows:\ndf=df.drop_duplicates()\n\n# extract titles from columns\ndf.update(\n    df.Neighbourhood.loc[\n        lambda x: x.str.contains('title')\n    ].str.extract('title=\\\"([^\\\"]*)',expand=False))\n\ndf.update(\n    df.Borough.loc[\n        lambda x: x.str.contains('title')\n    ].str.extract('title=\\\"([^\\\"]*)',expand=False))\n\n# delete Toronto annotation from Neighbourhood:\ndf.update(\n    df.Neighbourhood.loc[\n        lambda x: x.str.contains('Toronto')\n    ].str.replace(\", Toronto\",\"\"))\ndf.update(\n    df.Neighbourhood.loc[\n        lambda x: x.str.contains('Toronto')\n    ].str.replace(\"\\(Toronto\\)\",\"\"))\n\n# combine multiple neighborhoods with the same post code\ndf2 = pd.DataFrame({'Postcode':df.Postcode.unique()})\ndf2['Borough']=pd.DataFrame(list(set(df['Borough'].loc[df['Postcode'] == x['Postcode']])) for i, x in df2.iterrows())\ndf2['Neighborhood']=pd.Series(list(set(df['Neighbourhood'].loc[df['Postcode'] == x['Postcode']])) for i, x in df2.iterrows())\ndf2['Neighborhood']=df2['Neighborhood'].apply(lambda x: ', '.join(x))\ndf2.dtypes\n\ndf2.head()",
            "execution_count": 31,
            "outputs": [
                {
                    "output_type": "execute_result",
                    "execution_count": 31,
                    "data": {
                        "text/plain": "  Postcode           Borough                      Neighborhood\n0      M3A        North York                         Parkwoods\n1      M4A        North York                  Victoria Village\n2      M5A  Downtown Toronto                       Regent Park\n3      M6A        North York  Lawrence Manor, Lawrence Heights\n4      M7A  Downtown Toronto                     Queen's Park ",
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postcode</th>\n      <th>Borough</th>\n      <th>Neighborhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M3A</td>\n      <td>North York</td>\n      <td>Parkwoods</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M4A</td>\n      <td>North York</td>\n      <td>Victoria Village</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M5A</td>\n      <td>Downtown Toronto</td>\n      <td>Regent Park</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M6A</td>\n      <td>North York</td>\n      <td>Lawrence Manor, Lawrence Heights</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M7A</td>\n      <td>Downtown Toronto</td>\n      <td>Queen's Park</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
                    },
                    "metadata": {}
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "convert"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "#add Geo-spatial data\ndfll= pd.read_csv(\"http://cocl.us/Geospatial_data\")\ndfll.rename(columns={'Postal Code':'Postcode'}, inplace=True)\ndfll.set_index(\"Postcode\")\ndf2.set_index(\"Postcode\")\ntoronto_data=pd.merge(df2, dfll)\ntoronto_data.head()",
            "execution_count": 32,
            "outputs": [
                {
                    "output_type": "execute_result",
                    "execution_count": 32,
                    "data": {
                        "text/plain": "  Postcode           Borough                      Neighborhood   Latitude  \\\n0      M3A        North York                         Parkwoods  43.753259   \n1      M4A        North York                  Victoria Village  43.725882   \n2      M5A  Downtown Toronto                       Regent Park  43.654260   \n3      M6A        North York  Lawrence Manor, Lawrence Heights  43.718518   \n4      M7A  Downtown Toronto                     Queen's Park   43.662301   \n\n   Longitude  \n0 -79.329656  \n1 -79.315572  \n2 -79.360636  \n3 -79.464763  \n4 -79.389494  ",
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postcode</th>\n      <th>Borough</th>\n      <th>Neighborhood</th>\n      <th>Latitude</th>\n      <th>Longitude</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M3A</td>\n      <td>North York</td>\n      <td>Parkwoods</td>\n      <td>43.753259</td>\n      <td>-79.329656</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M4A</td>\n      <td>North York</td>\n      <td>Victoria Village</td>\n      <td>43.725882</td>\n      <td>-79.315572</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M5A</td>\n      <td>Downtown Toronto</td>\n      <td>Regent Park</td>\n      <td>43.654260</td>\n      <td>-79.360636</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M6A</td>\n      <td>North York</td>\n      <td>Lawrence Manor, Lawrence Heights</td>\n      <td>43.718518</td>\n      <td>-79.464763</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M7A</td>\n      <td>Downtown Toronto</td>\n      <td>Queen's Park</td>\n      <td>43.662301</td>\n      <td>-79.389494</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
                    },
                    "metadata": {}
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "address = 'Toronto, ON, Canada'\n\ngeolocator = Nominatim(user_agent=\"to_explorer\")\nlocation = geolocator.geocode(address)\nlatitude = location.latitude\nlongitude = location.longitude\nprint('The geograpical coordinate of Toronto, ON, Canada are {}, {}.'.format(latitude, longitude))",
            "execution_count": 39,
            "outputs": [
                {
                    "output_type": "stream",
                    "text": "The geograpical coordinate of Toronto, ON, Canada are 43.653963, -79.387207.\n",
                    "name": "stdout"
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "",
            "execution_count": 36,
            "outputs": [
                {
                    "output_type": "error",
                    "ename": "NameError",
                    "evalue": "name 'folium' is not defined",
                    "traceback": [
                        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
                        "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
                        "\u001b[0;32m<ipython-input-36-8fddb2b324e7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# create map of New York using latitude and longitude values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mmap_toronto\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfolium\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlocation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlatitude\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlongitude\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzoom_start\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# add markers to map\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mlat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlng\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mborough\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mneighborhood\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtoronto_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Latitude'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoronto_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Longitude'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoronto_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Borough'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoronto_data\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Neighborhood'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
                        "\u001b[0;31mNameError\u001b[0m: name 'folium' is not defined"
                    ]
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "",
            "execution_count": null,
            "outputs": []
        }
    ],
    "metadata": {
        "kernelspec": {
            "name": "python3",
            "display_name": "Python 3.6",
            "language": "python"
        },
        "language_info": {
            "name": "python",
            "version": "3.6.9",
            "mimetype": "text/x-python",
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "pygments_lexer": "ipython3",
            "nbconvert_exporter": "python",
            "file_extension": ".py"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 1
}