scrape_web_obesity.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraping the web for YMCA locations\n",
    "## the assumptions: \n",
    "   > 1. Selenium to enter location zip code on 'Find your Y' page and go to next page (can enter by state, but it will only list 20 'closest' locations)\n",
    "   > 2. page showing four of the closest 20 locations - Beautiful Soup to scrape/parse the addresses and put into DF    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib.request, urllib.parse, urllib.error\n",
    "import selenium \n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import time\n",
    "import os\n",
    "import pickle \n",
    "import sys\n",
    "sys.setrecursionlimit(10000)\n",
    "\n",
    "# path to your pickled zipcode files\n",
    "path = '/Volumes/ext200/Dropbox/metis/pklfiles/'  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_zips(state_name):\n",
    "    \"\"\"open the pickle file of zipcodes for a state\n",
    "    -----------\n",
    "    IN: state abbreviation\n",
    "    OUT: the data in the file \n",
    "    \"\"\"\n",
    "    \n",
    "    with open(path + state_name + \"_zips.pkl\", 'rb') as picklefile: \n",
    "        return pickle.load(picklefile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_ymca_locations(state_name):\n",
    "    \"\"\" #open the page to enter in zip code, parse HTML, save in a dataframe\n",
    "    -------------------\n",
    "    IN: 2-letter abbreviation of a state\n",
    "    OUT: data frame of ymca locations (name, address, city, zip) in the state\n",
    "    \"\"\"\n",
    "    \n",
    "    # open file that contains all zipcodes for selected state\n",
    "    zipcodes_for_site = [get_zips(state_name)]\n",
    "    \n",
    "    #create a data frame, name the cols we will fill up\n",
    "    y_df = pd.DataFrame(index=range(len(zipcodes_for_site[0])*20),columns=['zipcode','state','city',\n",
    "                                                                           'adds','name','locations'])\n",
    "    \n",
    "    row = 0\n",
    "    \n",
    "    for zipy in zipcodes_for_site[0]:\n",
    "        \n",
    "        #open chrome   \n",
    "        chromedriver = \"/Applications/chromedriver\"\n",
    "        os.environ[\"webdriver.chrome.driver\"] = chromedriver\n",
    "        driver = webdriver.Chrome(chromedriver)\n",
    "        \n",
    "        #url of YMCA's page\n",
    "        driver.get(\"http://www.ymca.net/find-your-y/\")\n",
    "        \n",
    "        # ID of the box where zipcode is entered\n",
    "        query = driver.find_element_by_id(\"address\")\n",
    "        \n",
    "        #put in zip code\n",
    "        query.send_keys(zipy)\n",
    "        \n",
    "        #equivalent to hitting enter on the keyboard\n",
    "        query.send_keys(Keys.RETURN)\n",
    "        \n",
    "        #parse HTML\n",
    "        soup2=BeautifulSoup(driver.page_source,\"html.parser\")\n",
    "        \n",
    "        #section with locations\n",
    "        locationsoup = soup2.find_all(style=\"padding-left: 17px; text-indent: -17px;\") \n",
    "        \n",
    "        #store the 20 closest locations\n",
    "        \n",
    "        for item in locationsoup:  \n",
    "            name1 = item.find('a')\n",
    "            name = name1.text\n",
    "            adds= name1.next_sibling.next_sibling\n",
    "            nn= adds.next_sibling.next_sibling\n",
    "            \n",
    "            # check to see if location is already there, if not, parse further and add to dataframe\n",
    "            \n",
    "            if adds not in y_df.adds.values:\n",
    "                y_df.name.iloc[row] = name\n",
    "                y_df.adds.iloc[row] = adds\n",
    "                y_df.city.iloc[row] = nn.split(',')[0]\n",
    "                y_df.state.iloc[row] = nn.split()[-2]\n",
    "                y_df.zipcode.iloc[row] = str(nn.split()[-1])[0:5]\n",
    "                \n",
    "                row +=1\n",
    "        \n",
    "        #close web driver!!!\n",
    "        driver.close()  \n",
    "    \n",
    "     #drop null rows\n",
    "    y_df = y_df[y_df['name'].notnull()]    \n",
    "    \n",
    "    # this is how we will sum later\n",
    "    y_df['locations'] = 1 \n",
    "    \n",
    "    #get rid of locations in nearby states\n",
    "    y_df = y_df.loc[(y_df.state == state_name)] \n",
    "    \n",
    "    return y_df    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15 µs, sys: 5 µs, total: 20 µs\n",
      "Wall time: 775 µs\n"
     ]
    }
   ],
   "source": [
    "# Call the function to scrape your locations\n",
    "# enter state abbreviation as a 2-letter string\n",
    "%time LA_y = get_ymca_locations('LA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Don't forget to save!\n",
    "with open(path + 'LA_y.pkl', 'wb') as picklefile:\n",
    "        pickle.dump(LA_y, picklefile)     "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}