_exercises.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Requests Check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import urllib3\n",
    "urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "r = requests.get('https://www.python.org')\n",
    "r"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Filtering rows by column value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>star_rating</th>\n",
       "      <th>title</th>\n",
       "      <th>content_rating</th>\n",
       "      <th>genre</th>\n",
       "      <th>duration</th>\n",
       "      <th>actors_list</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9.3</td>\n",
       "      <td>The Shawshank Redemption</td>\n",
       "      <td>R</td>\n",
       "      <td>Crime</td>\n",
       "      <td>142</td>\n",
       "      <td>[u'Tim Robbins', u'Morgan Freeman', u'Bob Gunt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9.2</td>\n",
       "      <td>The Godfather</td>\n",
       "      <td>R</td>\n",
       "      <td>Crime</td>\n",
       "      <td>175</td>\n",
       "      <td>[u'Marlon Brando', u'Al Pacino', u'James Caan']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9.1</td>\n",
       "      <td>The Godfather: Part II</td>\n",
       "      <td>R</td>\n",
       "      <td>Crime</td>\n",
       "      <td>200</td>\n",
       "      <td>[u'Al Pacino', u'Robert De Niro', u'Robert Duv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.0</td>\n",
       "      <td>The Dark Knight</td>\n",
       "      <td>PG-13</td>\n",
       "      <td>Action</td>\n",
       "      <td>152</td>\n",
       "      <td>[u'Christian Bale', u'Heath Ledger', u'Aaron E...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8.9</td>\n",
       "      <td>Pulp Fiction</td>\n",
       "      <td>R</td>\n",
       "      <td>Crime</td>\n",
       "      <td>154</td>\n",
       "      <td>[u'John Travolta', u'Uma Thurman', u'Samuel L....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   star_rating                     title content_rating   genre  duration  \\\n",
       "0          9.3  The Shawshank Redemption              R   Crime       142   \n",
       "1          9.2             The Godfather              R   Crime       175   \n",
       "2          9.1    The Godfather: Part II              R   Crime       200   \n",
       "3          9.0           The Dark Knight          PG-13  Action       152   \n",
       "4          8.9              Pulp Fiction              R   Crime       154   \n",
       "\n",
       "                                         actors_list  \n",
       "0  [u'Tim Robbins', u'Morgan Freeman', u'Bob Gunt...  \n",
       "1    [u'Marlon Brando', u'Al Pacino', u'James Caan']  \n",
       "2  [u'Al Pacino', u'Robert De Niro', u'Robert Duv...  \n",
       "3  [u'Christian Bale', u'Heath Ledger', u'Aaron E...  \n",
       "4  [u'John Travolta', u'Uma Thurman', u'Samuel L....  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "movies = pd.read_csv('http://bit.ly/imdbratings')\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "movies.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "type(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create python list of booleans with the same length of the dataframe. \n",
    "#Boolean will be true if it is > 200 mins and false if others\n",
    "booleans = []\n",
    "for length in movies.duration:\n",
    "    if length >= 200:\n",
    "        booleans.append(True)\n",
    "    else:\n",
    "        booleans.append(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "booleans[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(booleans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# convert booleans list to a pandas series\n",
    "is_long = pd.Series(booleans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "is_long.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pass is_long to the dataframe movies with bracket notation\n",
    "movies[is_long].head()\n",
    "\n",
    "#It shows up dataframe with all columns but only shows those with duration > 200 mins"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# There is a shorter way - without the for loop.\n",
    "\n",
    "#Instead:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This completely replaces the need for a 'for' loop.\n",
    "is_long = movies.duration >= 200 # Series (movies.duration)  ... comparison >=200 and returns series of trues and falses\n",
    "is_long.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "movies[is_long].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We can eliminate is_long itself\n",
    "movies[movies.duration >=200].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# If we are studying only the genre of the movies\n",
    "# you can use dot notation\n",
    "\n",
    "movies[movies.duration >=200].genre\n",
    "\n",
    "# or in bracket notations\n",
    "movies[movies['duration'] >=200]['genre']\n",
    "\n",
    "#The above code may sometimes cause strange behaviour. Not the best way to do things.\n",
    "#better practice is to use the .loc method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# use .loc with a comma\n",
    "movies.loc[movies['duration'] >=200, 'genre']\n",
    "\n",
    "#.loc allows selection of rows and columns by 'label'\n",
    "# in the above movies['duration'] >= 200 are the rows and 'genre' are the columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ...For Multiple filter criteria"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "movies = pd.read_csv('http://bit.ly/imdbratings')\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "movies[movies.duration >= 200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# how do we select only long movies of genre Drama?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "True and True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "movies[movies.duration >= 200 and movies.genre == 'Drama'] # will not work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add parenthesis to add evaluation order\n",
    "# add ampersand & instead of and\n",
    "\n",
    "movies[(movies.duration >= 200) & (movies.genre == 'Drama')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# or condition - gives a much bigger dataframe\n",
    "movies[(movies.duration >= 200) | (movies.genre == 'Drama')].head()\n",
    "movies[(movies.duration >= 200) | (movies.genre == 'Drama')].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# inside the bracket there is a boolean series which tells dataframe which rows display\n",
    "((movies.duration >= 200) & (movies.genre == 'Drama'))[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# what if there is a bunch of or conditions on the same series\n",
    "# either crime or drama or action\n",
    "# normally\n",
    "movies[(movies.genre == 'Crime') | (movies.genre == 'Drama') | (movies.genre == 'Action')].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# the above is very wordy.\n",
    "# to simplify we can use a series method called 'isin'\n",
    "# it generates a boolean series\n",
    "movies.genre.isin(['Crime', 'Drama', 'Action'])[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The above series can be passed to the DataFrame\n",
    "movies[movies.genre.isin(['Crime', 'Drama', 'Action'])].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web Scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kashir\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\urllib3\\connectionpool.py:858: InsecureRequestWarning: Unverified HTTPS request is being made. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings\n",
      "  InsecureRequestWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1.79 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Scraping all HTMLs table from a URL using BeautifulSoup\n",
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "res = requests.get(\"https://www.5paisa.com/5pit/spma.asp\", verify=False)\n",
    "soup = BeautifulSoup(res.content, 'lxml')\n",
    "table = soup.find_all('table')\n",
    "df = pd.read_html(str(table))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[                   0                        1\n",
       " 0  Margin Calculator  Span Margin May 4, 2018,\n",
       "              0      1       2        3        4              5             6   \\\n",
       " 0        Symbol   Mlot  SpMgn%  ExpMgn%  TotMgn%  SpMgnPerShare  ExpMgnPerShr   \n",
       " 1    PCJEWELLER   1500   40.21    31.04    71.25          44.56       34.3923   \n",
       " 2    JPASSOCIAT  34000   14.78    21.69    36.47           2.92        4.2838   \n",
       " 3          RCOM  28000    15.6    20.54    36.14           2.52        3.3172   \n",
       " 4           IRB   2500      11       20       31          29.81          54.2   \n",
       " 5    BALRAMCHIN   3500   13.71       15    28.71           9.16         10.02   \n",
       " 6        RNAVAL   9000   20.05     7.21    27.26           3.62        1.3014   \n",
       " 7    JETAIRWAYS   1200   10.76       15    25.76          63.86       88.9575   \n",
       " 8      JUSTDIAL   1400   12.31    13.38    25.69          50.25       54.5837   \n",
       " 9          DHFL   1500    8.32       15    23.32          54.32       97.9275   \n",
       " 10     INDIAVIX    900   16.88        5    21.88       216.3594       64.0875   \n",
       " 11     INDIAVIX    900   16.88        5    21.88       216.3594       64.0875   \n",
       " 12     INDIAVIX    900   16.88        5    21.88         216.43       64.0875   \n",
       " 13      RAYMOND    800    7.71     12.5    20.21          86.68      140.4375   \n",
       " 14     GMRINFRA  45000   11.04      8.1    19.14           2.22        1.6281   \n",
       " 15   WOCKPHARMA    900    8.27       10    18.27          67.52          81.6   \n",
       " 16     NIITTECH    750   13.04        5    18.04         144.48       55.3575   \n",
       " 17   CENTURYTEX    550   10.49      7.5    17.99         120.26       85.9725   \n",
       " 18     ADANIENT   4000   12.99        5    17.99          16.81        6.4675   \n",
       " 19          HCC  15000   12.82        5    17.82           3.02        1.1775   \n",
       " 20   ANDHRABANK  10000   11.81        5    16.81           4.53        1.9175   \n",
       " 21    REPCOHOME    900   11.39        5    16.39          67.27        29.525   \n",
       " 22   NATIONALUM   8000   11.08        5    16.08           8.86        3.9975   \n",
       " 23     HINDALCO   3500   11.01        5    16.01          25.28        11.475   \n",
       " 24         SAIL  12000   10.85        5    15.85           7.96         3.665   \n",
       " 25        CANBK   1600   10.85        5    15.85             28        12.895   \n",
       " 26     MINDTREE   1200    10.8        5     15.8         114.01        52.775   \n",
       " 27     SREINFRA   5000   10.79        5    15.79           9.06         4.195   \n",
       " 28   ORIENTBANK   6000   10.76        5    15.76           9.67        4.4925   \n",
       " 29         IDBI  10000   10.76        5    15.76           7.05         3.275   \n",
       " ..          ...    ...     ...      ...      ...            ...           ...   \n",
       " 188         IGL   2750     7.5        5     12.5          21.17        14.095   \n",
       " 189  IBULHSGFIN    400     7.5        5     12.5          97.97       65.2625   \n",
       " 190  HINDUNILVR    600     7.5        5     12.5         111.08       74.0125   \n",
       " 191  HEROMOTOCO    200     7.5        5     12.5         276.38      184.0225   \n",
       " 192    HDFCBANK    500     7.5        5     12.5         148.17        98.775   \n",
       " 193     HAVELLS   1000     7.5        5     12.5          41.02        27.315   \n",
       " 194    GRANULES   5000     7.5        5     12.5           8.06        5.3675   \n",
       " 195    GLENMARK    900     7.5        5     12.5          41.53       27.6775   \n",
       " 196  ENGINERSIN   3500     7.5        5     12.5          11.69        7.7875   \n",
       " 197   EICHERMOT     25     7.5        5     12.5           2302     1533.0525   \n",
       " 198   DALMIABHA    300     7.5        5     12.5         223.77         149.1   \n",
       " 199      CONCOR    625     7.5        5     12.5         100.19        66.775   \n",
       " 200      COLPAL    700     7.5        5     12.5          85.27         56.82   \n",
       " 201    CHOLAFIN    500     7.5        5     12.5         129.22       86.0975   \n",
       " 202   BRITANNIA    200     7.5        5     12.5         409.23       272.695   \n",
       " 203    BOSCHLTD     25     7.5        5     12.5        1473.16      981.7925   \n",
       " 204      BIOCON    900     7.5        5     12.5           50.1        33.375   \n",
       " 205  BALKRISIND    800     7.5        5     12.5          93.94         62.57   \n",
       " 206  BAJFINANCE    500     7.5        5     12.5         143.84       95.8575   \n",
       " 207  BAJAJFINSV    125     7.5        5     12.5         410.14       273.165   \n",
       " 208      ARVIND   2000     7.5        5     12.5          30.74       20.4925   \n",
       " 209   AMBUJACEM   2500     7.5        5     12.5          18.55       12.3525   \n",
       " 210  AJANTPHARM    500     7.5        5     12.5          101.3       67.5025   \n",
       " 211   NIFTYCPSE    250       5        5       10          119.7      119.5875   \n",
       " 212     NIFTYIT     50    5.01        3     8.01         691.64        413.88   \n",
       " 213    NIFTYPSE    200       5        3        8         191.15       114.624   \n",
       " 214  NIFTYMID50    200       5        3        8         269.77       161.802   \n",
       " 215  NIFTYINFRA    225       5        3        8            175       104.943   \n",
       " 216       NIFTY     75       5        3        8         537.03       322.182   \n",
       " 217   BANKNIFTY     40       5        3        8        1282.28       769.311   \n",
       " \n",
       "                7           8            9            10  \n",
       " 0    TotMgnPerShr  SpMgnPerLt  ExpMgnPerLt  TotMgnPerLt  \n",
       " 1         78.9523       66840     51588.48    118428.48  \n",
       " 2          7.2038       99280    145648.35    244928.35  \n",
       " 3          5.8372       70560     92881.88    163441.88  \n",
       " 4           84.01       74525       135500       210025  \n",
       " 5           19.18       32060        35070        67130  \n",
       " 6          4.9214       32580    11712.645    44292.645  \n",
       " 7        152.8175       76632       106749       183381  \n",
       " 8        104.8337       70350    76417.194   146767.194  \n",
       " 9        152.2475       81480    146891.25    228371.25  \n",
       " 10       280.5175   194723.46     57678.75    252465.75  \n",
       " 11       280.5175   194723.46     57678.75    252465.75  \n",
       " 12       280.5175      194787     57678.75    252465.75  \n",
       " 13       227.1175       69344       112350       181694  \n",
       " 14         3.8481       99900      73264.5     173164.5  \n",
       " 15         149.12       60768        73440       134208  \n",
       " 16       199.8375      108360    41518.125   149878.125  \n",
       " 17       206.2325       66143    47284.875   113427.875  \n",
       " 18        23.2775       67240        25870        93110  \n",
       " 19         4.1975       45300      17662.5      62962.5  \n",
       " 20         6.4475       45300        19175        64475  \n",
       " 21         96.795       60543      26572.5      87115.5  \n",
       " 22        12.8575       70880        31980       102860  \n",
       " 23         36.755       88480      40162.5     128642.5  \n",
       " 24         11.625       95520        43980       139500  \n",
       " 25         40.895       44800        20632        65432  \n",
       " 26        166.785      136812        63330       200142  \n",
       " 27         13.255       45300        20975        66275  \n",
       " 28        14.1625       58020        26955        84975  \n",
       " 29         10.325       70500        32750       103250  \n",
       " ..            ...         ...          ...          ...  \n",
       " 188        35.265       58218     38761.25     96979.25  \n",
       " 189      163.2325       39188        26105        65293  \n",
       " 190      185.0925       66648      44407.5     111055.5  \n",
       " 191      460.4025       55276      36804.5      92080.5  \n",
       " 192       246.945       74085      49387.5     123472.5  \n",
       " 193        68.335       41020        27315        68335  \n",
       " 194       13.4275       40300      26837.5      67137.5  \n",
       " 195       69.2075       37377     24909.75     62286.75  \n",
       " 196       19.4775       40915     27256.25     68171.25  \n",
       " 197     3835.0525       57550   38326.3125   95876.3125  \n",
       " 198        372.87       67131        44730       111861  \n",
       " 199       166.965       62619    41734.375   104353.375  \n",
       " 200        142.09       59689        39774        99463  \n",
       " 201      215.3175       64610     43048.75    107658.75  \n",
       " 202       681.925       81846        54539       136385  \n",
       " 203     2454.9525       36829   24544.8125   61373.8125  \n",
       " 204        83.475       45090      30037.5      75127.5  \n",
       " 205        156.51       75152        50056       125208  \n",
       " 206      239.6975       71920     47928.75    119848.75  \n",
       " 207       683.305       51268    34145.625    85413.625  \n",
       " 208       51.2325       61480        40985       102465  \n",
       " 209       30.9025       46375     30881.25     77256.25  \n",
       " 210      168.8025       50650     33751.25     84401.25  \n",
       " 211      239.2875       29925    29896.875    59821.875  \n",
       " 212       1105.52       34582        20694        55276  \n",
       " 213       305.774       38230      22924.8      61154.8  \n",
       " 214       431.572       53954      32360.4      86314.4  \n",
       " 215       279.943       39375    23612.175    62987.175  \n",
       " 216       859.212       40277     24163.65     64440.65  \n",
       " 217      2051.591       51291     30772.44     82063.44  \n",
       " \n",
       " [218 rows x 11 columns]]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %%time\n",
    "# Scraping an HTML table into pandas\n",
    "import pandas as pd\n",
    "url = \"https://www.5paisa.com/5pit/spma.asp\"\n",
    "df = pd.read_html(url)[1]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from lxml import etree, html\n",
    "import requests\n",
    "\n",
    "url = \"https://finance.google.com/finance?q=NSE:PFC\"\n",
    "\n",
    "page = requests.get(url)\n",
    "root = html.fromstring(page.content)\n",
    "dividend = float(root.findall('.//table')[2].text_content().strip().split(\"\\n\")[2].split('/')[0])\n",
    "dividend"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use of loc and iloc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Pandas Index\n",
    "## ...from Data School - ref: https://www.youtube.com/watch?v=OYZNk7Z9s6I\n",
    "import pandas as pd\n",
    "drinks = pd.read_csv('http://bit.ly/drinksbycountry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.read_table('http://bit.ly/movieusers', header = None, sep = '|').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks[drinks.continent == 'South America']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[23, 'beer_servings']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.set_index('country', inplace=True)\n",
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.loc['Brazil', 'beer_servings']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.index.name = None\n",
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.index.name = 'country'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.reset_index(inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.describe().loc['25%', 'beer_servings']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.continent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.set_index('country', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.continent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.continent.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.continent.value_counts()['Africa']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.continent.value_counts().sort_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.continent.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "people = pd.Series([3000000, 85000], index=['Albania', 'Andorra'], name = 'population')\n",
    "people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.beer_servings * people"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.concat([drinks, people], axis=1).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks = pd.read_csv('http://bit.ly/drinksbycountry')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.info(memory_usage = 'deep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.memory_usage(deep=True).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sorted(drinks.continent.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.continent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks['continent'] = drinks.continent.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.continent.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.continent.cat.codes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks['country'] = drinks.country.astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.memory_usage(deep=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.country.cat.categories"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Handling date and time in python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "ufo = pd.read_csv('http://bit.ly/uforeports')\n",
    "ufo.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Analyze sightings by year or time\n",
    "# Check the dtypes\n",
    "ufo.dtypes # time column shows an object - in this case a string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ufo.Time.str.slice(-5, -3).head()  #outputs as a string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ufo.Time.str.slice(-5, -3).astype(int).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# the above approach is very brittle. It easily breaks\n",
    "# overwrite the time column. Overwrite the Time column.\n",
    "ufo['Time'] = pd.to_datetime(ufo.Time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ufo.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# real thing that's changed is dtype is now datetime\n",
    "ufo.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# pandas just figures out the date. If not, there are lot of options in to_datetime\n",
    "ufo.Time.dt.hour # Pulls out the hour\n",
    "ufo.Time.dt.weekday_name[:4] # Pulls out the name of the week!\n",
    "# Search the reference page for '.dt.'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Let's pass a string instead of a series\n",
    "pd.to_datetime('1/1/1999') #outputs a timestamp. Did not have to specify month and year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ts = pd.to_datetime('1/1/1999') # Save it for comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ufo.loc[ufo.Time >= ts, :].head() #only shows othe ufo's sighted after 1/1/1999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# can do math operations\n",
    "ufo.Time.max() # Latest timestamp in the time series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ufo.Time.max() - ufo.Time.min() # time delta object tells the difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#timedelta objects have attributes like .days\n",
    "(ufo.Time.max() - ufo.Time.min()).days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# no of ufo reports by year. Plot!\n",
    "%matplotlib inline\n",
    "ufo['Year'] = ufo.Time.dt.year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ufo.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ufo.Year.value_counts().sort_index().plot()  #sort by order of index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Handling inputs in date format\n",
    "Following code converts any text input to appropriate date format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expiry: May 13, 2018\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "datetime.datetime(2018, 5, 13, 0, 0)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from dateutil.parser import parse\n",
    "\n",
    "while True:\n",
    "    expiry = input('Expiry: ')\n",
    "    try:\n",
    "        parse(expiry)\n",
    "    except ValueError:\n",
    "        print(\"Enter date in any proper format\")\n",
    "    expiry = parse(expiry)\n",
    "    break\n",
    "\n",
    "expiry"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Vectorize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests\n",
    "\n",
    "# x = ['~', 'A', 'Sun']   # Works\n",
    "x = ['~', 'Walter', 'A', 'Sun'] # Doesn't work because of Walter \n",
    "df = pd.DataFrame(x, columns=['x'])\n",
    "\n",
    "u = \"https://en.wikipedia.org/wiki/\"\n",
    "\n",
    "df['URL'] = u + df['x']\n",
    "\n",
    "def tbl10(u):\n",
    "    html = requests.get(u).content\n",
    "    tbl = pd.read_html(u)[10]\n",
    "    return tbl\n",
    "\n",
    "v = np.vectorize(tbl10)\n",
    "pd.concat(v(df.URL))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Try-Except Error detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "while True:\n",
    "    try:\n",
    "        x = int(input(\"Please enter a number: \"))\n",
    "        break\n",
    "    except ValueError:\n",
    "        pass\n",
    "        print(\"Oops! That was no valid number. Try again...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Examples of where"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd, numpy as np\n",
    "df = pd.DataFrame(np.arange(10).reshape(-1, 2), columns=['A', 'B'])\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "m = df % 3 == 0\n",
    "m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Map and Apply experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train = pd.read_csv('http://bit.ly/kaggletrain')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#translate sex to 1 and 0 using map\n",
    "train['Sex_num'] = train.Sex.map({'female':0, 'male':1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#compare sex and sex num\n",
    "train.loc[0:4, ['Sex', 'Sex_num']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#use apply method for a function\n",
    "train['Name_len'] = train.Name.apply(len)\n",
    "\n",
    "train.loc[0:4, ['Name', 'Name_len']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "train['Fare_ceil'] = train.Fare.apply(np.ceil)\n",
    "\n",
    "train.loc[0:4, ['Fare', 'Fare_ceil']]\n",
    "\n",
    "train.Name.str.split(',').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_element(my_list, position):\n",
    "    return my_list[position]\n",
    "\n",
    "train.Name.str.split(',').apply(get_element, position=0).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.Name.str.split(',').apply(lambda x: x[0]).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks = pd.read_csv('http://bit.ly/drinksbycountry')\n",
    "drinks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings'].apply(max, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings'].apply(max, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings'].apply(np.argmax, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings'].applymap(float).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "drinks.loc[:, 'beer_servings':'wine_servings']=drinks.loc[:, 'beer_servings':'wine_servings'].applymap(float)\n",
    "\n",
    "drinks.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments with arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def varargs(*args):\n",
    "    return args\n",
    "\n",
    "varargs(1,2,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def keyword_args(**kwargs):\n",
    "    return kwargs\n",
    "\n",
    "keyword_args(big=\"foot\", loch=\"ness\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def all_the_args(*args, **kwargs):\n",
    "    print(args)\n",
    "    print(kwargs)\n",
    "\n",
    "all_the_args(1, 2, a=3, b=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "args = (1, 2, 3, 4)\n",
    "kwargs = {\"a\": 3, \"b\": 4}\n",
    "all_the_args(*args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_the_args(*kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_the_args(**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_the_args(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}