07-03-Ensemble Methods 3 - Random Forest and Extra-Trees.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging Machine Learning Algorithm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **B**ootstrap **Agg**regat**ing** or [Bagging](https://en.wikipedia.org/wiki/Bootstrap_aggregating)\n",
    "* [Scikit- Learn Reference](http://scikit-learn.org/stable/modules/ensemble.html#bagging)\n",
    "* Bootstrap sampling: Sampling with replacement\n",
    "* Combine by averaging the output (regression)\n",
    "* Combine by voting (classification)\n",
    "* Can be applied to many classifiers which includes ANN, CART, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df = sns.load_dataset('titanic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(891, 15)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>class</th>\n",
       "      <th>who</th>\n",
       "      <th>adult_male</th>\n",
       "      <th>deck</th>\n",
       "      <th>embark_town</th>\n",
       "      <th>alive</th>\n",
       "      <th>alone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>man</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>no</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>First</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>C</td>\n",
       "      <td>Cherbourg</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>S</td>\n",
       "      <td>First</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>C</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>man</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>no</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   survived  pclass     sex   age  sibsp  parch     fare embarked  class  \\\n",
       "0         0       3    male  22.0      1      0   7.2500        S  Third   \n",
       "1         1       1  female  38.0      1      0  71.2833        C  First   \n",
       "2         1       3  female  26.0      0      0   7.9250        S  Third   \n",
       "3         1       1  female  35.0      1      0  53.1000        S  First   \n",
       "4         0       3    male  35.0      0      0   8.0500        S  Third   \n",
       "\n",
       "     who  adult_male deck  embark_town alive  alone  \n",
       "0    man        True  NaN  Southampton    no  False  \n",
       "1  woman       False    C    Cherbourg   yes  False  \n",
       "2  woman       False  NaN  Southampton   yes   True  \n",
       "3  woman       False    C  Southampton   yes  False  \n",
       "4    man        True  NaN  Southampton    no   True  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 3, 2])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['pclass'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    157\n",
       "2     15\n",
       "3     10\n",
       "Name: pclass, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['pclass'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['female', 'male'], dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sex'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "male      94\n",
       "female    88\n",
       "Name: sex, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sex'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEpJJREFUeJzt3X2MZXV9x/H3t+AD7OgCRaZ0l3Q0MVTDKLo3FKW1M1J0BSJNY1MINdDSzB+1iGaNXWJa4x+mNK1PaZs2RBHTEqYRsZrFqgSZkjaKnQV0FxfE6gZZkMUiSwdJdeu3f8zZOBlndu455z4cf/t+JZO598y553zuOed+cubMvb+JzESS9PPvF8YdQJI0GBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRDHj3Jlp556ak5NTfU9/zPPPMOmTZuGF6ihruaC7mbrai7obrau5oLuZutqLmiXbffu3d/PzBdtOGNmjuxr27ZtWcedd95Za/5R6WquzO5m62quzO5m62quzO5m62quzHbZgMXso2O95CJJhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYUY6Uf/pUGb2nnbmtP3X3fRiJNI4+cZuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFWLDQo+IGyLiYETsXeNn74qIjIhThxNPktSvfs7QbwS2r54YEWcAFwAPDziTJKmBDQs9M+8CnlzjRx8C3g3koENJkuprdA09It4MHMjMrw04jySpocjc+AQ7IqaAXZl5VkScCNwJvCEzD0XEfqCXmd9f57FzwBzA5OTktvn5+b7DLS0tMTEx0ff8o9LVXNDdbMPKtefAoTWnT2/Z3PcyjrVtNghdzdbVXNAu2+zs7O7M7G00X5NCnwbuAH5Y/Xgr8ChwTmZ+72jL6fV6ubi4uOH6jlhYWGBmZqbv+Uelq7mgu9mGlWsQw+cea9tsELqarau5oF22iOir0GuPh56Ze4DTVqxoP0c5Q5ckjUY/b1u8GfgycGZEPBIRVw0/liSprg3P0DPzsg1+PjWwNJKkxvykqCQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQvTzP0VviIiDEbF3xbS/iogHIuLrEfHpiDhpuDElSRvp5wz9RmD7qmm3A2dl5iuAbwLXDjiXJKmmDQs9M+8Cnlw17YuZebi6+xVg6xCySZJqGMQ19D8E/nUAy5EktRCZufFMEVPArsw8a9X09wA94HdynQVFxBwwBzA5Obltfn6+73BLS0tMTEz0Pf+odDUXdDfbsHLtOXBozenTWzb3vYxjbZsNQlezdTUXtMs2Ozu7OzN7G813fKOlAxFxBXAxcP56ZQ6QmdcD1wP0er2cmZnpex0LCwvUmX9UupoLupttWLmu3HnbmtP3X97/uo61bTYIXc3W1VwwmmyNCj0itgN/CvxmZv5wsJEkSU3087bFm4EvA2dGxCMRcRXwt8ALgNsj4r6I+Ich55QkbWDDM/TMvGyNyR8bQhZJUgt+UlSSCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqRD//JPqGiDgYEXtXTDslIm6PiIeq7ycPN6YkaSP9nKHfCGxfNW0ncEdmvhS4o7ovSRqjDQs9M+8Cnlw1+RLgE9XtTwC/PeBckqSaml5Dn8zMxwCq76cNLpIkqYnIzI1nipgCdmXmWdX9pzLzpBU//0FmrnkdPSLmgDmAycnJbfPz832HW1paYmJiou/5R6WruaC72YaVa8+BQ2tOn96yue9ljGqb1c3a1X0J3c3W1VzQLtvs7OzuzOxtNN/xjZYOj0fE6Zn5WEScDhxcb8bMvB64HqDX6+XMzEzfK1lYWKDO/KPS1VzQ3WzDynXlztvWnL7/8v7XNaptVjdrV/cldDdbV3PBaLI1veTyWeCK6vYVwGcGE0eS1FQ/b1u8GfgycGZEPBIRVwHXARdExEPABdV9SdIYbXjJJTMvW+dH5w84iySpBT8pKkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhWg6OJcKMLXeYFHXXVTsutda747pw8wMda3SaHiGLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhWhV6BHxzoi4PyL2RsTNEfH8QQWTJNXTuNAjYgvwdqCXmWcBxwGXDiqYJKmetpdcjgdOiIjjgROBR9tHkiQ1EZnZ/MER1wDvB54FvpiZl68xzxwwBzA5Obltfn6+7+UvLS0xMTHRON+wdDUX1Mu258ChNadPb9k8yEjAz+Ya1LrrLmet+SdPgNNOGfxz7mfdsH7WUo6zUepqLmiXbXZ2dndm9jaar3GhR8TJwKeA3wOeAj4J3JKZ/7TeY3q9Xi4uLva9joWFBWZmZhrlG6au5oJ62UY54uHqXINad93lrDfa4tWXX1JrvU3UzVrKcTZKXc0F7bJFRF+F3uaSy28B38nMJzLzx8CtwGtbLE+S1EKbQn8YODciToyIAM4H9g0mliSprsaFnpl3A7cA9wB7qmVdP6BckqSaWv3Hosx8L/DeAWWRJLXgJ0UlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhWj1wSIN17AHsBqkUQ70JWltnqFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhWhV6RJwUEbdExAMRsS8iXjOoYJKketp+9P8jwOcz8y0R8VzgxAFkkiQ10LjQI+KFwOuAKwEy80fAjwYTS5JUV5tLLi8BngA+HhH3RsRHI2LTgHJJkmqKzGz2wIge8BXgvMy8OyI+AjydmX+2ar45YA5gcnJy2/z8fN/rWFpaYmJiolG+YRpVrj0HDtWaf3rL5jWzNVlOXeut48iyVueqm6mu9Z7DWuudPAFOO6X+c65ro2202sEnD/H4s/3PP0rH+muziTbZZmdnd2dmb6P52hT6LwFfycyp6v5vADszc93xUnu9Xi4uLva9joWFBWZmZhrlG6ZR5ao77O3+6y5aM1uT5dS10fC5q3MNe0jf9Z7DWuvdMX2Yqy+/ZKh51ls3rJ/1b276DB/Y87NXRbswJPGx/tpsok22iOir0BtfcsnM7wHfjYgzq0nnA99oujxJUjtt3+VyNXBT9Q6XbwN/0D6SJKmJVoWemfcBG/4aIEkaPj8pKkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhWj70f+RqTuwkZob9sBZpRjUMbnecnZMD3e9dfP4Wus+z9AlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQrQs9Io6LiHsjYtcgAkmSmhnEGfo1wL4BLEeS1EKrQo+IrcBFwEcHE0eS1FTbM/QPA+8GfjKALJKkFiIzmz0w4mLgwsz844iYAd6VmRevMd8cMAcwOTm5bX5+vu91LC0tMTExAcCeA4dq5ZvesrnW/HWszDVMTZ7zWtnqLmcYJk+Ax58dd4q1HS3b0Y6j9bbreo+pux/qbrO6622Tc2W2Yb7W6hrVa7OJNtlmZ2d3Z2Zvo/naFPpfAG8FDgPPB14I3JqZv7/eY3q9Xi4uLva9joWFBWZmZoD6Q7oOc6jPlbmGqclzXitbF4bD3TF9mA/s6eZozUfLdrTjqO4ws3X3Q91tVne9bXKuzNalYXVH9dpsok22iOir0BtfcsnMazNza2ZOAZcCXzpamUuShsv3oUtSIQbyO3BmLgALg1iWJKkZz9AlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5Jhejm4BrHmEGNtTK18zZ2TB/myg6M3XIsG9fYOV0Ys0fj5Rm6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEI0LvSIOCMi7oyIfRFxf0RcM8hgkqR62ozlchjYkZn3RMQLgN0RcXtmfmNA2SRJNTQ+Q8/MxzLznur2/wD7gC2DCiZJqicys/1CIqaAu4CzMvPpVT+bA+YAJicnt83Pz/e93KWlJSYmJgDYc+BQrUzTWzbXmr+OlbnqqPscmpg8AR5/duirqa2rueDo2Y52HA17fw57m6333Pp5XiuzDXIbtX3dNn1tjkKbbLOzs7szs7fRfK0LPSImgH8D3p+Ztx5t3l6vl4uLi30ve2FhgZmZGaD+0KD7r7uo1vx1rMxVxyiGN90xfZgP7OneqMhdzQVHz3a042jY+3PY22y959bP81qZbZDbqO3rtulrcxTaZIuIvgq91btcIuI5wKeAmzYqc0nScLV5l0sAHwP2ZeYHBxdJktREmzP084C3Aq+PiPuqrwsHlEuSVFPjC3SZ+e9ADDCLJKkFPykqSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKkQ3R0saoqMNFlR3YKD1ljXMgcE0WqMYUG1cBvXcxrmNVq97x/Rhrhzga7xtnpVu3L5pqOsGz9AlqRgWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQrQo9IrZHxIMR8a2I2DmoUJKk+hoXekQcB/wd8Cbg5cBlEfHyQQWTJNXT5gz9HOBbmfntzPwRMA9cMphYkqS62hT6FuC7K+4/Uk2TJI1BZGazB0b8LvDGzPyj6v5bgXMy8+pV880Bc9XdM4EHa6zmVOD7jQIOV1dzQXezdTUXdDdbV3NBd7N1NRe0y/YrmfmijWZqM3zuI8AZK+5vBR5dPVNmXg9c32QFEbGYmb1m8Yanq7mgu9m6mgu6m62ruaC72bqaC0aTrc0ll/8EXhoRL46I5wKXAp8dTCxJUl2Nz9Az83BE/AnwBeA44IbMvH9gySRJtbT6j0WZ+TngcwPKspZGl2pGoKu5oLvZupoLuputq7mgu9m6mgtGkK3xH0UlSd3iR/8lqRCdLPQuDSkQETdExMGI2Lti2ikRcXtEPFR9P3kMuc6IiDsjYl9E3B8R13Qo2/Mj4qsR8bUq2/uq6S+OiLurbP9c/TF95CLiuIi4NyJ2dSzX/ojYExH3RcRiNa0L+/OkiLglIh6ojrfXdCTXmdW2OvL1dES8oyPZ3lkd+3sj4ubqNTH046xzhd7BIQVuBLavmrYTuCMzXwrcUd0ftcPAjsx8GXAu8LZqO3Uh2/8Cr8/MVwJnA9sj4lzgL4EPVdl+AFw1hmwA1wD7VtzvSi6A2cw8e8Xb27qwPz8CfD4zfxV4Jcvbbuy5MvPBaludDWwDfgh8etzZImIL8Hagl5lnsfymkUsZxXGWmZ36Al4DfGHF/WuBa8ecaQrYu+L+g8Dp1e3TgQc7sN0+A1zQtWzAicA9wK+x/KGK49fazyPMs5XlF/nrgV1AdCFXte79wKmrpo11fwIvBL5D9fe2ruRaI+cbgP/oQjZ++in6U1h+48ku4I2jOM46d4bOz8eQApOZ+RhA9f20cYaJiCngVcDddCRbdVnjPuAgcDvwX8BTmXm4mmVc+/XDwLuBn1T3f7EjuQAS+GJE7K4+YQ3j358vAZ4APl5dpvpoRGzqQK7VLgVurm6PNVtmHgD+GngYeAw4BOxmBMdZFws91pjmW3HWERETwKeAd2Tm0+POc0Rm/l8u/yq8leWB3F621myjzBQRFwMHM3P3yslrzDqu4+28zHw1y5cb3xYRrxtTjpWOB14N/H1mvgp4hvFc9llXdS36zcAnx50FoLpmfwnwYuCXgU0s79PVBn6cdbHQ+xpSYMwej4jTAarvB8cRIiKew3KZ35SZt3Yp2xGZ+RSwwPJ1/pMi4shnH8axX88D3hwR+1keHfT1LJ+xjzsXAJn5aPX9IMvXgs9h/PvzEeCRzLy7un8LywU/7lwrvQm4JzMfr+6PO9tvAd/JzCcy88fArcBrGcFx1sVC/3kYUuCzwBXV7StYvn49UhERwMeAfZn5wY5le1FEnFTdPoHlA3wfcCfwlnFly8xrM3NrZk6xfFx9KTMvH3cugIjYFBEvOHKb5WvCexnz/szM7wHfjYgzq0nnA98Yd65VLuOnl1tg/NkeBs6NiBOr1+mRbTb842ycf8g4yh8VLgS+yfJ11/eMOcvNLF8H+zHLZytXsXzd9Q7goer7KWPI9ess/8r2deC+6uvCjmR7BXBvlW0v8OfV9JcAXwW+xfKvx88b436dAXZ1JVeV4WvV1/1HjvuO7M+zgcVqf/4LcHIXclXZTgT+G9i8YtrYswHvAx6ojv9/BJ43iuPMT4pKUiG6eMlFktSAhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiH+H59OIbKbrAMjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['age'].hist(bins=50);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "X = df[['pclass', 'sex', 'age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "lb = preprocessing.LabelBinarizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/anthonyn/anaconda3/envs/tensorflow/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "X['sex'] = lb.fit_transform(X['sex'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>54.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>58.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    pclass  sex   age\n",
       "1        1    0  38.0\n",
       "3        1    0  35.0\n",
       "6        1    1  54.0\n",
       "10       3    0   4.0\n",
       "11       1    0  58.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(182, 3)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>182.000000</td>\n",
       "      <td>182.000000</td>\n",
       "      <td>182.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.192308</td>\n",
       "      <td>0.516484</td>\n",
       "      <td>35.623187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.516411</td>\n",
       "      <td>0.501107</td>\n",
       "      <td>15.671615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.920000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>24.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>36.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>47.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>80.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           pclass         sex         age\n",
       "count  182.000000  182.000000  182.000000\n",
       "mean     1.192308    0.516484   35.623187\n",
       "std      0.516411    0.501107   15.671615\n",
       "min      1.000000    0.000000    0.920000\n",
       "25%      1.000000    0.000000   24.000000\n",
       "50%      1.000000    1.000000   36.000000\n",
       "75%      1.000000    1.000000   47.750000\n",
       "max      3.000000    1.000000   80.000000"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 182 entries, 1 to 889\n",
      "Data columns (total 3 columns):\n",
      "pclass    182 non-null int64\n",
      "sex       182 non-null int64\n",
      "age       182 non-null float64\n",
      "dtypes: float64(1), int64(2)\n",
      "memory usage: 5.7 KB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "y = df['survived']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    123\n",
       "0     59\n",
       "Name: survived, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[paper](http://ect.bell-labs.com/who/tkh/publications/papers/odt.pdf)\n",
    "\n",
    "* Ensemble of Decision Trees\n",
    "\n",
    "* Training via the bagging method (Repeated sampling with replacement)\n",
    "  * Bagging: Sample from samples\n",
    "  * RF: Sample from predictors. $m=sqrt(p)$ for classification and $m=p/3$ for regression problems.\n",
    "\n",
    "* Utilise uncorrelated trees"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Forest\n",
    "* Sample both observations and features of training data\n",
    "\n",
    "Bagging\n",
    "* Samples only observations at random\n",
    "* Decision Tree select best feature when splitting a node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "from sklearn.model_selection import cross_val_score, cross_val_predict\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "def print_score(clf, X_train, y_train, X_test, y_test, train=True):\n",
    "    '''\n",
    "    print the accuracy score, classification report and confusion matrix of classifier\n",
    "    '''\n",
    "    if train:\n",
    "        '''\n",
    "        training performance\n",
    "        '''\n",
    "        print(\"Train Result:\\n\")\n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_train, clf.predict(X_train))))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_train, clf.predict(X_train))))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_train, clf.predict(X_train))))\n",
    "\n",
    "        res = cross_val_score(clf, X_train, y_train, cv=10, scoring='accuracy')\n",
    "        print(\"Average Accuracy: \\t {0:.4f}\".format(np.mean(res)))\n",
    "        print(\"Accuracy SD: \\t\\t {0:.4f}\".format(np.std(res)))\n",
    "        \n",
    "    elif train==False:\n",
    "        '''\n",
    "        test performance\n",
    "        '''\n",
    "        print(\"Test Result:\\n\")        \n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_test, clf.predict(X_test))))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_test, clf.predict(X_test))))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_test, clf.predict(X_test))))    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import cross_val_score, cross_val_predict\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score\n",
    "def print_score(clf, X_train, X_test, y_train, y_test, train=True):\n",
    "    '''\n",
    "    v0.1 Follow the scikit learn library format in terms of input\n",
    "    print the accuracy score, classification report and confusion matrix of classifier\n",
    "    '''\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(y_train)\n",
    "    if train:\n",
    "        '''\n",
    "        training performance\n",
    "        '''\n",
    "        res = clf.predict(X_train)\n",
    "        print(\"Train Result:\\n\")\n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_train, \n",
    "                                                                res)))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_train, \n",
    "                                                                            res)))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_train, \n",
    "                                                                  res)))\n",
    "        print(\"ROC AUC: {0:.4f}\\n\".format(roc_auc_score(lb.transform(y_train), \n",
    "                                                      lb.transform(res))))\n",
    "\n",
    "        res = cross_val_score(clf, X_train, y_train, cv=10, scoring='accuracy')\n",
    "        print(\"Average Accuracy: \\t {0:.4f}\".format(np.mean(res)))\n",
    "        print(\"Accuracy SD: \\t\\t {0:.4f}\".format(np.std(res)))\n",
    "        \n",
    "    elif train==False:\n",
    "        '''\n",
    "        test performance\n",
    "        '''\n",
    "        res_test = clf.predict(X_test)\n",
    "        print(\"Test Result:\\n\")        \n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_test, \n",
    "                                                                res_test)))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_test, \n",
    "                                                                            res_test)))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_test, \n",
    "                                                                  res_test)))   \n",
    "        print(\"ROC AUC: {0:.4f}\\n\".format(roc_auc_score(lb.transform(y_test), \n",
    "                                                      lb.transform(res_test))))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "rf_clf = RandomForestClassifier(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.9325\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.90      0.88      0.89        50\n",
      "          1       0.95      0.96      0.95       113\n",
      "\n",
      "avg / total       0.93      0.93      0.93       163\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[ 44   6]\n",
      " [  5 108]]\n",
      "\n",
      "ROC AUC: 0.9179\n",
      "\n",
      "Average Accuracy: \t 0.7901\n",
      "Accuracy SD: \t\t 0.0766\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.7368\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.70      0.78      0.74         9\n",
      "          1       0.78      0.70      0.74        10\n",
      "\n",
      "avg / total       0.74      0.74      0.74        19\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[7 2]\n",
      " [3 7]]\n",
      "\n",
      "ROC AUC: 0.7389\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_score(rf_clf, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(rf_clf, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "rf_clf = RandomForestClassifier(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "params_grid = {\"max_depth\": [3, None],\n",
    "               \"min_samples_split\": [2, 3, 10],\n",
    "               \"min_samples_leaf\": [1, 3, 10],\n",
    "               \"bootstrap\": [True, False],\n",
    "               \"criterion\": ['gini', 'entropy']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "grid_search = GridSearchCV(rf_clf, params_grid,\n",
    "                           n_jobs=-1, cv=5,\n",
    "                           verbose=1, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 280 tasks      | elapsed:    2.3s\n",
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    2.8s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'max_depth': [3, None], 'min_samples_split': [2, 3, 10], 'min_samples_leaf': [1, 3, 10], 'bootstrap': [True, False], 'criterion': ['gini', 'entropy']},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=1)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7975460122699386"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'class_weight': None,\n",
       " 'criterion': 'entropy',\n",
       " 'max_depth': None,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 10,\n",
       " 'n_jobs': 1,\n",
       " 'oob_score': False,\n",
       " 'random_state': 42,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_estimator_.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.9325\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.90      0.88      0.89        50\n",
      "          1       0.95      0.96      0.95       113\n",
      "\n",
      "avg / total       0.93      0.93      0.93       163\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[ 44   6]\n",
      " [  5 108]]\n",
      "\n",
      "ROC AUC: 0.9179\n",
      "\n",
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n",
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    2.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    2.6s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    2.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    2.4s finished\n",
      "[Parallel(n_jobs=-1)]: Done 280 tasks      | elapsed:    1.6s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    1.9s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    1.9s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    2.0s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    1.9s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 72 candidates, totalling 360 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    2.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Accuracy: \t 0.7779\n",
      "Accuracy SD: \t\t 0.0906\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.7368\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.70      0.78      0.74         9\n",
      "          1       0.78      0.70      0.74        10\n",
      "\n",
      "avg / total       0.74      0.74      0.74        19\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[7 2]\n",
      " [3 7]]\n",
      "\n",
      "ROC AUC: 0.7389\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 360 out of 360 | elapsed:    2.1s finished\n"
     ]
    }
   ],
   "source": [
    "print_score(grid_search, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(grid_search, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extra-Trees (Extremely Randomized Trees) Ensemble\n",
    "\n",
    "[scikit-learn](http://scikit-learn.org/stable/modules/ensemble.html#bagging)\n",
    "\n",
    "* Random Forest is build upon Decision Tree\n",
    "* Decision Tree node splitting is based on gini or entropy or some other algorithms\n",
    "* Extra-Trees make use of random thresholds for each feature unlike Decision Tree\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "xt_clf = ExtraTreesClassifier(random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',\n",
       "           max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "           oob_score=False, random_state=42, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xt_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.9448\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.87      0.96      0.91        50\n",
      "          1       0.98      0.94      0.96       113\n",
      "\n",
      "avg / total       0.95      0.94      0.95       163\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[ 48   2]\n",
      " [  7 106]]\n",
      "\n",
      "ROC AUC: 0.9490\n",
      "\n",
      "Average Accuracy: \t 0.7967\n",
      "Accuracy SD: \t\t 0.0704\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.6842\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.64      0.78      0.70         9\n",
      "          1       0.75      0.60      0.67        10\n",
      "\n",
      "avg / total       0.70      0.68      0.68        19\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[7 2]\n",
      " [4 6]]\n",
      "\n",
      "ROC AUC: 0.6889\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_score(xt_clf, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(xt_clf, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "def eval_gini(y_true, y_prob):\n",
    "    \"\"\"\n",
    "    Original author CPMP : https://www.kaggle.com/cpmpml\n",
    "    In kernel : https://www.kaggle.com/cpmpml/extremely-fast-gini-computation\n",
    "    \"\"\"\n",
    "    y_true = np.asarray(y_true)\n",
    "    y_true = y_true[np.argsort(y_prob)]\n",
    "    ntrue = 0\n",
    "    gini = 0\n",
    "    delta = 0\n",
    "    n = len(y_true)\n",
    "    for i in range(n-1, -1, -1):\n",
    "        y_i = y_true[i]\n",
    "        ntrue += y_i\n",
    "        gini += y_i * delta\n",
    "        delta += 1 - y_i\n",
    "    gini = 1 - 2 * gini / (ntrue * (n - ntrue))\n",
    "    return gini"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}