Assignment_04_Random_Forest_GS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Machine Learning Assignment 4\n",
    "\n",
    "Robert Knox"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Imports needed for this notebook\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "p\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "#to make nice looking confusion matrices\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.ensemble.partial_dependence import plot_partial_dependence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'pd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-bdf47a19371e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#NOTE: I manually deleted the row column\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_excel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mio\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"default of credit card clients.xls\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msheet_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'Data'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mheader\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'pd' is not defined"
     ]
    }
   ],
   "source": [
    "#NOTE: I manually deleted the row column\n",
    "df = pd.read_excel(io=\"default of credit card clients.xls\",sheet_name='Data',header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleaned_df= df.dropna(how='any', axis = 0)\n",
    "#check that all NA's are dropped\n",
    "cleaned_df.isna().any().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LIMIT_BAL</th>\n",
       "      <th>AGE</th>\n",
       "      <th>PAY_0</th>\n",
       "      <th>PAY_2</th>\n",
       "      <th>PAY_3</th>\n",
       "      <th>PAY_4</th>\n",
       "      <th>PAY_5</th>\n",
       "      <th>PAY_6</th>\n",
       "      <th>BILL_AMT1</th>\n",
       "      <th>BILL_AMT2</th>\n",
       "      <th>BILL_AMT3</th>\n",
       "      <th>BILL_AMT4</th>\n",
       "      <th>BILL_AMT5</th>\n",
       "      <th>BILL_AMT6</th>\n",
       "      <th>PAY_AMT1</th>\n",
       "      <th>PAY_AMT2</th>\n",
       "      <th>PAY_AMT3</th>\n",
       "      <th>PAY_AMT4</th>\n",
       "      <th>PAY_AMT5</th>\n",
       "      <th>PAY_AMT6</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>3.000000e+04</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>3.000000e+04</td>\n",
       "      <td>30000.00000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "      <td>30000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>167484.322667</td>\n",
       "      <td>35.485500</td>\n",
       "      <td>-0.016700</td>\n",
       "      <td>-0.133767</td>\n",
       "      <td>-0.166200</td>\n",
       "      <td>-0.220667</td>\n",
       "      <td>-0.266200</td>\n",
       "      <td>-0.291100</td>\n",
       "      <td>51223.330900</td>\n",
       "      <td>49179.075167</td>\n",
       "      <td>4.701315e+04</td>\n",
       "      <td>43262.948967</td>\n",
       "      <td>40311.400967</td>\n",
       "      <td>38871.760400</td>\n",
       "      <td>5663.580500</td>\n",
       "      <td>5.921163e+03</td>\n",
       "      <td>5225.68150</td>\n",
       "      <td>4826.076867</td>\n",
       "      <td>4799.387633</td>\n",
       "      <td>5215.502567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>129747.661567</td>\n",
       "      <td>9.217904</td>\n",
       "      <td>1.123802</td>\n",
       "      <td>1.197186</td>\n",
       "      <td>1.196868</td>\n",
       "      <td>1.169139</td>\n",
       "      <td>1.133187</td>\n",
       "      <td>1.149988</td>\n",
       "      <td>73635.860576</td>\n",
       "      <td>71173.768783</td>\n",
       "      <td>6.934939e+04</td>\n",
       "      <td>64332.856134</td>\n",
       "      <td>60797.155770</td>\n",
       "      <td>59554.107537</td>\n",
       "      <td>16563.280354</td>\n",
       "      <td>2.304087e+04</td>\n",
       "      <td>17606.96147</td>\n",
       "      <td>15666.159744</td>\n",
       "      <td>15278.305679</td>\n",
       "      <td>17777.465775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>10000.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-2.000000</td>\n",
       "      <td>-165580.000000</td>\n",
       "      <td>-69777.000000</td>\n",
       "      <td>-1.572640e+05</td>\n",
       "      <td>-170000.000000</td>\n",
       "      <td>-81334.000000</td>\n",
       "      <td>-339603.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>50000.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>3558.750000</td>\n",
       "      <td>2984.750000</td>\n",
       "      <td>2.666250e+03</td>\n",
       "      <td>2326.750000</td>\n",
       "      <td>1763.000000</td>\n",
       "      <td>1256.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>8.330000e+02</td>\n",
       "      <td>390.00000</td>\n",
       "      <td>296.000000</td>\n",
       "      <td>252.500000</td>\n",
       "      <td>117.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>140000.000000</td>\n",
       "      <td>34.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>22381.500000</td>\n",
       "      <td>21200.000000</td>\n",
       "      <td>2.008850e+04</td>\n",
       "      <td>19052.000000</td>\n",
       "      <td>18104.500000</td>\n",
       "      <td>17071.000000</td>\n",
       "      <td>2100.000000</td>\n",
       "      <td>2.009000e+03</td>\n",
       "      <td>1800.00000</td>\n",
       "      <td>1500.000000</td>\n",
       "      <td>1500.000000</td>\n",
       "      <td>1500.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>240000.000000</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>67091.000000</td>\n",
       "      <td>64006.250000</td>\n",
       "      <td>6.016475e+04</td>\n",
       "      <td>54506.000000</td>\n",
       "      <td>50190.500000</td>\n",
       "      <td>49198.250000</td>\n",
       "      <td>5006.000000</td>\n",
       "      <td>5.000000e+03</td>\n",
       "      <td>4505.00000</td>\n",
       "      <td>4013.250000</td>\n",
       "      <td>4031.500000</td>\n",
       "      <td>4000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1000000.000000</td>\n",
       "      <td>79.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>964511.000000</td>\n",
       "      <td>983931.000000</td>\n",
       "      <td>1.664089e+06</td>\n",
       "      <td>891586.000000</td>\n",
       "      <td>927171.000000</td>\n",
       "      <td>961664.000000</td>\n",
       "      <td>873552.000000</td>\n",
       "      <td>1.684259e+06</td>\n",
       "      <td>896040.00000</td>\n",
       "      <td>621000.000000</td>\n",
       "      <td>426529.000000</td>\n",
       "      <td>528666.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            LIMIT_BAL           AGE         PAY_0         PAY_2         PAY_3  \\\n",
       "count    30000.000000  30000.000000  30000.000000  30000.000000  30000.000000   \n",
       "mean    167484.322667     35.485500     -0.016700     -0.133767     -0.166200   \n",
       "std     129747.661567      9.217904      1.123802      1.197186      1.196868   \n",
       "min      10000.000000     21.000000     -2.000000     -2.000000     -2.000000   \n",
       "25%      50000.000000     28.000000     -1.000000     -1.000000     -1.000000   \n",
       "50%     140000.000000     34.000000      0.000000      0.000000      0.000000   \n",
       "75%     240000.000000     41.000000      0.000000      0.000000      0.000000   \n",
       "max    1000000.000000     79.000000      8.000000      8.000000      8.000000   \n",
       "\n",
       "              PAY_4         PAY_5         PAY_6      BILL_AMT1      BILL_AMT2  \\\n",
       "count  30000.000000  30000.000000  30000.000000   30000.000000   30000.000000   \n",
       "mean      -0.220667     -0.266200     -0.291100   51223.330900   49179.075167   \n",
       "std        1.169139      1.133187      1.149988   73635.860576   71173.768783   \n",
       "min       -2.000000     -2.000000     -2.000000 -165580.000000  -69777.000000   \n",
       "25%       -1.000000     -1.000000     -1.000000    3558.750000    2984.750000   \n",
       "50%        0.000000      0.000000      0.000000   22381.500000   21200.000000   \n",
       "75%        0.000000      0.000000      0.000000   67091.000000   64006.250000   \n",
       "max        8.000000      8.000000      8.000000  964511.000000  983931.000000   \n",
       "\n",
       "          BILL_AMT3      BILL_AMT4      BILL_AMT5      BILL_AMT6  \\\n",
       "count  3.000000e+04   30000.000000   30000.000000   30000.000000   \n",
       "mean   4.701315e+04   43262.948967   40311.400967   38871.760400   \n",
       "std    6.934939e+04   64332.856134   60797.155770   59554.107537   \n",
       "min   -1.572640e+05 -170000.000000  -81334.000000 -339603.000000   \n",
       "25%    2.666250e+03    2326.750000    1763.000000    1256.000000   \n",
       "50%    2.008850e+04   19052.000000   18104.500000   17071.000000   \n",
       "75%    6.016475e+04   54506.000000   50190.500000   49198.250000   \n",
       "max    1.664089e+06  891586.000000  927171.000000  961664.000000   \n",
       "\n",
       "            PAY_AMT1      PAY_AMT2      PAY_AMT3       PAY_AMT4  \\\n",
       "count   30000.000000  3.000000e+04   30000.00000   30000.000000   \n",
       "mean     5663.580500  5.921163e+03    5225.68150    4826.076867   \n",
       "std     16563.280354  2.304087e+04   17606.96147   15666.159744   \n",
       "min         0.000000  0.000000e+00       0.00000       0.000000   \n",
       "25%      1000.000000  8.330000e+02     390.00000     296.000000   \n",
       "50%      2100.000000  2.009000e+03    1800.00000    1500.000000   \n",
       "75%      5006.000000  5.000000e+03    4505.00000    4013.250000   \n",
       "max    873552.000000  1.684259e+06  896040.00000  621000.000000   \n",
       "\n",
       "            PAY_AMT5       PAY_AMT6  \n",
       "count   30000.000000   30000.000000  \n",
       "mean     4799.387633    5215.502567  \n",
       "std     15278.305679   17777.465775  \n",
       "min         0.000000       0.000000  \n",
       "25%       252.500000     117.750000  \n",
       "50%      1500.000000    1500.000000  \n",
       "75%      4031.500000    4000.000000  \n",
       "max    426529.000000  528666.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#clean up the columns that should be category and boolean respectively\n",
    "dt = {'SEX':'category',\n",
    "        'EDUCATION':'category',\n",
    "        'MARRIAGE':'category',\n",
    "        'default payment next month':'bool'}\n",
    "cleaned_df = df.astype(dt)\n",
    "cleaned_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfX = cleaned_df.iloc[:,0:cleaned_df.columns.size-1]\n",
    "dfy = cleaned_df.iloc[:,cleaned_df.columns.size-1]\n",
    "#Build train & test using sklearn's train_test_split function\n",
    "X_train, X_test, y_train, y_test = train_test_split(dfX, dfy, test_size=0.30, random_state=42)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 1 - no optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Bob\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:248: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "yhat: [False False False ... False False False] \n",
      "yhat_probs: [[0.7 0.3]\n",
      " [0.8 0.2]\n",
      " [0.7 0.3]\n",
      " ...\n",
      " [0.5 0.5]\n",
      " [0.8 0.2]\n",
      " [1.  0. ]]\n"
     ]
    }
   ],
   "source": [
    "rforest1 = RandomForestClassifier()\n",
    "rforest1.fit(X_train,y_train)\n",
    "yhat = rforest1.predict(X_test)\n",
    "yhat_probs = rforest1.predict_proba(X_test)\n",
    "print('yhat:',yhat,'\\nyhat_probs:',yhat_probs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 1 Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[6626  414]\n",
      " [1343  617]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUgAAAEYCAYAAAA+mm/EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xe8FPW9//HXGxAVsWBDBRULIsZrQUQsUWLBmoDGFjUSG8EYfxpjYgnX7rXEq7FEYxQVjQVixYqo8caCSBGNHUSNKCpIsSFSPr8/5nvICrtnFzx79uye99PHPM7Md74z892z8jnfNjOKCMzMbHEtKl0AM7OmygHSzKwAB0gzswIcIM3MCnCANDMrwAHSzKwAB0hD0vKSHpQ0S9Lfv8d5Dpf0eEOWrVIk/VDSW5Uuh1WWPA+yekg6DDgF2BT4AhgPXBgRz37P8/4cOBHYISLmfe+CNnGSAugcERMrXRZr2lyDrBKSTgH+BPwP0B5YD7gW6NMAp18feLs5BMdSSGpV6TJYExERXpr4AqwMfAkcVE+eZckC6Edp+ROwbNrXC5gM/Bb4FJgCHJX2nQt8C8xN1zgGOAf4W865OwEBtErbvwAmkdVi3wUOz0l/Nue4HYDRwKz0c4ecfU8D5wPPpfM8Dqxe4LPVlf/3OeXvC+wDvA1MB87Myd8DGAnMTHmvAVqnff9Mn+Wr9HkPyTn/acDHwG11aemYjdI1uqXtdYBpQK9K/7/hpbyLa5DVYXtgOeC+evL8AegJbAVsSRYkBubsX4ss0HYgC4J/ltQuIs4mq5UOiYi2ETGovoJIWgG4Ctg7IlYkC4Lj8+RbFXg45V0NuBx4WNJqOdkOA44C1gRaA6fWc+m1yH4HHYCzgBuAI4BtgB8CZ0naMOWdD/wGWJ3sd7cb8CuAiNg55dkyfd4hOedflaw23T/3whHxDlnwvF1SG+Bm4JaIeLqe8loNcICsDqsB06L+JvDhwHkR8WlETCWrGf48Z//ctH9uRDxCVnvqspTlWQBsLmn5iJgSEa/lybMvMCEibouIeRFxJ/Am8OOcPDdHxNsRMRsYShbcC5lL1t86F7iLLPhdGRFfpOu/BmwBEBFjI+KFdN33gOuBXUr4TGdHxJxUnu+IiBuACcAoYG2yP0hW4xwgq8NnwOpF+sbWAd7P2X4/pS08xyIB9mug7ZIWJCK+ImuWDgCmSHpY0qYllKeuTB1ytj9egvJ8FhHz03pdAPskZ//suuMlbSLpIUkfS/qcrIa8ej3nBpgaEd8UyXMDsDlwdUTMKZLXaoADZHUYCXxD1u9WyEdkzcM666W0pfEV0CZne63cnRExPCL2IKtJvUkWOIqVp65MHy5lmZbEdWTl6hwRKwFnAipyTL3TOSS1JevXHQSck7oQrMY5QFaBiJhF1u/2Z0l9JbWRtIykvSVdmrLdCQyUtIak1VP+vy3lJccDO0taT9LKwBl1OyS1l/ST1Bc5h6ypPj/POR4BNpF0mKRWkg4BNgMeWsoyLYkVgc+BL1Pt9vhF9n8CbLjYUfW7EhgbEceS9a3+5XuX0po8B8gqERGXk82BHAhMBT4Afg3cn7JcAIwBXgH+BYxLaUtzrRHAkHSusXw3qLUgGw3/iGxkdxfSAMgi5/gM2C/l/YxsBHq/iJi2NGVaQqeSDQB9QVa7HbLI/nOAwZJmSjq42Mkk9QH2IutWgOx76Cbp8AYrsTVJnihuZlaAa5BmZgU4QJqZFeAAaWZWgAOkmVkBNXtTvlotH2q9YqWLYYvYuut6lS6C5TFu3NhpEbFGQ56z5UrrR8xb7KakxcTsqcMjYq+GvHZDqd0A2XpFlu1SdAaHNbLnRl1T6SJYHssvo0XvevreYt7skv4NfjP+z8XucqqYmg2QZlZhErRoWelSfC8OkGZWPqruYQ4HSDMrHxW7Bb5pc4A0szKRa5BmZnkJ90GameUnN7HNzApyE9vMrADXIM3M8vA8SDOzeriJbWaWj6f5mJkV1sJ9kGZmi/M8SDOzQtzENjMrzNN8zMwKqPIaZHWX3syarrp5kMWWkk6lVSTdLelNSW9I2l7SqpJGSJqQfrZLeSXpKkkTJb0iqVvOefql/BMk9St2XQdIMysfqfhSmiuBxyJiU2BL4A3gdODJiOgMPJm2AfYGOqelP3BdVhStCpwNbAf0AM6uC6qFOECaWZmkQZpiS7GzSCsBOwODACLi24iYCfQBBqdsg4G+ab0PcGtkXgBWkbQ2sCcwIiKmR8QMYARQ77twHCDNrHxKq0GuLmlMztJ/kbNsCEwFbpb0kqQbJa0AtI+IKQDp55opfwfgg5zjJ6e0QukFeZDGzMpDghYlhZhpEdG9nv2tgG7AiRExStKV/Kc5nffKedKinvSCXIM0s/JpmD7IycDkiBiVtu8mC5ifpKYz6eenOfnXzTm+I/BRPekFOUCaWfk0QB9kRHwMfCCpS0raDXgdGAbUjUT3Ax5I68OAI9Nodk9gVmqCDwd6S2qXBmd6p7SC3MQ2s/JpuIniJwK3S2oNTAKOIqvgDZV0DPBv4KCU9xFgH2Ai8HXKS0RMl3Q+MDrlOy8iptd3UQdIMyuPBnweZESMB/L1U+6WJ28AJxQ4z03ATaVe1wHSzMpGvtXQzGxxwgHSzCw/kX9iTRVxgDSzMhEtWlT3RBkHSDMrGzexzcwKcIA0M8vHfZBmZvnJfZBmZoW5iW1mVoADpJlZPu6DNDPLz32QZmb1cBPbzKyQ6o6PDpBmViZyDdLMrCD3QZqZ5SFU9TXI6g7vVWzltstzxx+PYfy9A3npnoFst8UGABx/6C68fN9/M/buP3DhSX0A2HW7TXnu9t8zeuiZPHf779ll200WnmeZVi25ZuDPeOX+sxh/70D67rZVRT5PrZo/fz49u2/NAX32A+C6P1/DDzbdmOWXEdOmTVss/5jRo1lh2Zbce8/djV3UpkklLE2Ya5AVctnvD+Tx51/nsN8NYplWLWmzXGt27t6Z/Xr9F9sefBHfzp3HGu3aAvDZzC858OTrmTJ1FptttDYPXnsCG+05EIDTjt2TqdO/YIu+5yGJVVduU8mPVXOuuepKunTtyheffw7A9jvsyD777kfv3Xstlnf+/PkMPPM09ui9ZyOXsolS9Texq7v0VWrFFZZjp24bcct9IwGYO28+s76cTf+DfshlN4/g27nzAJg640sAXn5rMlOmzgLg9XemsGzrZWi9TPa3rV+f7fnjTY8DEBF8NvOrxv44NWvy5Mk89ujDHHX0sQvTttp6a9bv1Clv/muvuZq++/+UNdZYM+/+5khS0aUpc4CsgA06rMa0GV/y13OPYOSdp3HtWYfRZrnWbLz+muy49Ub889ZTefzGk9hms/UWO3b/3bfi5bc+4Nu581i57fIAnH3Cfjx/x2ncfunRrLnqio39cWrW7357MhdedGlJtaAPP/yQYQ/cx3G/HNAIJasiVd7ELluAlDRf0vicpVM9eTtJerVcZWlqWrVqyVabrssNf3+G7X92CV/PnsOpR+9Bq5YtaLdSG3Y+8jLOvOJ+/nbp0d85ruuGa3HB/+vDry+4K52nBR3XasfI8ZPY4bBLGPXKe1z0m/0r8ZFqziMPP8Saa6xJt222KSn/7357Mhf8zyW0bNkwb/GrFdVegyxnH+TsiPCIQR4ffjKDDz+dyehX3wfgvifG89uj9uDDT2Zy/5MvAzDmtfdZsCBYvV1bps34kg5rrsKQy/tz7H/fxruTs8GBz2Z+xVez5/DAU9kx944YR7++21fmQ9WYkc8/x0MPDeOxxx5hzjff8Pnnn3PUkUdw861/y5t/3NgxHHnEoQB8Nm0awx97hFatWvGTPn0bs9hNilT9txo2aulTTfEZSePSskOePD+Q9GKqdb4iqXNKPyIn/XpJVfun+pPPvmDyxzPovH7WV9WrRxfenPQxDz79Cr16ZCPUG6+3Jq2XacW0GV+yctvluffqAZx19TBGvjzpO+d65J+vsnP3zjnnmdK4H6ZGnX/hRbzz3mTemvget95+F71+tGvB4Ajw5oR3eWvie7w18T32P+BA/nT1tc06ONap9hpkOQPk8jnN6/tS2qfAHhHRDTgEuCrPcQOAK1PtszswWVLXlH/HlD4fOHzRAyX1lzRG0piYN7scn6nBnHLJ37n5f37Bi0POYMsuHbh00HAG3z+SDTqsxpi/n8mtFx/FsWfdBsCAQ3dmo3XX4PTj9uKFu07nhbtOXzjCPfDK+xk4YB9eHHIGh+3bg9Mvv6++y9r39Oerr2KjTh35cPJktu22Bcf3P7b4Qc1ZA/VBSnpP0r9SPBmT0laVNELShPSzXUqXpKskTUyVrG455+mX8k+Q1K/odSNiyT5wiSR9GRFtF0lbGbgGqAtym0REm9Q/+VBEbC7pMOAPwK3AvRExQdKvgTPJAizA8sCdEXFOoeu3aLNmLNvl4Ab+VPZ9zRh9TaWLYHksv4zGRkT3hjznsu07R4fDryya790r9i16bUnvAd0jYlpO2qXA9Ii4WNLpQLuIOE3SPsCJwD7AdmQVru0krQqMIat4BTAW2CYiZhS6bmPPg/wN8AmwJVnt9ZtFM0TEHZJGAfsCwyUdS/Z3ZnBEnNGYhTWzpSdBixZlbUL3AXql9cHA08BpKf3WyGp/L0haRdLaKe+IiJielU8jgL2AOwtdoLF7UFcGpkTEAuDnwGL9iJI2BCZFxFXAMGAL4EngQElrpjyrSlq/8YptZkuueP9j6oNcva5rLC3985wsgMcljc3Z3z4ipgCkn3UTUDsAH+QcOzmlFUovqLFrkNcC90g6CPgHkG9W8yHAEZLmAh8D50XEdEkDyX5BLYC5wAnA+41UbjNbCiWOwUwroXm/Y0R8lCpJIyS9Wd9l86RFPekFlS1ALtr/mNImkNUI65yR0t8DNk/rFwEX5Tl2CDCkHGU1s/JoqFHqiPgo/fw0Dfr2AD6RtHZETElN6LoxisnAujmHdwQ+Sum9Fkl/ur7rVvckJTNrsiRo2VJFl+Ln0QqSVqxbB3oDr5J1wdWNRPcDHkjrw4Aj02h2T2BWaoIPB3pLapdGvHuntIL8sAozK5sGqkC2B+5LtdFWwB0R8Zik0cBQSccA/wYOSvkfIRvBngh8DRwFkLrqzgdGp3zn1Q3YFOIAaWZl0xBN7IiYRDbzZdH0z4Dd8qQH2RhFvnPdBNxU6rUdIM2sPNRgNciKcYA0s7Lwa1/NzOrhGqSZWQFN/WEUxThAmll5uA/SzCw/UfZ7scvOAdLMysZNbDOzAqo8PjpAmlmZyDVIM7O8snmQDpBmZnlVeQXSAdLMysdNbDOzfDwP0swsv2wepO/FNjPLyzVIM7MC3AdpZpaP+yDNzPLzPEgzs3q0qPIqZMEAKWml+g6MiM8bvjhmVkuqPD7WW4N8jcVftl23HcB6ZSyXmVU51fK92BGxbqF9ZmalaFnlfZAlzeKUdKikM9N6R0nblLdYZlYLpOJLU1Y0QEq6BvgR8POU9DXwl3IWysyqn8hGsov9V9K5pJaSXpL0UNreQNIoSRMkDZHUOqUvm7Ynpv2dcs5xRkp/S9KepVy3lBrkDhHxS+AbgIiYDrQu6VOZWbPWQsWXEp0EvJGzfQlwRUR0BmYAx6T0Y4AZEbExcEXKh6TNgEOBHwB7AddKalm0/CUUbK6kFmQDM0haDVhQyicys2ZM2TzIYkvx06gjsC9wY9oWsCtwd8oyGOib1vukbdL+3VL+PsBdETEnIt4FJgI9il27lAD5Z+AeYA1J5wLPkqKymVkhIpsHWWwpwZ+A3/OfitlqwMyImJe2JwMd0noH4AOAtH9Wyr8wPc8xBRWdKB4Rt0oaC+yekg6KiFeLHWdmVuIgzOqSxuRs/zUi/podr/2ATyNirKRedafNc44osq++Ywoq9U6alsDcdMLqfn6RmTUKqeTXvk6LiO4F9u0I/ETSPsBywEpkNcpVJLVKtcSOwEcp/2RgXWCypFbAysD0nPQ6uccUVMoo9h+AO4F10knvkHRGsePMzL5vEzsizoiIjhHRiWyQ5amIOBz4B3BgytYPeCCtD0vbpP1PRUSk9EPTKPcGQGfgxWLlL6UGeQSwTUR8DSDpQmAscFEJx5pZM1bGaY6nAXdJugB4CRiU0gcBt0maSFZzPBQgIl6TNBR4HZgHnBAR84tdpJQA+f4i+VoBk0r9FGbWfDXkrYYR8TTwdFqfRJ5R6Ij4BjiowPEXAhcuyTXre1jFFWR9jl8Dr0kanrZ7k41km5kVJKnqbzWsrwZZN1L9GvBwTvoL5SuOmdWSpn4rYTH1PaxiUKF9ZmalqNmn+dSRtBFZu30zsmF2ACJikzKWy8yqXDZRvNKl+H5KmdN4C3Az2efdGxgK3FXGMplZjWigO2kqppQA2SYihgNExDsRMZDs6T5mZgVJ1R8gS5nmMyfd7P2OpAHAh8Ca5S2WmdWCJh7/iiolQP4GaAv8P7K+yJWBo8tZKDOrDTU/SBMRo9LqF/znoblmZvUSNTwPUtJ91PO0i4g4oCwlMrPaUAWvVCimvhrkNY1WijLYrHNH7n300koXwxYxZ27R21+thtRsEzsinmzMgphZ7an2ZyOW+jxIM7MlIqr/ta8OkGZWNlUeH0sPkJKWjYg55SyMmdWO7L3X1R0hS3mieA9J/wImpO0tJV1d9pKZWdVrwNe+VkQpfahXAfsBnwFExMv4VkMzK6KuD7LY0pSV0sRuERHvL1JV9lwNMyuqOYxifyCpBxCSWgInAm+Xt1hmVguqvAuypAB5PFkzez3gE+CJlGZmVpCq4Gk9xZRyL/anpDeDmZktiZZV3sYu5YniN5DnnuyI6F+WEplZTcieKF7jNUiyJnWd5YD9gQ/KUxwzqyVVHh9LamIPyd2WdBswomwlMrPaUAXzHItZmh6CDYD1G7ogZlZbBLSUii5FzyMtJ+lFSS9Lek3SuSl9A0mjJE2QNERS65S+bNqemPZ3yjnXGSn9LUl7Frt2KXfSzJA0PS0zyWqPZxb9VGbW7DXQnTRzgF0jYktgK2AvST2BS4ArIqIzMAM4JuU/BpgRERsDV6R8SNqMbMD5B8BewLVp6mLh8te3M72LZktgjbS0i4gNI2JoSR/LzJo1SUWXYiLzZdpcJi0B7ArcndIHA33Tep+0Tdq/W4plfYC7ImJORLwLTAR61HftegNkRARwX0TMT0vBJ4ybmeWqey92CTXI1SWNyVkWmyEjqaWk8cCnZK3Yd4CZETEvZZkMdEjrHUgDyWn/LGC13PQ8x+RVyij2i5K6RcS4EvKamWVU8vMgp0VE9/oyRMR8YCtJqwD3AV3zZfvPlfPuK5ReUH3vpGmVou9OwHGS3gG+SheJiOhW34nNrHmrq0E2pIiYKelpoCewSk6c6gh8lLJNBtYFJktqRfYm1uk56XVyj8mrvhrki0A3/tOuNzNbIg0xD1LSGsDcFByXB3YnG3j5B3AgcBfQD3ggHTIsbY9M+5+KiJA0DLhD0uXAOkBnsjhXUH0BUgAR8c7SfjAza85Ei7yt2iW2NjA4jTi3AIZGxEOSXgfuknQB8BIwKOUfBNwmaSJZzfFQgIh4TdJQ4HVgHnBCaroXVF+AXEPSKYV2RsTlpX02M2uOpIa5FzsiXgG2zpM+iTyj0BHxDXBQgXNdCFxY6rXrC5Atgbbk79g0Myuqlu/FnhIR5zVaScyspojavhe7yj+amVVaLdcgd2u0UphZzcnuxa50Kb6fggEyIqY3ZkHMrMbUwGtfS34vtpnZkqru8OgAaWZlUve4s2rmAGlmZVPl8dEB0szKpbTHmTVlDpBmVhZi6V5Z0JQ4QJpZ2dTyPEgzs6XnaT5mZvm5iW1mVg/XIM3MCqj292I7QJpZWWRN7OqOkA6QZlY2Vd7CdoA0s3IRcg3SzGxxvhfbzKwQuYltZlZQtQfIap/HWbXO+M0Att98ffbr1X1h2p8uOY8f79qDPrv35OhDfswnH0/5zjGvjB9L1w4r8thD9wHw4Qf/5oDeO9Jn957su0t37hx8Y6N+hlo3c+ZMjjzsYLbd6gf02HpzXhw1kvvvvZue22xBuxWW4aWxYxbmHXrXHey03TYLl3YrLMMrL4+vYOmbBpXwX1PmAFkhBxx8BDfecf930o791ck8+NSLPPDEC/TaY2/+fPlFC/fNnz+fyy4YyE69dl+Ytkb7tbjrwad44IkXGPrI09xwzf8uFlRt6Z3+u9+w+x57Mnr8azw7ahybdOlK181+wG13/p0ddvrhd/IefOhhPDtqLM+OGsv1g25hvfU7scWWW1Wo5E1DXR9ksaUpcxO7Qrbdficmf/D+d9LarrjSwvXZX3/1nbsQbht0HXvu25d/jR+7MK1169YL17+dM4cFCxaUscTNy+eff87zzz7DdX+9Cch+161bt2aVVVYpeuw9Q+/iwIMOKXcRq0ITj39FuQbZxFxx0Tnsss0mPHjvEE763UAAPpnyEU88+iCHHnnsYvmnfDiZH+/ag17bdOG4X59C+7XWbuwi16T33p3E6quvzq9+eQw/7NmdE4/vz1dffVXSsffe83d+evChZS5hdWiIJrakdSX9Q9Ibkl6TdFJKX1XSCEkT0s92KV2SrpI0UdIrkrrlnKtfyj9BUr9i126UAClpNUnj0/KxpA9ztlsXP0Pz8ZszzuH/xr7Njw84hL/dfD0AF571e04deD4tW7ZcLP/aHTry4FMv8vjIf3Hf0NuZNvWTxi5yTZo/bx4vj3+JY479Jc+8MIY2K6zAFZddUvS4MS+Ook2bNmz2g80boZRNm8huNSy2lGAe8NuI6Ar0BE6QtBlwOvBkRHQGnkzbAHsDndPSH7gOsoAKnA1sB/QAzq4LqoU0SoCMiM8iYquI2Ar4C3BF3XZEfAsLo75rtMl++x/C4w9nfZSvvjyOUwb0Y9dtuzL8ofs59/STeeLRB7+Tv/1aa9O5S1fGjHq+EsWtOet06Mg6HTrSvcd2APTZ/wBeGf9S0ePuuXsIP3XzOiPRooSlmIiYEhHj0voXwBtAB6APMDhlGwz0Tet9gFsj8wKwiqS1gT2BERExPSJmACOAveq7dkUDkqSNJb0q6S/AOGBdSTNz9h8q6ca03l7SvZLGSHpRUs9Klbtc3ps0ceH6U48/zIYbd8nWX3ydp0a/wVOj32DP/fpy9sV/Yve9f8zHH33IN7NnAzBr5gzGjX6BDTbqXJGy15r2a61Fx44dmfD2WwD83z+eokvXrvUes2DBAh649x4HyBwqYQFWT/+u65b+Bc8ndQK2BkYB7SNiCmRBFFgzZesAfJBz2OSUVii9oKYwSLMZcFREDJBUX3muAi6NiBfSL+kh4DvtmPSL7Q+wTod1y1PaBnLK8f148flnmDH9M3bu1pkTTx3IP58czrvvvI1atKBDx/U495Kr6j3HOxPe5OJzz0ASEcHRA06iS1c37RrKJf97JccddSTfzv2WTp024NrrB/HgA/dz2m9PYtq0qRz805/wX1tsyb3DHgXguWf/yTodOtBpgw0rXPKmIWtil9SGnhYR3YtlktQWuAc4OSI+r+dRavl2RD3pha8ZUe/+BifpHODLiLhM0sbAo6kPgRQgp0XEKmn7UGD3iDhW0md8N/qvAWwcEbPzXWfzLbvFvcOfLedHsaXQfuVlK10Ey2OVNq3GlhKklkTX/9o6br7vH0Xzbd+5XdFrS1qGrFI0PCIuT2lvAb0iYkpqQj8dEV0kXZ/W78zNV7dExC9T+nfy5dMU+vxyhwYX8N0ov1zOuoAeOX2XHQoFRzNrGiQVXUo4h4BBwBt1wTEZBtSNRPcDHshJPzKNa/QEZqUm+HCgt6R2aXCmd0orqCkEyIUiYgEwQ1LnNGCzf87uJ4AT6jYkNe9ZuGZVQCq+lGBH4OfArjmzX/YBLgb2kDQB2CNtAzwCTAImAjcAvwKIiOnA+cDotJyX0gpqCn2QizoNeAz4N/A6UNcmOwG4TtJRZOX+BzkB08yanoaYJx4Rz9Zzqt3y5A8KxIaIuAm4qdRrN3qAjIhzctYnAlstsn8IMCTPcVOBA8tdPjNrQFV+J01TrEGaWQ2Q/F5sM7OCqjs8OkCaWTlVeYR0gDSzMmn6z3ssxgHSzMqi7mEV1cwB0szKxwHSzCw/N7HNzAqo8lk+DpBmViZ+7auZWWFuYpuZ5SFcgzQzK6jK46MDpJmVTynPe2zKHCDNrGyqPD46QJpZ+VR5fHSANLPyyAZpqjtEOkCaWXl4HqSZWWFVHh8dIM2sjKo8QjpAmlmZyK9cMDPLR1R9BdIB0szKqMojpAOkmZVNtT+sokWlC2BmtauFii/FSLpJ0qeSXs1JW1XSCEkT0s92KV2SrpI0UdIrkrrlHNMv5Z8gqV9J5V/yj2xmVoI0D7LYUoJbgL0WSTsdeDIiOgNPpm2AvYHOaekPXAdZQAXOBrYDegBn1wXV+jhAmlkZqYSlfhHxT2D6Isl9gMFpfTDQNyf91si8AKwiaW1gT2BEREyPiBnACBYPuotxH6SZlcUSPA9ydUljcrb/GhF/LXJM+4iYAhARUyStmdI7AB/k5Juc0gql18sB0szKpsTXvk6LiO4NdMl8V4x60uvlJraZlY1K+G8pfZKazqSfn6b0ycC6Ofk6Ah/Vk14vB0gzK5/v3wVZyDCgbiS6H/BATvqRaTS7JzArNcWHA70ltUuDM71TWr3cxDazsmmIWZCS7gR6kfVVTiYbjb4YGCrpGODfwEEp+yPAPsBE4GvgKICImC7pfGB0yndeRCw68LMYB0gzKwuJBrkXOyJ+VmDXbnnyBnBCgfPcBNy0JNd2gDSz8qnuG2kcIM2sfKo8PjpAmln5VPnTzhwgzaw8VAPPg/Q0HzOzAlyDNLOyqfIKpAOkmZVPtT8P0gHSzMpCJT7vsSlzgDSz8nGANDPLz01sM7MCPEhjZlaAA6SZWQHV3sRW9vCL2iNpKvB+pcvRQFYHplW6ELaYWvpe1o+INRryhJIeI/sdFTMtIoq+H6YSajZA1hJJYxrwkfTWQPy91D7famhmVoADpJlZAQ6Q1aHYKzCtMvy91Dj3QZqZFeAapJlZAQ6QZmYFOECamRXgAGlmVoADZJWRqv3u1tpS6Pvw91QbPIpdRSQpvRgdSfsCAXwCjAt/kY1uke+ARp0eAAAHQklEQVTjOGB5YOWIOL+yJbOG4odVVJGcf4ynAvsCzwPbAZcAIypYtGYp5/sYABwGHA+8ImlqRPylooWzBuEmdpWRtD6wXUT8CJgDfAM8KWm5ypas+ahrPktqIWl5YBvgp8AuwHDgRkmtK1hEayAOkE1cnr6sOcC3km4AegA/jYgFwD6S1mn0AjZDOd0ZK0bEbGAucDnwI7LvYx5woqT9KlVGaxgOkE3YIn1cR0raluzxWu8DWwOnRMQcSUcDZwMLKlfa5kVSD+BKSasCz5I1sU+LiNmSDgF+DrxeyTLa9+c+yKatBTBf0q+B44ADImKepIfJguHNkkYDewAHR8THFSxrTav7Y5X7Rwv4GDgLOAP4PTBU0lvABsARETGpQsW1BuJR7CZI0jbAGxHxtaRNgcFkAfB9SXuS/WH7DFgOaJPyvlu5EjcfkraPiJFpvRuwP7AycCqwBtl3MjsiPqpcKa2hOEA2ManP8Tpgc6A38C1wJdkUEoB1yPoh742IwRUpZDMlaTXgTeDWiPhtSusJnAt8CJwTEf+uYBGtgbkPsolJzbeTgZeAe8jeLDyUrD/rsvRo+heAbcETkstJUqec9QHAL4DuwE8kXQwQES8AE4EvyP6YWQ1xDbKJWKRvizRN5FqgPVnzenZKP4KsOfeziHijIoVtBiTtQ1Zz7wbsDewKXBoRkyR1IBuYuZ+sRnkIWZ+jm9U1xjXIJkBSi5zR6k0kbRAR30bEsWR3ytwvafk0B7I32T9GB8cySf28lwE/j4gvgL7AAcCnABHxIbA90JasJn+yg2Ntcg2yCZF0EnAgWX/WlylAIukvZH2SuwIt62qT1vAk9QZuA54BzoyItyWtBNwOzI2IA3LytiD7NzS/MqW1cnMNsoIkrZWzfjhwENmUnXeBX0h6ECAiBpD1SbZ3cCwfSbsB1wCnACOBYyT9MCI+Bw4HvpJ0V12/b0QscHCsbQ6QFZIeNjFMUt27iN8iC5DHAF3JpotsmRMkT4yIDypS2Objc+AXEXE78BDZoMu+knZMQfIEsu/l5gqW0RqRm9gVIGkv4A/AhRHxmKRWaQL4ssCNwC0R8aSkC8mCZi/3cTWe1Ce8QFJnsjtiWgPDIuJ5SSuS3WLo76MZcA2ykaVb0x4B/jcFx42AQWmOXZDdndFT0plAJ2An/2NsXOnediJiAll/5GzgZ5K2i4gv/H00Hw6QjSwipgM/Bs6StAXZq0NfiojPIuJb/vPYsp2AiyPi0woV1VgYJIcAH5H1DVsz4iZ2haRm9iNkI6UX1zWzc/YvExFzK1dCy+Xvo3lygKwgSXsAV5M933GWpNapFmlmTYADZIVJ2hv4E7B9an6bWRPhx51VWEQ8mm4rfEJS9yzJf7XMmgLXIJsISW0j4stKl8PM/sMB0sysAE/zMTMrwAHSzKwAB0gzswIcIM3MCnCArHGS5ksaL+lVSX+X1OZ7nKuXpIfS+k8knV5P3lUk/WoprnGOpFNLTV8kzy2SDlyCa3WS9OqSltGaDwfI2jc7IraKiM3JHt81IHenMkv8/0FEDIuIi+vJsgqwxAHSrClxgGxengE2TjWnNyRdC4wD1pXUW9JISeNSTbMtZPeMS3pT0rNkrx0gpf9C0jVpvb2k+yS9nJYdgIuBjVLt9Y8p3+8kjZb0iqRzc871B0lvSXoC6FLsQ0g6Lp3nZUn3LFIr3l3SM5LelrRfyt9S0h9zrv3L7/uLtObBAbKZkNSK7OVT/0pJXcheX7o18BUwENg9IroBY4BTJC0H3ED29KEfAmstduLMVcD/RcSWZC+5eg04HXgn1V5/l15l0BnoAWwFbCNpZ2XvAD8U2JosAG9bwse5NyK2Tdd7g+whw3U6AbsA+wJ/SZ/hGGBWRGybzn+cpA1KuI41c77VsPYtL2l8Wn8GGET2bu330ytLAXoCmwHPpbcJtCZ75cCmwLvpkV9I+hvQP881dgWOBEivIJglqd0ieXqn5aW03ZYsYK4I3BcRX6drDCvhM20u6QKyZnxbYHjOvqHpeY4TJE1Kn6E3sEVO/+TK6dpvl3Ata8YcIGvf7IjYKjchBcGvcpOAERHxs0XybUX2EN+GIOCiiLh+kWucvBTXuAXoGxEvS/oF0Ctn36LninTtEyMiN5B+573XZvm4iW0ALwA7StoYQFIbSZuQvfN5g/TUc4CfFTj+SeD4dGzL9BbAL8hqh3WGA0fn9G12kLQm8E9gf2WvtV2RrDlfzIrAFEnLkL1MK9dBklqkMm9I9q6f4cDxKX/dq3VXKOE61sy5BmlExNRUE7szvRcHYGB65Wl/4GFJ04BnyV4/u6iTgL9KOgaYDxwfESMlPZem0Tya+iG7AiNTDfZLsvd7j5M0BBgPvE/WDVDMfwOjUv5/8d1A/Bbwf0B7YEBEfCPpRrK+yXHKLj6V7F3XZvXywyrMzApwE9vMrAAHSDOzAhwgzcwKcIA0MyvAAdLMrAAHSDOzAhwgzcwK+P+ArRdNkT2mrAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(y_test,yhat)\n",
    "\n",
    "#helper function from http://scikit-learn.org/stable/\n",
    "#auto_examples/model_selection/plot_confusion_matrix.html\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    fmt = '.2f' if normalize else 'd'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    plt.tight_layout()\n",
    "\n",
    "classes = ['False','True']\n",
    "plot_confusion_matrix(cm,classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       False       0.83      0.94      0.88      7040\n",
      "        True       0.60      0.31      0.41      1960\n",
      "\n",
      "   micro avg       0.80      0.80      0.80      9000\n",
      "   macro avg       0.71      0.63      0.65      9000\n",
      "weighted avg       0.78      0.80      0.78      9000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cr=classification_report(y_test,yhat)\n",
    "print(cr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7242343677527829\n"
     ]
    }
   ],
   "source": [
    "# calculate roc_auc_score\n",
    "# limit to probability for class = 1 (TRUE)\n",
    "print(roc_auc_score(y_test, yhat_probs[:,1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 2 : GridSearch Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a dictionary of parameters \n",
    "param_grid = {'n_estimators':[500,750,1000,1500,2000],\n",
    "'max_features':[2,4,6,8],\n",
    "'max_depth':[2,4,6,8],\n",
    "'random_state':[5]}\n",
    "\n",
    "# create Random Forest model \n",
    "rf_obj=RandomForestClassifier()\n",
    "\n",
    "# Create gridsearch object with various combinations of parameters\n",
    "#cv = 10, up the cross validation to 10\n",
    "#n_jobs =-1, use all processors\n",
    "#verbose = 5, show more messages\n",
    "rf_Grid = GridSearchCV(rf_obj, param_grid, cv = 10, scoring = 'roc_auc',\n",
    "                       refit = True, n_jobs=-1, verbose = 5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 80 candidates, totalling 800 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:   20.7s\n",
      "[Parallel(n_jobs=-1)]: Done  64 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 154 tasks      | elapsed: 10.9min\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-397bf62cd972>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# next, just fit this object\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mrf_Grid\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[0;32m    720\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mresults_container\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    721\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 722\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    723\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    724\u001b[0m         \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresults_container\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[1;34m(self, evaluate_candidates)\u001b[0m\n\u001b[0;32m   1189\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1190\u001b[0m         \u001b[1;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1191\u001b[1;33m         \u001b[0mevaluate_candidates\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1192\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1193\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[1;34m(candidate_params)\u001b[0m\n\u001b[0;32m    709\u001b[0m                                \u001b[1;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    710\u001b[0m                                in product(candidate_params,\n\u001b[1;32m--> 711\u001b[1;33m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[0;32m    712\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    713\u001b[0m                 \u001b[0mall_candidate_params\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcandidate_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    994\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    995\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 996\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    997\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    998\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    897\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    898\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'supports_timeout'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 899\u001b[1;33m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    900\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    901\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\externals\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[1;34m(future, timeout)\u001b[0m\n\u001b[0;32m    515\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[0;32m    516\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 517\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    518\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mLokyTimeoutError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    519\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\concurrent\\futures\\_base.py\u001b[0m in \u001b[0;36mresult\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    425\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    426\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 427\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    428\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    429\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mCANCELLED\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCANCELLED_AND_NOTIFIED\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\threading.py\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    293\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m    \u001b[1;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    294\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 295\u001b[1;33m                 \u001b[0mwaiter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    296\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    297\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# next, just fit this object\n",
    "rf_Grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "bestparams = rf_Grid.best_params_\n",
    "bestparams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "bestmodel = rf_Grid.best_estimator_\n",
    "bestmodel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 2 Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_hat_best = bestmodel.predict(X_test)\n",
    "y_hat_best_proba = bestmodel.predict_proba(X_test)\n",
    "cmbest = confusion_matrix(y_test,y_hat_best)\n",
    "plot_confusion_matrix(cmbest,classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "crbest = classification_report(y_test,y_hat_best)\n",
    "print(crbest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculate roc_auc_score\n",
    "# limit to probability for class = 1 (TRUE)\n",
    "print(roc_auc_score(y_test, y_hat_best_proba[:,1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Questions\n",
    "\n",
    "**What are the best parameters from the Grid Search?**\n",
    "\n",
    "The best parameters were found to be:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(bestparams)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Does the Model from #3 outperform Model #2?**\n",
    "\n",
    "The model from #3 does outperform the Model from #2. The ROC_AUC score is 4 percent higher and both the precision and recall improved for False & True."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the feature importance from the best model\n",
    "feat_imp = bestmodel.feature_importances_\n",
    "c_names = cleaned_df.columns[:len(cleaned_df.columns)-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#function to help pull out the names from a dataframe in a sorted order\n",
    "#feature_importance is a numpy array\n",
    "#sorted_column_names is a list of column names in the same order as \n",
    "#feature_importance default\n",
    "#returns a zip of column names & feature importance values sorted in order\n",
    "def sorted_feature_importance(feature_importance,column_names):\n",
    "    #turn that into an enumerated list so we know the index value of the importance\n",
    "    list_feat_imp = list(enumerate(feature_importance))\n",
    "    #sort our list in descending fashion\n",
    "    list_feat_imp_sorted = sorted(list_feat_imp,key=lambda list_feat_imp:list_feat_imp[1],reverse=True)\n",
    "    #do list comprehension to extract the value of the importance of the feature in\n",
    "    feat_imp_sorted = [i[1] for i in list_feat_imp_sorted]\n",
    "    #take the sorted list of features and extract their IDs\n",
    "    sortednameids = [i[0] for i in list_feat_imp_sorted]\n",
    "    #map the indices to the actual names from the dataframe\n",
    "    sortednames = [column_names[i] for i in sortednameids]\n",
    "    return zip(sortednameids,sortednames,feat_imp_sorted)\n",
    "\n",
    "#use the helper function to get sorted features\n",
    "srt_feats = sorted_feature_importance(feat_imp,c_names)\n",
    "#unzip the values into lists for plotting\n",
    "sorted_feat_ids,sorted_names,sorted_feat_vals = zip(*srt_feats)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Importance Plot (top 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#needed to invert the lists for the bar chart to get the most \n",
    "#significant feature at the top\n",
    "plt.barh(sorted_names[4::-1], sorted_feat_vals[4::-1])\n",
    "plt.xlabel('Feature Importance')\n",
    "plt.title('Top 5 Features for Best Model')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}