Digital Model-with crossval score.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary library\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "\n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from sklearn import ensemble\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedKFold\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn import metrics\n",
    "from math import sqrt\n",
    "from numpy import mean\n",
    "from numpy import std\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wellhead Temp. (C)</th>\n",
       "      <th>Wellhead Press (psi)</th>\n",
       "      <th>MMCFD- gas</th>\n",
       "      <th>BOPD (barrel of oil produced per day)</th>\n",
       "      <th>BWPD (barrel of water produced per day)</th>\n",
       "      <th>BSW - basic solid and water (%)</th>\n",
       "      <th>CO2 mol. (%) @ 25 C &amp; 1 Atm.</th>\n",
       "      <th>Gas Grav.</th>\n",
       "      <th>CR-corrosion defect</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>53.35</td>\n",
       "      <td>1105.13</td>\n",
       "      <td>12.87</td>\n",
       "      <td>1378.93</td>\n",
       "      <td>2812.62</td>\n",
       "      <td>75.64</td>\n",
       "      <td>3.3628</td>\n",
       "      <td>0.7205</td>\n",
       "      <td>0.2245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>72.25</td>\n",
       "      <td>1026.31</td>\n",
       "      <td>3.42</td>\n",
       "      <td>1028.75</td>\n",
       "      <td>919.92</td>\n",
       "      <td>44.21</td>\n",
       "      <td>3.8679</td>\n",
       "      <td>0.8940</td>\n",
       "      <td>0.2262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>65.08</td>\n",
       "      <td>722.96</td>\n",
       "      <td>6.23</td>\n",
       "      <td>2017.92</td>\n",
       "      <td>1212.42</td>\n",
       "      <td>17.55</td>\n",
       "      <td>2.3552</td>\n",
       "      <td>0.7661</td>\n",
       "      <td>0.2305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>60.71</td>\n",
       "      <td>1557.23</td>\n",
       "      <td>11.71</td>\n",
       "      <td>558.22</td>\n",
       "      <td>1716.09</td>\n",
       "      <td>65.79</td>\n",
       "      <td>1.7253</td>\n",
       "      <td>0.7738</td>\n",
       "      <td>0.2303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>46.19</td>\n",
       "      <td>1304.42</td>\n",
       "      <td>8.58</td>\n",
       "      <td>1280.47</td>\n",
       "      <td>1929.22</td>\n",
       "      <td>37.45</td>\n",
       "      <td>1.8327</td>\n",
       "      <td>0.7611</td>\n",
       "      <td>0.2202</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Wellhead Temp. (C)  Wellhead Press (psi)  MMCFD- gas  \\\n",
       "0               53.35               1105.13       12.87   \n",
       "1               72.25               1026.31        3.42   \n",
       "2               65.08                722.96        6.23   \n",
       "3               60.71               1557.23       11.71   \n",
       "4               46.19               1304.42        8.58   \n",
       "\n",
       "   BOPD (barrel of oil produced per day)  \\\n",
       "0                                1378.93   \n",
       "1                                1028.75   \n",
       "2                                2017.92   \n",
       "3                                 558.22   \n",
       "4                                1280.47   \n",
       "\n",
       "   BWPD (barrel of water produced per day)  BSW - basic solid and water (%)  \\\n",
       "0                                  2812.62                            75.64   \n",
       "1                                   919.92                            44.21   \n",
       "2                                  1212.42                            17.55   \n",
       "3                                  1716.09                            65.79   \n",
       "4                                  1929.22                            37.45   \n",
       "\n",
       "   CO2 mol. (%) @ 25 C & 1 Atm.  Gas Grav.  CR-corrosion defect  \n",
       "0                        3.3628     0.7205               0.2245  \n",
       "1                        3.8679     0.8940               0.2262  \n",
       "2                        2.3552     0.7661               0.2305  \n",
       "3                        1.7253     0.7738               0.2303  \n",
       "4                        1.8327     0.7611               0.2202  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('generated_dataset.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 10292 entries, 0 to 10291\n",
      "Data columns (total 9 columns):\n",
      "Wellhead Temp. (C)                         10292 non-null float64\n",
      "Wellhead Press (psi)                       10292 non-null float64\n",
      "MMCFD- gas                                 10292 non-null float64\n",
      "BOPD (barrel of oil produced per day)      10292 non-null float64\n",
      "BWPD (barrel of water produced per day)    10292 non-null float64\n",
      "BSW - basic solid and water (%)            10292 non-null float64\n",
      "CO2 mol. (%) @ 25 C & 1 Atm.               10292 non-null float64\n",
      "Gas Grav.                                  10292 non-null float64\n",
      "CR-corrosion defect                        10292 non-null float64\n",
      "dtypes: float64(9)\n",
      "memory usage: 723.8 KB\n"
     ]
    }
   ],
   "source": [
    "# Check data types\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Wellhead Temp. (C)</th>\n",
       "      <th>Wellhead Press (psi)</th>\n",
       "      <th>MMCFD- gas</th>\n",
       "      <th>BOPD (barrel of oil produced per day)</th>\n",
       "      <th>BWPD (barrel of water produced per day)</th>\n",
       "      <th>BSW - basic solid and water (%)</th>\n",
       "      <th>CO2 mol. (%) @ 25 C &amp; 1 Atm.</th>\n",
       "      <th>Gas Grav.</th>\n",
       "      <th>CR-corrosion defect</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>10292.000000</td>\n",
       "      <td>10292.000000</td>\n",
       "      <td>10292.000000</td>\n",
       "      <td>10292.000000</td>\n",
       "      <td>10292.000000</td>\n",
       "      <td>10292.000000</td>\n",
       "      <td>10292.000000</td>\n",
       "      <td>10292.000000</td>\n",
       "      <td>10292.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>57.352401</td>\n",
       "      <td>1361.783583</td>\n",
       "      <td>8.851533</td>\n",
       "      <td>1103.563396</td>\n",
       "      <td>4636.556014</td>\n",
       "      <td>44.876740</td>\n",
       "      <td>2.502574</td>\n",
       "      <td>0.821436</td>\n",
       "      <td>0.211285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>9.423529</td>\n",
       "      <td>559.275420</td>\n",
       "      <td>4.968737</td>\n",
       "      <td>565.392744</td>\n",
       "      <td>2685.799834</td>\n",
       "      <td>25.710558</td>\n",
       "      <td>1.041002</td>\n",
       "      <td>0.063142</td>\n",
       "      <td>0.037283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>41.070000</td>\n",
       "      <td>382.080000</td>\n",
       "      <td>0.230000</td>\n",
       "      <td>129.470000</td>\n",
       "      <td>40.610000</td>\n",
       "      <td>0.130000</td>\n",
       "      <td>0.678600</td>\n",
       "      <td>0.711100</td>\n",
       "      <td>0.000900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>49.220000</td>\n",
       "      <td>880.000000</td>\n",
       "      <td>4.570000</td>\n",
       "      <td>611.645000</td>\n",
       "      <td>2295.520000</td>\n",
       "      <td>22.887500</td>\n",
       "      <td>1.608650</td>\n",
       "      <td>0.766800</td>\n",
       "      <td>0.192100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>57.360000</td>\n",
       "      <td>1364.900000</td>\n",
       "      <td>8.880000</td>\n",
       "      <td>1106.085000</td>\n",
       "      <td>4591.995000</td>\n",
       "      <td>45.075000</td>\n",
       "      <td>2.518100</td>\n",
       "      <td>0.821600</td>\n",
       "      <td>0.213500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>65.412500</td>\n",
       "      <td>1848.252500</td>\n",
       "      <td>13.090000</td>\n",
       "      <td>1589.710000</td>\n",
       "      <td>6997.442500</td>\n",
       "      <td>67.212500</td>\n",
       "      <td>3.406125</td>\n",
       "      <td>0.876300</td>\n",
       "      <td>0.232800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>73.870000</td>\n",
       "      <td>2317.230000</td>\n",
       "      <td>17.540000</td>\n",
       "      <td>2087.430000</td>\n",
       "      <td>9314.260000</td>\n",
       "      <td>89.260000</td>\n",
       "      <td>4.298200</td>\n",
       "      <td>0.931900</td>\n",
       "      <td>0.405200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Wellhead Temp. (C)  Wellhead Press (psi)    MMCFD- gas  \\\n",
       "count        10292.000000          10292.000000  10292.000000   \n",
       "mean            57.352401           1361.783583      8.851533   \n",
       "std              9.423529            559.275420      4.968737   \n",
       "min             41.070000            382.080000      0.230000   \n",
       "25%             49.220000            880.000000      4.570000   \n",
       "50%             57.360000           1364.900000      8.880000   \n",
       "75%             65.412500           1848.252500     13.090000   \n",
       "max             73.870000           2317.230000     17.540000   \n",
       "\n",
       "       BOPD (barrel of oil produced per day)  \\\n",
       "count                           10292.000000   \n",
       "mean                             1103.563396   \n",
       "std                               565.392744   \n",
       "min                               129.470000   \n",
       "25%                               611.645000   \n",
       "50%                              1106.085000   \n",
       "75%                              1589.710000   \n",
       "max                              2087.430000   \n",
       "\n",
       "       BWPD (barrel of water produced per day)  \\\n",
       "count                             10292.000000   \n",
       "mean                               4636.556014   \n",
       "std                                2685.799834   \n",
       "min                                  40.610000   \n",
       "25%                                2295.520000   \n",
       "50%                                4591.995000   \n",
       "75%                                6997.442500   \n",
       "max                                9314.260000   \n",
       "\n",
       "       BSW - basic solid and water (%)  CO2 mol. (%) @ 25 C & 1 Atm.  \\\n",
       "count                     10292.000000                  10292.000000   \n",
       "mean                         44.876740                      2.502574   \n",
       "std                          25.710558                      1.041002   \n",
       "min                           0.130000                      0.678600   \n",
       "25%                          22.887500                      1.608650   \n",
       "50%                          45.075000                      2.518100   \n",
       "75%                          67.212500                      3.406125   \n",
       "max                          89.260000                      4.298200   \n",
       "\n",
       "          Gas Grav.  CR-corrosion defect  \n",
       "count  10292.000000         10292.000000  \n",
       "mean       0.821436             0.211285  \n",
       "std        0.063142             0.037283  \n",
       "min        0.711100             0.000900  \n",
       "25%        0.766800             0.192100  \n",
       "50%        0.821600             0.213500  \n",
       "75%        0.876300             0.232800  \n",
       "max        0.931900             0.405200  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Descriptive statistics\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Wellhead Temp. (C)                         0\n",
       "Wellhead Press (psi)                       0\n",
       "MMCFD- gas                                 0\n",
       "BOPD (barrel of oil produced per day)      0\n",
       "BWPD (barrel of water produced per day)    0\n",
       "BSW - basic solid and water (%)            0\n",
       "CO2 mol. (%) @ 25 C & 1 Atm.               0\n",
       "Gas Grav.                                  0\n",
       "CR-corrosion defect                        0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check for missing values\n",
    "data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform some graphical analysis on the data\n",
    "# from scipy.stats import norm\n",
    "# ax = sns.pairplot(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data.iloc[:,1:7].values\n",
    "y = data.iloc[:,8].values\n",
    "X = MinMaxScaler().fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8233, 6) (2059, 6)\n"
     ]
    }
   ],
   "source": [
    "# Splitting the dataset into the Training set and Test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Boosting Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ensemble.GradientBoostingRegressor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the grid of values to search\n",
    "grid = dict()\n",
    "grid['n_estimators'] = [10, 50, 100, 500]\n",
    "grid['learning_rate'] = [0.0001, 0.001, 0.01, 0.1, 1.0] \n",
    "grid['subsample'] = [0.5, 0.7, 1.0]\n",
    "grid['max_depth'] = [3, 7, 9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1, cv=cv, scoring='neg_mean_absolute_error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-48-8577bfd97366>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# execute the grid search\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mgrid_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrid_search\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m# summarize the best score and configuration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Best: %f using %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mgrid_result\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_score_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrid_result\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_params_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# summarize all scores that were evaluated\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda3/lib/python3.7/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     71\u001b[0m                           FutureWarning)\n\u001b[1;32m     72\u001b[0m         \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 73\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     74\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    734\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    735\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 736\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    737\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    738\u001b[0m         \u001b[0;31m# For multi-metric evaluation, store the best_index_, best_params_ and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[0;34m(self, evaluate_candidates)\u001b[0m\n\u001b[1;32m   1186\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1187\u001b[0m         \u001b[0;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1188\u001b[0;31m         \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda3/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[0;34m(candidate_params)\u001b[0m\n\u001b[1;32m    713\u001b[0m                                \u001b[0;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    714\u001b[0m                                in product(candidate_params,\n\u001b[0;32m--> 715\u001b[0;31m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[1;32m    716\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    717\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda3/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m    932\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    933\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 934\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    935\u001b[0m             \u001b[0;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    936\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda3/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    831\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    832\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'supports_timeout'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 833\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    834\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    835\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda3/lib/python3.7/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[0;34m(future, timeout)\u001b[0m\n\u001b[1;32m    519\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[1;32m    520\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mLokyTimeoutError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda3/lib/python3.7/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    428\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    429\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 430\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    431\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    432\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mCANCELLED\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mCANCELLED_AND_NOTIFIED\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Applications/anaconda3/lib/python3.7/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    294\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 296\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    297\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "# execute the grid search\n",
    "grid_result = grid_search.fit(X, y)\n",
    "# summarize the best score and configuration\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "# summarize all scores that were evaluated\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: -0.012 (0.001)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# define the evaluation procedure\n",
    "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "# evaluate the model\n",
    "n_scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1,error_score='raise')\n",
    "# report performance\n",
    "print('MAE: %.3f (%.3f)' % (mean(n_scores), std(n_scores)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The scikit-learn library makes the MAE negative so that it is maximized instead of minimized. This means that larger negative MAE are better and a perfect model has a MAE of 0.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingRegressor()"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# gbm_model = ensemble.GradientBoostingRegressor(n_estimators=15000, max_depth=4, min_samples_leaf=15, \n",
    "#                                            min_samples_split=10, learning_rate=0.01, loss='huber', random_state=5)\n",
    "\n",
    "# # Reshape train_target to be a 1d array\n",
    "# y_train = y_train.as_matrix().flatten()\n",
    "\n",
    "# Fit model\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions with model\n",
    "y_pred = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score: 0.5703278461174064\n",
      "Mean Absolute Error: 0.012459934003917646\n",
      "Mean Squared Error: 0.0006166638486893732\n",
      "Root Mean Squared Error: 0.02483271730377836\n"
     ]
    }
   ],
   "source": [
    "print(\"Score:\", model.score(X_test, y_test))\n",
    "print('Mean Absolute Error:', metrics.mean_absolute_error(y_test, y_pred))  \n",
    "print('Mean Squared Error:', metrics.mean_squared_error(y_test, y_pred))  \n",
    "print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_test, y_pred)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}