03-02-Multiple Regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multiple Regression\n",
    "\n",
    "Simple Linear Regression:\n",
    "\n",
    "$$y = \\beta_0 + \\beta_1X$$\n",
    "\n",
    "Multiple Linear Regression:\n",
    "\n",
    "$$y = \\beta_0 + \\beta_1X_1 + \\beta_2X_2 + ...$$\n",
    "\n",
    "Well studied field in statistics\n",
    "\n",
    "Focus will be on what is relevant for Data Science - practical and relevant for prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.16.5\n",
      "0.25.1\n",
      "3.7.4 (default, Aug 13 2019, 15:17:50) \n",
      "[Clang 4.0.1 (tags/RELEASE_401/final)]\n",
      "0.21.3\n"
     ]
    }
   ],
   "source": [
    "print(np.__version__)\n",
    "print(pd.__version__)\n",
    "import sys\n",
    "print(sys.version)\n",
    "print(sklearn.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "boston_data = load_boston()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(boston_data.data, columns=boston_data.feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  \n",
       "0     15.3  396.90   4.98  \n",
       "1     17.8  396.90   9.14  \n",
       "2     17.8  392.83   4.03  \n",
       "3     18.7  394.63   2.94  \n",
       "4     18.7  396.90   5.33  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506, 13)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = boston_data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([24. , 21.6, 34.7, 33.4, 36.2, 28.7, 22.9, 27.1, 16.5, 18.9, 15. ,\n",
       "       18.9, 21.7, 20.4, 18.2, 19.9, 23.1, 17.5, 20.2, 18.2, 13.6, 19.6,\n",
       "       15.2, 14.5, 15.6, 13.9, 16.6, 14.8, 18.4, 21. , 12.7, 14.5, 13.2,\n",
       "       13.1, 13.5, 18.9, 20. , 21. , 24.7, 30.8, 34.9, 26.6, 25.3, 24.7,\n",
       "       21.2, 19.3, 20. , 16.6, 14.4, 19.4, 19.7, 20.5, 25. , 23.4, 18.9,\n",
       "       35.4, 24.7, 31.6, 23.3, 19.6, 18.7, 16. , 22.2, 25. , 33. , 23.5,\n",
       "       19.4, 22. , 17.4, 20.9, 24.2, 21.7, 22.8, 23.4, 24.1, 21.4, 20. ,\n",
       "       20.8, 21.2, 20.3, 28. , 23.9, 24.8, 22.9, 23.9, 26.6, 22.5, 22.2,\n",
       "       23.6, 28.7, 22.6, 22. , 22.9, 25. , 20.6, 28.4, 21.4, 38.7, 43.8,\n",
       "       33.2, 27.5, 26.5, 18.6, 19.3, 20.1, 19.5, 19.5, 20.4, 19.8, 19.4,\n",
       "       21.7, 22.8, 18.8, 18.7, 18.5, 18.3, 21.2, 19.2, 20.4, 19.3, 22. ,\n",
       "       20.3, 20.5, 17.3, 18.8, 21.4, 15.7, 16.2, 18. , 14.3, 19.2, 19.6,\n",
       "       23. , 18.4, 15.6, 18.1, 17.4, 17.1, 13.3, 17.8, 14. , 14.4, 13.4,\n",
       "       15.6, 11.8, 13.8, 15.6, 14.6, 17.8, 15.4, 21.5, 19.6, 15.3, 19.4,\n",
       "       17. , 15.6, 13.1, 41.3, 24.3, 23.3, 27. , 50. , 50. , 50. , 22.7,\n",
       "       25. , 50. , 23.8, 23.8, 22.3, 17.4, 19.1, 23.1, 23.6, 22.6, 29.4,\n",
       "       23.2, 24.6, 29.9, 37.2, 39.8, 36.2, 37.9, 32.5, 26.4, 29.6, 50. ,\n",
       "       32. , 29.8, 34.9, 37. , 30.5, 36.4, 31.1, 29.1, 50. , 33.3, 30.3,\n",
       "       34.6, 34.9, 32.9, 24.1, 42.3, 48.5, 50. , 22.6, 24.4, 22.5, 24.4,\n",
       "       20. , 21.7, 19.3, 22.4, 28.1, 23.7, 25. , 23.3, 28.7, 21.5, 23. ,\n",
       "       26.7, 21.7, 27.5, 30.1, 44.8, 50. , 37.6, 31.6, 46.7, 31.5, 24.3,\n",
       "       31.7, 41.7, 48.3, 29. , 24. , 25.1, 31.5, 23.7, 23.3, 22. , 20.1,\n",
       "       22.2, 23.7, 17.6, 18.5, 24.3, 20.5, 24.5, 26.2, 24.4, 24.8, 29.6,\n",
       "       42.8, 21.9, 20.9, 44. , 50. , 36. , 30.1, 33.8, 43.1, 48.8, 31. ,\n",
       "       36.5, 22.8, 30.7, 50. , 43.5, 20.7, 21.1, 25.2, 24.4, 35.2, 32.4,\n",
       "       32. , 33.2, 33.1, 29.1, 35.1, 45.4, 35.4, 46. , 50. , 32.2, 22. ,\n",
       "       20.1, 23.2, 22.3, 24.8, 28.5, 37.3, 27.9, 23.9, 21.7, 28.6, 27.1,\n",
       "       20.3, 22.5, 29. , 24.8, 22. , 26.4, 33.1, 36.1, 28.4, 33.4, 28.2,\n",
       "       22.8, 20.3, 16.1, 22.1, 19.4, 21.6, 23.8, 16.2, 17.8, 19.8, 23.1,\n",
       "       21. , 23.8, 23.1, 20.4, 18.5, 25. , 24.6, 23. , 22.2, 19.3, 22.6,\n",
       "       19.8, 17.1, 19.4, 22.2, 20.7, 21.1, 19.5, 18.5, 20.6, 19. , 18.7,\n",
       "       32.7, 16.5, 23.9, 31.2, 17.5, 17.2, 23.1, 24.5, 26.6, 22.9, 24.1,\n",
       "       18.6, 30.1, 18.2, 20.6, 17.8, 21.7, 22.7, 22.6, 25. , 19.9, 20.8,\n",
       "       16.8, 21.9, 27.5, 21.9, 23.1, 50. , 50. , 50. , 50. , 50. , 13.8,\n",
       "       13.8, 15. , 13.9, 13.3, 13.1, 10.2, 10.4, 10.9, 11.3, 12.3,  8.8,\n",
       "        7.2, 10.5,  7.4, 10.2, 11.5, 15.1, 23.2,  9.7, 13.8, 12.7, 13.1,\n",
       "       12.5,  8.5,  5. ,  6.3,  5.6,  7.2, 12.1,  8.3,  8.5,  5. , 11.9,\n",
       "       27.9, 17.2, 27.5, 15. , 17.2, 17.9, 16.3,  7. ,  7.2,  7.5, 10.4,\n",
       "        8.8,  8.4, 16.7, 14.2, 20.8, 13.4, 11.7,  8.3, 10.2, 10.9, 11. ,\n",
       "        9.5, 14.5, 14.1, 16.1, 14.3, 11.7, 13.4,  9.6,  8.7,  8.4, 12.8,\n",
       "       10.5, 17.1, 18.4, 15.4, 10.8, 11.8, 14.9, 12.6, 14.1, 13. , 13.4,\n",
       "       15.2, 16.1, 17.8, 14.9, 14.1, 12.7, 13.5, 14.9, 20. , 16.4, 17.7,\n",
       "       19.5, 20.2, 21.4, 19.9, 19. , 19.1, 19.1, 20.1, 19.9, 19.6, 23.2,\n",
       "       29.8, 13.8, 13.3, 16.7, 12. , 14.6, 21.4, 23. , 23.7, 25. , 21.8,\n",
       "       20.6, 21.2, 19.1, 20.6, 15.2,  7. ,  8.1, 13.6, 20.1, 21.8, 24.5,\n",
       "       23.1, 19.7, 18.3, 21.2, 17.5, 16.8, 22.4, 20.6, 23.9, 22. , 11.9])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Statsmodels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to add a constant term to allow statsmodel.api to calculate the bias / intercepts."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$y = mX$\n",
    "\n",
    "$y = mX + c$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/anthonyn/anaconda3/envs/ml_python_2019/lib/python3.7/site-packages/numpy/core/fromnumeric.py:2389: FutureWarning: Method .ptp is deprecated and will be removed in a future version. Use numpy.ptp instead.\n",
      "  return ptp(axis=axis, out=out, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "X_constant = sm.add_constant(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>const</th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>501</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.06263</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.593</td>\n",
       "      <td>69.1</td>\n",
       "      <td>2.4786</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>391.99</td>\n",
       "      <td>9.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>502</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.04527</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.120</td>\n",
       "      <td>76.7</td>\n",
       "      <td>2.2875</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>503</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.06076</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.976</td>\n",
       "      <td>91.0</td>\n",
       "      <td>2.1675</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>504</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.10959</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.794</td>\n",
       "      <td>89.3</td>\n",
       "      <td>2.3889</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>393.45</td>\n",
       "      <td>6.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>505</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.04741</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573</td>\n",
       "      <td>6.030</td>\n",
       "      <td>80.8</td>\n",
       "      <td>2.5050</td>\n",
       "      <td>1.0</td>\n",
       "      <td>273.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>396.90</td>\n",
       "      <td>7.88</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>506 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     const     CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD  \\\n",
       "0      1.0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0   \n",
       "1      1.0  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0   \n",
       "2      1.0  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0   \n",
       "3      1.0  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0   \n",
       "4      1.0  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0   \n",
       "..     ...      ...   ...    ...   ...    ...    ...   ...     ...  ...   \n",
       "501    1.0  0.06263   0.0  11.93   0.0  0.573  6.593  69.1  2.4786  1.0   \n",
       "502    1.0  0.04527   0.0  11.93   0.0  0.573  6.120  76.7  2.2875  1.0   \n",
       "503    1.0  0.06076   0.0  11.93   0.0  0.573  6.976  91.0  2.1675  1.0   \n",
       "504    1.0  0.10959   0.0  11.93   0.0  0.573  6.794  89.3  2.3889  1.0   \n",
       "505    1.0  0.04741   0.0  11.93   0.0  0.573  6.030  80.8  2.5050  1.0   \n",
       "\n",
       "       TAX  PTRATIO       B  LSTAT  \n",
       "0    296.0     15.3  396.90   4.98  \n",
       "1    242.0     17.8  396.90   9.14  \n",
       "2    242.0     17.8  392.83   4.03  \n",
       "3    222.0     18.7  394.63   2.94  \n",
       "4    222.0     18.7  396.90   5.33  \n",
       "..     ...      ...     ...    ...  \n",
       "501  273.0     21.0  391.99   9.67  \n",
       "502  273.0     21.0  396.90   9.08  \n",
       "503  273.0     21.0  396.90   5.64  \n",
       "504  273.0     21.0  393.45   6.48  \n",
       "505  273.0     21.0  396.90   7.88  \n",
       "\n",
       "[506 rows x 14 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(X_constant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[0;31mInit signature:\u001b[0m \u001b[0msm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOLS\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mendog\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexog\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmissing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'none'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhasconst\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
       "\u001b[0;31mDocstring:\u001b[0m     \n",
       "A simple ordinary least squares model.\n",
       "\n",
       "\n",
       "Parameters\n",
       "----------\n",
       "endog : array-like\n",
       "    1-d endogenous response variable. The dependent variable.\n",
       "exog : array-like\n",
       "    A nobs x k array where `nobs` is the number of observations and `k`\n",
       "    is the number of regressors. An intercept is not included by default\n",
       "    and should be added by the user. See\n",
       "    :func:`statsmodels.tools.add_constant`.\n",
       "missing : str\n",
       "    Available options are 'none', 'drop', and 'raise'. If 'none', no nan\n",
       "    checking is done. If 'drop', any observations with nans are dropped.\n",
       "    If 'raise', an error is raised. Default is 'none.'\n",
       "hasconst : None or bool\n",
       "    Indicates whether the RHS includes a user-supplied constant. If True,\n",
       "    a constant is not checked for and k_constant is set to 1 and all\n",
       "    result statistics are calculated as if a constant is present. If\n",
       "    False, a constant is not checked for and k_constant is set to 0.\n",
       "\n",
       "\n",
       "Attributes\n",
       "----------\n",
       "weights : scalar\n",
       "    Has an attribute weights = array(1.0) due to inheritance from WLS.\n",
       "\n",
       "See Also\n",
       "--------\n",
       "GLS\n",
       "\n",
       "Examples\n",
       "--------\n",
       ">>> import numpy as np\n",
       ">>>\n",
       ">>> import statsmodels.api as sm\n",
       ">>>\n",
       ">>> Y = [1,3,4,5,2,3,4]\n",
       ">>> X = range(1,8)\n",
       ">>> X = sm.add_constant(X)\n",
       ">>>\n",
       ">>> model = sm.OLS(Y,X)\n",
       ">>> results = model.fit()\n",
       ">>> results.params\n",
       "array([ 2.14285714,  0.25      ])\n",
       ">>> results.tvalues\n",
       "array([ 1.87867287,  0.98019606])\n",
       ">>> print(results.t_test([1, 0]))\n",
       "<T test: effect=array([ 2.14285714]), sd=array([[ 1.14062282]]), t=array([[ 1.87867287]]), p=array([[ 0.05953974]]), df_denom=5>\n",
       ">>> print(results.f_test(np.identity(2)))\n",
       "<F test: F=array([[ 19.46078431]]), p=[[ 0.00437251]], df_denom=5, df_num=2>\n",
       "\n",
       "Notes\n",
       "-----\n",
       "No constant is added by the model unless you are using formulas.\n",
       "\u001b[0;31mFile:\u001b[0m           ~/anaconda3/envs/ml_python_2019/lib/python3.7/site-packages/statsmodels/regression/linear_model.py\n",
       "\u001b[0;31mType:\u001b[0m           type\n",
       "\u001b[0;31mSubclasses:\u001b[0m     \n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sm.OLS?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = sm.OLS(y, X_constant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = model.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>            <td>y</td>        <th>  R-squared:         </th> <td>   0.741</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.734</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   108.1</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Wed, 02 Oct 2019</td> <th>  Prob (F-statistic):</th> <td>6.72e-135</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>21:18:57</td>     <th>  Log-Likelihood:    </th> <td> -1498.8</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   506</td>      <th>  AIC:               </th> <td>   3026.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   492</td>      <th>  BIC:               </th> <td>   3085.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>    13</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "     <td></td>        <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>const</th>   <td>   36.4595</td> <td>    5.103</td> <td>    7.144</td> <td> 0.000</td> <td>   26.432</td> <td>   46.487</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>CRIM</th>    <td>   -0.1080</td> <td>    0.033</td> <td>   -3.287</td> <td> 0.001</td> <td>   -0.173</td> <td>   -0.043</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ZN</th>      <td>    0.0464</td> <td>    0.014</td> <td>    3.382</td> <td> 0.001</td> <td>    0.019</td> <td>    0.073</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>INDUS</th>   <td>    0.0206</td> <td>    0.061</td> <td>    0.334</td> <td> 0.738</td> <td>   -0.100</td> <td>    0.141</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>CHAS</th>    <td>    2.6867</td> <td>    0.862</td> <td>    3.118</td> <td> 0.002</td> <td>    0.994</td> <td>    4.380</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>NOX</th>     <td>  -17.7666</td> <td>    3.820</td> <td>   -4.651</td> <td> 0.000</td> <td>  -25.272</td> <td>  -10.262</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>RM</th>      <td>    3.8099</td> <td>    0.418</td> <td>    9.116</td> <td> 0.000</td> <td>    2.989</td> <td>    4.631</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>AGE</th>     <td>    0.0007</td> <td>    0.013</td> <td>    0.052</td> <td> 0.958</td> <td>   -0.025</td> <td>    0.027</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>DIS</th>     <td>   -1.4756</td> <td>    0.199</td> <td>   -7.398</td> <td> 0.000</td> <td>   -1.867</td> <td>   -1.084</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>RAD</th>     <td>    0.3060</td> <td>    0.066</td> <td>    4.613</td> <td> 0.000</td> <td>    0.176</td> <td>    0.436</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>TAX</th>     <td>   -0.0123</td> <td>    0.004</td> <td>   -3.280</td> <td> 0.001</td> <td>   -0.020</td> <td>   -0.005</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>PTRATIO</th> <td>   -0.9527</td> <td>    0.131</td> <td>   -7.283</td> <td> 0.000</td> <td>   -1.210</td> <td>   -0.696</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>B</th>       <td>    0.0093</td> <td>    0.003</td> <td>    3.467</td> <td> 0.001</td> <td>    0.004</td> <td>    0.015</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>LSTAT</th>   <td>   -0.5248</td> <td>    0.051</td> <td>  -10.347</td> <td> 0.000</td> <td>   -0.624</td> <td>   -0.425</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>178.041</td> <th>  Durbin-Watson:     </th> <td>   1.078</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td> 783.126</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 1.521</td>  <th>  Prob(JB):          </th> <td>8.84e-171</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td> 8.281</td>  <th>  Cond. No.          </th> <td>1.51e+04</td> \n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.<br/>[2] The condition number is large, 1.51e+04. This might indicate that there are<br/>strong multicollinearity or other numerical problems."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:                      y   R-squared:                       0.741\n",
       "Model:                            OLS   Adj. R-squared:                  0.734\n",
       "Method:                 Least Squares   F-statistic:                     108.1\n",
       "Date:                Wed, 02 Oct 2019   Prob (F-statistic):          6.72e-135\n",
       "Time:                        21:18:57   Log-Likelihood:                -1498.8\n",
       "No. Observations:                 506   AIC:                             3026.\n",
       "Df Residuals:                     492   BIC:                             3085.\n",
       "Df Model:                          13                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "const         36.4595      5.103      7.144      0.000      26.432      46.487\n",
       "CRIM          -0.1080      0.033     -3.287      0.001      -0.173      -0.043\n",
       "ZN             0.0464      0.014      3.382      0.001       0.019       0.073\n",
       "INDUS          0.0206      0.061      0.334      0.738      -0.100       0.141\n",
       "CHAS           2.6867      0.862      3.118      0.002       0.994       4.380\n",
       "NOX          -17.7666      3.820     -4.651      0.000     -25.272     -10.262\n",
       "RM             3.8099      0.418      9.116      0.000       2.989       4.631\n",
       "AGE            0.0007      0.013      0.052      0.958      -0.025       0.027\n",
       "DIS           -1.4756      0.199     -7.398      0.000      -1.867      -1.084\n",
       "RAD            0.3060      0.066      4.613      0.000       0.176       0.436\n",
       "TAX           -0.0123      0.004     -3.280      0.001      -0.020      -0.005\n",
       "PTRATIO       -0.9527      0.131     -7.283      0.000      -1.210      -0.696\n",
       "B              0.0093      0.003      3.467      0.001       0.004       0.015\n",
       "LSTAT         -0.5248      0.051    -10.347      0.000      -0.624      -0.425\n",
       "==============================================================================\n",
       "Omnibus:                      178.041   Durbin-Watson:                   1.078\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              783.126\n",
       "Skew:                           1.521   Prob(JB):                    8.84e-171\n",
       "Kurtosis:                       8.281   Cond. No.                     1.51e+04\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "[2] The condition number is large, 1.51e+04. This might indicate that there are\n",
       "strong multicollinearity or other numerical problems.\n",
       "\"\"\""
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are a lot of statistical tests and information. Mostly for the purpose of statistical analysis.\n",
    "\n",
    "You do not need all of these for data science.\n",
    "\n",
    "Data science focus is on prediction and having models that work on predicting real data. It is not concerned as much with  correct specifications of statistical problems."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Statistical Outputs:\n",
    "\n",
    "**Dep. Variable**: The dependent variable or target variable\n",
    "\n",
    "**Model**: Highlight the model used to obtain this output. It is OLS here. Ordinary least squares / Linear regression\n",
    "\n",
    "**Method**: The method used to fit the data to the model. Least squares\n",
    "\n",
    "**No. Observations**: The number of observations\n",
    "\n",
    "**DF Residuals**: The degrees of freedom of the residuals. Calculated by taking the number of observations less the number of parameters\n",
    "\n",
    "**DF Model**: The number of estimated parameters in the model. In this case 13. The constant term is not included.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**R-squared**: This is the coefficient of determination. Measure of goodness of fit.\n",
    "$$R^2=1-\\frac{SS_{res}}{SS_{tot}}$$\n",
    "\n",
    "> From [wiki](https://en.wikipedia.org/wiki/Coefficient_of_determination),\n",
    "\n",
    "  > The total sum of squares, $SS_{tot}=\\sum_i(y_i-\\bar{y})^2$\n",
    "\n",
    "  > The regression sum of squares (explained sum of squares), $SS_{reg}=\\sum_i(f_i-\\bar{y})^2$\n",
    "\n",
    "  > The sum of squares of residuals (residual sum of squares), $SS_{res}=\\sum_i(y_i-f_i)^2 = \\sum_ie^2_i$\n",
    "\n",
    "**Adj. R-squared**: This is the adjusted R-squared. It is the coefficient of determination adjusted by sample size and the number of parameters used.\n",
    "$$\\bar{R}^2=1-(1-R^2)\\frac{n-1}{n-p-1}$$\n",
    "\n",
    "> $p$ = The total number of explanatory variables not including the constant term\n",
    "\n",
    "> $n$ = The sample size\n",
    "\n",
    "**F-statistic**: A measure that tells you if you model is different from a simple average.\n",
    "\n",
    "**Prob (F-statistic)**: This measures the significance of your F-statistic. Also called p-value of F-statistic. In statistics, p-value equal or lower than 0.05 is considered significant.\n",
    "\n",
    "**AIC**: This is the Akaike Information Criterion. It evaluatess the model based on the model complexity and number of observations. The lower the better. \n",
    "\n",
    "**BIC**: This is the Bayesian Information Criterion. Similar to AIC, except it pushishes models with more parameters."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters Estimates and the Associated Statistical Tests\n",
    "\n",
    "**coef**: The estimated coefficient. Note that this is just a point estimate.\n",
    "\n",
    "**std err**: The standard error of the estimate of the coefficient. Another term for standard deviation\n",
    "\n",
    "**t**: The t-statistic score. \n",
    "\n",
    "**P > |t|**: The p-value. A measure of the probability that the coefficient is different from zero.\n",
    "\n",
    "**[95.0% Conf. Interval]**: The 95% confidence interval of the coefficient. Shown here as [0.025, 0.975], the lower and upper bound."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Residual Tests\n",
    "\n",
    "**Omnibus D'Angostino's test**: This is a combined statistical test for skewness and kurtosis.\n",
    "\n",
    "**Prob(Omnibus)**: p-value of Omnibus test.\n",
    "\n",
    "**Skewness**: This is a measure of the symmetry of the residuals around the mean. Zero if symmetrical. A positive value indicates a long tail to the right; a negative value a long tail to the left.\n",
    "\n",
    "**Kurtosis**: This is a measure of the shape of the distribution of the residuals. A normal distribution has a zero measure. A negative value points to a flatter than normal distribution; a positive one has a higher peak than normal distribution.\n",
    "\n",
    "**Durbin-Watson**: This is a test for the presence of correlation among the residuals. This is especially important for time series modelling\n",
    "\n",
    "**Jarque-Bera**: This is a combined statistical test of skewness and kurtosis.\n",
    "\n",
    "**Prob (JB)**: p-value of Jarque-Bera.\n",
    "\n",
    "**Cond. No**: This is a test for multicollinearity. > 30 indicates unstable results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# statsmodels.formula.api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "form_lr = smf.ols(formula = 'y ~ CRIM + ZN + INDUS + CHAS + NOX + RM + AGE + DIS + RAD + TAX + PTRATIO + B + LSTAT', \n",
    "              data=df)\n",
    "mlr = form_lr.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>            <td>y</td>        <th>  R-squared:         </th> <td>   0.741</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.734</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   108.1</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Wed, 02 Oct 2019</td> <th>  Prob (F-statistic):</th> <td>6.72e-135</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>21:32:12</td>     <th>  Log-Likelihood:    </th> <td> -1498.8</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   506</td>      <th>  AIC:               </th> <td>   3026.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   492</td>      <th>  BIC:               </th> <td>   3085.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>    13</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "      <td></td>         <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th> <td>   36.4595</td> <td>    5.103</td> <td>    7.144</td> <td> 0.000</td> <td>   26.432</td> <td>   46.487</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>CRIM</th>      <td>   -0.1080</td> <td>    0.033</td> <td>   -3.287</td> <td> 0.001</td> <td>   -0.173</td> <td>   -0.043</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ZN</th>        <td>    0.0464</td> <td>    0.014</td> <td>    3.382</td> <td> 0.001</td> <td>    0.019</td> <td>    0.073</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>INDUS</th>     <td>    0.0206</td> <td>    0.061</td> <td>    0.334</td> <td> 0.738</td> <td>   -0.100</td> <td>    0.141</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>CHAS</th>      <td>    2.6867</td> <td>    0.862</td> <td>    3.118</td> <td> 0.002</td> <td>    0.994</td> <td>    4.380</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>NOX</th>       <td>  -17.7666</td> <td>    3.820</td> <td>   -4.651</td> <td> 0.000</td> <td>  -25.272</td> <td>  -10.262</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>RM</th>        <td>    3.8099</td> <td>    0.418</td> <td>    9.116</td> <td> 0.000</td> <td>    2.989</td> <td>    4.631</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>AGE</th>       <td>    0.0007</td> <td>    0.013</td> <td>    0.052</td> <td> 0.958</td> <td>   -0.025</td> <td>    0.027</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>DIS</th>       <td>   -1.4756</td> <td>    0.199</td> <td>   -7.398</td> <td> 0.000</td> <td>   -1.867</td> <td>   -1.084</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>RAD</th>       <td>    0.3060</td> <td>    0.066</td> <td>    4.613</td> <td> 0.000</td> <td>    0.176</td> <td>    0.436</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>TAX</th>       <td>   -0.0123</td> <td>    0.004</td> <td>   -3.280</td> <td> 0.001</td> <td>   -0.020</td> <td>   -0.005</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>PTRATIO</th>   <td>   -0.9527</td> <td>    0.131</td> <td>   -7.283</td> <td> 0.000</td> <td>   -1.210</td> <td>   -0.696</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>B</th>         <td>    0.0093</td> <td>    0.003</td> <td>    3.467</td> <td> 0.001</td> <td>    0.004</td> <td>    0.015</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>LSTAT</th>     <td>   -0.5248</td> <td>    0.051</td> <td>  -10.347</td> <td> 0.000</td> <td>   -0.624</td> <td>   -0.425</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>178.041</td> <th>  Durbin-Watson:     </th> <td>   1.078</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td> 783.126</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 1.521</td>  <th>  Prob(JB):          </th> <td>8.84e-171</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td> 8.281</td>  <th>  Cond. No.          </th> <td>1.51e+04</td> \n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.<br/>[2] The condition number is large, 1.51e+04. This might indicate that there are<br/>strong multicollinearity or other numerical problems."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:                      y   R-squared:                       0.741\n",
       "Model:                            OLS   Adj. R-squared:                  0.734\n",
       "Method:                 Least Squares   F-statistic:                     108.1\n",
       "Date:                Wed, 02 Oct 2019   Prob (F-statistic):          6.72e-135\n",
       "Time:                        21:32:12   Log-Likelihood:                -1498.8\n",
       "No. Observations:                 506   AIC:                             3026.\n",
       "Df Residuals:                     492   BIC:                             3085.\n",
       "Df Model:                          13                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "Intercept     36.4595      5.103      7.144      0.000      26.432      46.487\n",
       "CRIM          -0.1080      0.033     -3.287      0.001      -0.173      -0.043\n",
       "ZN             0.0464      0.014      3.382      0.001       0.019       0.073\n",
       "INDUS          0.0206      0.061      0.334      0.738      -0.100       0.141\n",
       "CHAS           2.6867      0.862      3.118      0.002       0.994       4.380\n",
       "NOX          -17.7666      3.820     -4.651      0.000     -25.272     -10.262\n",
       "RM             3.8099      0.418      9.116      0.000       2.989       4.631\n",
       "AGE            0.0007      0.013      0.052      0.958      -0.025       0.027\n",
       "DIS           -1.4756      0.199     -7.398      0.000      -1.867      -1.084\n",
       "RAD            0.3060      0.066      4.613      0.000       0.176       0.436\n",
       "TAX           -0.0123      0.004     -3.280      0.001      -0.020      -0.005\n",
       "PTRATIO       -0.9527      0.131     -7.283      0.000      -1.210      -0.696\n",
       "B              0.0093      0.003      3.467      0.001       0.004       0.015\n",
       "LSTAT         -0.5248      0.051    -10.347      0.000      -0.624      -0.425\n",
       "==============================================================================\n",
       "Omnibus:                      178.041   Durbin-Watson:                   1.078\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              783.126\n",
       "Skew:                           1.521   Prob(JB):                    8.84e-171\n",
       "Kurtosis:                       8.281   Cond. No.                     1.51e+04\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "[2] The condition number is large, 1.51e+04. This might indicate that there are\n",
       "strong multicollinearity or other numerical problems.\n",
       "\"\"\""
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlr.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>            <td>y</td>        <th>  R-squared:         </th> <td>   0.741</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.735</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   128.2</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Wed, 02 Oct 2019</td> <th>  Prob (F-statistic):</th> <td>5.54e-137</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>21:33:03</td>     <th>  Log-Likelihood:    </th> <td> -1498.9</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   506</td>      <th>  AIC:               </th> <td>   3022.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   494</td>      <th>  BIC:               </th> <td>   3072.</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>    11</td>      <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>    \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "      <td></td>         <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th> <td>   36.3411</td> <td>    5.067</td> <td>    7.171</td> <td> 0.000</td> <td>   26.385</td> <td>   46.298</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>CRIM</th>      <td>   -0.1084</td> <td>    0.033</td> <td>   -3.307</td> <td> 0.001</td> <td>   -0.173</td> <td>   -0.044</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ZN</th>        <td>    0.0458</td> <td>    0.014</td> <td>    3.390</td> <td> 0.001</td> <td>    0.019</td> <td>    0.072</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>CHAS</th>      <td>    2.7187</td> <td>    0.854</td> <td>    3.183</td> <td> 0.002</td> <td>    1.040</td> <td>    4.397</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>NOX</th>       <td>  -17.3760</td> <td>    3.535</td> <td>   -4.915</td> <td> 0.000</td> <td>  -24.322</td> <td>  -10.430</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>RM</th>        <td>    3.8016</td> <td>    0.406</td> <td>    9.356</td> <td> 0.000</td> <td>    3.003</td> <td>    4.600</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>DIS</th>       <td>   -1.4927</td> <td>    0.186</td> <td>   -8.037</td> <td> 0.000</td> <td>   -1.858</td> <td>   -1.128</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>RAD</th>       <td>    0.2996</td> <td>    0.063</td> <td>    4.726</td> <td> 0.000</td> <td>    0.175</td> <td>    0.424</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>TAX</th>       <td>   -0.0118</td> <td>    0.003</td> <td>   -3.493</td> <td> 0.001</td> <td>   -0.018</td> <td>   -0.005</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>PTRATIO</th>   <td>   -0.9465</td> <td>    0.129</td> <td>   -7.334</td> <td> 0.000</td> <td>   -1.200</td> <td>   -0.693</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>B</th>         <td>    0.0093</td> <td>    0.003</td> <td>    3.475</td> <td> 0.001</td> <td>    0.004</td> <td>    0.015</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>LSTAT</th>     <td>   -0.5226</td> <td>    0.047</td> <td>  -11.019</td> <td> 0.000</td> <td>   -0.616</td> <td>   -0.429</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>178.430</td> <th>  Durbin-Watson:     </th> <td>   1.078</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td> 787.785</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 1.523</td>  <th>  Prob(JB):          </th> <td>8.60e-172</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td> 8.300</td>  <th>  Cond. No.          </th> <td>1.47e+04</td> \n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.<br/>[2] The condition number is large, 1.47e+04. This might indicate that there are<br/>strong multicollinearity or other numerical problems."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:                      y   R-squared:                       0.741\n",
       "Model:                            OLS   Adj. R-squared:                  0.735\n",
       "Method:                 Least Squares   F-statistic:                     128.2\n",
       "Date:                Wed, 02 Oct 2019   Prob (F-statistic):          5.54e-137\n",
       "Time:                        21:33:03   Log-Likelihood:                -1498.9\n",
       "No. Observations:                 506   AIC:                             3022.\n",
       "Df Residuals:                     494   BIC:                             3072.\n",
       "Df Model:                          11                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "Intercept     36.3411      5.067      7.171      0.000      26.385      46.298\n",
       "CRIM          -0.1084      0.033     -3.307      0.001      -0.173      -0.044\n",
       "ZN             0.0458      0.014      3.390      0.001       0.019       0.072\n",
       "CHAS           2.7187      0.854      3.183      0.002       1.040       4.397\n",
       "NOX          -17.3760      3.535     -4.915      0.000     -24.322     -10.430\n",
       "RM             3.8016      0.406      9.356      0.000       3.003       4.600\n",
       "DIS           -1.4927      0.186     -8.037      0.000      -1.858      -1.128\n",
       "RAD            0.2996      0.063      4.726      0.000       0.175       0.424\n",
       "TAX           -0.0118      0.003     -3.493      0.001      -0.018      -0.005\n",
       "PTRATIO       -0.9465      0.129     -7.334      0.000      -1.200      -0.693\n",
       "B              0.0093      0.003      3.475      0.001       0.004       0.015\n",
       "LSTAT         -0.5226      0.047    -11.019      0.000      -0.616      -0.429\n",
       "==============================================================================\n",
       "Omnibus:                      178.430   Durbin-Watson:                   1.078\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              787.785\n",
       "Skew:                           1.523   Prob(JB):                    8.60e-172\n",
       "Kurtosis:                       8.300   Cond. No.                     1.47e+04\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "[2] The condition number is large, 1.47e+04. This might indicate that there are\n",
       "strong multicollinearity or other numerical problems.\n",
       "\"\"\""
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "form_lr = smf.ols(formula = 'y ~ CRIM + ZN + CHAS + NOX + RM + DIS + RAD + TAX + PTRATIO + B + LSTAT', \n",
    "              data=df)\n",
    "mlr = form_lr.fit()\n",
    "mlr.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exercise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a model using the following features\n",
    "\n",
    "CRIM, ZN, CHAS, NOX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"simpletable\">\n",
       "<caption>OLS Regression Results</caption>\n",
       "<tr>\n",
       "  <th>Dep. Variable:</th>            <td>y</td>        <th>  R-squared:         </th> <td>   0.300</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Model:</th>                   <td>OLS</td>       <th>  Adj. R-squared:    </th> <td>   0.295</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Method:</th>             <td>Least Squares</td>  <th>  F-statistic:       </th> <td>   53.72</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Date:</th>             <td>Wed, 02 Oct 2019</td> <th>  Prob (F-statistic):</th> <td>1.14e-37</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Time:</th>                 <td>21:40:55</td>     <th>  Log-Likelihood:    </th> <td> -1749.9</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>No. Observations:</th>      <td>   506</td>      <th>  AIC:               </th> <td>   3510.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Residuals:</th>          <td>   501</td>      <th>  BIC:               </th> <td>   3531.</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Df Model:</th>              <td>     4</td>      <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Covariance Type:</th>      <td>nonrobust</td>    <th>                     </th>     <td> </td>   \n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "      <td></td>         <th>coef</th>     <th>std err</th>      <th>t</th>      <th>P>|t|</th>  <th>[0.025</th>    <th>0.975]</th>  \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Intercept</th> <td>   32.8818</td> <td>    2.157</td> <td>   15.245</td> <td> 0.000</td> <td>   28.644</td> <td>   37.119</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>CRIM</th>      <td>   -0.2515</td> <td>    0.044</td> <td>   -5.676</td> <td> 0.000</td> <td>   -0.339</td> <td>   -0.164</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>ZN</th>        <td>    0.0768</td> <td>    0.017</td> <td>    4.462</td> <td> 0.000</td> <td>    0.043</td> <td>    0.111</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>CHAS</th>      <td>    6.9811</td> <td>    1.367</td> <td>    5.109</td> <td> 0.000</td> <td>    4.296</td> <td>    9.666</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>NOX</th>       <td>  -19.4631</td> <td>    3.768</td> <td>   -5.166</td> <td> 0.000</td> <td>  -26.865</td> <td>  -12.061</td>\n",
       "</tr>\n",
       "</table>\n",
       "<table class=\"simpletable\">\n",
       "<tr>\n",
       "  <th>Omnibus:</th>       <td>172.725</td> <th>  Durbin-Watson:     </th> <td>   0.826</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Prob(Omnibus):</th> <td> 0.000</td>  <th>  Jarque-Bera (JB):  </th> <td> 480.442</td> \n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Skew:</th>          <td> 1.680</td>  <th>  Prob(JB):          </th> <td>4.71e-105</td>\n",
       "</tr>\n",
       "<tr>\n",
       "  <th>Kurtosis:</th>      <td> 6.390</td>  <th>  Cond. No.          </th> <td>    327.</td> \n",
       "</tr>\n",
       "</table><br/><br/>Warnings:<br/>[1] Standard Errors assume that the covariance matrix of the errors is correctly specified."
      ],
      "text/plain": [
       "<class 'statsmodels.iolib.summary.Summary'>\n",
       "\"\"\"\n",
       "                            OLS Regression Results                            \n",
       "==============================================================================\n",
       "Dep. Variable:                      y   R-squared:                       0.300\n",
       "Model:                            OLS   Adj. R-squared:                  0.295\n",
       "Method:                 Least Squares   F-statistic:                     53.72\n",
       "Date:                Wed, 02 Oct 2019   Prob (F-statistic):           1.14e-37\n",
       "Time:                        21:40:55   Log-Likelihood:                -1749.9\n",
       "No. Observations:                 506   AIC:                             3510.\n",
       "Df Residuals:                     501   BIC:                             3531.\n",
       "Df Model:                           4                                         \n",
       "Covariance Type:            nonrobust                                         \n",
       "==============================================================================\n",
       "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
       "------------------------------------------------------------------------------\n",
       "Intercept     32.8818      2.157     15.245      0.000      28.644      37.119\n",
       "CRIM          -0.2515      0.044     -5.676      0.000      -0.339      -0.164\n",
       "ZN             0.0768      0.017      4.462      0.000       0.043       0.111\n",
       "CHAS           6.9811      1.367      5.109      0.000       4.296       9.666\n",
       "NOX          -19.4631      3.768     -5.166      0.000     -26.865     -12.061\n",
       "==============================================================================\n",
       "Omnibus:                      172.725   Durbin-Watson:                   0.826\n",
       "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              480.442\n",
       "Skew:                           1.680   Prob(JB):                    4.71e-105\n",
       "Kurtosis:                       6.390   Cond. No.                         327.\n",
       "==============================================================================\n",
       "\n",
       "Warnings:\n",
       "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
       "\"\"\""
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "form_lr = smf.ols(formula = 'y ~ CRIM + ZN + CHAS + NOX', \n",
    "              data=df)\n",
    "mlr = form_lr.fit()\n",
    "mlr.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlation Matrix\n",
    "\n",
    "Useful diagnostic tool to identify collinearity between predictors\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>CRIM</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>0.41</td>\n",
       "      <td>-0.06</td>\n",
       "      <td>0.42</td>\n",
       "      <td>-0.22</td>\n",
       "      <td>0.35</td>\n",
       "      <td>-0.38</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.29</td>\n",
       "      <td>-0.39</td>\n",
       "      <td>0.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ZN</td>\n",
       "      <td>-0.20</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-0.53</td>\n",
       "      <td>-0.04</td>\n",
       "      <td>-0.52</td>\n",
       "      <td>0.31</td>\n",
       "      <td>-0.57</td>\n",
       "      <td>0.66</td>\n",
       "      <td>-0.31</td>\n",
       "      <td>-0.31</td>\n",
       "      <td>-0.39</td>\n",
       "      <td>0.18</td>\n",
       "      <td>-0.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>INDUS</td>\n",
       "      <td>0.41</td>\n",
       "      <td>-0.53</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.76</td>\n",
       "      <td>-0.39</td>\n",
       "      <td>0.64</td>\n",
       "      <td>-0.71</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.38</td>\n",
       "      <td>-0.36</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>CHAS</td>\n",
       "      <td>-0.06</td>\n",
       "      <td>-0.04</td>\n",
       "      <td>0.06</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.09</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>-0.04</td>\n",
       "      <td>-0.12</td>\n",
       "      <td>0.05</td>\n",
       "      <td>-0.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>NOX</td>\n",
       "      <td>0.42</td>\n",
       "      <td>-0.52</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.09</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-0.30</td>\n",
       "      <td>0.73</td>\n",
       "      <td>-0.77</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.19</td>\n",
       "      <td>-0.38</td>\n",
       "      <td>0.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>RM</td>\n",
       "      <td>-0.22</td>\n",
       "      <td>0.31</td>\n",
       "      <td>-0.39</td>\n",
       "      <td>0.09</td>\n",
       "      <td>-0.30</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-0.24</td>\n",
       "      <td>0.21</td>\n",
       "      <td>-0.21</td>\n",
       "      <td>-0.29</td>\n",
       "      <td>-0.36</td>\n",
       "      <td>0.13</td>\n",
       "      <td>-0.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>AGE</td>\n",
       "      <td>0.35</td>\n",
       "      <td>-0.57</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.73</td>\n",
       "      <td>-0.24</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-0.75</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.51</td>\n",
       "      <td>0.26</td>\n",
       "      <td>-0.27</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>DIS</td>\n",
       "      <td>-0.38</td>\n",
       "      <td>0.66</td>\n",
       "      <td>-0.71</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>-0.77</td>\n",
       "      <td>0.21</td>\n",
       "      <td>-0.75</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-0.49</td>\n",
       "      <td>-0.53</td>\n",
       "      <td>-0.23</td>\n",
       "      <td>0.29</td>\n",
       "      <td>-0.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>RAD</td>\n",
       "      <td>0.63</td>\n",
       "      <td>-0.31</td>\n",
       "      <td>0.60</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>0.61</td>\n",
       "      <td>-0.21</td>\n",
       "      <td>0.46</td>\n",
       "      <td>-0.49</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.46</td>\n",
       "      <td>-0.44</td>\n",
       "      <td>0.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>TAX</td>\n",
       "      <td>0.58</td>\n",
       "      <td>-0.31</td>\n",
       "      <td>0.72</td>\n",
       "      <td>-0.04</td>\n",
       "      <td>0.67</td>\n",
       "      <td>-0.29</td>\n",
       "      <td>0.51</td>\n",
       "      <td>-0.53</td>\n",
       "      <td>0.91</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.46</td>\n",
       "      <td>-0.44</td>\n",
       "      <td>0.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PTRATIO</td>\n",
       "      <td>0.29</td>\n",
       "      <td>-0.39</td>\n",
       "      <td>0.38</td>\n",
       "      <td>-0.12</td>\n",
       "      <td>0.19</td>\n",
       "      <td>-0.36</td>\n",
       "      <td>0.26</td>\n",
       "      <td>-0.23</td>\n",
       "      <td>0.46</td>\n",
       "      <td>0.46</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-0.18</td>\n",
       "      <td>0.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>B</td>\n",
       "      <td>-0.39</td>\n",
       "      <td>0.18</td>\n",
       "      <td>-0.36</td>\n",
       "      <td>0.05</td>\n",
       "      <td>-0.38</td>\n",
       "      <td>0.13</td>\n",
       "      <td>-0.27</td>\n",
       "      <td>0.29</td>\n",
       "      <td>-0.44</td>\n",
       "      <td>-0.44</td>\n",
       "      <td>-0.18</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-0.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>LSTAT</td>\n",
       "      <td>0.46</td>\n",
       "      <td>-0.41</td>\n",
       "      <td>0.60</td>\n",
       "      <td>-0.05</td>\n",
       "      <td>0.59</td>\n",
       "      <td>-0.61</td>\n",
       "      <td>0.60</td>\n",
       "      <td>-0.50</td>\n",
       "      <td>0.49</td>\n",
       "      <td>0.54</td>\n",
       "      <td>0.37</td>\n",
       "      <td>-0.37</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         CRIM    ZN  INDUS  CHAS   NOX    RM   AGE   DIS   RAD   TAX  PTRATIO  \\\n",
       "CRIM     1.00 -0.20   0.41 -0.06  0.42 -0.22  0.35 -0.38  0.63  0.58     0.29   \n",
       "ZN      -0.20  1.00  -0.53 -0.04 -0.52  0.31 -0.57  0.66 -0.31 -0.31    -0.39   \n",
       "INDUS    0.41 -0.53   1.00  0.06  0.76 -0.39  0.64 -0.71  0.60  0.72     0.38   \n",
       "CHAS    -0.06 -0.04   0.06  1.00  0.09  0.09  0.09 -0.10 -0.01 -0.04    -0.12   \n",
       "NOX      0.42 -0.52   0.76  0.09  1.00 -0.30  0.73 -0.77  0.61  0.67     0.19   \n",
       "RM      -0.22  0.31  -0.39  0.09 -0.30  1.00 -0.24  0.21 -0.21 -0.29    -0.36   \n",
       "AGE      0.35 -0.57   0.64  0.09  0.73 -0.24  1.00 -0.75  0.46  0.51     0.26   \n",
       "DIS     -0.38  0.66  -0.71 -0.10 -0.77  0.21 -0.75  1.00 -0.49 -0.53    -0.23   \n",
       "RAD      0.63 -0.31   0.60 -0.01  0.61 -0.21  0.46 -0.49  1.00  0.91     0.46   \n",
       "TAX      0.58 -0.31   0.72 -0.04  0.67 -0.29  0.51 -0.53  0.91  1.00     0.46   \n",
       "PTRATIO  0.29 -0.39   0.38 -0.12  0.19 -0.36  0.26 -0.23  0.46  0.46     1.00   \n",
       "B       -0.39  0.18  -0.36  0.05 -0.38  0.13 -0.27  0.29 -0.44 -0.44    -0.18   \n",
       "LSTAT    0.46 -0.41   0.60 -0.05  0.59 -0.61  0.60 -0.50  0.49  0.54     0.37   \n",
       "\n",
       "            B  LSTAT  \n",
       "CRIM    -0.39   0.46  \n",
       "ZN       0.18  -0.41  \n",
       "INDUS   -0.36   0.60  \n",
       "CHAS     0.05  -0.05  \n",
       "NOX     -0.38   0.59  \n",
       "RM       0.13  -0.61  \n",
       "AGE     -0.27   0.60  \n",
       "DIS      0.29  -0.50  \n",
       "RAD     -0.44   0.49  \n",
       "TAX     -0.44   0.54  \n",
       "PTRATIO -0.18   0.37  \n",
       "B        1.00  -0.37  \n",
       "LSTAT   -0.37   1.00  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.options.display.float_format = '{:,.2f}'.format\n",
    "corr_matrix = df.corr()\n",
    "corr_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>CRIM</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ZN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.66</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>INDUS</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.64</td>\n",
       "      <td>-0.71</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>CHAS</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>NOX</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.76</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.73</td>\n",
       "      <td>-0.77</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>RM</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>AGE</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.64</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.73</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-0.75</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>DIS</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.66</td>\n",
       "      <td>-0.71</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.77</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.75</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>RAD</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.61</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>TAX</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.67</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.91</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PTRATIO</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>B</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>LSTAT</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.61</td>\n",
       "      <td>0.60</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         CRIM   ZN  INDUS  CHAS   NOX    RM   AGE   DIS  RAD  TAX  PTRATIO  \\\n",
       "CRIM     1.00 0.00   0.00  0.00  0.00  0.00  0.00  0.00 0.63 0.00     0.00   \n",
       "ZN       0.00 1.00   0.00  0.00  0.00  0.00  0.00  0.66 0.00 0.00     0.00   \n",
       "INDUS    0.00 0.00   1.00  0.00  0.76  0.00  0.64 -0.71 0.00 0.72     0.00   \n",
       "CHAS     0.00 0.00   0.00  1.00  0.00  0.00  0.00  0.00 0.00 0.00     0.00   \n",
       "NOX      0.00 0.00   0.76  0.00  1.00  0.00  0.73 -0.77 0.61 0.67     0.00   \n",
       "RM       0.00 0.00   0.00  0.00  0.00  1.00  0.00  0.00 0.00 0.00     0.00   \n",
       "AGE      0.00 0.00   0.64  0.00  0.73  0.00  1.00 -0.75 0.00 0.00     0.00   \n",
       "DIS      0.00 0.66  -0.71  0.00 -0.77  0.00 -0.75  1.00 0.00 0.00     0.00   \n",
       "RAD      0.63 0.00   0.00  0.00  0.61  0.00  0.00  0.00 1.00 0.91     0.00   \n",
       "TAX      0.00 0.00   0.72  0.00  0.67  0.00  0.00  0.00 0.91 1.00     0.00   \n",
       "PTRATIO  0.00 0.00   0.00  0.00  0.00  0.00  0.00  0.00 0.00 0.00     1.00   \n",
       "B        0.00 0.00   0.00  0.00  0.00  0.00  0.00  0.00 0.00 0.00     0.00   \n",
       "LSTAT    0.00 0.00   0.60  0.00  0.00 -0.61  0.60  0.00 0.00 0.00     0.00   \n",
       "\n",
       "           B  LSTAT  \n",
       "CRIM    0.00   0.00  \n",
       "ZN      0.00   0.00  \n",
       "INDUS   0.00   0.60  \n",
       "CHAS    0.00   0.00  \n",
       "NOX     0.00   0.00  \n",
       "RM      0.00  -0.61  \n",
       "AGE     0.00   0.60  \n",
       "DIS     0.00   0.00  \n",
       "RAD     0.00   0.00  \n",
       "TAX     0.00   0.00  \n",
       "PTRATIO 0.00   0.00  \n",
       "B       1.00   0.00  \n",
       "LSTAT   0.00   1.00  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corr_matrix[np.abs(corr_matrix) < 0.6] = 0\n",
    "corr_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAq0AAAHVCAYAAADbxcKaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xl4VNX9x/H3mUCAQAIBsoIlCGoVyqLRYllcAGVTqlYr0sXqT7FWgVK1itZiVUBxX6rQqlClyGLdQHEJQgIqEBUEkSWyhSUkJIEQAkKY8/tjQsieYCYz9zqfF888T+7cM/d+7pmZcPKdc+8Yay0iIiIiIk7mCXYAEREREZHaaNAqIiIiIo6nQauIiIiIOJ4GrSIiIiLieBq0ioiIiIjjadAqIiIiIo6nQauIiIiIOJ4GrSIiIiLieBq0ioiIiIjjNQp2gEBo9pMRjv7arxEzRwU7goiEsNQnM4MdoVb9/nxKsCOIlPNy3wtNsDNAw4xxDm2f5Yhjq0iVVhERERFxvJCotIqIiIj8GBkTOvXH0DlSEREREXEtVVpFREREXMqEUP0xdI5URERERFxLlVYRERERlwqlOa0atIqIiIi4VCgNWkPnSEVERETEtVRpFREREXEpYxz5PQANQpVWEREREXE8VVpFREREXCt06o8atJ6EF6eMYnD/nuTkFpA88K5gx6lW/tq1bJ41B7xe4vr2of2QQcGOVIky+o8bciqjfzghY7+eidx3QzJhHsOcjzOY+uY3ldoM+UUHRv+6G9bCt1vzGffUUhJjmvPPuy7A4zE0DvPwn/fWM+vDTQHPf5wT+rI2yugfbshYHzoRqwEYY+KNMa8bY74zxqwzxrxnjDndGHPIGLOq5L7/GGMal7S/0Bgzv+Tn640x1hjTv8z2rii571eBOoZX5y5h+O8mB2p3P4j1etk8cxZdxt5OzwcnkLNiJUW7dgU7VjnK6D9uyKmM/uGEjB6PYcJN53HjQ4sYNOZdhvVNonP7luXadEiI5JYru3LN+A8YPPZdHnolHYCc/ENcc89CLv/LAq66+31GXdmV2OhmAc1/nBP6sjbK6B9uyCh1F5BBq/HNEn4TWGyt7WStPQsYD8QB31lrewA/A9oD11SzmTXAiDLL1wKrGy51ZctWrCdvX2Egd3nSDmzZQtPYWJrGxOBp1IiY85LJWxXQbqqVMvqPG3Iqo384IWP3zm3YtvsAmXsKOVrsZcHSbQw475RybX494DReW7iBgoNHAMjbfxiAo8VejhR7AQhv5METxJNHnNCXtVFG/3BDxvoyxuP3m1MFKtlFwFFr7YvH77DWrgIyyywfA1YA7arZRhpwnjGmsTGmBdAZWNVwkd3pSP4+wqOjS5fDo6P5Pn9fEBNVpoz+44acyugfTsgY1yaC3bkHS5ezcg8S17p8tbRjYhRJCVHMnngp8yYPol/PxNJ1CW0imP/EMNL+dRXT3lxLdv6hgGUvywl9WRtl9A83ZJS6C9SgtSvwRU0NjDFNgZ8DC6tpYoGPgUuB4cA7/gz4Y+aGq2Eoo/+4Iacy+kegM1a1O1thOSzMkJQYyci/fcjYJ5Yy8dZeREY0BmB3bhHDxs2n/61vccVFnWjTsmmDZ64rPd/+oYyBZ/D4/eZUTkjWyRizCsgFtltrv66h7ev4pgVcC8yqaaPGmJuNMenGmPTiwgz/pXW48OhWHMnPL10+kp9PeKtWQUxUmTL6jxtyKqN/OCFjVm4RCW2aly7Ht2lOdt6hSm0+XpFJ8THLjuxCNu8sICkxqlyb7PxDbMrcx7lnxQYkd0VO6MvaKKN/uCFjfWl6gP99A5xTzbrjc1o7A72MMZdXtxFr7Qp8Vdu21tqNNe3QWjvNWptsrU1u1KLzD83tOpFJSRzak83hnL14i4vJWZFO6+7dgx2rHGX0HzfkVEb/cELGrzNy6ZAQSfvYFjRu5GFonw6krMws1+bjFZn06hoPQHRkEzomRpGZdYD4NhE0CQ8DIKp5OOf8NJbNOwsCmv84J/RlbZTRP9yQUeouUJe8WgRMNMbcZK39F4Ax5lwg4ngDa+1uY8zdwD3U/NH/PcDhhgxbnRnP3k7f88+kbXQkGcuf48En5jFj9uJgRKmWCQvj1Ouu5Zunngavl9jevYlol1j7AwNIGf3HDTmV0T+ckPGY1/LAv1fwyv39CfMY5qZksClzP2Ou7c7a73JJWbmD1K920ad7AgufvoxjXsvkGV+yr/AIvTu14Z7fn4PFYjD8++11bNwenLmFTujL2iijf7ghY305uTLqb8baijOSGmhHxiQCT+GruB4GtgJjgTettV1L2hh8J1fdBoQBd1hrhxljrgeSrbW3VdjmdGC+tXZeTftu9pMRgTnIH2jEzFHBjiAiISz1yczaGwVZvz+fUnsjkQB6ue+Fjpgd2+b00X4f4+RufMYRx1ZRwL5cwFq7i6ovZ9W1TBsLlK3bLy65fzowvYptXu/HiCIiIiKuEqxKqzFmEPA0viLjv621kyus7wC8DMQAecBvrLU76rPP0Kkpi4iIiEi9GWPCgOeBwcBZwAhjzFkVmj0G/Mda2w34BzCpvvvVoFVERETEpUwD/KuD84AMa+1ma+0RfFd3Gl6hzVlASsnPn1Sx/qRp0CoiIiLiUkG65FU7ynxBFLCDyl8OtRq4quTnK4BIY0yb+hyrBq0iIiIiUqrste5LbjdXbFLFwyqeEHYHcIEx5ivgAmAnUFyfXAE7EUtERERE/KshTsSy1k4DptXQZAdQ9pIe7YFdFbaxC7gSwBjTArjKWru/PrlUaRURERGRk7ESOM0Y09EYE47vm0rLXWPfGNPWnBhR34PvSgL1okGriIiIiEsFY06rtbYY3zX1PwC+BeZYa78xxvyjzDebXghsMMZsBOKAh+t7rJoeICIiIuJawak/WmvfA96rcN/9ZX6eB9T45U8nS5VWEREREXE8VVpFREREXCpY34gVDKFzpCIiIiLiWiFRaR0xc1SwI9Ro1sipwY5QK6f3oYhTpU7aEuwItep3T8dgRxCRHyiUKq0hMWgVERER+TEyIfSheegcqYiIiIi4liqtIiIiIi4VStMDQudIRURERMS1VGkVERERcSljTLAjBIwGrSIiIiIupekBIiIiIiIOokqriIiIiEvpklciIiIiIg6iSquIiIiIS2lOq4iIiIiIg6jSKiIiIuJSoVRp1aD1JOWvXcvmWXPA6yWubx/aDxkU7EjlvDhlFIP79yQnt4DkgXcFO061nN6P4I6M4I6cylg3/c5px32jfk6YxzDng41MnbumUpshfZMYPbIn1lq+3ZLHuEdTAUiIac6kMb2Jb9scgBvv/4id2YUBzQ/O6Me6cENOZfQPN2Ssj1A6Ectxg1ZjzBXA3yvc3Q34E/BPYLS19tmSts8B6dba6YHIZr1eNs+cRZdxYwmPjmb1Q5No3aMbEYmJgdh9nbw6dwkvzviAfz95a7CjVMsN/eiGjOCOnMpYNx6PYcKtvfj9vR+QtbeI/z11GSmfbycjc39pmw6JUdxyTTeuuWMBBYVHaN2yaem6x/7Sl3/O/pplX+0iomkjvNYGLPtxTujHunBDTmX0DzdklLpz3PDcWvumtbbH8Ru+gWoa8AGQDYwxxoQHI9uBLVtoGhtL05gYPI0aEXNeMnmrVgcjSrWWrVhP3r7AV1dOhhv60Q0ZwR05lbFuup/elm27DpCZVcjRYi8LUjcz4PyflGvz60Gn89r8bykoPAJA3v7DAHQ+pSVhYR6WfbULgKLDxRz+/lhA84Mz+rEu3JBTGf3DDRnrzXj8f3Mo5yYDjDGnA/cDvwW8QA6QAvw+GHmO5O8jPDq6dDk8Oprv8/cFI4qruaEf3ZAR3JFTGesmrk0Eu/ceLF3O2ltEXJvm5dp0bBdFUruWzH5sCPOeGEq/c9oBkNS+JQUHj/D8vRfzzrOX89cbkvF4Av/Vjk7ox7pwQ05l9A83ZJS6c+yg1RjTGPgvcIe1dnuZVZOBvxhjwmp5/M3GmHRjTPqGd95twJwNtumQ4oZ+dENGcEdOZaxqf5V3aCt8xB8W5iEpMYqRf32fsY8sYeKY3kQ2D6eRx3Bulzgmv7SCK8a8yykJkVw1oHOgotfIDc81uCOnMvqHGzKeDGM8fr85lXOTwYPAN9ba18veaa3dAqwArqvpwdbaadbaZGtt8hmXX+aXQOHRrTiSn1+6fCQ/n/BWrfyy7VDihn50Q0ZwR05lrJusvQdJaHuishrfNoLsvKJKbT7+fDvFxyw79hSyecd+khKjyNpbxLrvcsnMKuSY1/LxZ9vp0rlNQPODM/qxLtyQUxn9ww0Z68sY4/ebUzly0GqMuRC4CritmiYTgb8S4PyRSUkc2pPN4Zy9eIuLyVmRTuvu3QMZ4UfBDf3ohozgjpzKWDdfb9xLh8Qo2se1oHEjD0P7nUrK55nl2nz82XZ6dUsAIDqqCR3btSQz6wBfb9pLVIsmtI5qAkCv7glkbA/8R6BO6Me6cENOZfQPN2SUunPi1QOigVeA66y1B6pqY61db4xZBwzDV3UNTLawME697lq+eepp8HqJ7d2biHbOOgNxxrO30/f8M2kbHUnG8ud48Il5zJi9ONixynFDP7ohI7gjpzLWzTGv5YEXPueVhy4hzGOY++EmNm3fx5jf9GTtpr2kLM8k9Yud9Dk7kYUvXsExr2XySyvZd+B7ACa/tJL/TBqEMYa1m/Yye+HGgOYHZ/RjXbghpzL6hxsy1lcoXfLKVJwzFWzGmHuA+4BNFVbNAn5rre1a0q478BVwQ22XvLohbbGzDrKCWSOnBjtCrUbMHBXsCCKulDppS7Aj1KrfPR2DHUHEdV7ue6EjPkc/LflZv49xNqXf7ohjq8hxlVZr7SRgUjWrHynTbjUOnd4gIiIiEghOPnHK3xw3aBURERGROnLwiVP+FjrDcxERERFxLVVaRURERNwqhMqPIXSoIiIiIuJWqrSKiIiIuJXmtIqIiIiIOIcqrSIiIiJuFUKVVg1aRURERNwqhD4zD6FDFRERERG3UqVVRERExKWspgdIII2YOSrYEepk1sipwY5QKzf05dI71wU7Qq36TDkr2BFqlTplW7Aj1Mm383sGO0KtRi3bF+wItUr728ZgR6hV3wdPD3aEH43UZ3cGO0Lt+gY7QOjRoFXqxA0DVhERkZATOoVWDVpFREREXMsTOqNWnYglIiIiIo6nSquIiIiIW4XQiViqtIqIiIiI46nSKiIiIuJWoVNo1aBVRERExLV0IpaIiIiIiHOo0ioiIiLiVjoRS0RERETEOVRpFREREXGr0Cm0qtIqIiIiIs6nSutJyl+7ls2z5oDXS1zfPrQfMijYkSpxQ8YXp4xicP+e5OQWkDzwrmDHqZJT+rFvr1O49899CPN4mPvOOqa9+lW59feM6U2vc9oB0LRpI9pENyN54EsAJMS14OHxF5EQ1wJrLTeNW8DO3QcCfgxO6Mt+Zydy3/+dR1iYYc6Hm5j6xtpKbYb07sDoET2wwLdb8hj3eFrpuhbNGrPwn7/ko8+388DU5Q2S0VrLpInTSUv9iqZNm/DwxD9yVpdTy7U5ePAQv/vN30uX92TlMeyyPtw9/nrSV67jkUkz2LhxO1MeH8Mll/ZqkJw1ccJzDdD3vPbcd9svfM/3gvVM++/qcuvH/+l8evVMAKBpE9/75pxhMzizcxse+HMfWkQ05pjX8sJrX/HeJ5uDcQiO6cuaOC1jv+4J3PeHZMI8hjkpGUx9e12lNkPO/wmjr+6GtZZvt+1j3DPLgpDUj0Lo6gF+HbQaYwqttS2MMUnAFmC0tfbZknXPAenW2unGmOnABUAB0Az4HLjHWruz7HbKbPd6INlae5sx5gxgKtAKaAKkWWtv9udxVMd6vWyeOYsu48YSHh3N6ocm0bpHNyISEwOx+zpxQ0aAV+cu4cUZH/DvJ28NdpQqOaUfPR7D3+/oxx9Gv0tWdiFvvPIrUtK28t3W/NI2k54+8Qv3t1f/jDNPb1u6/Ojf+/PC9C/4dMUOIpo1wusNaHzAGX3p8RgmjOrF7+//kKzcIv73+FBSVmSSkbm/tE2HhEhuufpnXPPX9yk4eITWLZuW28bYkT1YsTarQXOmpa5i+7Ys3lv4NF+v3sSD/3iJWbMfLtemefNmvPHmo6XL11x1NwMGngdAQmJbHpp0K9NffrdBc1bHCc81lDzfY/pw/R0LyMo5yBsvXsGiZdvI2LavtM3E5z8r/fm3V3ThrNN875tDh4u5c+InbNtZQGybCN6cdiVpK3dwoPBIQI/BKX1ZE6dl9BjDhBvP5fcPLfK9zycNIiV9Bxk7C0rbdIiP5JZfduGav33oe59HNQlKVr8KnTFrg04PyAbGGGPCq1l/p7W2O3AG8BXwSQ1ty3oGeNJa28NaeybwrH/i1u7Ali00jY2laUwMnkaNiDkvmbxVq2t/YAC5ISPAshXrydtXGOwY1XJKP3Y7K5ZtO/aTuauAo8VeFnyUwYB+HattP3Tgacz/aBMAnZKiaRTm4dMVOwAoOlTM4e+LA5K7LCf0ZffT2rJtdwGZewp9/Zi2hQE/P6Vcm19fejqvLdhAwUHf4CRv/+HSdV06taZtq2Ys/WpXg+b8ZNFKLh/eD2MM3XuczoGCg+Rk51fbftvW3eTmFXBO8pkAtGsXyxlndMDjCc7MLyc81wDdfhrDtp37ydx9wPd8L/qO/r2Tqm0/rH8n5qdkALB1x362lQxysnOLyM0/VOkPmEBwSl/WxGkZu3duw7asA2RmF3L0mJcFn25jwLkV3uf9O/PaBxtPvM8Lvg9GVPmBGvI3Ww6QAvy+pkbW50kgCxhch+0mADvKPH5NfUKejCP5+wiPji5dDo+O5vv8fTU8IvDckNENnNKPcTHNyco+MbjPyi4kLqZ5lW0T41vQPjGSz9N3AtDxJ60oOPA9z00exFszruau287HE4SPkZzQl3FtIti992DpctbeIuLalO/HjolRJLWLYvYjg5k3ZQj9zvZVi4yB8TecyyPT0xs85549+cTHtzmRO74Ne7Lzqm3/3oJlDBp8PsYhl7xxwnMNEB/TnN05ZZ7vnIPVv2/iWtA+IYrPqviDpNtPYwhv7GH7roIqHtmwnNKXNXFaxrjWzdidW1S6nJVbRFzrZuXadEyMJCkhitn/uIR5D11Kv+4JgY7pd9YYv9+cqqH/HJ8M/MUYE1aHtl8CP61DuyeBRcaY940xfzbGtKpXwnpy8HNbyg0Z3SAY/VjVYMRiq2w7dOBpfPDJd3i9vvVhYYbkHgk88synXHXDPE5pF8WVQ+vyFmt4ge7LqvZnbfl+DAszJCVEMXL8QsY+lsrE235BZPPG/GbIT1n8xQ527y2qvBE/q5gJqn4NHPf++58yZGjvhoxUb075/VNV3wIMu7gTC5dsLn3fHBfTuhlTxl/E3Y8soZqHBpxT+rImwcxY5e/LCs9dmMdDUkIkIx/4iLFPL2XiLb2IjGgcoIRSXw16Ipa1dosxZgVwXR2a1/ZStyXbfMUY8wEwCBgOjDLGdLfWlqvxG2NuBm4G+MUd4zjj8stOOn9F4dGtOJJ/4qO6I/n5hLcK6pi5EjdkdAOn9GNWdiHxsaXTu4mPbUF2TtWDp6EDOvPAYydOHMrKPsi6jXvJLKkSfbxkCz26xjEvwNMdndCXWXuLSGh7otIW3zaC7LyiSm1Wbcih+Jhlx55CNu8sICkhih5nxHBul1hGDv4pEc0aEd7IQ9Gho0z5z5d+yTZr5gfMm5cCQNeuncjKyi1dtycrl9iY6Coft379Vo4Ve+lS4UStYHLCcw2+ympCmcpqfExzsqv5o2PoxZ2Y8FT5E3FaRDTmX5MH8+RLK1m1LrtBs1bHKX1ZE6dlzMotIqFNROlyfJsIsvMPlW+TV8SqjXt97/Ocg2zeVUBSQiRrvqv+Ew3HC6ETsQIx8Wki8Nc67Ksn8G3Jz4cqzG9tDew9vmCt3WWtfdlaOxwoBrpW3Ji1dpq1Ntlam+yPAStAZFISh/ZkczhnL97iYnJWpNO6e3e/bNtf3JDRDZzSj2u+zSbplJa0T4ikcSMPQwd2JiVtS6V2HX/SiqioJny1JqvcY1tGNiG6lW8+Xq/kdmRsqX5+ZENxQl9+vWkvHRKjaB/XwtePfTuSsnxHuTYfL99Or27xAERHNqFjYhSZewr5yxNp9LvxDS686Q0mv5zOm59s9tuAFWDEyEt5481HeePNR7m4/7m883Yq1lpWr9pIi8gIYmKrHrS+v+BTBg/9hd9y+IMTnmuANRtySGrfkvbxJe+bizuR8um2Su06ntKSqMgmfPXNntL7Gjfy8PyDl/DWhxtZuKTyey1QnNKXNXFaxq+/y6VDQiTtY5rTOMzD0F90ICW9wvt8RSa9usYBJe/zhEgy9zj3/Io6MQ1wc6gGv+SVtXa9MWYdMAxYUXG98dXzb8c3V3Vhyd1LgN8ALxtjmgHXAHeVtB8EpFhrjxpj4oE2wM6GPg4AExbGqdddyzdPPQ1eL7G9exPRzjlncoI7MgLMePZ2+p5/Jm2jI8lY/hwPPjGPGbMXBztWKaf047Fjln88lsZLT19GmMcwb/56MrbkM/qmc1m7PodFaVsBGHbJabz3UUa5x3q9lsnPfsqM54ZjgG825DCnisu/NDQn9OUxr+WBqct5ZcIA36XDPt7Epsx9jLmuB2szcklZkUnql7vo0yORhc8N55jXMnl6OvsOBPYkjX4X9CQt9SsGXzqGZk3DeXDiH0vXXXXFXeWuGvDBws/459S7yz1+zZoMxt7+OAUFB1n8yRc8/+xc3p7/eMDyO+G5Bt/75oGnl/HylMGEeTzMe38DGVvzGfOHc1izYS+LSgaww/p3ZsGi78o9dvBFp3Ju9wSiWzbhykGnA/DXyUv4NiO30n4aklP6siZOy3jMa3ng5XReufdiwjyGuZ98x6Yd+xlzTTfWfpdLyhc7SV29mz7dE1j4xDDf+/y1r9gX4CtDyA9nqpvn84M2Vv6SV/OttV1L7u+O7woBN1RxyasITlzyakdJ+3b4LmvVHt+Y/z/W2sdL1j0BDAWOn9o7xVr7Wk25bkhb7JAZSe41a+TUYEeokxEzRwU7Qq2W3hn4gePJ6jPlrGBHqFXqlMqVMyf69i1nVceqMmqZs07wqUra3zYGO0Kt+j54erAj/GikPhuQWlS9ZMwZ6YiaZOfLZ/h9jJPxzu8dcWwV+bXSevzaqtbarZT5yN5au5oy0wOstdfXsp2d+CqzVa0bB4yrf1oRERERcQt9I5aIiIiIW4XQiVgatIqIiIi4VeiMWQNy9QARERERkXpRpVVERETErdzwrRN+okqriIiIiDieKq0iIiIibqVKq4iIiIhI1Ywxg4wxG4wxGcaYu6tpc40xZp0x5htjzH/ru09VWkVERETcKgjlR2NMGPA8MBDYAaw0xrxjrV1Xps1pwD1Ab2ttvjEmtr771aBVRERExK2CMz3gPCDDWrvZF8G8DgwHyn7l403A89bafABrbXZ9d6rpASIiIiJSyhhzszEmvczt5gpN2gGZZZZ3lNxX1unA6caYZcaYz40xg+qbS5VWEREREbdqgEKrtXYaMO0k92orLDcCTgMuBNoDacaYrtbafT80lwatUicjZo4KdoQ6mTVyarAj1Motfel0/e7sEOwIdTJq2Q/+/Sxl9H3w9GBHkADqd3vFop04zA7glDLL7YFdVbT53Fp7FNhijNmAbxC78ofuVNMDRERERFzKeozfb3WwEjjNGNPRGBMOXAu8U6HNW8BFAMaYtvimC2yuz7Gq0ioiIiLiVkE4EctaW2yMuQ34AAgDXrbWfmOM+QeQbq19p2TdJcaYdcAx4E5rbW599qtBq4iIiIicFGvte8B7Fe67v8zPFhhXcvMLDVpFRERE3Cp0vhBLc1pFRERExPlUaRURERFxq7qdOPWjoEGriIiIiFsF5xuxgkLTA0RERETE8VRpFREREXGr0Cm0qtIqIiIiIs6nSquIiIiIW4XQiViqtIqIiIiI46nSepLy165l86w54PUS17cP7YcMCnakSpTRP16cMorB/XuSk1tA8sC7gh2nWm7oS2X0D2X0HzfkVEb/cEPGelGl1X+MMfHGmNeNMd8ZY9YZY94zxpxujFlbod0EY8wdZZYbGWP2GmMmVWg3zBjzlTFmdcn2RjX0MRxnvV42z5xFl7G30/PBCeSsWEnRrl2B2n2dKKP/vDp3CcN/NznYMWrkhr5URv9QRv9xQ05l9A83ZKwva/x/c6oGHbQaYwzwJrDYWtvJWnsWMB6Iq8PDLwE2ANeUbAdjTGNgGnCZtbY70BNY3BDZq3JgyxaaxsbSNCYGT6NGxJyXTN6q1YHafZ0oo/8sW7GevH2FwY5RIzf0pTL6hzL6jxtyKqN/uCGj1F1DV1ovAo5aa188foe1dhWQWYfHjgCeBrYDvUrui8Q3pSG3ZFvfW2s3+DVxDY7k7yM8Orp0OTw6mu/z9wVq93WijKHFDX2pjP6hjP7jhpzK6B9uyFhvHuP/m0M19KC1K/BFNes6GWNWHb8BtxxfYYxpBvQH5gOz8A1gsdbmAe8A24wxs4wxI40xVR6DMeZmY0y6MSZ9wzvv+vGQKu6nwTbtN8oYWtzQl8roH8roP27IqYz+4YaMUrVgXj3gO2ttj+M34MUy64YBn1hri4A3gCuMMWEA1tr/wzegXQHcAbxc1cattdOstcnW2uQzLr/ML4HDo1txJD+/dPlIfj7hrVr5Zdv+ooyhxQ19qYz+oYz+44acyugfbshYb8b4/+ZQDT1o/QY45wc8bgQwwBizFV+ltg2+qQYAWGvXWGufBAYCV/khZ51EJiVxaE82h3P24i0uJmdFOq27dw/U7utEGUOLG/pSGf1DGf3HDTmV0T/ckLHeQmh6QENf8moRMNEYc5O19l8AxphzgYjqHmCMiQL6AKdYa78vue8PwAhjzOdAsrV2cUnzHsC2BsxfPltYGKdedy3fPPU0eL3E9u5NRLvEQO2+TpTRf2Y8ezt9zz+TttGRZCx/jgefmMeM2YuDHascN/SlMvqHMvqPG3Iqo3+4IaPUnbHWNuwOjEkEnsJXcT0MbAXGAm9aa7uWaTcBKAT2AoOstdeWWdca35WB3I2lAAAgAElEQVQEOuOb49oJOAQcBMZYa9NrynBD2uKGPUhxjFkjpwY7Qq1GzAzYVdpERKSBvNz3QkeUJE/90//8PsbZ/PyVjji2ihr8ywWstbuAa6pY1bVCuwllFqdXWJcHxJQsDvFjPBERERFxAX0jloiIiIhbOfjEKX/ToFVERETErRx84pS/BfOSVyIiIiIidaJKq4iIiIhL2RCaHqBKq4iIiIg4niqtIiIiIm4VQuXHEDpUEREREXErVVpFRERE3CqErh6gQauIiIiIW+lELBERERER51ClVeps6Z3rgh2hViNmjgp2hFrNGjk12BFq5YZ+TBu/IdgR6mRT2kXBjlCrzlevCHaE2h08GuwEtep3T8dgR5BQFELTA1RplTpxw4BVREREfrxUaRURERFxq9AptGrQKiIiIuJWVtMDREREREScQ5VWEREREbdSpVVERERExDlUaRURERFxqxD6cgENWkVERETcKoQ+Mw+hQxURERERt1KlVURERMStQmh6gCqtIiIiIuJ4qrSKiIiIuFUIXfJKg9aTlL92LZtnzQGvl7i+fWg/ZFCwI1XihIx9e53CvX/uQ5jHw9x31jHt1a/Krb9nTG96ndMOgKZNG9EmuhnJA18CICGuBQ+Pv4iEuBZYa7lp3AJ27j4Q8GNwQj/W5sUpoxjcvyc5uQUkD7wr2HGq5YS+7HveKdw35heEeQxz5q9n2sxV5daPv/18evVMBEpek62acc6Q6STGteD5hy8hzGNo1MjDq2+sZdbb3zZIRmstDz88jSVLvqBp0yZMnjyGLl06l2tTWFjEyJF3ly5nZe3l8ssv4t57b2LixH+xfPkaAA4f/p7c3P2kp7/u14z9eiRw3x/O9fVjSgZT3/qmUpsh5/+E0dd0w1r4dls+455eBsDL915Mj9Pakr4+m5snL/Zrrko5z2nHfaN+7sv5wUamzl1TOWffJEaP7Im1lm+35DHu0VR6dYtn/E3nlbbpdEpLxjyyhI8/296geavihPdNbZRRAingg1ZjjAWesNb+pWT5DqCFtXZCyfLNwLiS5gXAOGvtUmNMGLAC+LO1NrWk7YfAv6y1cwOR3Xq9bJ45iy7jxhIeHc3qhybRukc3IhITA7H7OnFCRo/H8Pc7+vGH0e+SlV3IG6/8ipS0rXy3Nb+0zaSS/8QAfnv1zzjz9Laly4/+vT8vTP+CT1fsIKJZI7zegEUv5YR+rItX5y7hxRkf8O8nbw12lGo5oS89HsOEcb25/s8LyMo5yBv/upJFy7aSsXVfaZuJz35W+vNvr+rCWaf5XpM5uUX8+o9vceSol4hmjVgw4xpSlm4jO7fI7zlTU79g69ZdfPjhVFav3sCECS8wd+7j5dq0aBHB228/U7p85ZVjueSS8wEYP/6m0vtfffVd1q3b7Nd8Ho9hwo3n8fsHU8jKK+J/kwaTkr6DjB37S9t0iI/kliu6cs19H1Jw8Aito5qUrvvX2+to1iSMawee5tdcVea8tRe/v/cDsvYW8b+nLiPl8+1kZJbJmRjFLdd045o7FlBQeITWLZsC8PnXWVx++zsAtGwRTspLv2LplzsbNG9VnPC+qY0yOkQIVVqDMaf1e+BKY0zbiiuMMcOAUUAfa+1PgVuA/xpj4q21x4BbgeeNMY2NMSMAG6gBK8CBLVtoGhtL05gYPI0aEXNeMnmrVgdq93XihIzdzopl2479ZO4q4GixlwUfZTCgX8dq2w8deBrzP9oEQKekaBqFefh0xQ4Aig4Vc/j74oDkLssJ/VgXy1asJ29fYbBj1MgJfdntzFi27Swgc/cB32syJYP+fZKqbT+sf2fmf5wBwNFiL0eO+v5yCm8chqcBf2umpHzOL395McYYevT4KQUFB8nOzqu2/datu8jN3U9ycpdK6xYsSGXYsH5+zde9cxu2ZR0gM7vQ14/LtjIguX25Nr8e0JnXFm6k4OARAPIKvi9d99naLA4eavj3c/fT27Jt1wEys0pypm5mwPk/KZ9z0Om8Nv9bCgpLcu4/XGk7g/oksSR9B4e/P9bgmStywvumNsroDNYYv9+cKhiD1mJgGvDnKtb9FbjTWrsXwFr7JTAD+FPJ8nLgU2ACMPH4/YFyJH8f4dHRpcvh0dF8n7+vhkcEnhMyxsU0Jyv7xEAqK7uQuJjmVbZNjG9B+8RIPk/3VTI6/qQVBQe+57nJg3hrxtXcddv5eILwV6QT+vHHwgl9GR8Twe6yr8mcg8S1reY1Ged7TX725a4Tj49tzrvTf0XqGyOZNnN1g1RZAfbsySU+/sTf8/HxbdizJ7fa9vPnL2HIkD6YCv/J7NyZzY4de+jVq5tf88W1jmB3mWPPyisirk1EuTYdE6JISoxk9oOXMO/hS+nXI8GvGeoirk0Eu/ceLF3O2ltEXJvyz3fHdlEktWvJ7MeGMO+JofQrma5U1rALTmX+Ev9Wq+vKCe+b2iijBFqwrh7wPDDSGNOywv1dgC8q3Jdecv9x9wBjgf9aazOq24Ex5mZjTLoxJn3DO+/6I3M1+2mwTftNoDNW/A8UwGKrbDt04Gl88Ml3eL2+9WFhhuQeCTzyzKdcdcM8TmkXxZVDf9qgeevKDc+1WwS+L6t6TVZtWP9OLFy8pfQ1CZCVfZDLrp/HgGtf54pBp9MmulmDpLRVhKrq/XTce++lMXToBZXuX7AglUsv7U1YWJg/41XRi755uGWFhRmSEiIZOeEjxj69lIm39CIyorFfc9Smyt9BlXJ6SEqMYuRf32fsI0uYOKY3kc3DS9fHRDfjjKRo0r4I/NSA6rjhd5AyBoGnAW4OFZRo1toC4D/A6Do0N5T//6UfsB/oWss+pllrk621yWdcftkPzlpWeHQrjuSfmJd5JD+f8Fat/LJtf3FCxqzsQuJjW5Qux8e2IDun6srU0AGdmf/hib89srIPsm7jXjJ3FXDsmOXjJVvockalmSQNzgn9+GPhhL7MyjlIQtnXZExzsstU4soaWmZqQEXZuUVkbM3n3O7xfss2c+YChg8fzfDho4mNbU1W1t4TubNyiY1tXeXj1q/fwrFjx+jatXOldb7BrH+nBoCvsppQprIa3zqC7LxD5dvkFvHxyh0UH7PsyD7I5l0FJCVE+T1LjTn3HiShTCU9vm0E2XlFldp8/Pl2X849hWzesZ+kxBM5h/TryIefbqP4WHV/3jQsJ7xvaqOMEmjBHE8/BdwIlP3MZh1wToV2Z5fcjzGmOfAocDEQY4wZEoCcpSKTkji0J5vDOXvxFheTsyKd1t27BzJCrZyQcc232SSd0pL2CZE0buRh6MDOpKRtqdSu409aERXVhK/WZJV7bMvIJkS38p0U0Su5HRlb8is9tqE5oR9/LJzQl2vWZ5PUvsxrsn9nUpZuq9Su4yktiYpswldr95TeFx/TnCbhvoplVItwzv5ZHJu376/02B9q5MihvP32M7z99jMMGNCLt95ahLWWVavWExkZUe2gdf78JVUOTDdv3kFBQSE9e/r/E4qvM3LpkBBJ+9jmvn7snURK+o5ybT5emUmvLnEAREc2oWNCFJl7Anv1j6837qVDYhTt41r4cvY7lZTPM8vn/Gw7vbr5pi5ERzWhY7uWZGadyHnZBR2DNjUAnPG+qY0yOoQx/r85VNAueWWtzTPGzME3cH255O5HgUeMMYOstbnGmB7A9cDPS9bfD8yx1q43xtwKzDbGLLLWVp5B3wBMWBinXnct3zz1NHi9xPbuTUQ7Z52B6ISMx45Z/vFYGi89fRlhHsO8+evJ2JLP6JvOZe36HBalbQVg2CWn8d5H5StaXq9l8rOfMuO54Rjgmw05zHl7XUDzgzP6sS5mPHs7fc8/k7bRkWQsf44Hn5jHjNmLgx2rHCf05bFjlgeeXMrLjw/xvSYXbCBjaz5jbkxmzfocFi3zDWCHDejMgpTyr8lOHVpx923nY63vd/lLs75m4+bqT46qjwsuSGbJknQGDryZZs2aMHHimNJ1w4ePLnfVgPffX8q0aX+vtI0FC1IZMqRvjdMKfqhjXssDL63klXv7E+YxzP3kOzbt2M+YX3dj7Xd5pKTvIHXVbvp0T2Thk8M45rVMfvVL9pWc7DTrH5fQqV0UEU0bsfTFK7jnhc9JW727YXK+8DmvPOS7VNncDzexafs+xvymJ2s37SVleSapX+ykz9mJLHzxCl/Ol1ay74DvpLF2sS2Ib9uc5WX+oA40J7xvaqOMDhFCVw8wFef5NPgOjSm01rYo+TkO2AI8WuaSV3/EN2fVAgeAv1hrU40xZwFvAd2ttYdK2j4D5FprH6hpnzekLQ7O5zs/IkvvDPzA8YfoM+WsYEeo1ayRU4MdoVYjZo4KdoRapY3fEOwIdbIp7aJgR6hV56tXBDtC7Q4eDXaCWvW7p/qrpMiPz8t9L3TEaLHDIyl+H+Ns+2t/RxxbRQGvtB4fsJb8vAeIqLD+BeCFKh63Dji9wn11mRMrIiIi8uMUQpVWB58jJiIiIiLio69xFREREXGr0Cm0atAqIiIi4lZW0wNERERERJxDlVYRERERt3LwdVX9TZVWEREREXE8VVpFRERE3CqE5rRq0CoiIiLiVqEzZtX0ABERERFxPlVaRURERFzKE0LlxxA6VBERERFxK2OtDXaGBndD2uIf/0GK+NGskVODHaFWI2aOCnYEEWkgz/TyBjtCrVo0vtgRs0k7Pr/E72OcLX+6wBHHVpEqrSIiIiLieBq0ioiIiLiUMf6/1W2/ZpAxZoMxJsMYc3cV628xxqwxxqwyxiw1xpxV32PVoFVERETEpYwxfr/VYZ9hwPPAYOAsYEQVg9L/Wmt/Zq3tATwKPFHfY9WgVUREREROxnlAhrV2s7X2CPA6MLxsA2ttQZnF5kC9597qklciIiIiLlXXj/NPbpvmZuDmMndNs9ZOK7PcDsgss7wD+HkV2/kTMA4IBy6uby4NWkVERESkVMkAdVoNTaoaKleqpFprnweeN8ZcB9wH/L4+uTRoFREREXGphqi01sEO4JQyy+2BXTW0fx14ob471ZxWEREREZcyHv/f6mAlcJoxpqMxJhy4FninXC5jTiuzOBTYVN9jVaVVREREROrMWltsjLkN+AAIA1621n5jjPkHkG6tfQe4zRgzADgK5FPPqQGgQauIiIiIawVpegDW2veA9yrcd3+Zn8f4e5+aHiAiIiIijqdKq4iIiIhLeYJUaQ0GDVpPUv7atWyeNQe8XuL69qH9kEHBjlSJMvqHGzKC83O+OGUUg/v3JCe3gOSBdwU7TrWc3o+gjP7khpzKWDfWWqZMmsOytG9o2jScCQ//jjPP+kmldkePFvPIw7P5YuVGjMfwp9GX03/g2XyZvonHHplLxsadTJxyIwMuOTvgx1AfwZoeEAyOmh5gjDlW8h21a40x7xpjWpXcn2SMscaYB8u0bWuMOWqMeS5Q+azXy+aZs+gy9nZ6PjiBnBUrKdpV0xUeAk8Z/cMNGcEdOV+du4Thv5sc7Bg1ckM/KqP/uCGnMtbdsrRvyNyezVvvPcB9E65j0oOzqmz30tT3ad26BW8ueIB5b9/P2cmnAxCf0JoHHvodg4acG8jY8gM4atAKHLLW9rDWdgXygD+VWbcZGFZm+Wrgm0CGO7BlC01jY2kaE4OnUSNizksmb9XqQEaolTL6hxsygjtyLluxnrx9hcGOUSM39KMy+o8bcipj3S35ZDVDL++FMYafdT+VwgNF5OTsr9TunTc/4w//56sEezweoqNbAJDYrg2nndEe49LP2Y3x/82pnDZoLeszfF8Tdtwh4FtjTHLJ8q+BOYEMdCR/H+HR0aXL4dHRfJ+/L5ARaqWM/uGGjOCenE7nhn5URv9xQ05lrLvsPfuIiz+RIzYumpw95XMcKCgC4IXn3uW6qydy17h/kbu3IKA5pf4cOWg1xoQB/alwoVp836hwrTGmPXCMGr59wRhzszEm3RiTvuGddxswa4Nt2m+U0T/ckBHck9Pp3NCPyug/bsipjFWzlb48tHKO4mNe9uzJp3vPU/nv3PF0696Rpx57IzABG5gxxu83p3LaiVjNjDGrgCTgC+CjCusXAg8Ce4DZNW2o7Pfm3pC2uIqX9MkLj27Fkfz80uUj+fmEt2rlj037jTL6hxsygntyOp0b+lEZ/ccNOZWxZnNmLebNecsAOKtrB/ZknciRvSeftrHlc7Rq1ZymzcK5qH8PAAZccjZv/+/TgGQV/3FapfWQtbYH0AEIp/ycVqy1R/ANZv8CBPxPpMikJA7tyeZwzl68xcXkrEindffugY5RI2X0DzdkBPfkdDo39KMy+o8bcipjza4ZcSGz3riXWW/cy4UXd2fBO59jrWXN6s20aNGMmJiW5dobY+h3wc9IX7kRgBXLN9CxU0JAsja0IH2Na1A4rdIKgLV2vzFmNPC2MeaFCqsfB5ZYa3MDXcI2YWGcet21fPPU0+D1Etu7NxHtEgOaoTbK6B9uyAjuyDnj2dvpe/6ZtI2OJGP5czz4xDxmzF4c7FjluKEfldF/3JBTGeuuT7+uLEtby/DB99O0WTgTHvxd6boRVz3MrDfuBWD0uCv42z3TeXzyPKJbt+DvD/nafbNmK3eMnUpBQRFpi9cw9fn5zH37/ir35UQO/jTf74ytajJIkBhjCq21Lcosv4vvZKs0YH7JVQXKtr8eSLbW3lbTdv01PUAkVMwaOTXYEWo1YuaoYEcQkQbyTC9vsCPUqkXjix0xXOz2aprfxzhf/7avI46tIkdVWssOWEuWLyuz2LVCc6y104HpDZtKRERExJlCqdLq4JkLIiIiIiI+jqq0ioiIiEjdhVKlVYNWEREREZdy6Rd5/SCaHiAiIiIijqdKq4iIiIhLhdL0AFVaRURERMTxVGkVERERcalQqrRq0CoiIiLiUiaEzsTS9AARERERcTxVWkVERERcStMDRCpInbIt2BHqpN+dHYIdoVZp4zcEO0KtRswcFewItZo1cmqwI9TJwW1/C3aEWv3f0uxgRxApp8fI3cGOUKuMOcFOEHo0aBURERFxqVCqtGpOq4iIiIg4niqtIiIiIi4VSpVWDVpFREREXCqErnil6QEiIiIi4nyqtIqIiIi4VChND1ClVUREREQcT5VWEREREZcyIVR+1KBVRERExKU0PUBERERExEFUaRURERFxKRNCpVZVWkVERETE8VRpPUn5a9eyedYc8HqJ69uH9kMGBTtSJU7I2O/sRO77v/MICzPM+XATU99YW6nNkN4dGD2iBxb4dkse4x5PK13XolljFv7zl3z0+XYemLo8gMlPcEI/AvQ97xTuG/MLwjyGOfPXM23mqnLrx99+Pr16JgLQtGkj2rRqxjlDppMY14LnH76EMI+hUSMPr76xlllvfxuMQ3BMX1bnxSmjGNy/Jzm5BSQPvCtoOay1THz4JVJTv6Bp0yZMnHQ7Xbp0KtfmYOEhfvOb8aXLWVm5XHb5BYwffyNv/m8RU6bMIC6uNQDXjRzC1VcPDOgxOP25Ps4NOZXx5PXrnsB9f0j2/b5MyWDq2+sqtRly/k8YfXU3rLV8u20f455ZFoSk/hNChdbgDlqNMVcA/wPOtNauL7nvNOBJ4ExgH1AA/N1am2qMuR6YAuwss5nrrLWVX5UNwHq9bJ45iy7jxhIeHc3qhybRukc3IhITA7H7OnFCRo/HMGFUL35//4dk5Rbxv8eHkrIik4zM/aVtOiREcsvVP+Oav75PwcEjtG7ZtNw2xo7swYq1WQHLXJET+hFK+nJcb67/8wKycg7yxr+uZNGyrWRs3VfaZuKzn5X+/NurunDWaW0ByMkt4td/fIsjR71ENGvEghnXkLJ0G9m5RQE9Bqf0ZU1enbuEF2d8wL+fvDWoOVJTv2Tbtl0s/OCfrF69kX88MJXZcx4t16Z5i2a8+daTpctXXfkXBg7sVbo8eHBv/nb/zQHLXJYbnmtwR05lPHkeY5hw47n8/qFFvv97Jg0iJX0HGTsLStt0iI/kll924Zq/fej7vyeqSVCy+lMoDVqDPT1gBLAUuBbAGNMUWABMs9Z2staeA9wOnFrmMbOttT3K3AIyYAU4sGULTWNjaRoTg6dRI2LOSyZv1epA7b5OnJCx+2lt2ba7gMw9hRwt9rIgbQsDfn5KuTa/vvR0XluwgYKDRwDI23+4dF2XTq1p26oZS7/aFdDcZTmhHwG6nRnLtp0FZO4+4OvLlAz690mqtv2w/p2Z/3EGAEeLvRw56gUgvHEYniC9253SlzVZtmI9efsKgx2DRSkrGD78Iowx9OhxBgUFB8nOzqu2/datu8jL209y8lkBTFk9NzzX4I6cynjyunduw7asA2RmF3L0mJcFn25jwLkV/u/p35nXPth44v+egu+DEVV+oKANWo0xLYDewI2UDFqBkcBn1tp3jrez1q611k4PfMLKjuTvIzw6unQ5PDqa7/P31fCIwHNCxrg2Eezee7B0OWtvEXFtmpdr0zExiqR2Ucx+ZDDzpgyh39m+v8yNgfE3nMsj09MDmrkiJ/QjQHxMBLuzTwymsnIOEte2eZVtE+Na0D4xks++PDHYj49tzrvTf0XqGyOZNnN1wKus4Jy+dIM9e3KJT2hTuhwf34bsPdUPWhcsSGPw4D7lTsT48KPPGX75WMaMfpTdu/c2aN6K3PJcuyGnMp68uNbN2F3md1xWbhFxrZuVa9MxMZKkhChm/+MS5j10Kf26JwQ6pt8Z4/+bUwWz0vpLYKG1diOQZ4w5G+gCfFnL435tjFlV5tasqkbGmJuNMenGmPQN77zr5+hl99Ngm/abQGesan/W2nLLYWGGpIQoRo5fyNjHUpl42y+IbN6Y3wz5KYu/2MHuvYEfXNUmOM915Z3aKloBDOvfiYWLt+D1nmiRlX2Qy66fx4BrX+eKQafTJrrKt0vAueF9EwxVPbc1nRn8/ntLGTq0b+nyhRclk5IylbffeYrzf9GNe+5+ugFSnhy3PNduyKmMte27it+XFd5UYR4PSQmRjHzgI8Y+vZSJt/QiMqJxgBJKfQVzTusI4KmSn18vWS7HGPMmcBqw0Vp7Zcnds621t9W2cWvtNGAawA1pi6v7f/6khEe34kh+funykfx8wlu18sem/cYJGbP2FpFQphoY3zaC7LyiSm1Wbcih+Jhlx55CNu8sICkhih5nxHBul1hGDv4pEc0aEd7IQ9Gho0z5T21/y/iXE/oRfJXVhNgWpcvxMc3JLlPFLmto/85MeHJpleuyc4vI2JrPud3jWbh4S4NkrY5T+tKpZs58j3lzPwKg6886k7U7t3RdVlYuMbHRVT5u/fotFBcfo0vXEydqRUdHlf589dUDefyxVxsoddXc8ly7Iacynrys3CIS2kSULse3iSA7/1D5NnlFrNq41/d/T85BNu8qICkhkjXfVf+JhtN5XPDHjL8EpdJqjGkDXAz82xizFbgT+DXwDXD28XbW2iuA64HWgU9ZWWRSEof2ZHM4Zy/e4mJyVqTTunv3YMcqxwkZv960lw6JUbSPa0HjRh6G9u1IyvId5dp8vHw7vbrFAxAd2YSOiVFk7inkL0+k0e/GN7jwpjeY/HI6b36yOeADVnBGPwKsWZ9NUvuWtE+I9PVl/86kLN1WqV3HU1oSFdmEr9buKb0vPqY5TcLDAIhqEc7ZP4tj8/b9lR7b0JzSl041cuQQ3nzrSd5860n69/85b7/9CdZaVq3aQGRkBLGxVf/6W7AgrVyVFSg3/3XRopWc2ql9g2avyC3PtRtyKuPJ+/q7XDokRNI+pjmNwzwM/UUHUtIr/N+zIpNeXeOAkv97EiLJ3BP8+exSN8GqtP4K+I+1dtTxO4wxS4CNwD3GmMvLzGuNqGoDwWDCwjj1umv55qmnwesltndvIto550xOcEbGY17LA1OX88qEAYR5PMz9eBObMvcx5roerM3IJWVFJqlf7qJPj0QWPjecY17L5Onp7DvgnAnxTuhHgGPHLA88uZSXHx9CmMcwb8EGMrbmM+bGZNasz2HRMt8AdtiAzixIySj32E4dWnH3bedjre8ju5dmfc3GzYGvJjilL2sy49nb6Xv+mbSNjiRj+XM8+MQ8ZsxeHPAcF1xwDqmpX3DpJX/0XfJq4u2l66745Z/LXTVg4fufMnXafeUe/9qrC1j0yUoahYXRsmULJk26nUByw3MN7sipjCfvmNfywMvpvHLvxYR5DHM/+Y5NO/Yz5ppurP0ul5QvdpK6ejd9uiew8Ilhvv97XvuKfYVHgpbZH0Kp0moqzjUMyE6NWQxMttYuLHPfaHyXuXoaeAL4KbAHOAA8aq39uJpLXt1qrf20pv35a3pAKEudUrm650T97uwQ7Ai1Shu/IdgRatV34hnBjlCrWSOnBjtCnRzc9rdgR6jV/y3NDnYEkXJSn91Ze6Mgy5gz0hHDxUs/WOr3Mc4Hl/ZxxLFVFJRKq7X2wirue6bM4pBqHjcdmN4goURERETEsfSNWCIiIiIuFUrTA4L95QIiIiIiIrVSpVVERETEpUKp+qhBq4iIiIhLeUzonGseSgN0EREREXEpVVpFREREXEonYomIiIiIOIgqrSIiIiIuFUrVRw1aRURERFxK0wNERERERBxElVYRERERlzIhdMkrDVodIHXSlmBHqNW383sGO0KdjFq2L9gRarUp7aJgR6jVDWm7gh2hVge3/S3YEeqkeYcHgx2hViNmjgp2BJFy+t3eLtgRxIE0aBURERFxKc1pFRERERFxEFVaRURERFwqlKqPGrSKiIiIuJQnhE7ECqUBuoiIiIi4lCqtIiIiIi6lE7FERERERBxElVYRERERlwql6mMoHauIiIjIj4rH+P9WF8aYQcaYDcaYDGPM3VWsb2KMmV2yfrkxJqnex1rfDYiIiIhI6DDGhAHPA4OBs4ARxpizKjS7Eci31nYGngQeqe9+NWgVERERcSmPsX6/1cF5QIa1drO19gjwOjC8QpvhwK13tnMAACAASURBVIySn+cB/Y0x9TptTINWERERESlljLnZGJNe5nZzhSbtgMwyyztK7quyjbW2GNgPtKlPLp2IdZLy165l86w54PUS17cP7YcMCniGfue0475RPyfMY5jzwUamzl1Tqc2QvkmMHtkTay3fbslj3KOpACTENGfSmN7Et20OwI33f8TO7EK/Z7TWMmnidNJSv6Jp0yY8PPGPnNXl1HJtDh48xO9+8/fS5T1ZeQy7rA93j7+e9JXreGTSDDZu3M6Ux8dwyaW9/J6xNk54rsHXlw8/PI0lS76gadMmTJ48hi5dOpdrU1hYxMiRJ6YUZWXt5fLLL+Lee29i4sR/sXy57zVy+PD35ObuJz399YAegxP60lrLxIdfIjXV148TJ91Oly6dyrU5WHiI3/xmfOlyVlYul11+AePH38ib/1vElCkziItrDcB1I4dw9dUDA3oML04ZxeD+PcnJLSB54F0B3XddOeG5rgs35FRG/3BDxvpoiEteWWunAdNqaFLVXiuWaOvS5qQ4ZtBqjDkGrAEaA8X4SspPWWu9xpgLgTustcOMMXHAS8ApJW23WmuHBCKj9XrZPHMWXcaNJTw6mtUP/X97dx4fVXnvcfzzSyBCIEDYQgBLENdqWTT1oixFEWVzwVqVcm+1vV7x2iKWWtfqRbHgVvdWsahFpYjgAoqCCrJoWS0goAjILiRASNgVSZ77x5yEyT6Bycw55PvmNS/mnPPMOd95zpmZZ555zskoGndsT3LLlrHYPAAJCcbwmztz3T3Tydp5gLeevJQZ8zexdvPuojJtWjbgpqvbc/VtU9mz7xCNG9YpWvbYH7rxtwlf8NmSrSTXqUWBq56/pDF3zlI2bczi/WlP8cWyNYx44EXGT/hzsTL16tXlzbcfKZq++ud3clGvcwFIb9mUB0fdzD9eerda8lXGD/u60Jw5n7Nhw1Y+/HA0y5Z9zfDhzzFx4l+KlalfP5nJk58umr7yylu5+OLzALj77v8pmv/qq+/y5ZfrYhPc45e6nDPn32zcuJVp0//GsmWreeD+0Ux445FiZerVr8vb7zxRNP3zK/9Ar15HvjD16dOFe+8r2eEQO69OnM3zY6cz5omb45ahIn7Z15UJQk5ljI4gZDxWcfrJfAuhdlih1sDWcspsMbNaQENg17Fs1E/DAw465zo6584EegF9gf8ro9wDwEfOuQ7OuR8Dpc5Yqy5716+nTvPm1GnWjIRatWh2bia7li6L1eYB6HBqUzZu3cvmrH38cLiAqXPWcdF5PypW5prep/Lae1+xZ98hAHbt/g6Ak09sSGJiAp8tCR1XB747zHff51dLzk9mLuKyy7tjZnToeCp79+xnx/bccstv3LCNnF17OCfzDABatWrOaae1ISEhPoeoH/Z1oRkz5nPFFRdiZnTseDp79uxn+/byX/cbNmwlJ2c3mZlnllo2deoc+vfvXp1xS/FLXc6csZDLL7/Aq8fTIqrHXbt2k5lZ8tyC+Pls4Sp25UX/l5Fo8cu+rkwQcipjdAQhY0AtAk4xs7ZmlgRcC0wpUWYKcJ13/ypgpnPH1lPmp0ZrEefcduBG4HdlDNpNJ9R6Lyz7RaxyHcrNIyk1tWg6KTWV73PzYrV5ANKaJLNt5/6i6aydB0hrUq9YmbatGpDRqiETHuvLpMf70f2c0DCTjNYN2bP/EH+950KmPHMZd/wmk4Rq+lMa2dm5tGhxZOhKWosmZFfQQHh/6mf07nMexzhGO2r8sK8LZWfn0KJF06LpFi2akJ2dU275996bTd++XUvV5bffbmfLlmw6d25fbVnL4pe6zM7OoUX6kWOyRYsmbM8u/5icOnUuffoUr8cPP5rP5ZfdytBbHmHbtp3VmjeI/LKvKxOEnMoYHUHIeKzicSKWN0b1d8B04CvgDefcSjN7wMwu84q9CDQxs7XAMKLQyejLRiuAc24doXzNSyz6K/CimX1iZveYWZl9/OGDiL+eUn0/Mce6jVVWo67kF5fExAQyWjZg0B0fcOvDsxk5tAsp9ZKolWD89Mw0HnpxIQOGvsuJ6Sn8/KKTS60vGsr6MlVRg/SDD/5F335dqiVLtMSrPV3W99KK6vL99+fSr9/PSs2fOnUOl1zShcTExGjGOyrxqMuy3oYrPCbf/5R+/boVTfe4IJMZM0YzecqTnHd+e+6686lqSHn88cn30EoFIacyRkcQMgaBc+5959ypzrl2zrk/e/Puc85N8e5/55z7hXPuZOfcuV677pj4ZkxrOUodWs656WZ2EtCb0PXBlpjZWc65HSXKFQ0i/s3cWVEZuJmU2ohDuUd+4j6Um0tSo0bRWHXEsnbuJ73pkZ7VFk2T2b7rQKkyS1ft4HC+Y0v2PtZt2U1GywZk7TzAl9/ksDkr9PPix/M20fH0Zkz8cE1Uso0fN51Jk2YAcNZZ7cjKOtIbmJ2VQ/NmqWU+btWqDeQfLuDMEidqxVO89/W4cVN5443pAPzkJ6eQlXWkVy8rK4fmzRuX+bhVq9aTn5/PWWeV/jLy/vtzue++m6oncAXiWZfjxr3PpIkfAXDWT04ma9uRYzIrK4dmzcs7Jtdz+HA+Z5515ESt1NQGRfd/8Yte/OWxV6spdXDF+3UTqSDkVMboCELGY1VNP5j6km97Wr2GaT6wveQy59wu59w/nXP/RWhcRUwG6aVkZHAwezvf7dhJweHD7Fi4mMYdOsRi00W+WL2TNi0b0DqtPrVrJdCv+0nMmL+5WJmP522ic/t0AFIbnEDbVg3ZnLWXL9bspEH9E2jc4AQAOndIZ+2m6P1MMnDQJbz59iO8+fYjXNjzp0yZPAfnHMuWrqZ+SnK5DYQPpv6LPv3Oj1qOaIj3vh40qB+TJz/N5MlPc9FFnXnnnZk451i6dBUpKcnlNlrfe282/fqVfjmsW7eFPXv20anT6dUdvZR41uWgQX15+50nePudJ+jZ8z+YPPkTrx6/rrAep06dW6yXFSg2/nXmzEWc1K51tWYPoni/biIVhJzKGB1ByCiR82VPq5k1A54HnnXOufCf8MzsQmC+c+6AmaUA7YBNMcmVmMhJv7yWlU8+BQUFNO/SheRWsT0DMb/Acf9z83n5wYtJTDAmfriGNZvyGPqfnVixZiczFmxmzuff0vXslkx7fgD5BY6HXlxE3t7vAXjoxUW8Mqo3ZsaKNTuZMG11teTs/rNOzJ2zhD6XDKVunSRGjPzfomU/H3B7sasGTJ82j7+NLj7UZfnytdw65C/s2bOfWZ98zl+fmcjk94qfMV+d/LCvC/3sZ5nMnr2YXr1upG7dExg5cmjRsssvv6XYVQM++OBTXnih9PmLU6fOoW/fbnEZM+yXuvzZz85hzpzPueTi/w1d8mrkkKJlA674fbGrBkz74F+MfuFPxR7/2qtTmfnJImolJtKwYX1GjRpCrI19ZgjdzjuDpqkprF3wLCMen8TYCbNinqM8ftnXlQlCTmWMjiBkPFY1qafVjvFErqgp45JXrwKPl3HJqz8Cv/bKJAAvO+cqbM1Ea3hAdZkzan28I1Tqq/c6xTtCRAZ/5v8B9i918/8b5m/mlrxyif+M6VpyuLs/1WszIt4RKjVw3OB4RxAJnJe69fBFc/GWeZ9EvY3z9HkX+OK5leSbnlbnXLlnhzjnZgGzvPuPAo/GJpWIiIiI+IFvGq0iIiIiUjWRXKLqeOHbE7FERERERAqpp1VEREQkoGrSiVhqtIqIiIgEVE36ybwmPVcRERERCSj1tIqIiIgEVE0aHqCeVhERERHxPfW0ioiIiASU1aBLXqnRKiIiIhJQGh4gIiIiIuIjNaKndc4Tm+MdoULd72ob7wiVGvxZXrwjHDdO/sXCeEeoVPdbWsc7QqVu+HR7vCNEZOC4wfGOUKnxg0bHO0Kl6tVNi3eESl025op4R5AaqCb1Ptak5yoiIiIiAVUjelpFREREjkcJNehELPW0ioiIiIjvqadVREREJKBq0tUD1GgVERERCaia1GjV8AARERER8T31tIqIiIgEVGK8A8SQelpFRERExPfU0yoiIiISUDXpkldqtIqIiIgElE7EEhERERHxEfW0ioiIiARUTeppVaO1hO6dWvKn32SSmGC88fFaRr+9slSZvue34ZZr2uMcfLUhl2FPfkrLZvX42+0/IyHBqJ2YwCvvr2L8h2vi8Awgd8UK1o1/AwoKSOvWldZ9e8clR0WUMXLdO6bzp1//NHRMzljL6HfKOCbP+xG3XO0dkxtzGfbUZwC8dM+FdDylKYtXbefGh2bFOPkRfqnLiihjdDz/6GD69OzEjpw9ZPa6PW45Lux2OiPvuZKEBOO1ifN5+u8zii1v3TKVp0cOpEnj+uTlHeCmP77KtuzdAEwYM5jMDhks+Hwdv7zp7/GIDwRjfyujxFJcGq1mlg8s97a/Hvgv51xe2PLfA6OANOfcbm9eD2AysA5IBrKBR5xz70UrV0KCMfx/zuW6+z8mK+cAbz3ShxmLtrB2y+6iMm3SU7jpyrO4+u7p7Nl/iMYN6wCwI/cgV981jUOHC0iuU4v3n7yUGYu2sD33YLTiRcQVFLBu3HjOHHYrSampLHtwFI07tie5ZcuY5qiIMkYuIcEY/t/nct2IGWTtOsBbo/owY3GJY7JFCjcNOIur//Rh6JhscELRsr9P/pK6JyRyba9TYpo7nF/qsiLKGD2vTpzN82OnM+aJm+OWISHBePi+q7jq18+xNTuPjyYNY9rMFaz+JruozP13XM6EdxYx4Z1FdOt8Cvf+oT833z4OgGfHzCS5bhLXXXN+vJ5CIPa3MvpDYg3qaY3XmNaDzrmOzrmzgF3Ab0ssHwgsAgaUmD/XOdfJOXcacAvwrJn1jFaoDic3YeO2vWzO3scPhwuY+ulGLjr3xGJlrrnoFF6b9jV79h8CYNfu7wD44XABhw4XAJBUK4EEi89RtHf9euo0b06dZs1IqFWLZudmsmvpsrhkKY8yRq7DyU3YmLWXzdu9Y/KzDVyU2bpYmWsuOpnXpq0+ckzu+b5o2bwVWew/eDimmUvyS11WRBmj57OFq9iVty+uGc5u34b1G3eycUsOP/yQz9tTl9Cn50+KlTmtXRpz5q0GYO78NcWWz52/hn37vyeegrC/ldEfEiz6N7/yw4lY84BWhRNm1g6oD/yJUOO1TM65pcADwO+iFSStSTLbcvYXTWfl7Cetcd1iZdq2bEBGegMmjLyESQ/1pnunI9/W0psk897j/Zn795/zwtsrYt7LCnAoN4+k1NSi6aTUVL7PzavgEbGnjJFLa5zMtpwDRdNZuw6Q1iS5WJm26Q3IaJnChBEXM+nPl9C9Y3qsY1bIL3VZEWU8vqSnNWRrVm7R9NbsPNLTGhYrs3LVVi69pAMA/Xq1J6V+HVIbFX9txVMQ9rcySqzFtdFqZolAT2BK2OyBwHhgLnCamTWvYBX/Bk4vZ903mtliM1u8Z/0nkeUpY17Jq58lJhoZLVMYdO+H3Pr4p4y8uTMpybUB2JZzgP7D3qPnze8w4IJ2NPGGDsRbnDp9q0QZy9lmGfOcK35UJiYaGekpDBr+Ebc+9SkjbzpyTPqV9nd0BCFjPJRVLyVfN//3yGTO/2k7Zr59G+ef246tWXkc9n4t86sg7G9ljL0Ec1G/+VW8Gq11zWwpkAM0Bj4KW3Yt8LpzrgB4C/hFBesp99Bzzr3gnMt0zmU2aHtBRKGycg6Q3qRe0XSLJvXYvutgqTIfL9zM4XzHlu37WPftHjJaNihWZnvuQdZszuOnP66ovV09klIbcSj3SA/Dodxckho1inmOiihj5LJ2HSA9rGe1RePkso/JRVu8Y3I/67buISO9QclVxY1f6rIiynh82Zq1m5YtjvSutUxrRNb2PcXKZG3fw/VDXubCAY8x8ompAOzd911Mc1YkCPtbGSXW4jqmFWgDJOGNaTWz9sApwEdmtoFQA7bcIQJAJ+CraIX6Ym0ObdJTaN28PrVrJdCvaxtmLNpcrMzHCzfT+awWAKSmnEDblg3YnLWXFk2SOSEp9BeAG9RL4pzTm7Pu2z2ltlHdUjIyOJi9ne927KTg8GF2LFxM4w4dYp6jIsoYuSPHZL3QMdklgxmLtxQr8/GizXQ+Mw3wjsn0BmzO3hvzrOXxS11WRBmPL0uWb+KkjKb8qHVjatdOZEC/TkybuaJYmcap9TCvy23ojRfxzzcXxCNquYKwv5XRH2rSmNa4XvLKObfbzG4BJpvZc4QaqMOdc6MKy5jZejNrU/KxXgP3XuCGaOXJL3DcP2YhL9/Xk8QEY+KMtazZvJuh13ZgxTc5zFi0hTlLttK1QzrTnrqU/ALHQ2P/Td6+Q3Rp14S7rjsHh8Mwxkz+ktWbYj9uxhITOemX17LyyaegoIDmXbqQ3MpfZ0kqY+TyCxz3v7iIl+/xjslPvmHNlt0MvaY9K77ZxYzFW5izdBtdO7Rk2hP9Q8fkq6FjEmD8AxfTrlUDkuvU4tPnB3DXc/OZu2xbTJ+DX+qyIsoYPWOfGUK3886gaWoKaxc8y4jHJzF2wqyYZsjPL+DOB95k4pibSEhM4J9vLuDrtVnceUsflq7YxLSZK+ly7sncO6w/zjnmLf6G2++fVPT4d8cN4ZST0qiXnMQXs4cz9J7X+eTTVTF9DkHY38oosWYlx/nEZKNm+5xz9cOm3wXeIHRiVR/n3KqwZY8TurzVAopf8mo7oUtevVvZ9k6+8lX/DtAAuv/+xMoLyXFjztNbKi8UZ91vaV15ITlujB80Ot4RKlWvblq8I1TqsjFXxDuCxNBL3Xr4ok/yb19+GPU2zs0/vtgXz62kuPS0hjdYvelLvbuvllF2WNhkw5LLRURERGoqP/+cH21+uOSViIiIiEiF9GdcRURERALKz5eoijb1tIqIiIiI76mnVURERCSgEmvQmFY1WkVEREQCSidiiYiIiIj4iHpaRURERAJKPa0iIiIiIj6inlYRERGRgKpJPa1qtIqIiIgEVGINuk5rjWi0dv/9ifGOcFyYe+/qeEeoVLcRp8Y7QuX2/xDvBJWaM2o93e9qG+8YEiP16qbFO0Kl9h/MjneESo0fNJqB4wbHO4bIcatGNFrl2AWhwSrRowarSNWpwSrxUJNOTqpJz1VEREREAko9rSIiIiIBpROxRERERMT3alKjVcMDRERERMT31NMqIiIiElA16ZJX6mkVEREREd9TT6uIiIhIQGlMq4iIiIiIj6inVURERCSgalJPqxqtIiIiIgFVkxqtGh4gIiIiIr6nntYqyl2xgnXj34CCAtK6daV1397xjlSKHzJ2O7c1f/rd+SQmGm9MXcUL/1xWbPndvz2Pzp3SAahzQi2apNblnP5jOePkJtz/+67UT65NfoHjudeW8P4n62KeH/xRjwDdz2nFnwb/B4kJxhvTVzN64vJSZfp2y+CWQZ1wzvHV+l0Me2QOndu34O7/ObeoTLsTGzL04dl8PG9TLOMD/qnLiihjZC7sdjoj77mShATjtYnzefrvM4otb90yladHDqRJ4/rk5R3gpj++yrbs3QBMGDOYzA4ZLPh8Hb+86e8xz17o+UcH06dnJ3bk7CGz1+1xy1EZP+zvyihj/CXWoJ7WuDVazawJUPhu1wLIB3Z40+cC/YC3gDOcc6u8x2QC/wDOds4dMrN2wEdAR+fcnurO7AoKWDduPGcOu5Wk1FSWPTiKxh3bk9yyZXVvOmJ+yJiQYAwf2pXrb5tK1o79vPn8AGZ+tpG1G/OKyoz867yi+/814Ex+fEpTAA5+d5g/jvyEjd/uoXmTZN5+4UrmLtrC3n2HYpYf/FGP4NXlzZ257p7pZO08wFtPXsqM+ZtYu3l3UZk2LRtw09Xtufq2qezZd4jGDesAMP+LLC4bMgWAhvWTmPHiVXz6729jmh/8U5cVUcbIJCQYD993FVf9+jm2Zufx0aRhTJu5gtXfZBeVuf+Oy5nwziImvLOIbp1P4d4/9Ofm28cB8OyYmSTXTeK6a86PWeayvDpxNs+Pnc6YJ26Oa46K+GF/V0YZJdbiNjzAOZfjnOvonOsIPA88UTjtnDsEDAQ+Ba4Ne8xiYA5wmzfrr8A9sWiwAuxdv546zZtTp1kzEmrVotm5mexauqzyB8aQHzK2P70ZG7/dzeZte/nhcAFTZ35Dzy4Z5Zbv37Md781YC8CGLbvZ+G1od27POUBO7sGiRlgs+aEeATqc2pSNW/eyOWtfqC7nrOOi835UrMw1vU/ltfe+Yo/XsN+1+7tS6+ndNYPZi7fw3ff5Mckdzi91WRFljMzZ7duwfuNONm7J4Ycf8nl76hL69PxJsTKntUtjzrzVAMydv6bY8rnz17Bv//cxzVyWzxauYlfevnjHqJAf9ndllNEfEsxF/eZXvhzTamb1gS7AfxPWaPXcDdxgZrcDtZ1z42OV61BuHkmpqUXTSampfJ+bV8EjYs8PGVs0q8e2HfuLprN27CetWb0yy7ZMq0/r9AbMW7K11LL2pzcjqXYCm7bG5DtJMX6oR4C0Jsls2xlWlzsPkNakeF22bdWAjFYNmfBYXyY93o/u57QqtZ7+PzuJ92bHZ5iFX+qyIsoYmfS0hmzNyi2a3pqdR3paw2JlVq7ayqWXdACgX6/2pNSvQ2qj5JjmPB74YX9XRhn9IaEabsfKzBqb2Udmtsb7P7WMMm3M7HMzW2pmK83spsrW68tGK3AFMM05txrYZWZnFy5wzuUBDwOjgLj/tmMBGEvih4zOlf3Nrf+F7Zg2ex0FBcWXN2tcl0fvvoA7H55NOQ+NuXjUo5Wx0ZJ1mZiYQEbLBgy64wNufXg2I4d2IaVeUtHyZql1OS0jlbmfx35oQHn8cExWRhkj217J4/H/HpnM+T9tx8y3b+P8c9uxNSuPw4cLYpTw+KZjMjqCkPE4cCcwwzl3CqGhoHeWUWYbcL73i/t/AHeaWYXjNvzaaB0IvO7df92bDtcHyAZ+XN4KzOxGM1tsZou/nvJuVEIlpTbiUO6RXoZDubkkNWoUlXVHix8yZu3YT3pYz2qLZvXYvvNAmWX7XdiO92Z8U2xe/eTa/P2hPjzx4iKWfrm9WrOWxw/1CJC1cz/pTcPqsmky23cdKFXm4/mbOJzv2JK9j3VbdpPRskHR8r7d2/LhvzZyOD8+rX+/1GVFlDEyW7N207LFkQ6TlmmNyNpe/JeQrO17uH7Iy1w44DFGPjEVgL37Sg9ZkYr5YX9XRhn9IcGif4uCy4Gx3v2xhDoji3HOHXLOFY4XOoEI2qS+a7R6J2hdCIwxsw3AH4FrzOtyMrP+QEPgEuBRMyvzdyfn3AvOuUznXOZpl10alWwpGRkczN7Odzt2UnD4MDsWLqZxhw5RWXe0+CHj8q93kNG6Ia1bpFC7VgL9LmzHjH9tLFWu7YkNaZByAktWHjmJo3atBP464mLe+XA102avj2XsYvxQjwBfrN5Jm5YNaJ1WP1SX3U9ixvzNxcp8PG8TnduHrsSQ2uAE2rZqyOasvUXLL/1Z27gNDQD/1GVFlDEyS5Zv4qSMpvyodWNq105kQL9OTJu5oliZxqn1in4hGHrjRfzzzQUxzXi88MP+rowyHr/CO/68241VXEWac24bgPd/83K2c6KZfQFsBh52zpUeKxjGj5e8ugp4xTk3uHCGmc0GuprZYuAvwADn3JdmNhm4x7tVO0tM5KRfXsvKJ5+CggKad+lCcit/nYHoh4z5+Y77n/qMlx7tQ2JCApM++Jq1G3IZ+utzWP71TmZ6Ddj+PU9m6szivax9LjiJn3ZIJ7XhCVzZ+1QA7nhoNl+tzYnpc/BDPQLkFzjuf24+Lz94MYkJxsQP17BmUx5D/7MTK9bsZMaCzcz5/Fu6nt2Sac8PIL/A8dCLi8jbG/ry2qp5fVo0rceC5Vkxz17IL3VZEWWMTH5+AXc+8CYTx9xEQmIC/3xzAV+vzeLOW/qwdMUmps1cSZdzT+beYf1xzjFv8Tfcfv+kose/O24Ip5yURr3kJL6YPZyh97zOJ5+uiulzABj7zBC6nXcGTVNTWLvgWUY8PomxE2bFPEdF/LC/K6OM/lAdl7xyzr0AvFBRGTP7mNDVn0qKuE3mnNsMtPeGBbxjZpOcc9nllbfyxhrGkpkNB/Y55x4zs1nAQ865aWHLbwHOAHKBROfcHd78FGAp0Ns5t6a89f9m7qz4P8mAm3vv6nhHiEi3EafGO0Kl5oyKXw9ypLrf1TbeESSGptzwTrwjVGr/wXI/x3xj4LjBlReS48ZL3Xr4YnTs3KypUW/jdGvR75iem5l9DfRwzm0zs3RglnPutEoe8zIw1Tk3qbwyvuhpdc4ND7vfo4zlT5fzuL1Au2oLJiIiIiJVNQW4DnjI+39yyQJm1hrIcc4d9K4u0AV4vKKV+m5Mq4iIiIhExqcnYj0E9DKzNUAvbxozyzSzMV6ZM4AFZrYMmA085pwr/Scfw/iip1VEREREjg/OuRygZxnzFwM3ePc/AtpXZb1qtIqIiIgEVJR6RgNBwwNERERExPfU0yoiIiISUDWp91GNVhEREZGAqkl/lrYmNdBFREREJKDU0yoiIiISUDWoo1U9rSIiIiLif+ppFREREQmomjSmVY1WERERkYCqST+Zq9EqEek24tR4RzhudL+rbbwjiBRz2Zgr4h3huDB+0Oh4R6jUwHGD4x1B5Kip0SoiIiISUGYu3hFipib1KouIiIhIQKmnVURERCSgatB5WGq0ioiIiARVTbp6gIYHiIiIiIjvqadVREREJKBqUEerelpFRERExP/U0yoiIiISUAk1qKtVPa0iIiIi4nvqaRUREREJqBrU0apGq4iIiEhQ6ZJXIiIiIiI+op7WKspdsYJ149+AggLSunWldd/e8Y5UijJGRxAyQjByKmN0KGP0+D3n848Opk/PTuzI2UNmr9vjHadcfq9HCEbGY1GDOlor72k1s3wzW2pmK8xsopm18qaXmlmWmX0bNp1Uovy7QK6l8AAAG6FJREFUZtaoxPp+b2bfmVlDb/qSsMfvM7OvvfuvmFkPM3sv7LFXmNkXZrbKzJab2RXRr5LyuYIC1o0bz5m3DqHTiOHsWLiIA1u3xjJCpZQxOoKQEYKRUxmjQxmjJwg5X504m8t/9VC8Y1QoCPUYhIwSuUiGBxx0znV0zp0FHAKu8aY7As8DTxROO+cOlSi/C/htifUNBBYBAwCcc9PD1rcYGORN/yr8QWbWAXgMuNw5dzpwGfCYmbU/6mdfRXvXr6dO8+bUadaMhFq1aHZuJruWLovV5iOijNERhIwQjJzKGB3KGD1ByPnZwlXsytsX7xgVCkI9BiHjsbJquPlVVce0zgVOrkL5eUCrwgkzawfUB/5EqPFaFbcBI51z6wG8/0cBf6zieo7aodw8klJTi6aTUlP5PjcvVpuPiDJGRxAyQjByKmN0KGP0BCWn3wWhHoOQ8VglWPRvfhVxo9XMagF9gOURlk8EegJTwmYPBMYTavyeZmbNI4/KmcDnJeYt9uaXtf0bzWyxmS3+esq7VdhM1QThrD1ljI4gZIRg5FTG6FDG6AlKTr8LQj0GIaOULZJGa10zW0qogbgJeDHC8jlAY+CjsGXXAq875wqAt4BfVCGrAS6CeQA4515wzmU65zJPu+zSKmymfEmpjTiUm1s0fSg3l6RGjSp4ROwpY3QEISMEI6cyRocyRk9QcvpdEOoxCBmPlYYHFHcwbMzqEG/caqXlgTZAEt6YVm/s6SnAR2a2gVADtipDBFYCmSXmnQ18WYV1HJOUjAwOZm/nux07KTh8mB0LF9O4Q4dYbT4iyhgdQcgIwcipjNGhjNETlJx+F4R6DEJGiVy1XfLKObfbzG4BJpvZc4QaqMOdc6MKy5jZejNr45zbGMEqHwMmmtlM59wGM8sA7gauqob4ZbLERE765bWsfPIpKCigeZcuJLdqGavNR0QZoyMIGSEYOZUxOpQxeoKQc+wzQ+h23hk0TU1h7YJnGfH4JMZOmBXvWMUEoR6DkPFYmZX5g/NxyZyr+Mma2T7nXP1ylg0H9jnnHiuvvJm9C7wBPAD0cc6tClv2OJDtnHvYm54F3OacW+xN9/Cm+3vTVwL3A7WBH4D/c869VdmT/M3cWTVnj4qISFyMHzQ63hEqNXDc4HhHOG681K2HL35J/2bPu1Fv47RrcKkvnltJlfa0ltdg9ZYNr6y8c65wQOmrZZQdVmK6R4npWcCssOm3CI2FFREREZEaRH8RS0RERCSgatLVEKp6nVYRERERkZhTT6uIiIhIQNWk3sea9FxFREREJKDU0yoiIiISUDVpTKsarSIiIiIBVYParBoeICIiIiL+p55WERERkYCqScMD1NMqIiIiIr6nnlYRERGRgKpBHa1qtIqIiETDwHGD4x2hUuMHjY53hIgEoS79IqEGtVo1PEBEREREfE89rSIiIiIBVYM6WtXTKiIiIiL+p55WERERkYAyc/GOEDNqtIqIiIgElIYHiIiIiIj4iHpaRURERAJKfxFLRERERMRH1NMqIiIiElA1qKNVPa0iIiIi4n/qaRUREREJqJrU+6hGq4iIiEhA1aQTsdRoraLcFStYN/4NKCggrVtXWvftHe9IpShjdAQhIwQjpzJGhzJGTxBy+j3j848Opk/PTuzI2UNmr9vjHadcfq9HiVzgepXNLN/MlprZMjP7t5mdH6ttu4IC1o0bz5m3DqHTiOHsWLiIA1u3xmrzEVHG6AhCRghGTmWMDmWMniDkDELGVyfO5vJfPRTvGBUKQj0eO6uGmz8FrtEKHHTOdXTOdQDuAkbFasN716+nTvPm1GnWjIRatWh2bia7li6L1eYjoozREYSMEIycyhgdyhg9QcgZhIyfLVzFrrx98Y5RoSDUo0QuiI3WcA2A3Fht7FBuHkmpqUXTSampfJ+bF6vNR0QZoyMIGSEYOZUxOpQxeoKQMwgZg6Am1KNVwz+/CuKY1rpmthSoA6QDF8YzTBAGQCtjdAQhIwQjpzJGhzJGTxByBiFjEBxv9WgW9P7HyAXxmRYODzgd6A28Ylb6EDSzG81ssZkt/nrKu1HZcFJqIw7lHunYPZSbS1KjRlFZd7QoY3QEISMEI6cyRocyRk8QcgYhYxCoHo8vQWy0FnHOzQOaAs3KWPaCcy7TOZd52mWXRmV7KRkZHMzeznc7dlJw+DA7Fi6mcYcOUVl3tChjdAQhIwQjpzJGhzJGTxByBiFjENSMeqw5J2IFcXhAETM7HUgEcmKyvcRETvrltax88ikoKKB5ly4kt2oZi01HTBmjIwgZIRg5lTE6lDF6gpAzCBnHPjOEbuedQdPUFNYueJYRj09i7IRZ8Y5VTBDqUSJnzrl4Z6gSM8sHlhdOAnc756ZW9JjfzJ0VrCcpIiJSDcYPGh3vCBEZOG5wvCNU6qVuPXzRJbn70LSot3EaJvX2xXMrKXA9rc65xHhnEBEREfEHX7Yvq0Wgx7SKiIiIiL+YWWMz+8jM1nj/p5ZT7kdm9qGZfWVmX5pZRkXrVaNVREREJKDMEqJ+i4I7gRnOuVOAGd50WV4BHnXOnQGcC2yvaKVqtIqIiIhINF0OjPXujwWuKFnAzH4M1HLOfQTgnNvnnDtQ0UrVaBUREREJrOhf8ir8Wvfe7cYqhkpzzm0D8P5vXkaZU4E8M3vLzJaY2aNmVuF5S4E7EUtEREREqo9z7gXghYrKmNnHQIsyFt0T4WZqAd2ATsAmYAJwPfBiRQ8QERERkQCyOF09wDl3UXnLzCzbzNKdc9vMLJ2yx6puAZY459Z5j3kH6EwFjVYNDxAREREJKKuGf1EwBbjOu38dMLmMMouAVDMr/KumFwJfVrRSNVpFREREJJoeAnqZ2RqglzeNmWWa2RgA51w+cBsww8yWExpQ+/eKVqrhASIiIiKB5b/+R+dcDtCzjPmLgRvCpj8C2ke6Xv89UxERERGREtTTKiIiIhJQZjXnz7iq0SoRmfPMt/GOEJHuQ1rFO8Jx4enOBfGOUKmOg7bFO0JEdEyKnwwcNzjeESIyftDoeEeo1EubesQ7gqfmNFo1PEBEREREfE89rSIiIiIBFa/rtMaDelpFRERExPfU0yoiIiISWDWn/1GNVhEREZGA0vAAEREREREfUU+riIiISEDVpOu0qqdVRERERHxPPa0iIiIigaWeVhERERER31BPq4iIiEhAWQ3qf1SjVURERCSwas7wADVaqyh3xQrWjX8DCgpI69aV1n17xztSKX7L2L1DOn/6dSaJCcYbM9YyevKXpcr0Pe9H3PKL9jjn+GpjHsOe/iwOSYvzWz2Wxw85nXM8OuoNPpu7kjp1khj+519xxo9/VKrcDz8c5uE/T+DzRauxBOO3t1xGz15n8+/Fa3js4YmsXf0tIx/9by66+OxqzatjsvoEISMEI6cyRsfzjw6mT89O7MjZQ2av2+MdR45B1ButZrbPOVe/xLzTgNFAI+AEYC7wJvCwV+Rk4FvgIPCFc+5X3uOeAq4CTnTOFZjZr4Gh3mN+DHwN5APTnHN3Rvu5lOQKClg3bjxnDruVpNRUlj04isYd25PcsmV1bzpifsuYYMbw//4p1z04k6ycA7w1qjczFm9h7bd7isq0aZHCTVecydX3fsie/Ydo3OCEuGQN57d6LI9fcn42dyWbN23nnffvZ8UX6xk1YjyvjL+jVLkXR39A48b1eXvq/RQUFLB79wEAWqQ35v4Hf8Wr//i42rPqmKw+QcgIwcipjNHz6sTZPD92OmOeuDneUaqFLnkVfU8DTzjnOjrnzgCecc5N96Y7AouBQd50YYM1ARgAbAa6AzjnXg57zFbgAm+62husAHvXr6dO8+bUadaMhFq1aHZuJruWLovFpiPmt4wdTm7Cxqy9bN6+jx/yC5j6r41c9NMTi5W5pufJvDZ9NXv2HwJg157v4xG1GL/VY3n8knP2J8vod1lnzIyfdDiJfXsPsGPH7lLlprw9j1/fEOqJSUhIIDU19P22ZasmnHJaayyh+t98dUxWnyBkhGDkVMbo+WzhKnbl7Yt3DImCWDVa04EthRPOueURPOYCYAXwHDCwmnJVyaHcPJJSU4umk1JT+T43L46JSvNbxrTGddmWc6BoOivnAGmN6xYr07ZlChnpDZjwwMVMevASundIj3XMUvxWj+XxS87t2XmktTiSo3laKjuyi+fYuyd0HDz37Lv88hcjuX3Y38nZuYdY0zFZfYKQEYKRUxklclYNN3+KVaP1CWCmmX1gZr83s0YRPGYgMB54G+hvZrWrskEzu9HMFpvZ4q+nvHsUkSPdTrWtOmrimbGsny2cKz6dmJBARnoKg+7/iFuf+pSRN3UmJblKuzsmgrCvIT45S+7TsnIczi8gOzuXDp1O4p8T76Z9h7Y8+dibsQlYLJeOyVgKQkYIRk5llLIYCVG/+VVMkjnnXgbOACYCPYD5ZlbuIDEzSwL6Au845/YAC4CLq7jNF5xzmc65zNMuu/Sos4dLSm3EodzcoulDubkkNYqk/R07fsuYlXOA9CbJRdMtmiSzPfdg8TK7DvDxoi0cznds2bGfdVv3kJGeEuuoxfitHssTz5xvjJ/FwJ//mYE//zPNmjckO+tIju3ZuTRtXjxHo0b1qFM3iQt6dgTgoovPZtVXm2OSNZyOyeoThIwQjJzKKFJazJrTzrmtzrmXnHOXA4eBsyoo3htoCCw3sw1AV3wwRCAlI4OD2dv5bsdOCg4fZsfCxTTu0CHesYrxW8YvvsmhTXoKrZvVo3ZiAv3Ob8OMxVuKlfl44WY6n5UGQGrKCbRNT2FzdnzHH/mtHssTz5xXD+zB+DfvYfyb99Djwg5MnTIf5xzLl62jfv26NGvWsFh5M6P7z37C4kWrAVi44Gvatov9z+46JqtPEDJCMHIqo0Su5gwPiMklr8ysNzDDOfeDmbUAmhC6WkB5BgI3OOfGe4+vB6w3s2Tn3IEKHletLDGRk355LSuffAoKCmjepQvJrfx1lqTfMuYXOO5/aTEv33MhiQnGxE++Yc2W3Qy9uj0rvslhxuffMmfZNrp2SGfa4/3JL3A89NoS8vYdiltm8F89lscvObt2P4vP5q7g8j73UaduEsNH/Kpo2cCf/5nxb94DwC3DBnDvXf/gLw9NIrVxff7vwVC5lcs3cNuto9mz5wBzZy1n9F/fY+Lk+6olq47J6hOEjBCMnMoYPWOfGUK3886gaWoKaxc8y4jHJzF2wqx4x5KjYK6swWjHskKzAkJn9hd6HGgN9AO+8+Y96px7Lewxs4DbnHOLzSyZ0ElbGd7QgMIybwETnHMTvOkNQKZzbmdlmX4zd1Z0n2QNNOeZir5j+Ef3Ia3iHeG48HTngnhHqFTHQdviHSEiOiZFqm78oNHxjlCpg5vG+6JL8nDBsqi3cWoldPDFcysp6j2tzrnyhhwMq+AxPcLuHwAal1HmyhLTGUeXUEREROT4oOu0ioiIiIj4iP6Mq4iIiEhg1Zz+x5rzTEVEREQksNTTKiIiIhJQ5uNLVEWbelpFRERExPfU0yoiIiISWDWnp1WNVhEREZGA0iWvRERERER8RD2tIiIiIoFVc/ofa84zFREREZHAUk+riIiISEDVpEtemXMu3hkCx8xudM69EO8cFQlCRghGTmWMDmWMniDkVMboUMboCUpOKZ+GBxydG+MdIAJByAjByKmM0aGM0ROEnMoYHcoYPUHJKeVQo1VEREREfE+NVhERERHxPTVaj04QxsQEISMEI6cyRocyRk8QcipjdChj9AQlp5RDJ2KJiIiIiO+pp1VEREREfE+NVhERERHxPTVaPWbWwsxeN7NvzOxLM3vfzE41s4NmttSb94qZ1fbK9zCz97z715uZM7OeYesb4M27qppzD/Dyhd8KzOx/ve0PCSv7rJldXw0Z9nn/Z1S0TTP7h5mtN7NlZrbaq89WJdcTNn29mT3r3T/NzGZ5z+8rMzvqsUkV7OsVJcoNN7PbwqZrmdlOMxtVolx/M1viPa8vzWzw0WYrI6szs7+ETd9mZsPDpm80s1XebaGZdfXmJ5rZ52bWPazsh2b2i2hlKydvvrePVpjZu2bWyJtfeGyMCCvb1Mx+KNzHsRD2ujw9bN4pZvaedzx8bmafFNabdwzuKPH6+nEMchbW40rvuBpmZgnesvD3njQve+Gx934MsxXbx2HLf29m35lZw7B5Pcxst/c6+drM5phZ/2rM2CRsf2WZ2bdh00nlHAeZ3nNK8qbbmdk6M2tQxW2H189EM2tVSZYq1aeZXRL2+H1efS610Ptp0bHhlb3CzL7w3h+Wm9kVx1az5T7XZWb2bzM7P5rrL2eb+8qYV+rzoaJ6CnvcU97+KHxt/TrsMYe8OltqZg9V9/OSCDnnavwNMGAecFPYvI5AN2CFN50IzAQGedM9gPe8+9cDXwBjwh4/AVgKXBXj53IjMBs4CcgG1gJJ3rJngeurYZv7vP8zKtom8I/C+vDq/PfA6rCy+0qs93rgWe/+dODysGU/qa59HTZ/OHBb2HRf4DPgG46MB68NbAVae9MnAKdFsW6/A9YDTb3p24Dh3v3+wOdhy84GNgEtvOn/AJZ7GQcC02Nw/O0Luz8WuCfs2PgGWBK2/H+918iz1Z0rbJtvAHPD6rCOdwxeFlbmrLBj9vpY5iunHpsDHwP3e9M9OPLeMxoYGla2fbz2cdi8hV4dXx82ryizN90R2AD0jEHeYq/jso6DsPl/A+727k8DBh5j/YwDhlWSpcr1GbZsFpBZVj0DHQi9F7f1ptt601E7RkpkvwSYHcvjL2xehZ8PJevJm5dA6P1yPtCjjHVuwHtv1c0/N/W0hlwA/OCce75whnNuKbA5bDqf0JtHq9IPB0JvKueaWW0zqw+cTOgDOWbM7FTgPuC/gAJgBzADuC6GMSLapgt5AsgC+kSw3nRgS9jjlx9lvkr3dQUGAk8ReqPr7M1LIfTnkHO8dX3vnPv6KLOV5TChM15/X8ayO4A/Oud2etv+N6EPvd960wuAfxH6oBxZOD+G5lH89XIQ+MrMMr3pawg1HmLCe112Af4buNabPQiY55ybUljOObfCOfePWOWqjHNuO6Evo78zs5J/r7Hk6+KLWGajxD42s3ZAfeBPhF4vZfJecw8Av6vugCWVcxwUuhu4wcxuB2o758Yf4+bmEvosiNRR1Wc5bgNGOufWA3j/jwL+WMX1RKoBkFtN667M0Xw+XACsAJ6j6nUrcaJGa8hZhHqsymVmdQj1XE0rp4gj1BtyCXA5MKWcctXCQsMW/knoW/ymsEUPAX8ws8QYxqnKNv8NnF5pKXgCmGlmH3g/lzWq9BFlq2hftwv7aWgpcFPhAjOrC/QE3gPG473JOed2EdrXG81svJkNKvypKYr+CgwK/7nVc2YZz2WxN7/QXcCtwD+dc2ujnKtc3r7vSenXwevAtWbWGsgn1EsdK1cA05xzq4FdZnY2obr6dyWPu8aKDw+oW+1JS3DOrSP0ft28xKK/Ai9aaEjDPWbWMlaZytnHAwm9PuYCp5lZybzhIn3tR1tZxwEAzrk84GFCjbubj2UjZlaL0BfyiL5gR6E+S4rk/eFY1fVeE6uAMcCIyh5QTY7m86Gwbt8G+nufoeJzarRWrp3XgMkBNlXSk/E6oW/u1xJ6McTSCGClc+718Jnet+uFwC9jFaSK2yzZc1Rqdd46XwbOACYS+glsvpmdcAwxy/KNc65j4Q14PmxZf+AT59wB4E1gQGGj3Dl3A6EPm4WEejdeimYo59we4BXglgiKG16deboDuwk11mOhbtjrpTHwUYnl04BehD4wJsQoU6GBhF6jeP+X6l0xs7e9sYVvhc2eEH5cOOcOxiJsGUq9Vpxz0wkNBfo7oQbgEjNrVs05KtrH1wKvO+cKgLeAisZQV/bary6VHQd9CA1zOtqxy4X1s5jQrzIvRlj+WOuzpJLvBeXNOxYHvdfE6UBv4JUyfg2odlX9fLDQuOW+wDve++sC4OIYRJVjpEZryErgnHKWfeM1YE4GOpvZZeWtxDm3kFDjoKn3LT4mzKwH8HPK/6ltJKGfkmO5vyPdZifgK+/+Qe/NpFBjYGfhhHNuq3PuJefc5YR+Nj+ahlhF+7oiA4GLzGwDod6LJoR+XirMttwb7tCL0L6IticJ/ZxZL2zel5R+Lmd78zGzesAjwIVAMzPrWw25SjrovV7aAEmUGJLgnDtEqP7+QKjxHxNm1oRQPYzx9uEfCQ1PWEmozgrzDSA0jrVxrLJFwsxOItQzvb3kMufcLufcP51z/wUsIvRFpTqVuY/NrD1wCvCRV8fXUvHPruGv/Zgo7zgobGhZ6OSwhoR+MXvUzJKPYjMHw77gDPGO+UrLc+z1WdJKILPEvKL3h2hzzs0DmgLV/aWpvO1X5fOhN6H9vNyr265oiEAgqNEaMhM4wcz+p3CGmf2U0JsIAM65bcCdhH5urchdhMZFxYSZpQIvA79yzu0tq4xzbhWhN6pqO1u3qtu0kFsIjUUqHHIxG/hPb3ld4GrgE2+6tx25ckMLQo3Gb48iWqX7uoysDQi9qf3IOZfhnMsg9MEy0Mzqe18aCnUENh5Frgp5wxDeINRwLfQI8LD3QYyZdSTU4Pqbt/w+4A1vX9wMPOENc6l2zrndhHqGbyvjZ7e/AHc453JikcVzFfCKc66Ntw9PJHSC22qgS4kvo0fTUKk2Xs/p84ROCHMlll1Y2LAysxSgHaHevWpXxj4eSOjEpgzv1hJoZWalXlteg+xeQsMbYqm846Cr957zF+C33pjIycA9sQp2LPVZjseAu8wsA0JX8CD02fSXch9xDCx0JYZEvPH9sXQUnw8DgRvC3s/bAhcf5ZcUiaFa8Q7gB845Z2YDgCfN7E5CZ2xvIDQWMNw7wHAz61bBuj6otqBlu4nQOLfnSvwqU3J4wp+BJbEKVcE2HzWzewk1DOYDF4T1RAwFRnuNWSP04TLHW3Yx8JSZfedN/9E5l1XVQFXY1+GuBGY6574PmzeZUKNxGHC7mY0mdKLRfkINx+rwF8J6051zUyx0ybB/mZkD9gL/6ZzbZqHLMg0gdAYxzrmlZjadUO/3/dWUrxjn3BIzW0aoh2hu2PyVhHqBYmkgobHW4d4kNISlP/C4mT1J6GfhvcCDYeWuMe9SYp6bnXP/qs6wHPnJuDahXqNXgcfLKHcO8KyZHSbUCTHGObeomrMVKbGPr6X0SZVve/MXAN3MbAmh1/524Bbn3IxYZfVUdBz0IfRzcWFP5HBgqZn9wzm3JhbhqlCfD0ewrqVmdgfwrteg+wG43TsJLloKj1MIvWdf50InLVenZDPbEjb9ONCaCD8fvIbpJUDRpQmdc/vN7FPgUmI/bEmqQH/GVURERER8T8MDRERERMT31GgVEREREd9To1VEREREfE+NVhERERHxPTVaRURERMT31GgVEREREd9To1VEREREfO//ASR9Z59uMXvSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12,8))\n",
    "sns.heatmap(corr_matrix, annot=True, cmap='YlGnBu')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Detecting Collinearity with Eigenvectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "eigenvalues, eigenvectors = np.linalg.eig(df.corr())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8    0.0635\n",
       "10   0.1693\n",
       "11   0.1860\n",
       "12   0.2202\n",
       "9    0.2769\n",
       "7    0.3961\n",
       "6    0.5354\n",
       "5    0.6574\n",
       "4    0.8348\n",
       "3    0.8576\n",
       "2    1.2426\n",
       "1    1.4333\n",
       "0    6.1268\n",
       "dtype: float64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.options.display.float_format = '{:,.4f}'.format\n",
    "pd.Series(eigenvalues).sort_values()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that index 8, eigenvalue of 0.0635, is near to zero or very small compared to the others. Small value represents presence of collinearity. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9    0.7202\n",
       "8    0.6335\n",
       "2    0.2511\n",
       "1    0.0809\n",
       "0    0.0460\n",
       "5    0.0456\n",
       "4    0.0436\n",
       "6    0.0386\n",
       "3    0.0359\n",
       "12   0.0244\n",
       "10   0.0234\n",
       "7    0.0183\n",
       "11   0.0045\n",
       "dtype: float64"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.abs(pd.Series(eigenvectors[:,8])).sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that index 9, 8, 2 have very high loading when compared against the rest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INDUS RAD TAX\n"
     ]
    }
   ],
   "source": [
    "print(df.columns[2], df.columns[8], df.columns[9])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the factors that are causing multicollinearity problem."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Revisiting Feature Importance and Extractions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check:\n",
    "\n",
    "1. Direction of the coefficient\n",
    "2. Impact of the variable / factor on the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.0063</td>\n",
       "      <td>18.0000</td>\n",
       "      <td>2.3100</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.5380</td>\n",
       "      <td>6.5750</td>\n",
       "      <td>65.2000</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>296.0000</td>\n",
       "      <td>15.3000</td>\n",
       "      <td>396.9000</td>\n",
       "      <td>4.9800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.0273</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>7.0700</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.4690</td>\n",
       "      <td>6.4210</td>\n",
       "      <td>78.9000</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0000</td>\n",
       "      <td>242.0000</td>\n",
       "      <td>17.8000</td>\n",
       "      <td>396.9000</td>\n",
       "      <td>9.1400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.0273</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>7.0700</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.4690</td>\n",
       "      <td>7.1850</td>\n",
       "      <td>61.1000</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0000</td>\n",
       "      <td>242.0000</td>\n",
       "      <td>17.8000</td>\n",
       "      <td>392.8300</td>\n",
       "      <td>4.0300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.0324</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2.1800</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.4580</td>\n",
       "      <td>6.9980</td>\n",
       "      <td>45.8000</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0000</td>\n",
       "      <td>222.0000</td>\n",
       "      <td>18.7000</td>\n",
       "      <td>394.6300</td>\n",
       "      <td>2.9400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.0691</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>2.1800</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.4580</td>\n",
       "      <td>7.1470</td>\n",
       "      <td>54.2000</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0000</td>\n",
       "      <td>222.0000</td>\n",
       "      <td>18.7000</td>\n",
       "      <td>396.9000</td>\n",
       "      <td>5.3300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    CRIM      ZN  INDUS   CHAS    NOX     RM     AGE    DIS    RAD      TAX  \\\n",
       "0 0.0063 18.0000 2.3100 0.0000 0.5380 6.5750 65.2000 4.0900 1.0000 296.0000   \n",
       "1 0.0273  0.0000 7.0700 0.0000 0.4690 6.4210 78.9000 4.9671 2.0000 242.0000   \n",
       "2 0.0273  0.0000 7.0700 0.0000 0.4690 7.1850 61.1000 4.9671 2.0000 242.0000   \n",
       "3 0.0324  0.0000 2.1800 0.0000 0.4580 6.9980 45.8000 6.0622 3.0000 222.0000   \n",
       "4 0.0691  0.0000 2.1800 0.0000 0.4580 7.1470 54.2000 6.0622 3.0000 222.0000   \n",
       "\n",
       "   PTRATIO        B  LSTAT  \n",
       "0  15.3000 396.9000 4.9800  \n",
       "1  17.8000 396.9000 9.1400  \n",
       "2  17.8000 392.8300 4.0300  \n",
       "3  18.7000 394.6300 2.9400  \n",
       "4  18.7000 396.9000 5.3300  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEHRJREFUeJzt3X+MZWV9x/H3p6yAYHWBHch2FzvQblBqFOiEQmmMBW1BDPAHJhBTN3aTTVPaYrXRpSaS/mECaSNq0pJuBVkTglLUQoCqZMWYNmXt8PvHSlmRwgiyY/lhq4nt6rd/3LN4ux12Zu6Zy+w8vl/JzT3nOc+55/vk3v3suc+9506qCklSu35huQuQJI2XQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklq3KrlLgBgzZo1NTk5udxlSNKKcvfdd3+/qibm63dABP3k5CTT09PLXYYkrShJ/n0h/Zy6kaTGzRv0Sa5NsjvJQ3Ns+7MklWRNt54kn0qyK8kDSU4ZR9GSpIVbyBn9dcDZ+zYmORZ4B/DkUPM5wIbuthm4un+JkqQ+5g36qvoG8Nwcm64CPgQM/87x+cBna+AuYHWStUtSqSRpJCPN0Sc5D/huVd2/z6Z1wFND6zNd21yPsTnJdJLp2dnZUcqQJC3AooM+yWHAR4CPzrV5jrY5/7JJVW2tqqmqmpqYmPfbQZKkEY3y9cpfAY4D7k8CsB64J8mpDM7gjx3qux54um+RkqTRLfqMvqoerKqjq2qyqiYZhPspVfU94Bbgvd23b04DXqyqZ5a2ZEnSYizk65U3AP8CnJBkJsmm/XS/HXgc2AX8HfCHS1KlJGlk807dVNXF82yfHFou4JL+ZUnSK2dyy23Lduwnrjh37MfwylhJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxs0b9EmuTbI7yUNDbX+Z5FtJHkjypSSrh7ZdlmRXkkeT/O64CpckLcxCzuivA87ep+0O4E1V9Wbg34DLAJKcCFwE/Fq3z98kOWjJqpUkLdq8QV9V3wCe26ftq1W1p1u9C1jfLZ8PfK6qflxV3wF2AacuYb2SpEVaijn63wf+sVteBzw1tG2ma/t/kmxOMp1kenZ2dgnKkCTNpVfQJ/kIsAe4fm/THN1qrn2ramtVTVXV1MTERJ8yJEn7sWrUHZNsBN4FnFVVe8N8Bjh2qNt64OnRy5Mk9TXSGX2Ss4EPA+dV1Y+GNt0CXJTkkCTHARuAb/YvU5I0qnnP6JPcALwNWJNkBricwbdsDgHuSAJwV1X9QVU9nORG4BEGUzqXVNVPxlW8JGl+8wZ9VV08R/M1++n/MeBjfYqSJC0dr4yVpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklq3Mh/eETLZ3LLbct27CeuOHfZji1pNJ7RS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUuHmDPsm1SXYneWio7cgkdyR5rLs/omtPkk8l2ZXkgSSnjLN4SdL8FnJGfx1w9j5tW4DtVbUB2N6tA5wDbOhum4Grl6ZMSdKo5g36qvoG8Nw+zecD27rlbcAFQ+2frYG7gNVJ1i5VsZKkxRt1jv6YqnoGoLs/umtfBzw11G+ma5MkLZOl/q2bzNFWc3ZMNjOY3uH1r3/9EpfxyljO35yRpIUa9Yz+2b1TMt397q59Bjh2qN964Om5HqCqtlbVVFVNTUxMjFiGJGk+owb9LcDGbnkjcPNQ+3u7b9+cBry4d4pHkrQ85p26SXID8DZgTZIZ4HLgCuDGJJuAJ4F3d91vB94J7AJ+BLxvDDVLkhZh3qCvqotfZtNZc/Qt4JK+RUmSlo5XxkpS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1rlfQJ/nTJA8neSjJDUkOTXJckh1JHkvy+SQHL1WxkqTFGznok6wD/gSYqqo3AQcBFwFXAldV1QbgeWDTUhQqSRpN36mbVcCrk6wCDgOeAc4Ebuq2bwMu6HkMSVIPIwd9VX0X+CvgSQYB/yJwN/BCVe3pus0A6/oWKUkaXZ+pmyOA84HjgF8CDgfOmaNrvcz+m5NMJ5menZ0dtQxJ0jz6TN28HfhOVc1W1f8AXwR+E1jdTeUArAeenmvnqtpaVVNVNTUxMdGjDEnS/vQJ+ieB05IcliTAWcAjwJ3AhV2fjcDN/UqUJPXRZ45+B4MPXe8BHuweayvwYeADSXYBRwHXLEGdkqQRrZq/y8urqsuBy/dpfhw4tc/jSpKWjlfGSlLjDHpJapxBL0mNM+glqXG9PoyVXimTW25btmM/ccW5y3ZsaSl4Ri9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJalyvoE+yOslNSb6VZGeS05McmeSOJI9190csVbGSpMXre0b/SeDLVfUG4C3ATmALsL2qNgDbu3VJ0jIZOeiTvBZ4K3ANQFX9d1W9AJwPbOu6bQMu6FukJGl0fc7ojwdmgc8kuTfJp5McDhxTVc8AdPdHL0GdkqQR9Qn6VcApwNVVdTLwQxYxTZNkc5LpJNOzs7M9ypAk7U+foJ8BZqpqR7d+E4PgfzbJWoDufvdcO1fV1qqaqqqpiYmJHmVIkvZn5KCvqu8BTyU5oWs6C3gEuAXY2LVtBG7uVaEkqZdVPff/Y+D6JAcDjwPvY/Cfx41JNgFPAu/ueQxJUg+9gr6q7gOm5th0Vp/HlSQtHa+MlaTGGfSS1DiDXpIaZ9BLUuP6futm2U1uuW25S5CkA5pn9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktS43kGf5KAk9ya5tVs/LsmOJI8l+XySg/uXKUka1VKc0V8K7BxavxK4qqo2AM8Dm5bgGJKkEfUK+iTrgXOBT3frAc4Ebuq6bAMu6HMMSVI/fc/oPwF8CPhpt34U8EJV7enWZ4B1c+2YZHOS6STTs7OzPcuQJL2ckYM+ybuA3VV193DzHF1rrv2ramtVTVXV1MTExKhlSJLmsarHvmcA5yV5J3Ao8FoGZ/irk6zqzurXA0/3L1OSNKqRz+ir6rKqWl9Vk8BFwNeq6j3AncCFXbeNwM29q5QkjWwc36P/MPCBJLsYzNlfM4ZjSJIWqM/UzUuq6uvA17vlx4FTl+JxJUn9eWWsJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJatySfL1SPz8mt9y23CVIWiTP6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS40YO+iTHJrkzyc4kDye5tGs/MskdSR7r7o9YunIlSYvV54x+D/DBqnojcBpwSZITgS3A9qraAGzv1iVJy2TkoK+qZ6rqnm75P4GdwDrgfGBb120bcEHfIiVJo1uSOfokk8DJwA7gmKp6Bgb/GQBHL8UxJEmj6R30SV4DfAF4f1X9YBH7bU4ynWR6dna2bxmSpJfRK+iTvIpByF9fVV/smp9NsrbbvhbYPde+VbW1qqaqampiYqJPGZKk/ejzrZsA1wA7q+rjQ5tuATZ2yxuBm0cvT5LU16oe+54B/B7wYJL7urY/B64AbkyyCXgSeHe/EiVJfYwc9FX1T0BeZvNZoz6uJGlpeWWsJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuPGFvRJzk7yaJJdSbaM6ziSpP0bS9AnOQj4a+Ac4ETg4iQnjuNYkqT9WzWmxz0V2FVVjwMk+RxwPvDImI4njc3kltuW5bhPXHHushxX7RnX1M064Kmh9ZmuTZL0ChvXGX3maKv/0yHZDGzuVv8ryaNjqmUN8P0xPfaBxHG2ZU2u/PkYJz8nzycvM85c2etxf3khncYV9DPAsUPr64GnhztU1VZg65iO/5Ik01U1Ne7jLDfH2RbH2ZblHue4pm7+FdiQ5LgkBwMXAbeM6ViSpP0Yyxl9Ve1J8kfAV4CDgGur6uFxHEuStH/jmrqhqm4Hbh/X4y/C2KeHDhCOsy2Osy3LOs5U1fy9JEkrlj+BIEmNW9FBn+TYJHcm2Znk4SSXdu1HJrkjyWPd/RFde5J8qvtZhgeSnLK8I1iYJIcm+WaS+7tx/kXXflySHd04P9998E2SQ7r1Xd32yeWsf7GSHJTk3iS3duutjvOJJA8muS/JdNfW1GsXIMnqJDcl+Vb3b/X01saZ5ITuedx7+0GS9x8o41zRQQ/sAT5YVW8ETgMu6X5qYQuwvao2ANu7dRj8JMOG7rYZuPqVL3kkPwbOrKq3ACcBZyc5DbgSuKob5/PApq7/JuD5qvpV4Kqu30pyKbBzaL3VcQL8dlWdNPTVu9ZeuwCfBL5cVW8A3sLguW1qnFX1aPc8ngT8OvAj4EscKOOsqmZuwM3AO4BHgbVd21rg0W75b4GLh/q/1G+l3IDDgHuA32BwAcaqrv104Cvd8leA07vlVV2/LHftCxzfegb/IM4EbmVw8V1z4+xqfgJYs09bU69d4LXAd/Z9Xlob5z5j+x3gnw+kca70M/qXdG/bTwZ2AMdU1TMA3f3RXbcV+9MM3XTGfcBu4A7g28ALVbWn6zI8lpfG2W1/ETjqla14ZJ8APgT8tFs/ijbHCYOrxb+a5O7uSnFo77V7PDALfKabjvt0ksNpb5zDLgJu6JYPiHE2EfRJXgN8AXh/Vf1gf13naFsRXzuqqp/U4G3hegY/GvfGubp19ytynEneBeyuqruHm+fouqLHOeSMqjqFwdv4S5K8dT99V+pYVwGnAFdX1cnAD/nZ9MVcVuo4Aeg+PzoP+Pv5us7RNrZxrvigT/IqBiF/fVV9sWt+NsnabvtaBmfBsICfZjjQVdULwNcZfCaxOsneayGGx/LSOLvtrwOee2UrHckZwHlJngA+x2D65hO0N04Aqurp7n43g/ncU2nvtTsDzFTVjm79JgbB39o49zoHuKeqnu3WD4hxruigTxLgGmBnVX18aNMtwMZueSODufu97e/tPvE+DXhx79uqA1mSiSSru+VXA29n8IHWncCFXbd9x7l3/BcCX6tuIvBAVlWXVdX6qppk8Pb3a1X1HhobJ0CSw5P84t5lBvO6D9HYa7eqvgc8leSEruksBj9X3tQ4h1zMz6Zt4EAZ53J/cNHzQ4/fYvB25wHgvu72TgbztNuBx7r7I7v+YfAHUb4NPAhMLfcYFjjONwP3duN8CPho13488E1gF4O3iod07Yd267u67ccv9xhGGPPbgFtbHWc3pvu728PAR7r2pl67Xe0nAdPd6/cfgCMaHedhwH8ArxtqOyDG6ZWxktS4FT11I0man0EvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1Lj/hdlApt/J0ghfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(df['TAX']);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADFJJREFUeJzt3W+MZfVdx/H3p6ykNlKh3aEhLNuhCa2SxgadkGoTo8UHbTFAIjUQNYvBbmIioCXK6pMafbIYIzWxMVmLujEVimtTsDQ2SCFG0xJ3gVphRZCudAXLlkD9k2iLfn0wx7iBWe6Zmftn5zvvVzKZOXfO3fv9MZv3Hs6990yqCknS1ve6RQ8gSZoOgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqYkd83ywnTt31vLy8jwfUpK2vCNHjny9qpYm7TfXoC8vL3P48OF5PqQkbXlJ/mnMfp5ykaQmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCbm+k5Rrc/yvnsX8rjH9l++kMeVtDkeoUtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqYlTQk/xCkseS/F2SO5K8PsmFSR5K8mSSTyY5c9bDSpJObWLQk5wP3AisVNU7gTOAa4Bbgduq6iLgReD6WQ4qSXptY0+57AC+PckO4A3Ac8B7gUPD9w8CV01/PEnSWBODXlX/DPwm8AyrIf8GcAR4qapeHnY7Dpw/qyElSZNN/CXRSc4BrgQuBF4C/gR4/xq71inuvxfYC7B79+4ND7ooi/pFzZK0XmNOufwI8JWqOlFV3wI+BfwAcPZwCgZgF/DsWneuqgNVtVJVK0tLS1MZWpL0amOC/gzw7iRvSBLgMuBx4AHg6mGfPcDdsxlRkjTGmHPoD7H65OfDwJeH+xwAbgE+nOQp4M3A7TOcU5I0wcRz6ABV9RHgI6+4+Wng0qlPpG1tUc9ZHNt/+UIeV5om3ykqSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmRr1s8XTgW/Al6bV5hC5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1MSroSc5OcijJ3yc5muT7k7wpyX1Jnhw+nzPrYSVJpzb2CP23gT+vqu8C3gUcBfYB91fVRcD9w7YkaUEmBj3JG4EfBG4HqKpvVtVLwJXAwWG3g8BVsxpSkjTZjhH7vA04AfxBkncBR4CbgLdU1XMAVfVcknPXunOSvcBegN27d09laM3W8r57Fz2CpA0Yc8plB/C9wO9W1SXAf7CO0ytVdaCqVqpqZWlpaYNjSpImGRP048Dxqnpo2D7EauC/luQ8gOHz87MZUZI0xsSgV9W/AF9N8o7hpsuAx4F7gD3DbXuAu2cyoSRplDHn0AFuAD6R5EzgaeCnWf3H4K4k1wPPAB+czYiSpDFGBb2qHgVW1vjWZdMdR5K0Ub5TVJKaGHvKRdKMLOplosf2X76Qx9XseIQuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSE6ODnuSMJI8k+cywfWGSh5I8meSTSc6c3ZiSpEnWc4R+E3D0pO1bgduq6iLgReD6aQ4mSVqfUUFPsgu4HPj4sB3gvcChYZeDwFWzGFCSNM7YI/SPAr8E/M+w/Wbgpap6edg+Dpw/5dkkSeswMehJfhR4vqqOnHzzGrvWKe6/N8nhJIdPnDixwTElSZOMOUJ/D3BFkmPAnayeavkocHaSHcM+u4Bn17pzVR2oqpWqWllaWprCyJKktUwMelX9clXtqqpl4Brg81X1E8ADwNXDbnuAu2c2pSRpos28Dv0W4MNJnmL1nPrt0xlJkrQROybv8v+q6kHgweHrp4FLpz+SJGkjfKeoJDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqYl1/Qo6qavlffcuegRp0zxCl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpiYlBT3JBkgeSHE3yWJKbhtvflOS+JE8On8+Z/biSpFMZ80uiXwZurqqHk5wFHElyH3AdcH9V7U+yD9gH3DK7USVN0yJ/Mfax/Zcv7LE7m3iEXlXPVdXDw9f/BhwFzgeuBA4Oux0ErprVkJKkydZ1Dj3JMnAJ8BDwlqp6DlajD5x7ivvsTXI4yeETJ05sblpJ0imNDnqS7wD+FPj5qvrXsferqgNVtVJVK0tLSxuZUZI0wqigJ/k2VmP+iar61HDz15KcN3z/POD52YwoSRpjzKtcAtwOHK2q3zrpW/cAe4av9wB3T388SdJYY17l8h7gp4AvJ3l0uO1XgP3AXUmuB54BPjibESVJY0wMelX9FZBTfPuy6Y4jSdoo3ykqSU0YdElqwqBLUhMGXZKaGPMqF0lqYVHXr5nXtWs8QpekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1MSmgp7kfUmeSPJUkn3TGkqStH4bDnqSM4CPAe8HLgauTXLxtAaTJK3PZo7QLwWeqqqnq+qbwJ3AldMZS5K0XpsJ+vnAV0/aPj7cJklagB2buG/WuK1etVOyF9g7bP57kic28ZiT7AS+PsM//3Tm2refLbvu3LrpP2JLrX0K633rmJ02E/TjwAUnbe8Cnn3lTlV1ADiwiccZLcnhqlqZx2Odblz79lv7dl03bO+1v5bNnHL5G+CiJBcmORO4BrhnOmNJktZrw0foVfVykp8DPgecAfx+VT02tckkSeuymVMuVNVngc9OaZZpmMupndOUa99+tuu6YXuv/ZRS9arnMSVJW5Bv/ZekJrZk0MdeciDJ1UkqSZtnwyetPcl1SU4keXT4+JlFzDltY37mSX48yeNJHkvyx/OecVZG/MxvO+nn/Q9JXlrEnLMwYu27kzyQ5JEkf5vkA4uY87RRVVvqg9UnYP8ReBtwJvAl4OI19jsL+Evgi8DKouee19qB64DfWfSsC1j3RcAjwDnD9rmLnntea3/F/jew+gKFhc8+p5/7AeBnh68vBo4teu5FfmzFI/Sxlxz4deA3gP+c53Aztl0vtzBm3R8CPlZVLwJU1fNznnFW1vszvxa4Yy6Tzd6YtRfwxuHr72SN98JsJ1sx6BMvOZDkEuCCqvrMPAebg7GXW/ix4X8/DyW5YI3vbzVj1v124O1J/jrJF5O8b27TzdboS2wkeStwIfD5Ocw1D2PW/qvATyY5zuor7m6Yz2inp60Y9Ne85ECS1wG3ATfPbaL5GXO5hT8Dlqvqe4C/AA7OfKrZG7PuHayedvkhVo9SP57k7BnPNQ+jLrExuAY4VFX/PcN55mnM2q8F/rCqdgEfAP5oaMC2tBUXPumSA2cB7wQeTHIMeDdwT5MnRidebqGqXqiq/xo2fw/4vjnNNktjLjNxHLi7qr5VVV8BnmA18FvdqEtsDK6hz+kWGLf264G7AKrqC8DrWb3Oy7a0FYP+mpccqKpvVNXOqlquqmVWnxS9oqoOL2bcqZp4uYUk5520eQVwdI7zzcqYy0x8GvhhgCQ7WT0F8/Rcp5yNUZfYSPIO4BzgC3Oeb5bGrP0Z4DKAJN/NatBPzHXK08iWC3pVvQz83yUHjgJ3VdVjSX4tyRWLnW62Rq79xuFle18CbmT1VS9b2sh1fw54IcnjwAPAL1bVC4uZeHrW8ff9WuDOGl7u0cHItd8MfGj4+34HcF2n/wbr5TtFJamJLXeELklam0GXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2Smvhf+LStC86G6XcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(df['NOX']);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Standardise Variable to Identify Key Feature(s)\n",
    "\n",
    "In order to perform point 2 properly, one needs to standardise the variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LinearRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coefficient</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>NOX</td>\n",
       "      <td>17.7666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>RM</td>\n",
       "      <td>3.8099</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>CHAS</td>\n",
       "      <td>2.6867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>DIS</td>\n",
       "      <td>1.4756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PTRATIO</td>\n",
       "      <td>0.9527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>LSTAT</td>\n",
       "      <td>0.5248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>RAD</td>\n",
       "      <td>0.3060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>CRIM</td>\n",
       "      <td>0.1080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ZN</td>\n",
       "      <td>0.0464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>INDUS</td>\n",
       "      <td>0.0206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>TAX</td>\n",
       "      <td>0.0123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>B</td>\n",
       "      <td>0.0093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>AGE</td>\n",
       "      <td>0.0007</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         coefficient\n",
       "name                \n",
       "NOX          17.7666\n",
       "RM            3.8099\n",
       "CHAS          2.6867\n",
       "DIS           1.4756\n",
       "PTRATIO       0.9527\n",
       "LSTAT         0.5248\n",
       "RAD           0.3060\n",
       "CRIM          0.1080\n",
       "ZN            0.0464\n",
       "INDUS         0.0206\n",
       "TAX           0.0123\n",
       "B             0.0093\n",
       "AGE           0.0007"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = pd.DataFrame(list(zip(model.coef_, df.columns)), columns=['coefficient', 'name']).set_index('name')\n",
    "np.abs(result).sort_values(by='coefficient', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler  \n",
    "from sklearn.pipeline import make_pipeline  \n",
    "scaler = StandardScaler()  \n",
    "standard_coefficient_linear_reg = make_pipeline(scaler, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coefficient</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>LSTAT</td>\n",
       "      <td>3.7436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>DIS</td>\n",
       "      <td>3.1040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>RM</td>\n",
       "      <td>2.6742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>RAD</td>\n",
       "      <td>2.6622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>TAX</td>\n",
       "      <td>2.0768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>PTRATIO</td>\n",
       "      <td>2.0606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>NOX</td>\n",
       "      <td>2.0567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ZN</td>\n",
       "      <td>1.0816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>CRIM</td>\n",
       "      <td>0.9281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>B</td>\n",
       "      <td>0.8493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>CHAS</td>\n",
       "      <td>0.6817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>INDUS</td>\n",
       "      <td>0.1409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>AGE</td>\n",
       "      <td>0.0195</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         coefficient\n",
       "name                \n",
       "LSTAT         3.7436\n",
       "DIS           3.1040\n",
       "RM            2.6742\n",
       "RAD           2.6622\n",
       "TAX           2.0768\n",
       "PTRATIO       2.0606\n",
       "NOX           2.0567\n",
       "ZN            1.0816\n",
       "CRIM          0.9281\n",
       "B             0.8493\n",
       "CHAS          0.6817\n",
       "INDUS         0.1409\n",
       "AGE           0.0195"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "standard_coefficient_linear_reg.fit(X,y)\n",
    "result = pd.DataFrame(list(zip(standard_coefficient_linear_reg.steps[1][1].coef_, df.columns)), \n",
    "                      columns=['coefficient', 'name']).set_index('name')\n",
    "np.abs(result).sort_values(by='coefficient', ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use $R^2$ to Identify Key Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Compare $R^2$ of model against $R^2$ of model without a feature. \n",
    "\n",
    "* A significant change in $R^2$ signify the importance of the feature.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7406426641094095"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear_reg = smf.ols(formula = 'y ~ CRIM + ZN + INDUS + CHAS + NOX + RM + AGE + DIS + RAD + TAX + PTRATIO + B + LSTAT', \n",
    "              data=df)\n",
    "benchmark = linear_reg.fit()\n",
    "r2_score(y, benchmark.predict(df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "without LSTAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6842042799773889"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear_reg = smf.ols(formula = 'y ~ CRIM + ZN + INDUS + CHAS + NOX + RM + AGE + DIS + RAD + TAX + PTRATIO + B', \n",
    "              data=df)\n",
    "lr_without_LSTAT = linear_reg.fit()\n",
    "r2_score(y, lr_without_LSTAT.predict(df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "without AGE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7406412165505145"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear_reg = smf.ols(formula = 'y ~ CRIM + ZN + INDUS + CHAS + NOX + RM + DIS + RAD + TAX + PTRATIO + B + LSTAT', \n",
    "              data=df)\n",
    "lr_without_AGE = linear_reg.fit()\n",
    "r2_score(y, lr_without_AGE.predict(df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}