ISLR Chapter 4 Lab.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as lda\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as qda\n",
    "from sklearn.neighbors import KNeighborsClassifier as knn\n",
    "import sklearn.linear_model as skl_lm\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Year', 'Lag1', 'Lag2', 'Lag3', 'Lag4', 'Lag5', 'Volume', 'Today',\n",
       "       'Direction'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Smarket =pd.read_csv('./data/Smarket.csv',index_col=[0])\n",
    "Smarket.columns\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Lag1</th>\n",
       "      <th>Lag2</th>\n",
       "      <th>Lag3</th>\n",
       "      <th>Lag4</th>\n",
       "      <th>Lag5</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Today</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1250.000000</td>\n",
       "      <td>1250.000000</td>\n",
       "      <td>1250.000000</td>\n",
       "      <td>1250.000000</td>\n",
       "      <td>1250.000000</td>\n",
       "      <td>1250.00000</td>\n",
       "      <td>1250.000000</td>\n",
       "      <td>1250.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2003.016000</td>\n",
       "      <td>0.003834</td>\n",
       "      <td>0.003919</td>\n",
       "      <td>0.001716</td>\n",
       "      <td>0.001636</td>\n",
       "      <td>0.00561</td>\n",
       "      <td>1.478305</td>\n",
       "      <td>0.003138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.409018</td>\n",
       "      <td>1.136299</td>\n",
       "      <td>1.136280</td>\n",
       "      <td>1.138703</td>\n",
       "      <td>1.138774</td>\n",
       "      <td>1.14755</td>\n",
       "      <td>0.360357</td>\n",
       "      <td>1.136334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2001.000000</td>\n",
       "      <td>-4.922000</td>\n",
       "      <td>-4.922000</td>\n",
       "      <td>-4.922000</td>\n",
       "      <td>-4.922000</td>\n",
       "      <td>-4.92200</td>\n",
       "      <td>0.356070</td>\n",
       "      <td>-4.922000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>2002.000000</td>\n",
       "      <td>-0.639500</td>\n",
       "      <td>-0.639500</td>\n",
       "      <td>-0.640000</td>\n",
       "      <td>-0.640000</td>\n",
       "      <td>-0.64000</td>\n",
       "      <td>1.257400</td>\n",
       "      <td>-0.639500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2003.000000</td>\n",
       "      <td>0.039000</td>\n",
       "      <td>0.039000</td>\n",
       "      <td>0.038500</td>\n",
       "      <td>0.038500</td>\n",
       "      <td>0.03850</td>\n",
       "      <td>1.422950</td>\n",
       "      <td>0.038500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2004.000000</td>\n",
       "      <td>0.596750</td>\n",
       "      <td>0.596750</td>\n",
       "      <td>0.596750</td>\n",
       "      <td>0.596750</td>\n",
       "      <td>0.59700</td>\n",
       "      <td>1.641675</td>\n",
       "      <td>0.596750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2005.000000</td>\n",
       "      <td>5.733000</td>\n",
       "      <td>5.733000</td>\n",
       "      <td>5.733000</td>\n",
       "      <td>5.733000</td>\n",
       "      <td>5.73300</td>\n",
       "      <td>3.152470</td>\n",
       "      <td>5.733000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Year         Lag1         Lag2         Lag3         Lag4  \\\n",
       "count  1250.000000  1250.000000  1250.000000  1250.000000  1250.000000   \n",
       "mean   2003.016000     0.003834     0.003919     0.001716     0.001636   \n",
       "std       1.409018     1.136299     1.136280     1.138703     1.138774   \n",
       "min    2001.000000    -4.922000    -4.922000    -4.922000    -4.922000   \n",
       "25%    2002.000000    -0.639500    -0.639500    -0.640000    -0.640000   \n",
       "50%    2003.000000     0.039000     0.039000     0.038500     0.038500   \n",
       "75%    2004.000000     0.596750     0.596750     0.596750     0.596750   \n",
       "max    2005.000000     5.733000     5.733000     5.733000     5.733000   \n",
       "\n",
       "             Lag5       Volume        Today  \n",
       "count  1250.00000  1250.000000  1250.000000  \n",
       "mean      0.00561     1.478305     0.003138  \n",
       "std       1.14755     0.360357     1.136334  \n",
       "min      -4.92200     0.356070    -4.922000  \n",
       "25%      -0.64000     1.257400    -0.639500  \n",
       "50%       0.03850     1.422950     0.038500  \n",
       "75%       0.59700     1.641675     0.596750  \n",
       "max       5.73300     3.152470     5.733000  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Smarket.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Lag1</th>\n",
       "      <th>Lag2</th>\n",
       "      <th>Lag3</th>\n",
       "      <th>Lag4</th>\n",
       "      <th>Lag5</th>\n",
       "      <th>Volume</th>\n",
       "      <th>Today</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Year</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.029700</td>\n",
       "      <td>0.030596</td>\n",
       "      <td>0.033195</td>\n",
       "      <td>0.035689</td>\n",
       "      <td>0.029788</td>\n",
       "      <td>0.539006</td>\n",
       "      <td>0.030095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lag1</th>\n",
       "      <td>0.029700</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.026294</td>\n",
       "      <td>-0.010803</td>\n",
       "      <td>-0.002986</td>\n",
       "      <td>-0.005675</td>\n",
       "      <td>0.040910</td>\n",
       "      <td>-0.026155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lag2</th>\n",
       "      <td>0.030596</td>\n",
       "      <td>-0.026294</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.025897</td>\n",
       "      <td>-0.010854</td>\n",
       "      <td>-0.003558</td>\n",
       "      <td>-0.043383</td>\n",
       "      <td>-0.010250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lag3</th>\n",
       "      <td>0.033195</td>\n",
       "      <td>-0.010803</td>\n",
       "      <td>-0.025897</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.024051</td>\n",
       "      <td>-0.018808</td>\n",
       "      <td>-0.041824</td>\n",
       "      <td>-0.002448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lag4</th>\n",
       "      <td>0.035689</td>\n",
       "      <td>-0.002986</td>\n",
       "      <td>-0.010854</td>\n",
       "      <td>-0.024051</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.027084</td>\n",
       "      <td>-0.048414</td>\n",
       "      <td>-0.006900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lag5</th>\n",
       "      <td>0.029788</td>\n",
       "      <td>-0.005675</td>\n",
       "      <td>-0.003558</td>\n",
       "      <td>-0.018808</td>\n",
       "      <td>-0.027084</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.022002</td>\n",
       "      <td>-0.034860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Volume</th>\n",
       "      <td>0.539006</td>\n",
       "      <td>0.040910</td>\n",
       "      <td>-0.043383</td>\n",
       "      <td>-0.041824</td>\n",
       "      <td>-0.048414</td>\n",
       "      <td>-0.022002</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.014592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Today</th>\n",
       "      <td>0.030095</td>\n",
       "      <td>-0.026155</td>\n",
       "      <td>-0.010250</td>\n",
       "      <td>-0.002448</td>\n",
       "      <td>-0.006900</td>\n",
       "      <td>-0.034860</td>\n",
       "      <td>0.014592</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Year      Lag1      Lag2      Lag3      Lag4      Lag5    Volume  \\\n",
       "Year    1.000000  0.029700  0.030596  0.033195  0.035689  0.029788  0.539006   \n",
       "Lag1    0.029700  1.000000 -0.026294 -0.010803 -0.002986 -0.005675  0.040910   \n",
       "Lag2    0.030596 -0.026294  1.000000 -0.025897 -0.010854 -0.003558 -0.043383   \n",
       "Lag3    0.033195 -0.010803 -0.025897  1.000000 -0.024051 -0.018808 -0.041824   \n",
       "Lag4    0.035689 -0.002986 -0.010854 -0.024051  1.000000 -0.027084 -0.048414   \n",
       "Lag5    0.029788 -0.005675 -0.003558 -0.018808 -0.027084  1.000000 -0.022002   \n",
       "Volume  0.539006  0.040910 -0.043383 -0.041824 -0.048414 -0.022002  1.000000   \n",
       "Today   0.030095 -0.026155 -0.010250 -0.002448 -0.006900 -0.034860  0.014592   \n",
       "\n",
       "           Today  \n",
       "Year    0.030095  \n",
       "Lag1   -0.026155  \n",
       "Lag2   -0.010250  \n",
       "Lag3   -0.002448  \n",
       "Lag4   -0.006900  \n",
       "Lag5   -0.034860  \n",
       "Volume  0.014592  \n",
       "Today   1.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Smarket.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x24ed16ef310>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABJqElEQVR4nO2dd9wU1fX/P2f3qfDQpEkTRBBFpQgCFtRgLxFNjC2WFKNJ1PiN33wTWxI1mhg15pdolFgTe4olRjGCNRa6ooiIIB0EHnp54Cm79/fHzN29M3On7c5s47xfr+f17E67Z3bunHvuueeeS0IIMAzDMOVPotgCMAzDMNHACp1hGKZCYIXOMAxTIbBCZxiGqRBYoTMMw1QIVcUquFu3bmLAgAHFKp5hGKYsmTNnzgYhRHfdvqIp9AEDBmD27NnFKp5hGKYsIaLlbvvY5cIwDFMhsEJnGIapEFihMwzDVAis0BmGYSoEVugMwzAVAit0hmGYCoEVOsMwTIXACp1hGCYAa7fuxusL1hVbDE9YoTMMwwTga/e9h+/+tbQnQ7JCZxiGCcCarbuLLYIvrNAZhmEqBFboDMMwFQIrdIZhmAqBFTrDMEwIhBDFFsEVVugMwzAhKGF9zgqdYRgmDCWsz/0VOhHVEdFMIvqIiOYT0c2aY4iI/khEi4noYyI6NB5xGYZhikspu1yCrFjUDGCCEGIHEVUDeJeIXhFCTFeOOQXAYPNvLID7zf8MwzBMgfC10IXBDvNrtflnb6ImAnjMPHY6gM5E1CtaURmGYYpP6drnAX3oRJQkorkA1gOYKoSYYTukD4CVyvdV5jb7dS4jotlENLuxsTFHkRmGYYpHCXtcgil0IURKCDECQF8AY4joYNshpDtNc50HhBCjhRCju3fXLlrNMAxT0ogSttFDRbkIIbYAeAvAybZdqwD0U773BbAmH8EYhmFKkbK20ImoOxF1Nj/XAzgewGe2w14EcLEZ7TIOwFYhxJdRC8swDMO4EyTKpReAvxJREkYD8HchxEtE9H0AEEJMAjAZwKkAFgNoAvDtmORlGIZhXPBV6EKIjwGM1GyfpHwWAK6IVjSGYZjSo6xdLgzDMEyWihkUZRiG2dNhC51hGKZCKGF9zgqdYRimUmCFzjAME4JSTs7FCp1hGCYEpavOWaEzDMOEooQNdFboDMMwoWCFzjAMUzlsaWrBgGtfxgsfri62KBZYoTO+CCFw7p+n4T+frC22KAxTdAQElm7YCQB49P1lxRXGBit0xhchgBlLN+H7T8wptigMU3SEANKmIz2hSxxeRFihM76kSnkUiGEKjACQNl+JBJWWRmeFzviSSrNCZxiJECLzTiRZoTPlBhvoDJNFAEibCj2haNBdLSn88l+fYPvu1uIIhmD50Jk9HHa5MIwV1eXy+6mfY3i/Tli4dgf+Om05OrerwY9P2L8ocrGFzvgiu5cl1rtkGADASx+vwc9f+KRg5VkHRQl/eH0RvvOX2RnLvKaqeGqVFTrjSynnrmCYK5/6EI9PX16w8gREpteaUMJcmlpSAIB2NcmCyWKHFTrjy4YdLcUWgWFKB5E1cpJKr3VncxsAVuhMiXP83W8XWwSGKRkEgFTa+KyGLTa1GhZ6fU3xhiZZoTMMw4QkrXG5tLQZWr4myT50hmGYskAoLhd1pmgmlFHZtnzjTry3eEPBZOOwRYZhmBAICK3LRQ6UqiEEx9z5FgBg2e2nFUQ2ttAZhmFCIAS0US4yvNctKGxJ4w5sbYp30hErdMaTNxeuL7YIDFNSCKhRLlmFLqNc3BKmT/jd2zjtnndilY0VOuPJtx+dVWwRGKakUHO5qP7yD1ZsAZCdRapj1eZdMUrGCp0JAU8UZRjgmZkrPbMtFnMenq9CJ6J+RPQmES0govlEdLXmmGOJaCsRzTX/fhGPuAzDMMXl3jcXa8MWJaKIa9QFiXJpA/C/QogPiKgDgDlENFUI8antuHeEEKdHLyLDMExpoQtRzOwrZQtdCPGlEOID8/N2AAsA9IlbMIZhmFIlpSTnslPM3EehfOhENADASAAzNLsPJ6KPiOgVIjrI5fzLiGg2Ec1ubGwMLy3DMIwLhVSk0kJ/ZtbKgpUZhMAKnYgaADwL4H+EENtsuz8A0F8IMRzAPQBe0F1DCPGAEGK0EGJ09+7dcxSZYRjGSSEN4zYPv0q61C10IqqGocyfFEI8Z98vhNgmhNhhfp4MoJqIukUqKcMwjAeFVKReyzIWM8rFd1CUiAjAwwAWCCHudjlmbwDrhBCCiMbAaCg2RiopwzCMB4UcjPRT6Nc9Nw9Pz1xROIFMgkS5HAngIgDziGiuue16APsAgBBiEoCzAfyAiNoA7AJwnuBVERiGKSCFDBf0crkIwKHM0wVqbXwVuhDiXfjMKRFC3Avg3qiEYkoT3Yh+rqTTAtOXbsQR+7FnjomGoCbkzKWb8OXWXZg4IvdgPS8LXef6KZQ7iGeKMoFJ6oJuc+Sv05bhggdn4NX5ayO7JrNnE0Rp3vD8PJzz52m4+pm5eZXlZaHrOgqFWmidFToTmCgV+rINOwEAX26JN7cFs+cQxKvx5Ixo/NpeLhSd62fO8s2RlOsHK3QmMMkIXS4MEzWFHLbzstB/9uw8x7YLHtRN3YkeVuhMYHR5K/KFR86ZqGhLCdz16kJs2hndouYbdzRj/B1vYPH67ZbtqXQ6sjKihBU6E5goXS7E1j6TA0IIvL94g9bl8ebC9bj3zcX45YvzIyvvtQXrsHLTLvz57SWW7ToLvXenusjKzRVW6DHx5sL1mUVjK4UoFTqz57JhRzNufGFeTu/Hq/PX4oKHZuDx6csd+3a3GtdraUvlLaOkKmGoSLsCb0s5FXpVEReHlhRfggpkzvLN+Pajs3D0HW9WlFJnHzoTBTf/+1M8MX0FpnwaPsJp9ZbdAIBlG3c69rWZbpAoDY+qJJnXtil0jculqgQMHlboMbDZ9OGt3bYbd0/9vMjSRAdb6EwUxOV//sW/DFfL5HlrIxsglXXeLnNLm85CL/77wQo9BlRDdu3WygnLS3BtYSKE8lgDy09fL9vYlPO1VTIuF5uLpTWls9CL/4IUX4IKp5IG/9jlwkRBPsZz0BoYVWeyKmOhB1DobKFXJqreqyQdGEfYIrPnIRV6nO9GggjnPzAdL3/8ZV7XSbr40PUWevHfD1boMaB2JfPpVpYabKEzUZJLbZJV8LUF63yPm7ZkI6546oMcSski67x9ELSVo1z2TEqg0Y6MOAZFOSfnnkc+WRFlDVy1eRfaNFZypoyI65Xdh85RLnsobNQyjJ58343fvPKZ6z6dSyQXZFIthw9dG+VSfHVafAmYsoGtaSYK8hoUVVqBh99dive/2KA9TucSyQUZ/mhvfFo0DUY1W+gVivJco8whXmwKuYAAU7lka1H+78b0JZu021dsiiZs0S1kXtcD6NulPpIy84EVesxUkD4v6BJfTOWTy7thP6epuU173Pcem52DRAZbm1qxq8VIHyBdLvbghvlrtlm+n3zQ3rjmxCE5lxkVrNBjQH30lRSHzqsKMlEQZTXa2RJd3hbJ8Fum4Pi73wag1Hmf1/jro/qitqr46rT4ElQgqhKvHHXOPnQmKqTVGx77OXGlEVhtLrySDqbPUZ2kkuiNs0KPmVJ4yFHB+pyJkpx6r7ZzPJeCC8G6bbu129Mug6J2apKJkphzwgo9ZkrhIefCgi+3OUK12OXCREGU1UiXxjYs//lkLcb++nW8t9gZMSPfAb/3uLoqEWjOSV11vCqXFXrMlEAkU2jmr9mKU/7wDu55Y5FlexyDotxE7LlE4XKRE3y6NdSGvtb9b30BAJi+ZCMA4LO12x3H+DU+NWbseXUyEajHEfdCR6zQY6DcB0XXmDmnP1m91bI9bNjium27MW/VVu2+MvxZmIiIshFvNhe1qMkhMdZv/2NMTJIRLe1qko5jvFwuPTrU4oBeHYz9CNZApWLu5bJCZxzIbqZ9ckZY6+LoO97EV+99NyqxmArBbbJOLuxsMcIWq/OIMGlqdVfo8l3Q+djvv3CU5bvb/ai9B7sbM2pYocdAuWdblFbJ25835nWd5gparYmJniji0HeZFnp1HtPud5mNQn21U6FLg/qLRucKSaP6d7HJpr+h8w7rZ/m+tak1FzEDwQo9Bso926Jb5MDqLbvw8aothRWGqTjysVHt71NrW3CFrhvU37CjGa8tWJ+5hv2YtI+LpFN9NQDvXOh22e6c4p6DJl9YocdMOVroXtEsZ9z7XgElYSqR7Fyd8C+H00I33CVBokd01frCh2ZkPr+1cD32vW6yZb+bh0RmHv39uSNw42kHYmivjq7l1tjcQbrEXlHh+ysQUT8iepOIFhDRfCK6WnMMEdEfiWgxEX1MRIfGI275EbXPbEtTC2aYo/JxEbefj2GiYusuw30hLWUvdNb2EsWVIi11FbdBTPmOdGuoxaXjB3oGP1TbrPdkjCsbBbHQ2wD8rxDiQADjAFxBRENtx5wCYLD5dxmA+yOVsoyJ2o98yaOzcO4D0yNLD6rjmr9/FNu1GSajIiPQa1Khd6zzV+g61axGbuny/Ucx98JuoceZldFXoQshvhRCfGB+3g5gAYA+tsMmAnhMGEwH0JmIekUubZmgNtZd2vlXtDDIUEI/3x7DlDrRTP033oOO9VX645UTdO+MukmXoTEdQW/Vvnh0PgO4foS6MhENADASwAzbrj4AVirfV8Gp9EFElxHRbCKa3diYXwRFudAhgOUQBlkpy0mfe70UPPt0zyOOZ+7mclEbAF2xfpJEkVbdvpJRv73a5X9RFwIrdCJqAPAsgP8RQmyz79ac4vgphBAPCCFGCyFGd+/ePZykZYT6Y0RtScvLlZMe1PkhyzH6h4mWXCbduZ3SrkZvofvh17jY91961L6hy7C7cqLKP6MjkEInomoYyvxJIcRzmkNWAVCDLfsCWJO/eGVKAXRVIRabiMrV91uPpcKYPZfcXC76s+TAo1e0i8648tOt9nOuPn4wnrp0LBbfdor2+Du+PsyxzR7S6LUWar4EiXIhAA8DWCCEuNvlsBcBXGxGu4wDsFUI8WWEcpYtUfjgtNctgIWeTxmqZfPQu0sjkIapFDK9zAivKf3U1TZ/tdoLCNurFUJgZ7M133pNVQJHDOrmun7oObZJREB21TJpqcdpoQfppxwJ4CIA84horrntegD7AIAQYhKAyQBOBbAYQBOAb0cuaYSs374bBEL3DuET+oQlrkdX6oOiyzdGswTYnsxbC9ejf9f22Ldb+5yvsX77btRVJwNFgRSanHzpLma9nPrvNcEnbGn3vfUF/vB6NkHd0ft3zyTjCoP0oSeJkIKINULNV6ELId6FT+9IGE/miqiEipsxt70OAFh2+2mxlxWX4hUlPqu+NuY0oXsC33p0FoD86umY215Ht4YazL7xhKjEyhvpLozyzZChgEm7ha58DvsuPv/hasv3hy4enZPfP+NDJ8ONGUXKXzf4rYsB1c8XlyFdCAt97451OZ9b4h2IsmfDjmZs3x0sJ8iGHS0xSxOOTN2IzkDPhALaJ/Go+jcXl4uKLk49CPK8dFqgKpmI1UJnhR4D6oBlXGF5cSr0MQP2AgDUa7LPBaXQLqFX56/F/1bQhCi/gbPRt76GY+98qzDCxESUA/vS1WJXumo1DPsu2o/ONUigS/saAIbvvDpBjiymUcIKPQ6E9mOkxDkoKl+0fJRyoS30yx+fg2c/WFXYQiOmcXszrn32YzS3pTIpXb3YuNPb8o4zmiIf8gm9dXN5yO32STudlYl9YcuzBzTkurZBl3aGQj9m/+5IJCjWCLXcgjcZT9THFZsPPUaNKS+dj+ycDyY8t778Kf41dw0O368rxg3smte1LntsNr5o3BGRZNGS8aHnUEXc6r1UtXYLvUcHw2047YuNmdWNgrIsooF9AjDvphNRW5XE6FunxmrssEIPgBACW5paM10n/+Ozn+PSawUJW8zDwPtyq37RXUkpZKH8fN12rNu2G+MHl8YkN/WZNrX4W+heTPl0XZ7SxE8uVdjvHPuszK4NNbjh+Xl4csaKHEqLDjljnIhiNcbY5RKAv81aiZG/moqFmjUHdVh96PHIVAgf9eotu3I+9/wHp0coSTyc+Pv/4qKHZxZbDC0tZlK3XBq+h95ZErE00ZJ1ueRionvvtoct7mhuK7oyV59hguJdR5cVegD+u8jIO7N4fbAubD4DMUGJs1KUk7MklRYYcO3LxRYjcmQkRCIHjX7rywsCHZdKC3zvsdmYs3xz6DKiIJd65mfIqImwDhvQpSA9WT8sC94QxWqMsUIPQD6/f2yDojHU1L++v0y7dmIYPlyxGdsChtNFQZwhYIVGVd1yNmGMmVaxbttuTP10Ha586oP4CtGQiVoMWIU/Wb0Vr85faznXDdXlQkRI+fgNG2oL63VOULwBA6zQA5BZYSXgy2UZFI3JRJi/Zmuk11u1uQm/fHE+vvfY7Jx7FW2pNM66731ckKO7ZdXmJvxuysJQ5QcZfB1w7cv4+Quf5CRTsZARKrlGVgQhTuP1nUWNeMrm6mhNpZFOC+X5BpPg9HvexeWPzzHOUE4Z1KMh8/mQPp1QlSBcffzgzDaC/ySeiSN6O7Z964gBgeQKivURUqy9BlboMaAqpDDPbuOOZrz/xYZAx37/iQ8idedIQ2ZzU+6TUGRWxU9W25NxZvO4e3H543NwzxuLA7u2gOCz7h6fvjzwNYuNEMjEKhdi7DiOMi56eCauf35e5vuqzU0YfMMr+NvslWhJ5RHlorxR3RqyQQqd21Vj8a9PtQxwJ4h886bYF58AgIN6uy8nly9GB4JdLmVFrmGL5z0wHRc8OCOwoo4yyY+0IoRwr27fGNXXM/+NV+92d4C46iDH2Gm1FWr/7cox3/rs5Zsyg8q5+NALyY7mtkAutqUbjKXe/v3RmsyAb05RLi4n6Xoy1VX+szJ1uVm88sEE4cUrj8RLVx2lyAbL53yix/xghR6CwI9ZDVsMoXQXmZZp0FOiXt4O8Laaaqqcq6KruK2/CLi7DiwDyJljs9s+XLEZA659GStcYoLtFrpdhFIYFAuKvO8npmddFTofupebqdAN2CE3vYphN00BYERFvbVwPa56+sPMfimrND6M3kc683nDjmY88u5SV7lvfCFr5afT1ik5amOn+51qkuSbJK5WY6Hb88GEZVjfzji4TyftvgTxxKK8EELgH7NX4fThvXJPgh/yAajHt+QwzTeVFoHyRjS3pmIZ1NGujj5uHxDIU5mkPO413OBe9uC/zzZmf769qBEXde3vONI+WcQuQblPcNJZ6F4TZFoKPEis1pXT//gONjdZrfUdu9vQqV21pW60KIbItx6diU9Wb8NJB++NPp3rHddXG7fmtrRF8fdUcg3pfqcgS73pjol6zU/VmCHEa2RUvIU+fckm/PTZj3Hzi58WpXxZeZvbUpaK7EVQJRSlhe7Xs7/1zEOQIO/K6GWh2184bXGa0zPvlsu17Ra63cVV6mmG/dA9F89GNYS2iNqatytzAGhqbQOQbYSIoLhcRGa8JUhPdsOOZss7dNbI7CqXYRV6v72MxkOX1zzXJFxuqFczJhZFenkLFa/Qm1qMCtW4o7lgZaoPrLnN8AsfctMUjL51qud5sh6lhMD67bsxb5X3QGK0Ct0oXAj3/kgi4R1D66VMJv7pPXy6xjlYqiLPVt8n+aK6XdlhoQv7fv+3p7kthZv/PR9bNQqp2BARnpm5Ag8ri4R43ZPbM9ApzGz0Vnx+elmsKrPqcpG0pQXWbNnlOY4y/o43MzH2066bgKP3zw6A2m/h+AN7uir02846GHu1N8aCVJfLU98bi6cuHZu3D90LonjdYhWv0CVR/IiBwxYtCt2ovC1taWzb3eZzfaOAVErghLv/i6/e+67n8UEt/jAIwNUaTvhYF37W4Z/eXOxdtnlxSxfV/OhmwdmVm73B0cnUmkrjK3e9lfn+woer8eh7y3DXlIWe8lnKSYvIQ1J11StBwLXPzcOvXsr2ML1cW27PYOayTYFkmLFkY84TjXTuv0XrtuP2Vz7L9KSsFnqWtlQaR9z+Bi5+ZCYem7YMa3xmKddVWTOBqhb6Z786GZMuPNT13G+O7Z95dnXVxnX6dqnHEft1wxGDuuXtQ7djHxSNs89Y8Qo9CuMjdB5l5XMYpStFTQmBrbv8rUVp/UeB2uC56akE+XT3fX6o6iRh8A2TcY+yCoxFBs028tgHeIctptMCT2jCFRu3N2eiLoDsOIcq/7bdrZ7ZCkfdOhXH3PWm6/5c0N2J3oceXqHrHo2ut3XuA9Px9fvfd72+F7qVlb716CxMevsLSxqJloyFrlrtxueZSzfhF/+aj0lvf+FZlv1nUXt1ddVJVCUTmLvSvWFKZRS6oQLVehT1ZC51pmiCc7mUH+oDa25L4YMVwSwe+fIWw4eupjR1c6tUJROeA3J+Fmt1MoHWlMDvpn7uKYPd5wi4NzL2sDRV9rc/b8Sdr2atbtmdtzey8nklzbJ2t6Yw7KYp+MGT7jMoNze1YuUmbysyiAXfuL0Z5z0wDRtcXII6d4jXM3BrVHV+4agH53SDmhL1N9eFEtoHc/0yHdoXi9Y1fDuanT3iwwZ0AZCtJ/Wmha7+pm4LUUcBD4pGRC6/4cYdzdjsk3Par6zpSzbha/cFtHikDz3gE48ygiO7cK9wrXC1VYZCHnDty1imWLgSP3/1P+YEy1du76Ia8gVzuaiH2f2xPzQVtF15SMUrdZ7MdDg1j4yFH63cgoHXT8a0LzZ6Hvf4tGWYvmQTnpqxQqtGtu5y1j/vXol+u66nGuWAcUtbGv8xp+frUBVmWjEeJPZn9d/PG/GhhyFENs2lu78m2wLP3TvU4u5zRljKlkslqotORD2kYE3ORexyyQfZ2uZSd0fd+hpG/mqq8gDCPekeIRehVgdFgxBWoX/RuANXPPmB1g0UZJUldVbdR6u25C2Po1xNnuzMoKjLpb2iXOwLQLzx2XoATgtdii2t4Sjyw8w2/dDnPzjdM+1As1mW2wCebnUb3e98zd/m4tX5a13rjs6NEGXXf5OP4SPv48MVW7LlK89+l2YwdL7HILr9dnQ9mb57tbN8H963E/qZ2+TvJH3x6m+qe8vPHtXXVRY/bDP/OTlXXkTY2oYdFG0fMkZcNj5eg14qYWeKXvfsPLw870utC0i1mtzqmzqrTt+FDy6PbBz+NXcNZpkDdtKIU6+S9aG7WejucehuLin7dil3MkGYs3wTxv769ez1U+lAaQvsdFCevVfagdY2o+zqJCHolAVnr0TguQ9X4/LH57jWHZ1LwqvdWr7R2QPzwu/Zy0ZUzfNuCR7QKHTdtHxJkMicv37nMHSoyz4HtXGUvbJa0+ViacQ1l77rG8N9ywtCIuZR0cpX6Cb5/IbhG1SzsnhUSB3hLfRsJZyzfLOvZSkH7+0vX1sqjTPuMSJqhGa/DA9T70c7WBdiEpVsEOav2YZvTJpm2adajgnzuCnz12ktSsdMUeUncLNAnRZ61uViX+n9zlcX4vR73sWidf658FWfeUNdsMZcPrOaqoRvZkCJak3ubk1ZGii3uqNTgF5K+Ni73goki04mHboJT2rxuvEI3bR8SRDbqkeHOhx3QI/MdzWIQN67HBS1WuiRj4paPrKFngfZeSlxeq6syKKCzFRTyQ6KBnuxpTJbvH4Hvn7/+7jNJw+2m/tic1MrtpsDSLpB0QcvHgXAajHpFHqYimo/dL2StlfVDbKU2cs3Y/I8p4/Wbq3+5J8fYb/rJ2NLU4trXnCHD928xIPvLLXMTASyriW3eQxvf96YkX/g9ZPx9MwVWLW5KbAR0Kq4XPxWebKfAxhLq6lWr5ti1W2Xz0sXGhr2dfFT6Lr4cvWM2yY7n5WXhR5UPLUhUxu+rw43siz26mQM5EaZF8khgyPKJbai9gCFXoTkRvJ5JcLGP2UGRYMdLl+iXeYLPXOpd6xxIhMx4l2j7HtrTT+j+oLpXC6hZinaSpm7cos+rapSzBWavN320MKpn65DKi3w3mL3wUg3Cz0XLnnEWPFIKvzrnpuHo377Jhq3B1POsnFpaUtb/Ms6pIWp/s6tqTR2mo1xMkF4zmWhbN092jcF7Rnq0EWUqOjcX0IIzxBBr0Wwg/Z+1cs3t2Zl+PHx+2PeTSdiL3NZyetOOSCzL/KwRXtyLrbQS4Ogz1k+r7ATzvzCFu2x6dKqkMp1Z4vfxCXjv/3yaoXb3ZrCkka9/7RWmcyhq/R2K+eqCYNcZbHX6eqqhGXhg2dmrsDwm6e4ni9pdYu79rDh7Ap9rocinb7Ef0JOOi1wz+vWSVObAs46lX7dII3hkBv/AyGE5XduSwu8u3hD5hr3vaWP396lWaM0yCSsoLiFXUp0PnIB72ySboPJZ43sE7z3q9ZtxeWSSFBmnc9lt5+Gy4/ZL7Ova0O4YIYwEEe55E5rKo1r/jYXQH6rhLy2IFz4mlQmVSFnnFHGQtcLe9ljsy3f5XGyC756s3dcdMZC9wj187K0VJ9mEJfLkYO6uV7L0QtIJjJypIUxO3Lrrlbf6fhuk3+8dJPdes53QeUvGnc4Qva8JiVJhBD490drAGTD5/w48773LbMoVQu9XU3S7TRc/MhMDLj2ZcxduSWzzf4b5eN2+O1/vGfZageohT5V7cF9vPORV4ewlNR62rdLO48jswzq0YBnf3AEvjd+38DleKFK27i9Oa9wWD98axERPUJE64lI21wS0bFEtJWI5pp/v4hezNyYvWyzZ7ctbsLOIM5Y6C6tjz1UsM2m0P1eSGnJ2xVv0PEF1eXy7b/McsSi2xsir9dOb6ELh3zPzFrpKZPbPXvd0yqfhi8sOl+vXxgfYPXLBzU4Plq5xZKe9p1FG/APMyNlkK68uoC0w0K3DTBvaWoJ1DABwIIvvfP06PYLCK2l/aMJgx3bVMKMTck6eOLQnrjnvJGBzxvVvwtuOG0oxg/uhtu/dkjg87QyKI2K7MkEqR+5EOSX+QuAk32OeUcIMcL8uyV/saIhzrzDnuVKl4uPM+6s+97DH5Vp8PLol0yrDTB8yy99bHy3uwrk4Kk6yGd3ywhhLPm1o7ktE4dt14FBfad2xfWebXWlJh+Xj4p9are6FmSYGbBhImsAI7/6To37IR+OufMtxza/hggAvtyS7Snk6u7455xVWGhG4ARJM2HJP29X6LbvI26Zip8++3FOctnZsMOpwITQR7L4pblWFfrlRw/0jIaRRtKEA3qgU7vqoOJmePy7Y3HemH1Cn+dHHHmYgAAKXQjxXwDBMvuUGmrljUC5Bx1glSXJacVufLhiC+5WpsHLyz+kZNY780/v4cqnDIvMrYusVo5F67ZbFOt1z83DvtdNxsK1WQvpmr/NxfMfZgfPgioTu0Jft83qN122wXu6thdpkVU2On+vG27T4N0SoZ113/t4euYK7T4v4pgOrjZcUURZBLmEfNZbm1rxf/8wlPWqzbvQ1NKm/S2f+2C1Y1tUCOiNnjof95Pqnrru1APx+W2nuB7rNm5USHQ1xyt9Qz5E5UM/nIg+IqJXiOggt4OI6DIimk1EsxsbGyMq2h31GRYyLba0fEJPLPJoMJY0OtfZzPrQszd39qRpOP/BGZnvWUsxe+3tzW348d8+ynwPWrfslpC9O75O8U1/+PMTQkUYCZFNObDbJenYkYO64rf/+QznTJqGKfPXYumGna4WetSLQp//4PSc0kB4oTak6m+57PbTIi1HRbpZ7nljEZYoLrNfT16gbdirog75UHBb6NxuoQ/u0YCjlPGY9iEWqsmkhS5Sb92QwbktbM8yKFEo9A8A9BdCDAdwD4AX3A4UQjwghBgthBjdvXt3t8Miw0+Jh41ND3t82BWSvF6dCb9727FNVgp79+0jZeArCEHWhAT8LadtirunS/sajyOdqMpEFxEBGA3P/W99gZnLNuGyx+fgK3e9FZulo2PFpiZTjvAvo73xS6eFJeOjtNBvPsOwhw7Yu0OuYnoiRbe7VzZsb9EqmaC5wXOJDn5i+gqH5fzxTSc6Boin/Pho3H3u8MxgaJgJe6VgoeuIa2WpvBW6EGKbEGKH+XkygGoicg9vKCDW/CTO/WG7uUGPlmUdvl9Xx77G7YabQlVgc5YbHq2wCwLbo1x8pHLdIxck9qMmaXUh2a+4bVdwH7od41bMuHoXhT5PMwVfl+skLqQSzCWO+D1bkq6B10/G9c9n18uUz7LejFR58cqjsOCWk3H4QGcdCkpXTaMqjRJ7o5RMEMbf8abj+OqAI/u594AFRvXvkvnWsa46k/VSQkTo0aEO3xzb35Aph0HRgnbRHTI432s1Jj5K8lboRLQ3mf0aIhpjXtM7xVwE3D1lIV740Nu/ZxkA0ii0sN2esBb6IZqFYg+77TUA1mnIX7/fmPoe1sppSwvMWb4ZW5r8XQFeim+7z8IbEq+Ze4DT0g9zP2khMgNnu10quy6kMq6uq46MMsyhyEsemZkZG9BFOEjjQjbqNVUJ1Nckccohe+coLfCzUw7Aa9ccbdn2usvA+KrN+vGPOFfvAYx31N4bcQsmkI1emAY1k60zN/GiQXM7Ua5loOLrEyCipwEcC6AbEa0C8EsA1QAghJgE4GwAPyCiNgC7AJwnCjDP/o9vGBM5zlTWFbRjf/Czlm1C/67ZWNSWVBr18B64VAl6V36TLACn0tqwoznQeSprtuwKvBiBzuoVQvj6ua2pP237bMfaewphaoE9P0nw83K3dM4c0RsvzF3jf2CmLON/rjP9lm7YiaG9O2pX45H3Yf+Nw85lsJ5Lrr0+u4HTuZ3eReZmDW/e2YL6mqS5pFrOIkJAs1iFi0I/8aCeeHz68sDx5EB+2VbjJK4oF1+FLoQ432f/vQDujUyiAPitTSlRn2FaAN+YNM2i0MOmSQ1qmckcIl6q0t5Cj771tVCyAM4QRS90kSPfe2wOHrpktGO7uhi0eg8d661hX/afIx8/paokw1jdrWmB6iThN18bhp/84yP/ExQO6t0plEJvTaWRSoucI1Kkv3+b5rnZLXRJPhZyMkEe1q71u9v4yPrtzUinhUPJjvzVVIzq3wXViURof/Dvzx2eGZQ3pv9br+02Djt+cHe8f+0E9OpUF7gsea1C5nKyo2tT3dyK+VKWM0VP/eM7gY5TZ07Kl2i5shKKqtCPvP0N3PhC1qepIy2MxZuD4mX8hgnNcyNM3LeuPLcZsFXJBO44exgAa+RNXXUS//2/rziO/88nX+Jvs1Y4Xhp5apAX0KLQQyjMtlQaVYlETtEYYS3t3a0pHHvXmzj4l6+GLgvIur10IZXqmpsqYWZF2qlKJLSr2gNOBffqfPfZi79/Tb/C1Jzlm3MalD7+wJ4Y1b8L+napR1o4DR+7D12ld+f6UNFTfiteFQKdtMWcWFS23KiErulmjMpc1ACwessuR6Y9O7+ZvABjbnvd82GoL4pX7LLfgtFBCDMI+cLc4PHE1Yms5HY9qfMAfP+JD/CzZ+e5vjReS5NJVL0QdHYiYCj/qiTlFGUR9iXf3Zr2XXbOiymfGukBtmuiilJuFnoeLpdkgtDeJR1AmLZMNxgtSQtj4YgwVCcTGNS9Aa2ptNbtFzqpXQBKzOMS2wz2ilPoG3c0a10pupcobFdxjZnedKdHvpNF67Px4m5Kpi2V1na7w6IOQvpZqO8s2uC5X6W+pirzktkbJbUL//A7S/Gucl11oQiVIC9TKkcLfeqn67Cjuc3Xavu+knwpK1e2nA4B8pfnGyL557eX4NpnP8bCtc7c6m0ZH3p0yiyZoEzUjJ0wmRVVmTbuaLakDwCcKwMBwMMaV56kKkGoShJSaX10uJeFHhYZaTYsZKMTJbq6uTHkeFlQKkqht6bSGHXra7j+OafrRDedPNelxrxmVgbp+g+64ZXMdP58+ExRDAf19k5oFIYOdVUZNe5cXT27oSWVxoUPZycxpQUwZsBemHfTica5IcpUf1PZMNxpun28WLV5F4TwT3kqe05qb0HVaeeO7hdKxqAcNagbrvxKNuvkM7NWYs1W3aCotNCDl6ku3qAjlU67TosPFSmifP7pPz925JlXewF3nD0Ms2443jPtRdIcrE0LACL4oGgunHTQ3pj7ixNw2IC9IrtmWHR3c80JQ2Ipq6IUuhw5/rdGWerq7+WPz8FD7ywJPWDyu6mfY0tTCz5csdnhflFfFC9D4++zgy2YrKO+OomR+3S2lWs9JuhkIR3ta5MZ2b0Uuh0hBBrqqkJPqAKsjav0NQ/s3hD4fL+p+bJBP/ngbBigGovtlalQkotCf+LSsThhaE/LNl11a8340K33oZY5un8XHKo89z9981A88i13S7g1pY9iEkKEmtKvXmOLpmcpZ0Q31FbhnNH90L1DLcYN7IpTDt4bT39vHL57lDVrIRGZA+9Cn0I3Yv+IWwRPodC9Mm49p3wpO4Xu9VLJgaWg792KTU249eUFeH3B+lAy/PujNfjD64tw1n3v4xuTrGGDqrsgroH19rVVjtly9t/l24/Oyv36NVWZSmj34dotLzU2Pe2zYMFXh/fGDaceqN33qpJ+VobwhRkQ9HMhyKgiVXGrP5nMje1FruGK9t/slU+cKy8FsdCH9+uMy47Ouo6qEuQZEeT2roSN0rEs0KDZL6fiq+XVVSdx/4WjcPh+XTGoh9EwD+jaLpPWgIiQTgukhXBcs1O7avzqzINDycgYlJ1Ct8dvquF/0ice1uJuyiGEqM5MvPWFbTEI9QWLa1mrVDrtiA+2K5s55qrzuVBTlchYvHZlZPdvqi6mdNo9H82y20/DH88b4dqdVpeXU0P4Xv7RUZj646O156ioS9jpOHqwkWriK4qbQv3NgvjQc52tHWRmo+yh2H9vtaG65PABFssumSDPa8vfcfKPxlu3+4SFXnb0QMt3VSLd45UWuluDl9REmsil2ITpcnnzJ8filauzcl40rr+njOVEHInd3Cg7hW6P355473uZeGw5sNSaEr4LI6jk8nN3qtdbdG4Jl6IklRaOF3+cbYp4EBeCG8lENmrEbiXbgy7U9SzdLHTZwAYNN5MKJ5kgHNS7Ewb39M9r4hcGNnZgVyz9zak4dJ/sNHNV/dhj7CUPXjwaN55m9CpyHXPxS6MMuEe5SC45vD/26doODbXZ50pEOGb/7rjulAO04aSy/g21ja+0KoO79uLOH7MPTjrIOjtVlUmnnNqbMrkpdNmIq/uzLhfDLbRvt/Y4sFd040ClRCFXwSxDhW59qT5buz2zVJkahjj6tqmBr5nLD77FpcFQLaqo84z87GRj3cNUWlgs45+dfACuO/UAy7H11UksXu/M0LiPLSJBF5+uWuEOC91DORmDk8rL76HcvbAvrRcEtwZWYjRS1uup53R0cbmkhcBZ5mzk531STbgRxHUk79n+m501sg++dcQAXHOiMYhWZ0vJnEgQLj9mP+39u8WgtyrvkL0BOWd0X4cxsGj99mw90dxKO43LxSKHVOjK/kSCMimTi7Dsb8VSfgrdI6lNi2ZgLQi5hIrZF2iQqJU66ooqky212Sz0fbu1c0QzJBKE4+9+23ENexTO32c7F2JQlZ/dh+71Wy1ctz3UwKFbfHoqhxA++8CbA41Y6oSnjvV6l0ta+a3tKQm+feSAQLLlY6HXVSdx0xkHZRS2W6bBpK3R+N74fXHmiN7aY3+mLFphl6wqkbDMpgYMt+LN/54PQN/r9Mv7n7XQs9vkYslCuLsk7v/moXjmsnGe1y43zh/TD18ZEl+m2fJT6B5JbfxWvXdDrU7vLDLytOc6VVi6C04c2jPyFKjSz5sWwqJo08LpznALn7TrSJ3VaYSVZT/b93kRdH3O8w7rh1sm6lPnqy6XoFQlE67Jw44a1A0NGh/5CUN7Yr/u7QEgM3BnJyWy097tyla1ZJ/74RGusql+7iEu7iO3qf92OtXrIzbsYxs/Om6wq4X+mhIE4AwZNCxu+7ORa5HqJsTV13irkawPXXW5mD50CFfD55RDejlcieWIen+/+dowPPrtMbGVVXYK3S0THwBLOtJcuejhmQBynyosLa1Lxw8MNUXZjk6XtavNdm1VZadre9yUoV2mfppJIYkEZawme8OQ76QPKWtDbZWrjBmXi1LWrR5RD7LnUuuiwJ64dKy2rOpkAq//77FYdvtpaFdThb9ffrjjmEE9GjJy2McP1PDMQ/fpgse+o39R1bKvOXF/7THSDeLXhnXvUKtNi2uXLWhj6DZx7OLDB1jlMy3zYbYMot0aajNRLm71XT4W66CoaqEzUVF2Cj2OtJN2ffjRyi05r/EofehhrEsdVx/nfPHrTAs0LaxJm3SDUUHX5dS5dxOUHRS1J4fKd9KHnBtIZFUAZ43sg19+dSiAbISOqqQuHNcfPzjWOdsTAGbecDwA//S+fuzV3uqH/ugXJ+KAvTtmnqVd+dl9zW7PXJXLreck63UQI2DG9cdhwS3WZX7DuMYs2A5zk082snYLffp1E3zrupRF2Cx0GYde6T50jnLxIMwCwnZOdcktbVeIE//0XqAc4zqk/zeOpbvcFJau6ZELadixS6VLf7CrJZU5LplHLhGJTj6ibDUf0rMD7j5nOEaaESirzfSydkXxkxOH4M8XjXJcSx531QRjRqbXtHMvHIOm5qLCUiHZw1DtvmM3xaQep1N+CQJ2tQaz0AHDvWSfmGI/L7iFbpdFf540cNZus85yrUomlDQReqQs1kl3clDUmW2x0uAoFw/ysdAH99D7L3W5ib1cO17k4v+1M+nCUZZK0K2hFgAseaDVhi2Mv99euaaZK+lIZQgAO5pbXcMWw+E8VxVV7QWoCl5if9GTCXIsGjKwW/vM528duS8+v/UUjB+c26CTWtq+ynXls1zwpTVtsz0G3M0SU49TLenLjh6If11xJLq0q8EuM3Nmrj0ge2MU1DVmP8yt3sp3ZPNOZ3SXr4WuGRSVp+iyLTK5U34KPY+lm9zquFah59hwSEsm1zzWT146FicfvLelkl993CDMu+lEdO9Qm9mmhoBJJfn+tRNCl7fZDL88Z3Q/vHjlkQCArg21kK+Z18vqpzMGmVP3L1cmqkipCWpXXH89neWm/q4vXXWUYzCypirh2wiNH9wN3zpigGO7qhRfv+YYRQ79deydlyA6VD3n+lMPxPB+nVFblcjkx46qY2dvGD74+QmO6BXA2Qi5DaSu396Mm/89X+uK9JNZG7ZomX5a2Sq9kHcXPulGkenVuR5fP7Qv5q3egs/XOeOsvXCzoHRuhzCr5qjoBvSCcsfZw3Cksrq5pCqZyExN792pDpcfsx/eW2xkOaxJJjLLlPX2SVN70kE9sWyDfqmxZIIwrG9n3HvBSIwf3B3Tl2zM6T7UBqBTu2r3FezJv7uva0zU8MyDNUv8Af5+6Me/O9ZNpKwsStlu17PXpyAdJV063LrqZGa1qnwG0iUvXXWUY9teLot224tr5xGC+Oh7yyxGhQyL9HOZ6KNcFBk8zy5/onimQSk7C31Ev8743TnDHbMHn//QP9mVmyWhs9BzXYDiwxVbAGSV0Z8uODTwuaqyUuuAqtjev+44XHLEgMzLccfZw1BblX0JdVaY5DdfG+ZqDEkr6vRhvdGpvlrJhx6uMvodnbHGdRrdhq4xcbMgoyDse+fIKWXT6Ccf5Byz0TZSVYmMiy8Kf7JbQycnpqnYy9Mljdq7YzZeP22uEAVkx3TkNdxE18ehqw2m/rxKoZC3V3YKXfLrMw9BD8VakEtaeeFWcXQDrbtzHHx95L2lALKW2GnDevlOvJBUWxR6Vlj9AKv1pZJM/fEx2sknfTrXu1ppgFPR5GpV+J2mZsC2W7i6mGg7cQw2Z8rLU7OoCqtdTRKTlAHcP5w3Ak+6hE/WViczvcQYbw+nHtILnc2BXjlGYC9OV3fOH7NP5vPGnS2Zeio7tn4+dLk/JfQul0ofFC0kZavQO7WrxiUaP6gXbkrqzlcXOrb9yVyEOgg694x95l4Q3Hy/+hhqZzcWMBS8fXq4ittvYLd8yfEhGEHHZ4mU9R7NbUFe7CCJrgpFKi3w0lVH4c2fHAvAFsVhO3biiD44clA3bYNUozz3fEMv/ZB+7Acv1kcC6eqHfTyoJhM+a80Q6ebS1On7PcvlUriySuftyIH8IjC8mbks+KzTUb8y8sZsVhJEqe4C/bosTtwGUt0mxQD6hFG6qJcGc1JSf81EIsBp+VLmJQ2H351mXS5O5eHw52ryqgeNHvqJywQeL8K+eGlhuDektWufCalDJ7+qCN0WpIgKKaFMqDW8X2ffc+y30tRsGDCZdAU+zyTznG3ZFt2uX2mwDz0gQQL2H/vOGAzv2wkJ8rcAZXhgWHa2pPDsnFUY+atsQrBcwhbd/Ipaq860ktSEZG6MH9wND5sLIdz5jWE4XDOd2ulyccoRBL+c4aP7dwFgZIfMvOcyG6PyPC8ct09eoZ9XThiM+755KG4+Q59eQEfYF89+r5ZvPmMV1nKzn+O20KUfvVtDLZ7/4RG4/8LsGI/b+qP2dLtynOYUc7GQhE/rr3UYqhZ6pWv0AlJ2US4qOuUx5cdHY+qn6zJulNqqBP51pTHq75ZQS9KtoSYTbRCWNz6zLpKhvriqmDXJBH503CCMHrAXvvnQDGsyL+V8VbnpJvfIxFYyF7WK/Vf53viBmRj2DnXVmDiiN6aZUSw6edXyw85y83O5jB3YFZ/cfBIaaqsw29YLUkWIwrVy6iG9Qh0fVq2kbSF8as/ILdWuzpqNSqFffHh/3+isC8f1x4VmrvGRSiphAJhvm30qsSfk6t+1HaYqYZ1+kVA6hc1+83goa4Wu8xX37VKPZqVS63KVuFEbcPBSx84W65Ro1YeuvvZXHz8YV5hrTCYTZMvO6OLf1iiBKycMQv+u7fSzX5UCf3/ucIwfbA2FPGFoT9zx6kJLDnGHNRzgfct1RSbp/vFyuTRoGqq4ycXlYvmu6D23iWk6RaY2mrXJ3OvgLRNzX+VHl4t8cI8GpIRAS8recFmP83PPZcdK9GGLrNyjo6wVui56pL46mRlN/8mJ+1tis+2Ksa46YXnxqvPo4r+1sNHy3VKWZnYkYCSTUkMmLRa68sXNh/61Q/v6ynXyQb0cirNrQy3e+9kEHPiL/yjl2S10pxx27A1SWJzXzm7o1ck9pv67R+2LwwZ0cd2fK2EVi72HGGSJOl0JagcsbpeLjkW3naKVS1rhv3rpU8t2+336ucZ0vTxrnH9AQRlfytqHrrO+iSgTTmXv3n5zbH/rsbaKVlvtHbIVBjfloG6vtr281pVhsoQN1fuhssq8m4LwfQl9YouB/FZFAnQTi7Kfuza4h1j+/PShOPngcO6UXOTxw6nQ/c/R1Qu1US+GQq9OJjzj++0uF/tt+sWh67aTS11n8sO39hDRI0S0nog+cdlPRPRHIlpMRB8TUfCZNHly+H5dcdtZzm6mfNHsvr36miR+pOQsses0XYTBjacdiB8fHz5iwuJDh95Pbo/ScXshwg4O/uDY/fD+tRPwh/NGuJ7rb1XJ/+7H5esWcbpclMYuxggmV8K6XDx86KP663sQftZovlk648DP5eKXvy07+K2cQ879TP4EMQf+AkA/WmJwCoDB5t9lAO7PX6zgnDmiT+azjAeWboCgaT0l6oxLSfvaKlypNALP/sB9IQMVt3zlqgvImdxJ+ezjcvGjd+d6TFR+GztBL6l72f7y7cPw4pVHagdkw+BloRfDrxp2ANhukfcyn+0FY/dxzY1ejsrL3hOz90wyFrrL76d1ufDEoljwVehCiP8C8ArKngjgMWEwHUBnIoq+P+yCVCrHDumeiQf2W3BXcvzQnpbvbt1d9Sr799SvbOM4xxKHbvDni0bh9GHZn8bRI1CtFkuUS/QVPp9QsWOH9MCwvp1x3SnOqeRhcCTnivme/fD6SV644kjHNrtiG9GvM1666ijcOvFg18auHEP0rjnBu4fqF+UiLXj11yrBjkhFEIXDrg8AdWHKVeY2B0R0GRHNJqLZjY2NukNyYvaNx2PShdlp1kEs9PMO64fffn2YZZvbeo2WgUyNFR+UY4d0t7zQ9gbEbbKFLqFT3AQZ6jzuwJ6WNLMAcMVX9ItQ6LDrAUuvpAiKz8sAGKGZgKMbBD24TyfPiTa6PbIhu1IZ+ygl2tdW4VJlzVZXC93Nh665a+ucC9buURGFptA9Da0+EEI8IIQYLYQY3b17dAuldmuotYQwyigXrxdr7051DoXqaqHbfLtj9t0rlHxygV+7kpp04ShcfHj/zIQON2njslbtucVVcl1T9f9Oyt1qV3+efFdGyqn8kMfnEuCjazTkGIvMs1KKqIOmufrQLefwoGgsRKHQVwHop3zvC2BNBNfNGeEyKOpHkAgDItKuPenFP79/OG4+4yBHJMGAbu1xy8SDM8rLPc9KPFX+iUv1aWQBJW95jNaT00IvXZeLjlxCNr3KKGVLtdpjyUM/12YmDt0tfW7p3nbZEYVCfxHAxWa0yzgAW4UQX0Zw3ZzJulzCnRcmZCxMKOHA7g2BEom5TYeOS7l5XlfJuRIXGR+6XGdUs6+QBB0U/c6Rhvshl16MNoQvMyu3dFHdfuFv23lnqnHDg6LRESRs8WkA0wAMIaJVRPRdIvo+EX3fPGQygCUAFgN4EMAPY5M2IJk4dH0ArOXr49/NRiOEUdJxKFm3S8aVsClID8bvkHx+BUe63CJb6EFv5qyRxhDRVw7okUMRHv71EtZrahW0K3RpCF00zjrPQ6K7rw7KoHEp33e54Rt3JoQ432e/AHBFZBJFwLFDuuPZD1a5JvpXGT+4O9rVJNHUkgplKVQlCLllffFC71dUV4mJtDRPAz2YGXb+mH1w2+QFuZUPW5RLkQdFgxZ5SN9O7isx+eDVTpWypar2GHt2qrPsq04msPi2U1wb4WxPLEuHOkWhRyfmHk9ZzxR146vDe+PTW07S5qcY3tdQ8sP6ZpV9dpQ+eNXq5bHcm269yiBYXS7Zz175zfPBywoWAV0ul47fF0t/c2pO5Tt86MrnIgT2FEShetWxEtbnmd9m7451+LVmMl9VMuGxVJ+TQ9XEYKV842VGRSp0QJ9LGzBC7d6/dgImHJCNQZddxjBW4ePfHeNqOf/89KEhJFVnZTrpEZN1Dnjfb9Zq9v5NiCjvwTzdAGxRBkULUIbXbZWyWpNynzGid2Z92+DnWntigBHF9NXhxpqkpXzf5UbFKnQv7IspZ7P/Bb9Gr071OHd0P+2+XJWRLpSrR8f4FLqMrjn+wJ6OfRklG1vpzmuXi8slrzJ0v2imRS9d1Sbrpj3dQRDcbkuODZWyq6ncKOtsi1EhFfrO5jafI61Y3SIJ15SpuVxPWqtd2rknqYqCGdcf5xn/XMh3zepyKV6US5z3TB4mVCnPntQt9JwvMhleKtc8zIyDPdJCt/O1Q42oBa9FlHVYM8bl/zaq15CXzjejoR89O9ZpZ7/mOrEoH4qd36MwFrrXvtLV6LKxCZIi2HGuS0tVZ9a7Zp9FOZjgsIUOI7/2OYf1wxsL1vsfrKBW0y7tqrFPXTssXLc9Zzl0CkWX870QuC93rOfEoT2xuanF/0BdWXIJuiK7XAqBV0NVyrc8pGcHAMDwfv6RY3bcbkta6M1t+fVsmSys0GFY2h3rqtFvL8O3Xl+dxK4AVoN8ATvUVeHpy8ahW0Mttu8O57aR5Tu2mf/jinDxQxdK6MUDLqvIe+GMclEs9IqNctFss/0vRY4Y1A3TrpuAHh3q/A+24fa7Sgvdb9k8Jjis0BVG9d8Lr/7P0ejduQ6H3DTF9/ieHY3K/dOThqB/VyNJVT4pZdWKL3NQF0uhQzN7M25U/3I5TP3PqQzNLyp7Q6U+OOi1ipQXbrclx262NLXmKhJjgxW6jSF7d8CulmAWw7mj+6FDXRVOjWj1HLXiS6ulPmYfuh/FGhQtSpRLIcrwdqJXJG63JRX6phxddYwTVugaguqSRIJw+rDesZQrlyWzL4JRieji0IsS5ZJZqCHOMjTbbP8rDbd5Cgf1Nvzxo11Wd2LCwwpdQ6G7vrI4tTuea8bIqChMkIv13lQdXpx86Mb/G04LNzEsXBleg6KVqdLdbmtQjwbMuuF4dPNYP5YJByt0DYX230rlqRab1mwrJNmJRYUTQC0rrpTBnuUT5ZyjJXAZHvtKOQ49H7xuK648RXsqld+fz4FivVhkUejOUL5CEjbKJb/CnGXluwB1qVKuYYv5UOqDvZUEK3QNsuvbxyMBV7TlZT5lthVigQkvMjnK45w1SbIs63fjc2UqAa+MzqU8sSgfKvRRliSVaQZFwMOXjA6UfjdKdBZ6sa2bOJWMI5dLhSo0lXLNtpgPldo4+/Hq/xyd10TDXGCF7sJxmoRVcaMqb51fvZB0bzB8mwO7t/c5Mjr20Pc+Q6Uqvgq9LV+G7N0BQ/buUNAyWaGXEGq9l1ntivUyjB3YFU9eOhZjQy6InQ/F7o0Um0q9+0q9r1KEFXoJYXW5GP+LqeSOHNStIOVkcrkUpLTSpVLbsz29oS4kPChaAmQnlmQrfjbKpXJfBvu9VfCtepJdJLoyf4A99bkWA1boJYSu4ldqbDIAtK810hqM6NcZQGU3Xl7IiKJKfdaV2lCVIuxyKSFUfZZKl0aUS5z06FCHl646CoN6NBRblED89OQhWL8t+qXBJZX6qCv1vkoRVugBOWd0X+zfM94Ra9VCzYYtxlpk0Sl0aGg+/PDYQbFclxSnWyXCCr1wsEIPyB1nD4/t2rqkUMWeWMQUnkp91JXcyyw12IdeAgjNJCLdKj5MZVOpiq8y76o0YYVeQljCFs1VuSr1JWecVOqT5l5m4WCFXgLoXC57ig+dUXK5VOiz5jpcOFihlxJKxWcf+p5HpT5qrsOFI5BCJ6KTiWghES0moms1+48loq1ENNf8+0X0olYuXhOL2OWy58CKj8kX3ygXIkoC+BOAEwCsAjCLiF4UQnxqO/QdIcTpMchY8ejSxxY7ORdTePhRM/kSxEIfA2CxEGKJEKIFwDMAJsYrFlPsBS6YwlHoFbKKxVkj+xRbhIonSBx6HwArle+rAIzVHHc4EX0EYA2Anwgh5tsPIKLLAFwGAPvss094aSsU3etcCsm5mPhQV2SSC4HL2cGVyKe3nITaqmSxxah4gih0nUax17wPAPQXQuwgolMBvABgsOMkIR4A8AAAjB49unJrb46oCzPvCcm59lQm/2g8unXILowsLfS2Clbo7Wp4DmMhCOJyWQWgn/K9LwwrPIMQYpsQYof5eTKAaiIqTO7VSmUP9qF3qND1RCVDe3dEjw51me9V5kOuZAudKQxB3pxZAAYT0b4AVgM4D8AF6gFEtDeAdUIIQURjYDQUG6MWdk9iT41yefYHR6Bvl8Ks5VoqSAu9NZUusiRMueOr0IUQbUR0JYBXASQBPCKEmE9E3zf3TwJwNoAfEFEbgF0AzhNCsLkREJ3OvumMg9C+tgrHHdij8AIVkVH9uxRbhILDFjoTFYH6tqYbZbJt2yTl870A7o1WtD2bnh3rcNc34ksIxpQOyYTh+axkHzpTGHimaAlw7wWHYvzgbtirfY3/wUzFwRY6ExWVPfpUJowb2BXjBnYtthhMkahKVn6UC1MY2EJnmCIjLfQ2HhRl8oQVOsMUmXozRnvPimdi4oBdLgxTZK6aMAhCCJw3hmdPM/nBCp1hikz72ipcd+qBxRaDqQDY5cIwDFMhsEJnGIapEFihMwzDVAis0BmGYSoEVugMwzAVAit0hmGYCoEVOsMwTIXACp1hGKZCoGKlLSeiRgDLczy9G4ANEYpTDPgeik+5yw/wPZQKhbyH/kKI7rodRVPo+UBEs4UQo4stRz7wPRSfcpcf4HsoFUrlHtjlwjAMUyGwQmcYhqkQylWhP1BsASKA76H4lLv8AN9DqVAS91CWPnSGYRjGSbla6AzDMIwNVugMwzAVQtkpdCI6mYgWEtFiIrq22PLoIKJ+RPQmES0govlEdLW5fS8imkpEi8z/XZRzrjPvaSERnVQ86a0QUZKIPiSil8zvZXUPRNSZiP5JRJ+Zz+PwcroHIvqxWYc+IaKniaiu1OUnokeIaD0RfaJsCy0zEY0ionnmvj8SUcFW6XO5hzvNevQxET1PRJ1L7h6EEGXzByAJ4AsAAwHUAPgIwNBiy6WRsxeAQ83PHQB8DmAogDsAXGtuvxbAb83PQ817qQWwr3mPyWLfhynbNQCeAvCS+b2s7gHAXwFcan6uAdC5XO4BQB8ASwHUm9//DuBbpS4/gKMBHArgE2VbaJkBzARwOIzlVl8BcEqR7+FEAFXm59+W4j2Um4U+BsBiIcQSIUQLgGcATCyyTA6EEF8KIT4wP28HsADGyzkRhoKB+f9M8/NEAM8IIZqFEEsBLIZxr0WFiPoCOA3AQ8rmsrkHIuoI48V8GACEEC1CiC0oo3uAsUxkPRFVAWgHYA1KXH4hxH8BbLJtDiUzEfUC0FEIMU0YmvEx5ZzY0d2DEGKKEKLN/DodQF/zc8ncQ7kp9D4AVirfV5nbShYiGgBgJIAZAHoKIb4EDKUPoId5WKne1/8D8FMAaWVbOd3DQACNAB413UYPEVF7lMk9CCFWA7gLwAoAXwLYKoSYgjKR30ZYmfuYn+3bS4XvwLC4gRK6h3JT6Dr/U8nGXRJRA4BnAfyPEGKb16GabUW9LyI6HcB6IcScoKdothX72VTB6DbfL4QYCWAnjO6+GyV1D6afeSKMbnxvAO2J6EKvUzTbiv0M/HCTuWTvhYhuANAG4Em5SXNYUe6h3BT6KgD9lO99YXRBSw4iqoahzJ8UQjxnbl5ndsNg/l9vbi/F+zoSwBlEtAyGa2sCET2B8rqHVQBWCSFmmN//CUPBl8s9HA9gqRCiUQjRCuA5AEegfORXCSvzKmRdGur2okJElwA4HcA3TTcKUEL3UG4KfRaAwUS0LxHVADgPwItFlsmBOZL9MIAFQoi7lV0vArjE/HwJgH8p288joloi2hfAYBiDKUVDCHGdEKKvEGIAjN/5DSHEhSive1gLYCURDTE3HQfgU5TPPawAMI6I2pl16jgY4zHlIr9KKJlNt8x2Ihpn3vvFyjlFgYhOBvAzAGcIIZqUXaVzD4UaNY7qD8CpMKJGvgBwQ7HlcZHxKBhdq48BzDX/TgXQFcDrABaZ//dSzrnBvKeFKOBofsD7ORbZKJeyugcAIwDMNp/FCwC6lNM9ALgZwGcAPgHwOIxIipKWH8DTMHz+rTCs1O/mIjOA0eZ9fwHgXpgz24t4D4th+MrlOz2p1O6Bp/4zDMNUCOXmcmEYhmFcYIXOMAxTIbBCZxiGqRBYoTMMw1QIrNAZhmEqBFboDMMwFQIrdIZhmArh/wOhOA//SqUCmwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Smarket.Volume)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          Generalized Linear Model Regression Results                           \n",
      "================================================================================================\n",
      "Dep. Variable:     ['Direction[Down]', 'Direction[Up]']   No. Observations:                 1250\n",
      "Model:                                              GLM   Df Residuals:                     1243\n",
      "Model Family:                                  Binomial   Df Model:                            6\n",
      "Link Function:                                    logit   Scale:                          1.0000\n",
      "Method:                                            IRLS   Log-Likelihood:                -863.79\n",
      "Date:                                  Wed, 28 Apr 2021   Deviance:                       1727.6\n",
      "Time:                                          10:48:29   Pearson chi2:                 1.25e+03\n",
      "No. Iterations:                                       4                                         \n",
      "Covariance Type:                              nonrobust                                         \n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "Intercept      0.1260      0.241      0.523      0.601      -0.346       0.598\n",
      "Lag1           0.0731      0.050      1.457      0.145      -0.025       0.171\n",
      "Lag2           0.0423      0.050      0.845      0.398      -0.056       0.140\n",
      "Lag3          -0.0111      0.050     -0.222      0.824      -0.109       0.087\n",
      "Lag4          -0.0094      0.050     -0.187      0.851      -0.107       0.089\n",
      "Lag5          -0.0103      0.050     -0.208      0.835      -0.107       0.087\n",
      "Volume        -0.1354      0.158     -0.855      0.392      -0.446       0.175\n",
      "==============================================================================\n"
     ]
    }
   ],
   "source": [
    "model = smf.glm('Direction ~ Lag1+Lag2+Lag3+Lag4+Lag5+Volume', Smarket,family=sm.families.Binomial())\n",
    "result = model.fit()\n",
    "print(result.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coefficeients\n",
      "Intercept    0.126000\n",
      "Lag1         0.073074\n",
      "Lag2         0.042301\n",
      "Lag3        -0.011085\n",
      "Lag4        -0.009359\n",
      "Lag5        -0.010313\n",
      "Volume      -0.135441\n",
      "dtype: float64\n",
      "\n",
      "p-Values\n",
      "Intercept    0.600700\n",
      "Lag1         0.145232\n",
      "Lag2         0.398352\n",
      "Lag3         0.824334\n",
      "Lag4         0.851445\n",
      "Lag5         0.834998\n",
      "Volume       0.392404\n",
      "dtype: float64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Coefficeients\")\n",
    "print(result.params)\n",
    "print()\n",
    "print(\"p-Values\")\n",
    "print(result.pvalues)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.49291587 0.51853212 0.51886117 0.48477764 0.48921884 0.49304354\n",
      " 0.50734913 0.49077084 0.48238647 0.51116222]\n"
     ]
    }
   ],
   "source": [
    "predictions = result.predict()\n",
    "print(predictions[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Direction[Down]', 'Direction[Up]']\n"
     ]
    }
   ],
   "source": [
    "print(result.model.endog_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_nominal = [ \"Up\" if x < 0.5 else \"Down\" for x in predictions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[145 141]\n",
      " [457 507]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, accuracy_score\n",
    "print(confusion_matrix(predictions_nominal,Smarket[\"Direction\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5216\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(Smarket[\"Direction\"],predictions_nominal))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = Smarket[Smarket.Year < 2005][:]\n",
    "y_train = Smarket[Smarket.Year < 2005]['Direction']\n",
    "\n",
    "x_test = Smarket[Smarket.Year >= 2005][:]\n",
    "y_test = Smarket[Smarket.Year >= 2005]['Direction']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          Generalized Linear Model Regression Results                           \n",
      "================================================================================================\n",
      "Dep. Variable:     ['Direction[Down]', 'Direction[Up]']   No. Observations:                  998\n",
      "Model:                                              GLM   Df Residuals:                      991\n",
      "Model Family:                                  Binomial   Df Model:                            6\n",
      "Link Function:                                    logit   Scale:                          1.0000\n",
      "Method:                                            IRLS   Log-Likelihood:                -690.55\n",
      "Date:                                  Wed, 28 Apr 2021   Deviance:                       1381.1\n",
      "Time:                                          10:48:29   Pearson chi2:                     998.\n",
      "No. Iterations:                                       4                                         \n",
      "Covariance Type:                              nonrobust                                         \n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "Intercept     -0.1912      0.334     -0.573      0.567      -0.845       0.463\n",
      "Lag1           0.0542      0.052      1.046      0.295      -0.047       0.156\n",
      "Lag2           0.0458      0.052      0.884      0.377      -0.056       0.147\n",
      "Lag3          -0.0072      0.052     -0.139      0.889      -0.108       0.094\n",
      "Lag4          -0.0064      0.052     -0.125      0.901      -0.108       0.095\n",
      "Lag5           0.0042      0.051      0.083      0.934      -0.096       0.104\n",
      "Volume         0.1163      0.240      0.485      0.628      -0.353       0.586\n",
      "==============================================================================\n"
     ]
    }
   ],
   "source": [
    "model = smf.glm('Direction ~ Lag1+Lag2+Lag3+Lag4+Lag5+Volume', x_train,family=sm.families.Binomial())\n",
    "result = model.fit()\n",
    "print(result.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[77 97]\n",
      " [34 44]]\n"
     ]
    }
   ],
   "source": [
    "predictions = result.predict(x_test)\n",
    "predictions_nominal = [ \"Up\" if x < 0.5 else \"Down\" for x in predictions]\n",
    "print(confusion_matrix(predictions_nominal, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4801587301587302\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_test,predictions_nominal))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                          Generalized Linear Model Regression Results                           \n",
      "================================================================================================\n",
      "Dep. Variable:     ['Direction[Down]', 'Direction[Up]']   No. Observations:                  998\n",
      "Model:                                              GLM   Df Residuals:                      995\n",
      "Model Family:                                  Binomial   Df Model:                            2\n",
      "Link Function:                                    logit   Scale:                          1.0000\n",
      "Method:                                            IRLS   Log-Likelihood:                -690.70\n",
      "Date:                                  Wed, 28 Apr 2021   Deviance:                       1381.4\n",
      "Time:                                          10:48:29   Pearson chi2:                     998.\n",
      "No. Iterations:                                       4                                         \n",
      "Covariance Type:                              nonrobust                                         \n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "Intercept     -0.0322      0.063     -0.508      0.611      -0.156       0.092\n",
      "Lag1           0.0556      0.052      1.076      0.282      -0.046       0.157\n",
      "Lag2           0.0445      0.052      0.861      0.389      -0.057       0.146\n",
      "==============================================================================\n"
     ]
    }
   ],
   "source": [
    "model = smf.glm('Direction ~ Lag1+Lag2', x_train,family=sm.families.Binomial())\n",
    "result = model.fit()\n",
    "print(result.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 35  35]\n",
      " [ 76 106]]\n"
     ]
    }
   ],
   "source": [
    "predictions = result.predict(x_test)\n",
    "predictions_nominal = [ \"Up\" if x < 0.5 else \"Down\" for x in predictions]\n",
    "print(confusion_matrix(predictions_nominal, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5595238095238095\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_test,predictions_nominal))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.479146\n",
       "1    0.496094\n",
       "dtype: float64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions=result.predict(pd.DataFrame([[1.2, 1.1], [1.5, -0.8]], columns=['Lag1', 'Lag2']))\n",
    "1-predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Discriminant Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train.loc[:,'Lag1':'Lag2']\n",
    "\n",
    "\n",
    "x_test = x_test.loc[:,'Lag1':'Lag2']\n",
    "\n",
    "\n",
    "model=lda()\n",
    "result=model.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.49198397, 0.50801603])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.priors_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.04279022,  0.03389409],\n",
       "       [-0.03954635, -0.03132544]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.means_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.64201904],\n",
       "       [-0.51352928]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.scalings_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAADQCAYAAABStPXYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAO/klEQVR4nO3df/BldV3H8edLMBzFBo0VEVCYJA0TV9uhFEGUUhQSTc3FMjLHdRoc0Ulr1TGshtp+GFZYDgrD+iOUJhEKUxF/QNOou/xQQUR3YJMFYhdtzHWU2uXdH/d848uXu3x/3O/d8/ne+3zM3Lnnfu6557zvd89nX+ece+7npqqQJKk1D+m7AEmShjGgJElNMqAkSU0yoCRJTTKgJElNMqAkSU0yoDSSJL+V5Nw5bV9IsqavmqS9JckJSb6f5LokNye5Kskpfdc1KfbtuwDNL8m+VbWr7zqkPqyA7f/qqjoFIMlq4BNJflRVV/Zb1srnEVTPkrwzyTeTXJHkoiRv6dq/kORPknwRODPJid1e2teTXJBkv26+rUkO7KbXJPlCN/2uJB9K8rkk307yuq794G4v7/okNyQ5bszvb2eSdye5NsmVSVaNc31aWSZt+6+q64E/At7Qre8J3Xb/te7+8Un2SXJLBg5Icm+S47v5r07yxK7+C7q/wy1J3ricda4UBlSPutNgLwOeDvwqMPe02AFV9RzgvcCFwCur6qkMjnx/ZwGrOBo4GXgm8AdJHge8Cvh0Va0GngZcP6Suc7oOPPe2fglv8xHAtVX1DOCLwFlLWIYm0ARv/9cCT+6mzwU+WFVHAx8B/qaqdgPfAo4Cng1cAxzXhe6hVbWle+2TgRcAxwBnJXnoAtc/MTzF169nA5dW1Y8AkvzznOc/1t0/Cbi1qr7VPd4InAG8Z57lzyz7R0k+z2BD3wRc0G3sn+j2+O6nqt68iPewp7GyZtrv5b738WHg44tYtibbJGz/w2TW9DMZhC/Ah4A/76avBo4HjgD+FHgdgx24TbNee3lV3QPck2Q7cBCwbcTaVhSPoPqVeZ7/4QLm28V9/44Pm/Pc3PCoqrqKQce4HfhQkt98QFGL24P8LvCoOW2PBu7eQ70O/qgZk7D9D/N04KY9PDdT09XAcQxC85PAAcAJwFWz5r1n1vRupvCAwoDq178Bv5LkYUn2Z3A6YphvAocneWL3+NUM9rYAtgI/302/bM7rTu2W/VMMNv5NSZ4AbK+q9wPnA8+Yu7KqenNVrR5y2zCktk3AsUkeC/9/2mY/4Lbu+YcAL++mX9W9ZwkmY/u/nyRHA+9kcFoS4N+Btd30r3Pf9v9l4FnAvVX1YwanGl/PILjUmbpEbklVbUpyGfBV4D+AzcD3h8z34ySvAf4xyb4MQuF93dN/CJyf5O0MNvrZvgJcDjwe+OOquiPJ6cBbk/wvsBN4wB7kIt/DXUnOBD6Z5CHdMk+rqnu7WX4IPCXJNd17e+Uo69PkmITtv3NckuuAhwPbgTfOuoLvjQxOKb4V2AG8pntP9yS5DfhSN9/VwGnA15ehnokRf26jX0n2r6qdSR7O4PB+XVVduwzLfRews6r+ctRljVjHzqrav88a1K5J3/41Go+g+ndekqMYnD/fuBydU1pB3P61Rx5BSZKa5EUSkqQmGVCSpCY1EVAnnXRSMfh+gDdvk3pbMvuHtym4DdVEQN19956+0ynJ/qFp1URASZI0lwElSWqSASVJapIBJUlqkgElSWqSASVJapJj8U2hw9dfPvIytm7Y0y8jSNLyMKAkTaVRdtTcQds7PMUnSWqSASVJapIBJUlqkgElSWqSASVJapIBJUlqkgElSWqSASVJapIBJUlqkiNJrEDLMVSRJLXOIyhJUpMMKElSkwwoSVKTDChJUpMMKElSk+YNqCQXJNme5IZZbe9KcnuS67vbi2Y997YkW5LcnOQF4ypckjTZFnIEdSFw0pD2c6pqdXf7JECSo4C1wFO61/xdkn2Wq1hJ0vSYN6Cq6irgewtc3qnAR6vqnqq6FdgCHDNCfZKkKTXKZ1BvSPK17hTgo7q2Q4DbZs2zrWuTJGlRlhpQfw/8NLAauBN4d9eeIfPWsAUkWZdkc5LNO3bsWGIZ0mSyf0hLDKiququqdlfVvcD7ue803jbgsFmzHgrcsYdlnFdVa6pqzapVq5ZShjSx7B/SEgMqycGzHr4UmLnC7zJgbZL9khwBHAl8ZbQSJUnTaN7BYpNcBJwAHJhkG3AWcEKS1QxO320FXg9QVTcmuRj4BrALOKOqdo+lcknSRJs3oKrqtCHN5z/I/GcDZ49SlCRJjiQhSWqSASVJapIBJUlqkgElSWqSASVJapIBJUlqkgElSWqSASVJapIBJUlqkgElSWqSASVJapIBJUlqkgElSWqSASVJapIBJUlqkgElSWqSASVJapIBJUlqkgElSWqSASVJapIBJUlqkgElSWqSASVJapIBJUlq0r59FyBJS3X4+sv7LkFj5BGUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJ8wZUkguSbE9yw6y2Rye5Ism3u/tHzXrubUm2JLk5yQvGVbgkabIt5Iu6FwLnAh+c1bYeuLKqNiRZ3z3+/SRHAWuBpwCPAz6b5Geqavfyli1J/RnlC8JbN5y8jJVMtnmPoKrqKuB7c5pPBTZ20xuBl8xq/2hV3VNVtwJbgGOWp1RJ0jRZ6mdQB1XVnQDd/WO69kOA22bNt61re4Ak65JsTrJ5x44dSyxDmkz2D2n5L5LIkLYaNmNVnVdVa6pqzapVq5a5DGlls39ISw+ou5IcDNDdb+/atwGHzZrvUOCOpZcnSZpWSw2oy4DTu+nTgUtnta9Nsl+SI4Ajga+MVqIkaRrNexVfkouAE4ADk2wDzgI2ABcneS3wHeAVAFV1Y5KLgW8Au4AzvIJPkrQU8wZUVZ22h6dO3MP8ZwNnj1KUJEmOJCFJapIBJUlqkgElSWqSASVJatJCxuKTHmCUschmOCaZpAfjEZQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJBpQkqUkGlCSpSfuO8uIkW4EfALuBXVW1JsmjgY8BhwNbgV+rqv8arUxJ0rRZjiOo51bV6qpa0z1eD1xZVUcCV3aPJUlalHGc4jsV2NhNbwReMoZ1SJIm3KgBVcBnklyTZF3XdlBV3QnQ3T9m2AuTrEuyOcnmHTt2jFiGNFnsH9LoAXVsVT0DeCFwRpLjF/rCqjqvqtZU1ZpVq1aNWIY0Wewf0ogBVVV3dPfbgUuAY4C7khwM0N1vH7VISdL0WXJAJXlEkkfOTAPPB24ALgNO72Y7Hbh01CIlSdNnlMvMDwIuSTKznH+oqk8l2QRcnOS1wHeAV4xepiRp2iw5oKrqFuBpQ9q/C5w4SlGSJDmShCSpSQaUJKlJIw11pMU7fP3lfZcgqUej/B+wdcPJy1hJ+zyCkiQ1yYCSJDXJgJIkNcmAkiQ1yYCSJDXJgJIkNcmAkiQ1yYCSJDXJgJIkNcmAkiQ1yYCSJDXJgJIkNcnBYiX1ygGUtSceQUmSmmRASZKaZEBJkprkZ1DqzXJ89jBtP+AmTROPoCRJTTKgJElN8hSfpJF5qbjGwSMoSVKTDChJUpMMKElSkwwoSVKTDChJUpO8ik+SVohRrpZciV9qN6AWwUtpJWnvMaC0oo2607AS9yqlaWFASdIUWImnB8cWUElOAv4a2Af4QFVtGNe6FsLTcxrGAWvvYx9Ra8YSUEn2Ad4L/DKwDdiU5LKq+sZSl2nnkaR+9HX0Na7LzI8BtlTVLVX1P8BHgVPHtC5J0gQa1ym+Q4DbZj3eBvzC7BmSrAPWdQ93Jrl5hPUdCNw9wuvHqeXawPpGdWD+bEH1faqqTlroQpexfzT/98P6RtF8faP0j3EFVIa01f0eVJ0HnLcsK0s2V9Wa5VjWcmu5NrC+UY2rvuXqH9P691su1jeaUesb1ym+bcBhsx4fCtwxpnVJkibQuAJqE3BkkiOS/ASwFrhsTOuSJE2gsZziq6pdSd4AfJrBZeYXVNWN41hXZ1lOFY5Jy7WB9Y3K+kZjfaOZ6PpSVfPPJUnSXuZo5pKkJhlQkqQmTVRAJXlLkkpyYN+1zJbkL5J8M8nXklyS5IC+a4LBcFRJbk6yJcn6vuuZLclhST6f5KYkNyY5s++ahkmyT5LrkvxL37XMp8X+Yd9YvGnqGxMTUEkOYzC00nf6rmWIK4Cfq6qjgW8Bb+u5ntnDUb0QOAo4LclR/VZ1P7uA362qnwV+ETijsfpmnAnc1HcR82m4f9g3Fm9q+sbEBBRwDvB7zPlCcAuq6jNVtat7+CUG3wvrW9PDUVXVnVV1bTf9AwYb+iH9VnV/SQ4FTgY+0HctC9Bk/7BvLN409Y2JCKgkLwZur6qv9l3LAvw28K99F8Hw4aia2shnJDkceDrw5Z5Lmes9DP7Tv7fnOh7UCuof9o1FmvS+sWJ+DyrJZ4HHDnnqHcDbgefv3Yru78Hqq6pLu3neweDw/CN7s7Y9mHc4qhYk2R/4J+BNVfXffdczI8kpwPaquibJCT2X03T/sG+MxzT0jRUTUFX1S8PakzwVOAL4ahIYnCK4NskxVfWffdc3I8npwCnAidXGl8+aH44qyUMZdMCPVNXH+65njmOBFyd5EfAw4CeTfLiqfqOPYlruH/aN5TctfWPivqibZCuwpqqaGeG3+/HGvwKeU1U7+q4HIMm+DD6UPhG4ncHwVK8a84gfC5bB/6Ybge9V1Zt6LudBdXuJb6mqU3ouZV6t9Q/7xuJNU9+YiM+gVoBzgUcCVyS5Psn7+i6o+2B6Zjiqm4CLW+mAnWOBVwPP6/5m13d7ZJos9o3Fm5q+MXFHUJKkyeARlCSpSQaUJKlJBpQkqUkGlCSpSQaUJKlJBtSUSLJzD+0XJnn53q5HaoV9o10GlCSpSStmqCMtj+5b6H8LPA+4leHjjklTx77RHo+gps9LgScBTwVeBzyr33KkZtg3GmNATZ/jgYuqandV3QF8ru+CpEbYNxpjQE0nx7eShrNvNMSAmj5XAWuT7JPkYOC5fRckNcK+0Rgvkpg+lzD4EPjrDH5S4Iv9liM1w77RGEczlyQ1yVN8kqQmGVCSpCYZUJKkJhlQkqQmGVCSpCYZUJKkJhlQkqQm/R8AhbmG8D9HLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x216 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ld_sc = x_train.iloc[:, 0] * result.scalings_[0] + x_train.iloc[:, 1] * result.scalings_[1]\n",
    "\n",
    "ld = pd.DataFrame({'groups': y_train, 'ld': ld_sc})\n",
    "\n",
    "g = sns.FacetGrid(ld, col='groups')\n",
    "g.map(plt.hist, 'ld');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prob_0</th>\n",
       "      <th>prob_1</th>\n",
       "      <th>pred_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.490179</td>\n",
       "      <td>0.509821</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.479218</td>\n",
       "      <td>0.520782</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.466818</td>\n",
       "      <td>0.533182</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.474001</td>\n",
       "      <td>0.525999</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.492788</td>\n",
       "      <td>0.507212</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>0.500589</td>\n",
       "      <td>0.499411</td>\n",
       "      <td>Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>0.497221</td>\n",
       "      <td>0.502779</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>0.479199</td>\n",
       "      <td>0.520801</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>0.483167</td>\n",
       "      <td>0.516833</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>0.489259</td>\n",
       "      <td>0.510741</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>252 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       prob_0    prob_1 pred_class\n",
       "0    0.490179  0.509821         Up\n",
       "1    0.479218  0.520782         Up\n",
       "2    0.466818  0.533182         Up\n",
       "3    0.474001  0.525999         Up\n",
       "4    0.492788  0.507212         Up\n",
       "..        ...       ...        ...\n",
       "247  0.500589  0.499411       Down\n",
       "248  0.497221  0.502779         Up\n",
       "249  0.479199  0.520801         Up\n",
       "250  0.483167  0.516833         Up\n",
       "251  0.489259  0.510741         Up\n",
       "\n",
       "[252 rows x 3 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'prob_0': result.predict_proba(x_test)[:, 0], 'prob_1': result.predict_proba(x_test)[:, 1],'pred_class': result.predict(x_test)},columns=['prob_0', 'prob_1', 'pred_class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 35  35]\n",
      " [ 76 106]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(result.predict(x_test), y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5595238095238095\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_test, result.predict(x_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quadratic Discriminant Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = qda()\n",
    "result = model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.49198397, 0.50801603])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.priors_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.04279022,  0.03389409],\n",
       "       [-0.03954635, -0.03132544]])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.means_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prob_0</th>\n",
       "      <th>prob_1</th>\n",
       "      <th>pred_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.487324</td>\n",
       "      <td>0.512676</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.475901</td>\n",
       "      <td>0.524099</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.463691</td>\n",
       "      <td>0.536309</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.473925</td>\n",
       "      <td>0.526075</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.490343</td>\n",
       "      <td>0.509657</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>0.497670</td>\n",
       "      <td>0.502330</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>0.494787</td>\n",
       "      <td>0.505213</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>0.475864</td>\n",
       "      <td>0.524136</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>0.482695</td>\n",
       "      <td>0.517305</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>0.486446</td>\n",
       "      <td>0.513554</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>252 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       prob_0    prob_1 pred_class\n",
       "0    0.487324  0.512676         Up\n",
       "1    0.475901  0.524099         Up\n",
       "2    0.463691  0.536309         Up\n",
       "3    0.473925  0.526075         Up\n",
       "4    0.490343  0.509657         Up\n",
       "..        ...       ...        ...\n",
       "247  0.497670  0.502330         Up\n",
       "248  0.494787  0.505213         Up\n",
       "249  0.475864  0.524136         Up\n",
       "250  0.482695  0.517305         Up\n",
       "251  0.486446  0.513554         Up\n",
       "\n",
       "[252 rows x 3 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'prob_0': result.predict_proba(x_test)[:, 0], 'prob_1': result.predict_proba(x_test)[:, 1],'pred_class': result.predict(x_test)},columns=['prob_0', 'prob_1', 'pred_class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 30  20]\n",
      " [ 81 121]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(result.predict(x_test), y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5992063492063492\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_test, result.predict(x_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# K-Nearest Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = knn(n_neighbors=1)\n",
    "result = model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prob_0</th>\n",
       "      <th>prob_1</th>\n",
       "      <th>pred_class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>249</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Up</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>252 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     prob_0  prob_1 pred_class\n",
       "0       0.0     1.0         Up\n",
       "1       1.0     0.0       Down\n",
       "2       0.0     1.0         Up\n",
       "3       0.0     1.0         Up\n",
       "4       0.0     1.0         Up\n",
       "..      ...     ...        ...\n",
       "247     1.0     0.0       Down\n",
       "248     1.0     0.0       Down\n",
       "249     0.0     1.0         Up\n",
       "250     1.0     0.0       Down\n",
       "251     0.0     1.0         Up\n",
       "\n",
       "[252 rows x 3 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame({'prob_0': result.predict_proba(x_test)[:, 0], 'prob_1': result.predict_proba(x_test)[:, 1],'pred_class': result.predict(x_test)},columns=['prob_0', 'prob_1', 'pred_class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[43 58]\n",
      " [68 83]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(result.predict(x_test), y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_test, result.predict(x_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = knn(n_neighbors=3)\n",
    "result = model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[48 55]\n",
      " [63 86]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(result.predict(x_test), y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5317460317460317\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_test, result.predict(x_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# An Application to Caravan Insurance Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MOSTYPE</th>\n",
       "      <th>MAANTHUI</th>\n",
       "      <th>MGEMOMV</th>\n",
       "      <th>MGEMLEEF</th>\n",
       "      <th>MOSHOOFD</th>\n",
       "      <th>MGODRK</th>\n",
       "      <th>MGODPR</th>\n",
       "      <th>MGODOV</th>\n",
       "      <th>MGODGE</th>\n",
       "      <th>MRELGE</th>\n",
       "      <th>...</th>\n",
       "      <th>APERSONG</th>\n",
       "      <th>AGEZONG</th>\n",
       "      <th>AWAOREG</th>\n",
       "      <th>ABRAND</th>\n",
       "      <th>AZEILPL</th>\n",
       "      <th>APLEZIER</th>\n",
       "      <th>AFIETS</th>\n",
       "      <th>AINBOED</th>\n",
       "      <th>ABYSTAND</th>\n",
       "      <th>Purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5818</th>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5819</th>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5820</th>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5821</th>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5822</th>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5822 rows × 86 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      MOSTYPE  MAANTHUI  MGEMOMV  MGEMLEEF  MOSHOOFD  MGODRK  MGODPR  MGODOV  \\\n",
       "1          33         1        3         2         8       0       5       1   \n",
       "2          37         1        2         2         8       1       4       1   \n",
       "3          37         1        2         2         8       0       4       2   \n",
       "4           9         1        3         3         3       2       3       2   \n",
       "5          40         1        4         2        10       1       4       1   \n",
       "...       ...       ...      ...       ...       ...     ...     ...     ...   \n",
       "5818       36         1        1         2         8       0       6       1   \n",
       "5819       35         1        4         4         8       1       4       1   \n",
       "5820       33         1        3         4         8       0       6       0   \n",
       "5821       34         1        3         2         8       0       7       0   \n",
       "5822       33         1        3         3         8       0       6       1   \n",
       "\n",
       "      MGODGE  MRELGE  ...  APERSONG  AGEZONG  AWAOREG  ABRAND  AZEILPL  \\\n",
       "1          3       7  ...         0        0        0       1        0   \n",
       "2          4       6  ...         0        0        0       1        0   \n",
       "3          4       3  ...         0        0        0       1        0   \n",
       "4          4       5  ...         0        0        0       1        0   \n",
       "5          4       7  ...         0        0        0       1        0   \n",
       "...      ...     ...  ...       ...      ...      ...     ...      ...   \n",
       "5818       2       1  ...         0        0        0       1        0   \n",
       "5819       4       6  ...         0        0        0       1        0   \n",
       "5820       3       5  ...         0        0        0       1        0   \n",
       "5821       2       7  ...         0        0        0       0        0   \n",
       "5822       2       7  ...         0        0        0       0        0   \n",
       "\n",
       "      APLEZIER  AFIETS  AINBOED  ABYSTAND  Purchase  \n",
       "1            0       0        0         0        No  \n",
       "2            0       0        0         0        No  \n",
       "3            0       0        0         0        No  \n",
       "4            0       0        0         0        No  \n",
       "5            0       0        0         0        No  \n",
       "...        ...     ...      ...       ...       ...  \n",
       "5818         0       0        0         0        No  \n",
       "5819         0       0        0         0        No  \n",
       "5820         0       0        0         0       Yes  \n",
       "5821         0       0        0         0        No  \n",
       "5822         0       0        0         0        No  \n",
       "\n",
       "[5822 rows x 86 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caravan = pd.read_csv('./data/Caravan.csv', index_col=0)\n",
    "caravan.Purchase = caravan.Purchase.astype('category')\n",
    "caravan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MOSTYPE</th>\n",
       "      <th>MAANTHUI</th>\n",
       "      <th>MGEMOMV</th>\n",
       "      <th>MGEMLEEF</th>\n",
       "      <th>MOSHOOFD</th>\n",
       "      <th>MGODRK</th>\n",
       "      <th>MGODPR</th>\n",
       "      <th>MGODOV</th>\n",
       "      <th>MGODGE</th>\n",
       "      <th>MRELGE</th>\n",
       "      <th>...</th>\n",
       "      <th>ALEVEN</th>\n",
       "      <th>APERSONG</th>\n",
       "      <th>AGEZONG</th>\n",
       "      <th>AWAOREG</th>\n",
       "      <th>ABRAND</th>\n",
       "      <th>AZEILPL</th>\n",
       "      <th>APLEZIER</th>\n",
       "      <th>AFIETS</th>\n",
       "      <th>AINBOED</th>\n",
       "      <th>ABYSTAND</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "      <td>5822.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>24.253349</td>\n",
       "      <td>1.110615</td>\n",
       "      <td>2.678805</td>\n",
       "      <td>2.991240</td>\n",
       "      <td>5.773617</td>\n",
       "      <td>0.696496</td>\n",
       "      <td>4.626932</td>\n",
       "      <td>1.069907</td>\n",
       "      <td>3.258502</td>\n",
       "      <td>6.183442</td>\n",
       "      <td>...</td>\n",
       "      <td>0.076606</td>\n",
       "      <td>0.005325</td>\n",
       "      <td>0.006527</td>\n",
       "      <td>0.004638</td>\n",
       "      <td>0.570079</td>\n",
       "      <td>0.000515</td>\n",
       "      <td>0.006012</td>\n",
       "      <td>0.031776</td>\n",
       "      <td>0.007901</td>\n",
       "      <td>0.014256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>12.846706</td>\n",
       "      <td>0.405842</td>\n",
       "      <td>0.789835</td>\n",
       "      <td>0.814589</td>\n",
       "      <td>2.856760</td>\n",
       "      <td>1.003234</td>\n",
       "      <td>1.715843</td>\n",
       "      <td>1.017503</td>\n",
       "      <td>1.597647</td>\n",
       "      <td>1.909482</td>\n",
       "      <td>...</td>\n",
       "      <td>0.377569</td>\n",
       "      <td>0.072782</td>\n",
       "      <td>0.080532</td>\n",
       "      <td>0.077403</td>\n",
       "      <td>0.562058</td>\n",
       "      <td>0.022696</td>\n",
       "      <td>0.081632</td>\n",
       "      <td>0.210986</td>\n",
       "      <td>0.090463</td>\n",
       "      <td>0.119996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>10.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>30.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>35.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>41.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 85 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           MOSTYPE     MAANTHUI      MGEMOMV     MGEMLEEF     MOSHOOFD  \\\n",
       "count  5822.000000  5822.000000  5822.000000  5822.000000  5822.000000   \n",
       "mean     24.253349     1.110615     2.678805     2.991240     5.773617   \n",
       "std      12.846706     0.405842     0.789835     0.814589     2.856760   \n",
       "min       1.000000     1.000000     1.000000     1.000000     1.000000   \n",
       "25%      10.000000     1.000000     2.000000     2.000000     3.000000   \n",
       "50%      30.000000     1.000000     3.000000     3.000000     7.000000   \n",
       "75%      35.000000     1.000000     3.000000     3.000000     8.000000   \n",
       "max      41.000000    10.000000     5.000000     6.000000    10.000000   \n",
       "\n",
       "            MGODRK       MGODPR       MGODOV       MGODGE       MRELGE  ...  \\\n",
       "count  5822.000000  5822.000000  5822.000000  5822.000000  5822.000000  ...   \n",
       "mean      0.696496     4.626932     1.069907     3.258502     6.183442  ...   \n",
       "std       1.003234     1.715843     1.017503     1.597647     1.909482  ...   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000  ...   \n",
       "25%       0.000000     4.000000     0.000000     2.000000     5.000000  ...   \n",
       "50%       0.000000     5.000000     1.000000     3.000000     6.000000  ...   \n",
       "75%       1.000000     6.000000     2.000000     4.000000     7.000000  ...   \n",
       "max       9.000000     9.000000     5.000000     9.000000     9.000000  ...   \n",
       "\n",
       "            ALEVEN     APERSONG      AGEZONG      AWAOREG       ABRAND  \\\n",
       "count  5822.000000  5822.000000  5822.000000  5822.000000  5822.000000   \n",
       "mean      0.076606     0.005325     0.006527     0.004638     0.570079   \n",
       "std       0.377569     0.072782     0.080532     0.077403     0.562058   \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "25%       0.000000     0.000000     0.000000     0.000000     0.000000   \n",
       "50%       0.000000     0.000000     0.000000     0.000000     1.000000   \n",
       "75%       0.000000     0.000000     0.000000     0.000000     1.000000   \n",
       "max       8.000000     1.000000     1.000000     2.000000     7.000000   \n",
       "\n",
       "           AZEILPL     APLEZIER       AFIETS      AINBOED     ABYSTAND  \n",
       "count  5822.000000  5822.000000  5822.000000  5822.000000  5822.000000  \n",
       "mean      0.000515     0.006012     0.031776     0.007901     0.014256  \n",
       "std       0.022696     0.081632     0.210986     0.090463     0.119996  \n",
       "min       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "25%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "50%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "75%       0.000000     0.000000     0.000000     0.000000     0.000000  \n",
       "max       1.000000     2.000000     3.000000     2.000000     2.000000  \n",
       "\n",
       "[8 rows x 85 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caravan.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "No     5474\n",
       "Yes     348\n",
       "Name: Purchase, dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caravan.Purchase.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.05977327378907592"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "caravan.Purchase.value_counts()['Yes']/len(caravan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import zscore\n",
    "\n",
    "X = caravan.iloc[:, 0:85].apply(zscore)\n",
    "\n",
    "y = caravan.Purchase\n",
    "\n",
    "x_train = X.loc[1001:]\n",
    "y_train = y.loc[1001:]\n",
    "\n",
    "x_test = X.loc[:1000]\n",
    "y_test = y.loc[:1000]\n",
    "model = knn(n_neighbors=1)\n",
    "result = model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[873  50]\n",
      " [ 68   9]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(result.predict(x_test), y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.882\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(y_test, result.predict(x_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = knn(n_neighbors=3)\n",
    "result = model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[921  54]\n",
      " [ 20   5]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(result.predict(x_test), y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = knn(n_neighbors=5)\n",
    "result = model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[930  55]\n",
      " [ 11   4]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(result.predict(x_test), y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\prath\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:763: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regr = skl_lm.LogisticRegression()\n",
    "regr.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True        No  Yes\n",
      "Predicted          \n",
      "No         934   59\n",
      "Yes          7    0\n"
     ]
    }
   ],
   "source": [
    "pred = regr.predict(x_test)\n",
    "cm_df = pd.DataFrame(confusion_matrix(y_test, pred).T, index=regr.classes_,columns=regr.classes_)\n",
    "cm_df.index.name = 'Predicted'\n",
    "cm_df.columns.name = 'True'\n",
    "print(cm_df)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pred   No  Yes\n",
      "True          \n",
      "No    917   24\n",
      "Yes    48   11\n"
     ]
    }
   ],
   "source": [
    "pred_p = regr.predict_proba(x_test)\n",
    "cm_df = pd.DataFrame({'True': y_test, 'Pred': pred_p[:,1] > .25})\n",
    "cm_df.Pred.replace(to_replace={True:'Yes', False:'No'}, inplace=True)\n",
    "print(cm_df.groupby(['True', 'Pred']).size().unstack('True').T)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}