Logistic Regression & Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.read_csv(\"Wall robot navigation.csv\",sep=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.687</td>\n",
       "      <td>0.445</td>\n",
       "      <td>2.332</td>\n",
       "      <td>0.429</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.687</td>\n",
       "      <td>0.449</td>\n",
       "      <td>2.332</td>\n",
       "      <td>0.429</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.687</td>\n",
       "      <td>0.449</td>\n",
       "      <td>2.334</td>\n",
       "      <td>0.429</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.687</td>\n",
       "      <td>0.449</td>\n",
       "      <td>2.334</td>\n",
       "      <td>0.429</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.687</td>\n",
       "      <td>0.449</td>\n",
       "      <td>2.334</td>\n",
       "      <td>0.429</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      V1     V2     V3     V4  Class\n",
       "0  1.687  0.445  2.332  0.429      4\n",
       "1  1.687  0.449  2.332  0.429      4\n",
       "2  1.687  0.449  2.334  0.429      4\n",
       "3  1.687  0.449  2.334  0.429      4\n",
       "4  1.687  0.449  2.334  0.429      4"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5456 entries, 0 to 5455\n",
      "Data columns (total 5 columns):\n",
      "V1       5456 non-null float64\n",
      "V2       5456 non-null float64\n",
      "V3       5456 non-null float64\n",
      "V4       5456 non-null float64\n",
      "Class    5456 non-null int64\n",
      "dtypes: float64(4), int64(1)\n",
      "memory usage: 213.2 KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### V1,V2,V3,V4 are the numerical variables. Class is the categorical variable but is given as integer here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>V1</th>\n",
       "      <th>V2</th>\n",
       "      <th>V3</th>\n",
       "      <th>V4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5456.000000</td>\n",
       "      <td>5456.000000</td>\n",
       "      <td>5456.000000</td>\n",
       "      <td>5456.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.290307</td>\n",
       "      <td>0.681269</td>\n",
       "      <td>1.881819</td>\n",
       "      <td>1.273689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.626700</td>\n",
       "      <td>0.342594</td>\n",
       "      <td>0.562533</td>\n",
       "      <td>0.821750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.495000</td>\n",
       "      <td>0.340000</td>\n",
       "      <td>0.836000</td>\n",
       "      <td>0.367000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.826000</td>\n",
       "      <td>0.495000</td>\n",
       "      <td>1.472000</td>\n",
       "      <td>0.788000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.089500</td>\n",
       "      <td>0.612000</td>\n",
       "      <td>1.753000</td>\n",
       "      <td>1.066500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.519500</td>\n",
       "      <td>0.753000</td>\n",
       "      <td>2.139000</td>\n",
       "      <td>1.400500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                V1           V2           V3           V4\n",
       "count  5456.000000  5456.000000  5456.000000  5456.000000\n",
       "mean      1.290307     0.681269     1.881819     1.273689\n",
       "std       0.626700     0.342594     0.562533     0.821750\n",
       "min       0.495000     0.340000     0.836000     0.367000\n",
       "25%       0.826000     0.495000     1.472000     0.788000\n",
       "50%       1.089500     0.612000     1.753000     1.066500\n",
       "75%       1.519500     0.753000     2.139000     1.400500\n",
       "max       5.000000     5.000000     5.000000     5.000000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[['V1','V2','V3','V4']].describe() # Analyzing only the numerical columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalization of numerical variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "extracting only the numerical columns so as to normalize it between 0 and 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=data[['V1', 'V2', 'V3', 'V4']] # Extracting only the numerical columns to a new dataframe so as to normalize them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['V1', 'V2', 'V3', 'V4'], dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "x=df.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.687, 0.445, 2.332, 0.429],\n",
       "       [1.687, 0.449, 2.332, 0.429],\n",
       "       [1.687, 0.449, 2.334, 0.429],\n",
       "       ...,\n",
       "       [0.873, 0.642, 1.053, 1.105],\n",
       "       [0.967, 0.635, 1.034, 1.118],\n",
       "       [0.854, 0.628, 1.016, 1.168]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_max_scaler = preprocessing.MinMaxScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_scaled = min_max_scaler.fit_transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_normalized = pd.DataFrame(x_scaled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.022532</td>\n",
       "      <td>0.35927</td>\n",
       "      <td>0.013382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.023391</td>\n",
       "      <td>0.35927</td>\n",
       "      <td>0.013382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.023391</td>\n",
       "      <td>0.35975</td>\n",
       "      <td>0.013382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.023391</td>\n",
       "      <td>0.35975</td>\n",
       "      <td>0.013382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.023391</td>\n",
       "      <td>0.35975</td>\n",
       "      <td>0.013382</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1        2         3\n",
       "0  0.264595  0.022532  0.35927  0.013382\n",
       "1  0.264595  0.023391  0.35927  0.013382\n",
       "2  0.264595  0.023391  0.35975  0.013382\n",
       "3  0.264595  0.023391  0.35975  0.013382\n",
       "4  0.264595  0.023391  0.35975  0.013382"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_normalized.head() #normalized dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_normalized['class']=data['Class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.022532</td>\n",
       "      <td>0.35927</td>\n",
       "      <td>0.013382</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.023391</td>\n",
       "      <td>0.35927</td>\n",
       "      <td>0.013382</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.023391</td>\n",
       "      <td>0.35975</td>\n",
       "      <td>0.013382</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.023391</td>\n",
       "      <td>0.35975</td>\n",
       "      <td>0.013382</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.264595</td>\n",
       "      <td>0.023391</td>\n",
       "      <td>0.35975</td>\n",
       "      <td>0.013382</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1        2         3  class\n",
       "0  0.264595  0.022532  0.35927  0.013382      4\n",
       "1  0.264595  0.023391  0.35927  0.013382      4\n",
       "2  0.264595  0.023391  0.35975  0.013382      4\n",
       "3  0.264595  0.023391  0.35975  0.013382      4\n",
       "4  0.264595  0.023391  0.35975  0.013382      4"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_normalized.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2=df_normalized"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One hot encoding on the categorical variable which is 'class'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "onehotencoder = OneHotEncoder(categorical_features = [4]) # 'class' is column 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\VAISHAK\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:371: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.\n",
      "If you want the future behaviour and silence this warning, you can specify \"categories='auto'\".\n",
      "In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.\n",
      "  warnings.warn(msg, FutureWarning)\n",
      "C:\\Users\\VAISHAK\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:392: DeprecationWarning: The 'categorical_features' keyword is deprecated in version 0.20 and will be removed in 0.22. You can use the ColumnTransformer instead.\n",
      "  \"use the ColumnTransformer instead.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df2 = onehotencoder.fit_transform(df2).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2=pd.DataFrame(df2) #one hot encoded & normalized dataframe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1074</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.060155</td>\n",
       "      <td>0.031330</td>\n",
       "      <td>0.567723</td>\n",
       "      <td>0.203324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4337</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.066593</td>\n",
       "      <td>0.063948</td>\n",
       "      <td>0.190202</td>\n",
       "      <td>0.092812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2899</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.219534</td>\n",
       "      <td>0.076609</td>\n",
       "      <td>0.139529</td>\n",
       "      <td>0.074250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4225</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.129412</td>\n",
       "      <td>0.051502</td>\n",
       "      <td>0.072767</td>\n",
       "      <td>0.173754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5282</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.390233</td>\n",
       "      <td>0.036481</td>\n",
       "      <td>0.235591</td>\n",
       "      <td>0.234190</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        0    1    2    3         4         5         6         7\n",
       "1074  0.0  1.0  0.0  0.0  0.060155  0.031330  0.567723  0.203324\n",
       "4337  0.0  1.0  0.0  0.0  0.066593  0.063948  0.190202  0.092812\n",
       "2899  1.0  0.0  0.0  0.0  0.219534  0.076609  0.139529  0.074250\n",
       "4225  1.0  0.0  0.0  0.0  0.129412  0.051502  0.072767  0.173754\n",
       "5282  1.0  0.0  0.0  0.0  0.390233  0.036481  0.235591  0.234190"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performing logistic regression to classify"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    40.414223\n",
       "2    38.434751\n",
       "4    15.139296\n",
       "3     6.011730\n",
       "Name: Class, dtype: float64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Class'].value_counts(normalize=1)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create training and testing variables\n",
    "X_train, X_test, y_train, y_test = train_test_split(data[['V1','V2','V3','V4']],data['Class'], test_size=0.2,random_state=42,stratify=data['Class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4364, 4) (4364,)\n",
      "(1092, 4) (1092,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape, y_train.shape)\n",
    "print(X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression \n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\VAISHAK\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n",
      "C:\\Users\\VAISHAK\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:460: FutureWarning: Default multi_class will be changed to 'auto' in 0.22. Specify the multi_class option to silence this warning.\n",
      "  \"this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_class=model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = accuracy_score(y_test,predicted_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8205128205128205"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.75      0.88      0.81       441\n",
      "           2       0.87      0.98      0.92       420\n",
      "           3       0.89      0.83      0.86        66\n",
      "           4       1.00      0.25      0.41       165\n",
      "\n",
      "   micro avg       0.82      0.82      0.82      1092\n",
      "   macro avg       0.88      0.74      0.75      1092\n",
      "weighted avg       0.84      0.82      0.80      1092\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,predicted_class))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm=confusion_matrix(y_test, predicted_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[386,  55,   0,   0],\n",
       "       [  0, 413,   7,   0],\n",
       "       [  5,   6,  55,   0],\n",
       "       [123,   0,   0,  42]], dtype=int64)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classification using neural network model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "nnmodel = MLPClassifier(hidden_layer_sizes=(100,50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\VAISHAK\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:562: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "MLPClassifier(activation='relu', alpha=0.0001, batch_size='auto', beta_1=0.9,\n",
       "       beta_2=0.999, early_stopping=False, epsilon=1e-08,\n",
       "       hidden_layer_sizes=(100, 50), learning_rate='constant',\n",
       "       learning_rate_init=0.001, max_iter=200, momentum=0.9,\n",
       "       n_iter_no_change=10, nesterovs_momentum=True, power_t=0.5,\n",
       "       random_state=None, shuffle=True, solver='adam', tol=0.0001,\n",
       "       validation_fraction=0.1, verbose=False, warm_start=False)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nnmodel.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "nnpredicted_class=nnmodel.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9853479853479854"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(y_test,nnpredicted_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           1       0.99      0.98      0.98       441\n",
      "           2       1.00      0.99      0.99       420\n",
      "           3       0.97      0.97      0.97        66\n",
      "           4       0.96      1.00      0.98       165\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      1092\n",
      "   macro avg       0.98      0.98      0.98      1092\n",
      "weighted avg       0.99      0.99      0.99      1092\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,nnpredicted_class))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "nn_cm=confusion_matrix(y_test, nnpredicted_class)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[433,   1,   1,   6],\n",
       "       [  4, 414,   1,   1],\n",
       "       [  2,   0,  64,   0],\n",
       "       [  0,   0,   0, 165]], dtype=int64)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn_cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding out the false postive rate & false negative rate for both the methods - Logistic regression & Multi-layer neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def false_perc(matrix):\n",
    "    FP = matrix.sum(axis=0) - np.diag(matrix)\n",
    "    FN = matrix.sum(axis=1) - np.diag(matrix)\n",
    "    TP = np.diag(matrix)\n",
    "    TN = matrix.sum() - (FP + FN + TP)\n",
    "    FP = FP.astype(float)\n",
    "    FN = FN.astype(float)\n",
    "    TP = TP.astype(float)\n",
    "    TN = TN.astype(float)\n",
    "    FPR = FP/(FP+TN) # FPR - False Positive Rate\n",
    "    FNR = FN/(TP+FN) # FNR - False Negative Rate\n",
    "    fr=np.vstack((FPR,FNR))\n",
    "    fr=fr*100\n",
    "    false_rate=pd.DataFrame(fr,columns=[1,2,3,4],index=['FPR(%)','FNR(%)'])\n",
    "    print(false_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                1         2          3          4\n",
      "FPR(%)  19.662058  9.077381   0.682261   0.000000\n",
      "FNR(%)  12.471655  1.666667  16.666667  74.545455\n"
     ]
    }
   ],
   "source": [
    "false_perc(cm) # for logistic regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               1         2         3         4\n",
      "FPR(%)  0.921659  0.148810  0.194932  0.755124\n",
      "FNR(%)  1.814059  1.428571  3.030303  0.000000\n"
     ]
    }
   ],
   "source": [
    "false_perc(nn_cm) # for neural network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experimenting the neural network model with different number of hidden layers and nodes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9688644688644689"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nnmodel = MLPClassifier(hidden_layer_sizes=(100,50,20))\n",
    "nnmodel.fit(X_train,y_train)\n",
    "nnpredicted_class=nnmodel.predict(X_test)\n",
    "accuracy_score(y_test,nnpredicted_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The accuracy has decreased compared to that of the previous one. Hence a 3rd hidden layer is not required. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9853479853479854"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nnmodel = MLPClassifier(hidden_layer_sizes=(100,30))\n",
    "nnmodel.fit(X_train,y_train)\n",
    "nnpredicted_class=nnmodel.predict(X_test)\n",
    "accuracy_score(y_test,nnpredicted_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Accuracy remains almost the same"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\VAISHAK\\Anaconda3\\lib\\site-packages\\sklearn\\neural_network\\multilayer_perceptron.py:562: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  % self.max_iter, ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9862637362637363"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nnmodel = MLPClassifier(hidden_layer_sizes=(100,60))\n",
    "nnmodel.fit(X_train,y_train)\n",
    "nnpredicted_class=nnmodel.predict(X_test)\n",
    "accuracy_score(y_test,nnpredicted_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Accuracy has increased slightly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.989010989010989"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nnmodel = MLPClassifier(hidden_layer_sizes=(100,100))\n",
    "nnmodel.fit(X_train,y_train)\n",
    "nnpredicted_class=nnmodel.predict(X_test)\n",
    "accuracy_score(y_test,nnpredicted_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The accuracy has increased even further. This seems to be a good model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}