Assign1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Author: Rudra Panda"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q1) Load the dataset into a pandas dataframe and display the first 5 lines of the dataset along with the column headings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>MEDV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31     0  0.538  6.575  65.2  4.0900    1  296.0   \n",
       "1  0.02731   0.0   7.07     0  0.469  6.421  78.9  4.9671    2  242.0   \n",
       "2  0.02729   0.0   7.07     0  0.469  7.185  61.1  4.9671    2  242.0   \n",
       "3  0.03237   0.0   2.18     0  0.458  6.998  45.8  6.0622    3  222.0   \n",
       "4  0.06905   0.0   2.18     0  0.458  7.147  54.2  6.0622    3  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  MEDV  \n",
       "0     15.3  396.90   4.98  24.0  \n",
       "1     17.8  396.90   9.14  21.6  \n",
       "2     17.8  392.83   4.03  34.7  \n",
       "3     18.7  394.63   2.94  33.4  \n",
       "4     18.7  396.90   5.33  36.2  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "#loading the dataset\n",
    "dataset = pd.read_table('data.txt', delim_whitespace=True, \n",
    "                   names=('CRIM', 'ZN', 'INDUS', 'CHAS', \n",
    "                          'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO',\n",
    "                         'B', 'LSTAT', 'MEDV'))\n",
    "#display the first 5 lines of the dataset along with the column headings\n",
    "dataset.head(n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q2) Split the dataset into training (70%) and testing set (30%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#features and targt\n",
    "X = dataset.iloc[:, :-1].values\n",
    "y = dataset.iloc[:, 13].values\n",
    "\n",
    "#add intercept\n",
    "import statsmodels.formula.api as sm\n",
    "X = np.append(arr=np.ones((len(X),1)).astype(int), values=X, axis=1)\n",
    "\n",
    "#Splitting data into Training and Test Set\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3,random_state=0)\n",
    "\n",
    "#Normalizing the features\n",
    "#from sklearn.preprocessing import StandardScaler\n",
    "#sc_X = StandardScaler()\n",
    "#X_train = sc_X.fit_transform(X_train)\n",
    "#X_test = sc_X.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q3) Model 1: Build a linear regression model with all the variables using Normal Equations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for inverse matrix\n",
    "from numpy.linalg import inv\n",
    "\n",
    "#apply normal equation\n",
    "w = inv(X_train.T.dot(X_train)).dot(X_train.T).dot(y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q4) What are the weight parameters you get for Model 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 3.79371077e+01 -1.21310401e-01  4.44664254e-02  1.13416945e-02\n",
      "  2.51124642e+00 -1.62312529e+01  3.85906801e+00 -9.98516565e-03\n",
      " -1.50026956e+00  2.42143466e-01 -1.10716124e-02 -1.01775264e+00\n",
      "  6.81446545e-03 -4.86738066e-01]\n"
     ]
    }
   ],
   "source": [
    "print(w)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q5) Using Model 1 make a prediction on the test set. Calculate mean squared error."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27.195965766882875"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make a prediction on test\n",
    "y_pred_ne = X_test.dot(w.T)\n",
    "\n",
    "# Calculate mean squared error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "mean_squared_error(y_test, y_pred_ne)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q6) Model 2: Using scikit learn, build a Linear Regression model with all the variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "#features and targt for LR\n",
    "X_LR = dataset.iloc[:, :-1].values\n",
    "y_lr = dataset.iloc[:, 13].values\n",
    "\n",
    "#Splitting data into Training and Test Set\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_LR_train, X_LR_test, y_lr_train, y_lr_test = train_test_split(X_LR,y_lr,test_size=0.3,random_state=0)\n",
    "\n",
    "lrObj = LinearRegression()\n",
    "lrObj.fit(X_LR_train, y_lr_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q7) What are the weight parameters you get for Model 2."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1.21310401e-01  4.44664254e-02  1.13416945e-02  2.51124642e+00\n",
      " -1.62312529e+01  3.85906801e+00 -9.98516565e-03 -1.50026956e+00\n",
      "  2.42143466e-01 -1.10716124e-02 -1.01775264e+00  6.81446545e-03\n",
      " -4.86738066e-01]\n"
     ]
    }
   ],
   "source": [
    "print(lrObj.coef_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q8) Using Model 2 make a prediction on the test set. Calculate mean squared error."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27.195965766883425"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_lr = lrObj.predict(X_LR_test)\n",
    "\n",
    "# Calculate mean squared error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "mean_squared_error(y_lr_test, y_pred_lr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q9) Write a function forwardSelection(x, sl) to implement the forward feature selection technique."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.formula.api as sm\n",
    "\n",
    "def forwardSelection(x, sl):\n",
    "    numVars = len(x[0])\n",
    "    minimum = None\n",
    "    featureList = []\n",
    "    \n",
    "    #Step-1 Create models for each of the co-efficients (x), \n",
    "    #and select the model with lowest p-value\n",
    "    for i in range(0, numVars):\n",
    "        olsObj = sm.OLS(y,x[:,i]).fit()\n",
    "        if minimum == None or olsObj.pvalues < minimum:\n",
    "            minimum = (olsObj.pvalues).astype(float)\n",
    "            idx = i\n",
    "    \n",
    "    #keep this variable in the model\n",
    "    featureList.append(idx)\n",
    "    \n",
    "    #create models by adding one new variable each time and \n",
    "    #checking that the p-value of the new variable is less than significant level\n",
    "    for i in range(0, numVars):\n",
    "        if i != idx: #don't need to include the variable already added in setp 1\n",
    "            featureList.append(i)\n",
    "            featureList.sort()\n",
    "            varPosition = featureList.index(i)\n",
    "            olsObj = sm.OLS(y,x[:,featureList]).fit()\n",
    "            varPvalue = (olsObj.pvalues[varPosition]).astype(float)\n",
    "            if varPvalue > sl:\n",
    "                featureList.remove(i)\n",
    "    x = x[:,featureList]\n",
    "    print(featureList)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q10) Display the features selected by your function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 1, 2, 3, 4, 6, 7, 8, 10, 11, 12, 13]\n"
     ]
    }
   ],
   "source": [
    "sig_level = 0.05\n",
    "X_sig = forwardSelection(X, sig_level)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q11) Model 3: Using sckikit learn, build a Linear Regression model using only the features selected by the forward selection function you implemented."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Splitting the data into Training Set and Test Set\n",
    "X_sig_train, X_sig_test, y_sig_train, y_sig_test= train_test_split(X_sig, y, test_size=0.3,random_state=0)\n",
    "mlrObj_sig= LinearRegression()\n",
    "mlrObj_sig.fit(X_sig_train, y_sig_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Q12) Using Model 3 make a prediction on the test set. Calculate mean squared error."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30.17179214690381"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_sig_pred= mlrObj_sig.predict(X_sig_test)\n",
    "mean_squared_error(y_sig_test, y_sig_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}