3.1.kNNtemplate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# k-Nearest Neighbour Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load relevant packages \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "np.random.seed(42)\n",
    "\n",
    "from sklearn import neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the data\n",
    "fullData = pd.read_csv('data/monthlyData.csv')\n",
    "\n",
    "# Make a small dataset with \"labels\"\n",
    "smallData = fullData[[\"ISO_3DIGIT\",\"Annual_temp\", \"Annual_precip\", \"tropical\",\"labels\"]]\n",
    "# What happens to the PLA when the training data is not linearly separable? \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainData = smallData[:150]\n",
    "testData = smallData[150:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.scatterplot(x=\"Annual_temp\",y=\"Annual_precip\",hue=\"labels\",data=smallData, palette=sns.color_palette(\"hls\", 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare training and test data\n",
    "xTrain = trainData.iloc[:,1:-2].values\n",
    "nTrain = xTrain.shape[0]\n",
    "\n",
    "yTrain = trainData.iloc[:,-1].values\n",
    "\n",
    "## Process test data\n",
    "xTest = testData.iloc[:,1:-2].values\n",
    "nTest = xTest.shape[0]\n",
    "\n",
    "yTest = testData.iloc[:,-1].values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### As k-NN is based on computing distances between neighbours, it is imperative to perform feature scaling!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform standard normalization on the training set. \n",
    "# Use the obtained mean and standard deviation to standardize the test set\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Implement k-Nearest Neighbour classifier by completing this function\n",
    "\n",
    "# Functions such as np.argsort() can be useful here\n",
    "\n",
    "# Load the pairwise distance module\n",
    "from sklearn.metrics import pairwise_distances\n",
    "\n",
    "def kNN(X,Y,xTest,neighbors=1):\n",
    "    nTest = len(xTest)\n",
    "    yPred = np.zeros(nTest)\n",
    "    \n",
    "    # Compute pairwise distance between each test and training data point\n",
    "    \n",
    "    \n",
    "    # Obtain the indices of the k-nearest neighbours\n",
    "    \n",
    "    \n",
    "    # Obtain the votes of these k-NN\n",
    "    \n",
    "    \n",
    "    # Predict the test set labels based on majority voting\n",
    "    \n",
    "    return yPred\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Obtain predictions and print accuracy\n",
    "yPred = kNN(xTrain,yTrain,xTest,neighbors=1)\n",
    "acc = 1-(np.sum(yTest != yPred))/len(yTest)\n",
    "print('Test Accuracy of the KNN classifier is %f'%acc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot the decision boundaries\n",
    "h = 0.1\n",
    "x_min, x_max = xTrain[:, 0].min() - 1, xTrain[:, 0].max() + 1\n",
    "y_min, y_max = xTrain[:, 1].min() - 1, xTrain[:, 1].max() + 1\n",
    "xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                         np.arange(y_min, y_max, h))\n",
    "zInput = np.c_[xx.ravel(), yy.ravel()]\n",
    "Z = kNN(xTrain, yTrain, zInput)\n",
    "Z = Z.reshape(xx.shape)\n",
    "plt.pcolormesh(xx, yy, Z,cmap=\"Spectral\")\n",
    "plt.scatter(xTrain[:,0],xTrain[:,1],c=yTrain,s=10)\n",
    "plt.scatter(xTest[:,0],xTest[:,1],c=yPred,s=10)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross validation on the dataset\n",
    "### Task: Perform 5 fold cross validation and report aggregate test performance of the KNN classifier with k=[1,3,11,21]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use 175 data points. Leave the last three out, for instance.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model selection using nested cross validation\n",
    "## Hyper-parameter of the k-NN algorithm is the \"k\". How do we choose k? \n",
    "* Split the data into say, 5 folds. \n",
    "* Use 3 folds for training and one fold for validation. \n",
    "* Try different k=[1,3,11,21]\n",
    "* Choose the model with best validation error\n",
    "* Report test set on the last fold.\n",
    "\n",
    "Repeat this process for each configuration of folds.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}