13.5-SVM-part-2-v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creative Commons CC BY 4.0 Lynd Bacon & Associates, Ltd. Not warranted to be suitable for any particular purpose. (You're on your own!)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  More Support Vector Classification: Grid Searching\n",
    "\n",
    "Here we'll try grid searching to find good parameter settings."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "from IPython.display import display, Markdown\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import scikitplot as skplt\n",
    "from sklearn import linear_model  \n",
    "from sklearn.model_selection import cross_val_predict, cross_validate\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score\n",
    "from sklearn.metrics import roc_curve, precision_score, recall_score\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "import os\n",
    "import shelve\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Data and Munge It"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "cervical=pd.read_csv('risk_factors_cervical_cancer.csv',na_values='?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "cervicalFeats=cervical.loc[:,'Age':'Hormonal Contraceptives (years)'].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "cervical2=pd.concat([cervical.Biopsy,cervicalFeats],axis=1).dropna(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(676, 9)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(676,)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X=cervical2.iloc[:,1:].to_numpy()  # features\n",
    "y=cervical2.Biopsy.to_numpy()\n",
    "X.shape\n",
    "y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set Up Train/Test Split, Pipeline, Grid Search Parameters\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y,\n",
    "            random_state=99,stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = Pipeline([(\"scaler\", preprocessing.StandardScaler()), \n",
    "                 (\"polynom\",PolynomialFeatures()),\n",
    "                  (\"svm\", LinearSVC(max_iter=100000,random_state=99))])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid={\"polynom__degree\":[1,2],\n",
    "           \"svm__C\":[0.001,0.01,0.1,1.0]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "    grid = GridSearchCV(pipe, param_grid=param_grid, cv=10,\n",
    "                       return_train_score=True,iid=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('scaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('polynom', PolynomialFeatures(degree=2, include_bias=True, interaction_only=False)), ('svm', LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "     intercept_scaling=1, loss='squared_hinge', max_iter=100000,\n",
       "     multi_class='ovr', penalty='l2', random_state=99, tol=0.0001,\n",
       "     verbose=0))]),\n",
       "       fit_params=None, iid=False, n_jobs=None,\n",
       "       param_grid={'polynom__degree': [1, 2], 'svm__C': [0.001, 0.01, 0.1, 1.0]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best params:\n",
      "{'polynom__degree': 1, 'svm__C': 0.001}\n",
      "\n",
      "Best cross-validation score: 0.93\n",
      "Test-set score: 0.93\n"
     ]
    }
   ],
   "source": [
    "print(\"Best params:\\n{}\\n\".format(grid.best_params_))\n",
    "print(\"Best cross-validation score: {:.2f}\".format(grid.best_score_))\n",
    "print(\"Test-set score: {:.2f}\".format(grid.score(X_test, y_test)))o"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multiple Learners in a Grid Search\n",
    "\n",
    "Here we try training both logistic regression and a support vector classifier as part of our grid search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = Pipeline([('preprocessing', StandardScaler()), ('classifier', SVC())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = [\n",
    "    {'classifier': [LinearSVC(max_iter=1000000,random_state=99)],\n",
    "     'preprocessing': [StandardScaler(), MinMaxScaler(), None],\n",
    "     'classifier__C': [0.001, 0.01, 0.1, 1]},\n",
    "    {'classifier': [linear_model.LogisticRegression(max_iter=1000000,\n",
    "                            solver='lbfgs',random_state=99)],\n",
    "     'preprocessing': [StandardScaler(), MinMaxScaler(), None],\n",
    "     'classifier__C': [0.001, 0.01, 0.1, 1]}\n",
    "    ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid = GridSearchCV(pipe, param_grid=param_grid, cv=10,\n",
    "                   return_train_score=True,iid=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('preprocessing', StandardScaler(copy=True, with_mean=True, with_std=True)), ('classifier', SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False))]),\n",
       "       fit_params=None, iid=False, n_jobs=None,\n",
       "       param_grid=[{'classifier': [LinearSVC(C=0.001, class_weight=None, dual=True, fit_intercept=True,\n",
       "     intercept_scaling=1, loss='squared_hinge', max_iter=1000000,\n",
       "     multi_class='ovr', penalty='l2', random_state=99, tol=0.0001,\n",
       "     verbose=0)], 'preprocessing': [StandardScaler(copy=True, with_mea...rue), MinMaxScaler(copy=True, feature_range=(0, 1)), None], 'classifier__C': [0.001, 0.01, 0.1, 1]}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best params:\n",
      "{'classifier': LinearSVC(C=0.001, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000000,\n",
      "     multi_class='ovr', penalty='l2', random_state=99, tol=0.0001,\n",
      "     verbose=0), 'classifier__C': 0.001, 'preprocessing': StandardScaler(copy=True, with_mean=True, with_std=True)}\n",
      "\n",
      "Best cross-validation score: 0.93\n",
      "Test-set score: 0.93\n"
     ]
    }
   ],
   "source": [
    "print(\"Best params:\\n{}\\n\".format(grid.best_params_))\n",
    "print(\"Best cross-validation score: {:.2f}\".format(grid.best_score_))\n",
    "print(\"Test-set score: {:.2f}\".format(grid.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "cvresDF=pd.DataFrame(grid.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "mean_fit_time                                                 0.00151751\n",
       "std_fit_time                                                 9.97572e-05\n",
       "mean_score_time                                              0.000422573\n",
       "std_score_time                                               6.82548e-05\n",
       "param_classifier       LinearSVC(C=0.001, class_weight=None, dual=Tru...\n",
       "param_classifier__C                                                0.001\n",
       "param_preprocessing    StandardScaler(copy=True, with_mean=True, with...\n",
       "params                 {'classifier': LinearSVC(C=0.001, class_weight...\n",
       "split0_test_score                                               0.923077\n",
       "split1_test_score                                               0.923077\n",
       "split2_test_score                                               0.923077\n",
       "split3_test_score                                               0.921569\n",
       "split4_test_score                                                   0.94\n",
       "split5_test_score                                                   0.94\n",
       "split6_test_score                                                   0.94\n",
       "split7_test_score                                                   0.94\n",
       "split8_test_score                                                   0.94\n",
       "split9_test_score                                                   0.94\n",
       "mean_test_score                                                  0.93308\n",
       "std_test_score                                                0.00848537\n",
       "rank_test_score                                                        1\n",
       "split0_train_score                                              0.934066\n",
       "split1_train_score                                              0.934066\n",
       "split2_train_score                                              0.934066\n",
       "split3_train_score                                              0.934211\n",
       "split4_train_score                                              0.932166\n",
       "split5_train_score                                              0.932166\n",
       "split6_train_score                                              0.932166\n",
       "split7_train_score                                              0.932166\n",
       "split8_train_score                                              0.932166\n",
       "split9_train_score                                              0.932166\n",
       "mean_train_score                                                0.932941\n",
       "std_train_score                                              0.000949161\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cvresDF.iloc[cvresDF.mean_test_score.idxmax(),]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": false,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": false,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}