Kernel Ridge, SVR, PCA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load the data\n",
    "\n",
    "K = np.loadtxt('ncicancer_kernel_hf_ex0.txt')\n",
    "y = np.loadtxt('y1.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2305, 2305), (2305,))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 47.,  54.,  50., ...,  90.,  76.,  79.])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import ShuffleSplit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "cv = ShuffleSplit(K.shape[0], n_iter=5, test_size=500, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.metrics.regression import r2_score\n",
    "\n",
    "models = KernelRidge, SVR\n",
    "\n",
    "scores = {model.__name__: [] for model in models}\n",
    "\n",
    "for outer_train, outer_test in cv:\n",
    "    for model in models:\n",
    "        est = model(kernel='precomputed')\n",
    "        K_train_train = K[outer_train[:, np.newaxis], outer_train]\n",
    "        y_train = y[outer_train]\n",
    "        \n",
    "        est.fit(K_train_train, y_train)\n",
    "        K_test_train = K[outer_test[:, np.newaxis], outer_train]\n",
    "        y_test = y[outer_test]\n",
    "        pred = est.predict(K_test_train)\n",
    "        score = r2_score(y_test, pred)\n",
    "        scores[model.__name__].append(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'KernelRidge': (0.34823307447592955, 0.044513698294497035),\n",
       " 'SVR': (0.21432798313133766, 0.014427004325354193)}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{key: (np.mean(value), np.std(value)) for key, value in scores.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "alphas = np.logspace(-3, 3, 20)\n",
    "Cs = np.logspace(3, -3, 20)\n",
    "\n",
    "models = (KernelRidge, 'alpha', alphas), (SVR, 'C', Cs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import KFold\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluating KernelRidge\n",
      "Best parameter for KernelRidge: 0.162377673919\n",
      "evaluating SVR\n",
      "Best parameter for SVR: 26.3665089873\n",
      "evaluating KernelRidge\n",
      "Best parameter for KernelRidge: 0.162377673919\n",
      "evaluating SVR\n",
      "Best parameter for SVR: 112.883789168\n",
      "evaluating KernelRidge\n",
      "Best parameter for KernelRidge: 0.162377673919\n",
      "evaluating SVR\n",
      "Best parameter for SVR: 54.5559478117\n",
      "evaluating KernelRidge\n",
      "Best parameter for KernelRidge: 0.162377673919\n",
      "evaluating SVR\n",
      "Best parameter for SVR: 54.5559478117\n",
      "evaluating KernelRidge\n",
      "Best parameter for KernelRidge: 0.162377673919\n",
      "evaluating SVR\n",
      "Best parameter for SVR: 54.5559478117\n"
     ]
    }
   ],
   "source": [
    "scores = {model[0].__name__: [] for model in models}\n",
    "\n",
    "for outer_train, outer_test in cv:\n",
    "    inner_cv = KFold(len(outer_train), n_folds=3)\n",
    "    for model, param, param_vals in models:\n",
    "        print(\"evaluating {}\".format(model.__name__))\n",
    "        inner_scores = []\n",
    "        K_train_train = K[outer_train[:, np.newaxis], outer_train]\n",
    "        y_train = y[outer_train]\n",
    "        K_test_train = K[outer_test[:, np.newaxis], outer_train]\n",
    "        y_test = y[outer_test]\n",
    "        for param_val in param_vals:\n",
    "            #print(\"testing param {}\".format(param_val))\n",
    "            param_scores = []\n",
    "            est = model(kernel='precomputed')\n",
    "            setattr(est, param, param_val)\n",
    "            for inner_train, inner_test in inner_cv:\n",
    "                est.fit(K_train_train[inner_train[:, np.newaxis],\n",
    "                                      inner_train],\n",
    "                        y_train[inner_train])\n",
    "                pred = est.predict(K_train_train[\n",
    "                            inner_test[:, np.newaxis],\n",
    "                            inner_train])\n",
    "                score = r2_score(y_train[inner_test], pred)\n",
    "                param_scores.append(score)\n",
    "            inner_scores.append(param_scores)\n",
    "        mean_inner_scores = np.mean(inner_scores, axis=1)\n",
    "        argbest_param = np.argmax(mean_inner_scores)\n",
    "        best_param = param_vals[argbest_param]\n",
    "        print(\"Best parameter for {}: {}\".format(model.__name__, best_param))\n",
    "        setattr(est, param, best_param)\n",
    "        est.fit(K_train_train, y_train)\n",
    "        pred = est.predict(K_test_train)\n",
    "        score = r2_score(y_test, pred)\n",
    "        scores[model.__name__].append(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'KernelRidge': [0.44333070663271967,\n",
       "  0.36307819027440247,\n",
       "  0.49275022845054484,\n",
       "  0.33807395840899068,\n",
       "  0.4050741557000449],\n",
       " 'SVR': [0.41549767795010839,\n",
       "  0.39331693768951792,\n",
       "  0.47944727269104948,\n",
       "  0.40553377137278235,\n",
       "  0.42177057985852218]}"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'KernelRidge': 0.40846144789334049, 'SVR': 0.42311324791239607}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{key: np.mean(value) for key, value in scores.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}