02 - Overfitting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overfitting\n",
    "### Explore bias-variance tradeoff visually\n",
    "### See what overfitting looks like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ml_helper import helper\n",
    "import numpy as np\n",
    "import numpy.random as npr\n",
    "import numpy.linalg as nlg\n",
    "import matplotlib.pyplot as plt\n",
    "from ipywidgets import interact\n",
    "import ipywidgets as widgets\n",
    "from warnings import filterwarnings\n",
    "filterwarnings('ignore', message='.*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "def generate_fit(n_data, degree):\n",
    "    \n",
    "    lbd = 0.00\n",
    "    LB = 1\n",
    "    UB = 4\n",
    "    \n",
    "    if degree >= n_data:\n",
    "        print(\"Reducing degree to equal one less than the number of points!\")\n",
    "        degree = n_data - 1\n",
    "    \n",
    "    \n",
    "    YNOISE = 0.3\n",
    "    XNOISE = 0.0\n",
    "    \n",
    "    NVAL = 50\n",
    "    \n",
    "    def func(x):\n",
    "        return np.log(x) + 0.5 * np.sin(3 * x) + x * 0.5\n",
    "    \n",
    "    def x_to_X(x):\n",
    "        out = np.zeros((len(x), degree + 1))\n",
    "        for i in range(degree + 1):\n",
    "            out[:, i] = x ** i\n",
    "        return out\n",
    "    \n",
    "    npr.seed(13371387)\n",
    "    \n",
    "    x_raw = np.linspace(LB, UB, num=NVAL) + npr.normal(scale=XNOISE, size=NVAL)\n",
    "    y_raw = func(x_raw) + npr.normal(scale=YNOISE, size=NVAL)\n",
    "    \n",
    "    ixes = npr.permutation(NVAL)\n",
    "    \n",
    "    x = x_raw[ixes[:n_data]]\n",
    "    y = y_raw[ixes[:n_data]]\n",
    "    \n",
    "    xv = np.linspace(LB, UB, num=NVAL) + npr.normal(scale=XNOISE, size=NVAL)\n",
    "    yv = func(xv) + npr.normal(scale=YNOISE, size=NVAL)\n",
    "    \n",
    "    X = x_to_X(x)\n",
    "    \n",
    "    coef = nlg.inv(X.T @ X + lbd * n_data * np.eye(degree + 1)) @ X.T @ y\n",
    "    \n",
    "    x_fit = np.linspace(np.min(xv), np.max(xv), num=NVAL)\n",
    "    X_fit = x_to_X(x_fit)\n",
    "    y_fit = X_fit @ coef\n",
    "    \n",
    "    plt.cla()\n",
    "    \n",
    "    # training plot\n",
    "    plt.subplot(1, 2, 1)\n",
    "    plt.plot(x_fit, y_fit)\n",
    "    plt.scatter(x, y)\n",
    "    plt.ylim((0, 4))\n",
    "    \n",
    "    ytfit = X @ coef\n",
    "    loss = np.mean((y - ytfit)**2)\n",
    "    st2 = np.sum((y - np.mean(y))**2)\n",
    "    rt2 = np.sum((y - ytfit)**2)\n",
    "    plt.title(f\"Train set: $R^2$ {1- rt2/st2:0.2f}\")\n",
    "    \n",
    "    # validation plot\n",
    "    plt.subplot(1, 2, 2)\n",
    "    plt.plot(x_fit, y_fit)\n",
    "    plt.scatter(xv, yv)\n",
    "    plt.ylim((0, 4))\n",
    "\n",
    "    Xv = x_to_X(xv)\n",
    "    yvfit = Xv @ coef\n",
    "    loss = np.mean((yv - yvfit)**2)\n",
    "    sv2 = np.sum((yv - np.mean(yv))**2)\n",
    "    rv2 = np.sum((yv - yvfit)**2)\n",
    "    plt.title(f\"Test set: $R^2$ {1 - rv2/sv2:0.2f}\") \n",
    "    \n",
    "    # epilogue\n",
    "    plt.gcf().set_size_inches((14, 6))\n",
    "    plt.show()\n",
    "    \n",
    "    return (1 - rt2/st2, 1 - rv2/sv2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be4a7cd1cf954354930de0b0312542e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=10, description='n_data', max=30, min=2), IntSlider(value=3, description…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "interact(generate_fit,\n",
    "         n_data=widgets.IntSlider(min=2,max=30,step=1,value=10),\n",
    "         degree=widgets.IntSlider(min=1,max=5,step=1,value=3));"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Bias-Variance tradeoff\n",
    "Our observations in the \"Decision Surfaces\" notebook and the above demo bring us to a very important concept:\n",
    "the **bias-variance tradeoff**.\n",
    "\n",
    "Because our input data is imperfect, our modell will err, and there are two fundamental types of error we play off against each other. The first is **bias**, or the inability of our model to capture the true shape of our data. The second is **variance**, or the error from the noise in our model parameters we get as a result of noise in our input data.\n",
    "\n",
    "Generally, there is a sweet spot where our model is complex enough to model our input, but not so complex that it has enough parameters to \"chase noise\" and **fail to generalize**. We will illustrate this below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJztnX+MHdd1379nSZsyZMVWtLukTFKiK6VQKTlQnJUgak2Kot3CdFQLMIJoiyYIkj9UtzbgtDXcxgaMoE6BpkEbp0gbQ3EC1I2DtdDITWBIjW2RkmhSsrmSrFiWmJQyLIo0vVzSDmM5Fp3dd/rH7HBnZ+fN3Jn769yZ8wEW3F2+N3Pmzr7zvefcc88QM0NRFEVRpDER2wBFURRFqUIFSlEURRGJCpSiKIoiEhUoRVEURSQqUIqiKIpIVKAURVEUkahAKYqiKCJRgVIURVFEogKlKIqiiGRzjJNOTk7yrl27YpxaUax4+umnzzPzVGw7cvSzpKSI6ecoikDt2rULCwsLMU6tKFYQ0cuxbSgy9eqr+P2Lrzo/7i1zN1i9/4q5Wav3X7rpoNX7q9hy4pHW73lt/mjr9zw//1Lr9wyN2wGjz5Gm+BRFcUpXcbp008HLXz7ocuwu12Ir7soaKlCKoqzDxsF2ceg+RcnF+VSk4qECpSiKE9o68tDCZHN+25Sl0g0VKEVRrOkiTlIwFaq216hRlD1RiiQURZGJb6fqSpieOH39ht/t22FXv3LppoONhRRXzM22Kpy4Ze4GLZqwQAVKURQrTCOLruJUJUamr2srWipS9tRNctqOgwqUoigAukVPvsTJVJTaHMdUrFSk2tHm7+bya3/zr4xer2tQpoxGwOIiwBzbEkVJijbi9MTp652Jk82xdU2qnlvmbrj85RMVKBNGI+Duu4EdO4D9+7OfFWXgmDhoU3HyKUxdz6UitZEQolREBcqEpSXg2DFgeTn7d2kptkX9QqPT6LR1Oq7FKQYqUuaEFqYcFSgTpqeBO+8ENm/O/p2ejm2RLGwERqPTXiJdnNqc30dJfCoiFUuYcmQKlLQZNRFw+DBw+jTw2GPZz0qGrcBodJocrjatxhanHBci1bXbhFShkmKbPIGSOqOemAC2blVxKmMrMBqdRse1IzKJOKSIU04skQLkRFOhCh/aIK/MvMrhbd0a2yplHLnAHDvWTWDy6HRpKXuvTgBE0+SEQ4nTQ4dObvjd+w7caHXMJ05f31iK3lSC3rb8PCcXhRil6JIEqYw8gbJ1eEpYXAhMHp0qwQntnLqKU5UgNb2mi2CZiFQTXUUKCCdUkkWpiDyB0hl1eqjADALb6KmLOJkIU9N72wpVk0j52MhbxodQpSJKReQJFBDe4Y1GwxLEoV2vEp224mQjTOOO1UaoJIgUsFFUTAUrRTGqQqZAhSQvyshTiocPZwLZV1xdr4pc8rRxYjbRU0xxKh83RZEq0hfhMaXHntiQHpQ5t6rKd3G9Uistm5C2faEnuNwn5Eucuh6/SVx9bORV1lCBSrzMubVWuLjeFEU9IVElok1E9CwRfcHXOVxGT3W0iZ58i5Ov86hI+WNYAlU1g27ahCt81t1aK1xsOk5R1NMS1Q8BeDG2ESa4ip5CiVOX87nas3XF3KwKVUuGI1B1M+hxm3ATmHV30grbTccpdtZIRFSJaAeAnwPwaV/nkBY9hRanLud1kerLUZEyZzgC1WUGncCsO5pW2Iicy6jU9FjpiOonAXwEwNjZEBHdT0QLRLTwNysr4Swr4aIwIpY4dTm/a5FSoWqm/wKVO7CpqfYz6ERm3Ul1YXIZlbY9lvCBIqJ7AJxj5qfrXsfMDzDzDDPPvHnTplbnkFQFFlucclza0TblqUJVT78FqujA7r4bePTRxhl0rmcrK8DiOQIfSmLWnQ4uo9IEItyWzAJ4LxF9G8A8gANE9MexjKlznC7LylPCV/dzFapq+i1QZQd24ULtDLqoZ9dcA2zfDuw/MIHRlNxZd3K4jEoTiXBNYeZfZ+YdzLwLwByAQ8z8i66OH2rtyQQp0VOO66KJSzcdtBIqFasMZxt1iWgTgAUAZ5j5HlfHtaJlX7+inl28mP1O+9U6pk0rq6bNwNoWyxiXqT3b6EmaOOW02chr2rPPZDPvOKpEyuWmX9Nzhjp3FS47SeSlsT/h8Jh2tHRgRT278krg1Vd7MTGXh0krK9OOFz3tA8jMjwF4LMa5dfZuRgiRKhPz3sQQTCcpvhClsZ1psTBeLPS6cAE4c0aXnqLRdn1J+H61mGj0ZI4v+7qm/KTjOx3pag2qVWnskuDF7FzPNm0SXfDVmuT8d5v1pbpqvuQu3C1txUmjp3a0LQhRoWqHtUB1KY2dmpqyPa09jh2XZD+YwH7jjRTD2UOHgHPnxg/uuGgryQt3R8gn5fYhespx3a+vij4LlUtcRFCiSmONcOy4pPvBZKuxJyay/WsHDtQP7rhoK9kLj4NGT2t0ESkboeqTWLn8O7IWKN+lsV5w7Lik+8Gkq7GrBrccro7rEpH0hduh0VMcbPaAFcWqT4JlQ7/3QY3DseOanARmZrJ1K4l+MJ0uPxWU79XkZHW4WlUMk/SFd6eLOGn0tJGuoupqo3JZsFISLVd/T04fWBizNLYVDvfPjEZZBmphAbj99my5RKIfTLYau3yvzp3bGFHVXViyF94NH+Kk0VN78nExKUNvQ929cFXKLolhRlCAs75sxQzU8ePA+fOO7FPWKN6rAaftmvDRZy+lWbsPbMU1ZNunqogrZuTlIorSR75b0rJZhReSe/q6jcGuu0ckN3jVdBUn34/T6EP01PZR8WWK4+Q6ojKlSqRSiLiGG0E5IvYyh/QKwg24MNhVV/LkBq8aX+I09OjJB12r/XwQO8IyIT2BErjhyOdTHJouV3oF4QYkGSzJlo684Se3dHqfbfplKNFTjutryYVKimD5Eirbv7O0BKonM15TTC43uSUZSQZLskUYQ32cRh0+BVeKYEmLqNJag6qa8fa4QsvkcpNr6C3JYEm2BERTe92xXY8yZZxIhVrDctng1oa0IqiAM14JmUTTyxX+oNiNmBgc6gYkN3h2hEjtAf1K75WJeW3lSMtntCVhopKWQAWqSAiRSTTxv7ELMLzRdPEDS+WGwkScJDilFJAkwD4FK/bfQ1oCBQSZ8fpeO2/jf3s3wTe5+B4UL0jDhThp9LQeqdfpWqxsRcomak9PoJpwkBpykUmsM8OX/5WQlmzE5OK1eMEpGjn546FDJ8UKFSCrrL0L/RIoR6kh29Rakxk+/G/5nMvLZmI1TtS8iZ3Jxfc2txkeVz3RNHqqp+9CFWsC0y+Bchia2KTWmszw4X+L5zx6FNi3r1mnxwmp1yUg04vvXW4zPKbi5Cq1p6wJlVSxSk2k0iozb0JC3yFDM1z3MC2ec2Ym6wvYVI0/rozdezX/wBq4SsalOEl1yrGoGo8QJepN+Gpk64N+CZSQfS0xzCiec2oqi4CadHqckE5PA3v2ZL/fs0eXgFJE151kUifiocXridPXixepfgkUIGZ2HsOM4jlNBHKckDJn3+df+c++6Unf1ui4EieNnsIybgx9Cldbkeq6gfeKuVm8Nn+09fv6J1AKAHOBrHpdjIYd+bpXHs0dPpzZ5vQEA1C/0EURin/KwiUhTRiKfhVJKE6IUeXtdevTQDb+uiqKaItGT2FxXYgheTIiV6Dq6pyT2PDTDQmXFqPK21gUuwzQADb+uhQnyQ5LWY8roWpzz0OuXcoTqNEIOHt2/Iy3x7NhSZcWusrbSBS7DlDPN/7GFCeNnmTQ1/sgS6ByB7RzJ3DkSPWMt+VsWEJEYsoAJvq1NIpi1wHq8cbfiavfaPQ6jZz6j2005fv+d1kflSVQuQNaWckUpWrGW5oNjyanazOBUiISE3o+0bfHZoAGvPHXlzj1ddaeOiHuS6g0nyyBKjqgvXuBV17ZOOMtzIZHhx7D3QdorACVJ9wvvBAukuoSuYWY6KcUUW6gx5GQL1Schklf7o8sgSo6oMcfB7Ztq3ZCq7PhpfNUm/Ep6t2VVwK33moWSdk6cZvIzffj411FlNGEbsCRUFt0I+6w6SJS0tJ8sgQKaOWAmjI+ud49+yzw6qtZ5rBp6cKFE5e6luTKrtRSp8p4NHpSJCNPoFpgkvGZmABuvhmYnTVbunDhxKWuJbmyS6oAK2toak8B0r9fSQsUYBZwtVm6cOHEpS6VuLJLqgArGSpOig2mfxtdU8ht0nyDaXVk2vqHGZifz5y3zVJH+XxSOu246BEopCevUoGWkytlHjp0Mtn2SMlHUC7J11auuw647z53BQAmazapVdclUauQ2qAKRqMnJQYqUAV8ra00HVeLDjwwwEHV1J7SN1SgCvhaW2k6rhYdNNM6GBrYoKo4KX1EBaqAr+KGpuNq0UE9dcHQWOHSQV2HitOw8XUvfe+1U4Eq4Wttpe64Uqr+pC7ZjAuGarN4UgY1AD6chIqTIgEVKCHELjqQvGQzLhhqzOLFHlQhaNWekiqDKTNX6onxFF1TxpW158KVP4U39SweEe0E8BkA2wCMADzAzL9b9x5+w5tqjznU1F6xrLoP1zNUVKAUAPKdfdX+rR7ux1oG8G+Z+RkiugrA00T0JWZ+IZQBKTrzpj0+5f9P8RpD88Tp67Fvx8uxzVCBUjJSd/ajEXD+fJq25zDzWQBnV7//ARG9CGA7gE4C1eeHD9psPM3fm9L1DhXrNSgi2klEh4noRSL6JhF9yIVhQyZWsUJqSzbLy1mPxbe8BbjmGpnrZ10hol0AfgbAV+NaIov3HbjRWVeEVLsrSMNnJZ+LIok8LfGPANwB4ANEtNvBcY2RWn3WBZ/FCn0bp337gKeeyr6/eLE/W56I6I0A/hTArzHz31b8//1EtEBEC0tL3688Rt+iJ5fCVD7uEGh7nRLSe4ADgWLms8z8zOr3PwCQpyW8MxoBZ8/KrT7rgq/9paGq9EKJ4NIScPz42s9XXdWPLU9E9Dpk4vRZZn6o6jXM/AAzzzDzzNTU1WENjIBvERmKSKWI0zLzurTE+lmfvdfNHe7OncCRI/Fmz64dsq/9pabCZ3M9IUvVi+O0Zw/wve+lv+WJiAjAHwJ4kZn/a9fj9Cl6CiUeKlIycSZQTWmJ9bO+Kevz5Q53ZSVzpjFmzz4csq/9pSbCZ3s9IbsLFcfp6NHsulJaPxvDLIBfAnCAiL6++vWe2EbFwFdKT/GDr3UoJwJlkpZwTdHh7t0LvPJKuNlzHmWcO+fHIfsoVjARPluBCd1daGICmJrK7kMf1tWY+SvMTMz808x86+rXwz7PKTF6iiVMfRXElK/LRRWfk7RE+/OuOdzHHwe2bQsnTnmUcd99WXoplbWPJuGzFZjQ3YUkd7+IRepdI2I709jnV9bjYh9Unpb4BhF9ffV3H/U98wPcPHyvLeUo49SpzI7y/ptYDyi0Oa+LvVAh74nk7hcpIC16UnFwT8gxvXTTQWw58YjTY7qo4guelohJOcrYtm1jVBJrZu/ivCnthdKG5YoPVCjloJ0kWmISZcSa2Q8toki9+4WyhoqCe7qOqZQ9UIB2M++E77Wcrgwxokgp4lOqkShOEm1qQyz7XVfzaQTlgVAz+/J6k0YUwyb1AgnFDamLaxGNoDzhe2Y/br1JIwrFBCkFEpKdqWTbxmFrs4v0nssoSgUqUUJuilUUH6QgACnYmCPJVlcipSm+RJH+/CZFUcLgSpgkFUfkaASVKL43xfap87mykdiz7djnb4NkWyWLk4soSgWqAtfO2Zez97Xe5GMflwqekiPZ4Y9Dms2p9Cq8dNNBK6FSgSrh2jmn2I7H9fqW1zFQ5VMCIUEQfAhTiNReV5FSgSrh2jmnWMzgej+VtzFwqXwqdEGQ4ORtiNnI1se5Q6475dFUG7FSgSrh2jmnuHnWxfpW0d97GwNXypdimFuBxEXuPhLyGVU+U3kp/L0MqorPpJGq682uKWyeLY4L89r3Xdsk5f4+rzA8fNjTGLgqZRxaj6hV3nfgxqD7oVKPnork1+Jy/EKNTwrClDMYgapymhMTG19j65yrcNnh23WX9PK4MANPPjl+jEwY5+9tBK/yml2pv9bse6dP4lSkfF0mghVzLFISJ2BAAtU0STYRsNj4sLE4LkePZr9bWbELJFz6+8ZrbqH+3oUuQUJEUX0VpyqkXmtqwpQjzAWvx+W6ddM6SArFDD5szMeFKBOmK6+0XytyuUcr2DLTgHtESXWqij37drycrDgBggXK9bp1k9NMoZihq411Qk8EzM+vRSU//CHw7LNrY9RlkuAyDenqvqQwAXFBV2fkS6RU/OKQujDliBUoFw6l7FzrJsmhH1fehTY25te+stIs9Nu2AbOzmQjMzgI337wmTm0nCaEnFutOXKOkKUxAYuNSTFLZSNonclHqgzDliBUoW4fSxVG6zPLE7B5RvPZ3vKNZ6MeJQJdJgo9IpfGaDW52ChMQCbgQFhWmcPRRlIqIFShbhxIzpRN7W03x2o8fB267rVnoq0SgyyQhSqRieLOHsszkwll1ESqNmsLRZ1EqIrqKz6Y8O0Tl8Lj9QybbalyXixcpX/uhQ8D58+3P1aW4LUpBnJaJe0MFRw5DEKQyogXKBt+Osm7/0KFD9f7Sd0l71bV3FfoukwSX+76MGHCZ+Dj27XhZn7DbA4YoSkV6K1CAH0eZRz7M66Mk5rX9Q+fP1/vLEI0L2ly7z2guGMFVUVH8MHRRKiJ2DUoixbWlX/iF9Wst5XWXuvUOSRVlsdfLFH+oo0uHvhc7dKXXEZRripHPk08Cp05lQjQ9nUVPJ04Au3c3RyGSMlIDbUM3GDTVJxcVo2ZUoFpQXovftm1tv9A739luTUlKRkpsfUEv8o4yUJGSRShh2nLikU7vc/EkXFeoQNVQ9pHFyGdyEjh3zrxqTypeojlbcUmhMaIQ6EcXjV6nIhUfn8LUVYyajhVbrPRTP4ZxazMTE8DUFHDgwNr/TU7KWVMyoU2HjU4Ht13UGkpfosBoSikOPtaWtpx4ZN2XL3we2wQVqDHU+cjy/+VVe7G6FLTpWuG9KMKFuEiqIkmANk5ERSocroUphCCNO28sVKDGUOcjq/4vVpukVoIzGmHphSUcO8b+gpPpaWDPHmDTpuzfLuKifYlaoyIlB5fCFEuUquyIgQrUGOp8pE//2TbCqQtY1gnd6oGnb30L7rzyOWzezH6CE+a1BTui7s0Ih9KXKBIqUn5wLUySiGGPClQNdT7Slf8sR0ttM2TjIr0NQreYHZhWlnH41dtx+tnzfoKTLik+X511B0ZbB6L7btziYiwlClOR0LapQEWkLCJnz2YFGG2WX4w7kdOakk3M7sHWmyf9BCdt1490p7BTujgQFSl7bMdQujDFgjjCrHVmZoYXFhaCn1cai4uZX15ezoRlYiJ7HtOjjwIXLtiVfTNn/j6v1H7sMYA40N6iNmXmxUHYvDlTWsE1+kT0NDPPxLYjZ+bm7Xz8wQ9s+H3X8mAtRW+PjTilKkq25edv2Px2o8+RRlAlQmabisFGsZffhQv26cPKyCrUuk6b82jFnhe6Oj5N+7Wj61ilHjGFsl0FqkDobFMuIq+8Auzd695HJ1Fn4KriRNexNmDjRFSomrERJ8UM7SRRIEZHiImJrGXSY48NuLOPbd8n7Twxli0nHrFKx+ROWFN/bnAtTq/NH218zRVzs07PmWP7t2WCE4EioncD+F0AmwB8mpn/k4vjhsakL52vFnFSevNtIIWeeCn3mgpA7hRthUpFao0u0ZMLcTIRpLr3+BIrX1hPM4loE4D/DuAggN0A/hkR7bY9bgyask2DKzhL5YJ1HcsIWwepab/u2Iz9a/NHL3/Z4uo4Ob7TlS4iqNsBnGTmbwEAEc0DuBfACw6OHZy6SGZwE/VULljS80ss8Z2NcJGWiR1NPXTopNHrfD2uvq1Id3XiLoWk6tgpRFMuEvXbAbxS+Pn06u/WQUT3E9ECES0sJdr8c3ATdZsLDl20kERFSD1dshGj77/a+jwuKshiRFIPHTppLE7562PTZZxdRzm+z+MzinIhUFUeYYNXYuYHmHmGmWempqYcnDY8g2sR1/WCU0kNyuNyNoKZfwwgz0bU0tXJpCJSbYWp/F6XtLnmruIUGski5UKgTgPYWfh5B4DvODiuSHowUW9HlwuW/LgM2eXorbMR53/4YwB2ImVbju4TFwITI5JKRZwknLsOFwJ1HMBPEdFbiej1AOYA/LmD4yqpIjUXKj+ya52NmLzy9Zd/b5OykSxSLpCQ7htHqJSeb3xEUdYCxczLAD4I4C8AvAjgQWb+pu1xoyJ7li0fk3LIGOMrObLLcJKN6ItISRMV02tsM46ShEliqs/JbkZmfpiZ/yEz38DM/9HFMS9Tdma+nZv8WXZroujBuNRgzPGVGtmt4SwbEWtdyhU+xEma4ElEkmAC0lsdlZ3Z8rJ/5yZ/lt2KIHrQRgFjjq/wKhfX2YjQ61IppPp8k2r05BKXkxzZAlV2ZidOVDs3lyGC/Fl2K7zqwWiUPSNk/35zBYw9vsKrXFxnI2KtS9ngM9KREkVJFidXqT4Xfz+yBarszHbvXvv5ttuyhye5DhGqZtkOBbDtoWxP7U0P8nG/7jrgyBFzBRwXxURalxrKcmMokeprFGVyXaZjJVmcclzZaCtSsgWq7MwmJrKHJc3MAF/7WuYgFxfdhwjFWbZDAWx7KBen9pbVKoZmRMCmTeYKWI5iIq1L9XC5sZYUHKPSP/JoqotYyRYoYKMzu3ABWFhYe3gSkd+UkcMcWdtDtXl9XSTgJatVDM327rVTwEjrUj1bbjSii0hJKZzoAylNEnzY2lao5AtUmXLOautWvwvfDnNkbQ9l+vookUA5NNu2rfvYtxkYhzm52MthsfAtUjZpPilrRG3pq4jHFtT0ngdV1RiUyF8TU4eNSNseyvT13nu6jnvkRh6ajUbAuXPdx8f0QpeXgX37gOPHnTz3qUc9ZluTO542DUNDPP+nz8R29imSXgQFhK/Ecni+tocyeb1RJNA18mgKz1yFb00XOhplqcQnn3SakxNe1OcddZpKEzH/RtIUKF8kWtLVWAhhIyJNCzWhFnKWlrLIKee224aTkxOEaSpLcjVf2zSii2vRiUA3VKByEi/pqo0EbESkKTwLtZAzPQ3MzmbVgnfcAXzlK8MNexyjztOOvq4/FYn1N9JfgWobDUUu6fIavHUVkXxt6dCh8eFZqO4M+XnOnMnuj8Xak7KRNg5oCA5Z2UgMkernp7xLNBSxpMt78NZFRIpGHTiQbYoe975QCzlDXzDyjEZSijT6KVBdoqGIfdqCBG9tnbuETUKJrgmG5EffuxTbBGekWmLeRJ+EP/S19FOgukZDkWboIvfjxDYq8TXBkDw//5KzY5k6oJTTfO87cGNsE5ImpEj1U6CEd60uI9Lc2EZJiOASIoZINSG5ks8VMYX6+fmXLn+FJpRIpSVQbVI+jqMh39km58GbC4NjrvnEjuASJIaj6jtSRVbCvQ4hUukIVMSUT3LZpuQMriB2BJcoIR1Xymk+xQ2+RSodgfKd8qmJOFycOuh6f4z0mI8L1Kq9TrgQqT4t7PeRW+ZuiG3CZWyeOdZEOgLlM+XTEHHYntpXQDNWE0Knx/oQsfUMCSkgxS+3zN1w+UsCPoQqnWaxPjt7NnRbtT21j2auuSYcO1bRNzV0F1Tv3WqVLjw//5KV83pt/mirZrKKAriNvtOJoAB/KR+DiMPm1D4CmsYsXoj0WB7CTU1pQYNijZZ/K2VkC1SohRvPC/KuDz8aZUOyZ09ETSim9e6+O3vSsRY0iENTfetREUwLuQJls67RRdjGRByuNNJVQJMPy86d2bFOnYqkCYuL60O4Cxe0oGGAaCWf4hO5AtW1Es3hgn35UGfPmguVr+CvPCwTEx40ocn40Qi4777MCKJ4aT1thWSERlHdkLoHakjIFaiuCzcOS6yLhzpyJItaTDTPZ1Gb1wK90ShT4Sbjl5ayBwcCmUJ+7nPhIyetHEwCSU5e03vpIVegui7cOPTgxUMxAysrZprncxuSt+WyYu7wyJF644sDMzsbp2JPWyEFQfdDKTGRK1BAu4WbPN0DrPfgzJ3TQLkYvPJK9rRxU83zvQ3JS4Fe7vBXVrKxqjNeQpcHbYXUilTSfBrlKEVkC5Qp5XQPkHlwZus00MQEsG1b5odN/bEE/92aosPfuzdT5TrjY3d5SHKQlVjEFD7dS9addDbq1jFuo6jDDaS5P/b1+uiE3tzrguQGWVGUNvQjghqX7tE0UDtiR0WK4oEu0ZOk4o4h0w+BGpfu0TSQoiSHy3SclDUtTfN1ox8pPmB8ukfTQIoipqGoorShHxGUoiheiDXzdxH5SImecjSKao8KVCJo0wRlaNgIjDRxUrqhApUAo1FWJb99O3DXXS2r5VXZlIRpKzTvO3CjtTj5LJDQKKodKlAJsLiYNXdYWcn+zfcjN6LtgBT4XX+6dNNBb8fOMRWcVKImFSlzrIokiOi3AfxTAD8G8BKAX2Hmv3Fh2FAZjTZuRSLKvpjXvjdCHySo9ISi+Dx06OSG36XGFXOz2kbKANsI6ksAbmHmnwbw1wB+3d4kmYTIlI0LeLZuXWu1tHdvC41p2gdmclH5a1ZWwqUKuw62pjM3YBM9SZ3pu0jjSUDq+ErCSqCY+YvMvLz641MAdtibJI9QmbJx/U87b+eqe6PJRRVfc8012SJY1wEwFY+ug63pzA1oaXl7Qm/QvWJuVoWqBpdrUL8KYOzTy4jofiJaIKKFpcQ6T4dqnF0X8HRu8jDujSYXVXzNxYvm7dzLtBGProOt3c3XoeIUBldrcEMRqufnX2rVuLhRoIjoy0T0fMXXvYXXfAzAMoDPjjsOMz/AzDPMPDM1NWVsoAR8dEyqCiiCNL7ITzw11XxRxQt/05uATZu6DUAb8egy2KNRNpB79mhbK0cMwVmWkdDeqK9C1VaYchqLJJj5XXX/T0S/DOAeAO9k7mfy33Uf1TygOHYs86WHD2eBDuC58UX5xI8+mj2qfdxFFS98chI4f77bAOSik5+3TjzaDnb5mk6dytrPD7StVcjIKUQF31DJRaoPhRQ2j3qxreJ7N4AOE1puAAAR3UlEQVR/B+AuZv47m2NFo6psrgKXwhGtuK584gsXmk9cvPCuRrYVnTaDXb6miYkkxclFRawrcerjDN4nl246iC0nxq5uWFG8F6mJlYtnkNmuQf0egKsAfImIvk5En7K2KCSRFtajNVmP2d3dV6f0/nSst6qI1TUnOySk95rI038pTCBcPSDTKoJi5rRrPSOFMtEevZTiM5+acHFNhlG0T5j5i4UfnwLw8ybve8NPbnEqTik4P9e4ECefUVQVkiMrl09vHnYniYiz72iPXurDM5/KFSY21ySzPN24Ivb8D38c0Kw1dP1JDpKiKpfiBAxdoKQ+L0o3nI7HtaAELE/3URE7eeXrndnny8lJTp+5tC22aEsSKlcMW6AAeRGFzBm9HFwLSsAompnfxcy3VHz9GbCuIvafh66IbePYYjtiV/gQTgljE0uoXEdPgAqUCNYFTD3YcOo1ADQVFFMjhETRhYrY94auiO3brNsEn1GdBJEC+hFRqUBZYuuMNwRMk2lXpXkPAE0Epa0RMqLoJCpipThfG0KkHC/ddFDMWIUQKR/RE6ACNR4D5XHhjDcETOdlzOi7EiQAbBKULkZEXvdj5huZeScz37r69f4Q5019ht2W0OthUoQq1fusAlWFofK4cMaVGSsZM/pOiNiW1NaI5WXgHe8Y3LpfW6clwdHaELNYIxeqmIKVokhZ7YPqLYb7o9p08BlH37YmibieNkaMRtkzTJ56Kvt5IM/NGpI4SawiLI9nqD1UqT2HSgWqCkPlEeGMBWLdFsrFxllTI5aWgOPH136+7bbk1v3akOIsuisShWkcRcEKueFXOpriq6JFZZdtNk6rykuEHpDpaWB2NuvUfscdwFe+0tuZRldxSi162rfj5aTEqYzvdGBKkxQVqHE0KI+rNfUeVJW7JfSA5JORM2fWms32kCGIU+rCNA4fQpWKSPXz0+gZl5P8rkUFvW02EerhW0USLkoxoe/i1FdhKiOlIjAkKlAdcDnJ77JPtHdpwaKAuN4427vBMsdmo6Z0R5iL0hCEqYwroUohilKB6oDrSX7bCXyv0oJVAuIyojEdrJ6FpDbOR6o4DVmUqpB6n1ySnkAJcCSxu+NkAsnYvJlx552cdtGZb7U1mU30KMqyjZqkOT0VpXps75mrKMrX88jSEihBjiTmsgXxCIf5bpzmHXiM94M4XYfqfWevyWyiJyFpX6ImFaX2SLp/LklrH1S0Z6XLYrS4hKVj/w/TK98BPXku7XEIsZmsaU+Uix3XEbGdBUtwbipG9oR+aGII0hKoxB2JMTUbVUcj4O65aRwbvYw7cQyH93wcE6mPg/XOXksS3nE9cfUbO783pDDt2/Eynjh9/bqfFRlI7i6RVoov9uJPCBrSmFkQSVjmzTi2eS+WHjwsYhwELA3a0fNS8zIxoiZN3fknZjTsYx0qKYEajYDFpQnwtFtHIsq5NqyHrF+yIUxvje9QBS0NKgb4dmJPnL6+8ksJg4SUrSuSEShfTlCcc20oGmgTRIYS3p7UGPQen1V6KkRpI7WaLxmB8uUExTlXAwUyyUaFFF4Rj9hQavEtTIos+hJFJSNQvpygSOfqYD0kpPC6WBoMnmYVldf1h4+oSaMlpQ6XUVQyAuWrPqKvdRehhddGU4OnWcXldf3gS5iUNGh7/122PnIlUskIFOCv0Co/LnN/JtWmwishkAieZhWX13WPK3HSaEnpiguRSkqgfNLHSXWToEu55uBpVpF5XXe4ECcVpTD0fYxvmbvBSqjS2qjrkSE2qZByzV72ydY9lTfhjbl1uBImxR9V41v+nct9YlK6S+Qi9fz8S63epxHUKpIm1aMRcPYs8N3v+k29Sbpmp+lbk9CwZxtzbcVJIya/tBnfmPfB9yM42kZUKlCrSCmWyH3r9u3AtdcCd93lL/Um5ZqdM4A1piIuxEnxR5fx1XuSoQJVQMKkOveteeTUxr92KXiQcM3OkRQaeobf8KbO79WoSTZ6bxITKFcVZxIq18aR+9ZcMEz9q5SCBxH0NjR0hzq/MKQ4zpKetJuMQLlywNIdee5bz5zJ1qEef9zMvw4sq9VML0NDezRqUlIiGYFy5YAXF4GjR2U78omJbP1p2zZz/5pqVktyNNs3VJjCouNtTzICZeqA6xzeaATMza1FTXv2mDty6Y7Ue1bLwwBIj2b7hARnKcEGxQwpab5kBMrEATc5vGIBwubNwIMPmjnyVBypj6zWaAQsnh2B97sfAE1LhkGFQUmVZAQKaHbATQ6vHIVNTZkFBUN1pJeF+TrC/iOfwGh5xekARElLSg+FHaPipKSME4Eiog8TERPRpIvjdaXJ4RWjsEOHgAMHzIKCVNd3bFkTZsIxuhNLm651OgDBi+1SCYUdoeKk2CAhzWfd6oiIdgL4xwBO2Ztja0tzB5s8CltcNG/z09POOI3kwnzsGHDnnZsw/blngK1uByC/H0GQ0tspACpO6eOy5VGquIigfgfARwCIyJmYrsO0jYqGWLW8PsIh0LbEB2CoobCSHFLEKXYUZRVBEdF7AZxh5ueowXER0f0A7geA6667zua0ThhqVNSWoBFOHXXNX00ZyE3X6CltpIiTBBojKCL6MhE9X/F1L4CPAfi4yYmY+QFmnmHmmampKVu7nTDEqCgnqVoBl2tHQ77pSlC6CI1EcYoZRTUKFDO/i5lvKX8B+BaAtwJ4joi+DWAHgGeIaJtfkweG5P1HoVRuqGWUHdDoSRamgrNvx8texMnVozZiiVTnNShm/gYzTzPzLmbeBeA0gLcz83edWTd0PFWdOfH3ISvidO1ISZhx4pP/XmLUJAV9YKFkPFWdra/O6+jvQ1bEDWTtyBaNnmSTuhDlUdRr80eDndPZRt3VSOq8q+Mp8BY5ONl/FDqq0bUjRWmFryfpXjE3Gyzll1wE5aKYKxk8Rg611Xkmg6xRjaIMGiuR+s2HjV6WVKujgTUCyJiYyHoynTsXpuSuzSBrVOMcKV1ZlLTxFT2FJimB8lXMJbrkOrQqa8VcNCR1ZVHSpS/iBCQmUDbLHuNESHxUFlowtGIuJqK6sijp0SdxAhITqK6L+3UiJD5gKAjGaM8sFnnab6Snj0uPQrEri8Fr7yeiBSJaWFr6fgDrlBTomzgBiQkU0G3Zo06EQgUMndOIq4IxOnUad9Nh7NhJ9ZGei3ylri15wU9Xlqv9Gq0kQR/FCUhQoLpQJ0LeAoaCUFinEScmsDSxFceOUX2kJz5fOWy0K4vimi0nHumtOAEDEagmEeoaMIwNVkpCsbQ4sk4jGkV6DvOVwQpHRFeohEG7sihd6LMw5QxCoIBuIlTnO2uDlZJQTNOSdRrRKNJzlK8MFohpxKcorel71FRkEALVZZLe5Dtrg5WSUNDW6dZpxCqbG0XWUb6y8tp8RDriK1Ti0LUrS+qtdJR6hiRMOb0XqNEoE5jt24G77jKfpDf5ztpgpUIo2kRwVoGFgwKHDdc26SnS0ZJ2RTFiaMKUk1yro7YsLgJHjmTfHzmS/Xzttc3va2qo2tjpx+JJf7GfTL7h2s55MkjbJTln346XtWlsjxiqMOX0XqCIsi/mte9N39fkO309bdZJt3FL1l2biUFdmySKeWSvosihj8J06aaDhZ8+ZvSe3gvU1q3A3r1rvrWNL3TtO5eXgRMngN27s2OPoyiOk5NZGz5Tv++lmW6TWuc5yXyQDx+uv0DFKxpFpUkfRQkoC1M7eu9FpDRGWF7OxOZtbwPe/Gbg7/++/vV5j9gDB8yXfrwWxdWtbdkWO2ipuXO0YCId+lz8YCNOwAAECpDRGOHECeDixez7H/wA2LOnWUDa+v1oRXG2TRK11NwLKlLy6bMw2YoTMBCBksDu3cBVV639/NxzzQLS1u9HK4qzCVO11NwrKlIy0ajJDBWoQExMABcuAD/7s+YC0tbvR01ndg1TtdTcO1JESoodMemzMAFuxQkYQJGEJF73OuBrX2tXxNC2UCO5ojgtNQ9CLg5aPBGPPgsT4F6cABWo4CQnICHQQQlGLKEacvQUWphemz+64XdWj2ePiAqUogyQomD4FisVJ39UiVHT63yIlY/oCVCBUpTBUyUgtqI1ZFHK8SVOpqLU9P4UoioVKEVRNqACY4drcbIVpXHHdCFSvqInQAVKUZKGfnQxtglecZF+DC22LsXJhzClhAqUoiTOlhOPeJ3FhsTHeljxmL7FypU4hRImV1GUL3QflKL0gD7srwlRWejzHC7G/7X5o8GjJslRmkZQitIjcieZWkQVsuz9idPXO4+kbMVJskjU4fvvTCMoRekhKUVUMTYPS9qwLEGcJNhQhQqUovSYlIQqNK5EymZ8pQqDFIgjPOKAiJYA1MXYkwDOBzKnC2qfHSnbdz0zT4U0pg6Dz5IJku6H2lJN32wx+hxFEagmiGiBmWdi2zEOtc8OtU8Wkq5XbalmqLZoik9RFEURiQqUoiiKIhKpAvVAbAMaUPvsUPtkIel61ZZqBmmLyDUoRVEURZEaQSmKoigDRwVKURRFEYkIgSKi3yCiM0T09dWv94x53buJ6K+I6CQR/fuA9v02EZ0gor8kos8T0ZvHvO7bRPSN1WtYCGBX7XgQ0RYi+tzq/3+ViHb5tmn1vDuJ6DARvUhE3ySiD1W8Zj8RXSzc84+HsK1kQ+39ooz/tjp+f0lEbw9tY2iI6MNExEQ0GdEGo8+bx/NH8TMVdjR+jiLYtImIniWiLwQ5ITNH/wLwGwA+3PCaTQBeAvAPALwewHMAdgey758A2Lz6/W8B+K0xr/s2gMlANjWOB4B/BeBTq9/PAfhcINuuBfD21e+vAvDXFbbtB/CFyH93tfcLwHsAPAKAANwB4Ksx7Q0wHjsB/AWyjb9B/o7H2GH0efN07mh+psKWxs9RBJv+DYA/CfXZFRFBGXI7gJPM/C1m/jGAeQD3hjgxM3+RmZdXf3wKwI4Q523AZDzuBfA/V7//3wDeSUTk2zBmPsvMz6x+/wMALwLY7vu8HrgXwGc44ykAbyaia2Mb5ZHfAfARAFErpyJ/3qL5mTLSPkdEtAPAzwH4dKhzShKoD66G9H9ERFdX/P92AK8Ufj6NODfrV5HNqqtgAF8koqeJ6H7PdpiMx+XXrH7gLwK4xrNd61hNK/4MgK9W/PceInqOiB4hoptD2rVK0/2S8jfnHSJ6L4AzzPxcbFtK1H3efCDynjd8jkLxSWQTmFGoEwZ73AYRfRnAtor/+hiA3wfwCWQO4xMA/guyP8x1h6h4r7OZXp19zPxnq6/5GIBlAJ8dc5hZZv4OEU0D+BIRnWDmJ1zZWMJkPLyOWRNE9EYAfwrg15j5b0v//Qyyflyvrq45/h8APxXKtlWa7lfU8XNNw2fwo8hSa9FtafF582Jaxe+i3vOGz1EoG+4BcI6Znyai/aHOG0ygmPldJq8joj8AULUAdxpZjjxnB4DvODANQLN9RPTLAO4B8E5eTcZWHOM7q/+eI6LPI0sX+BIok/HIX3OaiDYDeBOA73myZx1E9DpkH6rPMvND5f8vftCY+WEi+h9ENMnMwRpiGtwvr39zoRn3N05EbwPwVgDPrWaAdwB4hohuZ+bvhrSlYFPj580Tou550+coILMA3rs6mbwCwE8Q0R8z8y96PWvMBbfCwtu1he//NYD5itdsBvAtZB+kfPHy5kD2vRvACwCmal5zJYCrCt8fA/BujzY1jgeAD2B9kcSDgcaLAHwGwCdrXrMNaxvFbwdwKv85kI2N9wtZvr1YJPG1UPbF/ELAYp8x52/8vHk8dzQ/U2FL4+cokl37EahIQsoTdf8zEd2KLJT+NoB/AQBE9BYAn2bm9zDzMhF9EFmV0SYAf8TM3wxk3+8B2IIsDQQATzHz+4v2AdgK4POr/78ZwJ8w8//1ZdC48SCi/wBggZn/HMAfAvhfRHQSWeQ058ueErMAfgnAN4jo66u/+yiA61Zt/xSAnwfwL4loGcCPAMzx6l9/ICrvFxG9v2Djw8gq+U4C+DsAvxLQviFT+XkLceLIfqZM5eeImR+OZE9wtNWRoiiKIhJJVXyKoiiKchkVKEVRFEUkKlCKoiiKSFSgFEVRFJGoQCmKoigiUYFSFEVRRKICpSiKoojk/wObHIsbdJ+9NAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# First, let's generate some nice planar data and split it into training and validation sets\n",
    "\n",
    "def get_and_plot_data(seed=9999):\n",
    "    npr.seed(9999)\n",
    "    X, y, true_model = helper.gen_planar_samples()\n",
    "       \n",
    "    f = plt.figure()\n",
    "    helper.plot_red_blue(X, y, ax=plt.subplot(1, 2, 1))\n",
    "    helper.plot_decision_surface(true_model, ax=plt.subplot(1, 2, 2))\n",
    "    \n",
    "    return X, y\n",
    "\n",
    "# We use a validation set to model data we have not seen before and want to learn\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_tr, X_v, y_tr, y_v = train_test_split(*get_and_plot_data())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The bias-variance effect is canonically illustrated with a training curve, showing both the training and validation loss as a function of model complexity. While the training loss goes down monotonically in all but pathological cases, the validation loss ticks up when the model becomes too complex for the amount of data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Knn is a good classifier to illustrate this effect\n",
    "from sklearn.neighbors import KNeighborsClassifier as KNN\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier as DTC\n",
    "from sklearn.neural_network import MLPClassifier as MLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exercise**: what is \"model complexity\" for the KNN classifier?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b0355a555eb4aec8e042baf68dfd199",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(RadioButtons(description='model', options=('svc', 'knn', 'tree', 'mlp'), value='svc'), O…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_training_curves(model='svc'):\n",
    "    scores_tr = []\n",
    "    scores_v = []\n",
    "    \n",
    "    model = model.lower()\n",
    "    \n",
    "    if model == 'svc':\n",
    "        complexityspace = np.logspace(-4, 2, num=25)\n",
    "    elif model == 'knn':\n",
    "        complexityspace = range(101, 0, -4)\n",
    "    elif model == 'tree':\n",
    "        complexityspace = range(1, 26, 1)\n",
    "    elif model == 'mlp':\n",
    "        complexityspace = range(1, 10, 1)\n",
    "    else:\n",
    "        raise ValueError('Unknown model!')\n",
    "\n",
    "    # average over a number of randomizations to get smoother curves\n",
    "    for smoothing_iteration in range(5):\n",
    "        \n",
    "        scores_tr.append([])\n",
    "        scores_v.append([])\n",
    "        \n",
    "        X_tr, X_v, y_tr, y_v = train_test_split(*helper.gen_planar_samples(seed=smoothing_iteration*999)[:2])\n",
    "        \n",
    "        for parameter in complexityspace:\n",
    "            \n",
    "            if model == 'svc':\n",
    "                m = SVC(kernel='rbf', gamma=parameter)\n",
    "            elif model == 'knn':\n",
    "                m = KNN(n_neighbors=parameter)\n",
    "            elif model == 'tree':\n",
    "                m = DTC(max_depth=parameter)\n",
    "            elif model == 'mlp':\n",
    "                m = MLP(hidden_layer_sizes=(20 * (1 + parameter // 3),) * (1 + parameter // 2))\n",
    "            \n",
    "            m.fit(X_tr, y_tr)\n",
    "            \n",
    "            scores_tr[-1].append(m.score(X_tr, y_tr))\n",
    "            scores_v[-1].append(m.score(X_v, y_v))\n",
    "    \n",
    "    # smooth over iterations\n",
    "    scores_tr = np.mean(scores_tr, axis=0)\n",
    "    scores_v = np.mean(scores_v, axis=0)\n",
    "    \n",
    "    # plot against \"abstract complexity\", abstract away actual parameter values\n",
    "    plt.plot(scores_tr, label='Training scores')\n",
    "    plt.plot(scores_v, label='Validation scores')\n",
    "    plt.xlabel('Complexity')\n",
    "    plt.ylabel('Score')\n",
    "    plt.legend()\n",
    "    plt.ylim((0.5, 1))\n",
    "    plt.gcf().set_size_inches(14, 8)\n",
    "    plt.show()\n",
    "    \n",
    "interact(plot_training_curves,\n",
    "         model=widgets.RadioButtons(options=['svc', 'knn', 'tree', 'mlp'])\n",
    "        );"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}