Leave-one-out cross-validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Leave-one-out cross-validation\n",
    "\n",
    "General steps for LOO cross-validation:\n",
    "- Remove i-th datum from input data set\n",
    "- Fit model to remaining data set\n",
    "- Use fitted model to predict the i-th datum and store difference to original i-th datum\n",
    "- Do this for all i and compute variance of the differences\n",
    "- Select model with smallest variance\n",
    "\n",
    "I demonstrate below how to select the optimal order for a polynomial model with leave-one-out (LOO) cross validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install numpy matplotlib iminuit tabulate\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from iminuit import Minuit \n",
    "from tabulate import tabulate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "rng = np.random.default_rng(seed=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2dfZBc1Xnmn7enZ0AEPFJJCMsgMcaSIVjYbdQo2FllAVOAWTDZwK6J17GdIFH2kq1NwKv1mip7Q9Ykyy647IWNF8mUIQb5CxsEhYMTgxeZIEQPDAiBRYQsIa0IMrIYo9JHz0y/+0d3j27fuR/nfn/086tSaWb69r3n3I/nvPc57zlHVBWEEEKKTyXrAhBCCIkHCjohhJQECjohhJQECjohhJQECjohhJSEalYHnjdvno6MjGR1eEIIKSSjo6NvquqJTp9lJugjIyNoNBpZHZ4QQgqJiOx0+4yWCyGElAQKOiGElAQKOiGElAQKOiGElAQKOiGElAQKOiGElAQKOiGEpMjY3jGs3bwWY3vHYt93ZnnohBDSb4ztHcOqn6xCc6qJoYEhrLloDWrza7HtnxE6IYSkROONBppTTbTQwkRrAo034h1cSUEnhJCUqJ9Ux9DAEAZkAIOVQdRPqse6f1/LRUSOBfAEgGM62/9AVb9s2+YYAPcAWAZgH4CPq+qOWEtKCCEFpza/hjUXrUHjjQbqJ9VjtVsAMw/9CIALVPWAiAwC+LmI/FhVN1q2uQbAflVdLCJXA/jvAD4ea0kJIaQE1ObXYhfyLr6Wi7Y50Pl1sPPPvhDpFQDu7vz8AwAfERGJrZSEEEJ8MfLQRWRARMYA7AXw96r6tG2TkwHsAgBVnQQwDmCuw36uFZGGiDR+9atfRSs5IYTkmCTTE90wSltU1SkANRGZDeBHIrJUVV+0bOIUjdujeKjqnQDuBIB6vT7jc0IIKQNJpye6ESjLRVXfAvAzAJfYPtoNYCEAiEgVwDCAX8dQPkIIKRxJpye64SvoInJiJzKHiMwCcCGAX9g2Ww/g052frwLwmKoyAieE9CVJpye6YWK5LABwt4gMoN0AfE9VHxaRmwA0VHU9gG8C+FsR2YZ2ZH51YiUmhJAUGNs7Fjq9MOn0RDckq0C6Xq8rl6AjhOSRrDxwE0RkVFUdQ36OFCWEEBtZeeBRoaATQoiNrDzwqHC2RUIIsRG3Bx7Fjw8CBZ0QQhyIa4h+mn48LRdCCEmQNP14CjohhCRImn48LRdCCIkJJ688zZx0CjohhMSAl1ee5JS5Vmi5EEJIDOQhd50ROiGEhMRqsXS98onWRGa56xR0QggJgZPFksX8LVYo6IQQEgIni2XlWSsznfOFHjohhIQgj9MDMEInhJAQZDVFrhcUdEIICUla6Yim0HIhhBAfsljwOQyM0AkhxIM8L3ZhhxE6IaS0xBFZ52HAkCmM0AkhpaI72Gd4aBi3PHNL5Mg6DwOGTKGgE0JKg9UeqUgFLW31RNZhBD2P2SxuUNAJIaXBao9AgYpUIJDIkXXeslncoKATQkqD3R5Zfc5qjDfHcx9ZxwUFnRBSGtKwR9JaHzQMFHRCSKmIYo/4iXXeUxgp6ISQvsMq3ACMs2KcUhgp6IQQkjJO6YzVSlsCJ1uTRlkxeU9hpKATQnJFEh61VzojACjUKCsm7ymMFHRCSG4I61H7NQJu6YwDMgAAmNIp46yYPKcwUtAJIbkhjEdt0gh4pTN2j5vHiDsoFHRCSG4I4lF3o/I9B/b4NgJ+VknRhbwLBZ0QkhtMPWprVF6tVFGtVKdtE7dGIM9WSVxQ0AkhucJEeK3WzJRO4colV2LB8QtKYZtEgYJOCCkcdmvm8vdc3tdC3oWCTggpDNZsljynD2aFr6CLyEIA9wB4J4AWgDtV9Wu2bc4D8CCAX3b+9ENVvSneohJC+hGv+c1XnrUy6+LlCpMIfRLADar6rIicAGBURP5eVV+ybbdBVS+Lv4iEkH4lifnNy4zvEnSq+rqqPtv5+W0ALwM4OemCEUKItfOzpS1UpIIBGcjlsPs8EMhDF5ERAB8E8LTDxx8SkecB7AHweVXd4vD9awFcCwCLFi0KWlZCSJ/R7/ObB0VU1WxDkeMB/F8AX1HVH9o+eweAlqoeEJFLAXxNVZd47a9er2ujkd/FVgkh+SDP849ngYiMqqrj64lRhC4igwDuB3CvXcwBQFV/Y/n5ERH53yIyT1XfDFtoQggB+mNAUFz4eugiIgC+CeBlVb3NZZt3draDiCzv7HdfnAUlhBDijUmE/rsA/gjAZhEZ6/ztiwAWAYCqfgPAVQA+JyKTAA4BuFpNvRxCCCGx4CvoqvpzAOKzze0Abo+rUIQQQoLja7kQQggpBhR0QogjY3vHsHbzWoztHfPfmOQCzuVCCJlBnKvbM+0wPSjohJAZxLW6fZwNA/GHlgshZAbdEZpRh9k7NQwkORihE0JmENfq9vah+8NDw1i7eS3tl4QwHvofNxz6T0h/4DX9LUU9OF5D/2m5EEISpTa/hpVnrcR4c5z2S8JQ0AkhqRCXL0/coYdOCEkFL1+eqY3xQEEnhMSCiShbZ06ktx4/FHRCSGSC5ptzablkoIdOCIlM0HxzLi2XDIzQCSGRCZpvzqXlkoF56IT0GUl1QAb1xNkRGo7IS9ARQspBknOrdDs8125eazQPDJeWix966IT0EX5edxxT5jLfPDsYoRPSB1jtEKt3bRXbuKL3uOaBIcGhoBNScuxC7dYBGWbKXDcfnHZKNlDQCSk5dqEeb45j5Vkrpz83id6tcEBQCHZtAnZsAEZWAAuXJ3YYCjohJceeIuhls/ilD3JAUAh2bQLu/hgw1QQGhoBPr09M1CnohJQcL0/bL3q3Y90eClSkAoGw89OLHRvaYq5T7f93bKCgE0LC4+Zpe0XvVtxsGQ4IMmBkRTsy70boIysSOxQHFhHS5/gN8AlqyxAHYvTQObCIEOKKX0ZKUFuGOLBweaKdoV04sIiQEhHHwCA7HChUHBihE5Iwac1ZktSwfg4UKg4UdEISJMm5U+yEGRhkCgcKFQNaLoQkSNB5wgEz28RpG1ojhBE6IQlimhbYxSSid9uG1gihoBOSIEFF1sQ28dqG1kh/Q0EnJGGCiKxJRB806if9AwcWEZIzTLJiuNpPRFKaLCsJvAYWUdAJIf1FipNlJYGXoPtmuYjIQhF5XEReFpEtIvIfHbYREfm6iGwTkRdE5Ow4Ck4IIbHjNFlWSTBJW5wEcIOq/jaAcwFcJyJn2rb5KIAlnX/XAvibWEtJCHElidGhhWbXJmDDre3/nehOliUDiU+WlTa+naKq+jqA1zs/vy0iLwM4GcBLls2uAHCPtv2bjSIyW0QWdL5LCOlg9b4BOP4cxBNPc+BSIbDbKZf8NXBoX69XvnB522YpqIfuRaAsFxEZAfBBAE/bPjoZwC7L77s7f+sRdBG5Fu0IHosWLQpWUkIKjlV8q5X2ozfZmuz5OagoJzk6tJBY7ZTJI8AjNwCqM73ylCbLShvjkaIicjyA+wH8mar+xv6xw1dm9Laq6p2qWlfV+oknnhispIQUHLv4TrQmHH82GU3ahaNDbVjtlEoF0FYpvXI3jCJ0ERlEW8zvVdUfOmyyG8BCy++nANgTvXiElAdr/viADAAApnSq5+egoszRoTasdsqsucDffSGVhSUcySA10jdtUUQEwN0Afq2qf+ayzb8C8KcALgXwOwC+rqqeNWDaIulH4vbQY6fA+dmOJFEfk30mmBoZdYGL3wXwRwA2i0i3G/2LABYBgKp+A8AjaIv5NgAHAfxx1EITUkbso0bdfnYi8cFEWeZnJ9WQxO2Vm54jr3VEE2w0TbJcfg5nj9y6jQK4Lq5CEUJ6CZPNErgBSHEx4x6KNNDH9By5rSOacF05lwshBSBoNkuodMYUFzPuIauGJAym58gtNTLhulLQCQlJGvOpdI8xPDQcaEKuUOmMWeVnZ9WQhCHIOXKyexKuKwWdkA5BBDqNAT32Y6w+ZzXGm+NG5Qs9I2MW+dlxNCRpduZGOUcJN5oUdEIQXKD9IuA4onf7Mcab41h51kqj7xYunTGKSBbJgwcSbTQp6IQguEXhFQHHFb1Hnfe8bxa7KJIHnzAUdEIQXDy9IuC4huPX5tewpvbnaGx/FPXTLu4PcQ5Dlh58zvL2KeiEIJxF4RYBx7ai0K5NqD1wPWpTTeCFR4HhxbkQDVeyErc0OnOd6pZDq4eCTkiHuCyK2PzrIlgJXaGzD7NPW9yievBejYGbcOfw+lDQCQmAaWdnLI1D3tP5rEIn0pkIq5WsuMX9FmASZbsJt9P1ydiCoaATYkjqc4/nad5uJ6GyCp1W2rMbQpJrfJKwOEyG6M+a69yw2q8PkLkFQ0EnxJBM5h7Pw7zdbkJqj1CdFpMIezynRiwJi8N0iL5b3azXZ8OtmVswFHTS1wTJF4+ts7NouAlpEm8QXlF4EhaU6RD9Q/uAFTd47ysHFhkFnfQtQS2Uwg3WiQsvoXJ7gwjrJbst4NzdVxIWVFxD9HNgkVHQSd8SxkIp5WAdP/ENKlRRvG67kM6aO3NffpFyHIQV54wtMgo66Vv61kKxYiq+QYQqitdtF9IsUwPz0H8REAo66Vv61kKxkmZHoyl2IY3Ll87ZqM4koKCTwhNlIqzELJQo4pGm8KTZ0ZjlvkzfRAou+hR0UmiiTDHrt9/QkXsU8Yiaax1UkMIIpskx4rQr4uh4NXkTyeFQ/qBQ0EmhsXZsNqeauPnpm9HSVqSBP5EHEEURjyhrUZrmTtsJIr55Eb2g5TB5E8nhUP6gUNBJobF2bAoELW1FHvgTeQBRFPGIshaldZ+TR4BHbgBU4xXevIhe0HKYvInkII88KhR0UmisHZvDQ8O45ZlbImetRM5+iSIepgNdnATMus+k5lYJI3pJ+NJh88Sjin7OEVXN5MD1el0bjUYmxyblJa51PtNYLzSQ0AX15ZOc/TCJciddjhIhIqOq6hhlUNAJKQpBBSwPgrfhVuCxr7TfLGQAWPYpYHhh34lwnHgJOi0XQopC0MyRPAyMsVojlQHgufuA1mRhs0jyTiXrAhBSWHZtakeguzZlXZL80vWlL7gR+OAn22Jun6eFxAYjdFJIgnrcsXviQbzhPFgfWZan+6awaxMwtq7QWSR5h4JOCkfQPPGoeeWOjYFp2lwaIxTz0knph9OCEBtuzU9DVwIo6KRwmOaJd4V4z4E9ofPKXRsD07S5pEcoBm0wxndnm0dujdbzMECpZFDQSeEwyRO3CnG1UkW1UsWUTmGwMojhoWGs3bx2pv3iEOm6Nh6mOctJjVAMItBW8awMAJUq0EK2tkdeBiiVDAo6yS8uVoLJLIlWIZ7SKVy55EosOH5Be/DRpr9CszWBocog1lx8V/v7LhGjZ+NhySJx9eiTGKEYVKCt4tlCPlIHSzAqM49Q0Ek+8Xkl95sl0S7El7/nctTm17B2w5fbQi+CiakmGlt/1N6PS8Ro0nj4evRxj1AMKtB28fzAJ7KPhkswKjOPUNBJPon4Su4mxPXDhzGkigkAg6qoHz7c/oJHxOjXeMSyeHSQnPGgAp1X8cxDnnzJoKCTfDKyAmOzjkNjqIJ6s4VaiFdyJyGunXEl1jx379H9Xnhl+wNT0XOwgVJf+SiMQFM8+wIO/SexE0fO99jeMax69E9met1BcEvnC5si6GEDGdc5bznppHBEGvovIncBuAzAXlVd6vD5eQAeBPDLzp9+qKo3hS8uSYskJqCKK+d7z4E9aLam0AIwoa3gNoaXBx82WvWwgYxWPmKqHkkYE8vlWwBuB3CPxzYbVPWyWEpEUiHyIg4uePnJPQ3IkWbvAJMdGzA2ZwFWjX3VMdUwsI2Rx7UymapHEsZX0FX1CREZSb4oJE1i6chzwM1P7mlAKgNY8/pe1A4dbKfdQYDWJBqzh9GcfQJa0J5Uw1BvEHlcK5OpeiRh4uoU/ZCIPA9gD4DPq+oWp41E5FoA1wLAokWLYjo0CUNSHXlu2SW9DYiiMVRB7eAUMNXqfFNRP3QIQ3OG2xkollTDUCSV2RGlczGv2SakNBh1inYi9IddPPR3AGip6gERuRTA11R1id8+2SmaPaks4mA51qqfrGo3IFJxjNAxMISx378NDT2YSpmMYUcmyRGRF7jwEnSHbXcAqKvqm17bUdDzjavYRxA3Pw89tGAmKbj93JHJhiyXJLrAhYi8E8AbqqoishztOdb3Rd0vyQ7XDtOI4jYjE8T63bCCkbTg9mtHZj83ZAXGd4ELEVkH4CkAp4vIbhG5RkQ+KyKf7WxyFYAXOx761wFcrVklt5NYcOowBeAsbmnhtpiEvUzP3xds0Qm/RSq6HZky0F8dmVleaxIakyyXP/T5/Ha00xpJSXDtME0oS8PXy/eKFqMscWYShUYYQVpomJFTSDj034E0OwvziOuEVAlkaRjlw+/YgLEq0Pit30L9yARqVtvDWqbx3cDo3eb2iKmd4pfZUkZ7ghk5hYSCbiOpATdFo3akidpb48DsZu8HJlPGBmCGvbP1R6ht/WmPiIzNWYBVJ81DU4AhBdbMWYCeo4Vd4iyuKLSsPjvnfykcFHQbSQ24yQteIjz9mRyH2gPXe0accTV8PfaOVFB/6pvAoYM9x23oQTQrFbSgmJAKGnoQjkcKGlXGFYXSniA5gYJuI/WZ81KgK9TDQ8O45ZlbHEW4R6BFsKYK1CbdI864Gr4ee+ef/wm17WtmRLrta3JM7zVx86yDRpVxRKG0J0hOoKDbMFnQIDQZdJxZhboiFbS05SjCPQKtFTRmzWrnirtEnHE2fNPpjLM3AU/fPSPSnXFNjjTz51nTniA5gILugNHMeUHJqOPMKtRQoCIVCGSGCM8Q6I98Bdj/umvjk0jD5xHp9lyTDbcm71mXLWuF9AUU9LRIoePMyR+3C/Xqc1ZjvDk+Q4TDCLRJwxe449Qk0k3asy5j1grpCyjoaZGwCLl1UgYRatfMlpjLFJmkPeuyZq2Q0kNBT4sQIhQkuvXqpOyJpL1W8YkQlTqVNdGMoaCedRALhVkrpKBQ0NMkgAgFjW6NOim9RNswKnUSbrey5iZjKGhjxawVUlAo6DklaHRrt1YAYO3mtb0zG47vdhdtg6jUTbjdyppoxlAQwlgozFohBaRUgl6mIfthotuuiHquDlSpAi3MFG2DqNRNuL3KmkjGUFBooZA+oTSCXrYh+1GiW9fVgVoAln0KGF7oLNouUal1YJKTcIcpa6qNLy0U0ieURtBTH7KfQp5y2Oh2xnD6Zuvo9K8f+MTR8hrUYWzvGFY9+idotiYwVBnE6uX/xTXt0WjVe9ti0Kk1vrRQSB9QGkFPtQNu1yaMrfsDNIYqqD/1VdQu+G/AoX2xiLvRSkGApxA7jqy0b2/YUdjY+qN2QymCiakmxve+iJUr/sK7Ek4NheV41sWgyzhfTj8xunM/Nm7fh3NPm4tlp87Jujh9T2kEPc0OuLFf3I9VJw6jKYIhVaz5hy8cHSYfYRCK0UpBtjU43Y7Xk1PuFJ36dBR2H9T37h/HkGp74WZV1A8f9q6EW0NhOZ59MegyzJfTj4zu3I9/t3YjmpMtDFUruHfluRT1jCmNoAMhLIqQtknj2GPRFGlHrQAaxw6hdvhQ6EEo3ah8z4E9zrZRj/i2Ot9S9+OZRN8eHYXWB3V59d248/i3MHrMAOrNFmoXXuldGbeGwnK82iSw5uzV+VsMmgRi4/Z9aE620FJgYrKFjdv3UdAzplSCHogIA2nqp/9rDO14CBOtSQxWqr0etWkGhYOfXK1UUa1UMaVT7isF2SN0p+OZpOl5dBRaH9RnJhfj1dP/F1bO/WW0QTm249UWLneeAreElNWWOPe0uRiqVjAx2cJgtYJzT5ubdZH6nv4V9DC5yR0Rro2swJqL73JfwX7Drf7LlTn4yVM6hSuXXIkFxy+YsVLQ2O/fhsb2R1E/7WLUhhd7v1mYpum5dBTaH9R3f/B84NQ/MDo3GFnhnlHShx2TZbYllp06B/euPLeUjVVR6V9BD5qbbIvoa59ej9pZK49+3l0xxyTqt/nJg7OH0YRiQKq4/D2XOy48MZ0VMvZS21tfccPRctnFM2KaXuAH1ane3fL1OWW3JZadOqdU9Sk6/SvoQUXPJKL32WY6g2XOAtQ6jcn7JxTv2n0ptg41MdFcjGN3vgls7Y3w7SmZD736kP/KQhGj4UAPKiezcsXLliirFROGPJ6LPJbJj/4VdCCY6JlE9B7b2DNYVl94Pcb3bsHbzbPx4uhitBSoV17Be378CUB7M1isKZkDMoAHtj2AydYkhkSwelYV45XBmYsnx4jvjR1xJGYRHxxT3N52ymzFBCWP5yKPZTKhLwQ9llGJJhG9bZuxY4bQ6MynYo2ym1NN3PzKOrS0hWplC5YefxFWHHoTp1T2oaoTgLba4vj8fUc9+05K5p4De3D/K/e396OCm+fORgsuiyfHgNGNHcHiKeqDEwSntx27FXP/s7tL26j5kUdbKo9lMqH0gh7rlAAmEX1nmxkR+Tmrp6NsgUwvBTfVmsDlx6/DNRNvAZUqRAbbGSyVAeC5+6azWbqe/djeMTz06kNH9wNML5784K/34cnHt8UqCsY3dkiLp6gPTlSsVsxARfCD0d2YnCpvo+ZFHrNl4ihTFm+epRf01KcEcDnueHN8OsruLtY80ZrAIIBzDh1CBa22B312Z66V8d3A6N0zfGnrACrrfgakiu9sGMSRA1tjFQXPGzuG6Q/y+DCngdWK2fPWIazb9FrfNWpd8pgtE7VMWb15ll7Q056T22siK+vApyVzlhzt2Hzt+plzrezaBIytc/Slnfaz8/+9E/e+JLGLguuNHdMybXl8mE2JGoF1rZjRnftx/7O7S9uomZynPGbLRClTVpaaqGpiO/eiXq9ro9FI5VhpzeznZLM4TWQ1A69VhAwj4G5E0BWFxCOCDbcCj32l/QYhA8AFNzqnKpZ0seW4I7AsO4aTPHY/9JE4YX0eByoCiMRmqYnIqKo6Rqalj9CB9ObkdrJZVlpz1d1w858D+NKpR7ommS0lXmw5bu8/qwg1acHt1z6SrCy1cgl6xtFgWvaOW0SVqiiYZLaUOD+9aN6/2z2TtOBmcZ7y9LaTtqVWHsvFHg1e8tfhp7SN0DAkbe8U6hV21ya0vnU5MDUBDAyi8pmHSiPoQDjhMPlO3ILkdc+kYdVZ6wMgUbG11/VLl70P+w82Ix/bpA5+5zmuc9Aflos1Gpw8AjxyA6BqLO7TQuw1+tKF3gfQzN4J+9AW6RV2tLUE/6P5RSzTLRideh/+U2sJloXdV8pRVxIdeSaNcRINttc9k4ZVZ41Ukw5GrHVtTrTwpQdfREsV1Qg+trXcXvvxO89pnIPCCbprBGz1dEXag3O0NUPcf3Hxt/HTAyM9N29PZ6YI1lSB2qSZTTC6cz8++e3vojW0Dbf/42J8+5Mf971AUS6q6StsHkZfbty+D5smF2OjLsaAIHRPf9pvJaaRVtAymDTGYRpsv+jP755Jy6qLIxjxO//WuooIWqrt400pAIUi+LF7yu2xH5NnM+mArFCC7jlIyOrpzpoL/N0XZoi7TjXxyEPfx+0TH+t5UHs6M7WCxqxZRxes8BnG/sDLT2LgXXdiQCYBfQwPvHwKlp16med3TC+q081rj6gA4A7bYCJTAbTvP6hYBXm4ogyeSfutxO14URsWkwc+qOdsEj3mJTU0qp9ucv6tdZ1z3BBuenhLT6bJ1FTwY9vvY7f9mJznpPsUCiXovoOErFkhJ505Q9wnpYonJ8+Y8aDaOzNPrN2Ip3Zsx5wzL8Dh8W1oPHdHe9rapZ+YUabqcb8EZBIiCsVk+3cfTC6q183r9/pmIoBOXuNND28xFqugD1eUnn6n85WkF+12fYI0LCaNsdN3g4qvafSYhzzvqA2L6fm31vX0d54Q2bt2CqLc9uN3npNuXH0FXUTuAnAZgL2qutThcwHwNQCXAjgI4DOq+myspexgF97hoWGs7cyVUptfsz1EM8X91WM/gMH1W3Bd5UGMyvtw7mkfBtC7fN0Jejr+8/cPoDm5AEt3P4Y971qPCQGG9o1iDTBD1K84YwXW77wbE60JDA0M4ooz/CemMrmoUV7Pw7z6/fjF1wNFwUEfrig9/U4PVJJetNv1CWJ3+TXGfscP09hFiULjJolMrDDRrf14pm/CYfZjSpKNq0mE/i0AtwO4x+XzjwJY0vn3OwD+pvN/7NTm17Cm9udobH8Uw/Pfh1ueuWXafvn8+2/Dl79/wPkB7kTuZ+zahPuGbu50eD6IF94+EWs3H+wZxXnH49vQnNyKlgKzj30eOwWdpeYUje2PzhD02vwavnnx2mlff+rQqUctkMo/uWbL+F3UKK/nYV79Prp0AZ7Z8WvjhyXowxU1MrGer/Y1it+Ldjte0DqkaREFiR5NcBO3IG87SfV5JBHdFiprzABfQVfVJ0RkxGOTKwDco+38x40iMltEFqjq6zGV8Si7NqH2wPWoTTWxdvYTPSvH/2T7P6I5udT7IdqxAZXWBIAWxgYFq569BU3VHj/eKlRvHf4ABnUnJqEYVKB+2sWOxeo2Br1rcW7DfUM3t48XYlBN1NfzMK9+1tdTkygy6MMVV2Ti1Zh0hWfOcUOJeJUmdYjTJ00zenRL+et60U6pgGHfLsMSd3RbpKwxE+Lw0E8GsMvy++7O32YIuohcC+BaAFi0aFHwI9lXjp/9jvbK8VLBRad9GE+OHvB+iCyZMI1Zs9DU1nSD0PXje4Xqwxh4e+nRpd8cPHQr1ptjmW5plxWt0INq4n499/tu0H1l5cs6RaV3PL7NU3i626SRCx1XJBk1egzah+CW8lexZItY/+5WpiINuipSWU2IQ9DF4W+Oo5VU9U4AdwLtgUWBjzSyAq3KIDAFvL85hTX//CYa3dXozzsGi1cu9b6BLZkw9TkLUH3utumZCq2jOnuF6hO+Qt7FenOMyvuAgQeBboQ+siJSypvRd0s6b4oTVm++K3pW4ZmYbGH/wSauO3+xcR5xFJxGCEbdb5ToMUxj4JbyB1VUKgKB9qYCupTJtEHLQ2pt0p2UaROHoO8GsNDy+ykA9sSw3xlYB8ZTNqgAAAjgSURBVKqcUtmHq/Ux1A612pND7diAZSuW+1+Qjp8+tXM/Dr72G7SGtmGiuRh7n30NT73xPcw58wKccc6F/mXxzWL4MCqVD00L7GhrSehoy+jhLPG8KV5YRc8qPK7ZKRHykd1IyoeNEj2GaQys96815W/QwX7xK5Nfg5Yn7zqrN80kiEPQ1wP4UxH5DtqdoeOJ+OfoHahSr7yCq6obMIjJUMuebdy+D0cOLERLF2KZvIIVT12DQUxiYvsa/ALrpkXdSbjNsxiOZtpsNOjI8yqr73c95k1Jc+h12thFz8nfTToTJCkfNkr0GLYxsN6/bn0qQfpa3Mjau87D20ESmKQtrgNwHoB5IrIbwJcBDAKAqn4DwCNopyxuQztt8Y+TKqz1Jn1x4Aw8vnwtTnhjYzuqDhiNWvf1oYGXMYhJVKUF6CT2v/QYcM6Frp1Ee946FPhmjBJtGX3XZfbDNOyGLAnTeQwEa9SCDKCK24cNGz16nRdTMXM7dhwRbZbedZ7eDuKmcJNzWbMYggyE6WKdOmDq0KnYuH0flhx56WiEjip2XtaO0O94fBtu/Uk7hbECoHrca5BZrwKH3wM9MjId5ZkeOwsP3VqHbmeHAhgQ4PqLTsd15y8OVI5+I+yo27ySJzHL6pxZn4kiPgelmpyrGx2Y5CLbcZo64LrzawAW4xdz12H/S4/1eOg9r+qzXsPQwjWATAJaxb84/kacNe8DgW7GODNSHHGYPz2vA0+KQpjRiUkRhwBmbXVYycq7Lltmi5XCCXqXMBfFa+qAM865ELB1hlpfW18+9DJ+tvfoEP8FJ+3Bdb93ZSJ1i5OodkNSxDGAJQ3y8vDHFVnnpT5ZEocdlVcKZ7lYMT353e3mzX0d//OF66enDuiZ3MuHsb1juObRldPf/ebFaxOb77zIN5QJbuLkNZd1HhufNLHbBB9fvggnz56VnH2XIVmVL092lBelslysmLyy2S/SX/yb2/C2bA28AIV9iH8Sc54X4YZK8rXfbWBL1uciD2ltduss7MyVQD7q40aWz0Ce7KiwFFrQTbBfpDf3LcB15wdLcewSZG3SMDdm3m+ouOYJd3vtdxvYkta5yHPkarUJ0lyjMm2cnoHu35O+LmWwo0or6EnP6eFHGHHO+w3lVqegjZebh+k1sCXpc1GEt6NuZJ3mGpVpY38G5hw3lMhiI06UYdRoKQXduopQpbkYX7rs0tS92DDinPcbyq1OYUcl+mWLxDGAxZS8vx1Zyet9EofA2usWVxAR5Ph5OZ9hKKWg21cRenn/KfjLS7xXEYqbsA9dnm8otzo5RVX2VZTCHi/PDXCW5O0+iVNg7XWLK4joB0op6GFWEUqCvD10ceBUJyerJM/WhRN5jXqLQtrTHxStAU6LUgp6mFWESDSiDPjKEqdZEklw0p7+IC8NcN460ksp6GFSDEk8FClyKkJHaFHIQmCzboDzeP+UUtCBYCmGJD7yEjmZQB82XrIW2LTJ4/1TWkEn2VGUB7tIbxMkf+Tx/in00H9CopI3D5SEI8vpAtI+bmmH/hMSlaK8TRB3svSy83b/VLIuACGERMFtuoB+hIJOCCk0XS97QJAbLzsraLkQQgpNkTKrkoaCTggpPHnzsrOClgshhJQECjohhJQECjohhJQECjohhJQECjohhJQECjohhJSEzOZyEZFfAdgZ8uvzALwZY3GKAOvcH7DO/UGUOp+qqic6fZCZoEdBRBpuk9OUFda5P2Cd+4Ok6kzLhRBCSgIFnRBCSkJRBf3OrAuQAaxzf8A69weJ1LmQHjohhJCZFDVCJ4QQYoOCTgghJSHXgi4il4jIVhHZJiJfcPj8GBH5bufzp0VkJP1SxotBna8XkZdE5AUR+amInJpFOePEr86W7a4SERWRwqe4mdRZRP5t51pvEZH70i5j3Bjc24tE5HERea5zf1+aRTnjQkTuEpG9IvKiy+ciIl/vnI8XROTsyAdV1Vz+AzAA4FUApwEYAvA8gDNt2/x7AN/o/Hw1gO9mXe4U6nw+gOM6P3+uH+rc2e4EAE8A2AignnW5U7jOSwA8B2BO5/f5WZc7hTrfCeBznZ/PBLAj63JHrPPvATgbwIsun18K4McABMC5AJ6Oesw8R+jLAWxT1e2q2gTwHQBX2La5AsDdnZ9/AOAjIiIpljFufOusqo+r6sHOrxsBnJJyGePG5DoDwF8CuAXA4TQLlxAmdV4F4A5V3Q8Aqro35TLGjUmdFcA7Oj8PA9iTYvliR1WfAPBrj02uAHCPttkIYLaILIhyzDwL+skAdll+3935m+M2qjoJYBxAkRcUNKmzlWvQbuGLjG+dReSDABaq6sNpFixBTK7zewG8V0SeFJGNInJJaqVLBpM6/1cAnxSR3QAeAfAf0ilaZgR93n3J8xJ0TpG2PcfSZJsiYVwfEfkkgDqAf5loiZLHs84iUgHwVQCfSatAKWBynato2y7nof0WtkFElqrqWwmXLSlM6vyHAL6lqreKyIcA/G2nzq3ki5cJsetXniP03QAWWn4/BTNfwaa3EZEq2q9pXq84ecekzhCRCwHcCOBjqnokpbIlhV+dTwCwFMDPRGQH2l7j+oJ3jJre2w+q6oSq/hLAVrQFvqiY1PkaAN8DAFV9CsCxaE9iVVaMnvcg5FnQnwGwRETeLSJDaHd6rrdtsx7Apzs/XwXgMe30NhQU3zp37If/g7aYF91XBXzqrKrjqjpPVUdUdQTtfoOPqWojm+LGgsm9/QDaHeAQkXloWzDbUy1lvJjU+TUAHwEAEflttAX9V6mWMl3WA/hUJ9vlXADjqvp6pD1m3RPs00t8KYBX0O4dv7Hzt5vQfqCB9gX/PoBtADYBOC3rMqdQ538A8AaAsc6/9VmXOek627b9GQqe5WJ4nQXAbQBeArAZwNVZlzmFOp8J4Em0M2DGAFyUdZkj1ncdgNcBTKAdjV8D4LMAPmu5xnd0zsfmOO5rDv0nhJCSkGfLhRBCSAAo6IQQUhIo6IQQUhIo6IQQUhIo6IQQUhIo6IQQUhIo6IQQUhL+P78kaNqL5/W+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# create some toy data that follows polynomials of increasing degree\n",
    "x = np.linspace(0, 1, 100)\n",
    "y_set = np.empty((3, len(x)))\n",
    "for poly_degree in (0, 1, 2):\n",
    "    y_set[poly_degree] = np.polyval(np.ones(poly_degree + 1), x) + rng.normal(0, 0.1, len(x))\n",
    "    \n",
    "for y in y_set:\n",
    "    plt.plot(x, y, \".\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true polynomial order 0; selected order 0\n",
      "  model order    LOO variance\n",
      "-------------  --------------\n",
      "            0       0.0105525\n",
      "            1       0.0105614\n",
      "            2       0.0107262\n",
      "            3       0.0107479\n",
      "            4       0.0107608\n",
      "\n",
      "true polynomial order 1; selected order 1\n",
      "  model order    LOO variance\n",
      "-------------  --------------\n",
      "            0       0.0903401\n",
      "            1       0.0121111\n",
      "            2       0.0122902\n",
      "            3       0.0124952\n",
      "            4       0.0124547\n",
      "\n",
      "true polynomial order 2; selected order 2\n",
      "  model order    LOO variance\n",
      "-------------  --------------\n",
      "            0       0.347969\n",
      "            1       0.017877\n",
      "            2       0.0117249\n",
      "            3       0.0118761\n",
      "            4       0.011895\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# apply leave-one-out cross-validation\n",
    "for true_poly_order, y in enumerate(y_set):\n",
    "\n",
    "    variances = []\n",
    "    tested_poly_orders = np.arange(5)\n",
    "    for tested_poly_order in tested_poly_orders:\n",
    "        deltas = []\n",
    "\n",
    "        # leave out the i-th (x, y) pair in the fit\n",
    "        for i in range(len(y)):\n",
    "            mask = np.ones(len(y), dtype=bool)\n",
    "            mask[i] = 0\n",
    "            x_without_i = x[mask]\n",
    "            y_without_i = y[mask]\n",
    "\n",
    "            # least-squares cost function to fit a polynomial to the toy data\n",
    "            def lsq(par):\n",
    "                ym = np.polyval(par, x_without_i)\n",
    "                return np.sum((y_without_i - ym) ** 2 / 0.1 ** 2)\n",
    "            \n",
    "            m = Minuit.from_array_func(lsq, np.zeros(tested_poly_order+1), pedantic=False)\n",
    "            m.strategy = 0 # faster and does not compute errors automatically\n",
    "            m.migrad()\n",
    "\n",
    "            # predict the i-th y value with the fitted model and \n",
    "            # store the delta to the real value\n",
    "            assert m.migrad_ok()\n",
    "            \n",
    "            yi_predicted = np.polyval(m.np_values(), x[i])\n",
    "            deltas.append(y[i] - yi_predicted)\n",
    "\n",
    "        variances.append(np.var(deltas))\n",
    "\n",
    "    print(f\"true polynomial order {true_poly_order}; selected order {np.argmin(variances)}\")\n",
    "    print(tabulate(np.transpose((tested_poly_orders, variances)),\n",
    "                   headers=(\"model order\", \"LOO variance\")) + \"\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}