MLE_MAP_Bayesian_linear_regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLE & MAP & Bayesian linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Name: [Your name]\n",
    "\n",
    "Skeleton framework for you to fill in (Code you need to provide is marked by `###`):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy.io as sio\n",
    "import matplotlib.pyplot as plt  \n",
    "import numpy as np \n",
    "from numpy import linalg\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_data(W, m=20, area_interval=[70,200], noise_mu=0, noise_sigma=400):\n",
    "    \"\"\"\n",
    "    Function: Creat the simulation data\n",
    "    Input:    \n",
    "              W:                The real weight\n",
    "              m:                The number of data\n",
    "              area_interval:    The area interval of house\n",
    "              noise_mu:         The expectation of Gaussian noise\n",
    "              noise_sigma:      The variance of Gaussian noise\n",
    "    Output:   \n",
    "              x:                The area of house\n",
    "              y:                The price of house\n",
    "    \"\"\"\n",
    "    \n",
    "    x = (area_interval[1]-area_interval[0]) * np.random.rand(20,1) + area_interval[0]\n",
    "    x = np.column_stack( (x , np.ones([m,1])))\n",
    "    y = x.dot(W.T) + (np.random.randn(m)*np.sqrt(noise_sigma) + noise_mu)\n",
    "    return [x, y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucXHV9//HXmxBCEEgChDRXE2xCTYomuKYqaluohB9a\nE1Ax/rA/UGyoTfFSBRMEWaQRNHLxVjSALSpIo0DIj6pAEFovQNgQLkkgEhsuWQIJYAJCEnL59I85\nm52Zvc3szsw5s/N+Ph772HO+c87M90D2vd/9nu/5fhURmJlZ/7dP2hUwM7PacOCbmTUIB76ZWYNw\n4JuZNQgHvplZg3Dgm5k1CAe+mVmDcOCbmTUIB76ZWYPYN+0KABx22GExfvz4tKthZlZXVqxY8XxE\nDC/1+EwE/vjx42lpaUm7GmZmdUXSk+Uc7y4dM7MG4cA3M2sQDnwzswbhwDczaxAOfDOzBpGJUTpm\nZvVoycpWFt62lme2bGPU0MGcPeNIZk0bnXa1uuTANzPrhSUrW5l/0yNs27kbgNYt25h/0yMAmQ19\nd+mYmfXCwtvW7g37Ntt27mbhbWtTqlHPHPhmZr3wzJZtZZVngQPfzKwXRg0dXFZ5Fjjwzcx64ewZ\nRzJ44ICCssEDB3D2jCNLf5M9e3JfNeLANzPrhVnTRnPxyUcxeuhgBIweOpiLTz6qtBu2e/bAt94C\nXx4Gdy2oel3beJSOmVkvzZo2uvwROQ/8AJae1b4/aUZlK9UNB76ZWS289ip8ZWRh2XmbYd/9alYF\nB76ZWbX9/Atw33fb9z/07zDlpJpXw4FvZtZHXT5x++wq+O4xhQdfsAWkVJ7SdeCbmfVBV0/czrpl\ncuGBn/gljHlLt+dAdZ/SLWmUjqShkn4q6TFJj0p6u6RDJN0h6fHk+7C84+dLWidpraTa3ZEwM6ux\n4idu37/Pb3l0wIcLD2reujfsOzsHavOUbqkt/G8Av4iID0raDzgAOBe4MyIukTQPmAd8QdJkYDYw\nBRgFLJM0KSJ2d/XmZmb1qv3J2uCJ/U8tfPEf74XD39jNOaWVV0qPLXxJQ4B3A9cARMRrEbEFmAlc\nmxx2LTAr2Z4J3BAROyJiPbAOmF7pipuZZcGooYO5db9zC8J+cxzMMfvf3GnYt51TTnmllNKlMwHY\nDPybpJWSrpb0OmBERGxMjnkWGJFsjwaezjt/Q1JmZta/vPI8v9l+En++zxN7i47afjXv3nNVt0/c\nVuQp3V4opUtnX+Bo4KyIuE/SN8h13+wVESEpyvlgSXOAOQDjxo0r51Qzs/Q1DynYfUaHc8y2K0oa\ncdP2WhZH6WwANkTEfcn+T8kF/nOSRkbERkkjgU3J663A2LzzxyRlBSJiEbAIoKmpqaxfFmZmqfmf\nu+EHMwvLvvQio/YZwPoy3qZXT+n2UY9dOhHxLPC0pLa/NY4D1gBLgdOSstOAW5LtpcBsSYMkTQAm\nAssrWmszszQ0DykM+yPfmxuBs8+Ars/JkFJH6ZwFXJeM0Pkf4GPkflkslnQG8CRwCkBErJa0mNwv\nhV3AXI/QMbO6dvv58NtvFpY1bwXqa5lDRaTfm9LU1BQtLS1pV8PMrKOivnpmfRemfgTo+AAV5G6+\nljxrZh9JWhERTaUe7ydtzcw689UJsO3FwrKkVd+muweougr8NP8icOCbmeXb/hJcMraw7KwH4NA3\ndDi0lAeo8gN+6AED+eP2Xezck+tZqfXC5w58M7M2xd030KFVn2/U0MG0dhL6bQ9QFXf5/OHVnR2O\n7ekvgkryildmZq0rOob9eZu7DXvo+QGqzrp8OlOrhc/dwjezxlYc9KOOhjl3lXRqTw9QlRrktVr4\n3IFvZo3p11fAsgsKy3po0XemuweouuryyVeLKRXauEvHzBpP85DCsJ9xca/CvieddfkMHCCGDh5Y\n/sLnFeAWvpk1jiuPgedWFZZVIejbpDVnTlcc+GbW/+3cDgtGFJad+d8w8s1V/+g05szpigPfzPq3\nModa9mcOfDPrnzb/Dr7z1sKy+a0w6MB06pMBDnwz63+KW/UHjYTPPZZOXTLEgW9m/ccDP4ClZxWW\nNWj3TWcc+GaWGX2aWKy4Vf+uz8Nx51e+knXMgW9mqVuyspXmpavZsq19rpmSJxb70Qdg3bLCMrfq\nO+XAN7NUdTanfJtuJxbbvQsuOrSw7PSfwfhjqlTT+ufAN7NU9TTBWKfz0XioZa+UNLWCpCckPSLp\nQUktSVmzpNak7EFJJ+YdP1/SOklrJc2oVuXNrP71NMFYwcRiL/y+Y9ifs95hX6JyWvh/HRHPF5Vd\nHhFfzy+QNBmYDUwBRgHLJE3yurZm1pnuJhgrmFisk1b9MfvfzNlrtzFrWjVr2H9UY/K0mcANEbEj\nItYD64DpVfgcM+sHOptgDGDYAQNzE4vtWdYh7Mdvv47x26/fe2N3ycrWWlW3rpUa+EGupb5C0py8\n8rMkPSzp+5KGJWWjgafzjtmQlJmZdTBr2mguPvkoRg8dvHcGySs+PJWVXzqeWbdMhv//qb3H3r/P\nmxi//XpAe8vabuxaz0rt0nlnRLRKOhy4Q9JjwJXAReR+GVwEXAp8vNQPTn5xzAEYN25cWZU2s/6l\nwwRjl/4Z3LKx8KDmrZwy7z87Pb9WK0bVu5Ja+BHRmnzfBNwMTI+I5yJid0TsAa6ivdumFchfAXhM\nUlb8nosioikimoYPH96XazCz/mLPnlz3zct5YX/CJXtvyna1MlStVoyqdz0GvqTXSTqobRs4Hlgl\naWTeYScBbZNMLwVmSxokaQIwEVhe2WqbWb/TPAS+PKyobCu87ZN7d3taQ9a6V0qXzgjgZkltx18f\nEb+Q9ENJU8l16TwBnAkQEaslLQbWALuAuR6hY2Zd+sMT8I2ieennLmfJhgNZeMkvO51mISsLitQb\nRUTadaCpqSlaWlrSroaZ1VoXD1B19vTt4IEDarocYD2QtCIimko93mvamlnt3fe9jmF//vN7++o7\ne/rWo3H6zlMrmFltlTAtQlejbjwap28c+GZWG1e/BzYUjd/oYkqErp6+9WicvnGXjplVX/OQwrCf\n9tFu57/xaJzqcAvfzKqnl7NaejROdTjwzazyXn0RvjahsOwTd8KYkgeUdHz61vrMgW9mleW56jPL\ngW9mlfG72+H6DxWWnbsR9jsgnfpYBw58M+s7t+rrggPfzHrvJ6fD6psLyxz0meXAN7PeKW7VH/Uh\n+MDVXR6+ZGWrR92kzIFvZuXpRfdN8dw4bStVAQ79GvKDV2ZWmtde7Rj2p95YUheO58bJBrfwzaxn\nfbwp67lxssGBb2Zde3o5XPOewrJz1sMBh5T1Np4bJxvcpWNmnWse0jHsm7eWHfbguXGywi18Myt0\n2xfhnm8XlvVxqKXnxsmGkgJf0hPAy8BuYFdENEk6BPgPYDy5JQ5PiYg/JMfPB85Ijv9URNxW8Zqb\n1bHMDlEs7qt//THwsZ9V5K09N076ymnh/3VEPJ+3Pw+4MyIukTQv2f+CpMnAbGAKMApYJmmS17U1\ny8nkEEU/KdsQ+tKHPxO4Ntm+FpiVV35DROyIiPXAOmB6Hz7HrF/J1BDF3Ts7hv3JVzns+6lSW/hB\nrqW+G/heRCwCRkTExuT1Z4ERyfZo4N68czckZWZGhoYoulXfcEoN/HdGRKukw4E7JD2W/2JEhKQo\n54MlzQHmAIwbN66cU83qWupDFDevhe8U/dH92TUwxO2y/q6kLp2IaE2+bwJuJtdF85ykkQDJ903J\n4a3A2LzTxyRlxe+5KCKaIqJp+PDhvb8CszqT6hDF5iEdw755q8O+QfQY+JJeJ+mgtm3geGAVsBQ4\nLTnsNOCWZHspMFvSIEkTgIlA0crFZo1r1rTRXHzyUYweOhgBo4cO5uKTj6ruDdvffqtjF84FW9yF\n02BK6dIZAdwsqe346yPiF5LuBxZLOgN4EjgFICJWS1oMrAF2AXM9QsesUE2HKBYH/ZCx8NlVtfls\ny5QeAz8i/gd4cyflLwDHdXHOAmBBn2tnZr132WR4qag31S36huYnbc36mwi4cGhh2fEL4B3/lE59\nLDMc+Gb9iYdaWjcc+Gb9wdYNcPmUwrK598PwSenUxzLJgW9W79yqtxI58M3q1UP/ATfPKSw7/wUY\n4B9r65z/ZZjVI7fqrRcc+Gb15JoZ8PS9hWUOeiuRA9+sXhS36t/xKTj+onTqYnXJgW+WdWV032R2\nYRXLBAe+WVa9+iJ8bUJh2SfuhDFNnR6eyYVVLFMc+GZZ1Iubst0trOLAN3Dgm2XL43fAdR8sLDt3\nI+x3QI+nZmZhFcssB75ZVvRxqGXqC6tY5vVlTVszq4QbP9Ex7Ju3lj3cMtWFVawuuIVvlqbioD/q\nQ/CBq3v1Vm399B6lY11x4JuloUpPytZ0YRWrOw58s1p67VX4ysjCslNvhIl/k059rKE48M1qxfPf\nWMpKvmkraYCklZJuTfabJbVKejD5OjHv2PmS1klaK2lGNSpuVjc2tHQM+3PWO+yt5spp4X8aeBQ4\nOK/s8oj4ev5BkiYDs4EpwChgmaRJXsjcGpJb9ZYhJbXwJY0B3guUMnxgJnBDROyIiPXAOmB676to\nVoduP78iQy3NKqnULp0rgHOAPUXlZ0l6WNL3JQ1LykYDT+cdsyEpKyBpjqQWSS2bN28ut95m2dU8\nBH77zfb9cW930Fsm9Bj4kt4HbIqIFUUvXQkcAUwFNgKXlvPBEbEoIpoiomn48OHlnGqWTc1DOm/V\nf/wX6dTHrEgpffjHAO9PbsruDxws6UcR8dG2AyRdBdya7LYCY/POH5OUmfVPu3fBRYcWlp20CN78\n4XTqY9aFHgM/IuYD8wEk/RXw+Yj4qKSREbExOewkYFWyvRS4XtJl5G7aTgSWV7riZpngm7JWR/oy\nDv9rkqYCATwBnAkQEaslLQbWALuAuR6hY/3O5t/Bd95aWPbZNTDET7ladiki0q4DTU1N0dLSknY1\nzErjVr1lhKQVEdH5ijid8JO2ZqW65ztw27mFZRdsASmd+piVyYFvDadX674Wt+oPHgP/vLp6lTSr\nAge+NZSy1329/CjY+lRhmbtvrE55ARRrKN2t+1ogIteqzw/74//FYW91zS18ayglrfvqm7LWTznw\nraF0u+7rS8/AZW8sfGHu/TB8Uo1qZ1Zd7tKxhtLVuq+/2X5Sx7Bv3uqwt37FLXxrKPnrvrZu2cYJ\nA1r47oDLCg86/3kYMDCF2plVlwPfGk5b6M+6ZXKH15bMXMMsh731Uw58azx3fplZvyqc3HX89usB\nGH3bWi8Cbv2WA98aS9EInIt3foTv7f7bvftdjeIx6w8c+NYYLv0zeHljQVFbqz7fqKGDa1Ujs5rz\nKB3r33b8Mdeqzw/7f2phycw1nY7WOXvGkTWuoFntuIVv/Vc3D1DNOiy3W/acOmZ1zIFv/c/z6+Db\nbyksO28T7DuooGjWtNEOeGsoDnzrX4pb9SPfDGf+dzp1McsYB771D6tuhJ9+vLDM89+YFSj5pq2k\nAZJWSro12T9E0h2SHk++D8s7dr6kdZLWSppRjYqb7dU8pDDsT77aYW/WiXJG6XwaeDRvfx5wZ0RM\nBO5M9pE0GZgNTAFOAP5V0gDMKu2WuR27cJq3wps+lE59zDKupMCXNAZ4L3B1XvFM4Npk+1pgVl75\nDRGxIyLWA+uA6ZWprhmwe2cu6Ff+qL1s7v1u1Zv1oNQ+/CuAc4CD8spGRETb4OZngRHJ9mjg3rzj\nNiRlBSTNAeYAjBs3rowqW0O76HDYvaOwzEFvVpIeW/iS3gdsiogVXR0TEQFEOR8cEYsioikimoYP\nH17OqdaItm7Iterzw/7cjQ57szKU0sI/Bni/pBOB/YGDJf0IeE7SyIjYKGkksCk5vhUYm3f+mKTM\nrHeK++knHg+n/qTLw3u1SLlZA+ixhR8R8yNiTESMJ3cz9pcR8VFgKXBacthpwC3J9lJgtqRBkiYA\nE4HlFa+59X+/u73zm7I9hP38mx6hdcs2gvZFypesdJvDrC/j8C8BFks6A3gSOAUgIlZLWgysAXYB\ncyNid9dvY9aJ4qA/8esw/e97PK27RcrdyrdGV1bgR8TdwN3J9gvAcV0ctwBY0Me6WSO640vwm28U\nlpXRT1/SIuVmDcpP2lo27NkDXx5WWDbnv2DU1LLepttFys0anKdHtvRd8aaOYd+8teywh64XKfe0\nx2Zu4VuaXnkeFr6hsGzeU7B/J9Malyh/kXKP0jEr5MC3dBTflB01DebcXZG39rTHZp1z4FttPXkP\n/NsJhWUXbAEpnfqYNRAHvtVOcav+2PPg3WenUxezBuTAt+r71WVw54WFZZ4SwazmHPhWPRFw4dDC\nstP/E8a/M536mDU4B75Vx9XvgQ1FM2q4VW+WKge+Vdb2l+CSsYVln18HB3pGVLO0OfCtcopvyh40\nCj73aOfHmlnNOfCt7zY+DN97V2HZl16EfbyypVmWOPCtb4pb9W+bCyd8JZ26mFm3HPjWOy3/Brd+\nprDMN2XNMs2Bb+UrbtV/5D/gyBM6P9bMMsOBb6X78Udg7c8Ky9yqN6sbDnzr2c5tsOBPCss+8wgM\nHZdOfcysV3qcD1/S/pKWS3pI0mpJFyblzZJaJT2YfJ2Yd858SeskrZU0o5oXYFXWPKQw7DUg16p3\n2JvVnVJa+DuAYyPij5IGAr+W9PPktcsj4uv5B0uaTG6x8ynAKGCZpEle17bOvPB7+NbRhWXnbYZ9\n90unPmbWZz0GfkQE8Mdkd2DyFd2cMhO4ISJ2AOslrQOmA/f0sa5WK8U3Zd/8f+GkK9Opi5lVTElL\nHEoaIOlBYBNwR0Tcl7x0lqSHJX1fUtsadaOBp/NO35CUWdatuqlj2Ddvddib9RMlBX5E7I6IqcAY\nYLqkPweuBI4ApgIbgUvL+WBJcyS1SGrZvHlzmdW2imseAj/9WPv+yVd5BI5ZP1PWKJ2I2CLpLuCE\n/L57SVcBtya7rUD+7FljkrLi91oELAJoamrqrouoX1uysjXd9VfvvRJ+Ma+wzEFv1i/1GPiShgM7\nk7AfDLwH+KqkkRGxMTnsJGBVsr0UuF7SZeRu2k4Elhe/r+XCfv5Nj7BtZ+5+duuWbcy/6RGA6of+\n7l1w0aGFZXPvh+GTqvu5ZpaaUlr4I4FrJQ0g1wW0OCJulfRDSVPJ3cB9AjgTICJWS1oMrAF2AXM9\nQqdzC29buzfs22zbuZuFt62tbuDfcCo8dmv7/pi3wieWVe/zzCwTShml8zAwrZPyv+vmnAXAgr5V\nrf97Zsu2ssr77JUXYOERhWXnbYJ9B1Xn88wsU/ykbYpGDR1MayfhPmro4Mp/WPNQCkbTvvscOPaL\nlf8cM8uskkbpWHWcPeNIBg8snDN+8MABnD3jyMp9yDMrk6GWeWHfvNVhb9aA3MJPUVs/fdVG6XhW\nSzPL48BP2axpoyt/g3blj+CWuYVlHmpp1vAc+P3Jnj3w5WGFZWc9AIe+IZ36mFmmOPD7i1vm5lr2\nbYZNgE8/mF59zCxzHPj1bvtWuKRoquL5rTDowHTqY2aZ5cCvZwsnwiub2venz4ETF6ZXHzPLNAd+\nPdr0GPzrXxSWXbAFpHTqY2Z1wYFfb4qHWn7gGjjqg71+u9QnbzOzmnHg14vVN8NPTi8s6+NQy1Qn\nbzOzmnPgZ10EXDi0sOyTv4URU/r81qlN3mZmqXDgZ9ltX4R7vt2+P2gIzH+qYm9f88nbzCxVDvws\n2vUa/MvwwrIvPAmDh3Z+fC/VdPI2M0udJ0/LmrsuLgz7N30411df4bCHGk3eZmaZ4RZ+VrzyPCws\nmgKhykMtqz55m5lligM/C378EVj7s/b9j/0cXv+Omnx0VSZvM7NM6rFLR9L+kpZLekjSakkXJuWH\nSLpD0uPJ92F558yXtE7SWkkzqnkBdW3rhty4+rawH/5nue6bGoW9mTWWUlr4O4BjI+KPkgYCv5b0\nc+Bk4M6IuETSPGAe8AVJk4HZwBRyi5gvkzTJ69oWWfKP8OB17fufeQSGjuv6eDOzPiplTdsA/pjs\nDky+ApgJ/FVSfi1wN/CFpPyGiNgBrJe0DpgO3FPJitetZ1bCor9q33/vZfDWMwoO8dOvZlYNJfXh\nSxoArAD+FPhORNwnaUREbEwOeRYYkWyPBu7NO31DUtbY9uyGq4/LBT7A/kPgc2thYOEQSD/9ambV\nUtKwzIjYHRFTgTHAdEl/XvR6ULBoas8kzZHUIqll8+bN5Zxaf9b+HL58SHvYn/pTmPdUh7CH7p9+\nNTPri7JG6UTEFkl3AScAz0kaGREbJY0E2ubpbQXG5p02Jikrfq9FwCKApqamsn5Z1I3XXoGvvQF2\nJQ83jXs7nP4z2Kfr37N++tXMqqWUUTrDJQ1NtgcD7wEeA5YCpyWHnQbckmwvBWZLGiRpAjARWF7p\nimfevd+Fr4xqD/t/+A18/Bfdhj10/ZSrn341s74qpYU/Erg26cffB1gcEbdKugdYLOkM4EngFICI\nWC1pMbAG2AXMbagROi8/C5fmPan6lo/B315R8ulnzziyoA8f/PSrmVWGct3v6WpqaoqWlpayz8vc\naJb//Bzcf3X7/j8/BgePLPttMnddZpZJklZERFOpx9ftk7aZGs3y3Bq48u3t+zO+Am+f2+u389Ov\nZlYNdRv4mZjLfc8e+MH74Ylf5fYH7AfnrPcC4maWSXUb+KmPZvn9L+GHJ7Xvf/g6eOP7avPZZma9\nULeBn9pc7ju3w+WT4dUXcvt/8iaYczfsM6C7s8zMUle38+GnMpf7imthwYj2sP/7X8I//Mphb2Z1\noW5b+DWdy/2VF2DhEe37b/ownLyo8p9jZlZFdRv4UKPRLLefD7/9Zvu+Z7U0szpV14FfVc8/Dt/O\nG9567Hnw7rPTq4+ZWR858ItFwPWnwOO3t5fNeyo3u6WZWR1z4Od74tfw7+9t3//ANXDUB9Orj5lZ\nBTnwAXa9Bt86GrY+nds/9E/hH++FAQPTrZeZWQU58B+6AW4+s33/47fBuLelVx8zsypp3MDf9gf4\n6vj2/Tf+LZzyQ5BSq5KZWTU1ZuDf9RX4r6+2739qJRxyRNfHm5n1A40V+C+uh29Obd9/1+fhuPPT\nq4+ZWQ01RuBHwE8/Bqtvbi87Zz0ccEh6dTIzq7H+H/hP3w/X/E37/sx/hWmnplcfM7OUlLKm7VhJ\nd0laI2m1pE8n5c2SWiU9mHydmHfOfEnrJK2VNKOaF9Cl3bvgO3/RHvYHj4bzNjnszaxhldLC3wV8\nLiIekHQQsELSHclrl0fE1/MPljQZmA1MAUYByyRNqum6tquXwE9Oa9//f0vhiL+s2cebmWVRj4Ef\nERuBjcn2y5IeBbqbsWwmcENE7ADWS1oHTAfuqUB9u7fjZbh4TPv+G46Dj97ooZZmZpQ5H76k8cA0\n4L6k6CxJD0v6vqRhSdlo4Om80zbQ/S+Iyvj1FYVhP3c5/N1NDnszs0TJgS/pQOBG4DMR8RJwJXAE\nMJXcXwCXlvPBkuZIapHUsnnz5nJOLbR1AzQPgWUX5PbfNheat8LwKi6EYmZWh0oapSNpILmwvy4i\nbgKIiOfyXr8KuDXZbQXG5p0+JikrEBGLgEUATU1N0ZvKs/0luHxK+/7nH4cDD+/VW5mZ9XeljNIR\ncA3waERcllc+Mu+wk4BVyfZSYLakQZImABOB5ZWrcp59B8GUk+C9l+Za9Q57M7MuldLCPwb4O+AR\nSQ8mZecCH5E0FQjgCeBMgIhYLWkxsIbcCJ+5VRuhs+8g+NC/V+Wtzcz6m1JG6fwa6OzO58+6OWcB\nsKAP9TIzswora5SOmZnVLwe+mVmDcOCbmTUIB76ZWYNw4JuZNQgHvplZg3Dgm5k1CEX0blaDilZC\n2gw8mXY9EocBz6ddiT7yNWRHf7gOX0N2FF/H6yNieKknZyLws0RSS0Q0pV2PvvA1ZEd/uA5fQ3b0\n9TrcpWNm1iAc+GZmDcKB39GitCtQAb6G7OgP1+FryI4+XYf78M3MGoRb+GZmDaKhA1/SZyWtlrRK\n0o8l7S/pEEl3SHo8+T6s53eqrWQN4U2SVuWVdVlvSfMlrZO0VtKMdGpdqItrWCjpsWSd5JslDc17\nrS6uIe+1z0kKSYfllWXuGqDr65B0VvL/Y7Wkr+WVZ+46uvj3NFXSvZIeTJZTnZ73WhavYaykuySt\nSf6bfzopr9zPdkQ05Be5hdXXA4OT/cXA6cDXgHlJ2Tzgq2nXtZO6vxs4GliVV9ZpvYHJwEPAIGAC\n8HtgQEav4Xhg32T7q/V4DUn5WOA2cs+WHJbla+jm/8VfA8uAQcn+4Vm+ji6u4Xbg/yTbJwJ3Z/wa\nRgJHJ9sHAb9L6lqxn+2GbuGTWwBmsKR9gQOAZ4CZwLXJ69cCs1KqW5ci4r+BF4uKu6r3TOCGiNgR\nEeuBdcB0UtbZNUTE7RGxK9m9l9x6yFBH15C4HDiH3GpwbTJ5DdDldXwSuCQidiTHbErKM3kdXVxD\nAAcn20PI/XxDdq9hY0Q8kGy/DDxKrmFasZ/thg38iGgFvg48BWwEtkbE7cCIiNiYHPYsMCKlKpar\nq3qPBp7OO25DUpZ1Hwd+nmzXzTVImgm0RsRDRS/VzTUkJgHvknSfpP+S9NakvJ6u4zPAQklPk/tZ\nn5+UZ/4aJI0HpgH3UcGf7YYN/KQfbCa5P4VGAa+T9NH8YyL3d1PdDWOq13q3kfRFcushX5d2Xcoh\n6QBy6z1/Ke26VMC+wCHA24CzgcWSOlvqNMs+CXw2IsYCnwWuSbk+JZF0IHAj8JmIeCn/tb7+bDds\n4AN/A6yPiM0RsRO4CXgH8JykkQDJ903dvEeWdFXvVnJ9ym3GJGWZJOl04H3Aqck/bqifa3gDuQbE\nQ5KeIFfPByT9CfVzDW02ADdFznJgD7l5XOrpOk4j93MN8BPauzsyew2SBpIL++sioq3uFfvZbuTA\nfwp4m6QDkpbLceT6zJaS+4dC8v2WlOpXrq7qvRSYLWmQpAnARGB5CvXrkaQTyPV9vz8iXs17qS6u\nISIeiYjDI2J8RIwnF5pHR8Sz1Mk15FlC7sYtkiYB+5GbtKueruMZ4C+T7WOBx5PtTF5DkkPXAI9G\nxGV5L1VcvUOLAAAAuUlEQVTuZzvtO9NpfgEXAo8Bq4AfkrvbfShwJ7l/HMuAQ9KuZyf1/jG5+w47\nyYXKGd3VG/giuTv4a0lGLaT91cU1rCPXJ/lg8vXderuGotefIBmlk9Vr6Ob/xX7Aj5KfjQeAY7N8\nHV1cwzuBFeRGstwHvCXj1/BOct01D+f9DJxYyZ9tP2lrZtYgGrlLx8ysoTjwzcwahAPfzKxBOPDN\nzBqEA9/MrEE48M3MGoQD38ysQTjwzcwaxP8Cwr7J2qdd4ZYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x4d63e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# make data\n",
    "W_real = np.array([3, 30])\n",
    "train_m = 20\n",
    "[train_x, train_y] = make_data(W_real,train_m)\n",
    "[test_x, test_y] = make_data(W_real)\n",
    "\n",
    "# draw the train data\n",
    "plt.plot(train_x[:,0], train_y, 'o')\n",
    "plt.plot(train_x[:,0], train_x.dot(W_real))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Maximum Likelihood Estimation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def MLE_train(x, y):\n",
    "    \"\"\"\n",
    "    Function: Calculate the weight and beta by Maximum Likelihood Estimation\n",
    "    Input:    \n",
    "              x:                Design matrix of training data, size:[num_sample dimension]\n",
    "              y:                Training real value, size: [num_sample 1]\n",
    "    Output:   \n",
    "              W:                The weight which be calculated by Maximum Likelihood Estimation\n",
    "              beta:             The beta which be calculated by Maximum Likelihood Estimation\n",
    "    \"\"\"\n",
    "    \n",
    "    # Your code\n",
    "    # Here you should calculate the weight.\n",
    "    # You may use two functions:\n",
    "    # 1. linalg.pinv(). This function is used to solve the pseudo-inverse of the matrix.\n",
    "    # You can know more about it from https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.pinv.html.\n",
    "    # 2. np.dot(). This function is dot product of two arrays.\n",
    "    # You can know more about it from https://docs.scipy.org/doc/numpy/reference/generated/numpy.dot.html.\n",
    "    W = ###\n",
    "    \n",
    "    tmp = y-x.dot(W.T)\n",
    "    beta = tmp.T.dot(tmp)/tmp.shape[0]\n",
    "    return [W, beta]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def MLE_test(x, y, W, problem_type='regression', retained_digit=0):\n",
    "    \"\"\"\n",
    "    Function: Predict the testing data\n",
    "    Input:    \n",
    "              x:                Design matrix of training data, size:[num_sample dimension]\n",
    "              y:                Training real value, size: [num_sample 1]\n",
    "              W:                The weight which be calculated by Maximum Likelihood Estimation\n",
    "              problem_type:     The type of problem which you want to slove. Only regression and classification\n",
    "              retained_digit:   The parameter used to classification\n",
    "    Output:   \n",
    "              t:                Prediction of testing data\n",
    "    \"\"\"\n",
    "    \n",
    "    # Your code\n",
    "    # Here, the variable t is the prediction of testing data, and you should calculate it.\n",
    "    t = ###\n",
    "    \n",
    "    # Three kinds of errors\n",
    "    er = t - y\n",
    "    MSE = (er**2).mean()                      # Mean Squared Error\n",
    "    MAE = abs(er).mean()                      # Average Absolute Error\n",
    "    SD = np.sqrt((( er-er.mean())**2).mean()) # Error Standard Deviation\n",
    "    \n",
    "    if problem_type=='classification':\n",
    "        acc = np.equal(np.round(t, retained_digit), y).mean() * 100\n",
    "        text = \"The Linear Regression's accuracy is %.2f%%.\" %(acc)\n",
    "        print(text)\n",
    "    \n",
    "    obj = pd.Series([MSE,MAE,SD], index=['MSE','MAE','SD'])\n",
    "    print(obj)\n",
    "    \n",
    "    return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  2.80295241  59.31179403]\n"
     ]
    }
   ],
   "source": [
    "[W_MLE,beta] = MLE_train(train_x, train_y)\n",
    "print(W_MLE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE    709.942263\n",
      "MAE     20.224034\n",
      "SD      26.471751\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X+cVVX97/HXh9+jKQOBxM8vmIBCP9AmrpVZaolfs0C7\nGV+zq2mRRZplKNCvUS83C8X6fr9dEywjjYwMB1KTxNR+XIUGQRGUQsFkQCEV1AKE4XP/2HuYs8+c\nM+ecmTNn73PO+/l4zGPOXmftM2ur857l2muvZe6OiIhUvm5xN0BEREpDgS8iUiUU+CIiVUKBLyJS\nJRT4IiJVQoEvIlIlFPgiIlVCgS8iUiUU+CIiVaJH3A0AGDBggI8cOTLuZoiIlJXVq1f/w90H5ls/\nEYE/cuRIGhsb426GiEhZMbPnCqmvIR0RkSqhwBcRqRIKfBGRKqHAFxGpEgp8EZEqkYhZOiIi5ahh\nTRNzl29k2649DKmtYcaksUw5fmjczcpKgS8i0gENa5qYtWQde/Y3A9C0aw+zlqwDSGzoa0hHRKQD\n5i7feCjsW+zZ38zc5RtjalFuCnwRkQ7YtmtPQeVJoMAXEemAIbU1BZUngQJfRKQDZkwaS03P7pGy\nmp7dmTFpbEwtyk03bUVEOqDlxqxm6YiIVIEpxw9NdMCn05COiEiVUOCLiFQJDemIiHRSR564jeMp\nXfXwRUQ6oeWJ26Zde3Ban7htWNPU7jnr7/oef957Nr/tdVVe5xRDXoFvZrVmdqeZPW1mT5nZe8ys\nv5ndb2Z/C7/3S6k/y8w2mdlGM5vUdc0XEYlXwU/c7vwrU5aO4+vdfgrAnc0n5z6nSPId0vkBcJ+7\n/08z6wUcBswGHnD368xsJjATuMrMxgFTgfHAEGCFmY1x9+ZsHy4iUq7yfuLWHf73UdD8xqGi0/bN\n5Rkfmv2cIsvZwzezvsDJwI8B3P0Nd98FTAYWhtUWAlPC15OBO9x9n7tvBjYBE4vdcBGRJMjridu1\ni+Dq2kNhf2+3Uxi5d1Ek7Nv7rGLJZ0hnFLATuNXM1pjZLWZ2ODDI3beHdV4ABoWvhwLPp5y/NSwT\nEak47T5x+/oOqO8LDV9ofXP2Nt746A9jeUo3nyGdHsAJwKXuvtLMfkAwfHOIu7uZeSE/2MymAdMA\nRowYUcipIiKJkfWJ26XjohXPng/v/GR4zuGZz+niWTr5BP5WYKu7rwyP7yQI/BfNbLC7bzezwcCO\n8P0mYHjK+cPCsgh3nw/MB6irqyvoj4WISJJEnrhd3wC/Sgv7+t3tn1MiOQPf3V8ws+fNbKy7bwRO\nAzaEXxcA14Xfl4anLAMWmdk8gpu2o4FVXdF4EZHEOHgQrukXLbtoOYw4MZ72ZJDvLJ1LgZ+HM3Se\nBT5DMP6/2MwuBp4DzgVw9/VmtpjgD8IBYLpm6IhIRavvGz3u0xdm/j2etrQjr8B397VAXYa3TstS\nfw4wpxPtEhFJvucegVvPiBT95iONfPTdo2NqUPu0tIKISEek9ervbZ7IF/dfTs2yZ2jucVgiV9FU\n4IuIFOK2s+GZ30eKRu5ddOh1yxOz2QI/jjV0WijwRUTy8ca/4P8MjhRd8sbl3Hew7XOlqU/MpgZ8\n7WE9eX3vAfYfDCYmtqyhA5Qk9LV4mohILvV924Q99btZd+QHMlZveWI2fWG1V/61/1DYtyjFGjot\nFPgiItk89Zu2M3Bmbz80rz7XvraZFlbLpKvX0GmhIR0RkUzSg/6tp8Kn74oU5drXNt8g7+o1dFoo\n8EVEUv3gnfDKlmhZhidlW7T3xOyQ2hqacoR+KdbQaaEhHRERgH++FPTqU8P+M/e1G/a5ZBry6dnd\nqK3piQFDa2v4zjlv1ywdEZGSSR++gU4FfYtcQz6lpsAXker1l1vgniuiZd98CboXLxrjWCQtGwW+\niFSn9F593UVw1o3xtKVEFPgiUl26aPimHCjwRaQ6vLIlmIGT6osr4ahjY2lOHBT4IlL5qrhXn0qB\nLyKVa8XV8Kd50bJv7wKzeNoTMwW+iCRG0VaSzLT71Gnfgvdfkbl+lVDgi0jsGtY0Ub9sPbv27D9U\n1uGVJDV8k5WetBWRWLWsKJka9i0KWkly29q2Yf/VpxT2KdTDF5FY5VpRMq8FyNKDvtebYHZTJ1tW\nefIKfDPbArwGNAMH3L3OzOqBzwE7w2qz3f3esP4s4OKw/mXuvrzI7RaRCpEr0NNXkkwd57/5sB9x\n+sE/RE9Qjz6rQnr4p7j7P9LKbnT361MLzGwcMBUYDwwBVpjZGHfPvSi0iFSd9laUTF9JsmX45+D+\nPWzucyEcTKk85SaYcF7XNrbMdcUY/mTgDnff5+6bgU1A2z3ARETIvKIkQL/DerZZSXLu8o081f2T\nbOxzYaTu+/rcpbDPQ749fCfoqTcDN7v7/LD8UjP7X0AjcIW7vwIMBR5NOXdrWBZhZtOAaQAjRozo\nYPNFpNzlvaLk+gb+vPeCSNHb997CaxyG7S3NjlHlLt/AP8ndm8zsKOB+M3sauAm4luCPwbXADcBF\n+f7g8I/GfIC6ujrPUV1EKljOFSXTbsoe8G4cs+/2Q8el2jGq3OUV+O7eFH7fYWZ3ARPd/dCdEjNb\nANwdHjYBw1NOHxaWiYgUJsOc+uOafxmZ1VPKHaPKXc4xfDM73MyOaHkNnA48aWapW7ifDTwZvl4G\nTDWz3mY2ChgNrCpus0WkorXsPpXq4z+G+t1855y3M7S2JpYdo8pdPj38QcBdFqw90QNY5O73mdlt\nZjaBYEhnC/B5AHdfb2aLgQ3AAWC6ZuiISN4y9Orf1+cuZhwYyxSStaFIuTH3+IfP6+rqvLGxMe5m\niEicfnM5rL41UnTs3lvZS28gGLpRbz7KzFa7e12+9fWkrYjEL/2mLN04Zu/tkbKWZRYU+B2nwBeR\n+GRZ6Gz0zHsyVs9rmQXJSouniUjp7Xi6bdh/4qeHlkXINs1S0y87Rz18ESmtPJYvnjFpLLOWrNP0\nyyJT4ItIaSy+ADY0RMu+9TJ0a7usQt5P30pBFPgi0rXc4eraaNkRg+GKp9s9TdMvi0+BLyJdR7tP\nJYoCX0SK7/lV8OMPR8umPQxDJsTTHgEU+CJSbOrVJ5YCX0SK478nwj/S9p9V0CeKAl9EOqd5P1w7\nIFo24XyY8sNIUerWhJp1Ew8Fvoh0XJ7DNy1bE7bMq2/atYdZS9YBKPRLSE/aikjhNixtG/aXr8s6\nhDN3+cbIQ1TQujaOlI56+CJSmA7clM22Bo7WxiktBb6I5KcTs2+G1NbQlCHctTZOaWlIR0Tat++1\ntmF/6jcLmoEzY9JYanpGl1DQ2jilpx6+iGRXpDn1WhsnGRT4ItLWIz+E5bOjZVc9BzW1mevnQWvj\nxC+vwDezLcBrQDNwwN3rzKw/8EtgJMGetue6+yth/VnAxWH9y9x9edFbLlLGEj0nXU/KVqxCevin\nuPs/Uo5nAg+4+3VmNjM8vsrMxgFTgfHAEGCFmY3RRuYigcTOSVfQV7zO3LSdDCwMXy8EpqSU3+Hu\n+9x9M7AJmNiJnyNSURI3J/3V7W3D/pxbFPYVKN8evhP01JuBm919PjDI3beH778ADApfDwUeTTl3\na1gmIiRsTrp69VUl38A/yd2bzOwo4H4zi+xc4O5uZl7IDzazacA0gBEjRhRyqkhZS8Sc9Hu+Bn9Z\nEC37xk7o0at0bZCSy2tIx92bwu87gLsIhmheNLPBAOH3HWH1JmB4yunDwrL0z5zv7nXuXjdw4MCO\nX4FImYl9Tnp932jY144IevUK+4qXs4dvZocD3dz9tfD16cA1wDLgAuC68PvS8JRlwCIzm0dw03Y0\nsKoL2i5SlmKbk67hm6qXz5DOIOAuM2upv8jd7zOzvwCLzexi4DngXAB3X29mi4ENwAFgumboiESV\ndE76i+vhpvdGyy68B0aeVJqfL4mRM/Dd/VngnRnKXwJOy3LOHGBOp1snIp2jXr2k0JO2IpVo4cdg\n88PRsm/vguD/1KVKKfBFKsnBg3BNv2jZ6NPhU7+Kpz2SKAp8kUqh4RvJQYEvUu6eeRBumxItm74K\nBmrpYYlS4IuUM/XqpQAKfJFyNPcY+OfOaJmCXnLQjlci5WT/3qBXnxr2J05X2Ete1MMXKRcavpFO\nUuCLJN3aX0DDJdGyK/4KRwzKXF8kCwW+SJIV2KtP9E5aEjsFvkgSdWD4JrE7aUli6KatSJL86+W2\nYX/m9XmN1SduJy1JHPXwRZKikzdlE7WTliSSAl8kbg9dBw99J1o2exv0Orygj0nETlqSaBrSEYlT\nfd9o2HfrEfTqCwx7SMBOWpJ46uGLxKEL5tTHtpOWlA0FvkgpvbIFfpC2n9DUX8CxZxbl40u6k5aU\nHQW+SKnoSVmJWd5j+GbW3czWmNnd4XG9mTWZ2drw68yUurPMbJOZbTSzSV3RcJGy8evPtQ37b72s\nsJeSK6SH/2XgKeDIlLIb3f361EpmNg6YCowHhgArzGyMNjKXquMOV9dGywZPgM8/nLm+SBfLK/DN\nbBjwEYKNyb+ao/pk4A533wdsNrNNwETgkc40VKSsaPhGEijfIZ3vA1cCB9PKLzWzJ8zsJ2bWspHm\nUOD5lDpbwzKRyrd1dduw/9zvFfaSCDkD38zOAna4++q0t24CjgYmANuBGwr5wWY2zcwazaxx586d\nuU8QSbr6vnDLqWllu2Hou+Jpj0iafIZ03gd8LLwp2wc40sxud/fzWyqY2QLg7vCwCRiecv6wsCzC\n3ecD8wHq6uq8Y80XSYAfvR9eeCJa9u1dYBZPe0SyyNnDd/dZ7j7M3UcS3Iz9vbufb2aDU6qdDTwZ\nvl4GTDWz3mY2ChgNrCpyu0Xi13wg6NWnhv07Phn06hX2kkCdmYf/PTObADiwBfg8gLuvN7PFwAbg\nADBdM3Sk4uimrJQhc49/NKWurs4bGxvjboZIbk/fA3ecFy378hPQ79/iaY9UNTNb7e51+dbXk7Yi\n+VKvXsqcAl8kl6v7gafNSFbQSxlS4EvVyXvf132vw3fSyj84Gz54VWkaKlJkCnypKnnv+6rhG6lA\n2gBFqkrOfV9XLWgb9lduVthLRVAPX6pKu/u+qlcvFU6BL1Ul076vW/qc17aigl4qkIZ0pKqk7vva\nn1fbhv3ZNyvspWKphy9VpeXG7JSl49q81zB5A1PeqYVdpXIp8KW6PPYzpiy7NFI0eu/P2E8PajLN\n1hGpIBrSkepR3xdSwv7h5ncwcu8i9of9nshsHZEKpB6+VL5rB0LzG5GiUXsXkWkVqWyzeEQqgXr4\nUrl2bw169alhf8mfoX43Q2prMp6SrVykEijwpTLV94Ubx6eV7Ya3vA2IztZpUdOzOzMmjS1VC0VK\nTkM6UlnW3A5Lp0fLMuw+1XJjNq81dUQqhAJfKoM7XF0bLfvof8K7Lsh6ypTjhyrgpaoo8KX8zRsH\nr6Ztm6yHp0TaUOBL+Xr5WfjP46NlV/wVjhgUT3tEEk6BL+UpfaGzYRPhs/fH0xaRMpH3LB0z625m\na8zs7vC4v5ndb2Z/C7/3S6k7y8w2mdlGM5vUFQ2XKvXHG9qGff1uhb1IHgqZlvll4KmU45nAA+4+\nGnggPMbMxgFTgfHAGcD/NbPuiHRG84Eg6B+4prVs6iKN1YsUIK/AN7NhwEeAW1KKJwMLw9cLgSkp\n5Xe4+z533wxsAiYWp7lSler7wrVvTivbDcd+JJ72iJSpfMfwvw9cCRyRUjbI3beHr18AWu6UDQUe\nTam3NSyLMLNpwDSAESNGFNBkqRrbH4ebT46WzXwe+hwZT3tEylzOHr6ZnQXscPfV2eq4u0PGpUmy\ncvf57l7n7nUDBw4s5FSpBvV9o2H/to8HvXqFvUiH5dPDfx/wMTM7E+gDHGlmtwMvmtlgd99uZoOB\nHWH9JmB4yvnDwjKR3O6dAavmR8sKHKdvWNOkJ2hFMsjZw3f3We4+zN1HEtyM/b27nw8sA1oeY7wA\nWBq+XgZMNbPeZjYKGA2sKnrLpbLsD/eUTQ37z9zXobCftWQdTbv24EDTrj3MWrKOhjXqc4h0Zh7+\ndcBiM7sYeA44F8Dd15vZYmADcACY7u7NnW6pVK4ibh4+d/lG9uyP/ufWss69evlS7QoKfHd/CHgo\nfP0ScFqWenOAOZ1sm1S6Zx+Gn30sWvaNHdCjd4c/Mtt69lrnXkRP2kpc0nv1770MTr+20x87pLaG\npgzhrnXuRbQevpTaL87L/KRsEcIetM69SHvUw5fS2PMKfHdktOwLj8CgcUX9MVrnXiQ7Bb50vfQe\nffde8M2dXfbjtM69SGYKfOk6Ty6BOz8TLfvWK9BNI4kicVDgS9dI79WfPgfe+6V42iIigAJfiu1H\nJ8EL66JlWtFSJBEU+FIcr26DecdFyy5/EmqHZ64vIiWnwJfOSx++efNouLQxnraISFYKfOm4VQvg\n3q9Fy769C8ziaY+ItEuBL4U7eBCu6RctO2cBvOPceNojInlR4EthvjMc9r0aLdNNWZGyoMCX/Oz8\nK/zw3dGyKzfDYf3jaY+IFEyBL7ml35Q95kNw/q/jaYuIdJgCX7J74Br44w3RMg3fiJQtBb601bwf\nrh0QLTt/CRyTcfsDESkTCnyJKuLuUyKSLAp8CTz/F/jxh6Jls7dBr8PjaY+IFF3OZQvNrI+ZrTKz\nx81svZldHZbXm1mTma0Nv85MOWeWmW0ys41mNqkrL0CKoL5vNOxPuCDo1SvsRSpKPj38fcCp7v66\nmfUE/mRmvw3fu9Hdr0+tbGbjgKnAeGAIsMLMxmgj8wT6f/8Nv/t6tEzDNyIVK2fgu7sDr4eHPcMv\nb+eUycAd7r4P2Gxmm4CJwCOdbKsUy/49MOct0bIvPgpHHZe5vohUhLx2ojCz7ma2FtgB3O/uK8O3\nLjWzJ8zsJ2bW8qz9UOD5lNO3hmWSBPPGR8N+xHuCXr3CXqTi5RX47t7s7hOAYcBEM3sbcBNwNDAB\n2A7c0M5HtGFm08ys0cwad+7suu3uJNS0Ohirf3Vra9k3X4KL7ouvTSJSUgXN0nH3XWb2IHBG6ti9\nmS0A7g4Pm4DURdCHhWXpnzUfmA9QV1fX3hBRRWtY09T1G26nT7U8ez6885PF/Rkiknj5zNIZaGa1\n4esa4MPA02Y2OKXa2cCT4etlwFQz621mo4DRwKriNrsyNKxpYtaSdTTt2oMDTbv2MGvJOhrWtPn7\n2DEr6tuGff1uhb1Ilcqnhz8YWGhm3Qn+QCx297vN7DYzm0BwA3cL8HkAd19vZouBDcABYLpm6GQ2\nd/lG9uyP/qPZs7+Zucs3dq6Xv3c3XDciWvbVp+DIIR3/TBEpe/nM0nkCOD5D+afbOWcOMKdzTat8\n23btKag8L+k9+vHnwCdu7fjniUjF0JO2MRpSW0NThnAfUltT+Ic98yDcNiVapt2nRCRFXrN0pGvM\nmDSWmp7dI2U1PbszY9LY/D/EPejVp4b9eb8KxuoV9iKSQj38GLWM03d4lk7DdFh7e7RMT8qKSBYK\n/JhNOX5o4TdoX98B14+Olmn3KRHJQYFfbtJvyv6PS+DfvxtPW0SkrCjwy8X6BvjVBdEyDd+ISAEU\n+El38CBc0y9adtFyGHFiPO0RkbKlwE+y2z8Om1a0HvfpCzP/Hl97RKSsKfCT6J8vwdyjo2XafUpE\nOkmBnzS3fgSe+1Pr8anfgJNndNmPK8nibSKSCAr8pNjyZ/jpma3HA4+F6Suz1y+ClsXbWtbzaVm8\nDVDoi1QgBX7cDjbDNWnz50u00FmXLd4mIomkpRXitL4hGvYfnB1MtSzRqpZdsnibiCSWevhx+NfL\n8L1Rrccj3gMX3gPdumc/pwsUdfE2EUk89fBL7b7Z0bCfvirYZrDEYQ9FWrxNRMqGevilsv1xuPnk\n1uMPzIRTZsXXHoqweJuIlBUFfldr3h8E/Y4NwXG3nnDls9DnyHjbFerQ4m0iUpYU+F1p7SJo+ELr\n8afuhNEfjq89IlLVcga+mfUB/gD0Duvf6e7fNrP+wC+BkQR72p7r7q+E58wCLgaagcvcfXmXtD6p\nXnsRbhjTejx6Epz3S21IIiKxyqeHvw841d1fN7OewJ/M7LfAOcAD7n6dmc0EZgJXmdk4YCowHhgC\nrDCzMVWzkfnS6bAmZVOSy9ZC/1HZ64uIlEg+m5g78Hp42DP8cmAy8MGwfCHwEHBVWH6Hu+8DNpvZ\nJmAi8EgxG544f18JPzm99fj0OfDeL3Xoo7TcgYh0hbzG8M2sO7AaOAb4obuvNLNB7r49rPICMCh8\nPRR4NOX0rWFZZdq/F/7rXfDq1uD48IFw+Tro2bG57FruQES6Sl7z8N292d0nAMOAiWb2trT3naDX\nnzczm2ZmjWbWuHPnzkJOTY6V82HOoNawv/AemLGpw2EP7S93ICLSGQXN0nH3XWb2IHAG8KKZDXb3\n7WY2GNgRVmsChqecNiwsS/+s+cB8gLq6uoL+WMRu1/Pw/ZS/eW//BJyzoCg3ZbXcgYh0lZw9fDMb\naGa14esa4MPA08AyoGXPvQuApeHrZcBUM+ttZqOA0cCqYjc8Fu7wy/OjYf/Vp+DjtxRtBk62ZQ20\n3IGIdFY+PfzBwMJwHL8bsNjd7zazR4DFZnYx8BxwLoC7rzezxcAG4AAwvSJm6DzzINw2pfX4rO9D\n3WeK/mNmTBobGcMHLXcgIsVhwfB7vOrq6ryxsbHg80oym+WNf8L1Y+CNcKJS/6PhiyuhR6/i/pwU\nmqUjIvkws9XuXpdv/bJ90rYks1n+OA8euLr1+LO/h2HvKs5nt0PLHYhIVyjbwO/SzTteegb+64TW\n47qL4ax5nftMEZGYlW3gd8lsloMH4faz4dmHWstmPAOHD+j4Z4qIJETZBn7RN+94+l644z9aj8+5\nBd7xiQ62TkQkeco28Is2m2XvbrhuROvx4Anw2Qege9n+oxERyahsU60om3esuBr+lDI2f8mf4S1v\ny15fRKSMlW3gQydms7y4AW56T+vx+74MH76meA0TEUmgsg78gh1shltOg21rWsuueg5qauNrk4hI\niVRP4K+7E359cevxJ38Ox50VX3tEREqs8gP/ny/B3KNbj0d9AD7dAN3yWihURKRiVHbg3/M1+MuC\n1uMvrYYBx8TXHhGRGFVm4Dc9BgtOaT0+9Ztw8tfia4+ISAJUVuAfeCOYffPSpuC415vgio3Q+03x\ntktEJAEqJ/BXL4TfXNZ6/Om74K2nxtceEZGEKf/Af3U7zDu29fi4j8K5txVtQxIRkUpR3oH/6jaY\nd1zr8eXroHZE9voiIlWsvAO/9xHB9zO+CydeEm9bREQSrvwDv3533K0QESkLevpIRKRK5Ax8Mxtu\nZg+a2QYzW29mXw7L682syczWhl9nppwzy8w2mdlGM5vUlRcgIiL5yWdI5wBwhbs/ZmZHAKvN7P7w\nvRvd/frUymY2DpgKjAeGACvMbIy7R/cjFBGRksrZw3f37e7+WPj6NeApoL01iScDd7j7PnffDGwC\nJhajsSIi0nEFjeGb2UjgeGBlWHSpmT1hZj8xs35h2VDg+ZTTtpLhD4SZTTOzRjNr3LlzZ8ENFxGR\nwuQd+Gb2JuDXwOXu/ipwE3A0MAHYDtxQyA929/nuXufudQMHDizkVBER6YC8At/MehKE/c/dfQmA\nu7/o7s3ufhBYQOuwTRMwPOX0YWGZiIjEKJ9ZOgb8GHjK3eellA9OqXY28GT4ehkw1cx6m9koYDSw\nqnhNFhGRjjB3b7+C2UnAH4F1wMGweDbwHwTDOQ5sAT7v7tvDc74OXEQww+dyd/9tjp+xE3iuw1dR\nXAOAf8TdiE7SNSRHJVyHriE50q/j39w97zHxnIFfbcys0d3r4m5HZ+gakqMSrkPXkBydvQ49aSsi\nUiUU+CIiVUKB39b8uBtQBLqG5KiE69A1JEenrkNj+CIiVUI9fBGRKlHVgW9mXwlXAH3SzH5hZn3M\nrL+Z3W9mfwu/98v9SaUVLmWxw8yeTCnL2u4krl6a5RrmmtnT4XIdd5lZbcp7ZXENKe9dYWZuZgNS\nyhJ3DZD9Oszs0vDfx3oz+15KeeKuI8t/TxPM7NFwNd9GM5uY8l4SryHbysTF+91296r8IljfZzNQ\nEx4vBi4EvgfMDMtmAt+Nu60Z2n4ycALwZEpZxnYD44DHgd7AKOAZoHtCr+F0oEf4+rvleA1h+XBg\nOcGzJQOSfA3t/Ls4BVgB9A6Pj0rydWS5ht8B/x6+PhN4KOHXMBg4IXx9BPDXsK1F+92u6h4+wfLQ\nNWbWAzgM2Eaw2ufC8P2FwJSY2paVu/8BeDmtOFu7E7l6aaZrcPffufuB8PBRgmU5oIyuIXQjcCXB\nQ4ktEnkNkPU6vgBc5+77wjo7wvJEXkeWa3DgyPB1X4Lfb0juNWRbmbhov9tVG/ju3gRcD/ydYPG3\n3e7+O2CQh08MAy8Ag2JqYqGytTuv1UsT6CKg5QntsrkGM5sMNLn742lvlc01hMYA7zezlWb2sJm9\nOywvp+u4HJhrZs8T/K7PCssTfw1pKxMX7Xe7agM/HAebTPC/QkOAw83s/NQ6Hvx/U9lNYyrXdrcI\nl+Y4APw87rYUwswOI1h25Ftxt6UIegD9gROBGcDicF2tcvIF4CvuPhz4CsGaYImXYWXiQzr7u121\ngQ98CNjs7jvdfT+wBHgv8GLLwnDh9x3tfEaSZGt3Wa1eamYXAmcBnwr/44byuYa3EnQgHjezLQTt\nfMzM3kL5XEOLrcASD6wiWEdrAOV1HRcQ/F4D/IoyWNE308rEFPF3u5oD/+/AiWZ2WNhzOY1gzGwZ\nwX8ohN+XxtS+QmVrd9msXmpmZxCMfX/M3f+V8lZZXIO7r3P3o9x9pLuPJAjNE9z9BcrkGlI0ENy4\nxczGAL0IFu0qp+vYBnwgfH0q8LfwdSKvIcyhNisTU8zf7bjvTMf5BVwNPE2wtPNtBHe73ww8QPAf\nxwqgf9ztzNDuXxDcd9hPECoXt9du4OsEd/A3Es5aiPsryzVsIhiTXBt+/ajcriHt/S2Es3SSeg3t\n/LvoBdwe/m48Bpya5OvIcg0nAasJZrKsBN6V8Gs4iWC45omU34Ezi/m7rSdtRUSqRDUP6YiIVBUF\nvohIlVBIZ94GAAAAJklEQVTgi4hUCQW+iEiVUOCLiFQJBb6ISJVQ4IuIVAkFvohIlfj/QTG8mzfC\nz5oAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xa017ac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "t = MLE_test(test_x, test_y, W_MLE)\n",
    "# draw the train data\n",
    "plt.plot(train_x[:,0], train_y, 'o')\n",
    "plt.plot(train_x[:,0], train_x.dot(W_MLE.T))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Maximum A Posteriori"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def MAP_train(x, y, beta, prior_mu_w=None, prior_cov_w=None):\n",
    "    \"\"\"\n",
    "    Function: Calculate the weight and beta by Maximum A Posteriori\n",
    "    Input:    \n",
    "              x:                Design matrix of training data, size:[num_sample dimension]\n",
    "              y:                Training real value, size: [num_sample 1]\n",
    "              beta:             The precision of Gaussian random variables\n",
    "              prior_mu_w:       The mean of conjugate prior distribution, size: [dimension 1]\n",
    "              prior_cov_w:      The covariance of conjugate prior distribution, size: [dimension dimension]\n",
    "    Output:   \n",
    "              W:                The weight which be calculated by Maximum A Posteriori\n",
    "    \"\"\"\n",
    "    \n",
    "    m, d = x.shape \n",
    "    # The variable m is the number of the training data. \n",
    "    #The variable d is the dimension of the training data.\n",
    "    if prior_mu_w is None:  prior_mu_w = np.zeros((d,1))\n",
    "    if prior_cov_w is None: prior_cov_w = 1*np.eye(d)\n",
    "    \n",
    "    alpha = np.diag(prior_cov_w)\n",
    "    \n",
    "    lamb = alpha / beta\n",
    "    # Your code\n",
    "    # Here you should calculate the weight.\n",
    "    W = ###\n",
    "    \n",
    "    return W"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def MAP_test(x, y, W, problem_type='regression', retained_digit=0):\n",
    "    \"\"\"\n",
    "    Function: Predict the testing data\n",
    "    Input:    \n",
    "              x:                Design matrix of training data, size:[num_sample dimension]\n",
    "              y:                Training real value, size: [num_sample 1]\n",
    "              W:                The weight which be calculated by Maximum A Posteriori\n",
    "              problem_type:     The type of problem which you want to slove. Only regression and classification\n",
    "              retained_digit:   The parameter used to classification\n",
    "    Output:   \n",
    "              t:                Prediction of testing data\n",
    "    \"\"\"\n",
    "    \n",
    "    # Your code\n",
    "    # Here, the variable t is the prediction of testing data, and you should calculate it.\n",
    "    t = ###\n",
    "    \n",
    "    # Three kinds of errors\n",
    "    er = t - y\n",
    "    MSE = (er**2).mean()                      # Mean Squared Error\n",
    "    MAE = abs(er).mean()                      # Average Absolute Error\n",
    "    SD = np.sqrt((( er-er.mean())**2).mean()) # Error Standard Deviation\n",
    "    \n",
    "    if problem_type=='classification':\n",
    "        acc = np.equal(np.round(t, retained_digit), y).mean() * 100\n",
    "        text = \"The Linear Regression's accuracy is %.2f%%.\" %(acc)\n",
    "        print(text)\n",
    "    \n",
    "    obj = pd.Series([MSE,MAE,SD], index=['MSE','MAE','SD'])\n",
    "    print(obj)\n",
    "    \n",
    "    return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  2.97185908  31.86866069]\n"
     ]
    }
   ],
   "source": [
    "W_MAP = MAP_train(train_x, train_y, beta=1, prior_cov_w=1*np.eye(2))\n",
    "print(W_MAP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE    571.793186\n",
      "MAE     18.677860\n",
      "SD      23.897232\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucVWW9x/HPj2GE8cZoIjEDHNCAAi3QiVNHS+0iaiTk\n6SgnO9GRxHqZZhrKaOlYGR4xtauFmXG8IRkgmoniJdNUzoAX7kkBwoACIqiJCMPv/LHXMHvN7D2z\nZ2bvvdbe+/t+vXjNXr+91uxnKfOdh7We9Tzm7oiISPHrFnUDREQkPxT4IiIlQoEvIlIiFPgiIiVC\ngS8iUiIU+CIiJUKBLyJSIhT4IiIlQoEvIlIiukfdAIDDDjvMBw4cGHUzREQKyqJFi7a6e+9M949F\n4A8cOJD6+vqomyEiUlDMbF1H9tclHRGREqHAFxEpEQp8EZESocAXESkRCnwRkRIRi1E6IiKFaO7z\nDUybv4qN23dSVVnB5NFDGTeyOupmpaXAFxHphLnPN1A7ewk7dzcC0LB9J7WzlwDENvR1SUdEpBOm\nzV+1L+yb7NzdyLT5qyJqUfsU+CIinbBx+84O1eNAgS8i0glVlRUdqseBAl9EpBMmjx5KRXlZqFZR\nXsbk0UMjalH7dNNWRKQTmm7MapSOiEgJGDeyumsBv/0V2O9A2P/Q7DWqDQp8EZF8e/dNuLZ/83bd\njrx8rAJfRCSf5nwdXry7efvrT+XtoxX4IiJdlNETt1tXw8+P3be5u/uBnNj9djbe9ApVlVvycv1f\ngS8i0gUZPXFbVwn4vmMePWke31ywk51v70x/TA5kNCzTzCrN7F4zW2lmK8zs42Z2qJk9YmYvB18P\nSdq/1sxWm9kqMxuds9aLiESszSduH58Kdb3YF/bDz4C6HVz5TGMkT+lm2sP/CfCQu3/RzPYD9gcu\nBx5192vNbAowBbjMzIYB44HhQBWwwMyGuHtjum8uIlKoUj1Z24P3ePrdL8Gfk4qXrYOKyrTHtFXP\nlnZ7+GbWC/gkcCuAu7/n7tuBscCMYLcZwLjg9Vhgprvvcvc1wGpgVLYbLiISBy2frJ2/36Ws6vnV\n5sLgkxOjcIKwT3VMe/VsyeSSziBgC3CbmT1vZr8xswOAPu6+KdjnVaBP8LoaWJ90/IagJiJSdJqe\nuD3CNrK255cY2m1D85tXboOzf5/2mGT5eEo3k8DvDhwD3OzuI4F/krh8s4+7O8l3JDJgZpPMrN7M\n6rds2dKRQ0VEYmPcyGpWlJ3FYz2+s6/2/EfqEr36bmVpj5l6xtFUV1ZgQHVlBVPPODoWo3Q2ABvc\n/blg+14Sgf+amfV1901m1hfYHLzfACQ9UUC/oBbi7tOB6QA1NTUd+mUhIhILs8+Dl2aGa3U7GJnB\noV1+SrcT2u3hu/urwHoza/q3xqeB5cA8YEJQmwDcF7yeB4w3sx5mNggYDCzMaqtFRKK0tzEx+iY5\n7Cfcn7cnZjsr01E6FwB3BiN0/gH8N4lfFrPMbCKwDjgTwN2XmdksEr8U9gDna4SOiBSNul6tSsf1\nnMPGX79FVeVjsZ5AzRKX36NVU1Pj9fX1UTdDRCS9bf+An4Yv1jx4yl+45I8bQ2PqK8rL8nI9HsDM\nFrl7Tab760lbEZH2pOjVU7eDa659LO0DVOkCP8qFzxX4IiLpLLwFHvxOuHbVdjADMnuAKjngK/cv\n5+1397B7b+LKSr4XPteKVyIiqdT1Cof9J76TuCkbhD20/wBV0zw7Ddt34sAb7+zeF/ZN8rnwuXr4\nIiLJfnYsvL46XEsz+mby6KGhidMg/ABVqnl2UsnXwucKfBERgF1vw9QWl1Um/RmqRqQ9pL1lDjMN\n8nwtfK7AFxFJc1M2E209QFVVWUFDO6Gfz4XPdQ1fRErX2qdah/13N2ftAapUc+aUlxmVFeV5nVKh\niXr4IlKaWgb9gH+Dc/6U1Y9o75JPvinwRaS03H8RLLotXMvhlAhRzJmTjgJfRErD3r3w/UPCtTN+\nAx/+j2jaEwEFvogUvy7clC0mCnwRKV7b18NNR4VrF6+Eg/tG056IKfBFpDipV9+KAl9EYiMrE4st\n/l+Yd0G4ljT/TSlT4ItI5OY+30DdvGVs37l7X61TE4u17NV/7Hw45UfZambBU+CLSKSaJhhLNedM\ne1MN7/OrT8CrL4VrJX75JhUFvohEqr0Jxtqcj+a9d+BHLW7ATlwA/T+apdYVl4wC38zWAm8BjcAe\nd68xszrgXGBLsNvl7v5gsH8tMDHY/0J3n5/ldotIkWhvgrG0E4vppmyHdaSHf5K7b21Ru9Hdr08u\nmNkwYDwwHKgCFpjZEK1rKyKptDXBWKqJxZ6e/3uOe+Zr4R0v3wT77Z+rJhaNXEyeNhaY6e673H0N\nsBoYlYPPEZEikGqCMYBD9i9vPbFYXa9Q2C/b+y98qPEe5i57Ix9NLXiZ9vCdRE+9Efi1u08P6heY\n2VeAeuASd38DqAaeTTp2Q1ATEWklownGZp4NKx8IHTfw3buCVxne2JWMA/94d28ws8OBR8xsJXAz\n8AMSvwx+APwYOCfTDzazScAkgAEDBnSo0SJSXNJOMOYOV1eGStfuHs+vGk8P1fK1YlShyyjw3b0h\n+LrZzOYAo9z9yab3zewWoOnXbwPQP+nwfkGt5fecDkwHqKmp8Zbvi0iJS3FT9riec2h4t3W452vF\nqELX7jV8MzvAzA5qeg2cDCw1s+SxUF8Algav5wHjzayHmQ0CBgMLs9tsESlaW1e3DvsLFkPdjpTX\n+/O5YlShy6SH3weYY4nHkrsDd7n7Q2Z2u5mNIHFJZy1wHoC7LzOzWcByYA9wvkboiEhGUvTqB717\nF1W3rGPy6J6xW1Ck0Jh79FdTampqvL6+PupmiEhUHrgY6m8NlYY13s07u5vzqaK8LK/LARYCM1vk\n7jWZ7q81bUUkWnW9wmHfrTvH9ZwTCntonmZBOk9TK4hINNp4UnbjlD+mPESjcbpGPXwRya9db7cO\n+zE3hqZFSDfqRqNxukY9fBHJnwznv5k8emirGTQ1GqfrFPgiknsvL4A7/z1cm/wPOOB9KXfXaJzc\nUOCLSG51clbLtE/fSqcp8EUkN343Btb+JVzT9MWRUuCLSHalmP+GY/8bPn9TNO2RfRT4IpI9WpQk\n1hT4ItJ1b6yDn3w4XJv0Z6gaEU17JCUFvoh0TYa9+rnPN2jUTcQU+CLSOU/dCAvqwrUrt0G31qtX\nzX2+ITSuvmH7TmpnLwFQ6OeRAl9EOq5lr/6A3jB5ddrdp81fFXqICprnxlHg548CX0Qy18mbsunm\nwNHcOPmluXREpH27320d9qf/POMROJobJx7UwxeRtmVhqKXmxokHBb6IpLbmSZjx+XBt8t/hgMM6\n/K00N048ZBT4ZrYWeAtoBPa4e42ZHQrcAwwkscThme7+RrB/LTAx2P9Cd5+f9ZaLFLDYD1HMwQNU\nmhsneh3p4Z/k7luTtqcAj7r7tWY2Jdi+zMyGAeOB4UAVsMDMhmhdW5GEWA9RvOss+NtD4ZqelC0a\nXblpOxaYEbyeAYxLqs90913uvgZYDYzqwueIFJW2hihGqq5XOOyPPlNhX2Qy7eE7iZ56I/Brd58O\n9HH3TcH7rwJ9gtfVwLNJx24IaiJCDIcoav6bkpFp4B/v7g1mdjjwiJmtTH7T3d3MPM2xKZnZJGAS\nwIABAzpyqEhBq6qsoCFFuOd9iOKbG+GGD4VrExdA/4/mtx2SNxld0nH3huDrZmAOiUs0r5lZX4Dg\n6+Zg9wagf9Lh/YJay+853d1r3L2md+/enT8DkQIzefRQKsrD0w/kfYhiXa/WYV+3Q2Ff5NoNfDM7\nwMwOanoNnAwsBeYBE4LdJgD3Ba/nAePNrIeZDQIGAwuz3XCRQjVuZDVTzzia6soKDKiurGDqGUfn\n54bts79qfQnne1t1CadEZHJJpw8wx8ya9r/L3R8ys/8DZpnZRGAdcCaAuy8zs1nAcmAPcL5G6IiE\nRTJEsWXQl/WA721Ova8UpXYD393/AXwkRf114NNpjrkGuKbLrRORrvv+YbB3d7imHn1J0lw6IsWq\ncXeiV58c9qdOU9iXME2tIFKMNNRSUlDgixST9Qvh1s+GaxevhIP7RtMeiRUFvkixUK9e2qHAFyl0\n906EpfeGawp6SUGBL1LIWvbqh5wKX5oZTVsk9hT4IoVIl2+kExT4IoXk7S1w/QfCtQkPwKBPRNMe\nKSgKfJFCkUGvPvYLq0ikFPgicbdoBtx/Ybh2xWtQ3jNUivXCKhILCnyROOvAtfq2FlZR4Aso8EXi\n6boj4Z2t4Vo7N2Vjt7CKxI7m0hGJk8Y9iV59cth/pi6jETjpFlDJ+8IqElvq4YvERReHWk4ePTR0\nDR8iWFhFYk2BLxK1jS/A9BPCtYuWQGXHlv5suk6vUTqSjgJfJEpZfoAqkoVVpGAo8EWiMO9CWDwj\nXLtqOyRWlhPJCQW+SL617NUP+iRMuD+atkhJyTjwzawMqAca3H2MmdUB5wJbgl0ud/cHg31rgYlA\nI3Chu8/PaqtFCpHmv5GIdaSH/y1gBXBwUu1Gd78+eSczGwaMB4YDVcACMxuihcylZL2zDa4bFK6d\n/QcY/Jlo2iMlK6PAN7N+wOdILEx+cTu7jwVmuvsuYI2ZrQZGAc90paEiBUm9eomRTB+8ugm4FNjb\non6Bmb1kZr81s0OCWjWwPmmfDUEtxMwmmVm9mdVv2bKl5dsihe2l37cO+8s3KewlUu0GvpmNATa7\n+6IWb90MHAGMADYBP+7IB7v7dHevcfea3r17d+RQkXir6wWzv9aitgP22z+a9ogEMrmkcxxwupmd\nBvQEDjazO9z9y007mNktwAPBZgPQP+n4fkFNpLj9dCRs+0e4ph69xEi7PXx3r3X3fu4+kMTN2Mfc\n/ctm1jdpty8AS4PX84DxZtbDzAYBg4GFWW63SHzs3Zvo1SeH/QlTFPYSO10Zh3+dmY0AHFgLnAfg\n7svMbBawHNgDnK8ROlK0dFNWCoi5e9RtoKamxuvr66NuhkjmXlsON388XLtgMbzvyGjaIyXJzBa5\ne02m++tJW5GOUq9eCpQCX0pOp9d9fagWnv1luKb5b6SAKPClpHR63deWvfqqkTDpidw0UiRHFPhS\nUjq87qsu30gRUeBLScl43dd334Rr+4drZ90JHxqTo5aJ5J4CX0pKVWUFDSlCP7Tuq3r1UqS0iLmU\nlMmjh1JRXhaq7Vv3dfm81mFfu0FhL0VDPXwpKcnrvjZs30mZGTt3NzLuvmGtd1bQS5FR4EvJaQr9\n2tlL+AVT+VTZC+EdFPRSpHRJR0rStIdWsqLsrFDYT9/zOY7rOSfCVonklnr4UnrqevF0i9LAd+8C\nwNKM4hEpBgp8KR1vboIbPhgqnbDrBtb5+/dth0briBQZBb6UhhRDLT/UeA87kyZy3TdaR6RI6Rq+\nFLeXZrUO+6u2Q90Opp5xNNWVFRhQXVnB1DOOzmxOHZECpR6+FK+WQV8zEcbcsG9z3MhqBbyUFAW+\nFJ/7vwWLfheuaailiAJfisie9+CHvcO1rz4IA4+Lpj0iMZNx4JtZGVAPNLj7GDM7FLgHGEhiicMz\n3f2NYN9aYCLQCFzo7vOz3G6RMM1/I9Kujty0/RawIml7CvCouw8GHg22MbNhJBY7Hw6cAvwy+GUh\nkn1b/tY67K94VWEvkkJGgW9m/YDPAb9JKo8FZgSvZwDjkuoz3X2Xu68BVgOjstNckSR1veAXH23e\nHn5GIujLNZZeJJVML+ncBFwKHJRU6+Pum4LXrwJ9gtfVwLNJ+20IaiFmNgmYBDBgwIAONFlK3v/d\nCn+8OFxTj16kXe0GvpmNATa7+yIzOzHVPu7uZuYd+WB3nw5MB6ipqenQsVKi3OHqynDtzNth2OnR\ntEekwGTSwz8OON3MTgN6Ageb2R3Aa2bW1903mVlfYHOwfwOQvFRQv6Am0nm3fBoa6sO1NL36Ti9S\nLlLk2r2G7+617t7P3QeSuBn7mLt/GZgHTAh2mwDcF7yeB4w3sx5mNggYDCzMesulNLyzLXGtPjns\nL17RZtjXzl5Cw/adOM2LlM99Xn0Oka6Mw78WmGVmE4F1wJkA7r7MzGYBy4E9wPnu3pj+24ik0XL0\nzUF94ZKVbR7S4UXKRUpIhwLf3Z8Anghevw58Os1+1wDXdLFtUqrWPAkzPh+uXfkGdGt/UFnGi5SL\nlCA9aSvx0rJXf9IVcMKlGR+e0SLlIiVKs2VKPDxU2zrs63Z0KOyhnUXKRUqcevgSrcbd8IPDwrVz\nH4PqYzv17ZIXKdcoHZEwBb5Ep64SaPEIRhYeoNK0xyKpKfAl/17/O/zsmHCttgF6HBhNe0RKhAJf\n8qvldfohp8CX7ommLSIlRoEv+bH4dpj3zXBN89+I5JUCX3KvZa/+32+Fo78YTVtESpgCX3Lnd2Ng\n7V/CNfXqRSKjwJfse3cHXNtiyuuLlkClpsEWiZICX7Kr5eWbnr1gyivRtEVEQhT4kh3rnoHbTgnX\nrtwG3bS6pUhcKPCl61r26o+/GD5zVTRtEZG0FPjSeY9cCU//JFzTTVmR2FLgS8ftbYTvHxqunfMw\nDPjXaNojIhlR4EvH/Kga3ns7XFOvXqQgKPAlM2+sg598OFybsh56HhxNe0Skw9qdD9/MeprZQjN7\n0cyWmdnVQb3OzBrM7IXgz2lJx9Sa2WozW2Vmo3N5ApIHdb3CYX/EiYlevcJepKBk0sPfBXzK3d82\ns3LgKTP7U/Deje5+ffLOZjaMxGLnw4EqYIGZDdG6tgXopVkw+9xwTZdvRApWu4Hv7g40XbQtD/54\n+iMYC8x0913AGjNbDYwCnuliWyWfWg61HPtLGHl2NG0RkazIaIlDMyszsxeAzcAj7v5c8NYFZvaS\nmf3WzA4JatXA+qTDNwQ1KQR3fDH1UoMKe5GCl9FN2+ByzAgzqwTmmNlRwM3AD0j09n8A/Bg4J9MP\nNrNJwCSAAQNKd46Vuc83xGM5vl1vwdR+4dqFL8Chg/LfFhHJiQ6N0nH37Wb2OHBK8rV7M7sFeCDY\nbAD6Jx3WL6i1/F7TgekANTU1bV0iKlpzn2+gdvYSdu5O3N5o2L6T2tlLAPIb+i179N3K4cqt+ft8\nEcmLTEbp9A569phZBfBZYKWZ9U3a7QvA0uD1PGC8mfUws0HAYGBhdptdHKbNX7Uv7Jvs3N3ItPmr\n8tOArS+3Dvvvva6wFylSmfTw+wIzzKyMxC+IWe7+gJndbmYjSFzSWQucB+Duy8xsFrAc2AOcrxE6\nqW3cvrND9axqGfQn1sKJU3L/uSISmUxG6bwEjExR/682jrkGuKZrTSt+VZUVNKQI96rKitx96Iv3\nwJxJ4ZqGWoqUBD1pG6HJo4eGruEDVJSXMXn00Ox/2N698P1DwrWvPw3vPyr7nyUisaTAj1DTjdmc\nj9K5dyIsvbd5+6C+cMnK7H6GiMSeAj9i40ZW525Ezj+3wrQjw7XaBuhxYG4+T0RiTYFfrFrelP34\nN2G0bquIlDIFfrFJtdTgVdvBLJr2iEhsKPCLScte/dn3wuDPRtMWEYkdBX4xeHIaPPbDcE1DLUWk\nBQV+IduzC354eLj27eXQS3PViUhrCvxC9avj4dUlzdtHnARfmRtde0Qk9hT4heb1v8PPjgnXvvc6\nlHXuf2VsZusUkZxT4BeSljdlT7seRp2bet8MxGa2ThHJi4wWQJGILZ2delGSLoQ9xGC2ThHJK/Xw\n48wdrq4M1yb9GapGZOXbRzpbp4jknQI/ru47H56/o3m7Zy+Y8kpWPyKS2TpFJDIK/Lh575/wo6pw\nbcoricDPsrzO1ikikVPgx8n9F8Gi25q3T7oCTrg0Zx+Xt9k6RSQWFPhxsG0N/DTpuryVwZWv52X+\nm5zO1ikisZLJmrY9zWyhmb1oZsvM7OqgfqiZPWJmLwdfD0k6ptbMVpvZKjMbncsTKHh//E447M/7\nC1y1TZOdiUjWZdLD3wV8yt3fNrNy4Ckz+xNwBvCou19rZlOAKcBlZjYMGA8MB6qABWY2ROvatvDq\nksTTsk2GnApfmhlde0Sk6GWypq0Dbweb5cEfB8YCJwb1GcATwGVBfaa77wLWmNlqYBTwTDYbXrD2\nNsKtJ0NDfWK7WzlctgZ6HLRvFz39KiK5kNE1fDMrAxYBHwB+4e7PmVkfd98U7PIq0Cd4XQ08m3T4\nhqAmKx6Ae85u3h5/N3zwtNAuevpVRHIlo8APLseMMLNKYI6ZHdXifTcz78gHm9kkYBLAgAEDOnJo\n4dn1FkztT+IfRkC/UXDOQ9CtrNWubT39qsAXka7o0NQK7r4deBw4BXjNzPoCBF83B7s1AP2TDusX\n1Fp+r+nuXuPuNb179+5M2wvDX34MU/uxL+y//jR87ZGUYQ96+lVEcieTUTq9g549ZlYBfBZYCcwD\nJgS7TQDuC17PA8abWQ8zGwQMBhZmu+Gx98baxPw3j34/sT3qvMT8N+8/qs3D0j3lqqdfRaSrMrmk\n0xeYEVzH7wbMcvcHzOwZYJaZTQTWAWcCuPsyM5sFLAf2AOeX1Agdd5j1FVgxr7n2ndVwYGb/itHT\nryKSK5YYhBOtmpoar6+v7/BxsRvNsu6vcNupzduf/ykcOyH9/mnE7rxEJJbMbJG712S6f8E+aRur\n0Sx73oOfHQs7gsnNDu4HFy6G7j069e309KuI5ELBzocfm7ncF98OP+zdHPZf/SNcvKzTYS8ikisF\n28OPfDTLP7fCtCObtz84Bs66Q1MiiEhsFWzgRzqX+0O18Owvm7cvfAEOHZT7zxUR6YKCvaQzefRQ\nKsrDY9lzPprltWWJoZZNYX/SdxNDLRX2IlIACraHn9e53PfuTYy+WZ80Y8SU9dDz4Ox/lohIjhRs\n4EOeRrOsegjuPqt5+8zbYdjpuf1MEZEcKOjAz6ldb8N1R0DjrsR21THwtQVpp0QQEYk7BX4qT/8E\nHrmyefu8J6HvR6Jrj4hIFijwk21/BW46unm75hwYc2N07RERySIFPiTmv/nD12Dpvc21S/4GB/VJ\nf4yISIFR4L/yHPz25Obtz90AH50YXXtERHKkdAN/z3vwi1HwxprE9gGHw0VLoLxntO0SEcmR0gz8\nF+6GuV9v3v7KPDjihOjaIyKSB6UV+O9sg+uSnoodcir8592a/0ZESkLpBP7D34W//qx5+4LF8L4j\n0+8vIlJkij/wN6+EX/5r8/YJl8FJl0fXHhGRiGSypm1/M3vczJab2TIz+1ZQrzOzBjN7IfhzWtIx\ntWa22sxWmdnoXJ5AWnv3wu/GhMP+snUKexEpWZn08PcAl7j7YjM7CFhkZo8E793o7tcn72xmw4Dx\nwHCgClhgZkPyuq7t3x6Gu/6jefuLt8FRZ+Tt40VE4qjdwHf3TcCm4PVbZrYCaGvGsrHATHffBawx\ns9XAKOCZLLS3be/9E6Z9AHa/k9juczRMegLKiv/KlYhIezo0H76ZDQRGAs8FpQvM7CUz+62ZHRLU\nqoH1SYdtoO1fENnx15/Dj6qaw/7cx+EbTynsRUQCGQe+mR0I/AG4yN3fBG4GjgBGkPgXwI878sFm\nNsnM6s2sfsuWLR05NGzHhsSiJA9fkdg+5iuJRUmqj+n89xQRKUIZdX/NrJxE2N/p7rMB3P21pPdv\nAR4INhuA/kmH9wtqIe4+HZgOUFNT451pPO++CTcOb96+eCUc3LdT30pEpNhlMkrHgFuBFe5+Q1I9\nOVm/ACwNXs8DxptZDzMbBAwGFmavyUm694Bh4+DUaYlevcJeRCStTHr4xwH/BSwxsxeC2uXAf5rZ\nCMCBtcB5AO6+zMxmActJjPA5P2cjdLr3gDNn5ORbi4gUm0xG6TwFpJp74ME2jrkGuKYL7RIRkSzr\n0CgdEREpXAp8EZESocAXESkRCnwRkRKhwBcRKREKfBGREqHAFxEpEebeuVkNstoIsy3AuqjbETgM\n2Bp1I7pI5xAfxXAeOof4aHke/+LuvTM9OBaBHydmVu/uNVG3oyt0DvFRDOehc4iPrp6HLumIiJQI\nBb6ISIlQ4Lc2PeoGZIHOIT6K4Tx0DvHRpfPQNXwRkRKhHr6ISIko6cA3s2+b2TIzW2pmd5tZTzM7\n1MweMbOXg6+HtP+d8itYQ3izmS1NqqVtt5nVmtlqM1tlZqOjaXVYmnOYZmYrg3WS55hZZdJ7BXEO\nSe9dYmZuZocl1WJ3DpD+PMzsguD/xzIzuy6pHrvzSPP3aYSZPWtmLwTLqY5Kei+O59DfzB43s+XB\nf/NvBfXs/Wy7e0n+IbGw+hqgItieBXwVuA6YEtSmAP8TdVtTtP2TwDHA0qRaynYDw4AXgR7AIODv\nQFlMz+FkoHvw+n8K8RyCen9gPolnSw6L8zm08f/iJGAB0CPYPjzO55HmHB4GTg1enwY8EfNz6Asc\nE7w+CPhb0Nas/WyXdA+fxAIwFWbWHdgf2AiMBZqW0ZoBjIuobWm5+5PAthbldO0eC8x0913uvgZY\nDYwiYqnOwd0fdvc9weazJNZDhgI6h8CNwKUkVoNrEstzgLTn8Q3gWnffFeyzOajH8jzSnIMDBwev\ne5H4+Yb4nsMmd18cvH4LWEGiY5q1n+2SDXx3bwCuB14BNgE73P1hoI+7bwp2exXoE1ETOypdu6uB\n9Un7bQhqcXcO8KfgdcGcg5mNBRrc/cUWbxXMOQSGAJ8ws+fM7M9m9tGgXkjncREwzczWk/hZrw3q\nsT8HMxsIjASeI4s/2yUb+MF1sLEk/ilUBRxgZl9O3scT/24quGFMhdruJmZ2BYn1kO+Mui0dYWb7\nk1jv+cqo25IF3YFDgY8Bk4FZZpZqqdM4+wbwbXfvD3wbuDXi9mTEzA4E/gBc5O5vJr/X1Z/tkg18\n4DPAGnff4u67gdnAvwGvmVlfgODr5ja+R5yka3cDiWvKTfoFtVgys68CY4Czg7/cUDjncCSJDsSL\nZraWRDsXm9n7KZxzaLIBmO0JC4G9JOZxKaTzmEDi5xrg9zRf7ojtOZhZOYmwv9Pdm9qetZ/tUg78\nV4CPmdn+Qc/l0ySumc0j8ReF4Ot9EbWvo9K1ex4w3sx6mNkgYDCwMIL2tcvMTiFx7ft0d38n6a2C\nOAd3X+K9SbziAAAA8klEQVTuh7v7QHcfSCI0j3H3VymQc0gyl8SNW8xsCLAfiUm7Cuk8NgInBK8/\nBbwcvI7lOQQ5dCuwwt1vSHorez/bUd+ZjvIPcDWwElgK3E7ibvf7gEdJ/OVYABwadTtTtPtuEvcd\ndpMIlYlttRu4gsQd/FUEoxai/pPmHFaTuCb5QvDnV4V2Di3eX0swSieu59DG/4v9gDuCn43FwKfi\nfB5pzuF4YBGJkSzPAcfG/ByOJ3G55qWkn4HTsvmzrSdtRURKRClf0hERKSkKfBGREqHAFxEpEQp8\nEZESocAXESkRCnwRkRKhwBcRKREKfBGREvH/X1XCtsai938AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xa20ca58>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "t = MLE_test(test_x, test_y, W_MAP)\n",
    "# draw the train data\n",
    "plt.plot(train_x[:,0], train_y, 'o')\n",
    "plt.plot(train_x[:,0], train_x.dot(W_MAP.T))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayesian linear regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def BLR_train(x, y, beta=100, prior_mu_w=None, prior_cov_w=None):\n",
    "    \"\"\"\n",
    "    Function: Calculate the expectation and covariance of posterior distribution\n",
    "    Input:    \n",
    "              x:              Design matrix of training data, size:[num_sample dimension]\n",
    "              y:                Training Label, size: [num_sample 1]\n",
    "              beta:             The precision of Gaussian random variables\n",
    "              prior_mu_w:       The mean of conjugate prior distribution, size: [dimension 1]\n",
    "              prior_cov_w:      The covariance of conjugate prior distribution, size: [dimension dimension]\n",
    "    Output:   \n",
    "              posterior_mu_w:   The expectation of posterior distribution\n",
    "              posterior_cov_w:  The covariance of posterior distribution\n",
    "    \"\"\"\n",
    "    y = y[:, np.newaxis]\n",
    "    m, d = x.shape \n",
    "    # The variable m is the number of the training data. \n",
    "    #The variable d is the dimension of the training data.\n",
    "    if prior_mu_w == None:  prior_mu_w = np.zeros((d,1))\n",
    "    if prior_cov_w == None: prior_cov_w = 0.1*np.eye(d)\n",
    "    \n",
    "    # your code. \n",
    "    # Here, you should calculate the expectation posterior_mu_w and the covariance posterior_cov_w of posterior distribution\n",
    "    posterior_cov_w = ###\n",
    "    posterior_mu_w = ###\n",
    "    \n",
    "    return posterior_mu_w, posterior_cov_w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def BLR_test(x, y, posterior_mu_w, posterior_cov_w, problem_type='regression', \n",
    "                  beta=100, sampling_size=1000, output_type='mode', retained_digit=0):\n",
    "    \"\"\"\n",
    "    Function: Predict the testing data\n",
    "    Input:\n",
    "              x:              Design matrix of testing data\n",
    "              y:                Testing Label\n",
    "              posterior_mu_w:   The expectation of posterior distribution\n",
    "              posterior_cov_w:  The covariance of posterior distribution\n",
    "              problem_type:     The type of problem which you want to slove. Only regression and classification\n",
    "              beta:             The precision of Gaussian random variables\n",
    "              sampling_size:    The sampling size of weight w\n",
    "              output_type:      There are three types of output\n",
    "                   'mode':      The mode of each data prediction sequences\n",
    "                   'mean':      The mean of each data prediction sequences \n",
    "                    'all':      Each intact data prediction sequences \n",
    "    Output:\n",
    "              t:                Prediction of testing data\n",
    "    \"\"\"\n",
    "    # your code\n",
    "    # Here, You need to sample the weight w from its distribution and calculate the prediction of testing data.\n",
    "    # You may use the function \"np.random.multivariate_normal\"  to sample the weight. \n",
    "    # If you don't know how to use the function, you can input \"help(np.random.multivariate_normal)\" or contact us.\n",
    "    posterior_mu_w = posterior_mu_w.reshape(-1)\n",
    "    \n",
    "    w = ###\n",
    "    t = ###\n",
    "    \n",
    "    if output_type=='mean':\n",
    "        t = np.mean(t,1)[:, np.newaxis]\n",
    "    else:\n",
    "        t = pd.DataFrame(np.round(t, retained_digit)).mode(1).as_matrix(columns=None)\n",
    "        t = t[:,0:1]\n",
    "    \n",
    "    # Three kinds of errors\n",
    "    er = t - y\n",
    "    MSE = (er**2).mean()                      # Mean Squared Error\n",
    "    MAE = abs(er).mean()                      # Average Absolute Error\n",
    "    SD = np.sqrt((( er-er.mean())**2).mean()) # Error Standard Deviation\n",
    "    \n",
    "    if problem_type=='classification':\n",
    "        acc = np.equal(np.round(t, retained_digit), y).mean() * 100\n",
    "        text = \"The Linear Regression's accuracy is %.2f%%.\" %(acc)\n",
    "        print(text)\n",
    "    \n",
    "    obj = pd.Series([MSE,MAE,SD], index=['MSE','MAE','SD'])\n",
    "    print(obj)\n",
    "    \n",
    "    if output_type=='all':\n",
    "        return x.dot(w.T)\n",
    "    return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  2.97185908]\n",
      " [ 31.86866069]]\n",
      "[[  1.95446164e-06  -2.84869904e-04]\n",
      " [ -2.84869904e-04   4.62827310e-02]]\n"
     ]
    }
   ],
   "source": [
    "[posterior_mu_w, posterior_cov_w] = BLR_train(train_x, train_y, beta=10)\n",
    "print(posterior_mu_w)\n",
    "print(posterior_cov_w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE    28077.089964\n",
      "MAE      135.549512\n",
      "SD       167.559870\n",
      "dtype: float64\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAloAAAJCCAYAAAAY3mkcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcHGWd+PHPNzdHCIHEEJJwaRAIyGEEEbxAIItH4AeL\nUVRQdoOKIuDqCui664IXCLoqIAiIChvARYisyOmxoIAJcuQgEDlyEMhwGYiQY+b5/dGdmuohx0xm\nuquPz/v1GqeequrkSZlMf+iuqo6UEpIkSep7/YqegCRJUrMytCRJkqrE0JIkSaoSQ0uSJKlKDC1J\nkqQqMbQkSZKqxNCSJEmqEkNLkiSpSgwtSZKkKhlQ9AQARowYkXbYYYeipyFJkrRBM2fOfDalNLI7\n+9ZFaO2www7MmDGj6GlIkiRtUEQ82d19fetQkiSpSgwtSZKkKjG0JEmSqsTQkiRJqhJDS5IkqUoM\nLUmSpCoxtCRJkqrE0JIkSaoSQ0uSJKlKDC1JkqQqMbQkSZKqxNCSJEmqEkNLkiSpSgwtSZKkKjG0\nJEmSqsTQkiRJqhJDS5IkqUoMLUmSpCoxtCRJkqrE0JIkSaoSQ0uSJKlKDC1JkqQqMbQkSVLje/ll\n2HxzuOiiomdSYUDRE5AkSeqVv/0NttyytPzAA8XOpQtf0ZIkSY3r+ec7I2vPPeHCC4udTxeGliRJ\nakxLl8LWW5eWDzwQ7r+/2PmshaElSZIaz1NPwahRpeVJk+D//o9DDoH99y92Wl15jpYkSWosTz4J\nO+xQWj76aLj2WiIKndE6+YqWJElqHPPnd0bWccfVdWSBoSVJkhrF3Lkwfnxp+dOfhp/85DWRlVLt\np7U+hpYkSap/DzwAu+1WWv6Xf4Ef/rDuIwsMLUmSVO/uvRf22qu0/G//Buec0xCRBZ4ML0mS6tmd\nd8Lb315a/uY34V//tWEiCwwtSZJUr26/Hd7zntLy975H+uzJ9GugyAJDS5Ik1aNf/xre+97S8sUX\n03HCP9O/ywlP9R5Z4DlakiSp3lx3XWdk/fSnrDr+n+nfv3KXRogsMLQkSVI9ueoqOOqo0vK11/LK\n0R9l0KDKXRolssDQkiRJ9eKyy+DYY0vL06fz4nuOZtNNK3dppMgCQ0uSJNWDH/4QTjihtHzLLTz1\n5vczfHjlLo0WWWBoSZKkop17LnzmM6Xl3/+eedsdwpgxlbs0YmSBoSVJkor0ta/BF75QWr77bv7A\nO9hll8pdGjWywNCSJElF+dKX4KtfLS3fdx8/nbcf73xn5S6NHFlgaEmSpCKcfDJ861ul5Vmz+PL/\n7M1xx1Xu0uiRBd6wVJIk1doJJ5SuMASYN4+jTt+Z666r3KUZIgsMLUmSVEtTpsDVV5eWH3+c3Q7f\ngblzK3dplsiCbr51GBFbRsQvIuLhiJgbEftHxFYRcWtEPFr+Pjy3/+kRMT8i5kXEYdWbviRJahjv\nf39nZC1cyBZvau7Igu6fo/U94DcppV2APYG5wJeA21NK44Hby2MiYjdgCjABmARcEBH91/qrSpKk\n1vCud8GNN5aWn36aGDeWl17q3Dx6dPNFFnQjtCJiGPAO4FKAlNLKlNKLwGTgivJuVwBHlJcnA9NS\nSitSSo8D84F9+3rikiSpQbzlLfD735eWn32W2GZUxeYDDoCnnipgXjXQnVe0dgTagMsj4i8R8eOI\n2AwYlVJaUt7naWDNURsDLMw9flF5nSRJajW77AIzZpSWX3iBGLF1xeYTT4Q77yxgXjXSndAaAOwD\nXJhS2htYTvltwjVSSgno0Qt+ETE1ImZExIy2traePFSSJDWCMWNg3rzS8rJlxPAtKzb/4Adw0UUF\nzKuGuhNai4BFKaV7yuNfUAqvZyJiNED5+9Ly9sXAuNzjx5bXVUgpXZxSmphSmjhy5MiNnb8kSapH\nQ4d2vh+4fDmxxdCKzXfcASedVMC8amyDoZVSehpYGBFvLK86GJgDTAfW3FrsOOCG8vJ0YEpEDI6I\nHYHxwL19OmtJklSfUoIIePnl0vjVV4nNNq3Y5ZFH4N3vLmBuBejufbQ+C1wZEYOAx4CPU4q0ayLi\nBOBJ4BiAlNLsiLiGUoytBk5KKbX3+cwlSVJ9SQn65V7DWbmSGDSwYpfnnoOttqrxvArUrdBKKd0P\nTFzLpoPXsf/ZwNm9mJckSWokHR3QP3c3p1WriIGVmbF8OWy6KS3FO8NLkqTeaW+HAQMqxtG/32t2\n6deCn7BsaEmSpI23ejUMzL092NFB9IuKXTo6SqdttaIWbEtJktQnVq7cYGStOTe+VRlakiSp5155\nBQYPLi0PGgQprTWyWp2hJUmSeiZ/VvuIEbBixWtetTKySgwtSZLUfcuWweabl5Z33BHa2oys9TC0\nJElS9zz/PAwbVlp+05vgsceMrA0wtCRJ0oYtXQpblz8Q+oAD6PjLA0ZWNxhakiRp/Z56CkaNKi0f\neiirf3dnxb1JwchaF0NLkiSt24IFMGZMafnII3n1hpsr7ugARtb6GFqSJGnt/vpX2H770vJHP8pL\nV1zHJptU7mJkrZ+hJUmSXmvuXHjDG0rLn/wkz5zzU7bYonIXI2vDDC1JklTpwQdht91Ky5//PI+c\neiHbbFO5i5HVPYaWJEnqNGMG7LlnafnLX+buo8/ljW+s3MXI6j5DS5Ikldx1F7zlLaXlr3+dn47/\nT/bfv3IXI6tnDC1JkgR33AEHHlha/u53OfPl0znuuMpdjKyeG1D0BCRJUsFuugkOP7y0/KMf8f5f\nTeXGGyt3MbI2jqElSVIr++Uv4f/9v9LyT3/K+K99lPnzK3cxsjaeoSVJUquaNg0+9KHS8jXXsNnx\n/8jf/965edNNYfnyYqbWLDxHS5KkVnTRRZ2RdcMNxDGVkbXrrkZWX/AVLUmSWs0ee8CsWaXlm28m\nDju0YvPRR8O11xYwrybkK1qSJLWSbbftjKxvfOM1kfW1rxlZfclXtCRJahWDB8PKlaXl888nTj2l\nYvN118GRRxYwryZmaEmS1AoiOpd//GPin06o2PzQQ7D77jWeUwswtCRJanb5yJo2jZjywYrNTz8N\no0bVeE4twtCSJKmZ5SPrV78i3v++is3Ll5du46DqMLQkSWpW+ci6/Xbi4IMqNq9aBQMsgary8EqS\n1IzykfWnPxH7v7Vic3s79PPeA1VnaEmS1GzykXX//cRee1Zs9iN1asfQkiSpmeQja+5cYtddKjYb\nWbVlaEmS1CzykfX448SOO1RsNrJqz9CSJKkZ5CNr8WJizLYVm42sYhhakiQ1unxktbURI0dUbDay\nimNoSZLUqFKqvHTwxReJLYe9ZhcVx9CSJKkRdYmsjpeW03/opq/ZRcXyDhqSJDWaLjfBWrnsVSOr\nThlakiQ1ki63c3/5hVUM3mJwxS5GVv0wtCRJahSvvgqDBmXDpUvaGTq88iwgI6u+GFqSJDWC5cth\nk02y4aPzOhg1uvJp3MiqP4aWJEn17sUXYfPNs+Hvf5fY+Y1RsYuRVZ8MLUmS6llbGwwfng0vuzTx\nrndV7mJk1S9DS5KkerV4Mbzuddnw5M8mTjihchcjq755Hy1JkurRY4/B61+fDQ88IHHX9yt3MbLq\nn6ElSVK9mTMHJkzIhiNHJJ69q3IXI6sxGFqSJNWT++6DN785GwYJnq3cxchqHJ6jJUlSvbj66tdG\nVhdGVmMxtCRJqgcXXwxTpmTDrpG1xRZGViMytCRJKtq3vgUnnpgNu0bWm98Mf/tbrSelvmBoSZJU\npC99qfRV1jWyPv1pmDGj1pNSX/FkeEmSijJ1KlxySTbsGlnTpsEHP1jrSakvGVqSJBXhqKPguuuy\nYdfIuu8+2HvvWk9Kfc3QkiSp1t7xDvi//8uGXSNrwQIYN67Wk1I1GFqSJNXSLrvAvHnZsGtk/f3v\nsMkmtZ6UqsXQkiSpVoYPhxdfzIZdI6u9Hfp5mVpTMbQkSaqFiMphl8jyHlnNyW6WJKna8pE1cqSR\n1UIMLUmSqikfWbvvTrQtrdhsZDW3boVWRDwREQ9FxP0RMaO87t8jYnF53f0RcXhu/9MjYn5EzIuI\nw6o1eUmS6lo+sg46iJj1UMVmI6v59eQcrXenlLp8fjjnp5TOza+IiN2AKcAEYFvgtojYOaXU3rup\nSpLUQPKRdcwxxDVXV2w2slpDNd46nAxMSymtSCk9DswH9q3C7yNJUn3KR9anPmVktbDuhlai9MrU\nzIiYmlv/2Yh4MCIui4jh5XVjgIW5fRaV11WIiKkRMSMiZrS1tW3U5CVJqjv5yDrjDOLCCyo2G1mt\npbuhdWBKaS/gH4CTIuIdwIXATsBewBLgOz35jVNKF6eUJqaUJo4cObInD5UkqT7lIqvj2+cQXz+7\nYrOR1Xq6FVoppcXl70uBXwL7ppSeSSm1p5Q6gEvofHtwMZD/4ICx5XWSJDWvXGStvvAS+n/xXyo2\nG1mtaYOhFRGbRcTQNcvAocCsiBid2+1IYFZ5eTowJSIGR8SOwHjg3r6dtiRJdSQXWX+//GoGfuqf\nKjYbWa2rO1cdjgJ+GaW/RAOAq1JKv4mIn0XEXpTO33oCOBEgpTQ7Iq4B5gCrgZO84lCS1LRykfX8\nBdPY+uPHVGw2slrbBkMrpfQYsOda1n90PY85Gzh7XdslSWoKuch6/LxfstOnj6jYbGTJO8NLkrQx\ncpF142l3sNNpRpZey9CSJKmncpF19pEzeP95767YbGRpjZ7cGV6SJOUia/LOc5n+y10qNhtZyjO0\nJEnqjpSgX+cbQTsPWcCjj4x7zS5SnqElSdKGdHRA//7ZcCRLefbVypttG1laG0NLkqT1WbUKBg3K\nhlvwN15ii4pdjCyti6ElSdK6/P3vsNlm2XAIr7CCIRW7GFlaH686lCRpbV58sSKyBrLSyFKPGVqS\nJHW1ZAkMH54N+9HOagZm49GjjSx1j6ElSVLeY4/Btttmw6CDlHu6POYYeOqpIiamRmRoSZK0xoMP\nwutfnw2DBHTeN+uCC+DqqwuYlxqWJ8NLkgRw111w4IHZsBRZne6+G/bbr9aTUqMztCRJ+vWv4b3v\nzYZdI+uFF2DLLWs9KTUD3zqUJLW2K69cb2S1txtZ2niGliSpdf3Xf8FHPpINu0ZWl0/dkXrMvz6S\npNb0b/8Gn/tcNlxbZEm95TlakqTWc+KJcPHF2dDIUrUYWpKk1jJ5Mkyfng2NLFWToSVJah377gt/\n/nM2NLJUbYaWJKk1jBsHixZlQyNLtWBoSZKaX0Tl0MhSjXjVoSSpuRlZKpChJUlqXkaWCmZoSZKa\nk5GlOmBoSZKaj5GlOmFoSZKai5GlOmJoSZKah5GlOmNoSZKag5GlOmRoSZIan5GlOmVoSZIam5Gl\nOmZoSZIaVy6yXmGIkaW6Y2hJkhpTLrLmsgub8krFZiNL9cDQkiQ1nlxk3cyh7Mbcis1GluqFoSVJ\naiy5yLqMjzOJmys2G1mqJ4aWJKlx5CLrLM7kBC6r2Gxkqd4YWpKkxpCLrM/xXb7CWdl4zBgjS/XJ\n0JIk1b9cZB3Lz/kvPtc5PhYWLSpiUtKGDSh6ApIkrVcusiZxEzczKRtffTUcc0wRk5K6x9CSJNWv\nXGS9lT9xD2/NxgsWwLhxRUxK6j5DS5JUf1KCfp1nt+zKHB5m12y8YgUMGlTExKSeMbQkSfWlowP6\n98+G2/EkC9kuG3vSuxqJoSVJqh8rV8LgwdlwBG08x4hsbGSp0RhakqT6sHw5bL55Ntycl1hO59jI\nUiPy9g6SpOI991xFZA1ihZGlpmBoSZKKtXAhjOh8e7Af7ayi80x3I0uNzNCSJBXn4Ydhu84T3YMO\nUu6pychSozO0JEnFuOce2LXzlg1BAjrvm2VkqRkYWpKk2rv5Znhr581HS5HVychSszC0JEm1ddVV\nMKnzY3SMLDUzQ0uSVDvf+17pU6DLjCw1O0NLklQbX/4ynHJKNjSy1Aq8YakkqfpOOAEuuywbGllq\nFYaWJKm6/uEf4De/yYZGllqJoSVJqp7dd4fZs7OhkaVWY2hJkqpj881Ln19YZmSpFRlakqS+F1E5\nNLLUorp11WFEPBERD0XE/RExo7xuq4i4NSIeLX8fntv/9IiYHxHzIuKwak1eklSHjCwp05PbO7w7\npbRXSmliefwl4PaU0njg9vKYiNgNmAJMACYBF0RE/z6csySpXhlZUoXe3EdrMnBFefkK4Ijc+mkp\npRUppceB+cC+vfh9JEmNwMiSXqO7oZWA2yJiZkRMLa8blVJaUl5+GhhVXh4DLMw9dlF5nSSpWRlZ\n0lp192T4A1NKiyPidcCtEfFwfmNKKUVEj/4ZlYNtKsB2223Xk4dKkuqJkSWtU7de0UopLS5/Xwr8\nktJbgc9ExGiA8vel5d0XA+NyDx9bXtf117w4pTQxpTRx5MiRG/8nkCQVZz2RNWiQkSVtMLQiYrOI\nGLpmGTgUmAVMB44r73YccEN5eTowJSIGR8SOwHjg3r6euCSpYOuJrP32gxUraj0hqf50563DUcAv\no/QPagBwVUrpNxHxZ+CaiDgBeBI4BiClNDsirgHmAKuBk1JK7VWZvSSpGOuJrLPPhjPOqPWEpPq0\nwdBKKT0G7LmW9c8BB6/jMWcDZ/d6dpKk+rOeyJo5E/bZp9YTkuqXd4aXJHXfeiLrpZdKn7ojqZOh\nJUnqnlxkrWAQQ+g8CcuT3qW1680NSyVJrSIXWfPY2ciSusnQkiStXy6ybuNgdmFeNjaypPUztCRJ\n65aLrCv4GIdwWzY2sqQNM7QkSWuXi6xv8q8cn328rZEldZcnw0uSXisXWafxHc7ntGxsZEndZ2hJ\nkirlIuuj/JSf89FsbGRJPWNoSZI65SLrcP6Xmzg8GxtZUs8ZWpKkklxk7c8fuZv9s7GRJW0cQ0uS\nWl1K0K/z2qgJzGIOEyo2S9o4hpYktbKODujfPxtux5MsZLtsbGRJvWNoSVKrWrUKBg3KhiNo4zlG\nZGMjS+o9Q0uSWtHy5RWfAL05L7GczrGRJfUNb1gqSa3m+ecrImsQK4wsqUoMLUlqJYsWwdZbZ8N+\ntLOKzrcPjSypbxlaktQq5s2DceOyYdBByj0NGFlS3zO0JKkV/PnPsMsu2TBIQOd9s4wsqToMLUlq\ndrfeCvvumw1LkdXJyJKqx9CSpGZ29dVw6KHZ0MiSasvQkqRm9YMfwJQp2dDIkmrP0JKkZvTVr8Jn\nP5sNjSypGN6wVJKazdSpcMkl2dDIkorjK1qS1Ew+85l1RtZWWxlZUq0ZWpLULP7xH+GHP8yG+cg6\n+mh47rkiJiW1Nt86lKRmcMAB8Mc/ZsN8ZE2bBh/8YBGTkmRoSVKj23FHeOKJbJiPrEWLYMyYAuYk\nCTC0JKmxDRgA7e3ZMB9Zq1aVNksqjv8EJalRRVQOc5HlSe9SffBkeElqREaW1BAMLUlqNEaW1DAM\nLUlqJEaW1FAMLUlqFEaW1HAMLUlqBEaW1JAMLUmqd0aW1LAMLUmqZ0aW1NAMLUmqV7nIeppRRpbU\ngAwtSapHucj6C3sxmqezsZElNQ5DS5LqTS6y/pfD2Ye/ZGMjS2oshpYk1ZNcZF3KJ3gf/5uNjSyp\n8RhaklQvcpH1dU7nn7g0GxtZUmPyQ6UlqR7kIutUzuO7nJqNjSypcRlaklS0XGQdx0/4KcdlYyNL\namy+dShJRTKypKbmK1qSVJRcZL2PX/G/vC8bG1lSczC0JKkIucg6gDv5IwdkYyNLah6GliTVWi6y\nduchZrN7NjaypOZiaElSrXR0QP/+2XB7nmAB22djI0tqPoaWJNXCqlUwaFA2HMlSnmVkNjaypOZk\naElStb36KmyySTYcyjJeZmg2NrKk5uXtHSSpmpYtq4iswbyaRdZOOxlZUrMztCSpWpYuhWHDsmE/\n2lnJYABOPhn++teiJiapVnzrUJKq4cknYYcdsmHQAZSuNvztb+Fd7ypkVpJqzNCSpL42ezbs3nnL\nhqDz/cHnn4fhw4uYlKQiGFqS1Jfuvhv23z8b5iOro6PiFlqSWoDnaElSX7n55nVGVkpGltSKuh1a\nEdE/Iv4SETeWx/8eEYsj4v7y1+G5fU+PiPkRMS8iDqvGxCWprlx9NUyalA27Rpak1tSTtw4/B8wF\ntsitOz+ldG5+p4jYDZgCTAC2BW6LiJ1TSu29nawk1aWLLoJPfSobGlmS1ujWK1oRMRZ4L/Djbuw+\nGZiWUlqRUnocmA/su/FTlKQ6dtZZRpakderuW4ffBb4IdHRZ/9mIeDAiLouINdfRjAEW5vZZVF4n\nSc3l5JPhK1/JhkaWpK42GFoR8T5gaUppZpdNFwI7AXsBS4Dv9OQ3joipETEjIma0tbX15KGSVLwP\nfhC+//1saGRJWpvuvKJ1APCBiHgCmAYcFBE/Tyk9k1JqTyl1AJfQ+fbgYmBc7vFjy+sqpJQuTilN\nTClNHDlyZNfNklS/3v52uOaabGhkSVqXDYZWSun0lNLYlNIOlE5yvyOl9JGIGJ3b7UhgVnl5OjAl\nIgZHxI7AeODePp63JBXjDW+AO+/MhkaWpPXpzQ1Lvx0RewEJeAI4ESClNDsirgHmAKuBk7ziUFJT\nGDwYVq7MhkaWpA2JVAc/HSZOnJhmzJhR9DQkad263G3UyJJaV0TMTClN7M6+3hlekjbEyJK0kQwt\nSVofI0tSLxhakrQuRpakXjK0JGltjCxJfcDQkqSujCxJfcTQkqQ8I0tSHzK0JGkNI0tSHzO0JAkq\nIquNEUaWpD5haElSLrIe4E28js4PujeyJPWGoSWpteUi6yYmsRcPZGMjS1JvGVqSWlcusi7neA7n\npmxsZEnqC4aWpNaUi6xv8q98gssBGDrUyJLUdwwtSa0nF1mf51xO55sAHHQQLFtW1KQkNSNDS1Jr\nyUXWx7mM8/g8ABdcALffXtSkJDWrAUVPQJJqJhdZk7me6UwGYNYsmDChqElJamaGlqTWkIusd/Fb\nfs+7AHj5Zdhss4LmJKnpGVqSml8usvbmPu5nbwBWr4b+/YualKRWYGhJam65yBrPI8xnPOCVhZJq\nw9CS1JxSgn6d1/tsy2KWsG22SZJqwdCS1HxWr4aBA7PhVjzHC2wFGFmSasvQktRcXn0VNtkkG27G\ny/yd0tnuRpakWvM+WpKax7JlFZE1iBVGlqRCGVqSmsPSpTBsWDbsz2pWMQgwsiQVx9CS1PgWLIBR\no7Jh0EEHpfs2GFmSimRoSWpsc+fC9ttnwyABpVs6GFmSimZoSWpc994Lu+2WDUuRVWJkSaoHhpak\nxnTrrbDfftnQyJJUjwwtSY3n2mvh0EOzoZElqV4ZWpIay8UXwzHHZEMjS1I9M7QkNY6vfx1OPDEb\nGlmS6p13hpfUGE45Bb73vWxoZElqBIaWpPr3oQ/BtGnZ0MiS1CgMLUn17Z3vhD/8IRsaWZIaiaEl\nqX7tvDM8+mg2NLIkNRpDS1J92nRTeOWVbGhkSWpEhpak+hNROTSyJDUob+8gqb4YWZKaiKElqX4Y\nWZKajKElqT6sI7K23trIktS4DC1JxVtHZE2eDM8+W8SEJKlvGFqSirWOyLrsMrj++iImJEl9x6sO\nJRVnHZE1Zw7sumsRE5KkvmVoSSrGOiJr2TIYOrSICUlS3/OtQ0m1l4us5xmeRdbq1UaWpOZiaEmq\nrVxkzWICW/M8ULqysH//oiYlSdVhaEmqnVxk3cIh7MEswNs3SGpehpak2shF1hV8jMO4BTCyJDU3\nQ0tS9eUi69t8geO5AjCyJDU/rzqUVF25yPoC3+ZcvgAYWZJag6ElqXpykfUJLuVyPgEYWZJah6El\nqTpykXUEv+QGjgCMLEmtxdCS1PdykfVu7uB3vBswsiS1HkNLUt/KRdY+zOQv7AMYWZJak6Elqe/k\nImtn5vEoOwNGlqTWZWhJ6r2UoF/n3WLGspDFjM02SVKrMrQk9U57Owzo/FGyNc/yPFsDRpYkGVqS\nNt6KFTBkSDbcnJdYzuaAkSVJ0IM7w0dE/4j4S0TcWB5vFRG3RsSj5e/Dc/ueHhHzI2JeRBxWjYlL\nKthLL1VE1mBeNbIkqYuefATP54C5ufGXgNtTSuOB28tjImI3YAowAZgEXBAR/ftmupLqwrPPwhZb\nZMP+rGYlgwEjS5LyuhVaETEWeC/w49zqyVD+wLLS9yNy66ellFaklB4H5gP79s10JRVu4UIYOTIb\nBh10UPpvKSNLkip19xWt7wJfBDpy60allJaUl58GRpWXxwALc/stKq+rEBFTI2JGRMxoa2vr2awl\nFePhh2G77bJhkIDSLR2MLEl6rQ2GVkS8D1iaUpq5rn1SSgno0Y/ZlNLFKaWJKaWJI3P/dSypTv35\nz7Drrtkwcv/kjSxJWrvuXHV4APCBiDgcGAJsERE/B56JiNEppSURMRpYWt5/MTAu9/ix5XWSGtXt\nt8N73pMNjSxJ6p4NvqKVUjo9pTQ2pbQDpZPc70gpfQSYDhxX3u044Iby8nRgSkQMjogdgfHAvX0+\nc0m18YtfGFmStJF6cx+tbwLXRMQJwJPAMQAppdkRcQ0wB1gNnJRSau/1TCXV3iWXwNSp2dDIkqSe\niVQHPy0nTpyYZsyYUfQ0JOV985tw+unZcE1kbbMNLFmyrgdJUvOLiJkppYnd2bcn99GS1CpOO22t\nkXXMMUaWJPWEH8EjqdKxx8JVV2XDNZF1+eVw/PEFzUmSGpShJanTwQfDHXdkwzWR9dBDsPvuRU1K\nkhqXoSWpZJddYN68bLgmsl54AbbcsqhJSVJjM7QkwdCh8PLL2XBNZK1aBQP8KSFJG80foVKri6gc\nliOro+M1myRJPWRoSa1sHZFVB3d9kaSm4O0dpFZlZElS1RlaUisysiSpJgwtqdUYWZJUM4aW1EqM\nLEmqKUNLahVGliTVnKEltQIjS5IKYWhJzS4XWcsYamRJUg0ZWlIzy0XWHHZlGMsAI0uSasXQkppV\nLrJu42AmMAcwsiSplgwtqRnlIutnfIRDuA0wsiSp1gwtqdnkIutcPs/H+BlgZElSEQwtqZnkIuuL\nfIsvcC5gZElSUfxQaalZ5CLrBH7MZZwAGFmSVCRDS2oGucg6il9wHUcBRpYkFc3QkhpdLrIO5jbu\n4GDAyJJbwsB+AAAXs0lEQVSkemBoSY0sF1kT+TMzmQgYWZJULwwtqVHlIuuNPMwjvBEwsiSpnhha\nUqNJCfp1XjA8jgUsYly2SZJUPwwtqZG0t8OAzn+2I2jjOUYARpYk1SPvoyU1ihUrKiJrKMt4jhGM\nG2dkSVK9MrSkRvDyyzBkSDYczKu8zFA+9CFYsKDAeUmS1svQkurds8/C0KHZsD+rWclgLr8crrqq\nwHlJkjbIc7SkerZoEYwblw2DDiC47z7Ye+/ipiVJ6h5DS6pX8+bBLrtkw6B0IlZbG4wYUdSkJEk9\nYWhJ9WjmTJg4MRuuiawVK2DQoKImJUnqKc/RkurNHXesNbI6OowsSWo0hpZUT667Dg4+OBuuiayU\nKm4EL0lqEIaWVC8uvRSOOiob5iNLktSYDC2pHnz72/BP/5QNjSxJag6eDC8V7QtfgHPPzYZGliQ1\nD0NLKtLHPgY/+1k2NLIkqbkYWlJRDjkEbrstGxpZktR8DC2pCBMmwJw52dDIkqTmZGhJtTZ8OLz4\nYjY0siSpeRlaUi11uRmWkSVJzc3bO0i1YmRJUssxtKRaMLIkqSUZWlK1GVmS1LIMLamajCxJammG\nllQtRpYktTxDS6oGI0uShKEl9T0jS5JUZmhJfcnIkiTlGFpSX8lF1osMM7IkSYaW1CdykfUQuzOc\n0kfsGFmS1NoMLam3cpF1E5N4Ew8BRpYkydCSeicXWZfxcQ7nJrbZxsiSJJUYWtLGykXW1zmdE7iM\nD3wAliwpcE6SpLpiaEkbIxdZp3A+Z/J1zj8fbrihwDlJkurOBkMrIoZExL0R8UBEzI6I/yiv//eI\nWBwR95e/Ds895vSImB8R8yLisGr+AaSay0XWR/kp3+MU/vAHOOWUAuckSapLA7qxzwrgoJTSyxEx\nELgzIm4qbzs/pXRufueI2A2YAkwAtgVui4idU0rtfTlxqRC5yHovN/Jr3suiRTBmTIFzkiTVrQ2G\nVkopAS+XhwPLX+s71XcyMC2ltAJ4PCLmA/sCf+rlXKVi5SLrAO7kjxzA8uWw6aYFzkmSVNe6dY5W\nRPSPiPuBpcCtKaV7yps+GxEPRsRlETG8vG4MsDD38EXldV1/zakRMSMiZrS1tfXijyDVQC6y9uBB\n/sgBtLcbWZKk9etWaKWU2lNKewFjgX0jYnfgQmAnYC9gCfCdnvzGKaWLU0oTU0oTR44c2cNpSzWS\nUkVk7chjzGIPUoJ+XkoiSdqAHj1VpJReBH4LTEopPVMOsA7gEkpvDwIsBsblHja2vE5qLO3tFTU1\niqd5gh29R5Ykqdu6c9XhyIjYsry8CXAI8HBEjM7tdiQwq7w8HZgSEYMjYkdgPHBv305bqrIVK2BA\n5ymMw3iRpYwysiRJPdKdqw5HA1dERH9KYXZNSunGiPhZROxF6cT4J4ATAVJKsyPiGmAOsBo4ySsO\n1VBeegm22CIbDuEVVjDEyJIk9VikOnj2mDhxYpoxY0bR05CgrQ1e97psOIBVtDPAyJIkZSJiZkpp\nYnf29XReaY0FCyoiK+gwsiRJvWJoSQBz5sD222fDIJX+18iSJPWCoSXdfTdMmJANo3w/XiNLktRb\nhpZa2803w/77Z0MjS5LUlwwtta7//m+YNCkbGlmSpL5maKk1ff/78OEPZ0MjS5JUDYaWWs9Xvwon\nn5wNjSxJUrV054alUvP45CfhRz/KhkaWJKmaDC21jsmTYfr0bGhkSZKqzdBSa5g4EWbOzIZGliSp\nFgwtNb9Ro2Dp0mxoZEmSasXQUnOLqBwaWZKkGvKqQzUvI0uSVDBDS83JyJIk1QFDS83HyJIk1QlD\nS81lLZG1zTZGliSpGIaWmsdaImvSJFiypKD5SJJanqGl5rCWyPrP/4SbbipoPpIk4e0d1AzWElk3\n3QSTJhU0H0mSygwtNba1RNYjj8D48QXNR5KkHENLjWstkfXiizBsWEHzkSSpC8/RUmPKRdYyhhIk\nVq40siRJ9cXQUuPJRdYcdmUYy+jogIEDC5yTJElrYWipseQi6xYOYQJzSOk17yJKklQXDC01jlxN\n/YTjOIxbvBGpJKmuGVpqDLnI+ib/ysf5iZElSap7XnWo+peLrNP4DudzmpElSWoIhpbqWy6yjudy\nruB4I0uS1DAMLdWvXGR9gBv4FR8wsiRJDcXQUn3KRdbb+QN38nYjS5LUcAwt1Z9cZL2JB3iINxlZ\nkqSGZGipfqQE/TovhN2Jv/I4OxlZkqSGZWipPrS3w4DOv47bsIRn2MbIkiQ1NENLxVu5EgYPzobD\neJFlDDOyJEkNzxuWqlgvv1wRWZvwdyNLktQ0DC0V59lnYejQbDiAVbzKJkaWJKlpGFoqxqJFMHJk\nNgw6aGeAkSVJaiqGlmrv8cdh3LhsGKTS/xpZkqQmY2iptmbPhp12yoalyMLIkiQ1JUNLtXP33bD7\n7tnQyJIkNTtDS7Vxyy2w//7Z0MiSJLUCQ0vVd+21cNhh2TBIDBhgZEmSmp+hpeq65BI45phsGCTG\nj4dVqwqckyRJNWJoqXrOOQemTs2GQeLoo+GRRwqckyRJNWRoqTouvBC++MVsGCTOOaf0LqIkSa3C\nzzpU3/vGN+CMM7JhkLjtNjj44ALnJElSAQwt9a0vfrH0lmFZkHj0UXjDGwqckyRJBTG01Hf++Z/h\nxz/OhkFi2bKKjzOUJKmlGFrqG0ccATfckA2DxIoVMGhQgXOSJKlghpZ678AD4a67smGQ6OiAiALn\nJElSHTC01Dvjx8P8+dkwSN6IVJKkMkNLG2+LLeCll7KhkSVJUiVDSxuny/uCRpYkSa/lDUvVc7nI\nWsI2RpYkSetgaKlncpF1P3uyLUuMLEmS1sHQUvflIusWDmFv7jeyJElajw2GVkQMiYh7I+KBiJgd\nEf9RXr9VRNwaEY+Wvw/PPeb0iJgfEfMi4rBq/gFUI7nIupIPcxi3GFmSJG1Ad17RWgEclFLaE9gL\nmBQRbwW+BNyeUhoP3F4eExG7AVOACcAk4IKI6F+NyatGcpH1fT7DR7jSyJIkqRs2GFqp5OXycGD5\nKwGTgSvK668AjigvTwampZRWpJQeB+YD+/bprFU7ucj6Gl/hZL5vZEmS1E3dOkcrIvpHxP3AUuDW\nlNI9wKiU0pLyLk8Do8rLY4CFuYcvKq9To8lF1qmcx1f5mpElSVIPdCu0UkrtKaW9gLHAvhGxe5ft\nidKrXN0WEVMjYkZEzGhra+vJQ1ULucj6BJfyXU41siRJ6qEeXXWYUnoR+C2lc6+eiYjRAOXvS8u7\nLQbG5R42tryu6691cUppYkpp4siRIzdm7qqWXGQdxS+4nE8YWZIkbYTuXHU4MiK2LC9vAhwCPAxM\nB44r73YccEN5eTowJSIGR8SOwHjg3r6euKokF1mHcjPXcZSRJUnSRurOR/CMBq4oXznYD7gmpXRj\nRPwJuCYiTgCeBI4BSCnNjohrgDnAauCklFJ7daavPpWLrLdxF3/ibUaWJEm9EKkOnkknTpyYZsyY\nUfQ0Wlsusvbkfh5kTyNLkqS1iIiZKaWJ3dnXD5VWRWSN5xHmM97IkiSpDxharS4XWWNZyGLGGlmS\nJPURQ6tVpQT9Oq+F2IK/8RJbGFmSJPUhP1S6FXWJrE1ZzphdjCxJkvqaodVqOjoqImsAqzjyw5sy\nd26Bc5IkqUn51mEraW+HAZ3/l/ejnV/f1I9JkwqckyRJTczQahWrV8PAgdkw6ODBB4M99ihwTpIk\nNTlDqxWsXAmDB2fDoINFi4IxftS3JElVZWg1u1dfhU02AWAlAxnMSl54AbbcsuB5SZLUAjwZvpkt\nX55F1nNsxWBWsmKFkSVJUq0YWs1q2TLYfHMAHmcHRvAcq1bBoEEFz0uSpBZiaDWjF16AYcMAeJA9\n2InH6eiouOBQkiTVgKHVbNraYKutALiLt7EnD9LRUfFJO5IkqUYMrWayZAm87nUA3MIhHMhdpGRk\nSZJUFEOrWSxYANtuC8B1HMlh3OJH6kiSVDBDqwmk+X+F7bcH4Gd8hKO4zsiSJKkOGFoNrn32w8T4\nNwBwESfyMX5mZEmSVCcMrQb26r0P0n/3XQH4DqfxKS4ysiRJqiOGVoN68bYZDNlvTwDO4kz+he8Y\nWZIk1RlDqwEtvPqPbHnIWwA4g7P5CmcZWZIk1SFDq8E88N3fMm7KAQCcwvl8gzOMLEmS6pSh1UBu\n/8Jv2PPUgwA4kYv4HqcYWZIk1TE/lKVB/Pzo6/nI/xwJwMe4gp/xMSNLkqQ6Z2g1gPP2ncZpf/4Q\nAMdwNddyjJElSVIDMLTq3JfH/oSzFn8cgA9wA7/iA0aWJEkNwtCqY6cOuZDzV3wagMP4DbdwmJEl\nSVIDMbTq1GlxHufzeQDeye/4A+80siRJajBedViHvhxncV45svbnj9wz2MiSJKkRGVp15utxBmfx\nFQD2YSZPbbc/r75a8KQkSdJGMbTqyHfjFM7gGwDswYNs8c59ePLJgiclSZI2mqFVJy6OqZzC9wB4\nIw/z9k/twe9+V+ycJElS7xhadeDKOJapXALATvyVT573Ri64oOBJSZKkXvOqw4LdEJM5lukAjGMB\nF904jve+t+BJSZKkPmFoFej2OJjJ3AHANizhxj9vw8SJBU9KkiT1GUOrIPfEfhzMvQCMoI2ZT4xg\n++0LnpQkSepThlYBZscE9mMOAFvyAo89tyVbbVXwpCRJUp8ztGpsYYxjAosAGMoy2l4ZypAhBU9K\nkiRVhaFVQ3+LYYxjGQCbspy/rdyUgQMLnpQkSaoaQ6sWUqKjX3+GUfocnSG8wisdQ4goeF6SJKmq\nDK1qSwn69ctuWDaIFazoGGRkSZLUAgytaipH1hoDWMXq5CGXJKlVeGf4Kkmr2ysiqx/tRpYkSS3G\n0KqC9hWriYGdURV00JE81JIktRqf/fvYK39bSf8hnZcSBh2k5AlZkiS1IkOrD7349KtssuVgAFYy\nkCAZWZIktTBDq488MXs5W47eBIDnGc5gVpJSwZOSJEmFMrT6wD23LmOH3TcH4Em2Y2ueN7IkSZKh\n1VvXX/4C+x06DIBZTGAHnjSyJEkSYGj1ynfPbOOIT5Q+DfpPvJU9mGVkSZKkjDd22kifOWoJP7hu\nWwBu42AO4TYjS5IkVTC0NsIRb17I9fdtB8D1TOZIrjeyJEnSaxhaPbTfyMe459nXA3AlH+YjXGlk\nSZKktTK0emCXeJiH2RWAHzGVT/IjI0uSJK2TJ8N30x7xUBZZ53GqkSVJkjbI0OqGN8dMHuJNAJzF\nmZw55DwjS5IkbZChtQH7x5+YyUQAzuBsfjT2LF55peBJSZKkhrDB0IqIcRHx24iYExGzI+Jz5fX/\nHhGLI+L+8tfhucecHhHzI2JeRBxWzT9ANb0rfsefeBsAp3Ief3znGSxcWPCkJElSw+jOyfCrgc+n\nlO6LiKHAzIi4tbzt/JTSufmdI2I3YAowAdgWuC0idk4ptfflxKttUvyG3/EPAHySC+HET/K7iwqe\nlCRJaigbfEUrpbQkpXRfefklYC4wZj0PmQxMSymtSCk9DswH9u2LydbKEXE9vylH1nH8hJ2+9Uku\nMrIkSVIP9egcrYjYAdgbuKe86rMR8WBEXBYRw8vrxgD5N9gWsf4wqysfjKu5niNLy0zjiOuO44tf\nLHhSkiSpIXU7tCJic+B/gFNSSsuAC4GdgL2AJcB3evIbR8TUiJgRETPa2tp68tCqOT5+wtVMAWAy\n13Pqnz7IkUcWPClJktSwuhVaETGQUmRdmVK6DiCl9ExKqT2l1AFcQufbg4uBcbmHjy2vq5BSujil\nNDGlNHHkyJG9+TP0iU/GRfyEjwMwiZs4Z95k3vrWgiclSZIaWneuOgzgUmBuSum83PrRud2OBGaV\nl6cDUyJicETsCIwH7u27Kfe9U+N8LuJTALyL33Lls5PYeeeCJyVJkhped646PAD4KPBQRNxfXncG\n8KGI2AtIwBPAiQAppdkRcQ0wh9IViyfV8xWHZ8bZnM+XAXgbd3HLS29j880LnpQkSWoKGwytlNKd\nQKxl06/X85izgbN7Ma+aODvO5Gy+DsA+zOTuFfswaFDBk5IkSU2jZT9U+rw4jTM5H4A9eJD7V+9B\n//4FT0qSJDWVlgytH8WJnMbFALyRh3m4443E2l6zkyRJ6oWWCq2U4Kp+x3IiVwGwE3/lrx07GVmS\nJKkqWia0OjrgV/0ncyzTARjHAhamcRt4lCRJ0sZrmdC6vf8hTOY2AEbzFEvS6A08QpIkqXd69BE8\njer88xKHlCNrJEuNLEmSVBMtEVof+nAwjgUMZCVtqfi70EuSpNbQEm8dbrMNno8lSZJqriVe0ZIk\nSSqCoSVJklQlhpYkSVKVGFqSJElVYmhJkiRViaElSZJUJYaWJElSlRhakiRJVWJoSZIkVYmhJUmS\nVCWGliRJUpUYWpIkSVViaEmSJFWJoSVJklQlhpYkSVKVGFqSJElVYmhJkiRViaElSZJUJYaWJElS\nlRhakiRJVWJoSZIkVYmhJUmSVCWGliRJUpVESqnoORARbcCTRc+jICOAZ4ueRIPzGPaOx6/3PIa9\n4/HrPY9h7/T0+G2fUhrZnR3rIrRaWUTMSClNLHoejcxj2Dsev97zGPaOx6/3PIa9U83j51uHkiRJ\nVWJoSZIkVYmhVbyLi55AE/AY9o7Hr/c8hr3j8es9j2HvVO34eY6WJElSlfiKliRJUpUYWjUWEadG\nxOyImBUR/x0RQyJiq4i4NSIeLX8fXvQ860VEXBYRSyNiVm7dOo9XRJweEfMjYl5EHFbMrOvLOo7h\nORHxcEQ8GBG/jIgtc9s8hjlrO365bZ+PiBQRI3LrPH5drOsYRsRny38PZ0fEt3PrPYZdrOPf8V4R\ncXdE3B8RMyJi39w2j2FORIyLiN9GxJzy37fPlddX//kkpeRXjb6AMcDjwCbl8TXA8cC3gS+V130J\n+FbRc62XL+AdwD7ArNy6tR4vYDfgAWAwsCPwV6B/0X+Gor/WcQwPBQaUl7/lMezZ8SuvHwfcTOke\ngCM8fj3+O/hu4DZgcHn8Oo9hj4/hLcA/lJcPB37nMVzn8RsN7FNeHgo8Uj5OVX8+8RWt2hsAbBIR\nA4BNgaeAycAV5e1XAEcUNLe6k1L6A/B8l9XrOl6TgWkppRUppceB+cC+tLi1HcOU0i0ppdXl4d3A\n2PKyx7CLdfwdBDgf+CKQP9HV47cW6ziGnwK+mVJaUd5naXm9x3At1nEME7BFeXkYpecT8Bi+Rkpp\nSUrpvvLyS8BcSi9+VP35xNCqoZTSYuBcYAGwBPhbSukWYFRKaUl5t6eBUQVNsVGs63iNARbm9ltU\nXqf1+wRwU3nZY9gNETEZWJxSeqDLJo9f9+0MvD0i7omI30fEW8rrPYbddwpwTkQspPTccnp5vcdw\nPSJiB2Bv4B5q8HxiaNVQ+b3fyZRehtwW2CwiPpLfJ5Ves/RS0G7yePVORJwJrAauLHoujSIiNgXO\nAP6t6Lk0uAHAVsBbgS8A10REFDulhvMp4NSU0jjgVODSgudT9yJic+B/gFNSSsvy26r1fGJo1dZ7\ngMdTSm0ppVXAdcDbgGciYjRA+fvS9fwaWvfxWkzpvJk1xpbXaS0i4njgfcCx5R8w4DHsjtdT+o+l\nByLiCUrH6L6I2AaPX08sAq5LJfcCHZQ+b85j2H3HUXoeAbiWzre2PIZrEREDKUXWlSmlNcet6s8n\nhlZtLQDeGhGblv/L7WBK7xNPp/QPhvL3GwqaX6NY1/GaDkyJiMERsSMwHri3gPnVvYiYROn8og+k\nlP6e2+Qx3ICU0kMppdellHZIKe1AKRj2SSk9jcevJ66ndEI8EbEzMIjSh/p6DLvvKeCd5eWDgEfL\nyx7DLsrPuZcCc1NK5+U2Vf/5pOgrAVrtC/gP4GFgFvAzSlc0bA3cTukfyW3AVkXPs16+gP+mdD7b\nKkpPaCes73gBZ1K6OmQe5atxWv1rHcdwPqXzD+4vf13kMez+8euy/QnKVx16/Lp/DCmF1c/LPwvv\nAw7yGPb4GB4IzKR0ddw9wJs9hus8fgdSelvwwdzPvcNr8XzineElSZKqxLcOJUmSqsTQkiRJqhJD\nS5IkqUoMLUmSpCoxtCRJkqrE0JIkSaoSQ0uSJKlKDC1JkqQq+f+GL2IqZLekEAAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xa2a4320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The blue lines are the image of  Bayesian Linear Regression estimated parameters. The red one is the true parameters.\n"
     ]
    }
   ],
   "source": [
    "t = BLR_test(test_x, test_y, posterior_mu_w, posterior_cov_w, sampling_size=100, output_type='all', retained_digit=2)\n",
    "plt.figure(figsize=(10,10)) \n",
    "plt.plot(test_x[:,0], t, 'b')\n",
    "plt.plot(test_x[:,0], test_x.dot(W_real.T), 'r')\n",
    "plt.show()\n",
    "print('The blue lines are the image of  Bayesian Linear Regression estimated parameters. The red one is the true parameters.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如果你有什么疑问或者收获, 你可以在这里记录下来.\n",
    "###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}