machine-learning/logistic-regression-using-theano.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# Logistic Regression using Theano"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Import the required modules\n",
      "%pylab inline\n",
      "from theano import tensor as T\n",
      "from theano import function\n",
      "from theano import pp\n",
      "from theano import Param\n",
      "from theano import grad\n",
      "from theano import shared\n",
      "import numpy as np\n",
      "\n",
      "class LogisticRegression:\n",
      "    '''\n",
      "    This class performs logistic regression.\n",
      "    '''\n",
      "    def __init__(self, alpha=0.001, iterations=10000, threshold=.5):\n",
      "        '''\n",
      "        1. alpha is the learning rate. \n",
      "            The default value of alpha is 0.001.\n",
      "        2. iterations is the number of times the weights will be updated.\n",
      "            The default value is 1000\n",
      "        3. threshold is the threshold used to convert the probability\n",
      "            to either 0 or 1 class label\n",
      "        '''\n",
      "        # Set the learning rate\n",
      "        self.alpha = alpha\n",
      "        # Set the number of iterations\n",
      "        self.iterations = iterations\n",
      "        # Set the threshold value\n",
      "        self.threshold = threshold\n",
      "        \n",
      "        # Define the symbolic Variables\n",
      "        X, W = T.dmatrices('training-set', 'weights')\n",
      "        y = T.matrix('testing-set') \n",
      "        m, a = T.scalars('no-of-samples', 'learning-rate')\n",
      "        \n",
      "        # Define the cost function and the gradient functions\n",
      "        sigmoid_fn = 1/(1 + T.exp(-T.dot(X, W)))\n",
      "        h = sigmoid_fn\n",
      "        cost_fn = -((T.dot(y, T.transpose(h)) + T.dot(1 - y, T.transpose(1 - h)))/(1 * m)).sum()\n",
      "        grad_fn = (T.dot(T.transpose(X), h - y))/(m)\n",
      "        updated_wts = W - a * grad_fn\n",
      "        \n",
      "        # Compile the Theano functions\n",
      "        self.cost = function([X, y, W, m], cost_fn)\n",
      "        self.new_wts = function([X, y, W, m, a], updated_wts)\n",
      "        self.pred = function([X, W], h)\n",
      "        \n",
      "    def fit(self, X, y):\n",
      "        '''\n",
      "        The fit member function is used to train the classifier.\n",
      "        1. X is array-like data vector\n",
      "        2. y is array-like target vector\n",
      "        '''\n",
      "        # Add the bias feature to the data samples\n",
      "        bias = np.ones((X.shape[0], 1))\n",
      "        X = np.hstack((X, bias))\n",
      "        \n",
      "        # m is the number of data samples used\n",
      "        m = X.shape[0]\n",
      "        \n",
      "        # Initialize all weights to 0\n",
      "        self.W = np.zeros((X[0].shape[0],1), dtype=np.float)\n",
      "        \n",
      "        # Iterate and update the cost function and the weights\n",
      "        for i in xrange(self.iterations):\n",
      "            cst = self.cost(X, y, self.W, m)\n",
      "            self.W = self.new_wts(X, y, self.W, m, self.alpha)    \n",
      "        \n",
      "        # Set the coef_ and intercept_\n",
      "        self.coef_ = self.W[:-1]\n",
      "        self.intercept_ = self.W[-1]\n",
      "            \n",
      "    def predict(self, X):\n",
      "        '''\n",
      "        This is function to predict target value. \n",
      "        \n",
      "        1. X is array-like data vector whose target vector is predicted\n",
      "        '''\n",
      "        # Add the bias feature to the data samples\n",
      "        bias = np.ones((X.shape[0], 1))\n",
      "        X = np.hstack((X, bias))\n",
      "        \n",
      "        # Return the predicted target label\n",
      "        return (self.pred(X, self.W) > self.threshold) * 1\n",
      "    \n",
      "    def predict_proba(self, X):\n",
      "        '''\n",
      "        This is function gives the probability of how sure. \n",
      "        \n",
      "        1. X is array-like data vector whose target vector is predicted\n",
      "        '''\n",
      "        # Add the bias feature to the data samples\n",
      "        bias = np.ones((X.shape[0], 1))\n",
      "        X = np.hstack((X, bias))\n",
      "        \n",
      "        # Return the predicted target label\n",
      "        prob = self.pred(X, self.W)\n",
      "        for index, value in enumerate(prob < self.threshold):\n",
      "            if value:\n",
      "                prob[index] = 1 - prob[index]\n",
      "        return prob"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Populating the interactive namespace from numpy and matplotlib\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Prepare the train data\n",
      "x1 = [[x, y] for x in range(3, 8) for y in range(1, 6) if x - 3 > y - 1 ]\n",
      "x2 = [[x, y] for x in range(1, 6) for y in range(3, 8) if y - 3 > x - 1 ]\n",
      "X_train = []\n",
      "for x in x1:\n",
      "    X_train.append(x)\n",
      "for x in x2:\n",
      "    X_train.append(x)\n",
      "X_train = np.array(X_train)\n",
      "\n",
      "# Prepare the train data labels\n",
      "y1 = [[0]]*len(x1)\n",
      "y2 = [[1]]*len(x2)\n",
      "y_train = [] \n",
      "for y in y1:\n",
      "    y_train.append(y)\n",
      "for y in y2:\n",
      "    y_train.append(y)\n",
      "y_train = np.array(y_train)\n",
      "\n",
      "# Prepare the test data\n",
      "X_test = [[randint(0, 8) , randint(0, 8)] for x in range(25)]\n",
      "X_test = np.array(X_test)\n",
      "\n",
      "# Fit and predict\n",
      "clf = LogisticRegression(iterations=10000)\n",
      "clf.fit(X_train, y_train)\n",
      "y_pred = clf.predict(X_test)\n",
      "y_pred_prob = clf.predict_proba(X_test)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# NOTE :: This graphic will only work for 2-D features, although the\n",
      "# code can work for any dimensions\n",
      "# Plot the boundary\n",
      "# Training set in stars\n",
      "# Testing set in circles\n",
      "# Also show the probability\n",
      "plot(X_train[:,0][y_train[:,0]==0], X_train[:,1][y_train[:,0]==0], \"*r\")\n",
      "plot(X_train[:,0][y_train[:,0]==1], X_train[:,1][y_train[:,0]==1], \"*b\")\n",
      "plot(X_test[:,0][y_pred[:,0]==0], X_test[:,1][y_pred[:,0]==0], \"or\")\n",
      "plot(X_test[:,0][y_pred[:,0]==1], X_test[:,1][y_pred[:,0]==1], \"ob\")\n",
      "x_min, x_max, y_min, y_max = 0 , 8, 0, 8\n",
      "axis([x_min, x_max, y_min, y_max])\n",
      "y_min_val = (clf.coef_[1][0] * x_min + clf.intercept_[0])/-clf.coef_[0][0]\n",
      "y_max_val = (clf.coef_[1][0] * x_max + clf.intercept_[0])/-clf.coef_[0][0]\n",
      "plot([x_min, x_max], [y_min_val, y_max_val])\n",
      "for index, (x, y) in enumerate(X_test):\n",
      "    text(x, y, round(y_pred_prob[index][0], 2))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEACAYAAACeQuziAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X1YVGXeB/DvUVBeNHSAfANDRxQGdJxSeR4vU0yRlGwl\nrRR3tfKtbUErtUutBE0w09xScjE13FwV15c0HHmRlGAz1zW1AkxZAx2wdc3HUV5EGOb3/IFOjAww\nyMycM8zvc11zXc6Ze4avR7z5cs+ZcwQiIjDGGJOcdmIHYIwxZhpP0IwxJlE8QTPGmETxBM0YYxLF\nEzRjjEkUT9CMMSZRzU7Qq1evRlBQEAYOHIioqCjcvXvXFrkYY8zhNTlBFxcXY8uWLThz5gx+/PFH\n1NbWIiUlxVbZGGPMoTk19eAjjzwCZ2dnVFZWon379qisrESvXr1slY0xxhxakw1aJpNh4cKF6N27\nN3r27IkuXbpg7NixtsrGGGMOrckJ+tKlS/joo49QXFyMq1evory8HDt37rRVNsYYc2hNLnGcPn0a\nw4cPh6enJwDgueeew4kTJzB9+nTDmH79+uHSpUvWTckYY22MXC7Hv//97ybHNNmgAwICcPLkSdy5\ncwdEhKysLCgUCqMxly5dAhFJ/hYbGyt6Bs7JOTmnY2asqSEkJBC8vAibNxP0ejKr2DbZoJVKJWbM\nmIEhQ4agXbt2ePzxxzF37txmX5Qxxlid/Hzg5ZcBDw/g9GngscfMf26TEzQAvPXWW3jrrbdak48x\nxhyOTgesXQusXw/ExwNz5gCC0LLXaHaCbitCQ0PFjmAWzmlZnNOy7CGnFDK2pjXXJxBRq07YLwgC\nWvkSjDHWJrSkNZszdzpMg2aMMWuyVGuuj0+WxBhjraDTAatXA6GhwOzZQGamZSZngBs0Y4w9NGu0\n5vq4QTPGWAtZszXXxw2aMcZawNqtuT5u0IwxZgZbteb6uEEzxlgzbNma6+MGzRhjjRCjNdfHDZox\nxkwQqzXXxw2aMcbqEbs118cNmjHG7pFCa66PGzRjzOFJqTXXxw2aMebQpNaa6+MGzRhzSFJtzfXx\nBM0YaxGVSgVBECAIAtzd3Rs83qlTJ8Pj929paWkA0GC7taSnpyMgIAD+/v5Ys2ZNg8eTk7Ph4uKB\nhAQVHn1Uhf/+d1WLT6ZvE9RKFngJxpid0Gq1BIBWrFhBV69eJQD06quvNjpeoVAYzREA6IsvvrBq\nRp1OR3K5nIqKiqi6upqUSiUVFBQQEVFNDVFCAtEjjxynQYMmkl5v1ShNMmfu5AbNGDPbCy+8AABY\nvnw5evToATc3N3z++eeNjj9//jy8vb2NttXW1lo146lTp9CvXz/4+fnB2dkZU6dOxaFDh5CfDwwf\nDhw7BmzeDPTuTdJszfU0O0FfuHABKpXKcPPw8MCGDRtskY0xJjGFhYVGSxNdunTB3bt3TY49d+4c\niAh//etfjbZPmTIFgiDAy8vLKhlLS0vh6+truN+jhw8OHSo1Wmvu0UPAiRMnoFQqMWHCBBQUFFgl\nS2s1O0EPGDAAZ8+exdmzZ/Hdd9/Bzc0NkZGRtsjGGJOYdu3M/6X7d7/7HQBg/Pjxhm2bNm0CEeFv\nf/sbbty4YZXrB9b/AZKfD6xYAZSU1B2hMXdu3SWoHn/8cWg0Gnz//feIiYnBpEmTLJ7DElq0xJGV\nlQW5XG7008keEBGWLPlA8tdO5JyWZS857UFwcCQEYTwuXSoHESE4uK6k3bx5Ex07djT5nCtXrqB3\n795G2/74xz8CAKZPnw4XFxd89913FsuYo1bjnfBwpMXH46svDmLuzJ8QGgoolRr86U8+RkdodO7c\nGW5ubgDqfoDU1NTg//7v/yyWxVJaNEGnpKQgKirKWlmsZv/+DGza9AsOHMgUO0qTOKdl2UtOqQsO\njkR+vheANAB5AID8/Mvo3z8cd+7cwYwZMxo85+jRowAAtVpt2Hbu3Dnk5uYCAP71r3+hqqoK/fv3\nt0jGHLUaGQsWYFVmJmLOVuHqjY44sudHfJyQgaKiPfjd7541Gn/t2jXDD+5Tp06BiCCTySySxaLM\nfcfx7t275OXlRf/9739b/E6kWJKSdpBCEUH+/ssI0JO//zJSKCIoKWmH2NGMcE7Lspec9gJ4mgCq\ndxtEAAgAubq6EhGRTCYjmUxmeE63bt1IEASj14mNjTU8r/5zLeHtceOoBu0pAUvIC/+lGDxN/QHq\n6upKCQkJRESUlJRESUlJRESUmJhIQUFBpFQq6X//93/p22+/tVgWc5kzd5o9ux48eJDCw8NNfpHY\n2FjD7fjx4y0KaU16vZ7+/vcj5Ou7hAAiX98ltHdvGunFPLbGBM5pWfaS014AUx6YoO/fpogdzeCP\nQ2bSUPyTxiKTitHbEDJ21CixoxkcP37caK40Z4I2+6Peu3fvxrRp00w+FhcX18oebx33D4bXaqug\nULwJjUZv9QPkHwbntCx7yWk/ylu43XZ0OmDtWuCv33+EP+MtzMEW1P9XrnVxES3bg0JDQ43eFF2x\nYkWzzzFrDbqiogJZWVl47rnnHjqcWAoLNUhOfhp5eR8iOXk8Cgs1YkcyiXNalr3ktAdBQS4A5jyw\ndfa97eKpf1xz8uYzuCw/ZjQ5L5PLERYTI1o+SxCIWvcWtyAI/C45Y21c3RuFVQA6AShHUJAL8vK+\nECXL/da8fj0QHw/MmVN36FyOWo2jGzeifVUVal1cEBYTg5EREaJkNIc5cydP0Iwxu1H/zHNbt0rv\n5EYtYc7cyR/1ZoxJnj2cec4a+HzQjDFJk/L5mq2NGzRjTJIctTXXxw2aMSY5jtya6+MGzRiTDG7N\nxrhBM8YkgVtzQ9ygGWOi4tbcOG7QjDHRcGtuGjdoxpjNcWs2DzdoxphNcWs2HzdoxphNcGtuOW7Q\njDGr49b8cLhBM8ashltz63CDZoxZBbfm1uMGzRizKG7NluMwE3T//v3Rrl07uDRxCRylUokOHTrA\n1dUVu3fvtmE6+5Seno6AgAD4+/tjzZo1DR6/efMmIiMjoVQqERISgvz8fADAhQsXoFKpDDcPDw9s\n2LDB1vElp7n9uW7dOsM+GzhwIJycnKDVagEAfn5+GDRoEFQqFYYNGyZazvx8IDg4G+++64FHH1Xh\nL39RIT5+lVXztGmtvRCiBV7CJjZu3Ei7du2ijh07mnw8Li6OvL29iYho69at5O7ubst4dken05Fc\nLqeioiKqrq4mpVJJBQUFRmMWLVpEK1euJCKin376icaMGdPgdWpra6l79+505coVm+SWKnP2Z32p\nqalG+9PPz49u3LghWs6aGqKEBCIvL6I33zxOEydOtHoWe2fO3OkwDTo6Ohq+vr6NPr5nzx7DRXFn\nzZqFmpoa5OXl2Sqe3Tl16hT69esHPz8/ODs7Y+rUqTh06JDRmPPnz2P06NEAgAEDBqC4uBjXr183\nGpOVlQW5XN7kv40jMGd/1rdr164GF3EmG1zZyFTOTz89ZLg24OnTwMSJtsniCJqdoLVaLaZMmYLA\nwEAoFAqcPHnSFrls7saNG1AoFIb77u7uOHfunIiJpK20tNRoUvXx8UFpaanRGKVSiQMHDgCo+499\n+fJllJSUGI1JSUlBVFSU9QNLnDn7877KykpkZGRg8uTJhm2CIGDs2LEYMmQItmzZYpOcOh1w5owP\nkpJKjdaaBUHAiRMnoFQqMWHCBBQUFFgtT1vX7FEcCxYswIQJE7Bv3z7odDpUVFTYIpfFxMVtQmLi\n1ygrE9Cx43+h09U2Olav1xvdFwShkZHWQURYunQtVq9ebPOvbS61OgcbNmTgu+++hYvLTajVOYiI\nGGly7JIlS7BgwQLDmqlKpUL79u0Nj1dXVyM1NdXkequjqNufmSgpuYhbtwqb3J/3paamYsSIEejS\npYth2zfffIMePXrg+vXrCAsLQ0BAAJ588kmL5cxRq5G5YQMulpSg8NYt/HXT1/hk+yhUVgIvvgjM\nnfvb2McffxwajQZubm5IS0vDpEmTcPHiRYtlcShNrX9otVrq06dPq9dRxBIb+wk5Oc0jgO7dcglo\nT7GxnzQYGxgYSDExMYb7HTp0oB9//NGWcWnv3jTq3Pl12rcv3aZf11yHD39Ncvmye/vyWwLCSS5f\nRocPf00JCQn0/vvvN/l8Pz8/KisrM9w/ePAghYeHWzu2ZD3s/pw0aRLt3r270deNi4ujdevWWSzn\n14cP0zK5nAigXLQjf/Qh13Y3aGH09xQfb96/uy3Wx+2NOXNnkyPOnj1Lw4YNo5deeolUKhXNnj2b\nKioqWvxFxOLu/rt6k/P9Cbojubv/rsHY+m8SbtmyxaZvEiYl7SCFIoL8/ZcRoCd//2WkUERQUtIO\nm2Uwh0Lxcr19WUNAXwKKKDBwhsk3tbRaLd29e5eIiD799FOaOXOm0eMvvvgibd++3VbxJWfcuLdN\n7s+wsKWNvkmo1WpJJpNRZWWlYVtFRQXdvn2biIjKy8tp+PDhlJGRYbGcb48bRwRQHhQ0BN+QK3pQ\nLnrS0rAwkzn/85//kF6vJyKif/7zn/TYY49ZLEtbYs7c2eQSh06nw5kzZ5CYmIihQ4fi9ddfx/vv\nv4+VK1cajYuLizP8OTQ0FKGhoZYu+g+lfXuPevceA1ACQI+Kii/x8ssvo7q6GgCwc+dOxMbGYv/+\n/XB2doaTkxO2bt1qs5xz506HTOaJhQtzAAioqtIjISEakyeH2yyDOby86r+R5wQgEUA4ioquY/ny\nxQgMDMTmzZsBAPPmzUNBQQFeeuklCIKA4OBgbNu2zfDsiooKZGVlWXW9VOru3q3/3++3/Zmba3p/\nAsDBgwcRHh4OV1dXwzOvXbuGyMhIAHX/Z6dPn45x48ZZLGe7Kh1WYwnW403E42344BfMAnA99/+w\nePnyBjn37duHv/zlL3BycoKbmxtSUlIslsWeZWdnIzs7u2VPamr2/uWXX8jPz89wPzc3lyIiIlr8\nU0Asnp4vPNCg626eni+KHa2B+8sbCsUb1LnzAkkucxg3vt9u4eHviB3NLtnD/szLI+rxyE80FplU\njN5GQd9x4OUpSzBn7mzyKI7u3bvD19fXsMCflZWFoKCgh/vxIYLo6FFo126u0bZ27eYgOrrpN2HE\nUFioQXLy08jL+xDJyeNRWKgRO1ID8+ePg0wWbbRNJotGTEyYSIns2/z54yCXv220TS5fJon9Wf/T\ngFEz7mJo3z/iMVwxPL5MLkdYTIx4AR2EcG8mb9T333+P2bNno7q6GnK5HMnJyfDw+G3pQBAESR/z\nWHcURw50Ohc4OVUhOnok4uJeEzuW3VKrc7Bx41FUVbWHi0stYmLCmj3qgDVOivuz/jk0tm6tO3Qu\nR63G0Y0b0b6qCrUuLgiLicHIiAhRc9o7c+bOZidoS3wRxpj06XTA2rXA+vVAfDwwZw4g0aM92wRz\n5k4+mx1jjM88J1EO81FvxlhDfOY5aeMGzZiD4tYsfdygGXMw3JrtBzdoxhwIt2b7wg2aMQfArdk+\ncYNmrI3j1my/uEEz1kZxa7Z/3KAZa4O4NbcN3KAZa0O4Nbct3KAZayO4Nbc93KAZs3PcmtsubtCM\n2TFuzW0bN2jG7BC3ZsfADZoxO8Ot2XFwg2bMTnBrdjw8QbM2Lz09HQEBAfD398eaNWsaPL5u3Tqo\nVCqoVCoMHDgQTk5O0Gq1AAA/Pz8MGjQIKpUKw4YNs3V0g/x8YPhw4NixutY8d654J9NvC/vTbtji\nwoeMiUWn05FcLqeioiKqrq4mpVJJBQUFjY5PTU2lMWPGGO77+fnRjRs3bBHVpJoaooQEIi8vos2b\nifR60aIQkf3vTykxZ+7kBs3atFOnTqFfv37w8/ODs7Mzpk6dikOHDjU6fteuXZg2bZrRNhLpkm5S\nas332fP+tEdmTdD8awmzV6WlpfD19TXc9/HxQWlpqcmxlZWVyMjIwOTJkw3bBEHA2LFjMWTIEGzZ\nssXqeQFprzXb4/60Z2YdxSEIArKzsyGTyaydxyqICEuXrsXq1YshiF1BmmAvOe2BWp2DDRsyUVJy\nEbduFUKtzmn2atmpqakYMWIEunTpYtj2zTffoEePHrh+/TrCwsIQEBCAJ5980mq5pXqERo5ajcwN\nG3CxpASFt24hR61u9qreUtif9s7sJQ57/rVk//4MbNr0Cw4cyBQ7SpPsJafUqdU5WLAgA5mZq1BQ\n8CZKS7thwYIMqNU50Gg08PHxMfm8lJSUBr+O9+jRAwDg7e2NyMhInDp1yiqZpdyac9RqZCxYgFWZ\nmXizoADdSkuRsWABctRqye7PNsOcxew+ffrQ4MGD6YknnqBPP/20xQvdYklK2kEKRQT5+y8jQE/+\n/stIoYigpKQdYkczYi857cW4cW8TQPduNQT0JaCIwsKWNvqmllarJZlMRpWVlYZtFRUVdPv2bSIi\nKi8vp+HDh1NGRobF8+blEQ0dSjR2LFFxscVfvtXeHjfu/s6kGoD6AlQE0NKwMEnuT3thztxp1hJH\nc7+WxMXFGf4cGhqK0NBQy/4UeUhz506HTOaJhQtzAAioqtIjISEakyeHix3NiL3ktBd379b/tnYC\nkAggHLm517F8+WIEBgZi8+bNAIB58+YBAA4ePIjw8HC4uroannnt2jVERkYCAHQ6HaZPn45x48ZZ\nLKdOB6xdC6xfD8THA3PmiP8moClOd+/+9mfc35vA9dxcLF6+XDL7U+qys7ORnZ3dsie1dNaPi4uj\ndevWteingJj27k2jzp1fJ4XiDerceQHt25cudiST7CWnPTBu0L/dwsPfETuagdRbc331G3T92zvh\n4WJHs2vmzJ3NrkFXVlairKwMAFBRUYHMzEwMHDiw5T8+RFJYqEFy8tPIy/sQycnjUVioETuSSfaS\n0x7Mnz8OcvnbRtvk8mWIiQkTKdFvpLzW3Jhx8+fjbbncaNsyuRxhMTEiJXIcwr2ZvFFFRUUNfi1Z\nunTpby8gCHb9BiJrm9TqHGzceBRVVe3h4lKLmJiwZo/isLb6R2hs3Sr9ibm+HLUaRzduRPuqKtS6\nuCAsJqbZozhY08yZO5udoC3xRRhzZPay1sxsy5y5k89mx5gVSfW4ZmYf+KPejFmBPa41M+nhBs2Y\nhXFrZpbCDZoxC+HWzCyNGzRjFsCtmVkDN2jGWoFbM7MmbtCMPSRuzczauEEz1kLcmpmtcINmrAW4\nNTNb4gbNmBm4NTMxcINmrBncmplYuEEz1ghuzUxs3KAZM4FbM5MCbtCM1cOtmUkJN2jG7uHWzKSG\nGzRzeNyamVQ51ASdnp6OgIAA+Pv7Y82aNQ0ev3nzJiIjI6FUKhESEoL8/HwAwIULF6BSqQw3Dw8P\nbNiwwdbxmRXk5wPDhwPHjtW15rlz+WT6TEJsceFDKdDpdCSXy6moqIiqq6tNXi5+0aJFtHLlSiIi\n+umnn2jMmDENXqe2tpa6d+9OV65csUluZh01NUQJCUReXkSbNxPp9WInYo7GnLnTYRr0qVOn0K9f\nP/j5+cHZ2RlTp07FoUOHjMacP38eo0ePBgAMGDAAxcXFuH79utGYrKwsyOVy+Pr62iw7syxuzcxe\nmDVB19bWQqVSYeLEidbOYzWlpaVGk6qPjw9KS0uNxiiVShw4cABA3YR++fJllJSUGI1JSUlBVFSU\n9QMzi+O1ZmZvzJqgP/74YygUCgh2WDPU6hyEh7+NV1/dhLS001Crcxodu2TJEmi1WqhUKiQmJkKl\nUqF9+/aGx6urq5Gamornn3/eFtGZBXFrdgxEhA+WLGkzF7Ju9jC7kpISHDlyBG+//TbWr19vi0wW\no1bnYMGCDFy6FA/gJIA4LFiQAQDQaDTw8fExGt+5c2d89tlnhvt9+vRB3759DffT0tLwxBNPwNvb\n2xbxmQXwFbUdS8b+/fhl0yZkDh2K8MmTxY7Tas026DfeeANr165Fu3b2t1z91lvb703OADAEQCEu\nXZqDxYu3Yc+ePXj22WeNxt+6dQvV1dUAgC1btmDUqFHo1KmT4fHdu3dj2rRpNkrPWotbs+P42+bN\neCYoCLnLlmF9WRlyli7FM0FB+NvmzWJHa5UmG/Thw4fx6KOPQqVSITs7u9FxcXFxhj+HhoYiNDTU\nQvFax8ur/ht5TgASAYSjqOg6li9fjMDAQGy+9w84b948FBQU4KWXXoIgCAgODsa2bdsMz66oqEBW\nVha2bNli078DazluzY5n+ty58JTJkLNwIQQA+qoqRCckSKpFZ2dnNzmPmiJQE4s1y5Ytw44dO+Dk\n5ISqqircvn0bkydPxueff/7bCwiCZNd7wsPfQWbmKhPb30V6+nsiJGLWVv/TgFu38puAjiR93z5k\nvPIKBF9f6DUajE9OltQE/SBz5s4m1y0SEhKg0WhQVFSElJQUPPXUU0aTs9TNnz8OMlm00TaZLBox\nMWEiJWLWwkdoME1hIZ5OTsaHeXkYn5wMTWGh2JFarUXn4rC3ozgiIkbi88+BjRvfRVVVe7i41CIm\n5gVERIwUOxqzID6HBgOAOUuXGv4s5ebcEk0ucZj1AhJe4mBtG681M3tmztzJZ7NjdolbM3ME9nfs\nHHNovNbMHAk3aGY3uDUzR8MNmkket2bmqLhBM0nj1swcGTdoJkncmhnjBs0kiFszY3W4QTPJ4NbM\nmDFu0EwSuDUz1hA3aCYqbs2MNY4bNBMNt2bGmsYNmtkct2bGzMMNmtkUt2bGzMcNmtkEt2bGWo4b\nNLM6bs2MPRxu0MxquDUz1joWmaBXrVqFjh07okOHDhg/fnyDx4uKitCzZ0+4urqiU6dOOHToEAAg\nLS0Nbm5uhpsgCJjcRq6E4Oj4itptV3p6OgICAuDv7481a9Y0ePzmzZuIjIyEUqlESEgI8vPzAQAX\nLlyASqUy3Dw8PLBhwwZbx7cv1EoAyMnJiXJzc6miooJcXFwoNTXVaMyQIUPoqaeeIiKiI0eOUNeu\nXRu8Tk1NDbVr145OnjzZ2khMRDU1RAkJRF5eRJs3E+n1YidilqTT6Ugul1NRURFVV1eTUqmkgoIC\nozGLFi2ilStXEhHRTz/9RGPGjGnwOrW1tdS9e3e6cuWKTXJLkTnTr0UatIeHB0aMGAE3NzeMGjUK\niYmJRo9fuXLF0IzHjx+PsrIynD9/3mjM2rVr4e7ujpCQEEtEYiLg1tz2nTp1Cv369YOfnx+cnZ0x\ndepUw2/E950/fx6jR48GAAwYMADFxcW4fv260ZisrCzI5XL4+vraLLs9anaCrqqqQkhICAYPHgyF\nQoGl9S7MeJ+Xl5fhz3369MHVq1eNHu/Xrx+2bdsGAEhOToZOp8O5c+eMxiQnJ2PChAkP9Zdg4uK1\nZsdRWlpqNKn6+PigtLTUaIxSqcSBAwcA1E3oly9fRklJidGYlJQUREVFWT+wnWt2gnZxccHx48dx\n7tw5/PDDDzh+/Dj+8Y9/GI0pLCxFXNymRl9j7969KCsrg5ubG1atWgV3d3c4OzsbHi8vL8e///1v\nxMbGtuKvwsTArdkx5KjVeDs8HJtefRWn09KQo1Y3OnbJkiXQarVQqVRITEyESqVC+/btDY9XV1cj\nNTUVzz//vMVzEhE+WLKkzVzI2qzD7Nzc3ADU7dja2lrIZDKjx/X6joiP/wHAJvz888/o1auX0eM9\ne/bExYsXDfednZ0xYsQIw/2EhATIZDIEBgY+7N+D2RhfUdtx5KjVyFiwAPGXLuEkgDgAGQsWAAA0\nGg18fHyMxnfu3BmfffaZ4X6fPn3Qt29fw/20tDQ88cQT8Pb2tnjWjP378cumTcgcOhThbeGAA3MW\ns2tra0mpVFKnTp1o8eLFDRa6AScCcqlr1+dMvkl4+fJlKisrIyKiP/zhDySXy40e9/X1pdmzZ5sT\nhUlAXh7R0KFEY8cSFReLnYZZ28sKBRFABFANQH0BKgJoRmCgyTcJtVot3b17l4iIPv30U5o5c6bR\n4y+++CJt377dohl3JCVRhEJBy/z9SQ/QMn9/ilAoaEdSkkW/jiWZM/226CgOrVZLISEhdPz4caMv\nAoQS0I4A0JAhQ4iIKCoqiqKiooiIaPPmzeTs7EwdOnSgXr160eXLlw3Pv3btGgmCQKWlpS2JwkTA\nR2g4puUjRxomaALoCED9Aerq4kIJCQlERJSUlERJ9ybDEydOUP/+/WnAgAE0efJk0mq1htcqLy8n\nT09Pun37tkUz6vV6OvL3v9MSX18igJb4+lLa3r2kl9A36fHjxyk2NtZwM2eCFohatljz3nvvwdXV\nFYsWLQIACIJQN0cD8PScil9/TbFkwWcSUf/TgFu38puAjuSd8HCsysxssP3d8HC8l54uQiLT0vft\nQ8Yrr0Dw9YVeo8H45GRJL3MIgtDsWnmzbxL++uuv0Gq1AIA7d+7g6NGjUKlUDcY5Oc1DdPTIh4zK\npIqP0GDj5s9H9APvO0XLZAiLiREpkWmawkI8nZyMD/PyMD45GZrCQrEjtVqzDfrHH3/EzJkzodfr\nodfr8Yc//AGLFy/+7QUEAZ6eLyI6eiTi4l6zemBmO9ya2X05ajWObtyI9lVVqHVxQVhMDEZGRIgd\ny66Z06BbvMTxMF+E2Rc+QoMx6zNn7uSz2TEjfOY5xqSDz2bHAPBaM2NSxA2acWtmTKK4QTswbs2M\nSRs3aAfFrZkx6eMG7WC4NTNmP7hBOxBuzYzZF27QDoBbM2P2iRt0G8etmTH7xQ26jeLWzJj94wbd\nBnFrZqxt4AbdhnBrZqxt4QbdRnBrZqzt4QZt57g1M9Z2cYO2Y9yaGWvbuEHbIW7NjDkGh5qgV61a\nhY4dO6JDhw4YP358g8efeeYZuLm5wc3NDS4uLhAEAZcvXwYAODk5wdXVFW5ubujUqZOtoxvk5wPD\nhwPHjtW15rlz+WT6jLVVDjNBV1dXY8WKFfjqq6+g1WqRnZ2Nw4cPG405fPgwKisrUVlZibfeegtd\nu3bFY/eqqSAIyMvLQ2VlJcrLy22en1szY47HYSbo7du3w8PDAyNGjICbmxtGjRqFxMTERsd//vnn\nePbZZ4226fV6a8c0iVszY46p2Qlao9Fg9OjRCAoKQnBwMDZs2GCLXBZ34cIFeHl5Ge736dMHV69e\nNTn2119IsyDxAAARfUlEQVR/xZUrV7BixQqj7UqlEu7u7pgxY4ZVs97HrZkxx9bsBO3s7Iw///nP\nyM/Px8mTJ/HJJ5/g/PnztshmEXFxm+Dl9SISEzNQWFiKuLhNzT7nvffeQ7du3QzLGwDwr3/9C5WV\nlTh9+jT27t3bZPu2BG7NjoOI8MGSJZK/+LI95LSHjC3R7ATdvXt3DB48GADQqVMnBAYGNto8pSYu\nbhPi43/AjRt7UF39OvT6joiP/wFxcZvw888/o1evXiaft3fvXjz//PNG2+7vg8DAQPzP//wPMjIy\nrJKZW7Pjydi/H79s2oTMAwfEjtIke8hpDxlbhFqgqKiIevfuTWVlZYZtLXwJm/L0fIEAune7Q4AT\nAbnUtetz5OLiQqmpqQ2ec/nyZRIEgW7cuGHYdv36dSotLSUiomvXrlHnzp0pISHB4nnz8oiGDiUa\nO5aouNjiL88kZkdSEkUoFLTM35/0AC3z96cIhYJ2JCWJHc2IPeS0h4wPMmfuNPuDKuXl5ZgyZQo+\n/vjjBoeZxcXFGf4cGhqK0NBQy/z0aCWdzrXePRcA7wJ4Cjdv6jBuXBieeeYZTJ8+HQCwc+dOAHXL\nG71794ZMJjM8s6CgAOHh4QDqfoUaNWoUli5dasGcwNq1wPr1QHw8MGcOL2c4gulz58JTJkPOwoUQ\nAOirqhCdkIDwyZPFjmbEHnLaQ8bs7GxkZ2e36DlmTdA1NTWYPHkyfv/732PSpEkNHq8/QUuJk9Od\nB7YsB7Acnp5TkZGRAuC3ifm+LVu2NHidkSNH4s6dB1/LMvjTgI5LEAQIgoAqrRZvKhTQazSGbVJi\nDzntIeOD5fXBgxBMaXYNmogwa9YsKBQKvP76660KaGvR0aPg5PSq0TYnp3mIjh4pUqLf8FozAwBN\nYSGeTk7Gh3l5GJ+cDE1hodiRTLKHnPaQsaWEe2shjfrHP/6BkSNHYtCgQYafRqtXr8bTTz9d9wKC\nIOl3TOPiNiExMQc6nQucnKoQHT0ScXGviZqpfmveupUnZsYckTlzZ7MTtCW+CKvDa82MsfvMmTv5\nbHY2wmvNjLGWcpiPeouF15oZYw+LG7QVcWtmjLUGN2gr4NbMGLMEbtAWxq2ZMWYp3KAthFszY8zS\nuEFbALdmxpg1cINuBW7NjDFr4gb9kLg1M8asjRt0C3FrZozZCjfoFuDWzBizJW7QZuDWzBgTAzfo\nZnBrZoyJhRt0I7g1M8bExg3aBG7NjDEpsEiDTk9PR0BAAPz9/bFmzZoGj69btw4qlQoqlQoDBw6E\nk5MTtFotAMDPzw+DBg2CSqXCsGHDLBHnoXFrZsz+vfLKK+jWrRsGDhzY6Jj58+fD398fSqUSZ8+e\ntWG6FrLElWnlcjkVFRVRdXU1KZVKKigoaHR8amoqjRkzxnDfz8/P6AraYuErajPWNuTk5NCZM2co\nODjY5ONqtZrGjx9PREQnT56kkJAQW8YzMGf6tUiD7tevH/z8/ODs7IypU6fi0KFDjY7dtWsXpk2b\n9uAPCUvEeCjcmhlrW5588kl07dq10ce//PJLzJw5EwAQEhICrVaLa9eu2SpeizQ7QZvz64Kvr6/h\nzz4+PigtLTU5rrKyEhkZGZhc71LogiBg7NixGDJkiMkraltTfj4wfDhw7FjdWvPcuXwJKsbautLS\n0gZzVklJiYiJGtfsBP3yyy8jPT3dIl8sNTUVI0aMQJcuXQzbvvnmG5w9exZpaWn45JNPkJuba5Gv\n1RRuzYw5tgd/axck2syanaCb+3UBAL744hjU6hwAgEajgY+Pj8lxKSkpDZY3evToAQDw9vZGZGQk\nTp06ZVbwh8Wt2XEQET5YskTyFzXmnJaRo1bj7fBwvNK7N9bOmIGKsjKT43r16gWNRmO4X1JSgl69\netkqZsuYs5hdVFTU6II7AAL6Uu/er9EXX2Q1+iahVqslmUxGlZWVhm0VFRV0+/ZtIiIqLy+n4cOH\nU0ZGhjmRWqymhighgcjLi2jzZiK93ipfhklI2t699HrnzpS+b5/YUZrEOVvv68OHaZlcTgQQAVQE\nkHeHDvT14cMNxtZ/k/Dbb7+V9JuEFpqgjxDQn1xdu1JCQgIRESUlJVFSUpJh3Pbt22natGlGz/35\n559JqVSSUqmkoKAgw3MtjY/QcCw7kpIoQqGgZf7+pAdomb8/RSgUtKPe96MUcE7LeVmhMEzOUwHq\nAZAzQG5OTrRt27YG89Gf/vQnksvlNGjQIPruu+9EyWzDCTqWgFh67LFRdPz48RaFtCZuzY5Jr9fT\nkb//nZb4+hIBtMTXl9L27iW9xL4BOKflLB850jBB178tHzlS7GgGx48fp9jYWMPNnAnaQp8kjAMA\nBAS8i9DQUMu8ZCvxpwEdlyAIEAQBVVot3lQooNdoDNukhHNaTq2Li8nteldXGydpXGhoqNH8uGLF\nimaf0+ybhNOmTcPw4cNx8eJF+Pr6Ijk52eQ4uXwZYmLCzE9rJXyEBgMATWEhnk5Oxod5eRifnAxN\nYaHYkUzinJYxbv58RMtkRtuiZTKExcSIlMgyBKLWvSUrCALCw99BTEwYIiJGWirXQ6nfmrdu5YmZ\nMUeSo1bj6MaNaF9VhVoXF4TFxGBkRITYsRolCEKzR8RYZIJu5Uu0mk4HrF0LrF8PxMcDc+bwoXOM\nMWkzZ+60+7PZ8VozY6ytstvzQfNaM2OsrbPLBs2tmTHmCOyqQXNrZow5Ertp0NyaGWOORvINmlsz\nY8xRSbpBc2tmjDkySTZobs2MMSbBBs2tmTHG6kimQXNrZowxY5Jo0NyaGWOsIVEbNLdmxhhrnGgN\nmlszY4w1zeYNmlszY4yZx6YNmlszY4yZzyYNmltz25Seno6AgAD4+/tjzZo1DR6/efMmIiMjoVQq\nERISgvz8fADAhQsXoFKpDDcPDw9s2LDB1vElpX///mjXrh1cGrl0EwAolUp06NABrq6u2L17tw3T\n2ZdXXnkF3bp1w8CBAxsdM3/+fPj7+0OpVOLs2bM2TNdCrb0QYnMvwVfUbpt0Oh3J5XIqKiqi6upq\nUiqVVFBQYDRm0aJFtHLlSiIi+umnn2jMmDENXqe2tpa6d+9OV65csUluqdq4cSPt2rWLOnbsaPLx\nuLg48vb2JiKirVu3kru7uy3j2ZWcnBw6c+ZMoxe6VqvVNH78eCIiOnnyJIWEhNgynoE506/VGjS3\n5rbt1KlT6NevH/z8/ODs7IypU6fi0KFDRmPOnz+P0aNHAwAGDBiA4uJiXL9+3WhMVlYW5HI5fH19\nbZZdiqKjo5vcB3v27MG0adMAALNmzUJNTQ3y8vJsFc+uPPnkk+jatWujj3/55ZeYOXMmACAkJARa\nrRbXrl2zVbwWaXaCbu7XWFPy84Hhw4Fjx+rWmufO5UtQtTWlpaVGE4qPjw9KS0uNxiiVShw4cABA\n3YR++fJllJSUGI1JSUlBVFSU9QPbuRs3bkChUBjuu7u749y5cyImsl+mvncf/L6UiiYn6NraWkRH\nRyM9PR0FBQXYvXs3zp8/3+h4Kbfm7OxssSOYxV5y3l9PbsqSJUug1WqhUqmQmJgIlUqF9u3bGx6v\nrq5Gamoqnn/+eavltJf9aU5OvV5vdF8QofXYw/40JyM9cC1AMfalOZo8iqP+r7EADL/GBgYGNhgr\n9SM0srOzERoaKnaMZkk9Z45ajcwNG3Dkhx9wq6YGOZMmYWREBDQaDXx8fIzGdu7cGZ999pnhfp8+\nfdC3b1/D/bS0NDzxxBPw9va2Wl6p789NcXH4OjERF8rLIevQAbU6nclxnp6eRuWooqICSqXSVjEN\npLw/739v/uPCBQT37o2KsjKT43r16gWNRmO4X1JSgl69etkqZos02aDN+TUWkG5rZpaVo1YjY8EC\nrMrMxDP/+Q9w4wb2vPYavjp4EHv27MGzzz5rNP7WrVuorq4GAGzZsgWjRo1Cp06dDI/v3r3bsK7q\niDbFxeGH+HjsuXEDk+7excqyMlBtLTbFxTUY++KLLyIlJQUAsHXrVjg7OyM4ONjGiaWr/vdm6OXL\nWJSbi/JffkGOWt1g7LPPPovPP/8cAHDy5El06dIF3bp1s3VkszTZoM2t/ffXmnlibtsyN2xA/KVL\nAOp+sicCeP3KFeyOisLid99FYGAgNm/eDACYN28eCgoK8NJLL0EQBAQHB2Pbtm2G16qoqEBWVha2\nbNkiwt9EGr5OTMSee435IwDvAdADiF6xAv+6fNnww23nzp2IjY3F/v374ezsDCcnJ2zdulW03FJU\n/3tzP4BPAWirq/HMlCn46JNPUFNTA6Du+3LChAk4cuQI+vXrB3d3dyQnJ4sXvDlNHeLx7bffUnh4\nuOF+QkICvf/++0Zj5HI5AeAb3/jGN7614CaXy5s9zE6gB1fL69HpdBgwYAC++uor9OzZE8OGDcPu\n3btNrkEzxhizrCaXOJycnJCYmIjw8HDU1tZi1qxZPDkzxpiNNNmgGWOMiadVnyR8mA+x2Jo5n8uX\nAo1Gg9GjRyMoKAjBwcGSPTdFVVUVQkJCMHjwYCgUCixdulTsSI2qra2FSqXCxIkTxY7SKD8/Pwwa\nNAgqlQrDhg0TO06jtFotpkyZgsDAQCgUCpw8eVLsSA3Y0zleVq9ejaCgIAwcOBBRUVG4e/eu6YHN\nrlI3wpxzMUhBc5/Ll4pffvmFzp49S0REZWVl1L9/f0nuTyKiiooKIiKqqamhkJAQys3NFTmRaR9+\n+CFFRUXRxIkTxY7SKD8/P7px44bYMZo1Y8YM2rZtGxHV/btrtVqREzVNyud4KSoqoj59+lBVVRUR\nEb3wwgu0fft2k2MfukGbcy4GKWjuc/lS0b17dwwePBgA0KlTJwQGBuLq1asipzLNzc0NQN0nAWtr\nayGTyURO1FBJSQmOHDmC2bNnN/jUmNRIPd+tW7eQm5uLV155BUDde1MeHh4ip2qalM/x8sgjj8DZ\n2RmVlZXQ6XSorKxs9IMyDz1Bm/shFtZyxcXFOHv2LEJCQsSOYpJer8fgwYPRrVs3jB492ugcEVLx\nxhtvYO3atWjXTjLXRTZJEASMHTsWQ4YMkewx4UVFRfD29sbLL7+Mxx9/HHPmzEFlZaXYsZok5XO8\nyGQyLFy4EL1790bPnj3RpUsXjB071uTYh/7ulepn1+1deXk5pkyZgo8//tjoU3dS0q5dO5w7dw4l\nJSXIycmR3PkZDh8+jEcffRQqlUry7fSbb77B2bNnkZaWhk8++QS5ubliR2pAp9PhzJkzeO2113Dm\nzBm4u7vj/fffFztWo2xxjpfWuHTpEj766CMUFxfj6tWrKC8vx86dO02OfegJ+sHPs5s6FwNrmZqa\nGkyePBm///3vMWnSJLHjNMvDwwMRERE4ffq02FGMnDhxAl9++SX69OmDadOm4dixY5gxY4bYsUzq\n0aMHAMDb2xuRkZE4deqUyIka8vHxgY+PD4YOHQoAmDJlCs6cOSNyqsbZ4hwvrXH69GkMHz4cnp6e\ncHJywnPPPYcTJ06YHPvQE/SQIUNQWFiI4uJiVFdXmzwXAzMfEWHWrFlQKBR4/fXXxY7TqF9//RVa\nrRYAcOfOHRw9ehQqlUrkVMYSEhKg0WhQVFSElJQUPPXUU4ZzL0hJZWUlyu6d0KeiogKZmZmSPNqo\ne/fu8PX1xcWLFwHUre8GBQWJnKpxUj/HS0BAAE6ePIk7d+6AiJCVldX4MmFr3o08cuQI9e/fn+Ry\nOSUkJLTmpaxm6tSp1KNHD+rQoQP5+PjQZ599JnYkk3Jzc0kQBFIqlTR48GAaPHgwpaWliR2rgR9+\n+IFUKhUplUoaOHAgffDBB2JHalJ2drZkj+L4+eefSalUklKppKCgIMn+HyIiOnfuHA0ZMoQGDRpE\nkZGRkj2Ko7y8nDw9Pen27dtiR2nSmjVrSKFQUHBwMM2YMYOqq6tNjuMPqjDGmERJ+y1uxhhzYDxB\nM8aYRPEEzRhjEsUTNGOMSRRP0IwxJlE8QTPGmETxBM0YYxLFEzRjjEnU/wPMqTv7wHk0lQAAAABJ\nRU5ErkJggg==\n",
       "text": [
        "<matplotlib.figure.Figure at 0x7fef3969e890>"
       ]
      }
     ],
     "prompt_number": 3
    }
   ],
   "metadata": {}
  }
 ]
}