SVM_with_scikit_learn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#####################################################################################\n",
    "#################################      Problem 1      ###############################\n",
    "#####################################################################################\n",
    "\n",
    "import random \n",
    "from sklearn import svm \n",
    "import scipy.io as spy\n",
    "from __future__ import division\n",
    "\n",
    "data_path=\"/Users/Luba/Desktop/CS189_spring/hw1/data/digit-dataset\"\n",
    "\n",
    "digit_train_data=spy.loadmat(data_path+\"/train.mat\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x=digit_train_data.get('train_images')\n",
    "vector=[]\n",
    "for k in range(len((x[0][0]))):\n",
    "    element=[]\n",
    "    for i in range(28):\n",
    "        for j in range(28):\n",
    "            element.append(x[i][j][k])\n",
    "    vector.append(element)\n",
    "    \n",
    "y=digit_train_data.get('train_labels')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: pylab import has clobbered these variables: ['random', 'spy']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from skimage.feature import hog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD8CAYAAABTq8lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADd5JREFUeJzt3V2MHfV5x/HfU9sV2A4YE7pYyDUG1QGBRayqVi23ZNSC\n2SiIECERKJVWlhPlInEjrgJI1If2opERwepNpAoTOTTCiQpxCQjXJMqo4BC/VHYMDXZiwQps8Br8\nAl6DsAtPL3bWOSy7/1mfOS/jfb4facWceeblYaSfZ86ZOedv7i4AMfxRrxsA0D0EHgiEwAOBEHgg\nEAIPBELggUBaDryZ9ZvZXjP7vZl9p51NAegMa+U+vJlNk7RP0g2SDkraIelOd3+laRlu8AM95O42\ndl6rZ/ilkva7+6C7n5a0UdKXx9nhmb81a9Z84nXd/uiP/qZSfxNpNfCXSXqj6fWBYh6AGms18Fyu\nA+eg6S2ud1DS/KbX8zVylv+ERqNxZnrOnDkt7qo7sizrdQtJ9FfNVO8vz3PleV66XKsf2k3XyId2\nfyvpTUnbNc6Hdq1sG0B1ZiYf50O7ls7w7v5/ZvYtSf8laZqk9c1hB1BPLZ3hJ7VhzvBAz0x0hudJ\nOyAQAg8EQuCBQAg8EAiBBwIh8EAgBB4IhMADgRB4IBACDwRC4IFACDwQCIEHAiHwQCAEHgiEwAOB\nEHggEAIPBELggUAIPBAIgQcCIfBAIAQeCITAA4EQeCAQAg8EQuCBQAg8EAiBBwIh8EAgBB4IZHqV\nlc1sUNJ7kj6SdNrdl7ajKUwN7733XrI+MDCQrK9atSpZv/nmm8+6p+gqBV6SS8rc/Wg7mgHQWe24\npLc2bANAF1QNvEv6uZntNLOvt6MhAJ1T9ZJ+ubu/ZWaXSHrOzPa6+/OjxUajcWbBLMuUZVnF3QEY\nT57nyvO8dDlz97bs0MzWSBp294eK196ubePcxId2vWNmcvdPvd1u+ZLezGaa2WeK6VmSVkh6qfUW\nAXRalUv6Pkk/NbPR7fzI3be0pSsAHdG2S/pPbZhL+ko2bdqUrPf39yfr5513Xjvbacn+/fuT9UWL\nFiXry5YtS9a3bt161j1F0fZLegDnHgIPBELggUAIPBAIgQcCIfBAIAQeCKTqs/Ro0bPPPpus33bb\nbcn6TTfdlKw/+eSTyXod7tOj+zjDA4EQeCAQAg8EQuCBQAg8EAiBBwIh8EAg3IfvkVOnTiXrZb8l\nsHnz5mT9xIkTyXo37sMvXLgwWS971uDNN99sZzsQZ3ggFAIPBELggUAIPBAIgQcCIfBAIAQeCIT7\n8D3yzDPPVFr/6quvTtbPP//8SttvhwMHDiTrTz/9dLI+b968ZP348ePJ+pw5c5L1iDjDA4EQeCAQ\nAg8EQuCBQAg8EAiBBwIh8EAgpffhzexRSV+SdNjdFxfz5kr6saQFkgYl3e7u6Zui+IQXXnih0vpb\ntmxJ1mfPnl1p++1w+vTpZP3DDz9M1gcHB5P1xx57LFlfvXp1sh7RZM7wP5DUP2bePZKec/dFkn5R\nvAZQc6WBd/fnJR0bM/sWSRuK6Q2Sbm1zXwA6oNX38H3uPlRMD0nqa1M/ADqo8rP07u5mNu4PsDUa\njTPTWZYpy7KquwMwjjzPled56XKtBn7IzC5190NmNk/S4fEWag48gM4Ze0J94IEHxl2u1Uv6pyQN\nFNMDkja1uB0AXVQaeDN7XNKvJH3OzN4ws5WSvivpRjP7naS/KV4DqDkr+/3zljds5p3a9rlg7dq1\nyfp9992XrH/88cfJ+tGjR5P1OnwX/O23307WlyxZkqyX/S79ddddl6zv2rUrWZ/KzEzubmPn86Qd\nEAiBBwIh8EAgBB4IhMADgRB4IBACDwTC79K36MiRI8n6RI82jiq7zz4VXHLJJcn69ddfn6xv3Lix\nne1AnOGBUAg8EAiBBwIh8EAgBB4IhMADgRB4IBDuw7eobGzzDz74oKP77+8f+8vhnzRz5syO7r8d\ndu/e3esWwuEMDwRC4IFACDwQCIEHAiHwQCAEHgiEwAOBcB++RVdeeWWyPm3atGT9o48+qrT/7du3\nV1p/Krj22mt73cI5hzM8EAiBBwIh8EAgBB4IhMADgRB4IBACDwRSOj68mT0q6UuSDrv74mJeQ9LX\nJI0OAH6vu28es17o8eHvuOOOZH3Pnj3J+t69e5P1LMuS9VmzZiXrBw8eTNbPhe+qb926NVlftmxZ\nlzqpnyrjw/9A0thfW3BJ33P3JcXf5nHWA1AzpYF39+clHRun9Kl/PQDUW5X38KvN7Ddmtt7M5rSt\nIwAd0+qz9N+X9E/F9D9LekjSqrELNRqNM9NZlpW+7wTQmjzPled56XItBd7dD49Om9kjkn423nLN\ngQfQOWNPqBMNZtrSJb2ZzWt6+RVJL7WyHQDdVXqGN7PHJX1B0mfN7A1JayRlZvZ5jXxa/5qkb3S0\nSwBtUXofvuUNB78PX+bdd99N1l9//fVk/aqrrkrWZ8yYkayfPHkyWX/11VeT9cl4+eWXk/W77rqr\n0va5Dz+xKvfhAUwRBB4IhMADgRB4IBACDwRC4IFACDwQCL9L3yMXXnhhsr548eKO7r/s+/Lt2P/7\n779feRtoL87wQCAEHgiEwAOBEHggEAIPBELggUAIPBAI9+HRMQ8++GCl9a+44opk/Zprrqm0/Yg4\nwwOBEHggEAIPBELggUAIPBAIgQcCIfBAINyHR8dcdNFFldbv6+tL1i+44IJK24+IMzwQCIEHAiHw\nQCAEHgiEwAOBEHggEAIPBJIcH97M5kv6oaQ/keSS/s3d/9XM5kr6saQFkgYl3e7ux8esy/jwwe3f\nvz9ZX7RoUbJ+8cUXJ+s7d+5M1hcsWJCsT2Wtjg9/WtLd7n6NpL+U9E0zu1rSPZKec/dFkn5RvAZQ\nc8nAu/shd99dTA9LekXSZZJukbShWGyDpFs72SSA9pj0e3gzu1zSEknbJPW5+1BRGpKUfgYSQC1M\n6ll6M5st6QlJ33b3E2Z/eGvg7m5m475ZbzQaZ6azLFOWZVV6BTCBPM+V53npcskP7STJzGZIelrS\ns+6+rpi3V1Lm7ofMbJ6kX7r7VWPW40O74PjQrnda+tDORk7l6yX9djTshackDRTTA5I2tatRAJ1T\ndkm/XNLfS9pjZruKefdK+q6kn5jZKhW35TrWIYC2SQbe3V/QxFcBN7S/HUwl06enzyczZsxI1o8c\nOZKsr1u3Lll/+OGHk/Uyp06dStZ37NiRrC9fvrzS/juBJ+2AQAg8EAiBBwIh8EAgBB4IhMADgRB4\nIJDSR2tb3jCP1qLEypUrk/UNGzYk62WP3r744ovJetnv3t94443J+r59+5L1Y8eOJeud1Or34QFM\nIQQeCITAA4EQeCAQAg8EQuCBQAg8EAj34dEzw8PDyXrZffBt27ZV2v8kft4tWZ87d26y/s4775x1\nT+3CfXgABB6IhMADgRB4IBACDwRC4IFACDwQCPfhUVsnT55M1st+l/7+++9P1letWpWsL126NFlf\nsWJFst7Loa64Dw+AwAOREHggEAIPBELggUAIPBBIMvBmNt/Mfmlm/2tmL5vZPxTzG2Z2wMx2FX/9\n3WkXQBXJ+/BmdqmkS919t5nNlvQ/km6VdLukE+7+vcS63IcHemSi+/DTUyu5+yFJh4rpYTN7RdJl\no9tse5cAOmrS7+HN7HJJSyT9upi12sx+Y2brzWxOB3oD0GaTCnxxOf8fkr7t7sOSvi9poaTPS3pL\n0kMd6xBA2yQv6SXJzGZIekLSv7v7Jkly98NN9Uck/Wy8dRuNxpnpLMuUZVm1bgGMK89z5XleulzZ\nh3YmaYOkI+5+d9P8ee7+VjF9t6S/cPe/G7MuH9oBPTLRh3Zlgf8rSf8taY+k0QXvk3SnRi7nXdJr\nkr7h7kNj1iXwQI+0FPiKOyTwQI/w9VgABB6IhMADgRB4IBACDwRC4IFACDwQCIEHAiHwQCAEHgiE\nwAOBEHggkK4FfjLf1e0l+quG/qrpVn8EvkB/1dBfNVMu8AB6j8ADgXT0BzA6smEAk9LVX7wBUD9c\n0gOBEHggkK4E3sz6zWyvmf3ezL7TjX2eDTMbNLM9xcCY22vQz6NmNmRmLzXNm2tmz5nZ78xsSy9H\n+5mgv1oMMJoYALUWx6/XA7R2/D28mU2TtE/SDZIOStoh6U53f6WjOz4LZvaapD9396O97kWSzOyv\nJQ1L+qG7Ly7mrZX0jruvLf7RvMjd76lRf2tUMsBol3qbaADUlarB8asyQGs7dOMMv1TSfncfdPfT\nkjZK+nIX9nu2ajM4prs/L+nYmNm3aGRQEBX/vbWrTTWZoD+pBsfQ3Q+5++5ieljS6ACotTh+if6k\nLhy/bgT+MklvNL0+oD/8D9aFS/q5me00s6/3upkJ9DUN9jEkqa+XzUygVgOMNg2Auk01PH69GKC1\nG4E/F+77LXf3JZK+KOmbxSVrbRUjfNTtuNZqgNHicvkJjQyAeqK5Vofj16sBWrsR+IOS5je9nq+R\ns3xtjI6T5+5vS/qpRt6G1M1Q8f5PZjZP0uGS5bvK3Q97QdIj6uExbBoA9bHRAVBVo+M30QCt3Th+\n3Qj8Tkl/ZmaXm9kfS/qqpKe6sN9JMbOZZvaZYnqWpBWSXkqv1RNPSRoopgckbUos23VFiEZ9RT06\nhsUAqOsl/dbd1zWVanH8JuqvW8evK0/amdkXJa2TNE3Senf/l47vdJLMbKFGzurSyPDZP+p1f2b2\nuKQvSPqsRt5v/qOk/5T0E0l/KmlQ0u3ufrwm/a2RlKlkgNEu9TbeAKj3StquGhy/KgO0tmX/PFoL\nxMGTdkAgBB4IhMADgRB4IBACDwRC4IFACDwQCIEHAvl/8mbqwHSnZgsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x169434f10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_digit(X):\n",
    "    X=np.array(X)\n",
    "    img = X.reshape(28,28)\n",
    "    plt.imshow(img, cmap='Greys',  interpolation='nearest')\n",
    "    #plt.title('true label: %d' % y[idx)\n",
    "    plt.show()\n",
    "plot_digit(vector[27558])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "features = np.array(vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#HOG featurization\n",
    "list_hog_fd = []\n",
    "for feature in features:\n",
    "    fd = hog(feature.reshape((28, 28)), orientations=9, pixels_per_cell=(4, 4), cells_per_block=(1, 1), \n",
    "             visualise=False)\n",
    "    list_hog_fd.append(fd)\n",
    "hog_features = np.array(list_hog_fd, 'float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import random \n",
    "index=range(60000)\n",
    "index=list(index)\n",
    "random.shuffle(index)\n",
    "\n",
    "s1=100\n",
    "s2=200\n",
    "s3=500\n",
    "s4=1000\n",
    "s5=2000\n",
    "s6=5000\n",
    "s7=10000\n",
    "valid=10000\n",
    "\n",
    "r1=random.sample(index,s1)\n",
    "rest_r1=list(set(index)-set(r1))\n",
    "r1_valid=random.sample(rest_r1, valid)\n",
    "\n",
    "r2=random.sample(index,s2)\n",
    "rest_r2=list(set(index)-set(r2))\n",
    "r2_valid=random.sample(rest_r2, valid)\n",
    "\n",
    "r3=random.sample(index,s3)\n",
    "rest_r3=list(set(index)-set(r3))\n",
    "r3_valid=random.sample(rest_r3, valid)\n",
    "\n",
    "r4=random.sample(index,s4)\n",
    "rest_r4=list(set(index)-set(r4))\n",
    "r4_valid=random.sample(rest_r4, valid)\n",
    "\n",
    "r5=random.sample(index,s5)\n",
    "rest_r5=list(set(index)-set(r5))\n",
    "r5_valid=random.sample(rest_r5, valid)\n",
    "\n",
    "r6=random.sample(index,s6)\n",
    "rest_r6=list(set(index)-set(r6))\n",
    "r6_valid=random.sample(rest_r6, valid)\n",
    "\n",
    "r7=random.sample(index,s7)\n",
    "rest_r7=list(set(index)-set(r7))\n",
    "r7_valid=random.sample(rest_r7, valid)\n",
    "\n",
    "\n",
    "d1=[hog_features[i] for i in r1]\n",
    "d1_valid=[hog_features[i] for i in r1_valid]\n",
    "\n",
    "d2=[hog_features[i] for i in r2]\n",
    "d2_valid=[hog_features[i] for i in r2_valid]\n",
    "\n",
    "d3=[hog_features[i] for i in r3]\n",
    "d3_valid=[hog_features[i] for i in r3_valid]\n",
    "\n",
    "d4=[hog_features[i] for i in r4]\n",
    "d4_valid=[hog_features[i] for i in r4_valid]\n",
    "\n",
    "d5=[hog_features[i] for i in r5]\n",
    "d5_valid=[hog_features[i] for i in r5_valid]\n",
    "\n",
    "d6=[hog_features[i] for i in r6]\n",
    "d6_valid=[hog_features[i] for i in r6_valid]\n",
    "\n",
    "d7=[hog_features[i] for i in r7]\n",
    "d7_valid=[hog_features[i] for i in r7_valid]\n",
    "\n",
    "all_data=[hog_features[i] for i in index]\n",
    "\n",
    "\n",
    "label1=[y[i][0] for i in r1]\n",
    "label1_valid=[y[i][0] for i in r1_valid]\n",
    "\n",
    "label2=[y[i][0] for i in r2]\n",
    "label2_valid=[y[i][0] for i in r2_valid]\n",
    "\n",
    "label3=[y[i][0] for i in r3]\n",
    "label3_valid=[y[i][0] for i in r3_valid]\n",
    "\n",
    "label4=[y[i][0] for i in r4]\n",
    "label4_valid=[y[i][0] for i in r4_valid]\n",
    "\n",
    "label5=[y[i][0] for i in r5]\n",
    "label5_valid=[y[i][0] for i in r5_valid]\n",
    "\n",
    "label6=[y[i][0] for i in r6]\n",
    "label6_valid=[y[i][0] for i in r6_valid]\n",
    "\n",
    "label7=[y[i][0] for i in r7]\n",
    "label7_valid=[y[i][0] for i in r7_valid]\n",
    "\n",
    "all_label=[y[i][0] for i in index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74.28\n"
     ]
    }
   ],
   "source": [
    "clf_hog=svm.LinearSVC()\n",
    "clf_hog.fit(d1,label1)\n",
    "pred1=clf_hog.predict(d1_valid)\n",
    "result_100=len([i for i,j in zip(label1_valid,pred1) if i==j])/len(pred1)*100\n",
    "print(result_100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "83.01\n"
     ]
    }
   ],
   "source": [
    "clf_hog=svm.LinearSVC()\n",
    "clf_hog.fit(d2,label2)\n",
    "pred2=clf_hog.predict(d2_valid)\n",
    "result_200=len([i for i,j in zip(label2_valid,pred2) if i==j])/len(pred2)*100\n",
    "print(result_200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87.78\n"
     ]
    }
   ],
   "source": [
    "clf_hog=svm.LinearSVC()\n",
    "clf_hog.fit(d3,label3)\n",
    "pred3=clf_hog.predict(d3_valid)\n",
    "result_500=len([i for i,j in zip(label3_valid,pred3) if i==j])/len(pred3)*100\n",
    "print(result_500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90.29\n"
     ]
    }
   ],
   "source": [
    "clf_hog=svm.LinearSVC()\n",
    "clf_hog.fit(d4,label4)\n",
    "pred4=clf_hog.predict(d4_valid)\n",
    "result_1000=len([i for i,j in zip(label4_valid,pred4) if i==j])/len(pred4)*100\n",
    "print(result_1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "91.21\n"
     ]
    }
   ],
   "source": [
    "clf_hog=svm.LinearSVC()\n",
    "clf_hog.fit(d5,label5)\n",
    "pred5=clf_hog.predict(d5_valid)\n",
    "result_2000=len([i for i,j in zip(label5_valid,pred5) if i==j])/len(pred5)*100\n",
    "print(result_2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93.33\n"
     ]
    }
   ],
   "source": [
    "clf_hog=svm.LinearSVC()\n",
    "clf_hog.fit(d6,label6)\n",
    "pred6=clf_hog.predict(d6_valid)\n",
    "result_5000=len([i for i,j in zip(label6_valid,pred6) if i==j])/len(pred6)*100\n",
    "print(result_5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94.59\n"
     ]
    }
   ],
   "source": [
    "clf_hog=svm.LinearSVC()\n",
    "clf_hog.fit(d7,label7)\n",
    "pred7=clf_hog.predict(d7_valid)\n",
    "result_10000=len([i for i,j in zip(label7_valid,pred7) if i==j])/len(pred7)*100\n",
    "print(result_10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7gAAAF/CAYAAABjS2V+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmU5HddL/z3J5N9J4SskoUo26NcAiQgPsCAinBBEBdW\nufiwGxbF43kuqMeb6/XBHeUSIAgCYZFNZBWRsAxwAdmXgIBGCCHbZN8zk8nM5/mjakin00v1TFdX\nd/XrdU6d+tWvfsunqud0n/d8t+ruAAAAwFq3x6QLAAAAgOUg4AIAADAVBFwAAACmgoALAADAVBBw\nAQAAmAoCLgAAAFNhrAG3qn6rqs6pqm9W1W8N951eVRdU1VeHj0eMswYAAADWhz3HdeGq+skkz0xy\nSpJtST5cVR9M0kle1t0vG9e9AQAAWH/GFnCT3D3J57t7S5JU1SeT/PLwvRrjfQEAAFiHxtlF+ZtJ\nHlRVh1XV/kn+a5I7D997QVV9var+rqoOHWMNAAAArBPV3eO7eNXTk5yW5IYk30qyNclLk1w+POR/\nJTm6u58xtiIAAABYF8YacG9zo6qXJjm/u8+cse+EJB/o7p+a4/iVKQwAAIAV193LPnR1nGNwU1VH\ndPelVXVckscluX9VHd3dFw8PeVySc+Y7f7Hwffrpp+f0009frnJXzb2Wy1qseS3yPQNL5fcGAOtd\n1XimZRprwE3yD1V1xwxmUT6tu6+tqjOq6t4ZzKb8/STPGXMNAAAArANjDbjd/eA59v23cd4TAACA\n9WmcsyiP3caNG6fyXstlLda8FvmegaXyewMAxmPFJplaqqrq1VobAADAtNmxI7nppuTGGwePnduz\nn0fdt9B7t9xSY5lkSsAFAABYpbqTLVvGEzJn77v55mS//ZL995//edR9i723zz4CLgAAwMR1J9u2\nLU/IXCyAbtmS7L338ofMufbts08ypsmNb6dqPAF33LMoAwAArIhbbhmEw3G2cu583mOPXQuZhxyy\ntOP3229wL0ajBRcAABibHTsGrZDjCJmz39u+ffzda3du76mpcLeMqwVXwAUAgHWmO9m6dXm70s53\n/Natyb77jr977X77DbryrlQXW3aPLsoAADDltm1bmVlsb7pp0AK5KyHzDndY2vH77it0snK04AIA\nwAK2b7/tuM5xju/sXlqr5a7MXrvzecOGSX+zrGe6KAMAwNDOpVNWYhbbbdvGEzLn2rfXXpP+ZmFl\nCLgAAKxqcy2dMq7xnVu2DJY0GUfIvP16nbrYwnIzBhcAgF2yc+mUlZjFdsOGXQuShxyytFC6776W\nTgFuTwsuAMAE7Ngx97jOcbR87lw6Zdzday2dAoxKF2UAgDGbvXTKOMd37lw6Zdzda3eO69TFFlhN\ndFEGANat2eM6xzW+c8uWQRjclZB52GFLO97SKQDLTwsuALBLdi6dMq5JhGa+lyQHHLD7LZmLHb/v\nvpZOAVgJuigDAIuauXTKci+VMnvftm0r0712v/0snQIwbQRcAJgCN96YXHJJcsMN45nFdufSKeMI\nmbPf23tvXWwB2DXG4ALAKrZjR3LppcmFF97+cdFFt25v2ZIcdVRy4IGjhcxDD11aALV0CgDrmRZc\nAFjE9dffNqTO9di8eRBGjz12/scxxwwmItLqCcB6p4syACyz7dsHwXSultaZj23bbhtS5wqvRx89\n6LILACxOwAWAJbjuusW7C1966aBFdXYr6+zweuihWl0BYDkJuACQ5JZbbtvqOl+I3b594a7CO1td\nzc4LACtPwAVgqnUn1167eHfhyy5LDj988bGuhxyi1RUAVisBF4A1a9u2wdI4C3UXvvDCQSBdbKzr\nUUcle1oDAADWNAEXgFWnO7n66oVnGL7oouSKK5I73Wn+rsI7HwcfPOlPBACsBAEXgBV1883JxRcv\n3F34oouSDRsWH+t65JFaXQGAWwm4ACyL7uSqqxbvLnzVVYNgOl9X4Z0B9qCDJv2JAIC1RsAFYFFb\nt97a6jpfi+tFFw3Wa11saZwjjhi0zgIALDcBF2Ad6x6MY11shuFrrhlMwrTYWNcDDpj0JwIA1jMB\nF2BKbdly+8A61+v99lt8rOsRRyR77DHpTwQAsDABF2CN2bHjtq2u8415ve665OijF14a55hjkv33\nn/QnAgBYHgIuwCpy002Ldxe++OLkwAMXH+t6+OFaXQGA9UXABVgBO3Ykl122+AzDN9wwd1idue+Y\nYwbdigEAuC0BF2A33Xjj4t2FL7kkOfjgxZfGOfzwpJb9VzIAwPog4ALMY/v25NJL5+8qvPOxZcvC\nXYWPPXYwFnbffSf9iQAAppuAC6xL11+/cFfhCy9MNm9ODj108bGuhx2m1RUAYDUQcIGpsn37IJgu\n1F34wguTm29efGmco49O9tln0p8IAIBRCbjAmnHttQvPLnzhhYMuxXe848JjXY89dtAyq9UVAGC6\nCLjAxN1yy2ASpsWWx9mxY7SxrnvtNelPBADAJAi4wNh039rqulB34csuG8wevNDSOMcemxxyiFZX\nAADmJ+ACu2TbtuTiixfuLnzRRYNjFxvretRRWl0BANh9Ai5wG93J1Vcv3l34iiuSI45YfKzrQQdp\ndQUAYGUIuLCO3HzzoNV1oe7CF16Y7Lnn4kvjHHnk4DgAAFgtBFyYAt3JlVcu3l34yisHwXSxsa4H\nHTTpTwQAAEsn4MIqt3XrbYPrXCH2oouSvfdeuKvwMccMwu2GDZP+RAAAMB4CLkzY1q3Jeecl//mf\nyfe+N3j+z/9Mzj9/EF6vuWYwCdNCS+Mcc0xy4IGT/iQAADBZAi6sgGuuuTW4zn5ccskgpJ500m0f\nxx8/2H/EEckee0z6EwAAwOon4MIy6B5M3jRfiL3pptsH2JNOSu5yl0GQtUQOAADsPgEXRnTzzckP\nfjB3gP3e9wZdhOcKsSedNBj7aqkcAAAYLwEXZrjuuvlbYS+6aDDWdb6W2IMPnnT1AACwvgm4rCvd\nyebN84fY668fhNW5Quzxxw9mKgYAAFYnAZeps23bYAbi+boS77ff/CH26KN1JQYAgLVKwGVNuv76\n2y6pM/NxwQWDoDpfV+JDDpl09QAAwDgIuKxK3cmll87flfi665ITT5w7xJ5wgq7EAACwHgm4TMwt\ntyzclXjvveeflfjoo60NCwAA3JaAy1jdcMP8XYl/+MPB8jnzhdhDD5109QAAwFoi4LJbupPLL5+/\nK/HVV9/alXj2xE4nnpjss8+kPwEAADAt1mTArarfSvLMJJXktd398qo6LMk7khyf5Lwkj+/uq+c4\nV8DdDe9/f/KZz9w2xO655/ytsMccoysxAACwMtZcwK2qn0zytiSnJNmW5MNJnpvkOUku7+4/r6r/\nnuQO3f3iOc4XcHfBDTckp52WfOELyVOfetsQe4c7TLo6AACA8QXcPZf7gjPcPcnnu3tLklTVJ5P8\nSpLHJHnI8JizkmxKcruAy9Kdc07y+McnD3hA8qUvJQccMOmKAAAAVs44O6V+M8mDquqwqto/yX9N\n8mNJjuzuzcNjNic5cow1rAvdyetelzzsYclLXpK84Q3CLQAAsP6MrQW3u79TVX+W5CNJbkjytSTb\nZx3TVaUf8m647rrkuc9NvvGN5FOfSu5xj0lXBAAAMBnj7KKc7n59ktcnSVX9f0kuSLK5qo7q7kuq\n6ugkl853/umnn/6j7Y0bN2bjxo3jLHfN+drXBl2SN24cjLndb79JVwQAAHB7mzZtyqZNm8Z+n3HP\nonxEd19aVccl+ZckD0jy+0mu6O4/q6oXJznUJFNL052ceWbyh3+YvPzlyZOfPOmKAAAARrfmZlFO\nkqr6VJI7ZjCL8ou6+xPDZYLemeS4WCZoya65Jnn2s5Pvfjd55zuTu9510hUBAAAszZoMuLtDwL29\nL385ecITkoc/PHnZy5J99510RQAAAEs3roA7zlmUWSbdyStekTzykclLX5q86lXCLQAAwGxjnWSK\n3XfVVckznpGcf37yuc8lJ5006YoAAABWJy24q9jnP5/c5z7Jj/1Y8pnPCLcAAAAL0YK7CnUnf/3X\nyZ/+afKa1ySPe9ykKwIAAFj9BNxV5oorkt/4jeTSSwdr255wwqQrAgAAWBt0UV5FPvvZQZfku90t\n+fSnhVsAAICl0IK7CuzYkfzFXwyW/nnd65Jf/MVJVwQAALD2CLgTdtllydOellxzTfLFLybHHTfp\nigAAANYmXZQn6FOfGnRJvte9kk2bhFsAAIDdoQV3ArZvT/7kT5Izzkje8IbkkY+cdEUAAABrn4C7\nwjZvTn7915OtW5Mvfzk59thJVwQAADAddFFeQR//+KBL8gMeMNgWbgEAAJaPFtwVsH178kd/lLz2\ntclZZyU///OTrggAAGD6CLhjdvHFyZOfnFQNuiQfffSkKwIAAJhOuiiP0Uc+MuiSvHFjcvbZwi0A\nAMA4acEdg1tuSf7H/0je+Mbk7/8+eehDJ10RAADA9BNwl9kFFwy6JO+7b/LVryZHHDHpigAAANYH\nXZSX0Yc+lNzvfskjHpF8+MPCLQAAwErSgrsMtm1Lfv/3k7e9LXnXu5IHPWjSFQEAAKw/Au5uOv/8\n5IlPTA49dNAl+fDDJ10RAADA+qSL8m54//uTU05JfumXkg9+ULgFAACYJC24u+Dmm5MXvzh597uT\n97wneeADJ10RAAAAAu4Sff/7yROekBx11KBL8mGHTboiAAAAEl2Ul+Qf/zG5//2TJz0ped/7hFsA\nAIDVRAvuCLZuTX73dwfjbD/4weTUUyddEQAAALMJuIvoTn7hFwattV/96mC2ZAAAAFYfAXcRn/50\ncsklycc/nuyhQzcAAMCqJbIt4owzkuc/X7gFAABY7aq7J13DnKqqJ13bhRcmP/VTyXnnJQcfPNFS\nAAAApkZVpbtrua+rXXIBZ56ZPPnJwi0AAMBaoAV3Hlu3Jscfn2zalNz97hMrAwAAYOpowV1h73rX\noHuycAsAALA2CLjzOOOM5AUvmHQVAAAAjErAncMXv5hs3pw86lGTrgQAAIBRCbhzOOOM5LTTkg0b\nJl0JAAAAozLJ1CyXXZbc9a7Juecmd7zjit8eAABg6plkaoW89rXJL/+ycAsAALDWaMGd4ZZbkhNP\nTN7//uTkk1f01gAAAOuGFtwV8L73Dda+FW4BAADWHgF3BksDAQAArF26KA9985vJwx+enHdesvfe\nK3ZbAACAdUcX5TE744zkOc8RbgEAANYqLbhJrr56MLnUt7+dHHXUitwSAABg3dKCO0ZveEPyyEcK\ntwAAAGvZum/B3bEjuetdkze/Ofnpnx777QAAANY9Lbhj8uEPJ4cckjzgAZOuBAAAgN2x7gPuzqWB\natn/7wAAAICVtK67KJ97bvLAByY/+EGy335jvRUAAABDuiiPwStfmTz96cItAADANFi3LbjXX58c\nf3zyla8MngEAAFgZWnCX2Vvekjz4wcItAADAtFiXAbf71smlAAAAmA7rMuBu2jQIuQ996KQrAQAA\nYLmsy4B7xhnJ859vaSAAAIBpsu4mmTr//OTe9x48H3jgsl8eAACARYxrkqk9l1DAEUlemGT/JK/u\n7v9Y7mJWwplnJk99qnALAAAwbUZuwa2qNyd5bZJO8rLuPmWshY2hBXfLluS445L/83+Su951WS8N\nAADAiFZ8maCq+peqevCMXXsn+f7wsc9yF7IS3vGO5D73EW4BAACm0UKTTD0hyWOq6u1VdVKSP0jy\nJ0n+d5LTRrl4Vb2kqr5VVedU1d9X1T5VdXpVXVBVXx0+HrH7H2Nx3ckrXmFpIAAAgGm1aBflYbj9\n4yQXJfnj7r5qpAtXnZDk40nu0d1bq+odST6U5IQk13X3yxY5f1m7KP/rvyZPeUry7/+ebNiwbJcF\nAABgiVZ8kqmq+vEkz01yc5LfTXJSkrdX1T8leWV3b1/k2tcm2ZZk/6ransHkVBdmEHBXfIGeM85I\nTjtNuAUAAJhWC3VRfluS9yTZlORN3f2pJI9Ick2Ssxe7cHdfmeSvkpyfQevv1d390eHbL6iqr1fV\n31XVobtR/0g2b07+6Z+Spz993HcCAABgUhYKuDMnldo/SXrgrCSPXuzCw67Nv51Bi+0xSQ6sqqck\neXWSE5PcO8nFGYTgsfrbv01+7deSO9xh3HcCAABgUhZaB/e0JK/IoJvxc2e+0d03jnDt+yX5bHdf\nkSRV9Y9JHtjdb915QFW9LskH5rvA6aef/qPtjRs3ZuPGjSPc9ra2bRusffvP/7zkUwEAAFgGmzZt\nyqZNm8Z+n5HXwV3yhav+S5K3JjklyZYkb0zyhSTv7u5Lhse8KMkp3f3kOc5flkmm3vnO5JWvTD75\nyd2+FAAAAMtgxSeZ2l3d/fWqelOSLyXZkeQrSV6b5HVVde8knUH35+eMq4ZkMLnUC184zjsAAACw\nGoytBXd3LUcL7te/njzqUcn3v5/stdcyFQYAAMBuGVcL7kKTTM0uYP/lvvm4nXFG8tznCrcAAADr\nwaItuFX1wCSvS3JQd9952L342d192lgL280W3CuvTE46Kfnud5MjjljGwgAAANgtk2zB/ZsM1r+9\nPEm6+2tJHrLchSy3178+efSjhVsAAID1YqRJprr7/KrbhOtbxlPO8ti+PXnVq5K3v33SlQAAALBS\nRgm451fVzyRJVe2d5IVJvj3WqnbThz6UHH54cuqpk64EAACAlTJKF+XfTPK8JMcmuTDJycPXq9YZ\nZyTPf/6kqwAAAGAljTLJ1M9092cW27fshe3iJFPf/W7y4AcnP/hBsu++YygMAACA3TLJSabOGHHf\nqvDKVybPfKZwCwAAsN7MOwa3qn46yQOT3KmqfifJznR9UJawfu5Kuu665C1vSb7+9UlXAgAAwEpb\naJKpvTMIsxuGzztdm+RXx1nUrnrTm5KHPSy5850nXQkAAAArbZQxuCd093krU85t7rukMbjdyT3v\nmZx5ZvKQVb9KLwAAwPo1rjG4oywTdGNV/WWSeybZb7ivu/thy13M7vjYx5I99xxMMAUAAMD6M8pY\n2rcm+U6SuyQ5Pcl5Sb40vpJ2zc6lgWrZ/w8AAACAtWCULspf6e77VNU3uvtew31f6u77jbWwJXRR\nPu+85L73Tc4/PznggHFWBQAAwO6aZBflm4fPl1TVo5NclOQOy13I7nj1q5OnPU24BQAAWM9GacH9\nxSSfTnLnJK9IcnCS07v7/WMtbMQW3JtuSo47Lvnc55If//FxVgQAAMBymFgLbnd/YLh5dZKNw2JO\nXe5CdtXb3paceqpwCwAAsN7NG3Crao8kj0tyUpJvdveHqup+SV6a5Igk916ZEufXnbziFclLXzrp\nSgAAAJi0ebsoV9XrkpyY5AtJHpLk4iR3T/L7Sd63pEVqd6WwEboof+YzyW/8RvLd7yZ7jDIfNAAA\nABM3iS7KD0hyr+7eUVX7JrkkyUndfcVyF7Grzjgjed7zhFsAAAAWbsH9anefPN/rsRe2SAvuxRcn\n97xn8v3vJ4ceulJVAQAAsLsm0YJ796o6Z8brk2a87p1r4k7Ka16TPPGJwi0AAAADCwXce6xYFbvg\nrLOS97530lUAAACwWiy6Du6kLNRFuTvZe+/khhsGzwAAAKwd4+qivCanZ7r22mS//YRbAAAAbrUm\nA+7llyeHHz7pKgAAAFhNFg24VfWYqlpVQVjABQAAYLZRgusTkpxbVX9eVXcfd0GjuPzy5I53nHQV\nAAAArCaLBtzufkqSk5N8L8kbq+pzVfXsqjpo7NXN44ortOACAABwWyN1Pe7ua5L8Q5J3JDkmyeOS\nfLWqXjjG2ualizIAAACzjTIG97FV9Z4km5LsleSU7n5kknsl+Z3xljc3ARcAAIDZ9hzhmF9O8tfd\n/amZO7v7xqp65njKWtjllyfHHTeJOwMAALBajRJw/2eSi3e+qKr9khzZ3ed190fHVtkCjMEFAABg\ntlHG4L4zyfYZr3dkMB53YnRRBgAAYLZRAu6e3X3zzhfdvTWDsbgTI+ACAAAw2ygB9/KqeuzOF8Pt\ny8dX0uKsgwsAAMBs1d0LH1D140nemsHyQElyQZKndve5Yy2squeqrTvZe+/khhsGzwAAAKwtVZXu\nrmW/7mIBd0YBByXp7r5+uYuY535zBtyrr06OPz655pqVqAIAAIDlNq6AO8osyqmqRye5Z5J9qwY1\ndPcfLXcxozD+FgAAgLksOga3ql6T5PFJXpikhtvHj7mueRl/CwAAwFxGmWTqgd3935Jc2d3/M8kD\nktxtvGXNzxq4AAAAzGWUgHvT8PnGqjo2yS1JjhpfSQvTRRkAAIC5jDIG9wNVdYckf5Hky8N9rx1f\nSQsTcAEAAJjLggG3qvZI8vHuvirJu6vqn5Ls291Xr0h1czAGFwAAgLks2EW5u3ckeeWM11smGW4T\nY3ABAACY2yhjcD9aVb9aO9cHmjBdlAEAAJjLKAH3uUnemeTmqrpu+Lh2zHXNS8AFAABgLotOMtXd\nB65EIaMyBhcAAIC5LBpwq+rBc+3v7k8tfzmLMwYXAACAuVR3L3xA1QeT7Dxo3ySnJvlydz9srIVV\n9ezaduxI9tknufHGZK+9xnl3AAAAxqWq0t3LPs/TKF2UHz2rkDsneflyFzKKa65JDjhAuAUAAOD2\nRplkarYLktxjuQsZhfG3AAAAzGeUMbivmPFyjyT3TvLlsVW0AONvAQAAmM+iATeDMLtzMOwtSf6+\nuz8zvpLmZ4kgAAAA5jNKwP2HJDd19/YkqaoNVbV/d9843tJuT8AFAABgPqOMwf1okv1mvN5/uG/F\nGYMLAADAfEYJuPt29/U7X3T3dRmE3BVnDC4AAADzGSXg3lBV9935oqrul+Sm8ZU0P12UAQAAmM8o\nY3B/O8k7q+ri4eujkzxhlItX1UuS/HqSHUnOSfL/JDkgyTuSHJ/kvCSP7+6rR7megAsAAMB8Fm3B\n7e4vZrDu7W8OH/fo7i8tdl5VnZDkWUnu090/lWRDkicmeXGSs7v7rkk+Nnw9EmNwAQAAmM+iAbeq\nnp/kgO4+p7vPSXJAVZ02wrWvTbItyf5VtWcG43YvSvKYJGcNjzkryS+NWqwxuAAAAMxnlDG4z+ru\nq3a+GG4/e7GTuvvKJH+V5PwMgu3V3X12kiO7e/PwsM1Jjhy1WF2UAQAAmM8oAXePqvrRcVW1Icle\ni51UVSdlMH73hCTHJDmwqn595jHd3Ul6lEJ37EiuvDI57LBRjgYAAGC9GWWSqX9J8vaqek2SSvKc\nJB8e4bz7Jflsd1+RJFX1j0l+OsklVXVUd19SVUcnuXS+C5x++uk/2r7PfTbmwAM3Zq9FozUAAACr\nyaZNm7Jp06ax36cGjagLHDBosX12kp8d7jo7yeu6e/si5/2XJG9NckqSLUnemOQLGcyefEV3/1lV\nvTjJod19u4mmqqpn1vYf/5E88pHJueeO+MkAAABYlaoq3V3Lfd1FW3CHQfbVw0eq6kFJ/neS5y1y\n3ter6k1JvpTBMkFfSfK3SQ7KYNmhZ2S4TNAohRp/CwAAwEJG6aKcqrpPkicl+bUMQum7Rzmvu/88\nyZ/P2n1lkp8bvcQBARcAAICFzBtwq+puGYTaJyS5LMm7kuzR3RtXprTbsgYuAAAAC1moBffbST6Y\n5Be6+/wkqarfWZGq5mANXAAAABay0DJBv5zkpiSfqqozq+pnM5hFeSJ0UQYAAGAh8wbc7n5vdz8h\nyU8m+XSSFyW5U1W9uqoevlIF7qSLMgAAAAtZqAU3SdLd13f3W7v70UnunOSrSW63rM+46aIMAADA\nQhYNuDN195Xd/bfd/bBxFTQfXZQBAABYyJIC7iQJuAAAACxkTQVcY3ABAACYT3X3pGuYU1X1ztp2\n7Ej23ju56aZkr70mXBgAAAC7parS3cu+Ss+aaMG9+urkoIOEWwAAAOa3JgKu8bcAAAAsZs0EXONv\nAQAAWMiaCLjWwAUAAGAxayLg6qIMAADAYgRcAAAApsKaCbjG4AIAALCQNRFwjcEFAABgMWsi4Oqi\nDAAAwGIEXAAAAKbCmgm4xuACAACwkDURcI3BBQAAYDHV3ZOuYU5V1d2d7duTffZJtmxJ9txz0lUB\nAACwu6oq3V3Lfd1V34J79dXJwQcLtwAAACxs1Qdc428BAAAYxaoPuMbfAgAAMIpVH3AtEQQAAMAo\nBFwAAACmwpoIuMbgAgAAsJhVH3CNwQUAAGAUqz7g6qIMAADAKARcAAAApsKaCLjG4AIAALCYVR9w\njcEFAABgFKs+4OqiDAAAwCiquyddw5yqqm+5pbPPPsmWLcmee066IgAAAJZDVaW7a7mvu6pbcK+6\nKjn4YOEWAACAxa3qgGv8LQAAAKNa1QHX+FsAAABGJeACAAAwFVZ9wLUGLgAAAKNY1QHXGFwAAABG\ntaoDri7KAAAAjErABQAAYCqs+oBrDC4AAACjWNUB1xhcAAAARrWqA64uygAAAIxKwAUAAGAqVHdP\nuoY5VVVv2NDZujXZsGHS1QAAALBcqirdXct93VXdgnvIIcItAAAAo1nVAVf3ZAAAAEYl4AIAADAV\nVnXAtQYuAAAAo1rVAVcLLgAAAKMScAEAAJgKAi4AAABTYVUHXGNwAQAAGNWqDrhacAEAABiVgAsA\nAMBU2HOcF6+quyV5+4xdd0nyh0nukOSZSS4b7n9Jd3949vkCLgAAAKOq7l6ZG1XtkeTCJKcmeXqS\n67r7ZQsc35dd1kIuAADAlKmqdHct93VXsovyzyU5t7t/mKSGjwXd4Q5jrwkAAIApsZIB94lJ3jbc\n7iQvqKqvV9XfVdWhc52wYcOK1QYAAMAatyJdlKtq7wy6J9+zuy+rqiNy6/jb/5Xk6O5+xqxzeqW6\nTwMAALByxtVFeayTTM3wyCRf7u7LkqS7L935RlW9LskH5jrp9NNP/9H2xo0bs3HjxrEWCQAAwPLb\ntGlTNm3aNPb7rFQL7tuT/HN3nzV8fXR3XzzcflGSU7r7ybPO0YILAAAwhcbVgjv2gFtVByT5QZIT\nu/u64b43Jbl3BmNxv5/kOd29edZ5Ai4AAMAUWrMBd1cJuAAAANNpGpYJAgAAgLERcAEAAJgKAi4A\nAABTQcBI9UdLAAAL6ElEQVQFAABgKgi4AAAATAUBFwAAgKkg4AIAADAVBFwAAACmgoALAADAVBBw\nAQAAmAoCLgAAAFNBwAUAAGAqCLgAAABMBQEXAACAqSDgAgAAMBUEXAAAAKaCgAsAAMBUEHABAACY\nCgIuAAAAU0HABQAAYCoIuAAAAEwFARcAAICpIOACAAAwFQRcAAAApoKACwAAwFQQcAEAAJgKAi4A\nAABTQcAFAABgKgi4AAAATAUBFwAAgKkg4AIAADAVBFwAAACmgoALAADAVBBwAQAAmAoCLgAAAFNB\nwAUAAGAqCLgAAABMBQEXAACAqSDgAgAAMBUEXAAAAKaCgAsAAMBUEHABAACYCgIuAAAAU0HABQAA\nYCoIuAAAAEwFARcAAICpIOACAAAwFQRcAAAApoKACwAAwFQQcAEAAJgKAi4AAABTQcAFAABgKgi4\nAAAATAUBFwAAgKkg4AIAADAVBFwAAACmgoALAADAVBBwAQAAmApjC7hVdbeq+uqMxzVV9cKqOqyq\nzq6qf6+qj1TVoeOqAQAAgPVjbAG3u7/b3Sd398lJ7pvkxiTvSfLiJGd3912TfGz4epds2rRpOUpd\ndfdaLmux5rXI9wwsld8bADAeK9VF+eeSnNvdP0zymCRnDfefleSXdvWiAu7C1mLNa5HvGVgqvzcA\nYDxWKuA+McnbhttHdvfm4fbmJEeuUA0AAABMsbEH3KraO8kvJnnX7Pe6u5P0uGsAAABg+tUgY47x\nBlWPTfKb3f2I4evvJNnY3ZdU1dFJPtHdd5/jPMEXAABgSnV3Lfc191zuC87hSbm1e3KSvD/J05L8\n2fD5vXOdNI4PCwAAwPQaawtuVR2Q5AdJTuzu64b7DkvyziTHJTkvyeO7++qxFQEAAMC6MPYuygAA\nALASVmoW5SWrqjtX1Seq6ltV9c2qeuFw/2FVdXZV/XtVfaSqDp1xzkuq6j+q6jtV9fAl3u+8qvpG\nVX21qr4wznvtqqp6fVVtrqpzZuxbco1Vdd+qOmf43stXova1ZDn/7fmuYf1Yrr8jfm8AsFaMO59U\n1T5V9Y7h/n+tquMXq2nVBtwk25K8qLv/ryQPSPK8qrpHkhcnObu775rkY8PXqap7JnlCknsmeUSS\nV1XVUj5fZzD51cndfepw37jutaveMLzfTEupcee45lcneUZ3/0SSn6iq2ddc75bj357vGtaf3f07\n4vcGAGvNuPPJM5JcMdz/1xnM47SgVRtwu/uS7v7acPv6JN9OcmySxyQ5a3jYWUl+abj92CRv6+5t\n3X1eknOTnJqlmT2x1TjvtWTd/ekkV+1Gjfcfzlx9UHd/YXjcm2acQ5bt357vGtan3fk74vcGAGvK\nCuSTmdd6d5KfXaymVRtwZ6qqE5KcnOTzSY7s7s3DtzYnOXK4fUySC2acdkEGoWRUneSjVfWlqnrW\ncN+47rWcllrj7P0XZnK1r3q7+W/Pdw3ry3L8HfF7A4C1bjn/9h2b5IdJ0t23JLmmBpMWz2sllgna\nLVV1YAZp/be6+7pbW7GT7u5aeL3cpcyg9TPdfXFV3SnJ2TVYr3dc9xqLEWpkCXbz3x6w/uzu3xEA\nmCqT+Nu3qltwq2qvDALGm7t753q5m6vqqOH7Rye5dLj/wiR3nnH6jw33jaS7Lx4+X5bkPRl0OR7L\nvZbZUmq8YLj/x2btn1Ttq9Yy/NvzXcM6swx/R/zeAGAaLMffvgtmnHPc8Fp7Jjmku69c6OarNuAO\nBxz/XZJ/6+6/mfHW+5M8bbj9tCTvnbH/iVW1d1WdmOQnknwhI6iq/avqoOH2AUkenuSccdxrDJZU\nY3dfkuTaqrr/8Dt+6oxzyPL92/Ndw/qxXH9H/N4AYAosx9++981xrV/NYNKqBa3adXCr6v9O8qkk\n38it3X9fkkGQfGcGSf68JI/v7quH5/xekqcnuSWDbqX/MuK9Tszgf9uTQbftt3b3nwz7dy/rvXZH\nVb0tyUOSHJ5Bf/Y/zOCHv6Qaq+q+Sd6YZL8kH+ruF4679rVkOf/t+a5hfVjOvyN+bwCwVow7n1TV\nPknenMGcOFckeeJwgqr5a1qtARcAAACWYtV2UQYAAIClEHABAACYCgIuAAAAU0HABQAAYCoIuAAA\nAEwFARcAAICpIOACsOKqakdV/eWM179bVf9jma79xqr6leW41iL3+bWq+req+tis/cdX1ZN28Zqf\nGeGY11bVPXbl+qtRVZ1QVedMug4ApoOAC8Ak3JzkcVV1x+Hr5VyUfZevVVV7LuHwZyR5Znf/7Kz9\nJyZ58q5cv7t/ZrGbdvezuvvbI1cJAOuIgAvAJGxL8rdJXjT7jdktsFV1/fB5Y1V9sqreW1X/WVV/\nWlVPraovVNU3quouMy7zc1X1xar6blU9anj+hqr6i+HxX6+qZ8+47qer6n1JvjVHPU8aXv+cqvrT\n4b4/TPIzSV5fVX8+65Q/TfKgqvpqVf12VT2tqt4/bOk9u6oOqKqPVtWXh9d9zDyfdVNVvauqvl1V\nb5lxzKaqus/O46vqj6vqa1X1uao6Yrj/pKr61+H1/7iqrpvrh1BVv15Vnx/WemZV7VFVpwy/n32G\ntX6zqu45X93DFtjvVNUbht/3W6vq4VX1mar696o6ZXjc6VX15qr67HD/M+eoZ86fEQCMSsAFYFJe\nleQpVXXwrP2zW2Bnvr5XkuckuUeSpyY5qbtPTfK6JC8YHlNJju/uU5I8KsmZVbVPBi2uVw+PPzXJ\ns6rqhOE5Jyd5YXffbeaNq+qYDALrQ5PcO8kpVfXY7v6jJF9K8uTu/n9n1fvfk3y6u0/u7r8Z1nNy\nkl/p7ocm2ZLkcd193yQPS/JX83zWeyf5rST3THKXqnrgHMfsn+Rz3X3vJJ9K8qzh/pcn+evuvleS\nH2YOw27Oj0/ywO4+OcmOJE/p7i8meX+SP07yZ0ne3N3/tkjdJyX5yyR3T3K3JE8Ytkb/bpLfm3Hc\nTw6/y59O8odVddSsshb6GQHAopbSFQsAlk13X1dVb0rywiQ3jXjaF7t7c5JU1blJ/mW4/5sZBKdk\nEADfObzHuVX1vQyC18OT/FRV/erwuIOT/HiSW5J8obt/MMf9Tknyie6+YnjPtyZ5cJL3Dd+vOc6Z\nva+TfKS7rx6+3iPJn1TVgzIIlcdU1RHdfems877Q3RcN7/u1JCck+eysY27u7n8abn85yc8Ptx+Q\nZGfL8NsyCJ+z/WyS+yb5UlUlyX5JLhm+tzPA35Rb/+NgzrqH732/u781rPVbST463P/NYd07v4f3\ndffWJFur6hNJ7p/k6zNqmu9ndN4c9QPA7Qi4AEzS3yT5SpI3zNh3S4Y9jKpqjyR7z3hv64ztHTNe\n78jCf9N2tno+v7vPnvlGVW1McsMC580MrJXbtqCOOt73xhnbT0lyeJL7dPf2qvp+kn3nOGfmZ92e\nuT/fthnbi30Hczmru39vjv2HJzkgyYYMgu+Ni9Q9++dy84g17Zhj3+1+RgAwKl2UAZiY7r4qg9bW\nZ+TWsHheBi2LyaAVcq8lXraS/FoNnJTkLkm+k0Fr72k7J3qqqrtW1f6LXOuLSR5SVXesqg1Jnpjk\nk4ucc22Sg2bVM9PBSS4dhsSHJjl+pE+1NP+aZGcr6BPnOeZjSX61qu6UJFV1WFUdN3zvNUn+IMnf\nZ9BNeTnqriSPHY7tvWOSjRl8vzPtys8IAH5ECy4AkzCz5fOvkjx/xuvXJnnfsFvuh5NcP895s6/X\nM7bPT/KFDELZc7r75qp6XQbdZb9Sgz65lyZ53Kxzb3vR7our6sVJPpFBQPtgd39gkc/2jSTbh/W/\nMclVs67/1iQfqKpvZNANeOaMyEttHZ59/M7Xv53kLVX1exmExmtud2L3t6vqD5J8ZNhSvi3J86rq\nIUm2dvfbh/s/O2zlHrXu+T5HZ/DdfCKDluA/6u5LhmNsdx4z388IAEZS3cu5MgMAMGlVtV933zTc\nfmIGkz5NNCjWYJ3j67v7rxY9GAB2kRZcAJg+962qMzJodb4qydMnXM9O/lcdgLHSggsAAMBUMMkU\nAAAAU0HABQAAYCoIuAAAAEwFARcAAICpIOACAAAwFQRcAAAApsL/D9q8s2W/k4OsAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x169790ad0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "x_axis=[100,200,500,1000,2000,5000,10000]\n",
    "x_ticks=[200, 500,1000,2000,5000,10000]\n",
    "y_axis=[result_100,result_200,result_500,result_1000,result_2000,result_5000,result_10000]\n",
    "plt.figure(figsize=(16,6))\n",
    "plt.xticks(x_ticks)\n",
    "plt.plot(x_axis, y_axis)\n",
    "plt.xlabel(\"Number of training example\")\n",
    "plt.ylabel(\"Accuracy Rate %\")\n",
    "plt.savefig(\"/Users/Luba/Desktop/CS189_spring/hw1/Accuracy_rate.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQUAAAD0CAYAAACSLzOZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHmtJREFUeJzt3XvcnPOd//HXOydJxCmUUiFBCa1TFMUPsbUWP9J22R9B\nD2x1ty2JaruEbCVd3cXj4bRtdZeojUPROnTZUsG6qWqRJnGKOAdpJIhDECF38vn98f1OMjO5Z+a6\nZ66Z65q5P8/H43pk5prv9b2+d5L53t/j55KZ4ZxzBf2yLoBzLl+8UnDOlfBKwTlXwisF51wJrxSc\ncyW8UnDOlfBKIQOShki6XdI7km5sIJ/jJd2VZtmyIml/SfOyLocD+TqFyiQdB5wO7AC8B8wBfmxm\nf2gw368ApwD7mNmqhguac5JWAduZ2YtZl8XV5i2FCiSdDlwMnAtsCowAfgaMSyH7rYFn+0KFUEQV\nP5AGtLIgWZNkvTlaXkAz86PsADYgtAyOqpJmHeAS4C/xuBgYFD8bCywgtDIWAwuBr8fPpgIfAR/H\ne5wETAGuKcp7JLAK6Bfffx14AVgKvAgcV3T+90XX7Qs8CrwDPEJoiRQ+6wJ+BDwY87kL2LjCz1Yo\n/w+A12P5vwQcDjwLLAHOLEq/F/BH4O2Y9ifAwPjZA/FneT/+vH9XlP8/Aa8B0+O5V+M128Z77B7f\nbwG8ARyQ9f+NlP5/2bkJj/AVbW35vKXQs32AwcCtVdKcTfgy7BqPvYDJRZ9vBqxP+A/998DPJG1g\nZucA/wrcYGbrmdkvCP/4PZK0LnApcKiZrR/LNqeHdMOB3xIqquHARcBvJW1UlGw8oSLZFBgEfL/K\nz7cZoeLbHPghMA04Htgd2B/4oaStY9puYCKwcSzfF4BvA5jZATHNLvHn/XVR/hsBWwH/UHxjM3sB\nOAO4VtIQ4CrgKjN7oEp528rAhEcWvFLo2cbAm1a9eX8c8CMze9PM3iS0AL5S9PmK+PlKM7uT8Jty\nh/iZKG1OV2xaR6uAnSUNMbPFZja3hzT/F3jGzK4zs1VmdgMwjzXdHSN8sZ43s+XAr4DdqtxzBWH8\nZCVwI6GiucTMPoj3n1u43sxmmdkj8b4vA5cDByb4mc4xsxWxPCXMbBrwPKHFsxmhEu4YAxIeWfBK\noWdLgE0kVfv72QJ4uej9K/Hc6jzKKpVlwLDeFsTMPgCOAf4RWCjpfyTt0EPSLWIZir1cVqZFRa8/\nrFGeJRbbujEthK5Q8fXrAkjaPpbrNUnvAj8mVKzVvGFmH9dIMw34DPATM1tRI21bGZLwyIJXCj37\nI6Hf/+UqaRYS+v4FW8Vz9XgfGFr0/pPFH5rZDDM7JJ6fB1zRQx5/IQxgFts6nm+2nxNaDtuZ2QaE\n3+q1/m9VHUCTNIzQFZoGTC3rBrU97z60GTN7l9CP/pmkL0oaKmmgpMMknR+TXQ9MlrSJpE1i+mvq\nvOUc4ABJIyRtAEwqfCBp01iGdQlN+g+AlT3kcSewvaTxkgZIOgYYDfxPUZpa3ZR6DSMMIi6TNBr4\nVtnniwmDh71xKfCImX2TMFbyHw2XMke8+9CGzOwiwuzBZMII/CuEwbPC4OO5wEzg8XjMjOdWZ1Et\n++LPzeweQr/9ccLswe1Fn/cDvkv4jb+EMMj3rfJ8zGwJcATwPeBNwiDiEWb2VoUyGbXLWO19se8T\nxliWEsYTbihLPwWYLultSUdXubcBSPoicAhrfs7TgTGSxlcpQ1vJc0sh14uXJB1KaEL2B6aZ2fk1\nLmlmWUYAVxNG7g243Mz+PavyxDL1J1RGC8zsyIzLsiFrxgAMOMnM/pRheSYBJxAGNJ8ATjSzj7Iq\nTzFJdkPCtMcCZtasFl6PcttSiP/hfwocCuwEjJe0Y4ZFWgF818w+A3we+E7G5YEwDTiXGv3zFrkU\nuMPMdgR2AZ7OqiCSRgInA2PMbGfCL5VjsypPT/LcUshtpUCY93/ezObHkecbgC9mVRgzW2Rmc+Lr\n9wn/6beoflXzSNqSsJhoGs0bK0halg2A/eOaC8ysO47LZGUpoRIfGldLDqU1A66JeaVQn08Brxa9\nXxDPZS7+JtodeDjDYlxMWHGYh6XSo4A3JF0laZakKyQNrXlVk8RxlAsJ40ALgXfiuE1u+JRkffLQ\nJF5LnCq7CZgYWwxZlOEI4HUzm03GrYRoADAGuMzMxhBmSM7MqjCStgVOI0wZbwEMk3R8VuXpic8+\n1OcvhE1IBSMIrYXMSBoI3Axca2a/ybAo+wLjJL1EmBr9K0lXZ1ieBYTBzkfj+5sIlURWPgc8ZGZL\nzKwbuIXwd5Yb3n2oz0zg05JGShpEWNV3W1aFkSTgSmCumV2SVTkAzOwsMxthZqMIA2j/a2ZfzbA8\ni4BXJW0fTx0MPJVVeQgLvD4f41YolqenpeGZyXNLIbdbVs2sW9IphN18/YErzSyzEW1gP8IU1+OS\nZsdzk8zsdxmWqSAPXa1TgetiBf4CcGJWBTGzx2LLaSZhzGUWYf1EbmTVCkgi1+sUnOtEkuyxhGl3\nxdcpONcnNDKmIGmipCckPSlpYjw3XNLdkp6VNCMuJiuknyTpOUnzJB1Sq2xeKTiXgXqnJCV9FvgG\nsCehIXFEnG05E7jbzLYH7o3vkbQTYTxuJ8JCwMtq7P71SsG5LDTQUhgNPGxmy2Osi/uBowhxM6bH\nNNMJkbIgLPi7PsatmE+IUbFXtbJ5peBcBhqYfXgS2D92F4YSVrVuCWxmZoV4F4sJgWkgrNMonsqv\nuQgwt7MPznWygRW+eQ+ugj+U72Utfms2L27fn0FYJDaHsq30ZlYr4GvV2YVMK4VMItU61yS9mSUY\nUOGbNzYeBResFagO4h6TXwBI+jHht/9iSZ80s0WSNids94e1FwFuSY19IJm3FFYNr/75lGUwpcYq\n+n5vnZdSaZKsNr8TOKxGmg9rfJ5Ud4I09xLipFaT1j/zZrWT8BvWdGcrmd94UYDSr09P/osQp7aW\nrgbLASFEZ3ID+9d/J0mbmtnrkrYC/pawa3cU8DXg/PhnYcXtbcAvJV1E6DZ8mhD3sqLMKwXn+qJK\nLYWEbpK0MWEn6LfN7F1J5wG/kvT3hFr3/wGY2VxJvyKs6OyO6fPbfXCurxq4Tv3XFoXNLz73FmE5\nd0/p/5XwWIFEmjr7IOnQuGDiOUln1JPH2NytB90u6wKUGZV1AcqMzroARapFsM9Yjjc/NO22RZGT\nDiYMbDwq6bbe7l/IX6Xw6awLUGabrAtQxiuFRHLcRm9mSyFXkZOcy5UctxSaWSnkNnKSc5nrn/DI\nQDPrIl+D4FwlOe4+NLNoiSInTVm25vXYgXkcQ3CuJ/NpaL1FA7MPzdbMSmF15CRC8MxjCE89LlFr\nYZJz+TSS0qcG3t+7y/tiSyGHkZOcy4++WCkAxEew39nMezjXljIaREwix/WVcx0sx9+8HBfNuQ6W\n429ejovmXAfL8Tcvx0VzroP10SnJRPq9dU7DeZyT0hPKpnJhKvkki4OQRFoLQOenlM9bKeVTI4hG\nYlk9bTEFDXzzJE0iPINkFfAE4Rkb6wI3AlsTt06b2TtF6U8iRGiaYGYzquXvMRqdy0Kdy5zjup+T\ngTFmtnNMdSwezdm5Nlf/hqilhOAqQyUNAIYSFgd6NGfn2lqdlUIMpnIh8AqhMnjHzO4mxWjOXik4\nl4X6uw/bAqcR1lhvAQyTdEJxmhhurT2jOTvXZ1X45nUthK7Xql75OeAhM1sCIOkWYB9gUcdEc3au\nTxrc8+mx24SjYOrstZLMA/5Z0hBgOSGy2SOEZ0B4NGfn2ladex/M7DFJVxN2Ia8CZgGXA+vh0Zyd\na2MNfPPM7ALggrLTqUVz9krBuSzk+JuX46I518F867RzrkSOv3k5LppzHSzH37wcF825Dua7JJ1z\nJXL8zctx0ZzrYDn+5uW4aM51MJ99cM6VyPE3LwdFG9NwDlO5LIVygL347VTy0TZvppLPmu3xjcpb\nhKK0nty9XUr5ZCAH37xKclw05zqYdx+ccyUq7JLMA68UnMtCjr95HnnJuSzUH3lpB0mzi453JU2Q\nNFzS3ZKelTRD0oZF10yS9JykeZIOqVW0plYKkkZIuk/SU5KelDShmfdzrm3UH6PxGTPb3cx2B/YA\nlgG30kbRnFcA3zWzzwCfB74jaccm39O5/Ks/mnOxg4HnzexV2iWas5ktMrM58fX7wNOEYJPO9W11\ndh/KHAtcH1+nFs25ZcMd8SEWuwMPt+qezuVWg7MPkgYBRwJnlH9mZiYp39GcJQ0DbgImxhaDc31b\nhVZA1yzoWjtYa08OA/5sZm/E94vbJpqzpIHAzcC1ZvabtVNcX/T6s8DOzS6ScymYT0PP6KzwzRu7\nVzgKpl5VMYfxlH55bqMdojlLEnAlMNfMLuk51fhmFsG5JhkZj4L7e3d5Yw+YXZcwyHhy0enzaJNo\nzvsRno77uKRCo2iSmf2uyfd1Lt8ai+b8AbBJ2bn2iOZsZg/iC6ScW5vvfXDOlcjxNy/HRXOug3mM\nRudciRx/83JcNOc6WI6/eRWLJul7Va4zM7sonSIsrp2kpqUp5AHaJqUITt/YpHaiBDTtnFTygeEp\n5TMwpXwW1E6SSFprWtL6uXqhHSsFwlNsq85nOufqY+04+2BmU1pYDuf6lJU5binUXEMQgzrcK+mp\n+H4XSZObXzTnOtfKAcmOLCRZWHQFcBbwcXz/BL422bmGfLTOoERHFpLURUPN7OGwjWH1tswVzS2W\nc51tZf/8DiokqRTekLQ6wL6ko4HXmlck5zrfyhyvc05SKZwCXA6MlrQQeAk4vqmlcq7DdbdzpWBm\nLwBfiNs1+5nZe80vlnOdbWWOFyokmX3YRNJPgAeB+yVdKmnj5hfNuc61kv6Jjp5I2lDSTZKeljRX\n0t6tDvF+AyG0098CRwNvADcm+smdcz1qpFIALgXuMLMdgV2AeaQY4j1JG+aTZvYvRe/PlXRMguuc\ncxV8RH3TjZI2APY3s68BmFk38K6kccCBMdl0oItQMawO8Q7Ml1QI8f6nSvdI0lKYIWm8pH7xOAaY\nUddP5JwDwphCkqMHowgzgldJmiXpijje1/wQ75LeZ83eh9OAa+LrfsAHQLUNU865Kip1DR7tWsbM\nrmXVLh0AjAFOMbNHJV1C7CoUNC3Eu5kNq3ahc65+lSqFMWPXY8zY9Va//4+pb5UnWQAsMLNH4/ub\ngEnAorRCvCeKnyhpI0l7STqgcCS5zjnXs276JzrKmdki4FVJ28dTBwNPAbcTQrvD2iHej5U0SNIo\n0gjxLulkYAKhtplNeCbkH4G/qnWtc65nDa5TOBW4Lj4l6gXgREIo2JaFeJ8I7An80cwOkjQa+Lc6\nfxjnHI0tczazxwjfyXItC/G+3Mw+lISkwWY2T9IOSW9QWwoLJAev9Ti9+iz/eSrZaNp5qeTzmqVT\n926u01LJp8agdS+sn1I+d6aUz34p5ZPcx3VOSbZCkkrhVUkbEfood0t6m4ael+Wca/e9D1+OL6dI\n6iJU8/6EJ+cakOe9D9XWKfQU7fPx+OcwYK25EudcMu26dXoWlRc5GLBNkhtI6g/MJMytHtm74jnX\nmdqyUjCzkSndYyJhOmS9Wgmd6yvyPKbQ1Ie/StoSOByYBqiZ93KunXzMOomOLDR7tONi4AekNwfl\nXEfIc/ehaS0FSUcAr5vZbLyV4FyJepc5t0JvZx9WM7Nasw/7AuMkHQ4MBtaXdLWZfbU02V1Fr7cF\ntsO5/JsTj/q05ZQka2YfBGwFvB3PbwS8TNjXXZGZnUV4XgSSDgS+v3aFAPA3vS2zczmwWzwKpvfq\n6jx3H2rOPki6ArjVzO6I7w8Dvlzpuir8uZTORXmuFJKMKexTqBAAzOxOQtcgMTO738zG9bZwznWq\nBmM0NlWSjs3C+OzIawldieOoEaTBOVfdRw1MN0qaDywFVgIrzGyvOAZ4I7A1ceu0mb0T008CTorp\nJ5hZ1XCKSVoK44FNgVuBW+Jrf5akcw1osKVgwFgz293M9ornWhfN2cyWABMkrWtmHyT5gZ1z1aXQ\nNSif5m9dNGdJ+0qaS4gtj6RdJV3W25/AObdGg+sUDLhH0swYGQ1aEc25yCWEZsd/Q4j6EqcYnXN1\nanCdwn5m9pqkTxBinMwr/rBp0ZzLbvJK4VH0UXeS65KpukYqmeXnN54HkN6PdXYquWye0jrQT9rf\npZLPIj2USj7wdDrZnDslnXwmX5tOPr1Qqfswv+tlXu56ueq1ZvZa/PMNSbcSugOL04rmnKRSeEXS\nfgAxUOQEUvtXda5vqlQpjBi7DSPGrolK8MDUB0s+lzQU6G9m78WHwBwCTCVEbf4acD5rR3P+paSL\nCN2GxqM5A98iPLvuU4QaZgbwnQTXOecqqPexcYSxgltjy30AcJ2ZzZA0kxZGc97ezI4rPhFbDn/o\n5Q/jnIvqHVMws5coXV9dOP8WKUVzTrJO4acJzznnEmrLFY2S9iEsZ/6EpNNZMy+6Hk0OzuJcp8vz\n3odqbZhBhAqgP6Wh1JYCRzezUM51ujyHY6u2S/J+4H5JV5lZ9TkS51yv5DmeQpJuwDRJGxbeSBou\n6a5qFzjnqmvLMYUinyjstoIwyilps2oXOOeqa/fHxq2UtHWhCyFpJLCqmYVyrtO15ZhCkbOB30t6\nIL4/APhm84rkXOfL85hCkq3Tv5O0B/B5wkaK08zszaaXzLkO1pZTkpJ2NLOnY4VgwML40VaStjKz\nWS0poXMdqC0rBeB04GTgQnreanlQU0rkXB/QlmMKZnZy/HNsy0rjXB/RlmMKko6iSjAGM7ulKSVy\nrg9o1ynJIwmVwqaEPRD/G88fBDxECOKagjSWPHyYQh4QVnCnYUpK+aRjka5OJR8bPTWVfDTvnFTy\nYXJXOvnQ+gW7jXYfJPUHZgILzOzIlkRzNrOvm9mJhD0QO5nZUWZ2FPCZeM45V6eVDEh0VDGRECOh\n0JpPLZpzkmXOI4BFRe8XEx4j55yrUyPLnCVtCRwOTGPN7uVxrHl23XTgS/H16mjOZjYfKERzrijJ\naMc9wF2SfhkLcAxwd4LrnHMVNDgleTHwA2D9onPVojkXh3NPJZrzqYRnR+4f3/+nmd2a4DrnXAX1\nVgqSjgBeN7PZksb2lKbp0ZzjDWYB75nZ3ZKGSlrPzN6rdW3cXTmNMA5hwElmVvEhFM71FZUeG/dB\n10yWdc2sdum+wDhJhwODgfUlXUMrozlL+iZhEdNwYNuY6c+BL9S6lhDw9Q4zO1rSAGDdBNc41/Eq\ntRQGj92bwWP3Xv3+zamXl3xuZmcBZwHE569838y+IukCWhjN+TsUPWbKzJ6VtGmtiyRtAOxvZl+L\n13UD7ya4n3MdL8VlzoWuwHm0MJrzR2b2UeFhMPE3ftVMo1HAG5KuAnYF/gxMNLNlCa51rqOlscy5\nEB0tvm5pNOf7JZ0NDJX018CvgdsTXDcAGANcZmZjgA+Ic6fO9XUprFNomiR3PQP4BvAE8A/AHYTB\nw1oWEFZbPRrf30SPlULxI7t2iYdzefci8FLdV7frLslCV+FJMxsNXF4tbbk4CvqqpO3N7FlC0+ap\ntVOe0JtsncuJbeJRcF+vrm7bSsHMuiU9UxyOrZdOBa6Lz6B8ATixnkI612k++ji/OwWSdB+GA09J\neoQwLgBh+cK4Whea2WPAng2Uz7mOtLK7DbdOF5kc/yx+MHqS2QfnXAUru9uw+yBpCPCPwHbA48Av\nzGxFqwrmXCdry0qBsNPqY+D3hB1ZOxG2azrnGtS9oj0rhR3NbGcASVcCj1ZJ65zrhVUr23NMobvw\nIs5CNKkIaQSFrroTNAMDU8qn5mryhJ5PJZe0IibZCSlFcLp2eCr5hF5yoybXTlKsTbsPu0gq3gk5\npOi9mdn6PV3knEtgeRu2FMwsv1WZc+2uu3aSrOS3unKuk3ml4JwrkeNKIckuSedc2lYkPMpIGizp\nYUlzJM2V9G/x/HBJd0t6VtKMGPWscM0kSc9JmifpkFpF80rBuSysTHiUMbPlwEFmththS/FBkv4P\nLQ7x7pxLW3fCowdFgYoGAf2Bt0kxxLtXCs5lYXnCoweS+kmaQwjlfp+ZPUX1EO8Lii5PJcS7cy5t\nDQw0mtkqYLcYB/UuSQeVfd7cEO/OuSaoVCk80QVPdiXKwszelfRbYA9aGeLdOdcElSqFHceGo+CG\n0iXhkjYBus3snbiT+a+BqYRQ7i0L8e6cS1v9QQg2B6bHGYR+wDVmdq+k2bQwxLtzLm09TDcmYWZP\nEKKkl59PLcS7VwrOZSHHKxq9UnAuCxWmG/PAKwXnsuAtBedcCa8UqhmSQh5LU8gDQjT7NKQV37bq\ndHIG0omro2vPTSUfOzydkKG6I4N4QV4pOOdK5DguulcKzmWhzinJVvBKwbks+OyDc65EjscUmrp1\nOkZ8eUrSE5J+KWmdZt7PubZRZ+SlVmhapSBpJHAyMCY+VKY/cGyz7udcW6kz8lIrNLP7sJRQ1w2V\ntBIYSv7m2JzLRl/sPsQNGhcCrwALgXfM7J5m3c+5ttJAOLZma2b3YVvgNGAkISTUMEnHN+t+zrWV\n+qM5j5B0Xxyre1LShHg+tWjOzew+fA54yMyWxILdAuwLXFea7K6i19sSnnzvXN7Ni0edPqr7yhXA\nd81sjqRhwJ8l3Q2cSIjmfIGkMwjRnM8si+b8KeAeSdvHkG49amalMA/45xgdZjlhr3cPEV/+polF\ncK5ZRsej4L97d3mdXQMzWwQsiq/fl/Q04cs+DjgwJpsOdBEqhtXRnIH5kgrRnP9U6R5NqxTM7DFJ\nVwMzgVWEx0tf3qz7OddWUphujDN8uwMPUz2ac3EFkG00ZzO7ALigmfdwri01ON0Yuw43AxPN7D1J\nqz/zaM7OtaNK3Yc3u2BJV9VLJQ0kVAjXmFkhQKtHc3aurVWqFDYcG46CZ9eK5izgSmCumV1S9JFH\nc3aurdU/prAfcALweIzgDDAJOA+P5uxcG6tzStLMHqTy+qJOiea8aQp5vJxCHgDvpZRPWhGcJqSU\nz0Up5fNhOtlseXYq2eiO0bUTJXC7XdZwHkeqdpoSOV7mnINKwbk+yCMvOedKeOQl51wJ7z4450p4\npeCcK+FjCs65EvXvkmw6rxScy4J3H5xzJbz74Jwr4VOSzrkS3n1wzpXwSsE5V8LHFJxzJXLcUmjq\nY+PS8XTWBSjzYtYFKNOVdQHK5Onv58msC5A6Sb+QtFjSE0XnUgvvDl4p1OGlrAtQpivrApTJ09/P\nU1kXoBmuAg4tO3cmIbz79sC98T1l4d0PBS6TVPM73waVgnOuwMx+D7xddnocIaw78c8vxderw7ub\n2XygEN69Kh9TcC4TqY40phbeHXJRKXwlQZpbm16K3rmvRff5dsJ0U2snaakafz8LJremGAD8umaK\nXkdNSkWlkcYH4lGfRsO7Q8aVgpll8s/hXPYqtRT2iUdBotCKqYV3Bx9TcC4jHyY8EimEd4e1w7sf\nK2mQpFEkCO8Oueg+ONcX1TemIOl6wjMjN5H0KvBDUgzvDqAEaZxzKQp9/qRTt6Na3s32loJzmcjv\nOmcfU8gBSRtLmh2P1yQtiK9nSUql4pbUJWmPGmnmS0r80ApJX5f0k8ZL1xd1Jzxaz1sKOWBmSwiP\nFEfSOcB7Zrb6CS6S+ptZozvwjdrTUQb0pqnqfc+65bel4JVCPknSfwHLgd2AP0haCrxvZhfGBE8C\nh5vZK5JOAE4FBgEPEwaUVlXJ/DJgT2AIcJOZTSn6+J8kHUYY+j7OzF6Q9Ang58BWMc1pZvZQej9u\nX5TS07aawLsP+WWEFWn7mNn3KnyOpB0Jo837mtnuwCrg+Bp5n21mewK7AgdK+mzRZ++Y2S7AT4HC\nU40vBS42s72Ao4Fp8byvM6mbdx9cfX5dYwpJwBeAPYCZ4SnlDAEW1cj3GEknE/79NydsmClsKbw+\n/nkDcHF8fTCwY8wfYD1J6/bi53Br8e6Dq8+yotfdlLbsBhe9nm5mZyXJMC5i+R7wOTN7V9JVZXkV\nK1RIAvY2s4/L8vIxhbrlN6CCdx/ax3xgDICkMcAowpf2XuDo2O8v7K3fqlImwPrAB8BSSZsBhxV9\nJsJWW+KfhXGDGRQ9AlvSbkXpXV1WJDxaz1sK+Vb8m/hm4KtxgPFh4BkAM3ta0mRgRtwrv4Kwk+qV\nHjM0e0zSbGAe8CrwYNn9NpL0GGGQc3w8PwH4WTw/ALg/3iPJjIbrUX5bCr6i0bkWC92umxOmPspX\nNDrXN+R3StIrBecy4bMPzrkS+R1T8NkH5zJR/+yDpENjdObnJJ2Rdsm8peBcJuprKUjqT1htejAh\nitKjkm4zs9TCnnul4Fwm6h5T2At4PkZnRtINhKjNXik4197qHlP4FGF9ScECYO+Gi1PEKwXnMlH3\nlGTTFxZ5peBcJqbUe2F5hOYRhNZCanxFo3NtJEbieoawO3YhITrzeB9odK6PMrNuSacAdwH9gSvT\nrBDAWwrOuTK+eMk5V8IrBedcCa8UnHMlvFJwzpXwSsE5V8IrBedcCa8UnHMlvFJwzpX4/zbFo1W3\nifPlAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x19058d9d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#### #################################      Problem 2      ###############################\n",
    "#####################################################################################\n",
    "\n",
    "\n",
    "from sklearn import metrics as metr\n",
    "plt.matshow(metr.confusion_matrix(label1_valid, pred1))\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('Predicted label')\n",
    "plt.xlabel('True label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQUAAAD0CAYAAACSLzOZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHkZJREFUeJzt3Xm4HVWZ7/HvjyRAQpgCgoKBpEEgIEiCIMNFQkvzgI2o\nF7oZVbAbbVFGbRmkhfRVLvA001Wwm8GIzDLZotAE0BPEgRAJYQhhNJAYEiBAmEzISd77x1o7qb2z\nhzp7V52qfc77eZ56zh6qVr07OXudtVatepfMDOecq1ij6ACcc+XilYJzropXCs65Kl4pOOeqeKXg\nnKvilYJzropXCgWQNFzSnZLelHRzB+UcJemeLGMriqS9Jc0uOg4H8nkKjUk6EjgV2BZ4G3gU+L6Z\n/a7Dcr8AfAPYw8xWdBxoyUlaAWxtZi8UHYtrzVsKDUg6FbgY+B6wCTAauAw4OIPitwSeGQwVQoIa\nviEN7c9AiibJ+rL1e4Bm5lvNBqxPaBkc0mSftYBLgL/E7WJgzfjeRGAeoZWxEJgPHBPfmwQsBd6P\n5/gycA5wbaLsMcAKYI34/BjgeeAt4AXgyMTrv00ctyfwMPAmMI3QEqm81wP8O/BgLOceYKMGn60S\n/78Cr8T4Pwd8GngGWAScnth/N+APwBtx3x8Aw+J7D8TP8k78vP+QKP/bwMvANfG1ufGYreI5xsfn\nmwGvAp8s+ncjo98v+17KLXxF+zc+bynUtwewNnBHk32+Q/gyfCxuuwFnJd7fFFiP8Av9T8BlktY3\ns7OBc4GbzGxdM/sx4T+/LknrAJcCB5jZejG2R+vsNwr4FaGiGgVcBPxK0oaJ3Y4gVCSbAGsC32ry\n+TYlVHwfAr4LXAUcBYwH9ga+K2nLuG8vcBKwUYzvU8DxAGb2ybjPTvHz3pIof0NgC+CryROb2fPA\nacB1koYDk4HJZvZAk3i7yrCUWxG8UqhvI+A1a968PxL4dzN7zcxeI7QAvpB4f1l8f7mZ3U34S7lt\nfE9UN6cbNq2jFcCOkoab2UIzm1Vnn78Hnjaz681shZndBMxmVXfHCF+s58xsCfAzYOcm51xGGD9Z\nDtxMqGguMbN34/lnVY43s0fMbFo874vAFcA+KT7T2Wa2LMZTxcyuAp4jtHg2JVTCA8bQlFsRvFKo\nbxGwsaRm/z6bAS8mnr8UX1tZRk2l8h4wsq+BmNm7wGHAvwDzJf1S0rZ1dt0sxpD0Yk1MCxKP/9oi\nnkUW27pxXwhdoeTx6wBI2ibG9bKkxcD3CRVrM6+a2fst9rkK2AH4gZkta7FvVxmeciuCVwr1/YHQ\n7/98k33mE/r+FVvE19rxDjAi8fyDyTfNbIqZ7R9fnw1cWaeMvxAGMJO2jK/n7UeElsPWZrY+4a96\nq9+tpgNokkYSukJXAZNqukFdz7sPXcbMFhP60ZdJ+qykEZKGSTpQ0vlxtxuBsyRtLGnjuP+1bZ7y\nUeCTkkZLWh84o/KGpE1iDOsQmvTvAsvrlHE3sI2kIyQNlXQYsB3wy8Q+rbop7RpJGER8T9J2wNdq\n3l9IGDzsi0uBaWb2FcJYyX92HGWJePehC5nZRYSrB2cRRuBfIgyeVQYfvwdMBx6L2/T42soimhWf\nfN/M7iP02x8jXD24M/H+GsAphL/4iwiDfF+rLcfMFgEHAd8EXiMMIh5kZq83iMloHWOz50nfIoyx\nvEUYT7ipZv9zgGskvSHp0CbnNgBJnwX2Z9XnPBWYIOmIJjF0lTK3FEo9eUnSAYQm5BDgKjM7v8Uh\necYyGvgpYeTegCvM7P8VFU+MaQihMppnZp8pOJYNWDUGYMCXzeyPBcZzBnA0YUDzceBYM1taVDxJ\nkuymlPseDphZXi28ukrbUoi/8D8EDgC2B46QNK7AkJYBp5jZDsDuwNcLjgfCZcBZtOif95NLgbvM\nbBywE/BUUYFIGgMcB0wwsx0Jf1QOLyqeesrcUihtpUC47v+cmc2JI883AZ8tKhgzW2Bmj8bH7xB+\n6TdrflR+JH2YMJnoKvIbK0gby/rA3nHOBWbWG8dlivIWoRIfEWdLjqB/BlxT80qhPZsDcxPP58XX\nChf/Eo0HHiowjIsJMw7LMFV6LPCqpMmSHpF0paQRLY/KSRxHuZAwDjQfeDOO25SGX5JsTxmaxKuJ\nl8puBU6KLYYiYjgIeMXMZlBwKyEaCkwALjezCYQrJKcXFYykrYCTCZeMNwNGSjqqqHjq6eTqg6ST\nJD0u6QlJJ8XXRkm6V9IzkqbEMZ7K/mdIelbSbEn7t4qtzJXCXwg3IVWMJrQWCiNpGHAbcJ2Z/bzA\nUPYEDpb0Z8Kl0b+V9NMC45lHGOx8OD6/lVBJFOXjwO/NbJGZ9QK3E/7NSqPd7oOkjwL/DOxKmF5/\nUKwETwfuNbNtgPvjcyRtT5j8tj1hfO7yFpPySl0pTAc+ImmMpDUJH+wXRQUjScDVwCwzu6SoOADM\n7EwzG21mYwkDaL82sy8WGM8CYK6kbeJL+wFPFhUPYYLX7jFvhWI89aaGF6aDlsJ2wENmtiROQZ8K\nHEKYzn5N3Ocawg1sEMbhbozTyecQpo7v1iy20lYKsYb/BuFuvlnAzWZW2Ig2sBfhEte+kmbE7YAC\n40kqQ1frBOB6STMJVx/OLSoQM5tJuHxcmUcCYf5EaXQw0PgEsHfsLowgDDZ/GNjUzCrT0BcS7heB\n0H1KtrBbjs2V+j72eCPR3UXHAWBmD1LCStTMphL+WhQdx0xCk7YUzOwC4IKi42ik3S+emc2Os2qn\nEMZuHqVmhquZtcrD0PSPSKkrBecGqkaXG6fFrZl46ffHAJK+T/jrv1DSB81sgaQPEWbhwupjcx+m\nxeXZUs9odG4gkmR/TrnvWFaf0ShpEzN7RdIWhO717oSb0BaZ2fmSTgc2MLPT40DjDYRxhM2B+wg3\nrjX84ntLwbkCdDgx6VZJGxEmaB1vZoslnQf8TNI/AXOAfwQws1mSfkYYl+uN+ze/Q9VbCs71L0m2\noPVuQLhXvr/vffCWgnMFGJb2m9ebaxh1FVopFJKp1rmc9OUv+lCvFBprdQvpLYT0v80cztkZRZOm\np3c/IS9pM+tmEAvA6613oYeQCLmZrG6tSfO57gYObLHP2xnEkkaa/yvI5mvQt1ndw4ZkcMqcFF4p\nODcYpW4pFKDEoTk3cA1bq+gIGst1hp6kA+KdWc9KOq2dMrbPOqiOjS06gBpjig6gxtZFB5BQtv+r\nhBInaczttInMSfsRZlA9LOkXfb1/YYc8guvI3xQdQI0xRQdQ4yNFB5BQtv+rhBK30fNsKZQqc5Jz\npVLilkKelUJpMyc5V7ghKbcC5FkX+RwE5xopcfchz9BSZU66JfF4e8o4huBcPc8TFgBvU4mvPuRZ\nKazMnERInnkYYdXjKq0mJjlXTltRvejV/X07fDC2FMysV1Ilc9IQ4OqCMyc5Vx6DsVKAcmVOcq5U\nSjzNuXTpxZwbFDq4JBlTtj8Z07zfIGmtwZLi3bmBq81KocmSeIMixbtzA1f7LYV6S+LNZzCkeHdu\nQFsr5VajwZJ49zKQUrxnkQvhbCZlEAlMyiwvQ5o8CGlktcjSIxmVk5VlGZWzaetdUlnYepestfnN\nq1kSbzFwi6Sjk/t4infnulGDqw89r0DPq02PXLkkHoCk24E9gAVZpXj3SsG5IjT45k3cLGwVk1Zf\n7G428G+ShgNLCHchTyMsDPMl4Pz4s7LW6S+AGyRdROg2fIQWS0t4peBcEdr85pnZzLiY8HRgBaFv\neAUhV14mKd69UnCuCB1MXmqwJN7rhFZDvf3PpQ9re3ql4FwRSvzNK3Fozg1gaxcdQGNeKThXhBLf\n++CVgnNFKPE3r8ShOTeAlfibV+LQnBvAvPvgnKtS4m9eiUNzbgAr8TevxKE5N4AN0sStzrlGSvzN\nK3Fozg1gJf7mlTg05wYwv/rgnKtS4m9eCULrPHvOJC7PIA6wM4/PpBydOyWTcuDxjMrJSm9G5Ywq\nWTndk3mpP3iORueK0OYCs5K2lTQjsS2WdKKneHeu262dcqthZk+b2XgzGw/sArwH3IGneHeuy3Ww\nGEzCfsBzZjYXT/HuXJdrs/tQ43Dgxvg4sxTvuVYKkkZL+k1c4uoJSSfmeT7nukaHLQVJawKfAW6p\nfS/mYCxtivdlwClm9qikkcCfJN3rq0+7Qa/BN6/nsbClcCDwJzOrJIRf2BUp3s1sAbAgPn5H0lOE\n5oxXCm5wa9A1mDg+bBWTbmhYwhGs6jpASOXeXSne48KY44GH+uuczpVWBzkaJa1DGGQ8LvHyeXRT\nivfYdbgVOMnM3umPczpXap2leH8X2Ljmte5J8S5pGHAbcJ2Z/Xz1PX6ZeLxN3Jwruzlxa1OJZzTm\nGpokAVcDs8zskvp7HZRnCM7lZEzcKqb27fASVwp5z1PYCzga2DcxLfOAnM/pXPllM3kpt9ByY2YP\n4hOknFud3zrtnKtS4m9eiUNzbgDzHI3OuSol/uaVODTnBrASf/Mahibpm02OMzO7KJsQlmVQxusZ\nlAE699pMyplMyzwWqRzLeZmUA8MyKuevGZUzPKNysvjdgez+ffqgGysFYF1a3E3lnGuPdePVBzM7\npx/jcG5QWV7ilkLLOQQxJ9z9kp6Mz3eSdFb+oTk3cC0fmm4rQpqJRVcCZwLvx+ePE27bdM61aela\na6baipCmLhphZg+F2xjCCKOkrEZ4nBuUlg8p76BCmpbCq5K2rjyRdCjwcn4hOTfwLWdIqq0eSRtI\nulXSU5JmSfpEf6d4/wbwX8B2kuYDpwBfS/fRnXP19DIk1dbApcBdZjYO2AmYTYYp3lt2H8zseeBT\nMdvLGmb2dqpP7ZxraHmbExUkrQ/sbWZfAjCzXmCxpIOBfeJu1wA9hIphZYp3YI6kSor3PzY6R5qr\nDxtL+gHwIDBV0qWSNmrrEznngI66D2MJXfrJkh6RdGX8g92vKd5vImSG/d/AocCrwM0pjnPONdBB\npTAUmABcbmYTgHeJXYWK/kjx/kEz+z+J59+TdFiK45xzDSyl/uXGaT1LmNazpNmh84B5ZvZwfH4r\ncAawoD9TvE+RdASrWgf/AGS1rLJzg1KjMYVdJo5kl4kjVz6/bNLiqvfjl36upG3M7BlCstYn45Zv\nindJ77CqmXEyULlbaA1Ck6XZDVPOuSYaXW5M6QTg+rhK1PPAsYRcTvmmeDezkY3ec851ppNKwcxm\nArvWeav/UrxL2pDQ7Fi5hIWZPZD2JM65ak3mIBSuZaUg6TjgRMJgxQxgd+APwN/mG5pzA1e78xT6\nQ5pLkicRJjvMMbN9CUu/LW5+iHOumU6mOectTXW1xMz+KglJa5vZbEnbZhdCFlmTxmVQBsBzmZSS\nVcYk+/jprXdKQdPPzqScQjIUNZXV5NrPZFBG37IJvN/gkmQZpKkU5sYxhZ8D90p6g47Wy3LOdfWY\ngpl9Pj48R1IPsB7wP3kG5dxAV+YxhWbzFEbVefmx+HMkWWVLdW4QKmq8II1m1dUjNJ4jbcDfpDmB\npCHAdMLUzCw6b851va6sFMxsTEbnOIkwm2rdjMpzruuVeUwh18VfJX0Y+DRwFaA8z+VcN3mftVJt\nRch7tONi4F8Jg5POuajM3YfcWgqSDgJeMbMZeCvBuSodpmPLVV+vPqxkZq2uPuwJHCzp04R7JtaT\n9FMz+2L1bj2Jx2Pi5lzZPUwYP29PV16SZNXVBwFbAG/E1zcEXiSkhWrIzM4krBeBpH2Ab61eIQBM\n7GPIzpXBrlTfqPiffTq6K7sPZjbGzMYC9wIHmdlGZrYR8Pfxtb7ydSmdizpM8T5H0mOSZkiaFl/r\n1xTve5jZXZUnZnY3oWuQmplNNbOD+3KMcwNZhzdEGTDRzMab2W7xtf5L8Q7Mj2tHXkfoShxJixxv\nzrnmlnZ+ubF28L7/UrwT1o3cBLgDuD0+9rUknetABi2F+yRNj/lOIMMU72luiFoEnChpHTN7t9X+\nzrnWGn3hX+iZyws98+q+l7CXmb0s6QOEO5dnJ9+M673ml+Jd0p6EGYnrAqMlfQz4qpkd3+pY51x9\njeYgbDFxDFtMHLPy+f2THlptHzN7Of58VdIdhO7AwqxSvKfpPlxCGKB4LQYyk1V9F+dcG5YzNNVW\nS9IISevGx+sA+wOPE1K5fynuVpvi/XBJa0oaSycp3pPM7KXKUvRRb5rj0hmeQRlPZVAGZBML8K3T\nMilG/5FJMVxu2fz7HK+JmZQTurxZyOoeu/syKie9DuYpbArcEb+PQ4HrzWyKpOnkneI94SVJewHE\nPPMnkt230LlBqd1Kwcz+DOxc5/XX6ccU718jLH29OaEvMgX4etoTOOdW12jZuDJIUylsY2ZHJl+I\nLYff5ROScwNfme99SDPQ+MOUrznnUurKFO+S9iBMZ/6ApFNZNYNqXXJOzuLcQFfmG6KatWHWJFQA\nQ6ge5n0LODTPoJwb6Mqcjq1ZjsapwFRJk83sxX6MybkBr9vHFK6quQ1zlKR7cozJuQGvK8cUEj5g\nZm9WnpjZ65I2bXaAc665bl82brmkLStdCEljgBV5BuXcQNeVYwoJ3wF+K+mB+PyTwFfyC8m5ga/M\nYwppbp3+H0m7ALsTbrk82cxeyz0y5wawrrwkKWmcmT0VKwQD5se3tpC0hZk90i8ROjcAdWWlAJwK\nHAdcSP2kDPvmEpFzg0BXjimY2XHx58R+i8a5QaIrxxQkHUKTtE1mdnsuETk3CHR6SbJ2Nfe4eNPN\nwJbEfAqVqQSSzgC+DCwHTjSzKc3KblZdfYZQKWxCuAfi1/H1fYHfE5K4ZmBcBmU8nkEZAH/Nppj/\nuCibcjJyvMZkUs4Ujs2knP05O5NyoNUiZWll9P/eBxl0H2pXc6+keL9A0mnx+ek1Kd43JyR83cbM\nGk4raLYYzDFmdizhHojtzewQMzsE2CG+5pxrU7vp2KDhau4HE1K7E39+Lj5emeLdzOYAlRTvDaXp\n2IwGFiSeLyQsI+eca1OHVx/qrebeLMV7co2HzlO8ExLY3SPpBkKtdBjtLRvnnIsaVQpv9zzC2z0z\nGh6XXM1dqp80M/cU78AJwOeBvePz/zKzO1Ic55xroFGlMGLiroyYuGrh2pcnTa7dpd5q7tfSnyne\nY+bXR4C7zOwUQqshVRpdSRtIulXSU5JmSdo9zXHODXRLWSvVVsvMzjSz0XHx58OBX5vZF+jPFO+S\nvkKYxDQK2IpQ0/wI+FSKz34poTI5VNJQYJ0Uxzg34GU4o7HSFTiPfkzx/nUSC1Ka2TOSNml1kKT1\ngb3N7EvxuF5gcYrzOTfgZVEpVBIhxcf9muJ9qZktrSwGE//iN61porHAq5ImAx8D/gScZGbvpQ3O\nuYGqzNOc02RemirpO8AISX8H3ALcmeK4ocAE4HIzmwC8S5hQ4dyg18k8hbylOetpwD8Tpg1+FbiL\nMGmilXmEKZgPx+e3UrdSuDHx+KPAjimKdq5oc+LWnm69S7LSVXjCzLYDruhLwfHSyNw4pfIZQn/n\nydX3PKIvxTpXEmPiVjG1T0d3baVgZr2Snk6mY+ujE4Dr4xqUz0NGk+ed63JL3y/vnQJpug+jgCcl\nTSOMC0CYvnBwqwPjsvW7ttrPucFmeW8X3jqdcFb8mVyLPs3VB+dcA8t7u7D7IGk48C/A1sBjwI/N\nbFl/BebcQNaVlQLh9sv3gd8SbtPcnnAPt3OuQ73LurNSGGdmOwJIuhp4uMm+zrk+WLG8O8cUeisP\n4lWInEJ4LoMyWs66TmlYRuVklREoi6xUEO5n61xWGZPsI5MyKUfPXppJOTAxgzL6+Jm6tPuwk6S3\nE8+HJ56bma1X7yDnXApLurClYGblrcqc63a9rXcpSpp7H5xzWetNudWQtLakhyQ9GnOU/N/4+ihJ\n90p6RtKUmpXiz5D0rKTZkvZvFZpXCs4Voc1KwcyWAPua2c7ATsC+kv4Xq7I5bwPcH59Tk835AOBy\nSU2/914pOFeEZSm3OhLpB9YEhgBvkGE2Z68UnCvC8pRbHZLWkPQoIWvzb8zsSZpnc56XODyTbM7O\nuax1MNAYF3LZOWY3u0fSvjXv557N2TmXtSUNXp/ZA4/1pCrCzBZL+hWwCxlmc/ZKwbkiNGop7DAx\nbBXXVU+KkrQx0Gtmb8b7k/6OMHOqks35fFbP5nyDpIsI3YbOszk753LQfvfhQ8A18QrCGsC1Zna/\npBn0YzZn51zW2qwUzOxxQu7T2tf7NZuzcy5rJU5C4JWCc0VocLmxDLxScK4IJb73wSsF54rQ6JJk\nCXil4FwRvKXgnKvilULJQ1hp09a7pDImo3IeyqicURmVk01eHT17YSbl2LezSRmqC7KJp0+8UnDO\nVfFLks65Kn5J0jlXxa8+OOeqlHhMIdckKzE33JOSHpd0g6S18jyfc12jg8xLecutUpA0BjgOmBAX\nlRkCHJ7X+ZzrKh1kXspbnt2Htwh13QhJy4ERtEju4NygMRi7D/FWzguBl4D5wJtmdl9e53Ouq7Sf\n4n20pN/EbvkTkk6Mr5c/xbukrYCTCTN5NgNGSjoqr/M511XaH1NYBpxiZjsAuwNflzSODFO859l9\n+DjwezNbFIO7HdgTuL56t7sTj7cmZItyruyeA55v//Cl7R1mZguABfHxO5KeIqRZOxjYJ+52DdBD\nqBhWpngH5kiqpHj/Y6Nz5FkpzAb+LeaRW0LIClMnN9yBOYbgXF62jlvFlL4dnsGYQhzMH0+YD98s\nxXuyAiguxbuZzZT0U2A6sIKw9PEVeZ3Pua7S4eVGSSOB24CTzOzt5KrwpU7xbmYXABfkeQ7nulKj\ny42v9sBrPU0PlTSMUCFca2aVrM2e4t25rtao+7DhxLBVzF4txbuAq4FZZnZJ4i1P8e5cV2t/TGEv\n4GjgsZjWHeAM4Dw8xbtzXazNMQUze5DGUwk8xbtzXavNS5L9QS1aEvmeXLIw6bFTb2VQBsCwjMrZ\nMqNyxmVUzp0ZlZOViRmVMyeTUuyYYzsuQz8BM1PLHYm/93uk/N79QanLzYq3FJwrgmdecs5V8cxL\nzrkqJb5L0isF54rglYJzroqPKTjnqpT4kqRXCs4VwbsPzrkq3n1wzlXxS5LOuSrefXDOVfFKwTlX\npcRjCrmuEOWca6D9FO8/lrRQ0uOJ1zJL7w5dUSk8V3QANV4oOoAa04sOoMacogNImF10AHmYTEjV\nnpRZenfoikqhgzTaufhz0QHU+FPRAdSYU3QACQOvUjCz3wJv1Lx8MCGtO/Hn5+LjlendzWwO4S/s\nbq3O0QWVgnOuhWbp3ecl9muZ3h18oNG5guQz0thpencoRaXwzRT79HGhjdz9pugAalxZdAA1prZ4\nf1KL97P03y330E/yj2J1ja5JPhC3PsksvTsUnI7NucEo/CVfnHLv9VdLxxZXhrrTzHaMzy8AFpnZ\n+ZJOBzYws9PjQOMNhHGEzYH7gK09m7NzpfTXto6SdCNhzciNJc0FvkuG6d3BWwrO9bvQUpibcu/R\nnrjVucGhvPOcvVJwrhDlnefs8xRKQNJGkmbE7WVJ8+LjRyRlUnFL6pG0S4t95kga1Ycyj5H0g86j\nG4zanOfcD7ylUAJmtggYDyDpbOBtM7uo8r6kIWbW6R34Rutr1Ab0pf/qA1JtK29LwSuFcpKknwBL\ngJ2B30l6C3jHzC6MOzwBfNrMXpJ0NHACsCbwEGGUeUWTwi8HdgWGA7ea2TmJt78t6UDC8PiRZva8\npA8APwK2iPucbGa/z+7jDkbtXX3oD959KC8jTFPdw8zqzfAyAEnjCJeg9jSz8cAK4KgWZX/HzHYF\nPgbsI+mjiffeNLOdgB8ClaXOLwUuNrPdgEOBq+Lr/ToqPrB498G155YW15UFfArYBZguCcJf/wUt\nyj1M0nGE//8PEe6ieyK+d2P8eRNwcXy8HzAulg+wrqR1+vA53Gq8++Da817icS/VLbu1E4+vMbMz\n0xQoaSxhbvnHzWyxpMk1ZSVVKiQBnzCz92vK8jGFtpX3kqR3H7rHHGACgKQJwFjCl/Z+4NDY768k\n3NiiUSHAesC7wFuSNgUOTLwnwv33xJ+VcYMpwIkrd5J2Tuzv2rIs5db/vKVQbsm/xLcBX4wDjA8B\nTwOY2VOSzgKmxAQay4DjgZfqFmg2U9IMQrKBucCDNefbUNJMwiDnEfH1E4HL4utDCXc8HU+6Kxqu\nrvK2FHyas3P9LHS7bku59yE+zdm5waG8lyS9UnCuEH71wTlXpbxjCn71wblCtH/1QdIBMWX7s5JO\nyzoybyk4V4j2WgqShhBmm+5HSK32sKRfmNlTWUXmlYJzhWh7TGE34LmYsh1JNxFSuXul4Fx3a3tM\nYXOq0zbNAz7RcTgJXik4V4i2L0nmPrHIKwXnCnFOuwfWpm0fTfWCLx3zGY3OdZGYietpwt2x84Fp\nwBE+0OjcIGVmvZK+AdwDDAGuzrJCAG8pOOdq+OQl51wVrxScc1W8UnDOVfFKwTlXxSsF51wVrxSc\nc1W8UnDOVfFKwTlX5f8D54fVOdPWUSwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1697a0790>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import metrics as metr\n",
    "plt.matshow(metr.confusion_matrix(label2_valid, pred2))\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('Predicted label')\n",
    "plt.xlabel('True label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAD0CAYAAACM5gMqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH21JREFUeJzt3Xm4XFWd7vHvS0KEMAkikwSDQBhsmRQUFAiKXvAqKHJF\nQEW0kZYWULEVlCuJF7HxeQjkqnBbAzQog4BIQ4sytUEcmhlEQmSQAIEQkBlkyDm894+1itQ5nKqz\nq86us6sqv8/z7Ce1d+299u8kZ6+saa8l24QQwmiWqzqAEEJviMwihFBIZBYhhEIiswghFBKZRQih\nkMgsQgiFRGZRAUkrSrpU0lOSfjaGdA6QdHmZsVVF0k6S5lcdR2hMMc6iMUn7A18BNgWeBW4FvmP7\n92NM91PAF4EdbL8y5kC7nKRXgI1t/7XqWEL7omTRgKSvACcBxwFrAVOAHwJ7lpD8m4G7loWMoo4a\nfiFNHM9AqibJrWxVx/sq27EN24DVSCWJjzU553XAycBDeTsJmJS/mw4sJJVKFgMPA5/J380EXgJe\nzvf4LDAD+Eld2lOBV4Dl8v5ngHuBZ4C/AvvXHb+27rodgRuAp4DrSSWX2ndzgW8Dv8vpXA68ocHP\nVov/X4BHc/wfAT4I3AU8DhxVd/72wB+BJ/O53weWz9/9Nv8sz+Wf93/Vpf81YBFwZj72YL5mo3yP\nbfL+esBjwM5V/26U9Pvl4wpu6RGtPmbbUbJoYAdgBeAXTc75Jukh2Spv2wPH1H2/NrAq6Rf9c8AP\nJa1m+1jgeOA826vYPp30SzEiSSsBs4Hdba+aY7t1hPPWAH5JysDWAGYBv5S0et1p+5EymLWAScBX\nm/x8a5MyxHWBbwFzgAOAbYCdgG9JenM+dwA4AnhDju99wKEAtnfO52yZf94L6tJfHdgAOKT+xrbv\nBb4O/FTSisAZwBm2f9sk3p6yfMGtm0RmMbI3AH9z82rC/sC3bf/N9t9IJYZP1X2/JH8/aPtXpP9Z\nN83fiaHF8oZF9OwV4G2SVrS92Pa8Ec75n8BfbJ9t+xXb5wHzWVptMumBu8f2i8D5wNZN7rmE1D4z\nCPyMlAGdbPv5fP95tett32z7+nzf+4EfAbsU+JmOtb0kxzOE7TnAPaQS0tqkzLlvTCy4dZPILEb2\nOLCmpGZ/P+sB99ftP5CPvZrGsMzm78DKrQZi+3lgX+CfgIcl/aekTUc4db0cQ737h8X0SN3nF0aJ\n53HnMnM+F1KVqv76lQAkTctxLZL0NPAdUobbzGO2Xx7lnDnAW4Hv214yyrk9ZcWC23CSTpe0WNLt\ndcfWkHSlpLskXSHp9fn4VEkvSLolb6fUXfN2SbdLulvS7CIxR2Yxsj+S2hU+2uSch0ltCzUb5GPt\neA6YXLe/Tv2Xtq+w/YF8fD7w4xHSeIjUcFrvzfl4p51KKmlsbHs1UilgtN+tpg13klYmVanmADOH\nVad63hiqIWcAuw87dhRwpe1pwNV5v+Ye29vk7dC646cCn7O9CbCJpOFpvkZkFiOw/TSpnv5DSXtJ\nmixpeUl7SDohn3YucIykNSWtmc//SZu3vBXYWdIUSasBR9e+kLRWjmElUtXgeWBwhDR+BUyTtJ+k\niZL2BTYD/rPunNGqO+1amdR4+XdJmwFfGPb9YlKjZStmA9fb/jypLeb/jTnKLtJuNcT2taSG5Hp7\nkhqJyX9+pNm9Ja0LrGL7+nzorNGugcgsGrI9i9SbcQypR+ABUqNdrdHzOOBG4E95uzEfezWJZsnX\nf2/7KlK7wJ9IvRmX1n2/HPBlUgnhcVLj4heGp2P7ceBDwJHA30iNlx+y/USDmMzoMTbbr/dVUhvO\nM6T2ivOGnT8DOFPSk5L2aXJvA0jaC/gAS3/OrwDbStqvSQw9peQGzrVt16qIi0ltPDUb5irIXEnv\nycfeROqNqnkoH2uqqwdl5aLRycAEYI7tE0a5pJOxTCHlwGuRfql/ZPv/VhVPjmkCKZNaaPvDFcfy\nepa2MRj4rO3/rjCeo4FPkhpSbwcOsv1SVfHUk+TzGnx3B6k+V/NzwPaQEqGkqcCltt+W95+0vXrd\n90/YXkPSJGAl209K2ha4mPTvsynwXdvvz+fvBHxttN+hbmtwfVV+EH4A7EbK+W6QdIntOysKaQnw\nZdu35vr0TZKurDAeSN2V84BVKoyhZjZwme198iCrlaoKJD9MBwOb234pD6n/BEuL6pVrVGrYmqFd\nVD8vltxiSevYfiRXMR4FyA3IL+fPN0u6F9iE9DytX3f9+hRo2+rmasj2pMaZBbkl/Dxgr6qCsf2I\n7Vvz5+eAOxna0zCuJK1PGiQ1h861RRSNZTVgpzxmBNsDud2nKs+QMvfJOeOazPg09BZWcjXkEuDA\n/PlAUgmC3J42IX9+Cymj+KvtRcAzkt4pSaQu/4tHu0k3ZxZvAh6s219IgXrVeMj/c20DXFdhGCeR\nRlh2w5DxDYHHJJ0h6WZJP5Y0edSrOiS305xIamd6GHgqtwt1jTF0nZ4L/AHYVNKDkg4C/hV4v6S7\ngPfmfYCdgdsk3QJcABxi+6n83aGk/2juJv2n/OvRYu7aagijdK1VJVdBLgSOyCWMKmL4EPCo7Vsk\nTa8ihmEmAtsCX7R9g6STSd1336oiGEkbAV8idW0/DVwg6QDbZ1cRz0jaffBsN2rk3W2Ecy8CLmqQ\nzk3A21q5dzeXLB4ivbxVM4WhLbjjTtLypGrkT22PWmzroB2BPSXdR+rCfa+ksyqMZyGpkfWGvH8h\nKfOoyjuAP9h+3PYA6YHZscJ4XiOGe5frRtJgkam5VXdfUt2sErludxowz/bJVcUBYPsbtqfY3pDU\ncPdftj9dYTyPAA9KmpYP7UZq2K/KfOBdSvOGKMcz0hD5yvTicO9ui+dVtgckfZH0duQE4LSKex7e\nTeqK+1OuAwIcXaSuNw66ocp2GHB2ztjvBQ6qKhDbt+WS1o2kNp2bSeM/uka3lRqK6OpxFiH0I0m+\nreC5W/HacRZV6dqSRQj9rBdLFpFZhFCBkbpFu11kFiFUIEoWIYRCevHB68WYQ+h5yxd98gY6GkZL\nKs0sumrm4hDGqJVei4mRWbTOazX/fsZzMGOUyej06LElRVOkJnk1aT7a8VBkJrm5pImxmymrOa3I\nb26Rv5+yZsgb7d+r6L9VGY/BUaOfUmf5CSXccpxVnlmEsCwqXLLoIj0Ycgi9b/nXVR1B6zr6boik\n3SXNzzMIf72dNKZPKjuqsdqw6gCGmVp1AMN0099PN8UyTA++HNKxcMqa6ar7Mou3VB3AMFOrDmCY\nbvr76aZYhumyjKCITpYsumqmqxC6Sg+WLDqZWXTtTFchVG5Cwa2LdDLvijEUITTSZaWGIjoZcqGZ\nrmbUTUw3fVI3tlGEMJJ7SQvat6kHe0M6mVm8OtMVadLUfUmreA8x2oCrELrTRgxdZO3q1i6PksVS\nXTjTVQjdowczi46Os7D9K9ub2t7Y9nc7ea8QesoYGjglHZFXQP+zpCPysRFXUs/fHZ3HOs2X9IF2\nQ+7mCXtD6F9tdp1K+gfgH4HtSLPufSgvfTDiSuqStiA1AWxBWn39FEltPfeRWYRQhfbHWWwGXGf7\nRduDwDXAx2i8kvpewLm2l9heANxDGgPVssgsQqhC+5nFn4GdcrVjMmkJy/VpvJL6egzthWx7vFMP\nNrOE0Afa7Dq1PV/SCcAVwPPArcDgsHM8ylwxbY2BqjyzKGMuimOZWUIkMJPjSkln/OZrKOqFktIp\nS1nza5T1c5X179WCBk/e3MfS1kxegPp0AEnfIZUWRlxJndeOdyq0YnoLIYcQOqpBT8f0ddJWM3P+\na8+RtJbtRyVtAOwNvIv0iu2BwAnUraROWsXvHEmzSNWPTYDr2wk5MosQqjC2J+9CSW8gFYkOtf20\npH8Fzpf0OWAB8HEA2/MknU9avnEgn9+b1ZAQlkljePJs7zzCsScYYSX1/N3xwPHt3zGJzCKEKnTZ\nG6VFRGYRQhV68MnrwZBD6AMrVB1A6yKzCKEKUQ0JIRTSg09eD4YcQh/owSevB0MOoQ9ENSSEUEgP\nPnk9GHIIfaAHn7weDDmEPhAT9oYQCunBJ68HQw6hD/Tgk9eDIYfQB6I3JIRQSA8+eV0Q8rvHnMJM\nflJCHOCvfaqUdPS9C0tJB+4vKZ1um7mrrJmyykrn2ZLSaUEXPHmt6sGQQ+gDUQ0JIRQSb52GEArp\nwSevB0MOoQ/0YDWko4sMSZoi6TeS7sjrMh7eyfuF0DPaX2SoMp0OZwnwZdu3SloZuEnSlbGaeljm\ndVlGUERHQ7b9CPBI/vycpDtJy6lFZhGWbVENaUzSVGAb4LrxumcIXWuFgtswkjaVdEvd9rSkIyTN\nkLSw7vgeddccLeluSfMlfaDdkMelMJSrIBcCR9h+bjzuGUJXa7NkYfsvpP90kbQcaSnCi4DPArNs\nz6o/X9IWwL7AFqQVya6SNM32K63eu+OZhaTlgZ8DP7V98WvPOKvu81Z5C6Hb/RW4r/3Ly3nydgPu\nsf2gJAEa4Zy9gHNtLwEWSLoH2B7471Zv1tHMIv8ApwHzbJ888lmf7mQIIXTIW/JW85vWLi/nyfsE\ncG7+bOAwSZ8GbgSOtP0UqY2wPmNYSCphtKzTbRbvBj4J7FpXl9q9w/cMofuNsetU0iTgw8AF+dCp\npMWRtwYWASc2uXv3rXVq+3eMYyNqCD2jQZvF3BvSVsAewE22HwOw/WjtC0lzgEvz7kPAlLrr1s/H\nWtaDvb0h9IEGT970HdJWM/PUhinsx9IqCJLWtb0o734UuD1/vgQ4R9IsUvVjE+D6EkMOIXTUGObg\nlLQSqXHz4LrDJ0jamlTFuA84BMD2PEnnA/OAAeBQ291XDQkhNDCGJ8/288Caw4417CmwfTxwfPt3\nTCKzCKEKPfjkNQxZ0pFNrvPwwR/tu7mENAZKSKO8Ga58zj6lpKP9jyslHdi8pHS6bZT+JiWl86eS\n0mlBP2UWwCq02cUSQmjOPfhuSMPMwvaMcYwjhGXKYA+WLEYdA5FfXLla0h15f0tJx3Q+tBD61+DE\nYls3KTJg6sfAN4CX8/7tpD7eEEKbXnrdpEJbNymSd022fV16zSO1bEoqa275EJZJgxN6r9GiSGbx\nmKSNazuS9iGNPQ8htGmwB2e/KZJZfBH4EbCZpIdJo8MO6GhUIfS5gX7MLGzfC7wvDzFdznYFyzeF\n0F8Ge3CgxagRS1oTOBZ4D2BJ1wLftv14p4MLoV/1YjWkSG/IecCjwN7APsBjwM86GVQI/W6QCYW2\nblKkLLSO7f9Tt3+cpH07FVAIy4KX6K5u0SKKlCyukLSfpOXyti9wRacDC6GfDTKx0NZNmr1I9hxL\n3w35EvCT/Hk54Hmg2YtmIYQmuq2KUUSzd0NWHs9AQliW9FVmUU/S6qT3gV9d9sT2bzsVVAj9ri/H\nWUg6GDicNOnnLcC7gD8C7+1saCH0r25rjyiiSAPnEaRFSRbY3pW0GtLTHY0qhD7Xr12nL9p+QRKS\nVrA9X9KmHY+sJauUlM7to59SgPY/tpR0fGQ5MwHoxHLiKc8LJaVT1gxX25aUTnEv92DXaZHM4sHc\nZnExcKWkJ4EFHY0qhD7Xl20Wtj+aP86QNBdYFfh1J4MKod+Npc1C0uuBOcBbScMbDgLuJo2sfjPp\nP/OP5+ULkXQ0aeHkQeBw222Nk2rYZiFpjeEbqdz3OyC6VUMYgzG2WcwGLrO9ObAlMB84CrjS9jTg\n6rw/fBX13YFT8urrLWuWvd1M4wl7zdBVYRuSNIG0UOtC2x9uLbwQ+lO7jZeSVgN2sn0ggO0B4GlJ\newK75NPOBOaSMozOr6Jue2qriTVwBGk1pLJaIUPoeWNos9iQNCHVGcBWwE2kEdZr216cz1kMrJ0/\n98Yq6pLWBz5Iql+pk/cKoZe8zOsKbSOYSOq+OcX2tqRXL46qPyEvT9hsGY+uXL7wJOBfSI2iIYSs\nUTVk3tzHuHPuY80uXUiq0tfWWr8QOBp4RNI6th+RtC5pWgnohVXUJX0IeNT2LZKmd+o+IfSiRtWQ\nadPXYdr0dV7d/8XMoavA5czgQUnTbN9FWiD5jrwdCJyQ/7w4X9L5VdRz70dDtp8YJe0dgT0lfZD0\nTsmqks567QKuV9Z9fguw0SjJhtANbstbe8Y43Psw4GxJk4B7SV2nE4DzJX2O3HUK47eKeq03RMAG\nwJP5+OrA/aSGloZsf4O03giSdgG+OvJKz+9vNeYQusBWeav5aUtXj2Uot+3bgO1G+Gq3Bud3dhX1\nWm+IpB8Dv7B9Wd7fA/hoo+uaiHVTQ8i67b2PIor0huxQyygAbP+KVMUozPY1tvdsNbgQ+lW/vkj2\ncF7b9KekKsn+tNmaGkJIXhq5W7SrFSlZ7AesBfwCuCh/jrVOQxiDvixZ5PVBDpe0ku3nxyGmEPpe\nt2UERYxaspC0o6R5pJdVkLSVpFM6HlkIfWyACYW2blKkzeJk0ttq/wGp2yZ3hYYQ2tSL0+oVitj2\nA9KQVzsGOhNOu0YbH1bU8iWl87ZSUilrhitfNLOUdLT3iaWkU95MWSuWlE5bAxrHpBerIUUyiwck\nvRsgjxg7HLiz+SUhhGb6NbP4AmmyjTeRukyvAP65k0GF0O96cfnCIpnFNNv71x/IJY3fdyakEPpf\nL7ZZFBln8YOCx0IIBfXVOAtJO5CGdb9R0ldYOnnNKnR40pwQ+l23ZQRFNCsLTSJlDBMYOiXeM8A+\nnQwqhH7XbWMoimj21uk1wDWSzrB9/zjGFELf69c2izl5nQLg1SUCLu9gTCH0vb5qs6jzxtpiJZBm\nyJK0drMLQgjN9evyhYOS3lyrikiaCrzSyaBC6Hd91WZR55vAtZJ+m/d3Bj7fuZBC6H+92GZR5BX1\nX0t6O/Au0tR4X7L9t45HFkIf67b2iCKajbPY3PadOaMw8HD+agNJG9i+eVwiDKEP9VVmAXwFOBg4\nkZEn2921IxGFsAzoqzYL2wfnP6ePWzQhLCPG2mYxfMFxSTOAfwRqy5l9I0+ujaSjgc8Cg8Dhtq9o\n557NqiEfo8n0/bYvaueGIYRSuk6HLzhuYJbtWfUnSdoC2BfYgvTm+FV5NbOWezSbZW8fzgGsRXpH\n5L/y8V2BP5Am7y1BGRPOlDVpzTMlpVNWc045P5f2nl1KOv7MEaWko38vZ1IfWFJSOuNvLNWQugXH\nv0NqLoD07tZIi4/vBZxrewmwQNI9wPYMXVm9kGbVkM/kwK4EtrC9KO+vC5zZ6o1CCEuNsRoy0oLj\nBg6T9GlS9eTIPJhyPYZmDAtJJYyWFRnuPQV4pG5/MWk5wxBCm9od7l2/4DhDSxKnkpYU3RpYROqY\naKT0tU5rrgIul3RODm5fhq5mHEJoUaOu02fn3syzc29pdumoC45LmgNcmncfIv2HX7M+bS4SViSz\nOIy0tulOef/fbP+inZuFEJJGmcXk6dsxefrSNY8XzTxjyPeNFhyXtG6tqYD0vN6eP18CnCNpFqn6\nsQltzlBcZASnJd0MPGv7SkmTJa1i+9nRrs1vq84B3koq+nzWdssNKyH0m5KWLxRLqxTfk7RV3r8P\nOATA9jxJ55N6TgaAQ213phoi6fOkwVlrABuRijGnAu8rkP5s4DLb+0iaCKzUTpAh9JsyRnDangvM\nzZ8/1eS844Hjx3q/ItWQf6auq8X2XZLWGu0iSasBO9k+MF83ADw9hlhD6Bv9Nty75iXbL9UWGcol\nhCLFmA2BxySdAWwF3AQcYfvv7QYbQr/oxeHeRbpOr5H0TWCypPcDF7C0pbWZicC2wCm2twWeB45q\nO9IQ+sggEwtt3aRINF8njTm/ndRochmp0XI0C0nj1m/I+xcyYmZRP0PfRsDGBZIOoWoL8taevquG\n5CrHn21vBvyolYRtPyLpwTwO/S5gN+CO1575P1pJNoQuMTVvNde0dHXfZRa2ByT9pX5avRYdBpyd\n10i9FzionSBD6Dcvvdyfc3CuAdwh6XpSuwOk4Rd7jnah7duA7UY7L4RlzeBAd7VHFFEk4mPyn/Xj\n0Nsa1BFCSAYH+qgaImlF4J9ILY5/Ak7Pr7mGEMaorzIL0mvoLwPXkt6d34I04UYIYYwGlvRXZrG5\n7bcBSDoNuKHJuSGEFrwy2F9tFgO1D7lXpEMhvNChdNuxRtUBDFPW380TpaRS1gxXPmJmKelo9nGl\npJPeZhhnfVYN2VJS/ZulK9bt2/aqI10UQijgxT4qWdjuvawvhF4xMPop3ab3srcQ+kFkFiGEQiKz\nCCEU0oMjliKzCKEKg1UH0LrILEKoQlRDQgiFvFh1AK2LzCKEKkTJIoRQSA9mFkXm4AwhlG2g4DaM\npBUkXSfpVknzJH03H19D0pWS7pJ0RV6zp3bN0ZLuljRf0gfaDTkyixCqsKTgNoztF4FdbW8NbAns\nKuk9pPltr7Q9Dbg67yNpC9KSo1sAuwOnSGrruY/MIoQqDBbcRlC3nMYkYALwJLAnaVoJ8p8fyZ/3\nAs61vcT2AuAe2nxzLjKLEKrQZjUEQNJykm4FFgO/sX0HsLbtxfmUxcDa+fN6pJn2axaS1jxtWTRw\nhlCFMXSd2n4F2Dqv+ne5pF2HfW9Jzaa+7MxapyGEDmjUG3LXXLh7bqEkbD8t6ZfA24HFktbJS3Cs\nCzyaT3sImFJ32fr5WMsiswihCo0yi7dMT1vNZUMnCpK0JjBg+6k8T+77gZnAJcCBwAn5z4vzJZcA\n50iaRap+bAJc307IXZBZrFJCGsuXkAaU93bPiiWlU9ZMWW1VUUdQTjyaPbuUdHxROVPCau8zSkmn\nJe2Ps1gXODP3aCwH/MT21ZJuAc6X9DnSUmkfB7A9T9L5wLx810NtRzUkhJ7R5v9Ltm8nrSE8/PgT\npFX/RrrmeOD49u64VGQWIVQh3joNIRQSL5KFEAqJd0OGymPS75B0u6RzJL2uk/cLoWe0Ody7Sh3L\nLCRNBQ4Gts2LFU0APtGp+4XQU8Yw3LsqnayGPEPKGydLGgQm0+ZgkBD6TlRDlspdOScCDwAPA0/Z\nvqpT9wuhp4zh3ZCqdLIashHwJWAq6WWWlSUd0Kn7hdBTerDNopPVkHcAf7D9OICki4AdgbOHnvar\nus8bk0ajhtDt5uetTS+VFsi46WRmMR/433n8+ouk0WUjjEnfo4MhhNApm+Wt5j9au7zLqhhFdCyz\nsH2bpLOAG4FXgJuBH3XqfiH0lC6rYhTR0UFZtr8HfK+T9wihJ3VZt2gRMYIzhCpENSSEUEhkFiGE\nQqLNIoRQSHSdtqOM2ZeeKCGNMq1aUjprlJROt42yf2cpqWjv40pJx4ccNOY09G8tXhDVkBBCIVEN\nCSEUEl2nIYRCohoSQiikBzOLWL4whCq0+dappNMlLZZ0e92xGZIWSrolb3vUfVfKCuoQJYsQqtF+\n1+kZwPeBs+qOGZhle1b9icNWUH8TcJWkaXn5w5ZFySKEKrQ5+Y3ta0mrpg+nEY6VtoI6RGYRQjXK\nn/zmMEm3STpN0uvzsdJWUIfILEKoRrkT9p4KbAhsDSwiTWfZSFtLF0K0WYRQjUa9IQNzYXBuS0nZ\nrq2YjqQ5wKV5t7QV1CEyixCq0bDrdDpMmF63P7PBeUtJWtf2orz7UaDWU1LaCuoQmUUI1WhzuLek\nc4FdgDUlPQgcC0yXtDWpinEfcAiUu4I6RGYRQjXaHJRle78RDp/e5PxSVlCHnmjgvLfqAIZZUHUA\nw/yl6gCGWVB1AHX+WnUAfaUHMotu+wdfUHUAw0Rm0dh9VQfQV3ogswghdINoswihEr03oYXG0Dg6\n9ptL1d08hJLZHmnI9Wuk3/u/F0x1cuF0O63SkkW3/CWEMP56r2QR1ZAQKlHG3LPjKzKLECoRJYsQ\nQiG9N1VWZBYhVKL3ShYxzqILSHpD3ZRoi+qmSLtZUikZuqS5kt4+yjkLJBVerETSZyR9f+zRLYva\nnP2mQlGy6AK2Hwe2AZB0LPBs/RRpkibYHuvk8Wb0uQzMyDMuNTs/tKX3ShaRWXQnSfp34EXShCa/\nl/QM8JztE/MJfwY+aPsBSZ8EDgMmAdeR3i5sOM+ipFOA7YAVgQttz6j7+mt5wtcXgP1t3yvpjaQJ\nVjbI53zJ9h/K+3GXRb3XGxLVkO5l0rRoO9g+ssH3SNoc+Diwo+1tgFeAA0ZJ+5u2twO2AnaR9A91\n3z1le0vgB8DJ+dhs4CTb2wP7AHPy8Rgn07aohoRyXTDK/AMC3ge8HbhREqTSwiOjpLuvpINJ//7r\nkmZ//nP+7tz853nASfnzbsDmOX2AVSSt1MLPEV4jqiGhXPVjggcYWhJcoe7zmba/USRBSRsCRwLv\nsP20pDOGpVWvllEJeKftl4elFW0WbeuuUkMRUQ3pHQuAbQEkbUuaoNXA1cA+uV0BSWtI2qBRIqQl\n3p8HnpG0NrBH3XcirTNB/rPWLnEFcPirJ6VZmWrnh7aUP713p0XJorvV/8/9c+DTuWHzOvJEFrbv\nlHQMcIWk5Ui/YYcCD4yYoH2bpFuA+cCDwO+G3W91SbeRGldrszIdDvwwH58IXJPvUaSHJYyo90oW\nlb51GsKyKFXffl7w7I91zQuXUbIIoRK913UamUUIleiu9ogiIrMIoRK912YRvSEhVKL93hBJu0ua\nL+luSV8fn3ijZBFCRdorWUiaQBpduxtpKcIbJF1i+84SgxtRZBYhVKLtNovtgXtsLwCQdB6wFxCZ\nRQj9qe02izeRxsfULATeOeZwCojMIoRKtN11WtnAqMgsQqjEjHYvfAiYUrc/hVS66LgYwRlCD8kz\np/2F9Lbxw8D1wH7RwBlCGML2gKQvApcDE4DTxiOjgChZhBAKikFZIYRCIrMIIRQSmUUIoZDILEII\nhURmEUIoJDKLEEIhkVmEEAqJzCKEUMj/ByrA+XtGz8biAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x18ffbe4d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import metrics as metr\n",
    "plt.matshow(metr.confusion_matrix(label3_valid, pred3))\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('Predicted label')\n",
    "plt.xlabel('True label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAD0CAYAAACM5gMqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHxVJREFUeJzt3Xm4HFW57/Hvj4QpTBKRQBgM8+BhCk6gQFDwIiLqgSsG\nVETFAQUc8AjIlcSLXPFeGY4K92qAA4ogonLhCEJEgzOCTGEIASQyJiAKAWRIwu/8sVYnvXf27l27\nd/Wu6s77eZ560l1dverdO6k3tYZaS7YJIYShrFR1ACGE7hDJIoRQSCSLEEIhkSxCCIVEsgghFBLJ\nIoRQSCSLCkhaXdKVkp6S9MMRlHOYpGvKjK0qkvaQNKfqOMLgFOMsBifpUOBzwDbAM8CtwFdt/26E\n5X4A+DSwm+2XRxxozUl6GdjS9l+qjiW0L+4sBiHpc8AZwCnA+sAmwLeBA0so/tXA3BUhUTTRoB9I\nY0czkKpJ8nC2quNdynZs/TZgHdKdxEEtjlkVOBN4JG9nAKvkz6YAD5PuShYAjwIfyp9NB14EXsrn\n+DAwDfheU9mTgJeBlfL7DwH3AwuBvwCHNu3/TdP3dgduBJ4C/kS6c2l8Ngv4CvDbXM41wCsH+dka\n8X8BeDzH/25gf2Au8CRwfNPxrwf+APwjH/tNYOX82a/zz/Js/nn/e1P5/wY8BlyQ9z2Uv7NFPscu\n+f1E4Algz6r/bZT078unFNzSJVp9zLbjzmIQuwGrAT9tccyXSBfJTnl7PXBS0+cTgLVJ/9A/Anxb\n0jq2TwZOBS6xvZbt80j/KAYkaQ3gLGA/22vn2G4d4LjxwM9ICWw8cDrwM0nrNh02lZRg1gdWAY5r\n8fNNICXEDYEvAzOAw4BdgD2AL0t6dT52MXAs8Moc31uBowBs75mP2TH/vD9qKn9dYFPg480ntn0/\n8EXg+5JWB84Hzrf96xbxdpWVC251EsliYK8E/ubW1YRDga/Y/pvtv5HuGD7Q9Pmi/PkS21eT/mfd\nJn8m+t6WD3qLnr0M7CBpddsLbN81wDHvAO6xfZHtl21fAsxhWbXJpAvuPtsvAJcCO7c45yJS+8wS\n4IekBHSm7efy+e9qfN/2zbb/lM/7V+A7wF4FfqaTbS/K8fRhewZwH+kOaQIpOfeMsQW3OolkMbAn\ngfUktfr9TAT+2vT+wbxvaRn9ks0/gTWHG4jt54BDgE8Aj0r6T0nbDHDoxBxDs7/2i2l+0+vnh4jn\nSed75nwspCpV8/fXAJC0dY7rMUlPA18lJdxWnrD90hDHzABeA3zT9qIhju0qqxfc6iSSxcD+QGpX\neE+LYx4ltS00bJr3teNZYFzT+w2aP7R9re235f1zgO8OUMYjpIbTZq/O+zvtHNKdxpa21yHdBQz1\nb6tlw52kNUlVqhnA9H7Vqa4X1ZAeYftpUj3925LeJWmcpJUlvV3Safmwi4GTJK0nab18/PfaPOWt\nwJ6SNpG0DnBC4wNJ6+cY1iBVDZ4DlgxQxtXA1pKmShor6RBgW+A/m44ZqrrTrjVJjZf/lLQt8Ml+\nny8gNVoOx1nAn2x/jNQW839HHGWNtFsNkXSepAWSZjftGy9ppqS5kq6V9Iq8f5Kk5yXdkrezm76z\nq6TZku6VdFaRmCNZDML26aTejJNIPQIPkhrtGo2epwA3Abfn7aa8b2kRrYpv/tz2L0jtAreTejOu\nbPp8JeCzpDuEJ0mNi5/sX47tJ4EDgM8DfyM1Xh5g+++DxGSGjrHV+2bHkdpwFpLaKy7pd/w04AJJ\n/5B0cItzG0DSu4C3sezn/BwwWdLUFjF0lRHcWZwP7Ndv3/HATNtbA9fl9w332d4lb0c17T8H+Ijt\nrYCtJPUvczm1HpSVf4AzgTHADNunDfGVTsayCXAhqSfBwHds/3tV8eSYxpCS1MO231lxLK9gWRuD\ngQ/b/mOF8ZwAvJ/UkDobOML2i1XF00ySLyl47PsA233uCCVNAq60vUN+PwfYy/YCSRsAs2xv2/+4\npu9vCPzS9nb5/fuAKbY/0SqW2t5Z5AvhW6Qsuj0wVdJ2FYa0CPis7dcAbwQ+VXE8kLor72KI+v8o\nOQu4Kv8D3BG4u6pA8kVyJDA5XyhjSNddbZTcZjHBdqPxeQGp96hhs1wFmSXpzXnfRqRxLg2P5H0t\n1TZZkMYt3Gd7Xm4JvwR4V1XB2J5v+9b8+lnSxTCx9bc6R9LGpEFSM+hcW0TRWNYB9shjRrC9OLf7\nVGUhKbmPy6NDxzE6Db2FdaqBM/dgNf7zeBTYxPYupKrcDySt1W7MdevKbbYR8FDT+4eBN1QUSx/5\nf65dgBsqDOMM0gjLtSuMoWEz4AlJ55MGqP0ZONb2P6sIxvbfJX2D1M70PHBNbheqjcG6RW/L2zAt\nkLSB7fm5ivE4QO6afim/vlnS/cBWpMS5cdP3N6ZAMq3znUUdbq2Xk7v0LiNdDM9WFMMBwOO2b6Hi\nu4psLDAZONv2ZFKPzfGtv9I5krYAPkPq2p4IrCnpsKriGchgvR+7ksb/N7aCrgAOz68PBy4HyD11\nY/LrzUmJ4i+2HwMWSnqDJJEGE14+1EnqnCweIT281bAJfetZo07SysCPge/bHvKX20G7AwdKeoDU\nhfsWSRdWGM/DpEbWG/P7y0jJoyqvBX5v+0nbi4GfkH5ntdFuNUTSxcDvgW0kPSTpCOBrwL6S5gJv\nye8B9gRuk3QL8CPg47afyp8dRarC3kuq7v98qJhr2xuS65r3kJ4zeJQ07Heq7UoaznIGvoA0svGz\nVcQwEEl7AcfVoDfk18BHbc+VNA1Y3fYXK4plJ+Ai4HXAC8B/kMZsfLuKePqTVLib6I0s3xtSldq2\nWdheLOnTpKcjxwDnVpUosjeRuuJuz5ka4IQiGXkU1CHjHw1cJGkV0hOyR1QViO3b8p3WTaSu05tJ\n4z9qo26jM4uo7Z1FCL1Kkos2Yu5E3FmEsELrxjuLSBYhVKBuT5QWEckihArEnUUIoZBuvPC6MeYQ\nut7KRa+8xR0NY1gqTRa1mrk4hBEaTq/F2EgWw+chZiiYNhum7dD6GF18SusDCivS7HQN8N+GOGZh\nCbEUNYs0MXYrZdWQu+33M4uhfzdQzu/npKEPaT7jmBJOOcoqTxYhrIgK31nUSBeGHEL3W3nVqiMY\nvo4+SCZpP0lz8jx/bT0nMGX9sqMaqeFOJdlpk6oOoJ86/X4mVR3A4LpwLYCOhdM009U+pCdIb5R0\nxXCf75gyYehjRteWVQfQz6SqA+inTr+fSVUHMLiaJYIiOnlnUauZrkKolS68s+hkshhopqsh5/kL\nYYUwpuBWI53MXTGGIoTB1OyuoYhOhlxopqtps5e9nrJ+HdsoQhjIX4AH2v96F/aGdDJZ3ERavGQS\naaarQ0irePcx1ICrEOpp87w1/Gp4X487i2VqONNVCPURyaIv21eT1uAMITSrWeNlEV2Y30LoAV14\n5XVhyCH0gC688uq8bkgIvWsEg7IkHStptqQ7JB2b942XNFPSXEnX5oWqG8efkB+5mCPpbe2GHMki\nhCqsWnDrR9K/AB8lrYmyE3BAXoHteGCm7a2B6/J7JG1P6oncnrTI+NmS2rruK78Z0sUnj7iMk4c5\nl8BgplPWvBhlKWseikU1K6csdfv9DEP7V962wA22XwCQdD1wEHAgsFc+5gLSZB7Hkx6xuDg/cjFP\n0n2kRzGKrnO0VNxZhFCF9od73wHskasd44D9SQsbT7C9IB+zAGgMb5xI38GQbT92UfmdRQgrpDav\nPNtzJJ0GXEtagPpWYEm/YzzElJVtPYoRySKEKgxy5c2aD7MWDPxZg+3zgPMAJH2VdLewQNIGtudL\n2hB4PB/e/7GLjfO+skIOIXTUIIOypmyUtobpty9/jKT1bT8uaVPgX0nrJ28GHA6clv+8PB9+BfAD\nSaeTqh9bkRYZH7ZIFiFUYWRX3mWSXklqmT3K9tOSvgZcKukjwDzgvQC275J0KXAXaa7wo9zmAseR\nLEKowmrtf9X2ngPs+ztpVrqBjj8VOLX9MyaRLEKoQjwbEkIopAuvvC4MOYQe0IVXXheGHEIPiGpI\nCKGQLrzyujDkEHpAF155XRhyCD0gJuwNIRTShVdeF4YcQg/owiuvC0MOoQdEb0gIoZAuvPJqEPLq\nIy6hrBmu/NFyZtzSjG+UUk7vznBV1pK3z5RUTlfNlFWZLgw5hB4Q1ZAQQiEjeOq0KpEsQqhCF155\nXRhyCD2gC6shHZ3dW9Imkn4l6c68IMoxnTxfCF1jBIsMVaXT4SwCPmv7VklrAn+WNDNWUw8rvJol\ngiI6vYr6fGB+fv2spLtJ6xhEsggrti6shoxafpM0CdgFuGG0zhlCbUVvyMByFeQy4Fjbz47GOUOo\ntbizWJ6klYEfA9+3ffnyR8xser05sEWnQwqhBPPy1qY2rzxJ2wCXNO3aHPgysC5pweQn8v4TbV+d\nv3MC8GHSymXH2L52FEMuRpKAc4G7bJ858FH7djKEEDpkUt4arh/e19tfvvAeUnWevBr6I8BPSMng\ndNunNx/fbxX1jYBfSNra9svDPXenF0Z+E/B+YG9Jt+Rtvw6fM4T6K6frdB/gPtsPAcpbf0tXUbc9\nD2isot5WyB1j+7fESu0hLK+cNov3ARfn1waOlvRB4Cbg87afIvU+/rHpO22voh4XcghVGOGdhaRV\ngHcCP8q7ziGtd7oz8BjQ6tHnWL4whK4xyBycs/6YtgLeDvzZ9hMAthurpiNpBnBlfhurqIfQ1Qa5\n8qa8OW0N0/990BKmsqwKgqQNbT+W374HmJ1fxyrqIXS1EVx5ktYgNW4e2bT7NEk7k6oYDwAfh3JX\nUddg35P0+Rbfc/8umrZOLplSZrkqa6ajN5VSiqe+rZRydPHJpZQDW5ZUzl9LKqes/6PWLqmcv5dQ\nxknYHqg3YjmSmioNQxy7PoXL7bRWf2tr0WZDSAihNffSCE7b00YxjhBWKEu6sAFgyK5TSdtIuk7S\nnfn9jpLKmdk2hBXUkrHFtjopMs7iu8CJwEv5/WxSS2wIoU0vrrpKoa1OiuSucbZvSI95pJZNSXWb\nWz6ErrJkTPc1WhRJFk9IWtqcLulg0gixEEKblnThM+pFksWnge8A20p6lNSHe1hHowqhxy3uxWRh\n+37grXkgyEq2y1oGKoQV1pIuHA85ZMSS1gNOBt4MWNJvgK/YfrLTwYXQq7qxGlKkN+QS4HHgX4GD\nSTPx/LCTQYXQ65YwptBWJ0XuhTaw/T+b3p8i6ZBOBRTCiuBF6tUtWkSRO4trJU2VtFLeDgHamsMv\nhJAsYWyhrU4GjUbSsyx7NuQzwPfy65WA54BWD5qFEFqoWxWjiFbPhqw5moGEsCLpqWTRTNK6pEkz\nli6NYvvXnQoqhF7Xk+MsJB0JHEOamusW4I3AH4C3dDa0EHpX3dojiijSwHksaerwebb3Jq1Z8HRH\nowqhx/Vq1+kLtp+XhKTVbM/JqyKVpIxn0iaVUAa0OTXhcsqa4crnTy+lHB1R1oxbK9esnIUllbND\nSeUU91IXdp0WSRYP5TaLy4GZkv7BiNZtCyH0ZJuF7ffkl9MkzSJNfPjzTgYVQq/rxjaLVuMsxg+w\n+/b855qUM8tpCCukurVHFNEqvd3M4BP2mrR685AkjSEtp/aw7XcOL7wQetNIkoWkVwAzgNeQrsUj\ngHtJz2y9mtRM8N68fGHnV1G3PamdAgdwLGnNgrVKKi+ErjfCNouzgKtsHyxpLLAG8CVgpu2vS/oi\ncDxwfNesoi5pY2B/UhasxdoHIdTBS6xaaOtP0jrAHrbPA7C92PbTwIHABfmwC4B359elraLe6YWR\nzwC+AAw7i4XQy0YwzmIz0lSX50u6WdJ388RUE2wvyMcsACbk1xNJK6c31G8VdUkHAI/bvoW4qwih\nj8WMKbQNYCwwGTjb9mTSQ53HNx+QlydstUBYuauoD9Ib0hzQUL0huwMHStqf9EzJ2pIutP3BvofN\nano9ifIGWIXQSbOBO9r+9mBdp3NmLeCeWQsG/Cx7mNRZcGN+fxlwAjBf0ga250vakDRhFYzSKuqN\n3hABmwL/yPvXJS16uVmrgm2fSFpvBEl7AcctnygApgwz5BDqYAf6jvy8ZFjfHqw3ZKspE9lqysSl\n76+Y3jch5WTwUG6knEtaIPnOvB0OnJb/vLxRBJ1eRb3RGyLpu8BPbV+V37+dtKT7cMW6qSFkIxxn\ncTRwkaRVgPtJXadjgEslfYTcdQrlrqJeZBjZbraXLu1u+2pJ/3s4J7F9PXD9cIMLoVeNJFnYvg14\n3QAf7TPI8acCp7Z9wqxIsng0r236fVKV5FDarPOEEJIXB+gWrbsivSFTgfWBnwI/ya9jrdMQRqAn\nH1HP64McI2kN28+NQkwh9Ly6JYIihryzkLS7pLuAOfn9TpLO7nhkIfSwEYyzqEyRNoszgf2A/w+p\ncSV3hYYQ2tRTj6g3s/2g1GcQ5uLOhNOueSWVs3qtyilrhiv/rKQZt97xtVLKKW+Gq7VLKmd2SeUU\n143VkCLJ4kFJbwLI/brHAHd3NKoQelyvJotPkh6J3YjUZXot8KlOBhVCr+vG5QuLJIutbR/avCPf\nafyuMyGF0Pu6sc2iyDiLbxXcF0IoqKfGWUjajfTk6KskfY5lj5mvRefnwQihp9UtERTR6l5oFVJi\nGEPfKfEWAgd3MqgQel3dxlAU0eqp0+uB6yWdb/uvoxhTCD2vV9ssZuTZhIE0KY6kazoYUwg9r6fa\nLJq8qjGlOKQZsiRNaPWFEEJrvbp84RJJr25URSRNIibgDWFEeqrNosmXgN9I+nV+vyfwsc6FFELv\n68Y2iyKPqP9c0q7AG0lT433G9t86HlkIPaxu7RFFtBpnsZ3tu3OiMPBo/mhTSZvavnlUIgyhB/VU\nsgA+BxwJfIOBJ9vduyMRhbAC6Kk2i8YkvbanjFo0IawgeqrNQtJBtJi+3/ZPOhJRCCuAkXadShoD\n3ERacOidkqYBHwWeyIecaPvqfGxnV1EH3klKFuuTnhH5Zd6/N/B70uS9JShjopjnSygDypvTp6x4\nyqF3nFJKOX7/8UMfVIC+X86kPuVZNOpnLKEacixpLZDGoxgGTrd9evNBo7KKuu0P2T6C9IzI9rYP\nsn0Q8Jq8L4TQpiWMLbQNRNLGwP7ADJY94CkGXlN4VFdR3wSY3/R+AWk5wxBCm0Y43PsM4Av0HRxp\n4GhJt0k6t+kRjVFdRf0XwDWSPiTpCOAqYGY7JwshJO0mC0kHAI/bvoW+dxLnkNYf3hl4jNSLOZiO\nLV94NGlt0z3y+/9n+6ftnCyEkAx21/DMrJt5dlbLIUy7AwdK2h9YDVhb0oXNi45LmgFcmd+Oyirq\nANi2pJuBZ2zPlDRO0lq2nxnqu/lWaAapncPAh23/sZ1AQ+glgy1fuMqU3Rg/Zbel7xdMP7fP57ZP\nBE4EyEtyHGf7g5I2tP1YPuw9LJuyvPOrqDdI+hhpcNZ4YAtSZjoHeGuB8s8CrrJ9sKSxwBrtBBlC\nrylpBKdYVqX4uqSd8vsHgI/D6K+i/ilS6+kf88nnSlp/qC9JWgfYw/bh+XuLgafbCTKEXlNGsrA9\nC5iVX3+gxXGjtor6i7ZfbCwylO8QimSmzYAnJJ0P7AT8GTjW9j/bDTaEXtGNw72L9IZcL+lLwDhJ\n+wI/YlnjSStjgcnA2bYnA88B5YzqCaHLjWScRVWKRPNF0jDS2aR60FWkRsuhPEwainpjfn8ZAyaL\n5l7YzUnNIiHU3TxGsmxmrz112qhy3GF7W+A7wynY9nxJD+WhpXOBfYA7lz9y3+EUG0JNTMpbw/XD\n+nbPJQvbiyXd0zyt3jAdDVyU10i9HziinSBD6DUvvtR9T0wUqYaMB+6U9CdSuwOk4RcHDvVF27cB\nrxtBfCH0pCWL69UeUUSRiE/KfzYPLW2rnzaEkCxZ3EPVEEmrA58AtgRuB86zPfrP8obQg3oqWQAX\nAC8BvyE9Drs96Rn6EMIILV7UW8liO9s7AEg6F7ixxbEhhGF4eUlvtVksnTYq94p0KIQyZpVau4Qy\noLwZk8oqp4xZxKCsGcDKmuHKb5peSjn63ddKKadvF+go6bFqyI6Smp8sXb3pvW2XdYWGsOJ5oYfu\nLGx3X+oLoVuUNd3rKOq+9BZCL4hkEUIoJJJFCKGQLhyxFMkihCosqTqA4YtkEUIVohoSQijkhaoD\nGL5IFiFUIe4sQgiFRLIIIRTShcmiyIS9IYSyLSq49SNpNUk3SLpV0l2S/lfeP17STElzJV3btNYp\nkk6QdK+kOZLe1m7IkSxCqMKSgls/tl8A9ra9M7AjsLekN5Mmw55pe2vguvweSdsDh5CmmNgPOFtS\nW9d9JIsQqrC44DaAprV3VgHGAP8ADiTNQUP+89359buAi20vsj0PuI+0aNiwRbIIoQovFNwGIGkl\nSbcCC4Bf2b4TmGB7QT5kATAhv55IWpaj4WHSmqfDFg2cIVRhsAbOubPg3lktv2r7ZWDnvEToNZL2\n7ve5JbWaJ7dja52GEMo2WLLYfEraGq4afKIg209L+hmwK7BA0gZ5vZ4NgcfzYY8AmzR9beO8b9hq\nkCzKmA2qrB9jfEnlLCypnLL618qacasc+t0ppZTj/1POapg67uxSyhmWNv9qJa0HLLb9VJ5Ue19g\nOnAFcDhwWv7z8vyVK4AfSDqdVP3YCvhTO+euQbIIYQXU/lOnGwIX5B6NlYDv2b5O0i3ApZI+QlpX\n8b0Atu+SdClwFylFHWU7qiEhdI02nzq1PZu04Hj//X8nLRE60HdOBU5t74zLRLIIoQrxIFkIoZAY\n7t1XHmZ6p6TZkn4gadVOni+ErtHmcO8qdSxZSJoEHAlMzosVjQHe16nzhdBV2hzuXaVOVkMWknLj\nOElLgHG02b8bQs+JasgyuXX2G8CDwKPAU7Z/0anzhdBVRvBsSFU6WQ3ZAvgMaW24icCakg7r1PlC\n6Cpd2GbRyWrIa4Hf234SQNJPgN2Bi/oeNrPp9ebAFh0MKYSyzM1bm14sLZBR08lkMQf4H3lI6guk\nASMDDDPdt4MhhNApW+et4arhfb1mVYwiOpYsbN8m6ULgJuBl4GbgO506XwhdpWZVjCI6OijL9teB\nr3fyHCF0pZp1ixYRIzhDqEJUQ0IIhUSyCCEUEm0WIYRCouu0HSuXUMYzJZRRZjn1mpkKnq9ZOTuU\nUoqOO6uUcvyWo0Zchn45zC9ENSSEUEhUQ0IIhUTXaQihkKiGhBAKiWQRQiikC9ssYvnCEKrwYsGt\nH0nnSVogaXbTvmmSHpZ0S97e3vRZKSuoQ9xZhFCN9qsh5wPfBC5s2mfgdNunNx/YbwX1jYBfSNo6\nL384bHFnEUIV2pz8xvZvSKum96cB9pW2gjpEsgihGuVP2Hu0pNsknSvpFXlfaSuoQySLEKpR7hyc\n5wCbATsDj5Hmvh1MW0sXQrRZhFCNwRLBklnw8qxhFWW7sWI6kmYAV+a3pa2gDpEsQqjGoF2nU/LW\nMH3IoiRtaPux/PY9QKOnpLQV1CGSRQjVaLM3RNLFwF7AepIeAk4GpkjamVTFeAD4OJS7gjp0RbK4\nD9iy6iCa/IU0C3ld1O33M4+0+kMd3Ev6z7R32J46wO7zWhxfygrq0BUNnPdXHUA/D1QdQD91+/3M\nqzqAJvdVHUBP6YJkEUKogy6ohoTQi7rv4RCNoL1j5CeXqjt5CCWzPdAoyuWkf/f/LFjquMLldlql\ndxZ1+SWEMPq6784iqiEhVKKs+UxHTySLECoRdxYhhEK6b6qsSBYhVKL77ixinEUNSHpl0yxHjzXN\nenSzpFISuqRZknYd4ph5ksYPo8wPSfrmyKNbEZX72OloiDuLGrD9JLALgKSTgWeaZz2SNMb2SCeP\nN0M/nmwGnkSl1fGhLd13ZxHJop4k6T+AF0hzFPxO0kLgWdvfyAfcAexv+0FJ7weOBlYBbiA9MDTo\n1GmSzgZeR1o67TLb05o+/rc8h+PzwKG275f0KtKcCZvmYz5j+/fl/bgrou7rDYlqSH2ZNNPRbrY/\nP8jnSNoOeC+wu+1dgJeBw4Yo+0u2XwfsBOwl6V+aPnvK9o7At4Az876zgDNsvx44GJiR98c4mbZF\nNSSU60dDPFIs4K3ArsBNkiDdLcwfotxDJB1J+vvfkDSh6x35s4vzn5cAZ+TX+wDb5fIB1pK0xjB+\njrCcqIaEcjWPCV5M3zvB1ZpeX2D7xCIFStoM+DzwWttPSzq/X1nNGolKwBtsv9SvrGizaFu97hqK\niGpI95gHTAaQNJk056KB64CDc7sCksZL2nSwQoC1geeAhZImAG9v+kykqePJfzbaJa4Fjll6UJpo\npXF8aEub03tXKO4s6q35f+4fAx/MDZs3APcA2L5b0knAtZJWIv0LOwp4cMAC7dsk3QLMAR4Cftvv\nfOtKuo3UuNqYaOUY4Nt5/1jg+nyOIj0sYUDdd2dR6VOnIayIUvXtxwWPPqg2D1zGnUUIlei+rtNI\nFiFUol7tEUVEsgihEt3XZhG9ISFUov3eEEn75VXR75X0xdGJN+4sQqhIe3cWksaQRtfuQ1pd7EZJ\nV9i+u8TgBhTJIoRKtN1m8XrgvrwqOpIuIa2WHskihN7UdpvFRqTxMQ0PA28YcTgFRLIIoRJtd51W\nNjAqkkUIlZjW7hf7r4y+CenuouNiBGcIXSTPnHYP6WnjR0mrok+NBs4QQh+2F0v6NHANMAY4dzQS\nBcSdRQihoBiUFUIoJJJFCKGQSBYhhEIiWYQQColkEUIoJJJFCKGQSBYhhEIiWYQQCvkvhVr+lSva\n70sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x18e64a750>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import metrics as metr\n",
    "plt.matshow(metr.confusion_matrix(label4_valid, pred4))\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('Predicted label')\n",
    "plt.xlabel('True label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAD3CAYAAAAKcnGEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XuYHVW55/HvjwTkKhiRABJMUAIE5aqgKBgEGXAU5MDI\nzUGRgxxRQNGjoIwkHswIz2OEUWEOchlULiKiBxSFgAbxeOQid0K4SYAQCMgd5JIOv/ljrSY7ne7d\ntXfX7tq1836ep57sql216u0k9XatVavWkm1CCGE4K1QdQAihHiJZhBAKiWQRQigkkkUIoZBIFiGE\nQiJZhBAKiWQRQo1IOlvSQkm3N2wbJ2mWpHskXSlprbx9oqSXJN2cl9MajtlW0u2S7pV0apFzR7II\noV7OAXYfsO1YYJbtycDVeb3ffba3zssRDdtPBw61vTGwsaSBZS4jkkUINWL7WuDpAZv3BM7Nn88F\nPt6sDEnrAWvYvj5v+vFwx0Aki0pIWkXSZZKekfSzEZRzkKQryoytKpJ2lDS36jhqarzthfnzQmB8\nw3eTchVktqQP5G1vBeY37PNI3tZUJIsmJB0o6UZJz0taIOlySe8voeh9gXWAcbb3a7cQ2+fZ/m8l\nxNNRkl6TtFGzfWxfa3vT0YqpVzm9v9H/DscCYILtrYFjgPMlrdFu2ZEshiDpGOB7wImkC3sC8EPS\nLd9IvQ24x/ZrJZRVFxryC2nsaAZSNUluZSlQ5EJJ6+ay1wMeB7D9qu2n8+ebgPuBjUl3Ehs0HL9B\n3tac7VgGLMCawPPAPk32eQNwSv5LfoSUWFbK300l3eYdQ7otXAB8On83HXgFeDWf4zPANOAnDWVP\nBF4DVsjrn87/0M8BfwMObNh+bcNxOwA3AM8A1wPva/huNvAt4E+5nCuANw/xs/XH/6/5P94CUp32\nI8A9wJPAsQ37bwf8F6kuvQD4PrBi/u6P+Wd5If+8/6Oh/K8Cj5Lq2VOBh/Mxb8/n2Dqvrw88AexU\n9f+Nkv5/+cSCC/lmYcDxE4HbG9ZPBr6WPx8LfCd/XhsYkz9vlP/O18rr1wHbk5L45cDuw8UddxaD\nex+wMvDLJvt8g3SRbJmX7YDjG74fD7yR9B/9UOCHkta0fQIwA7jQ9hq2z2bJbeMyJK0GnEr6x3xj\nju2WQfYbB/yGlMDGATOB30h6U8NuB5ASzDrASsBXmvx840kJcT3gm8CZwEHA1sCOwDclvS3v2wcc\nDbw5x7cLcASA7Z3yPlvkn/fnDeW/CdgQOLzxxLbvB74G/FTSKqQnAOfY/mOTeGtlxYLLQJIuAP4M\nbCLpYUmHAN8BPizpHuBDeR1gJ+BWSTcDPwcOt/1M/u4I0r/pvaQnJr8bLubl6vavBW8G/u7m1YQD\ngS/Y/juApOnAv5MuLIBFwLdyGb+V9AKwCek3vlj6tnzIW/TsNeBdkuY7NWQtHGSf/w7cbfu8vH6h\npKNY0lJu0gV3X473IppXqRYB37bt3Ah7BnCK7ReBOZLmAFsBDzrd4vZ7UNIZwAdJSa7Zz3SC7UXA\nImnpvwLbZ0r6GOnvazEpOfeMdi882wcM8dWug+x7CXDJEOX8FXhXK+eOO4vBPQmsLanZ38/6wIMN\n6w/lba+XMSDZ/ANYvdVA8sW5H/AvwAJJv5a0yRDxPDRg24MDYnqs4fNLw8TzpPP9at4Xlk5SLwGr\nAUianON6VNKzwLdJCbeZJ2y/Osw+ZwKbA9/PSaVnrFJw6SaRLAb3X6R2hb2b7LOAVHfst2He1o4X\ngFUb1tdt/NL2lbZ3y9vnAj8apIxHSA2njd5GkYarkTsdmAO8w/aapLuA4f5vNW24k7Q6qUp1JjB9\nQHWq9tqthlQpksUgbD9Lqk78UNJeklaVtKKkPSSdlHe7ADhe0tqS1s77/6TNU94C7CRpgqQ1geP6\nv5C0To5hNVLV4EXSbflAvwUmSzpA0lhJ+wGbAr9u2Ge46k67Vic1Xv5D0qbA5wZ8v5DUaNmKU4Hr\nbX+W1Bbzf0ccZRcZW3DpJpEshmB7JulpxvGkJwIPkRqF+hs9TwRuBG7Ly4152+tFNCu+8XvbVwE/\ny+XcAFzW8P0KwJdIdwhPkhoXPzewHNtPAh8Fvgz8ndR4+VHbTw0Rkxk+xmbrjb5CasN5jtS2ceGA\n/acB50p6WtK+Tc5tAEl7Abux5Oc8BthG0lD19dqp452FllRLu0/ur34KMAY40/ZJwxzSyVgmkLrF\nrkP6T32G7f9TVTw5pjGkJDXf9scqjmUtlrQxGPiM7b9UGM9xwCdJDam3A4fYfqWqeBpJ8oUF990f\nsN2pO8KWdO2dRb4QfkB6aWYKcICkzSoMaRHwJdubA+8FPl9xPJAeV85hmPr/KDkVuNz2ZsAWwF1V\nBSJpInAYsI3td5F+2exfVTyDqeOdRdcmC1K/hftsz8st4RcCe1UVjO3HbN+SP79AuhjWb35U50ja\ngNRJ6kw61xZRNJY1gR1znxFs9+V2n6o8R0ruq+beoasyOg29hUWyKNdbgYcb1udT4GWX0ZB/c21N\n6gVXle+Relh2Q5fxScATks6RdJOkH0laddijOiS303yX1M60AHgmtwt1jXh0Wq5uuLVeRn6kdzFw\ndL7DqCKGjwKP276Ziu8qsrHANsBptrchPbE5tvkhnSPp7cAXSY+21wdWl3RQVfEMJp6GlOsR0stb\n/Saw9Gu1o07SisAvgJ/a/lWFoewA7CnpAdIj3A9J+nGF8cwnNbLekNcvJiWPqrwb+LPtJ233kXox\n7lBhPMuIaki5biSN4DNR0kqkXoyXVhWMUn/ks4A5tk+pKg4A21+3PcH2JFLD3e9tH1xhPI8BD0ua\nnDftCtxZVTykjmvvzeOGKMczp8J4llHHO4tui+d1tvskfYH0duQY4CzblbWwA+8nPYq7Lb+YA3Bc\nkRdwRkE3VNmOBM7Lif1+4JCqArF9a77TupHUpnMTqf9H1+i2u4YiurqfRQi9SJJvLbjvlnRPP4uu\nvbMIoZfV8c4ikkUIFei2x6JFRLIIoQJxZxFCKKSOF14dYw6h9lYseuX1dTSMllSaLAqOXBxCLbTy\n1GJsJIvWeVLz76c9DdOGGSNJD5zYfIfCijQ7XQEMN1XHS8N8X1SRkeRmkwbGbmY0m9NmAR8eZp+y\n/n6G+7mKxALlxDO9pb1XHFPCKUdZ5ckihOVR4TuLLlLDkEOovxXfUHUErevouyGSdpc0V2la96+1\nU8bUlcuOaqRaHUqy0yZWHcAATWcpHGXdFMsANXw5pGPJoqyRrqZ2Xe+Vd1QdwAATqw5ggG5Kpt0U\nywAjSBaSjpZ0u6Q7JB2dt42TNEvSPZKuzMMc9u9/XP6FPVfSbu2G3Mk7i64a6SqErtJmspD0TuCf\ngfeQXh35aB6/41hglu3JwNV5HUlTSG9sTyH94j5tmPlwhtTJZNG1I12FULkxBZdlbQpcZ/tl24uB\na4B9WDLzHPnPj+fPewEX2F5kex5wH+kXecs6mSyiD0UIQ2m/GnIHsGOudqxKGod1A2B8ntoS0jwt\n4/Pn9Vl60Ki2f2l3sgml0EhX055e8nnqyt3YRhHCYOblpU1tPg2xPTdPdHUlafjCWxgw6VSen7aV\nOWEK6WSyeH2kK9KgqfuRZvFeynAdrkLoThNZunH5mtYOH+LKm/0czH6++aF5FPWzASR9m/RLeKGk\ndW0/Jmk90sRYsOwv7Q1oc6TzjiWLLhzpKoTuMcSVN3VcWvpNf3TZfSStY/txSRsC/0Sax2YS8Cng\npPxn/xixlwLnS5pJqn5sTJqZvqyQy2H7t6Q5OEMIjUbW3ftiSW8mvQ9whO1nJX0HuEjSoaT60ScA\nbM+RdBFpDNK+vH/XVUNCCEMZwZVne6dBtj1FGph4sP1nADPaP2MSySKEKtTwyqthyCH0gBpeeTUM\nOYQeUMMXySpPFnrghBGXcQLHlxAJTKescTGKjEMxmsoaP6KskSPLKqesn6sClV95rathyCH0gBj8\nJoRQSA2vvBqGHEIPqOGVV8OQQ+gBUQ0JIRRSwyuvhiGH0AO6brjI4UWyCKEKUQ0JIRRSwyuvhiGH\n0ANqeOXVMOQQekBUQ0IIhdTwyqthyCH0gBpeeTUMOYQeEG+dhhAKqeGVV8OQQ+gBNbzyahhyCD0g\nnoaEEAqp4ZXXBSGPfNSk6XynhDjA2x9bSjm67pxSykmz0JWhrBGlyhoBbPzwuxRS1n/fx4ffpWxd\ncOW1qoYhh9ADalgN6eTEyCGEoaxccBlA0iaSbm5YnpV0tKRpkuY3bN+j4ZjjJN0raa6k3doNOe4s\nQqhCm1ee7buBrQEkrUCat/QS4DPATNszG/eXNIU0z/AU0vSFV0mabPu1Vs8ddxYhVGFMwaW5XYH7\nbD8MKC8D7QVcYHuR7XnAfcB27YTc0WQhaYKkP0i6U9Idko7q5PlCqI2xBZfm9gcuyJ8NHCnpVkln\nSVorb1+fNMt6v/mkO4yWdfrOYhHwJdubk2Z6/rykzTp8zhC63wiThaSVgI8BP8+bTifNpL4V8Cjw\n3SZn776JkW0/BjyWP78g6S5Sprurk+cNoesNUcWYfSvMvq1QCXsAf7X9BIDt15//SjoTuCyvPgJM\naDhug7ytZaPWwClpIqlh5rrROmcIXWuIMTinbp+WftN/OmQJB7CkCoKk9Ww/mlf3Bm7Pny8Fzpc0\nk1T92Bi4vp2QRyVZSFoduBg42vYLo3HOELraCPpZSFqN1Lh5WMPmkyRtRapiPAAcDmB7jqSLgDlA\nH3CE7e6rhgBIWhH4BfBT279ado+rGz5PAjbqdEghlOBvpGuyTSO48my/CKw9YNvBTfafAcxo/4xJ\nR5OFJAFnAXNsnzL4Xrt0MoQQOmQjlv7F9ofWDq9hD6dOPw15P/BJYOeGnmW7d/icIXS/ch6djqpO\nPw35E9HxK4Rl1fDdkC7LXSEsJ2p45dUw5BB6QIzBGUIopIZXXg1DDqEH1PDKGzJkSV9ucpwHvgrb\ngRBaUM5IULru1FLK+RuHlFLORpxYSjkwsaRy5pVUzlMllbNGSeVUoJeSBelfoq2eXiGE5txLT0Ns\nTxvFOEJYriyu4Z3FsH0g8jBeV0u6M69vIen4zocWQu9aPLbY0k2KdJj6EfB14NW8fjvpjbcQQpte\necNKhZZuUiR3rWr7uvSaR2rZlFTWmPAhLJcWj6lfo0WRZPGEpHf0r0jalzQSTwihTYtr2N+7SLL4\nAnAGsKmkBaT3cg/qaFQh9Li+XkwWtu8HdskDbqxg+/nOhxVCb1tcw44Ww0YsaW3gBOADgCVdC3zL\n9pOdDi6EXlXHakiRpyEXkiaD/CdgX+AJ4GedDCqEXreYMYWWblLkXmhd2//WsH6ipP06FVAIy4NX\n6K7HokUUubO4UtIBklbIy37AlZ0OLIRetpixhZZu0uxFshdY8m7IF4Gf5M8rAC8CzV40CyE00W1V\njCKavRuy+mgGEsLypKeSRSNJbyJNTvL61Ci2/9ipoELodXXsZ1HkRbLDgD+S2immA1cA0zobVgi9\nbSRtFpLWknSxpLskzZG0vaRxkmZJukfSlQ0TIyPpOEn3Sporabd2Yy7SwHk0aYr2ebZ3Jk1B+Gy7\nJwwhjPjR6anA5bY3A7YA5gLHArNsTybN3HUsgKQpwH7AFGB34DRJbY24X6Qa8rLtlyQhaWXbcyVt\n0s7JBtdXQhlvLKEMgHI6p27ECaWU44+UMxKALi8nnu5TVmfidUoqp7hX23x0KmlNYEfbnwKw3Qc8\nK2lP4IN5t3OB2aSEsRdwge1FwDxJ95F++f+l1XMXSRYP5zaLXwGzJD1NeeOrhbBcGkGbxSTSy53n\nAFsCfyU9rRxve2HeZyEwPn9en6UTw3zSBMktK/JuyN754zRJs0m/xn/XzslCCMlQ7RG3zX6a22Y/\n0+zQscA2wBds3yDpFHKVo18eRqLZkJjlTowsadwgm2/Lf65OeaOuhrDcGao9YvOpa7P51CVzHp83\n/cGBu8wH5tu+Ia9fDBwHPCZpXduPSVqP9IoGwCPAhIbjN8jbWtbszuImhs5ApuB055LGADeSfsCP\ntRZeCL2p3X4WORk8LGmy7XuAXYE78/Ip4KT856/yIZcC50uaSap+bAxc3865m3XKmthOgYM4GphD\nrcdtD6FcI+xncSRwnqSVgPuBQ0izp14k6VBSm+InAGzPkXQR6RrsA46wXW41pAySNgA+AnwbOKaT\n5wqhTl4dwfyFtm8F3jPIV7sOsf8MYEbbJ8w6/abK94B/pbxnmyH0hDp2926rc0YRkj4KPG77ZkCd\nOk8IddTHmEJLN2n1acjrbA/3NGQHYE9JHyG9U/JGST+2ffDSu13d8HkSBdtNQ6jY3XlpT7e9fl5E\nkachAjYEns7b3wQ8SLqyh2T766T5RpD0QeAryyYKgF1ajTmELrBJXvr9uqWj61gNGfZpiKQfAb+0\nfXle3wPYe6jjmoh5U0PI6pgsirRZvK8/UQDY/i2pilGY7Wts79lqcCH0ql4dg3NBntv0p6QqyYG0\n2QMshJC8MoJHp1UpcmdxAOm1vF8Cl+TPMddpCCPQk3cWeX6QoyStZvvFUYgphJ7XbYmgiCIjZe0g\naQ5pgA0kbSnptI5HFkIP66l+Fg1OIY2w8x+QuprmR6EhhDb1Wj+L19l+SFqqE2YZw1tli7qkDIAV\nSypns1JKKWuEK39zeinl6FvfKaWc8v77rFJSOaPfXl/HakiRZPGQpPcD5LfcjgLu6mhUIfS4Xk0W\nnyMNEPpWUgq+Evh8J4MKodfVcfrCIslisu0DGzfkO43/7ExIIfS+OrZZFOln8YOC20IIBfVUPwtJ\n7yN1636LpGNY8pr5GnTw1fYQlgfdlgiKaHYvtBIpMYxh6SHxngP27WRQIfS6butDUUSzt06vAa6R\ndI7tZYYYDiG0r1fbLM4cMG/iOElXdDCmEHpeT7VZNHiL7ddnPbH9lKTxzQ4IITTX7vSFVSqSLBZL\nelt/VUTSROC1TgYVQq/rqTaLBt8ArpX0x7y+E/DZzoUUQu/ryTYL278DtgV+BlwIbJO3hRDaNNI2\nC0ljJN0s6bK8Pk3S/Lzt5jz8Zf++x0m6V9JcSbu1G3Ozfhab2b5L0rak8TMX5K82lLSh7ZvaPWkI\ny7sSGi8HzvRnYKbtmY07SZoC7AdMIb2ycVWe+rDlpoRm90LHAIcB32XwwXZ3bvVkIYRkJG0WQ8z0\nJwafn2cv4ALbi4B5ku4DtgP+0up5m/WzOCz/ObXVQkMIzY2wzWKwmf4MHCnpYNJE5F/OTzHXZ+nE\nMJ90h9GyZtWQfWgyfL/tS9o5YQhh6EenT8++jWdm3zbkcY0z/Uma2vDV6cC38ud/I9UIDh2imNIn\nRv5YLnQd0jsiv8/bdwb+TBq8twRNJz4raLjJ0YoqaxCd7hruo6xBa/zOY0spR3eUM6hPef9eo2+o\nasgaU7dmjalbv77+4PTzBu4y7Ex/ks4ELsurjwATGo7fgDZH+xnyaYjtT9s+hPSOyBTb+9jeB9g8\nbwshtGkxYwstA9n+uu0JticB+wO/t32wpPUadtsbuD1/vhTYX9JKkiYBGwPXtxNzkYrTBOCxhvWF\npOkMQwhtKqkrt1hSpThZ0pZ5/QHgcADbcyRdRHpy0gccYbv0aki/q4ArJJ2fg9sPmNXOyUIISRnJ\nwvZsYHb+/D+b7DcDmDHS8xVJFkeSbmt2zOv/bvuXIz1xCMuzbntJrIgikwxZ0k3A87ZnSVpV0hq2\nnx/u2Py26pmkdg4Dn7Hd8vPdEHpNHacvHDZZSPosqXPWOODtpNbU04FdCpR/KnC57X0ljQVWG0Gs\nIfSMnryzII3k/XqPL9v3SFpnuIMkrQnsaPtT+bg+4NkRxBpCz+jVZPGK7Vf6JxnKdwhFWlMnAU9I\nOgfYEvgrcLTtf7QbbAi9oo6vqBcZKesaSd8AVpX0YeDnLOnw0cxYYBvgNNvbAC8C5fTqCaHm2u1n\nUaUi0XwN+GdSJ4/DgctJjZbDmQ/Mt31DXr+YQZPFbxs+v4PUZySEbjcvL+3puWpIrnLcYXtT4IxW\nCrb9mKSH8+uw9wC7Ancuu+cey24KoetNzEu/a1o6uueShe0+SXc3DqvXoiOB8/IcqfcDh7QTZAi9\n5pVX6/fGRJFqyDjgTknXk9odIHW/2HO4A23fCrxnBPGF0JMW93VXe0QRRSI+Pv/ZOLBGW33LQwjJ\n4r4eqoZIWgX4F1Kr423A2Xm0nRDCCPVUsgDOBV4FriUN4TWFNO5fCGGE+hb1VrLYzPa7ACSdBdzQ\nZN8QQgteW9xbbRZ9/R/yU5EOhVDGKFdvHH6XQp4rqZxVSiqnjFHEoM2BkZZR1ghXf2F6KeW8lxNL\nKSf1HRxlPVYN2UJS45ulqzSs23ZZV2gIy5+Xe+jOwnb9Ul8IddE3/C7dpn7pLYReEMkihFBIJIsQ\nQiE17LEUySKEKiyuOoDWRbIIoQo1rIYUGfwmhFC2lwsuA0haWdJ1km6RNEfS/87bx0maJekeSVfm\nwbL7jzlO0r2S5krard2QI1mEUIW+gssAtl8Gdra9FbAFsLOkD5AGlpplezJwdV5H0hTSXD9TgN2B\n0yS1dd1HsgihCm0mC4CGcWxXAsYATwN7kt7nIv/58fx5L+AC24tszwPuIw3A3bJIFiFUYQTJQtIK\nkm4hTSX6B9t3AuNtL8y7LATG58/rk4a47DcfeGs7IUcDZwhVGMGjU9uvAVvl6TaukLTzgO8tqdmY\nMx2b6zSEULahHp3OmQ13zS5UhO1nJf0G2BZYKGndPPbtesDjebdHSJOb99uANt8sjGQRQhWGenQ6\neWpa+l2y9Bu6ktYG+mw/kweo+jAwHbgU+BRwUv7zV/mQS4HzJc0kVT82Bq5vJ+RIFiFUYZDHogWt\nB5ybn2isAPzE9tWSbgYuknQoaY6CTwDYniPpImAOKUUdYTuqISHURpudsmzfziADcNh+ijTdxmDH\nzABmtHfGJSJZhFCFGvbg7IJksWIJZbxUQhlQ3shUZSlnhKvyRhJbo5RSyhrhyl89fvidCtDJ55RS\nTksiWYQQCom3TkMIhcRbpyGEQtp/GlKZSBYhVKGGbRYdfTckvxp7p6TbJZ0v6Q2dPF8ItbGo4NJF\nOpYsJE0EDgO2yZMVjQH279T5QqiVxQWXLtLJashzpNy4qqTFwKqU9ywwhHqLasgSuUfZd4GHgAXA\nM7av6tT5QqiVEbyiXpVOVkPeDnwRmEh6p351SQd16nwh1EoN2yw6WQ15N/Bn208CSLoE2AE4b+nd\nrm74PAnYqIMhhVCWuXlp0yulBTJqOpks5gL/K79G+zLpJZdBXo3dpYMhhNApm+al33+0dniXVTGK\n6FiysH2rpB8DNwKvATcBZ3TqfCHUSpdVMYroaKcs2ycDJ3fyHCHUUpc9Fi0ienCGUIWohoQQColk\nEUIoJNosQgiFxKPTqpSVpp8qqZwyRv+C1J+tDAuH36WQsnrrTyylFJ18Winl+EOHjLgM/b7FA6Ia\nEkIoJKohIYRC4tFpCKGQGlZDYmLkEKrQ5lunks6WtFDS7Q3bpkmaL+nmvOzR8N1xku6VNFfSbiMJ\nOe4sQqhC+20W5wDfB37csM3ATNszG3eUNAXYD5hCmrrwKkmT88TKLYs7ixCq8ErBZQDb1wJPD1Ki\nBtm2F3CB7UW25wH3Adu1G3IkixCqUP7gN0dKulXSWZLWytvWB+Y37DOfdIfRlkgWIVSh3MFvTicN\nBrMV8ChphLqhtDUpMkSbRQjVGOrR6aLZ0De7paJsP97/WdKZwGV59RFgQsOuGzCCnnWRLEKowlBV\nDE2FFacuWX95+rBFSVrP9qN5dW+g/0nJpcD5kmaSqh8bM+gAVMVEsgihCm32s5B0AfBBYG1JDwMn\nAFMlbUWqYjwAHA5ge46ki4A5+YxH2I5qSAi10uajU9sHDLL57Cb7zwBmtHe2pUWyCKEK0YOzE/5W\ndQADzKs6gAFGMMJ0R8yrOoAG91QdQE+pQbJ4oOoABphXdQADRLIYWiSLMtUgWYQQukG0WYRQifoN\naKERPEkZ+cml6k4eQslsD/Z+xjLS//t/FCx11cLldlqldxbd8pcQwuir351FVENCqMRLVQfQskgW\nIVQi7ixCCIXUr1dWJIsQKlG/O4voZ9EFJL25YfzERxvGU7xJUikJXdJsSdsOs888SeNaKPPTkr4/\n8uiWR+WPftNpcWfRBWw/CWwNIOkE4PnG8RQljbE90sHjzfADn5jBh2drtn9oS/3uLCJZdCdJ+n/A\ny6TRj/5T0nPAC7a/m3e4A/iI7YckfRI4ElgJuI70KvKQg7JKOg14D7AKcLHtaQ1ffzWPDv0ScKDt\n+yW9hTQa04Z5ny/a/nN5P+7yqH5PQ6Ia0r1MGkPxfba/PMT3SNoM+ASwg+2tgdeAg4Yp+xu23wNs\nCXxQ0jsbvnvG9hbAD4BT8rZTge/Z3g7YFzgzb49+Mm2Lakgo18+HGaxEwC7AtsCNkiDdLTw2TLn7\nSTqM9O+/Hmmo+DvydxfkPy8Evpc/7wpslssHWEPSai38HGEZUQ0J5WrsE9zH0neCKzd8Ptf214sU\nKGkS8GXg3baflXTOgLIa9ScqAdvbfnVAWdFm0bbuumsoIqoh9TEP2AZA0jak0ZwNXA3sm9sVkDRO\n0oZDFQK8EXgReE7SeGCPhu9EmpSG/Gd/u8SVwFGv75SGcOvfP7Sl3OG9R0PcWXS3xt/cvwAOzg2b\n1wF3A9i+S9LxwJWSViD9DzsCeGjQAu1bJd1MGgjjYeBPA873Jkm3khpX+4dwOwr4Yd4+Frgmn6PI\nE5YwqPrdWVT61mkIy6NUfftFwb336ZoXLuPOIoRK1O/RaSSLECrRXe0RRUSyCKES9WuziKchIVSi\n/achknaXNFfSvZK+Njrxxp1FCBVp785C0hhS79pdSfOW3iDpUtt3lRjcoCJZhFCJttsstgPusz0P\nQNKFwF5AJIsQelPbbRZvJfWP6Tcf2H7E4RQQySKESrT96LSyjlGRLEKoxLR2D3wEmNCwPoF0d9Fx\n0YMzhBrJI6fdTXrbeAFwPXBANHCGEJZiu0/SF4ArgDHAWaORKCDuLEIIBUWnrBBCIZEsQgiFRLII\nIRQSySKgA/p4AAAAHklEQVSEUEgkixBCIZEsQgiFRLIIIRQSySKEUMj/B1Rxb828fDuyAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1903a5410>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import metrics as metr\n",
    "plt.matshow(metr.confusion_matrix(label5_valid, pred5))\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('Predicted label')\n",
    "plt.xlabel('True label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAD1CAYAAABHutCPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8HFWd9/HPNwlBVjUigUgwAUEII6viiKJhWB7wUZCR\nx7AoiAzyGEUUdABhJHFhxOdFhFFhVJbBhSAiMPCIkoAGcWMRCIEQWSRCQAKyBZAlCd/545ybdC73\n9q3urr7V3fm9X696pau66tTvJqlzzzl1FtkmhBCGMqLqAEII3SEyixBCIZFZhBAKicwihFBIZBYh\nhEIiswghFBKZRQhdRNJ5khZLmldzbIyk2ZLuljRL0mvy8QmSnpd0a97OqrlmJ0nzJN0j6cwi947M\nIoTucj6wd79jJwCzbW8JXJv3+9xre4e8Ta05fjZwhO0tgC0k9U/zFSKzqICktSRdKekpST9uIZ1D\nJF1dZmxVkbSrpAVVx9HpbF8PPNnv8L7ABfnzBcAH6qUhaWNgPds35kPfH+oaiMyiLkkHS7pZ0jOS\nHpZ0laR3lpD0AcCGwBjbU5pNxPaPbP+vEuJpK0kvS9qs3jm2r7e91XDFVCVJbmQrkORY24vz58XA\n2JrvJuYqyBxJ78rH3gAsqjnnoXysrlFFfrjVkaRjgeOBo4CrgZdIxb99gd+2mPwbgbttv9xiOt1E\ng34hjbK9bDiDqdpXBjn+Z+D+mv1fNZiu7doM5mFgvO0nJe0IXC5pmwaTXCFKFgOQ9GpgOjDV9uW2\nn7e93PbPbB+fz1lT0hmSHsrbNySNzt9NlrRI0rG5MephSR/N300H/g2YkkssH5M0TdIPau4/If82\nHpH3PyrpPklLJP1Z0sE1x6+vuW4XSTfl6s2Nkt5R890cSV+S9JucztWSXjfIz98X/+clPZrj/4Ck\n9+ZGtMclnVBz/s6Sfi/pyXzuNyWtkb/7dT5tbv55/09N+v8q6a/AufnYg/mazfM9dsj74yQ9Jund\nLf3DdpA1BtneTPqN1LcVtFjSRrCiivEogO2XbD+ZP98C3AdsQSpJbFJz/Sb5WF2RWQzsHcCrgMvq\nnHMSsDOwXd52Bk6u+X4ssD4wDjgC+LakV9s+BTgVuMj2erbPAwYtakpaBzgT2Nv2+jm22wY4bwzw\nM+AMYAwwA/iZpNfWnHYQ8FFSFWg08Lk6P99YYE1gY+CLwDnAIcAOwK7AFyW9MZ+7DDgGeF2Ob3dg\nKoDtvgd82/zz/qQm/dcCm5JKbyvYvo9UqvuhpLVIjXrn2/41PWJUwa2gK4DD8ufDgMsBJG0gaWT+\nvBkpo/iz7b8CSyS9XZKAj/RdU09kFgN7HfC3IaoJBwNfsv03238jlUQ+UvP90vz9cts/B54l/eKA\nVCSvLZYPWkTPXgbeImkt24ttzx/gnP8N/Cm3Y7xs+yJgAanaBClDOt/2vbZfAC4Gtq9zz6XAV20v\nB35MyoDOsP1cvv/8vutt32L7xnzfvwDfBd5T4Gc6xfbSHM8qbJ8D3AvcSMpYThoiva6yVsGtP0kz\ngd8Bb5b0oKTDga8Be0q6G/invA/wblKJ7lbgJ8BRtp/K300l/QK4h/TG5BdDxRxtFgN7HNhA0og6\nGcY44C81+w/kYyvS6Hft34F1Gw3E9nOSppBKAedK+i1wnO0/DRDPA/2O/aVfTI/UfH5+iHge98r5\nC57Pfy6u+f55YB0ASVuSSjI7AWuT/l/dXO/nAh6z/dIQ55wD/DdwpO2lQ5zbVdZo8jrbBw3y1R4D\nnHspcOkg6fwReEsj946SxcB+D7wI7F/nnIeBCTX7m+ZjzXiW9JD12aj2S9uzbO+Vjy8AvjdAGg+R\nGk5rvZECddESnE0qabzJ9qtJpYCh/m/VbeWXtC6pSnUOML1fdarrlVwNGRaRWQzA9tOkevq3Je0n\naW1Ja0jaR9Jp+bSZwMm5XrhBPv8Hg6U5hNuAd0sanxtXT+z7QtKGOYZ1SFWD54DlA6Txc2BLSQdJ\nGpVLI1sB/7/mnKGqO81aF3gG+LukrYBP9Pt+MbB5g2meCdxo++Oktpj/bDnKDjJYA2f/rZNEZjEI\n2zOAY0mNlo+SivhTWdno+RVSUfv2vN3Mqm/E6v3mdO33tq8htQvcDtwEXFnz/Qjgs6QSwuOkxsVP\n9E/H9uPA+4DjgL+Rqi3vs/3EIDGZoWOst1/rc6Q2nCWk9oqL+p0/Dbggvy05oM69DSBpP2AvVv6c\nxwI7ShqsCN51urFkoU6eVk+pC+oZwEjgHNunDXFJO2MZT+rptiHpP/V3bf9HVfHkmEaSMqlFtt9f\ncSyvIVUZtiH9/XzM9h8qjOdE4MOkhtR5wOG2X6wqnlqS/NOC534QsN2uEmFDOrZkkR+Eb5FeN08C\nDpK0dYUhLQU+a3sb4B+BT1YcD6TXlfMZov4/TM4ErrK9NbAtcFdVgUiaABwJ7Gj7LaRfNgdWFc9A\nohpSrp1Jr3QW5pbwi4D9qgrG9iO2b8ufnyU9DOPqX9U+kjYB3kv6bV7pb57czrJr7jOC7WW53acq\nS0iZ+9qSRpEaj4ejobewZl+dVqmTM4s3AA/W7C+iQP/14ZB/c+0A3FBhGN8APk8qZldtIvCYpPMl\n3SLpe5LWHvKqNsntNKeT2pkeBp7K7UIdoxvbLDo5s+iEovUr5Fd6lwDH5BJGFTG8D3jU9q1UXKrI\nRgE7AmfZ3pH0xuaE+pe0j6TNgc+QXm2PA9aVdEhV8QwkqiHleggYX7M/nlVHyg27PN7hp8APbQ/Z\nPbaNdgH2lXQ/6RXuP0n6foXxLCI1st6U9y8hZR5VeSvwO9uP5wFql5L+zjpGlCzKdTNpUo4JSgO0\nppD6wFci96E/F5hv+4yq4gCw/QXb421PJDXc/dL2oRXG8wjwYO7JCak34Z1VxUPquPaPSvOGKMcz\nUBf5ynRjyaLTMq8VbC+T9CnS8PCRwLm2K2thB95JehV3e+5rD3BikT71w6ATqmxHAz/KGft9wOFV\nBWJ7bi5p3Uxq07mF1P+jY3Tsg1dHR/ezCKEXSRpwJOBAJtE5/Sy6MYMLoet12mvRIiKzCKECndYe\nUURkFiFUoBsfvG6MOYSut0bRJ6+DZiatNLNQsZmLQ+gKjTREjorMonEeYnTFtCUwbf365+jhU0qK\npkhN8lrSFJP1DOekTnOAycN0ryLNcrOBPYc45/khvi9qqH+vIv9WUM6/1/SGzl5jZAm3HGaVZxYh\nrI4Klyw6SBeGHEL3W2PNqiNoXMdnFpM77i91YtUB9DOh6gD6qbvw2DDrtH+rGh3/5L1SW8eGSNpb\n0gKllZqPbyaNzsssOulhgM7LLBqdarOdOu3fqkYXjiRrWzg1M13tQRpBepOkKyoe3xFCZ+iwjKCI\ndpYsOmqmqxA6ysiC2wAkHSNpnqQ7JB2Tj42RNDsvLzkrz4nad/6JuXS/QNJezYbczsyiY2e6CqFy\nTVZDJP0D8C/A20jLZr4vT/ZzAjDb9pakd8Yn5PMnkaZ3mESaz/Ys5TV0mwm5XQp1uJq2ZOXnyWt2\nYhtFCANZmLcmNf//fCvghr4lHyVdR5oEfF9WLhl5AakDzgmk0vzMXLpfKOleUqm/4ZnX25lZFJrp\naqgOVyF0pgms2rh8XWOXN//k3QF8NS+E/QJp0uabgbG2+5aXXExaHxbStIK1GUPTJfx2ZhYrZroi\nTZo6hbSKdwhhkCdvztMwZ8nA3wHYXpBXxZtFmuv0NvqtUGfbQwylaGqYRdsyiw6c6SqEzjFI4+Xk\nMWnrM32ABQzykgvnAUj6Kqm0sFjSRrYfkbQxaRU9eGUJfxOaXBahrS9wbP+ctAZnCKFWC0+epA1t\nPyppU+CfSYteTQQOA07Lf/ZNKH0FcKGkGaTqxxbAjcMccgihaa09eZdIeh1pBNxU209L+hpwsaQj\nSC2vHwKwPV/SxaQJi5fl8zurGhJCqKOFJ8/2uwc49gSpA+RA558KnNr8HZPILEKoQhd2Eag8syhj\nLopTGpxLYDDTOb2UdMqbz6KsmRrLime45qFYDVT+5DWuC0MOoQfE5DchhEK68MnrwpBD6AFd+OR1\nYcgh9ICohoQQCunCJ68LQw6hB7yq6gAaF5lFCFWIakgIoZAufPK6MOQQekAXPnldGHIIPSCqISGE\nQrrwyevCkEPoAV345HVhyCH0gBh1GkIopAufvC4MOYQe0IVPXheGHEIPiLchIYRCuvDJ64CQ12o5\nhem0PtsWgPc/rpR0dFlZM27VWUCiq5X1325ZSelUoAOevEZ1Ycgh9ICohoQQCunCUaftXEU9hDCY\n5ldRf7OkW2u2pyUdI2mapEU1x/epueZESfdIWiBpr1ZCDiEMtyarIbb/BOwAIGkEaSnCS4GPATNs\nz6g9X9Ik0jrDk0grkl0jaUvbLzd677aWLCSNl/QrSXdKukPSp9t5vxC6RpMli372AO61/SCgvPW3\nHzDT9lLbC4F7gZ2bCbnd1ZClwGdtb0Naj/GTkrZu8z1D6HzlZBYHAjPzZwNHS5or6VxJr8nHx5EW\nTu6ziFTCaFhbMwvbj9i+LX9+FriLFHwIq7eRBbdBSBoNvB/4ST50Nmlx5O2Bv0LdFbM6e61TSRNI\nda0bhuueIXSsQd6GzLkV5txWKIV9gD/afgzA9qN9X0g6B7gy7z4EjK+5bpN8rGHDkllIWhe4BDgm\nlzBCWL0NUmqY/Na09Zn+X4OmcBArqyBI2tj2X/Pu/sC8/PkK4EJJM0jVjy2AG5sJue2ZhaQ1gJ8C\nP7R9+SvPmF3zeTNg83aHFEIJFuatSS08eZLWITVuHllz+DRJ25OqGPcDRwHYni/pYmA+qcvrVNud\nVw2RJOBcYL7tMwY+a892hhBCm0zIW5/rGru8hSfP9nPABv2OHVrn/FOBU5u/Y9LutyHvBD4M7FbT\nWWTvNt8zhM5XztuQYdXWcGz/huglGsIrxdiQEEIhXfjkdWHIIfSAmIMzhFBIFz55XRhyCD2gC5+8\nDgh5jRLSeL6ENECXnVVKOh43tZR09HA5M4DBW0pKZ97Qpwyr1mdZSyp4DDrgyWvUoCFLqjfHnPsP\nhQ0hFOceexuyHk0OOAkh1Le8l0oWtqcNYxwhrFZ6KrPoI+nNwFnARra3kbQtsK/tr7Q9uhB61Itr\nji545kttjaMRRXpXfg/4Aiujnkca8RZCaNLykSMLbZ2kSGFobds3pDFhqWVT0tL2hhVCb1vehf29\ni2QWj0l6U9+OpANIM/GEEJq0rEczi08B3wW2kvQwaaz8IW2NKoQet7wLO1oMGbHt+4Dd84QbI2w/\n0/6wQuhtPVkNkbQBcArwLsCSrge+ZPvxdgcXQq/qycwCuIg0DdA/k9YlOBj4MWlarxBCE16k6KvT\nzlEks9jI9pdr9r8iaUq7AgphddCNbRZF+lnMknSQpBF5mwLMandgIfSy5YwstHWSegPJnmXl2JDP\nAD/In0cAzwH1BpqFEOrotIygiHpjQ9YdzkBCWJ30aj8LJL2WtDjJinWUbP+6XUGF0Ot6ss1C0pHA\nr0ntFNOBq4Fp7Q0rhN7WSpuFpNdIukTSXZLmS3q7pDGSZku6W9KsmoWRkXSipHskLZC0V7MxF8ne\njgHeBvze9m6StgL+vdkbvlIZs1yVMdsWwBOlpFLWDFeeOL2UdHR/WTNudZolJaUzoaR0inuptVen\nZwJX2T5A0ihgHeAkYLbtr0s6HjgBOEHSJGAKMIm0fOE1kra0/XKjNy3yNuQF288DSHqV7QXAmxu9\nUQhhpWWMLLT1J+nVwK62zwOwvcz208C+wAX5tAuAD+TP+wEzbS+1vRC4F9i5mZiLlCwezG0WlwOz\nJT1JS4s8hhBaaLOYSBrceT6wHfBH0tvKsbYX53MWA2Pz53HAH2quX0QqYTSsyNiQ/fPHaZLmAOsD\nv2jmZiGEZLD2iNvnPMm8OU/Wu3QUsCPwKds3STqDVOVYIU8jUW9KzHIXRpY0ZoDDt+c/16WsCn4I\nq6HBMottJm/ANpNXrnk8c/r9/U9ZBCyyfVPevwQ4EXhE0ka2H5G0MfBo/v4hYHzN9ZvkYw2rV7K4\nhcFzIAObFbmBpJHAzaQf8P2NhRdCb2q2n0XODB7MjZR3k8Zo3Zm3w4DT8p+X50uuAC6UNINU/dgC\nuLGZe9frlDWhmQQHcAwwnzRbeAgBeKm19QuPBn4kaTRwH3A4aanliyUdQWpT/BCA7fmSLiY9g8uA\nqbbLrYaUQdImwHuBrwLHtvNeIXSTVrp7255L6s7Q34AjwW2fCpza9A2zdncj+wbweVKjaAgh69nu\n3s2Q9D7gUdu3Spo8+JnX1nyeSMGmkBAqtiBvzenG7t6Nvg1ZwfZQb0N2AfaV9F7SmJL1JX3f9qGr\nnrZ7sUhD6Chb5a3Pfzd0dU+NOmXl2xABmwJ9L39fC/yFVAwYlO0vkNYbQdJ7gM+9MqMIYfXUU5lF\n39sQSd8DLrN9Vd7fB9h/sOvqiHVTQ8h6KrOo8Q7bR/bt2P65pP/XyE1sX0eaxzOEALzY2qvTShTJ\nLB6WdDLwQ1ZO2NtUD7AQQtKNJYsio04PAjYELgMuzZ9jrdMQWtBTc3D2yeuDfFrSOrafG4aYQuh5\n3djPoshMWbtImk9+qSxpO0lntT2yEHrYckYV2jpJkWjOAPYmv0i2PTe/Ci1JLy7IXreLSmFlzXDl\no0qaces7p5eSTnkzXJXVMXj4m+A6rYpRRKGsy/YDkmoPLWtPOCGsHno1s3hA0jsB8ii3TwN3tTWq\nEHpcry5f+AnSBKFvIJXXZgGfbGdQIfS6TmuPKKJIxFvaPrj2QC5p/LY9IYXQ+7qxGlKkn8W3Ch4L\nIRTUU/0sJL2DNHL09ZKOJfXehDTjVZFMJoQwiG7sZ1GvGjKalDGMZNUp8ZYAB7QzqBB6XU+1WfQN\n/pJ0vu2/DGNMIfS8TqtiFFGkOnFOv3UTx0i6uo0xhdDzXmJ0oa2TFCkLvd72U307tp+QNLbeBSGE\n+nqtzaLPcklv7KuKSJoANLyoaghhpZ5qs6hxEnC9pF/n/XcDH29fSCH0vp5ss7D9C2An4MfARcCO\n+VgIoUmt9rOQNFLSrZKuzPvTJC3Kx27N01/2nXuipHskLZC0V7Mx1+tnsbXtuyTtRJo/8+H81aaS\nNrV9S7M3DWF1V0KbRf+V/gzMsD2j9iRJk4ApwCTSkI1r8tKHDTcl1KuGHAscCZzOwJPt7tbozUII\nSSttFoOs9CdWdpystR8w0/ZSYKGke4GdgT80et96/SyOzH9ObjTREEJ9Lb4WHWilPwNHSzqUtBD5\ncfkt5jhWzRgWkUoYDatXDfkgdabvt31pMzd8pbVKSOP5EtIo01DrLw0vfedrpaTz8pjjSklnxBPl\nTOpT3sRJwz8B02DVkKfmzOXpOXMHva7OSn9nA1/Kn79MqhEcMUgypS+M/P6c6IakMSK/zMd3A35H\nmrw3hNCEwaoh603eifUm77Ri/8HpP+x/ypAr/Uk6B7gy7z4EjK+5fhOanBps0Lchtj9q+3DSGJFJ\ntj9o+4PANvlYCKFJzb4Nsf0F2+NtTwQOBH5p+1BJG9ectj8wL3++AjhQ0mhJE4EtgBubiblIK8t4\n4JGa/cWk5QxDCE0qqZ+FWFml+Lqk7fL+/cBRALbnS7qY9OZkGTDVdunVkD7XAFdLujAHNwWY3czN\nQghJGZmF7TnAnPz5I3XOOxU4tdX7FcksjiYVa3bN+9+xfVmrNw5hddaTyxfatqRbgGdsz5a0tqT1\nbD8z1LV5tOo5pHYOAx+z3fD73RB6TTd29x4ys5D0cVLnrDHA5qTW1LOB3QukfyZwle0DJI0C1mkh\n1hB6Rk9mFqSZvFf0+LJ9t6QNh7pI0quBXW0flq9bBjzdQqwh9IxeHaL+ou0X+xYZyiWEIq2pE4HH\nJJ0PbAf8ETjG9t+bDTaEXtGrQ9Svk3QSsLakPYGprOzwMVTaOwKfsn2TpDOAE4Avrnpa7YuVzUg1\nnRA63cK8NadXqyHHA/9C6uRxFHAVqdFyKIuARbZvyvuXkDKLfvYsEmcIHWZC3vpc19DVPZdZ5CrH\nHba3Ar7bSMK2H5H0YB4OezewB3Bn86GG0DtefKn7OkHXzSxsL5P0p9pp9Rp0NPCjvEbqfcDhzQQZ\nQq9Zvqw32yzGAHdKuhF4Lh+z7X2HutD2XOBtLcQXQk9avqzHqiHZyfnP2ok1mupbHkJIeiqzkLQW\n8H+BNwG3A+fl2XZCCC1atrSHMgvgAuAl4HrSFF6TSPP+hRBa9PLy3mqz2Nr2WwAknQvcVOfcFpQx\ny9UaJaQBqy7p2oqyZsoqYxYxKGsmsbJmuJrF9FLS2YuyZtyaUFI6Deilaghp7Duw4q3IMIQTwmri\nhd4qWWwrqXZk6Vo1+7a9/kAXhRAKWDb0KZ2m3uze3VdOCqFb9FJmEUJoo8gsQgiFdGEnhMgsQqjC\n8qoDaFxkFiFUIaohIYRCXqg6gMYNushQCKGNlhXc+pH0Kkk3SLpN0nxJ/56Pj5E0W9LdkmblybL7\nrjlR0j2SFkjaq9mQI7MIoQpNZha2XwB2s709sC2wm6R3kSaWmm17S+DavI+kSaS1fiYBewNnSWrq\nuY/MIoQqNJlZANTMYzsaGAk8CexLGs9F/vMD+fN+wEzbS20vBO4lTcDdsMgsQqjC0oLbACSNkHQb\naSnRX9m+Exhre3E+ZTEwNn8eR5riss8i4A3NhBwNnCFUoYVXp7ZfBrbPy21cLWm3ft9bUr05Z9q2\n1mkIoWyDvTq9aw4smFMoCdtPS/oZsBOwWNJGee7bjYFH82kPkRY377NJPtawyCxCqMJgr04nTk5b\nn8tXHc4vaQNgme2n8gRVewLTgSuAw4DT8p+X50uuAC6UNINU/dgCuLGZkCOzCKEKzXfK2hi4IL/R\nGAH8wPa1km4FLpZ0BGlBkw8B2J4v6WJgfr7rVNtRDQmhazSZWdieR1q8q//xJ0jLbQx0zanAqc3d\ncaUOyCzKmOWqrBmlyppxq6x4yuoTXNbPVc5/l734Sinp+O0nD31SAbrhzFLSaUh09w4hFBKjTkMI\nhcSo0xBCIV04kCwyixCq0IVtFm3t7p1Hu90paZ6kCyWt2c77hdA1WujuXZW2ZRaSJgBHAjvm9UdG\nAge2634hdJXlBbcO0s5qyBJS3ri2pOXA2jTZzTSEnhPVkJVyJ5HTgQeAh4GnbF/TrvuF0FVaGKJe\nlXZWQzYHPkNaG24csK6kQ9p1vxC6She2WbSzGvJW4He2HweQdCmwC/CjVU+7tubzRGCzNoYUQlnu\nIc0j06QXSwtk2LQzs1gA/FseGfcCqd/6AKPddm9jCCG0yxZ56/OLxi7vsCpGEW3LLGzPlfR94Gbg\nZeAW4Lvtul8IXaXDqhhFtLVTlu2vA19v5z1C6Eod9lq0iOjBGUIVohoSQigkMosQQiHRZhFCKCRe\nnVZlSYelU9ZMWeuVlM4zJaXzfEnpjB36lAJ0Q0kzbk08puU0dH+DF0Q1JIRQSFRDQgiFxKvTEEIh\nUQ0JIRTShZlFLIwcQhWaHHUq6TxJiyXNqzk2TdIiSbfmbZ+a706UdI+kBZL2aiXkKFmEUIXmX52e\nD3wT+H7NMQMzbM+oPVHSJGAKMIm0dOE1krbMCys3LEoWIVShyclvbF8PPDlAihrg2H7ATNtLbS8k\njanfudmQI7MIoQrlT35ztKS5ks6V9Jp8bBywqOacRaQSRlOiGhJCFQZ7dbp0Diyb02hqZwNfyp+/\nTJrO8ohBzm1qUWSIzCKEagz2NkSTYY3JK/dfmD5kUrYfXXG5dA5wZd59CBhfc+omtDBpdlRDQqhC\niRP2Stq4Znd/oO9NyRXAgZJGS5pImtprgNnqiomSRQhVaLK7t6SZwHuADSQ9CJwCTJa0PamKcT9w\nFIDt+ZIuBuaTsp6ptpuuhqiFa1smyVDGYKBO62hf1kCystIpayBZWX/P5QwkgydKScUTT245Dd0P\ntgd6I/HKcyUXbzpQ4XTbrQuqIX+uOoB+FlYdQD/3VB1APwurDqBGp/3f6W5dkFk0Ova33RZWHUA/\nLUxH3xYLqw6gRqf93+luXZBZhBA6QTRwhlCJTmtnG1oHNHCG0Bsaa+D8e8FU1+6YBs5KSxad8pcQ\nwvDrvpJFVENCqERZ85kOn8gsQqhElCxCCIV031RZkVmEUInuK1lEP4sOIOl1NVOi/bVmirRbJJWS\noUuaI2mnIc5ZKGlMA2l+VNI3W49udVTiSLJhEiWLDmD7cWAHAEmnAM/UTpEmaaTtViePN0MPSDAD\nz7hU7/zQlO4rWURm0Zkk6b+AF4Dtgd9KWgI8a/v0fMIdwHttPyDpw8DRwGjgBtLowkHnWZR0FvA2\n0ki1S2xPq/n6X/OEr88DB9u+T9LrSROsbJrP+Yzt35X3466Ouu9tSFRDOpdJ06K9w/Zxg3yPpK2B\nDwG72N4BeBk4ZIi0T7L9NmA74D2S/qHmu6dsbwt8CzgjHzsT+IbtnYEDgHPy8egn07SohoRy/WSI\n+QcE7A7sBNwsCVJp4ZEh0p0i6UjSv//GpNmf78jfzcx/XgR8I3/eA9g6pw+wnqR1Gvg5witENSSU\nq7ZP8DJWLQm+qubzBba/UCTBPGPSccBbbT8t6fx+adXqy6gEvN32S/3SijaLpnVWqaGIqIZ0j4XA\njgCSdgQmkh7ma4EDcrsCksZI2nSwRID1geeAJZLGAvvUfCfSOhPkP/vaJWYBn15xUpqVqe/80JTy\np/dutyhZdLba39w/BQ7NDZs3AH8CsH2XpJOBWZJGkP6HTQUeGDBBe66kW4EFwIPAb/rd77WS5pIa\nVw/Kxz8NfDsfHwVcl+9R5A1LGFD3lSwqHXUawuooVd9+WvDsD3bMgMsoWYRQie57dRqZRQiV6Kz2\niCIiswihEt3XZhGZRQiV6L6SRbw6DaESzffglLS3pAWS7pF0/PDEGyWLECrSXMlC0khSV/w9SOuW\n3iTpCtt3lRjcgKJkEUIlmi5Z7Azca3uh7aWkbvn7DUfEUbIIoRJNvzp9A6kzXZ9FwNtbDqeAyCxC\nqMS0Zi+srBdlZBYhDLMWe2Q+BIyv2R9PKl20XbRZhNBdbga2kDRB0mjSgL8rhuPGUbIIoYvYXibp\nU8DVwEgN1K4MAAAAN0lEQVTg3OF4EwIxkCyEUFBUQ0IIhURmEUIoJDKLEEIhkVmEEAqJzCKEUEhk\nFiGEQiKzCCEU8j9ntKuy+AmVdgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x18e66f450>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import metrics as metr\n",
    "plt.matshow(metr.confusion_matrix(label6_valid, pred6))\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('Predicted label')\n",
    "plt.xlabel('True label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQsAAAD0CAYAAACM5gMqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHw1JREFUeJzt3XmYHVW57/Hvj4RAwiQRCTMBmT0yBCdQICj6IAdQD1wR\ncELFAQVU9BiQK4kXueJ9CHBUuAcDHEAGEZULRxAC5zTODDJFIATQyJgGUUaZkvzuH2ttstPpoXp3\n7a5du9/P89TD3rVrV73ddL1ZU60l24QQwlBWqjqAEEI9RLIIIRQSySKEUEgkixBCIZEsQgiFRLII\nIRQSyaICkiZKulLSU5J+NILzHCrpmjJjq4qk3STNrzqOMDDFOIuBSToE+DKwNfAscDvwLdu/GeF5\nPwJ8AdjF9tIRB9rhJC0FtrD9p6pjCa2LksUAJH0ZOBU4EVgX2Bj4PrB/CaffFFgwFhJFEw34gTR+\nNAOpmiQPZ6s63lfZjq3PBqxFKkkcMMgxqwCnAY/k7VRgQv5sOvAwqVTSCzwKfDx/Ngt4CXg5X+MT\nwEzggqZzTwWWAivl9x8HHgCeAf4EHNK0/1dN39sVuBl4CriJVHJpfNYDfBP4dT7PNcBrB/jZGvF/\nFXg8x/9+YB9gAfAkMKPp+LcAvwP+no/9LrBy/uyX+Wd5Lv+8/6Pp/P8KPAacl/c9lL/z+nyNnfL7\nDYAngN2r/tso6e/LJxbc0i1afcy2o2QxgF2AVYGfDXLM10k3yQ55ewtwfNPnU4A1SX/onwS+L2kt\n2ycAJwGX2F7D9jmkP4p+SVoNOB3Y2/aaObbb+zluMvBzUgKbDMwGfi5p7abDDiYlmHWBCcBXBvn5\nppAS4vrAN4A5wKHATsBuwDckbZqPXQwcDbw2x/cu4AgA27vnY7bPP++Pm86/NrAJ8JnmC9t+APga\n8ENJE4FzgXNt/3KQeGtl5YJbJ4lk0b/XAn/14NWEQ4Bv2v6r7b+SSgwfafr8lfz5EttXk/5l3Tp/\nJpYvlg9YRM+WAm+UNNF2r+27+znmn4F7bV9oe6ntS4D5LKs2mXTD3W/7ReBSYMdBrvkKqX1mCfAj\nUgI6zfbz+fp3N75v+1bbN+Xr/gU4C9ijwM90gu1XcjzLsT0HuJ9UQppCSs5dY3zBrZNEsujfk8A6\nkgb7/WwA/KXp/YN536vn6JNs/gGsPtxAbD8PHAR8FnhU0n9K2rqfQzfIMTT7S5+YFjW9fmGIeJ50\nLjPnYyFVqZq/vxqApK1yXI9Jehr4FinhDuYJ2y8Pccwc4A3Ad22/MsSxtTKx4NZJIln073ekdoUP\nDHLMo6S2hYZN8r5WPAdManq/XvOHtq+1/Z68fz7wg37O8Qip4bTZpnl/u51JKmlsYXstUilgqL+t\nQRvuJK1OqlLNAWb1qU7VXlRDuoTtp0n19O9Lep+kSZJWlvReSSfnwy4Gjpe0jqR18vEXtHjJ24Hd\nJW0saS3g2MYHktbNMaxGqho8Dyzp5xxXA1tJOljSeEkHAdsA/9l0zFDVnVatTmq8/IekbYDP9fm8\nl9RoORynAzfZ/jSpLeb/jjjKDtJqNUTSOZJ6Jc1r2jdZ0lxJCyRdK+k1ef9USS9Iui1vZzR9Z2dJ\n8yTdJ+n0IjFHshiA7dmk3ozjST0CD5Ia7RqNnicCtwB35u2WvO/VUwx2+ubPbV9Hahe4k9SbcWXT\n5ysBXyKVEJ4kNS5+ru95bD8J7AscA/yV1Hi5r+2/DRCTGTrGwd43+wqpDecZUnvFJX2OnwmcJ+nv\nkg4c5NoGkPQ+4D0s+zm/DEyTdPAgMdTKCEoW5wJ799k3A5hreyvg+vy+4X7bO+XtiKb9ZwKftL0l\nsKWkvudcQUcPyso/wGnAOGCO7ZOH+Eo7Y9kYOJ/Uk2DgLNv/VlU8OaZxpCT1sO39Ko7lNSxrYzDw\nCdu/rzCeY4EPkxpS5wGH2X6pqniaSfIlBY/9EGB7uRKhpKnAlbbfmN/PB/aw3StpPaDH9jZ9j2v6\n/vrAf9neNr//EDDd9mcHi6VjSxb5RvgeKYtuBxwsadsKQ3oF+JLtNwBvAz5fcTyQuivvZoj6/yg5\nHbgq/wFuD9xTVSD5JjkcmJZvlHGk+65jlNxmMcV2o/G5l9R71LBZroL0SHpH3rchaZxLwyN536A6\nNlmQxi3cb3thbgm/BHhfVcHYXmT79vz6OdLNsMHg32ofSRuRBknNoX1tEUVjWQvYLY8Zwfbi3O5T\nlWdIyX1SHh06idFp6C2sXQ2cuQer8Y/Ho8DGtnciVeUukrRGqzF3Wldusw2Bh5rePwy8taJYlpP/\n5doJuLHCME4ljbBcs8IYGjYDnpB0LmmA2h+Ao23/o4pgbP9N0imkdqYXgGtyu1DHGKhb9I68DVOv\npPVsL8pVjMcBctf0y/n1rZIeALYkJc6Nmr6/EQWSaSeXLDqhaL2C3KV3GelmeK6iGPYFHrd9GxWX\nKrLxwDTgDNvTSD02Mwb/SvtIej3wRVLX9gbA6pIOrSqe/gzU+7Ezafx/YyvoCuBj+fXHgMsBck/d\nuPx6c1Ki+JPtx4BnJL1VkkiDCS8f6iKdnCweIT281bAxy9ezRp2klYGfAD+0PeQvt412BfaX9GdS\nF+47JZ1fYTwPkxpZb87vLyMlj6q8Cfit7SdtLwZ+SvqddYxWqyGSLgZ+C2wt6SFJhwHfBt4taQHw\nzvweYHfgDkm3AT8GPmP7qfzZEaQq7H2k6v4vhoq5Y3tDcl3zXtJzBo+Shv0ebLuShrOcgc8jjWz8\nUhUx9EfSHsBXOqA35JfAp2wvkDQTmGj7axXFsgNwIfBm4EXgP0hjNr5fRTx9SSrcTfQ2VuwNqUrH\ntlnYXizpC6SnI8cBZ1eVKLK3k7ri7syZGuDYIhl5FHRCxj8SuFDSBNITsodVFYjtO3JJ6xZS1+mt\npPEfHaPTRmcW0bElixC6lSQXbcTcgShZhDCm1bFkEckihAp02hOlRUSyCKECUbIIIRRSxxuvjjGH\nUHsrF73zFrc1jGGpNFl01MzFIYzQcHotxkeyGD6vO/jnM5+DmUNMRqfHTygvoCH1kCai7hQ9DB1P\nWTXkIjPb9TB6v5+hfq7rSWP6hlLGjH2zhnX0yuNKuOQoqzxZhDAWFS5ZdJAahhxC/a28StURDF9b\nHySTtLek+Xmev5aeE5g+oeyoRmpq1QH0MbXqAPqYWnUATTarOoCB1XAtgLaF0zTT1V6kJ0hvlnTF\ncJ/viGQxlKlVB9DH1KoDaLJ51QEMrMMSQRHtLFl01ExXIXSUGpYs2pks+pvpash5/kIYE8YV3DpI\nO3NXjKEIYSAdVmooop0hF5rpambTxHTTJ3RiG0UI/VmYtxbVsDekncniFtLiJVNJM10dRFrFezlD\nDbgKoTNNZfnG3BuG9/UoWSzTgTNdhdA5Ilksz/bVpDU4QwjNOqzxsoga5rcQukAN77wahhxCF6jh\nndfJ64aE0L1GMChL0tGS5kn6o6Sj877JkuZKWiDp2rxQdeP4Y/MjF/MlvafVkCNZhFCFVQpufUj6\nJ+BTpDVRdgD2zSuwzQDm2t6K9Gz+jHz8dqSeyO1Ii4yfIaml+77ywlAZc1GcMMy5BAYyixNLOU85\n8yOUqdPiGc35NTpU63feNsCNtl8EkHQDcACwP7BHPuY80sQiM0iPWFycH7lYKOl+0qMYRdc5elWU\nLEKoQuvDvf8I7JarHZOAfUgLG0+x3ZuP6QWm5NcbsPxgyJYfu6i8ZBHCmNTinWd7vqSTgWtJC1Df\nDizpc4yHmLKypUcxIlmEUIUB7ryeRdDT2/9nDbbPAc4BkPQtUmmhV9J6thdJWh94PB/e97GLjfK+\nskIOIbTVAIOypm+YtoZZd654jKR1bT8uaRPgX0jrJ28GfAw4Of/38nz4FcBFkmaTqh9bkhYZH7ZI\nFiFUYWR33mWSXktq4T3C9tOSvg1cKumTpCfcPghg+25JlwJ3k+YKP8ItLnAcySKEKqza+ldt797P\nvr+RZqXr7/iTgJNav2ISySKEKsSzISGEQmp459Uw5BC6QA3vvBqGHEIXiGpICKGQGt55NQw5hC5Q\nwzuvhiGH0AViwt4QQiE1vPNqGHIIXaCGd14NQw6hC0RvSAihkBreeTUMeUVlzXDlDx9fynn0wzNK\nOQ/8raTzlDWjVFkzXE0s6TyTSzrPEM+Et0MN77wahhxCF4hqSAihkBE8dVqVSBYhVKGGd14NQw6h\nC9SwGtLW2b0lbSzpvyXdlRdEOaqd1wuhNkawyFBV2h3OK8CXbN8uaXXgD5LmxmrqYczrsERQRLtX\nUV8ELMqvn5N0D2kdg0gWYWyrYTVk1PKbpKnATsCNo3XNEDpW9Ib0L1dBLgOOtv3caFwzhI4WJYsV\nSVoZ+AnwQ9uXr3hET9PrqXkLodMtzFuLWrzzJG0NXNK0a3PgG8DapAWTn8j7j7N9df7OscAnSCuX\nHWX72lEMuRhJAs4G7rZ9Wv9HTW9nCCG0yVSW/4fthuF9vfXlC+8lVefJq6E/AvyUlAxm257dfHyf\nVdQ3BK6TtJXtpcO9drsXRn478GFgT0m35W3vNl8zhM5XTtfpXsD9th8ClLe+Xl1F3fZCoLGKeksh\nt43tXxMrtYewonLaLD4EXJxfGzhS0keBW4BjbD9F6n38fdN3Wl5FPW7kEKowwpKFpAnAfsCP864z\nSeud7gg8BpwyyNVj+cIQamOAOTh7fp+2At4L/MH2EwC2G6umI2kOcGV+G6uoh1BrA9x509+RtoZZ\n/zbgGQ5mWRUESevbfiy//QAwL7+OVdRDqLUR3HmSViM1bh7etPtkSTuSqhh/Bj4D5a6iroG+J+mY\nQb7nvl00LV1cMqXMclXWTFBrlnIWbznYr6443XdCKecpb0apsmbu6rQZt14o4RzHY7u/3ogVSGqq\nNAxx7LoUPm+7DZbf1qDFhpAQwuDcTSM4bc8cxThCGFOW1LABYMiuU0lbS7pe0l35/faSypnZNoQx\nasn4YlsnKTLO4gfAccDL+f08UktsCKFFL60yodDWSYrkrkm2b0yPeaSWTUlltSiGMCYtGVe/Rosi\nyeIJSVs03kg6kDRCLITQoiU1fEa9SLL4AnAWsI2kR0l9uIe2NaoQutzibkwWth8A3pUHgqxk+9n2\nhxVCd1tSw/GQQ0YsaR3gBOAdgCX9Cvim7SfbHVwI3aqO1ZAivSGXAI8D/wIcSJqJ50ftDCqEbreE\ncYW2TlKkLLSe7f/V9P5ESQe1K6AQxoKX6Kxu0SKKlCyulXSwpJXydhDQ0hx+IYRkCeMLbZ1kwGgk\nPceyZ0O+CFyQX68EPA+U87RUCGNQp1Uxihjs2ZDVRzOQEMaSrkoWzSStTZo049WlUWz/sl1BhdDt\nunKchaTDgaNIU3PdBrwN+B3wzvaGFkL36rT2iCKKNHAeTZo6fKHtPUlrFjzd1qhC6HLd2nX6ou0X\nJCFpVdvz86pIJemkZ9KeKeUsZc1w5Y1nlXIePVTWjFtlzXBVljJmuII0z9PoermGXadFksVDuc3i\ncmCupL8zonXbQghd2WZh+wP55UxJPaSJKn/RzqBC6HZ1bLMYbJxFf7O83pn/uzrlzd4awpjTae0R\nRQyW3m5l4Al7TVq9eUiSxpGWU3vY9n7DCy+E7jSSZCHpNcAc4A2ke/Ew4D7SM1ubkpoJPpiXL2z/\nKuq2p7Zywn4cTVqzYPRbkULoUCNsszgduMr2gZLGA6sBXwfm2v6OpK8BM4AZtVlFXdJGwD6kLNgR\nax+E0AleZpVCW1+S1gJ2s30OgO3Ftp8G9gfOy4edB7w/vy5tFfV2L4x8KvBVYNhZLIRuNoJxFpuR\npro8V9Ktkn6QJ6aaYrs3H9MLTMmvNyCtnN7QeauoS9oXeNz2bUSpIoTlLGZcoa0f44FpwBm2p5Ee\n6pzRfEBennCwBcLKXUV9gN6Q5oCG6g3ZFdhf0j6kZ0rWlHS+7Y8uf1hP0+upeQuh091HKtG3ZqCu\n0/k9vdzb09vvZ9nDpM6Cm/P7y4BjgUWS1rO9SNL6pAmroMRV1Adb63QhKQMJ2AT4e/5obeAvtjcr\nfBFpD+ArfXtD0lqnZY0u7D4xgnO0lNH2fvSw1jo9yx8pdNZP64IVzivpl8CnbC+QNBOYlD960vbJ\nkmYAr7HdaOC8iNROsSFwHbBFK4sjD9kbIukHwM9sX5Xfv5e0pPtwxbqpIWQjHGdxJHChpAnAA6Su\n03HApZI+Se46hXJXUS8yjGwX268u7W77akn/ZzgXsX0DcMNwgwuhW40kWdi+A3hzPx/tNcDxJwEn\ntXzBrEiyeDSvbfpDUpXkEFqs84QQkpf66RbtdEV6Qw4G1gV+Bvw0v461TkMYga58RD2vD3KUpNVs\nPz8KMYXQ9TotERQxZMlC0q6S7gbm5/c7SDqj7ZGF0MVGMM6iMkXaLE4D9gb+H6TGldwVGkJoUVc9\not7M9oPScl29i9sTTrdYs5Sz6KETSzmPLzi+lPPoI+XEU97saBNLOs/oz7ZQx2pIkWTxoKS3A+R+\n3aOAe9oaVQhdrluTxedIj8RuSOoyvRb4fDuDCqHb1XH5wiLJYivbhzTvyCWN37QnpBC6Xx3bLIqM\ns/hewX0hhIK6apyFpF1IT46+TtKXWfaY+Rq0fx6MELpapyWCIgYrC00gJYZxLP9Y3jPAge0MKoRu\n12ljKIoY7KnTG4AbJJ1r+y+jGFMIXa9b2yzm5NmEgTQpjqRr2hhTCF2vq9osmryuMaU4pBmyJE0Z\n7AshhMF16/KFSyRt2qiKSJpKTMAbwoh0VZtFk68Dv8pTeQHsDny6fSGF0P3q2GZR5BH1X0jaGXgb\naWq8L9r+a9sjC6GLdVp7RBGDjbPY1vY9OVEYeDR/tImkTWzfOioRhtCFuipZAF8GDgdOof/Jdvds\nS0QhjAFd1WbRmKTX9vRRiyaEMaKr2iwkHcAg0/fb/mlbIgphDBhp16mkccAtpAWH9svrh3wKeCIf\ncpztq/Ox7V1FHdiPlCzWJT0j8l95/57Ab0mT95agjAlMypqLp6xJWZ4p6TzlKGvSGr+ppEl0bilr\n0aP6zsFUQjXkaNJaII1HMQzMtj27+aBRWUXd9sdtH0Z6RmQ72wfYPgB4Q94XQmjREsYX2vojaSNg\nH2AOyx7wVNPrZqO6ivrGwKKm972k5QxDCC0a4XDvU4GvsvzgSANHSrpD0tlNj2iM6irq1wHXSPq4\npMOAq4C5rVwshJC0miwk7Qs8bvs2li9JnAlsBuwIPEbqxRxI25YvPJK0tulu+f2/2/5ZKxcLISQD\nlRqe7bmV53oGHcK0K7C/pH2AVYE1JZ1v+6ONAyTNAa7Mb0tbRb3ICE5LuhV41vZcSZMkrWH72aG+\nm4tCc0jtHAY+Yfv3rQQaQjcZaPnCCdN3YfL0XV593zvr7OU+t30ccBxAXpLjK7Y/Kml924/lwz4A\nzMuvrwAukjSbVP3YEriplZiHTBaSPk0anDUZeD0pM50JvKvA+U8HrrJ9oKTxwGqtBBlCtylpBKdY\nVqX4jqQd8vs/A5+B0V9F/fOk1tPf54svkLTuUF+StBawm+2P5e8tBp5uJcgQuk0ZycJ2D9CTX39k\nkONGbRX1l2y/1FhkKJcQimSmzYAnJJ0L7AD8ATja9j9aDTaEblHH4d5FekNukPR1YJKkdwM/Zlnj\nyWDGA9OAM2xPA54HZrQcaQhdZCTjLKpSJJqvkYaRziPVg64iNVoO5WHSUNSb8/vL6DdZNPfCbk5q\nFgmh0y3MW2u67anTRpXjj7a3Ac4azoltL5L0UB5augDYC7hrxSPfPZzThtAhpuat4YZhfbvrkoXt\nxZLubZ5Wb5iOBC7Ma6Q+ABzWSpAhdJuXXq7fExNFqiGTgbsk3URqd4A0/GL/ob5o+w7gzSOIL4Su\ntGRxZ7VHFFEk4sajhs1DS1vqpw0hJEsWd1E1RNJE4LPAFsCdwDm2y3qGO4QxrauSBXAe8DLwK9Lj\nsNuRnqEPIYzQ4le6K1lsa/uNAJLOBm4e5NgQwjAsXdJdbRavTkOUe0XaFMILJZxj5RLOAbBmSecp\n42cqUzm1x7JmuHp5rVmlnGfC0+XMAAYVLLDXZdWQ7SU1P1k6sem9bZd1Z4Uw9rzYRSUL2/VLfSHU\nRQ2nD61feguhG0SyCCEUEskihFBIDUcsRbIIoQpLqg5g+CJZhFCFqIaEEAp5seoAhi+SRQhViJJF\nCKGQSBYhhEJqmCyKTNgbQijbKwW3PiStKulGSbdLulvS/877J0uaK2mBpGub1jpF0rGS7pM0X9J7\nWg05kkUIVVhScOvD9ovAnrZ3BLYH9pT0DtJk2HNtbwVcn98jaTvgINIUE3sDZ0hq6b6PZBFCFRYX\n3PrRtPbOBGAc8Hdgf9IcNOT/vj+/fh9wse1XbC8E7ictGjZskSxCqMKLBbd+SFpJ0u1AL/Dftu8C\nptjuzYf0suy5+w1Iy3I0PExa83TYooEzhCoM1MC5oAfu6xn0q7aXAjvmJUKvkbRnn88tabB5ctu2\n1mkIoWwDJYvNp6et4aqBJwqy/bSknwM7A72S1svr9awPPJ4PewTYuOlrG+V9w9YByaKMWa7K+jHK\nmuGqrHjK6l+bWNJ5yomnrBmufMzxQx9UgE75dinnGZYWf5WS1gEW234qT6r9bmAWcAXwMeDk/N/L\n81euAC6SNJtU/dgSuKmVa3dAsghhDGr9qdP1gfNyj8ZKwAW2r5d0G3CppE+S1lX8IIDtuyVdCtxN\nSlFH2I5qSAi10eJTp7bnkRYc77v/b6QlQvv7zknASa1dcZlIFiFUIR4kCyEUEsO9l5eHmd4laZ6k\niySt0s7rhVAbLQ73rlLbkoWkqcDhwLS8WNE44EPtul4ItdLicO8qtbMa8gwpN06StASYRIv9uyF0\nnaiGLJNbZ08BHgQeBZ6yfV27rhdCrYzg2ZCqtLMa8nrgi8BU0vj01SUd2q7rhVArNWyzaGc15E3A\nb20/CSDpp8CuwIXLH3Z90+vNgM3bGFIIZXkA+FPrX3+ptEBGTTuTxXzgf+YhqS+SBoz0M8z0XW0M\nIYR2eX3eGq4f6MD+dVgVo4i2JQvbd0g6H7gFWArcCpzVruuFUCsdVsUooq2Dsmx/B/hOO68RQi11\nWLdoETGCM4QqRDUkhFBIJIsQQiHRZhFCKCS6TltRRoqtYZoeVWWVecv6Pa9Zyll0yimlnMf7HTPi\nc+jKYX4hqiEhhEJq+O9bJIsQqhBdpyGEQqIaEkIoJJJFCKGQGrZZxPKFIVThpYJbH5LOkdQraV7T\nvpmSHpZ0W97e2/RZKSuoQ5QsQqhG69WQc4HvAuc37TMw2/bs5gP7rKC+IXCdpK3y8ofDFiWLEKrQ\n4uQ3tn9FWjW9L/Wzr7QV1CGSRQjVKH/C3iMl3SHpbEmvyftKW0EdIlmEUI1y5+A8kzTN3I7AY6S5\nbwfS0tKFEG0WIVRjoESwpAeW9gzrVLYbK6YjaQ7QGHxe2grqEMkihGoM2HU6PW8Ns4Y8laT1bT+W\n334AaPSUlLaCOkSyCKEaLfaGSLoY2ANYR9JDwAnAdEk7kqoYfwY+A+WuoA61SBYLSasJdIqFdFY8\nf6KzZkRfSOf8fu4Htqg6iFLZPrif3ecMcnwpK6hDLRo4F1YdQB8Lqw6gjz9XHUAfC6sOoMkDVQfQ\nVWqQLEIInaAG1ZAQulH9Hg7RCNo7Rn5xqbqLh1Ay2/2NolxB+rv/R8GzTip83nartGTRKb+EEEZf\n/UoWUQ0JoRIvVB3AsEWyCKESUbIIIRRSv6myIlmEUIn6lSxinEUHkPTaplmOHmua9ehWSaUkdEk9\nknYe4piFkiYP45wfl/TdkUc3FpX72OloiJJFB7D9JLATgKQTgGebZz2SNM72SCePN0M/nmz6n0Rl\nsONDS+pXsohk0Zkk6T+AF0lzFPxG0jPAc7ZPyQf8EdjH9oOSPgwcCUwAbiQ9MDTg1GmSzgDeDEwE\nLrM9s+njf81zOL4AHGL7AUmvI82ZsEk+5ou2f1vejzsW1a83JKohncukmY52sd3f+noGkLQt8EFg\nV9s7AUuBQ4c499dtvxnYAdhD0j81ffaU7e2B7wGn5X2nA6fafgtwIDAn749xMi2Lakgo14+HeKRY\nwLuAnYFbJEEqLSwa4rwHSTqc9P9/fdKErn/Mn12c/3sJcGp+vRewbT4/wBqSVhvGzxFWENWQUK7m\nMcGLWb4kuGrT6/NsH1fkhJI2A44B3mT7aUnn9jlXs0aiEvBW2y/3OVe0WbSss0oNRUQ1pD4WAtMA\nJE0jzblo4HrgwNyugKTJkjYZ6CSkJcyfB56RNAV4b9NnIk0dT/5vo13iWuCoVw9KE600jg8taXF6\n7wpFyaKzNf/L/RPgo7lh80bgXgDb90g6HrhW0kqkv7AjgAf7PaF9h6TbgPnAQ8Cv+1xvbUl3kBpX\nGxOtHAV8P+8fD9yQr1GkhyX0q34li0qfOg1hLErVt58UPPqAjnngMkoWIVSifl2nkSxCqERntUcU\nEckihErUr80iekNCqETrvSGS9s6rot8n6WujE2+ULEKoSGslC0njSKNr9yKtLnazpCts31NicP2K\nZBFCJVpus3gLcH9eFR1Jl5BWS49kEUJ3arnNYkPS+JiGh4G3jjicAiJZhFCJlrtOKxsYFckihErM\nbPWLfVdG35hUumi7GMEZQo3kmdPuJT1t/ChpVfSDo4EzhLAc24slfQG4BhgHnD0aiQKiZBFCKCgG\nZYUQColkEUIoJJJFCKGQSBYhhEIiWYQQColkEUIoJJJFCKGQSBYhhEL+P0s3Deh4mczSAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x190480e90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import metrics as metr\n",
    "plt.matshow(metr.confusion_matrix(label7_valid, pred7))\n",
    "plt.title('Confusion matrix')\n",
    "plt.colorbar()\n",
    "plt.ylabel('Predicted label')\n",
    "plt.xlabel('True label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#####################################################################################\n",
    "#################################      Problem 3      ###############################\n",
    "#####################################################################################\n",
    "\n",
    "\n",
    "c_values=[0.0001, 0.01, 0.05, 0.1, 0.3, 0.5, 1, 10, 100, 1000]\n",
    "\n",
    "\n",
    "kfold=d7\n",
    "kfold_label=label7\n",
    "\n",
    "average_performance=[]\n",
    "for c_value in c_values:\n",
    "    #print(c_value)\n",
    "    clf_k=svm.LinearSVC(C=c_value)\n",
    "    accuracies=[]\n",
    "    for i in range(10):\n",
    "        index_range1_1=0\n",
    "        index_range1_2=i*1000\n",
    "        index_range2_1=(i+1)*1000\n",
    "        index_range2_2=10000\n",
    "        \n",
    "        ktrain_data=kfold[index_range1_1:index_range1_2]+kfold[index_range2_1:index_range2_2]\n",
    "        klabel_data=kfold_label[index_range1_1:index_range1_2]+kfold_label[index_range2_1:index_range2_2]\n",
    "        \n",
    "        test_range_1=i*1000\n",
    "        test_range_2=(i+1)*1000\n",
    "        ktest_data=kfold[test_range_1:test_range_2]\n",
    "        ktest_lab=kfold_label[test_range_1:test_range_2]\n",
    "        \n",
    "        clf_k.fit(ktrain_data, klabel_data)\n",
    "        pred_label=clf_k.predict(ktest_data)\n",
    "        accuracy=len([i for i,j in zip(ktest_lab,pred_label) if i==j])/len(pred_label)*100\n",
    "        accuracies.append(accuracy)\n",
    "    ave=sum(accuracies)/len(accuracies)\n",
    "    average_performance.append(ave)\n",
    "    #print(average_performance)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 84.86  94.07  94.91  95.07  95.08  94.84  94.52  93.77  93.08  92.88]\n"
     ]
    }
   ],
   "source": [
    "print(np.around(average_performance, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95.05\n"
     ]
    }
   ],
   "source": [
    "clf_hog=svm.LinearSVC(C=0.3)\n",
    "clf_hog.fit(d7,label7)\n",
    "pred7=clf_hog.predict(d7_valid)\n",
    "result_10000=len([i for i,j in zip(label7_valid,pred7) if i==j])/len(pred7)*100\n",
    "print(result_10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import random \n",
    "from sklearn import svm \n",
    "import scipy.io as spy\n",
    "from __future__ import division\n",
    "\n",
    "\n",
    "data_path=\"/Users/Luba/Desktop/CS189_spring/hw1/data/digit-dataset\"\n",
    "\n",
    "digit_test_data=spy.loadmat(data_path+\"/test.mat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "digit_test_image=digit_test_data.get('test_images')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "digit_test_vector=[]\n",
    "for k in range(len(digit_test_image)):\n",
    "    element=[]\n",
    "    for i in range(28):\n",
    "        for j in range(28):\n",
    "            element.append(digit_test_image[k][j][i])\n",
    "    digit_test_vector.append(element)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_features=np.array(digit_test_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "list_hog_fd_test = []\n",
    "for feature in test_features:\n",
    "    fd = hog(feature.reshape((28, 28)), orientations=9, pixels_per_cell=(4, 4), \n",
    "             cells_per_block=(1, 1), visualise=False)\n",
    "    list_hog_fd_test.append(fd)\n",
    "hog_features_test = np.array(list_hog_fd_test, 'float64')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf_hog=svm.LinearSVC(C=0.3)\n",
    "clf_hog.fit(d7,label7)\n",
    "\n",
    "digit_pred_test_hog=clf_k.predict(hog_features_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD8CAYAAABTq8lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADfpJREFUeJzt3WuMHfV5x/HfgzHgXMBFgcWgrcmLhBawZGTKRS7yqKWR\nrUg2eQGVK8QqSkMsUjcYIRn6oj4URIMlIugLgiqb4KQRiYUdFxBqbUcelYuIQ5U1LjGJI2GBMV6n\nBqsYAVrg6YuddQ7Lnv+sz5zL7D7fj7Rizjxz+TPSz/+ZM3Pmb+4uADGc0u8GAOgdAg8EQuCBQAg8\nEAiBBwIh8EAgbQfezJaa2Stmtt/M1nayUQC6w9q5D29msyT9RtK1kt6Q9EtJK919X9My3OAH+sjd\nbeK8dnv4KyT9zt0PuPuopJ9IWjHJDk/8rVu37hOf6/ZH+2jfTGpfK+0G/gJJrzd9PljMA1Bj7Qae\n03VgGjq1zfXekDTY9HlQY738JzQajRPTc+fObXNXvZFlWb+bkET7qpnp7cvzXHmely7X7pd2p2rs\nS7u/lHRI0m5N8qVdO9sGUJ2ZySf50q6tHt7dPzSzv5P0n5JmSdrYHHYA9dRWDz+lDdPD19ptt92W\nrG/durV0G8PDw8l63S/jZrJWPTxP2gGBEHggEAIPBELggUAIPBAIgQcCIfBAIO0+WouaO3bsWLK+\nY8eOZP21114r3cfRo0eTde7D1w89PBAIgQcCIfBAIAQeCITAA4EQeCAQAg8Ewn34aeqjjz5K1q+5\n5ppk/eWXX07WFy5cWNqG888/v3QZ1As9PBAIgQcCIfBAIAQeCITAA4EQeCAQAg8Ewn34aeqtt95K\n1svus5dZsmRJ6TJz5syptA/0Hj08EAiBBwIh8EAgBB4IhMADgRB4IBACDwRS6T68mR2Q9H+SPpI0\n6u5XdKJRKLdly5ZK65e9M75s/HhMT1UfvHFJmbunnwIBUAudOKW3DmwDQA9UDbxL2mlmL5rZNzvR\nIADdU/WUfrG7v2lm50jaYWavuPsz48VGo3FiwSzLlGVZxd0BmEye58rzvHQ5c/eO7NDM1kk67u73\nF5+9U9vGpz388MPJ+i233JKsl31pt2fPntI2DA4Oli6D/jAzufunLrfbPqU3s8+Y2eeL6c9K+oqk\nve03EUC3VTmlH5D0MzMb386P3X17R1oFoCvaDry7vyqp/OXl6Ip777230vo33XRTss7p+szEk3ZA\nIAQeCITAA4EQeCAQAg8EQuCBQAg8EAjvpa+pp556Klk/ePBgpe0vX7680vqYnujhgUAIPBAIgQcC\nIfBAIAQeCITAA4EQeCAQ7sP3yejoaLK+devWSts/66yzkvVLLrmk0vYxPdHDA4EQeCAQAg8EQuCB\nQAg8EAiBBwIh8EAg3Ifvk9tvvz1Zf/TRRyttf8mSJcn6wMBApe1jeqKHBwIh8EAgBB4IhMADgRB4\nIBACDwRC4IFAzN3TC5g9Iumrko64+4Ji3tmSfippvqQDkm5w92MT1vOybc9k77//frK+aNGiZH3f\nvn3J+kUXXVRpfcxsZiZ3t4nzp9LD/0DS0gnz7pC0w92/LOnnxWcANVcaeHd/RtLbE2Yvl7SpmN4k\n6boOtwtAF7R7DT/g7iPF9IgkntMEpoHKz9K7u5vZpBfrjUbjxHSWZcqyrOruAEwiz3PleV66XOmX\ndpJkZhdKerLpS7tXJGXuftjM5kna5e5/MmEdvrRL4Es7dFOVL+0m84SkoWJ6SNK2dhsGoHdKA29m\nj0l6XtJFZva6mX1d0ncl/ZWZ/VbSXxSfAdRc6TW8u69sUbq2w22ZUZ5++ulkveop90MPPVRpfcTE\nk3ZAIAQeCITAA4EQeCAQAg8EQuCBQAg8EAjvpW/ToUOHkvWhoaFkvczq1auT9Znwu4TnnnsuWS97\nPLmqc889N1lfsGBBV/ffD/TwQCAEHgiEwAOBEHggEAIPBELggUAIPBAI9+HbtHv37mT93XffrbT9\nFStWJOtmn3p7Uc89++yzyfqtt96arA8PDyfrH3/88Um36WTMnj07Wd+2Lf0ip2XLlnWyOT1BDw8E\nQuCBQAg8EAiBBwIh8EAgBB4IhMADgXAfvoX33nsvWb/rrrsqbf/KK69M1hcvXlxp+2XKhgHbsGFD\n6TZWrVpVaR/9Njo6mqw//vjjyfrSpRNHUf+kOjwrMRE9PBAIgQcCIfBAIAQeCITAA4EQeCAQAg8E\nUnof3swekfRVSUfcfUExryHpbyX9vljsTnf/j241sh+2b9+erO/Zs6fS9u++++5k/fTTT6+0/TKb\nN29O1m+++ebSbVS9zzx//vxk/Z577knWb7zxxmS9avv279+frH/wwQfJ+hlnnFFp/90wlR7+B5Im\nPmHgkr7n7pcVfzMq7MBMVRp4d39G0tuTlOr3GBGApCrX8KvNbI+ZbTSzuR1rEYCuafdZ+u9L+qdi\n+m5J90v6xsSFGo3Gieksy2bEeGhAHeV5rjzPS5drK/DufmR82sw2SHpysuWaAw+geyZ2qK1+3NXW\nKb2ZzWv6+DVJe9vZDoDemsptucckLZH0BTN7XdI6SZmZLdTYt/WvSvpWV1sJoCNKA+/uKyeZ/UgX\n2lIrL7zwQle3f/HFF3d1+2XK3qvfid9yr127Nlm//PLLk/WdO3cm61XbeOaZZybr69evT9breJ+9\nDE/aAYEQeCAQAg8EQuCBQAg8EAiBBwIh8EAgvJd+hvrwww+T9UOHDnW9Dffdd1/X95GyaNGiZP3B\nBx9M1q+66qpONqcW6OGBQAg8EAiBBwIh8EAgBB4IhMADgRB4IBDuw/fJihUrkvWy32qXKXtn+vPP\nP19p+70wa9asZH3NmjXJeqvXPI2bM2fOSbdpuqOHBwIh8EAgBB4IhMADgRB4IBACDwRC4IFAzN27\ns2Ez79a2e2HXrl3J+qpVq5L1srHFI7j00kuT9bLfy1999dXJ+ty5jGHaipnJ3T/14n56eCAQAg8E\nQuCBQAg8EAiBBwIh8EAgBB4IJHkf3swGJf1Q0rmSXNK/uvu/mNnZkn4qab6kA5JucPdjE9ad1vfh\nyxw9ejRZ37RpU7K+d+/eSuufckr63+qysc3POeecZP36669P1qeirI2nnXZa5X1gcu3ehx+VtMbd\nL5F0laRvm9mfSrpD0g53/7KknxefAdRcMvDuftjdh4vp45L2SbpA0nJJ413QJknXdbORADpjytfw\nZnahpMsk/ULSgLuPFKURSQMdbxmAjpvSO+3M7HOStkj6jru/Y/aHSwN3dzOb9GK90WicmM6yTFmW\nVWkrgBbyPFee56XLlQbezGZrLOw/cvdtxewRMzvP3Q+b2TxJRyZbtznwALpnYofa6gWeyVN6G+vK\nN0r6tbs/0FR6QtJQMT0kadvEdQHUT1kPv1jSjZJeMrNfFfPulPRdSZvN7Bsqbst1rYUAOobfwwMz\nEL+HB0DggUgIPBAIgQcCIfBAIAQeCITAA4EQeCAQAg8EQuCBQAg8EAiBBwIh8EAgBB4IhMADgRB4\nIBACDwRC4IFACDwQCIEHAiHwQCAEHgiEwAOBEHggEAIPBELggUAIPBAIgQcCIfBAIAQeCCQZeDMb\nNLNdZvaymf2Pmf19Mb9hZgfN7FfF39LeNBdAFcnx4c3sPEnnufuwmX1O0n9Luk7SDZLecffvJdZl\nfHigT1qND39qaiV3PyzpcDF93Mz2SbpgfJsdbyWArpryNbyZXSjpMkkvFLNWm9keM9toZnO70DYA\nHTalwBen849L+o67H5f0fUlflLRQ0puS7u9aCwF0TPKUXpLMbLakLZL+zd23SZK7H2mqb5D05GTr\nNhqNE9NZlinLsmqtBTCpPM+V53npcmVf2pmkTZKOuvuapvnz3P3NYnqNpD9z97+ZsC5f2gF90upL\nu7LA/7mk/5L0kqTxBf9B0kqNnc67pFclfcvdRyasS+CBPmkr8BV3SOCBPmkVeJ60AwIh8EAgBB4I\nhMADgRB4IBACDwRC4IFACDwQCIEHAiHwQCAEHgiEwAOB9CzwU/mtbj/RvmpoXzW9ah+BL9C+amhf\nNTMu8AD6j8ADgXT1BRhd2TCAKenpG28A1A+n9EAgBB4IpCeBN7OlZvaKme03s7W92OfJMLMDZvZS\nMTDm7hq05xEzGzGzvU3zzjazHWb2WzPb3s/Rflq0rxYDjCYGQK3F8ev3AK1dv4Y3s1mSfiPpWklv\nSPqlpJXuvq+rOz4JZvaqpEXu/la/2yJJZnaNpOOSfujuC4p56yX9r7uvL/7R/CN3v6NG7VunkgFG\ne9S2VgOgfl01OH5VBmjthF708FdI+p27H3D3UUk/kbSiB/s9WbUZHNPdn5H09oTZyzU2KIiK/17X\n00Y1adE+qQbH0N0Pu/twMX1c0vgAqLU4fon2ST04fr0I/AWSXm/6fFB/+B+sC5e008xeNLNv9rsx\nLQw0DfYxImmgn41poVYDjDYNgPoL1fD49WOA1l4Efjrc91vs7pdJWibp28Upa20VI3zU7bjWaoDR\n4nR5i8YGQH2nuVaH49evAVp7Efg3JA02fR7UWC9fG+Pj5Ln77yX9TGOXIXUzUlz/yczmSTpSsnxP\nufsRL0jaoD4ew6YBUH80PgCqanT8Wg3Q2ovj14vAvyjpS2Z2oZmdJumvJT3Rg/1OiZl9xsw+X0x/\nVtJXJO1Nr9UXT0gaKqaHJG1LLNtzRYjGfU19OobFAKgbJf3a3R9oKtXi+LVqX6+OX0+etDOzZZIe\nkDRL0kZ3/+eu73SKzOyLGuvVpbHhs3/c7/aZ2WOSlkj6gsauN/9R0r9L2izpjyUdkHSDux+rSfvW\nScpUMsBoj9o22QCod0rarRocvyoDtHZk/zxaC8TBk3ZAIAQeCITAA4EQeCAQAg8EQuCBQAg8EAiB\nBwL5fwP3x3RKlFLoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1903c2c10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_digit(test_features[4556])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "writer=csv.writer(open(\"/Users/Luba/Desktop/CS189_spring/hw1/digit_test_hog.csv\", 'wb'))\n",
    "for elem in digit_pred_test_hog:\n",
    "    l=[elem]\n",
    "    writer.writerow(l)       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#####################################################################################\n",
    "#################################      Problem 4      ###############################\n",
    "#####################################################################################\n",
    "\n",
    "spam_data_path=\"/Users/Luba/Desktop/CS189_spring/hw1/data/spam-dataset\"\n",
    "spam_data=spy.loadmat(spam_data_path+\"/spam_data.mat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "spam_training_data=spam_data.get('training_data')\n",
    "spam_training_labels=spam_data.get('training_labels')\n",
    "spam_training_labels=spam_training_labels[0]\n",
    "spam_test_data=spam_data.get('test_data')\n",
    "\n",
    "spam_tdata_vector=[]\n",
    "spam_ldata_vector=[]\n",
    "\n",
    "for i in range(len(spam_training_data)):\n",
    "    email=[]\n",
    "    for j in range(len(spam_training_data[0])):\n",
    "        email.append(spam_training_data[i][j])\n",
    "    spam_tdata_vector.append(email)\n",
    "for i in range(len(spam_training_labels)):\n",
    "    spam_ldata_vector.append(spam_training_labels[i])\n",
    "\n",
    "spam_test_data_vector=[]\n",
    "for i in range(len(spam_test_data)):\n",
    "    spam_test_data_vector.append(spam_test_data[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36\n"
     ]
    }
   ],
   "source": [
    "#number of features\n",
    "print(len(spam_tdata_vector[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s_average_performace=[]\n",
    "c_values=[0.0000001, 0.00001, 0.001, 0.1, 1, 10, 100, 1000, 10000]\n",
    "\n",
    "for c_value in c_values:\n",
    "    clf_spam=svm.LinearSVC(C=c_value)\n",
    "    accuracies=[]\n",
    "    for i in range(10):\n",
    "        range11=0\n",
    "        range12=i*517\n",
    "        range21=(i+1)*517\n",
    "        range22=5172\n",
    "        \n",
    "        s_train_data=spam_tdata_vector[range11:range12] + spam_tdata_vector[range21:range22]\n",
    "        s_train_labels=spam_ldata_vector[range11:range12] + spam_ldata_vector[range21:range22]    \n",
    "        \n",
    "        s_test_range1=i*517\n",
    "        s_test_range2=(i+1)*517\n",
    "        \n",
    "        s_test_data=spam_tdata_vector[s_test_range1:s_test_range2]\n",
    "        s_test_labels=spam_ldata_vector[s_test_range1:s_test_range2]\n",
    "        \n",
    "        clf_spam.fit(s_train_data, s_train_labels)\n",
    "        s_pred_labels=clf_spam.predict(s_test_data)\n",
    "        \n",
    "        accuracy=len([i for i,j in zip(s_test_labels, s_pred_labels) if i==j])/len(s_pred_labels)*100\n",
    "        accuracies.append(accuracy)\n",
    "    s_ave=sum(accuracies)/len(accuracies)\n",
    "    s_average_performace.append(s_ave)\n",
    "                                                                                     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[69.168278529980654, 69.92263056092844, 74.796905222437118, 77.911025145067711, 78.104448742746612, 78.665377176015483, 78.355899419729198, 77.311411992263046, 70.58027079303676]\n"
     ]
    }
   ],
   "source": [
    "print(s_average_performace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=10, class_weight=None, dual=True, fit_intercept=True,\n",
       "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "     verbose=0)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_spam=svm.LinearSVC(C=10)\n",
    "clf_spam.fit(spam_tdata_vector, spam_ldata_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s_pred_test=clf_spam.predict(spam_test_data_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5857\n"
     ]
    }
   ],
   "source": [
    "print(len(s_pred_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "writer_spam=csv.writer(open(\"/Users/Luba/Desktop/CS189_spring/hw1/spam_test.csv\", 'wb'))\n",
    "for elem in s_pred_test:\n",
    "    l=[elem]\n",
    "    writer_spam.writerow(l)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}