logistic_regression_baseline_new_alph.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import platform\n",
    "import random\n",
    "import shutil\n",
    "import sys\n",
    "\n",
    "import h5py\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import sklearn.metrics\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.saved_model import tag_constants\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import keras \n",
    "%matplotlib inline\n",
    "random.seed(0)\n",
    "\n",
    "from vis.visualization import visualize_saliency\n",
    "from vis.utils import utils\n",
    "from keras import activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy_plot(model, title): \n",
    "    plt.plot(model.history.history['acc'])\n",
    "    plt.plot(model.history.history['val_acc'])\n",
    "    plt.title(title)\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.legend(['Train', 'Test'], loc='upper left')\n",
    "    plt.show()\n",
    "\n",
    "def loss_plot(model, title): \n",
    "    plt.plot(model.history.history['loss'])\n",
    "    plt.plot(model.history.history['val_loss'])\n",
    "    plt.title(title)\n",
    "    plt.ylabel('Loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.legend(['Train', 'Test'], loc='upper left')\n",
    "    plt.show()\n",
    "    \n",
    "# plotting\n",
    "\n",
    "import itertools\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import svm, datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn import metrics\n",
    "\n",
    "def confusion_plot(y_true_labels, y_pred_labels, title, class_names): \n",
    "\n",
    "    def plot_confusion_matrix(cm, classes,\n",
    "                              normalize=False,\n",
    "                              title='Confusion matrix',\n",
    "                              cmap=plt.cm.Blues):\n",
    "        \"\"\"\n",
    "        This function prints and plots the confusion matrix.\n",
    "        Normalization can be applied by setting `normalize=True`.\n",
    "        \"\"\"\n",
    "        if normalize:\n",
    "            cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "            print(\"All data confusion matrix\")\n",
    "        else:\n",
    "            print('Confusion matrix, without normalization')\n",
    "\n",
    "        print(cm)\n",
    "\n",
    "        plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "        plt.title(title)\n",
    "        plt.colorbar()\n",
    "        tick_marks = np.arange(len(classes))\n",
    "        plt.xticks(tick_marks, classes, rotation=45)\n",
    "        plt.yticks(tick_marks, classes)\n",
    "\n",
    "        fmt = '.2f' if normalize else 'd'\n",
    "        thresh = cm.max() / 2.\n",
    "        for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "            plt.text(j, i, format(cm[i, j], fmt),\n",
    "                     horizontalalignment=\"center\",\n",
    "                     color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "        plt.tight_layout()\n",
    "        plt.ylabel('True label')\n",
    "        plt.xlabel('Predicted label')\n",
    "\n",
    "    # Compute confusion matrix\n",
    "    cnf_matrix = confusion_matrix(y_true_labels, y_pred_labels)\n",
    "    np.set_printoptions(precision=2)\n",
    "\n",
    "    # Plot non-normalized confusion matrix\n",
    "    plt.figure()\n",
    "    plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=False,\n",
    "                          title=title)\n",
    "\n",
    "    plt.show()\n",
    "    \n",
    "    # Plot normalized confusion matrix\n",
    "    plt.figure()\n",
    "    plot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True,\n",
    "                          title=title)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>On-Target Site</th>\n",
       "      <th>Off-Target Site</th>\n",
       "      <th>Score</th>\n",
       "      <th>Encoded Seqs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAAAGAGAAGCTGAGGCACAGGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKREKEKKEZREKEEPKPKEEE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAACGAGAAGAAGAGGCATAGGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKMEKEKKEKKEKEEPKMKEEE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAACGAGAAGAAGCTGCACATGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKMEKEKKEKKEZJEPKPKJEE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAATCATATGAAGAGGCACAAGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKLHKJKREKKEKEEPKPKYEE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAATGAGAAAAATAGGCAGAGGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKLEKEKKYKKJKEEPKHKEEE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  Unnamed: 0  Unnamed: 0.1           On-Target Site  \\\n",
       "0      0           0             0  AAATGAGAAGAAGAGGCACAGGG   \n",
       "1      1           1             1  AAATGAGAAGAAGAGGCACAGGG   \n",
       "2      2           2             2  AAATGAGAAGAAGAGGCACAGGG   \n",
       "3      3           4             4  AAATGAGAAGAAGAGGCACAGGG   \n",
       "4      4           5             5  AAATGAGAAGAAGAGGCACAGGG   \n",
       "\n",
       "           Off-Target Site  Score             Encoded Seqs  \n",
       "0  AAAAGAGAAGCTGAGGCACAGGG    1.0  KKKREKEKKEZREKEEPKPKEEE  \n",
       "1  AAACGAGAAGAAGAGGCATAGGG    1.0  KKKMEKEKKEKKEKEEPKMKEEE  \n",
       "2  AAACGAGAAGAAGCTGCACATGG    1.0  KKKMEKEKKEKKEZJEPKPKJEE  \n",
       "3  AAATCATATGAAGAGGCACAAGG    1.0  KKKLHKJKREKKEKEEPKPKYEE  \n",
       "4  AAATGAGAAAAATAGGCAGAGGG    1.0  KKKLEKEKKYKKJKEEPKHKEEE  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read in train, val, test dfs \n",
    "data_dir = './data/'\n",
    "# diff sheets, so need to read in \n",
    "file_name = data_dir + 'processed_binary_data' \n",
    "train_df = pd.read_csv(file_name+'_train.csv', sep = ',').reset_index()\n",
    "test_df = pd.read_csv(file_name+'_test.csv', sep = ',').reset_index()\n",
    "train_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>On-Target Site</th>\n",
       "      <th>Off-Target Site</th>\n",
       "      <th>Score</th>\n",
       "      <th>Encoded Seqs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAAAGAGAAGCTGAGGCACAGGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKREKEKKEZREKEEPKPKEEE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAACGAGAAGAAGAGGCATAGGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKMEKEKKEKKEKEEPKMKEEE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAACGAGAAGAAGCTGCACATGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKMEKEKKEKKEZJEPKPKJEE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAATCATATGAAGAGGCACAAGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKLHKJKREKKEKEEPKPKYEE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>AAATGAGAAGAAGAGGCACAGGG</td>\n",
       "      <td>AAATGAGAAAAATAGGCAGAGGG</td>\n",
       "      <td>1.0</td>\n",
       "      <td>KKKLEKEKKYKKJKEEPKHKEEE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  Unnamed: 0  Unnamed: 0.1           On-Target Site  \\\n",
       "0      0           0             0  AAATGAGAAGAAGAGGCACAGGG   \n",
       "1      1           1             1  AAATGAGAAGAAGAGGCACAGGG   \n",
       "2      2           2             2  AAATGAGAAGAAGAGGCACAGGG   \n",
       "3      3           4             4  AAATGAGAAGAAGAGGCACAGGG   \n",
       "4      4           5             5  AAATGAGAAGAAGAGGCACAGGG   \n",
       "\n",
       "           Off-Target Site  Score             Encoded Seqs  \n",
       "0  AAAAGAGAAGCTGAGGCACAGGG    1.0  KKKREKEKKEZREKEEPKPKEEE  \n",
       "1  AAACGAGAAGAAGAGGCATAGGG    1.0  KKKMEKEKKEKKEKEEPKMKEEE  \n",
       "2  AAACGAGAAGAAGCTGCACATGG    1.0  KKKMEKEKKEKKEZJEPKPKJEE  \n",
       "3  AAATCATATGAAGAGGCACAAGG    1.0  KKKLHKJKREKKEKEEPKPKYEE  \n",
       "4  AAATGAGAAAAATAGGCAGAGGG    1.0  KKKLEKEKKYKKJKEEPKHKEEE  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['index', 'Unnamed: 0', 'Unnamed: 0.1', 'On-Target Site',\n",
       "       'Off-Target Site', 'Score', 'Encoded Seqs'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Guide length:  23\n",
      "Off-target: 23\n"
     ]
    }
   ],
   "source": [
    "# check lengths -- confirm all are same when comparing across datasets\n",
    "seq_len = len(train_df['On-Target Site'][0])\n",
    "print('Guide length: ', seq_len) # NOTE: WT seq is the reverse complement of guide \n",
    "print('Off-target:', len(train_df['Off-Target Site'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/ipykernel_launcher.py:19: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.\n"
     ]
    }
   ],
   "source": [
    "from pysster.One_Hot_Encoder import One_Hot_Encoder\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "new_alph_map = {'AA': 'K', 'AT': 'R', 'AC': 'Z', 'AG': 'Y', 'TT': 'L', 'TC': 'M', 'TG':'J', \n",
    "                'CC': 'P', 'CG':'H', 'GG': 'E'}\n",
    "\n",
    "new_alph = list(new_alph_map.values())\n",
    "alph_len = len(new_alph)\n",
    "\n",
    "# one-hot encode\n",
    "#modified code from Luis to get correct format for TPOT w/ my alphabet\n",
    "one = One_Hot_Encoder(''.join(new_alph))\n",
    "    \n",
    "# one-hot encode\n",
    "def _get_one_hot_encoding(seq):\n",
    "  \"\"\"Generate one-hot encoding for a single nucleotide sequence.\"\"\"\n",
    "  return pd.get_dummies(\n",
    "      list(seq) + new_alph)[:-len(new_alph)].as_matrix().transpose()\n",
    "\n",
    "# now convert the data into one_hot_encoding \n",
    "input_col_name = 'Encoded Seqs'\n",
    "train_data = np.stack(\n",
    "    [_get_one_hot_encoding(s) for s in train_df[input_col_name]]).astype(np.float32)\n",
    "#val_data = np.stack(\n",
    "    #[_get_one_hot_encoding(s) for s in val_df[input_col_name]]).astype(np.float32)\n",
    "test_data = np.stack(\n",
    "    [_get_one_hot_encoding(s) for s in test_df[input_col_name]]).astype(np.float32)\n",
    "train_data_f = np.reshape(train_data, (-1, alph_len * seq_len))\n",
    "#val_data_f = np.reshape(val_data, (-1, alph_len * seq_len))\n",
    "test_data_f = np.reshape(test_data, (-1, alph_len * seq_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "alph_len = len(new_alph)\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "output_bin_col_name = 'Score'; \n",
    "\n",
    "# NOTE: need one-hot encoding of the labels \n",
    "train_bins = to_categorical(np.array(train_df[output_bin_col_name].astype(np.float32)))\n",
    "#val_bins = to_categorical(np.array(val_df[output_bin_col_name].astype(np.float32)))\n",
    "test_bins = to_categorical(np.array(test_df[output_bin_col_name].astype(np.float32)))\n",
    "\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras as keras\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras import metrics as metrics\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Input, Embedding\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "# new metrics from: http://www.deepideas.net/unbalanced-classes-machine-learning/\n",
    "from keras import backend as K\n",
    "\n",
    "def sensitivity(y_true, y_pred):\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "    return true_positives / (possible_positives + K.epsilon())\n",
    "\n",
    "def specificity(y_true, y_pred):\n",
    "    true_negatives = K.sum(K.round(K.clip((1-y_true) * (1-y_pred), 0, 1)))\n",
    "    possible_negatives = K.sum(K.round(K.clip(1-y_true, 0, 1)))\n",
    "    return true_negatives / (possible_negatives + K.epsilon())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:   41.1s remaining:    0.0s\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:  1.4min remaining:    0.0s\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n",
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:  3.4min finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "X = train_data_f\n",
    "y = np.array(train_df[output_bin_col_name].astype(np.float32))\n",
    "model = LogisticRegressionCV(cv=5, random_state=0, verbose=3).fit(X, y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new = test_data_f\n",
    "y_true = np.array(test_df[output_bin_col_name].astype(np.float32))\n",
    "preds = model.predict(X_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.5\n",
      "Accuracy: 0.9979249060104487 \n",
      "\n",
      "F1: 0.49948068769168735 \n",
      "\n",
      "Confusion matrix, without normalization\n",
      "[[81754     0]\n",
      " [  170     0]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWYAAAEmCAYAAABRfjp6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XmcHFW99/HPNwkhQZaw5sIMSyQRCCpLWIKgsgkBcgkuQAAhCLIIKF71CqjPg6JcQZ9HhEfFByUXokIIIhIlGELU68WHQAKyhXVYIllYQsK+xITf88c5nVTGmZ6eSTfdPfN9z6teU32q6tTpru5fnz516pQiAjMzaxz96l0AMzNbnQOzmVmDcWA2M2swDsxmZg3GgdnMrME4MJuZNZi6BmZJgyX9TtLLkq5fg3yOk3RrNctWD5JukTSh3uXoKUlPSzowz39N0s/rXaae6umxkPRhSY/WokyNTNJWkl6T1L9G+V8r6Ygq5neDpEOqlV/VRUSXE3AsMAd4DVgE3ALsU8m2XeR7PHAXMGBN86rFBOwLBHBju/SdcvqfK8znm8Av38VynwisyMfrNeBJ4HPvwn6fBg6swzGaX8f3SADDq5DPN4F/5OP1EvD/gL3q9bwaaQI+CDwEqJC2OXBljkevAo8A3wLek+dP6iCfs4E5eX4P4O56P7fOpi5rzJK+BPwQ+A9gKLAV8BNgXFfbVmBr4LGIWF6FvGrlBWAvSRsX0iYAj1VrB0qq/evljohYNyLWBT4JfE/SLlXeh1XXdfl4bQL8Cejxr8hyJA2oRb41dBrwq8gRVdJGwB3AYNKX13rAx4AhwLbA1cAJHeRzfF5GRNwFrC9pt9oXvwe6+KbagPQNfmSZddYmBe6FefohsHZeti8wH/gy8Dzp2+0zedm3gGWsqiWcTLuaJbANqUYyID8+kVT7exV4CjiukH57YbsPAbOBl/P/DxWW/Rn4NvDXnM+twCadPLdS+X8KnJnT+gMLgP9JocYMXAo8A7wC3A18OKePafc87yuU48JcjjeB4Tnts3n55cANhfwvBmZSqDWUOSarvR457S7g2MLj64Fn82v0F2DHwrJDSTWUV/Nz/Uph2VjgXlbV6j5YWPY0ucZcPJaF4zgB+DuwGPh6Ybt+wLnAE8CLwBRgowprU/vSSY2Z9P6dRPpynQd8A+hXOI7/O5flKeAsVn+vFY/FcOC/8mu1mBRAya9bAK/nY3t0+/IAWwK/yWV4EfhRJ2Vd+XrlxyNz3ptW+NrvCvwtH7PrgeuA77R7H5+Tj/kvKsjvnHzsXwUeBQ7I6XuQfj2/AjwH/KCTz+oWwFRgCdAGnNLuuU7Jx+ZVYC6wW5lj/CSFX+jAd4AHSseyg/VbgeXA1u1ez2UUPuvAz4DzK3mfvdtTV2/6MfkJdtrUAFwAzAI2AzbNB/jbhTfE8rzOWqQP/BvAhp28Gds/XnmwST9RXgG2y8s2JwcTCoEI2AhYSvp2HAAckx9vXPjAPQG8j/SN+2fgonIfelKgvzOnHQpMBz7L6oH508DGeZ9fJn0ABnX0vArl+DuwY95mLVYPBuuQauUnAh8mBYTWwvYv0UlzEv/8RbV7Xv99hbSTgPVY9cV6b2HZIlZ9sWwI7JrndyF9we5JCmwTSMG49EX8NOUD88/ya74T8DawQ15+Nuk91JrL83+BawvluZ/Cl0pHx6iTZZOAm/Lz3Ca/nifnZaeTvnxa83O8jc4D87XA10lfIINYPUis1pRRLE9+je4DLiG9f1fbtl1Zi6/XQOCifMwHdPXa5/Xn5ddxLeATpCBUDMzLSV/ua+djUC6/7UiVjC0Kx2/bPH8HcHyeXxcY3f6zmh//hfTLehCwM+mLaf/Cc32L9FnqD3wXmNXJ6/Ie/vkLahbwrS5i1wzgG4XH3wV+226dLwG/qXZQrcbUVWA+Dni2i3WeAA4tPD4YeLrwhniTQmDPb4bSwVz5Zuzk8cqDnQ/QS6Sf5YPbleFEVgXm44G72i2/Azix8IErHrAzgD909aEHHs9v2Mn5dVktMHew7VJgp46eV6EcF3SQ9tnC4z1JNY55wDEVH9T0eizPr9er+TX8P3RS2yb9BAxgg/z476Sfj+u3W+9y8pduIe1R4KN5/mnKB+biF8tdwPg8/zC5RpYfb076hdHluQc6CcykD/wyYGQh7bTSMQP+CJxWWHYgnQfmScAVxfIXtisXmPciBaRKnsc3c3lfIp0feBHYt5LXHvgIqXZbbIO9ndUD8zJyRaGC/IaTPqcHAmu1W+cvpF+7m7RLLx3jAaRfCSuA9QrLvwtcVXiutxWWjQTe7OR1acn5Fsv+OHB6F6/np4FH83w/0nv64+3WOQX4Y6Wfq3dz6qpd80Vgky7apLYgBY6SeTltZR6xehvyG6Rv2m6JiNdJPxVPBxZJulnS9hWUp1SmlsLjZ3tQnl+Qfu7uB9zYfqGkr0h6OPcweYn0M3qTLvJ8ptzCiLiT9DNOpJ9+3TErIoZEan/7F1LN/D9yWftLukjSE5JeIQVUCuX9JKk2M0/Sf0naK6dvDXxZ0kulifQhLB7vcjp73bcGbizk+TDpgz20m8+5aBNS7bH9e7P0PtiC1V//csfiq6RjcJekuZJOqrAMWwLzovJzKFMiYgjpeT8IjCosK/fabwEsiBxtOnk+L0TEW5XkFxFtwBdJAfR5SZMllY7xyaRfm49Imi1pbAfPYwtgSUS8Wkjr6jM4qJM481L+v14h7UXSl3c5vwE2lzSa9MW0DnBzu3XWK+TfULoKzHeQfnKW66aykHSQS7bKaT3xOukFLPmX4sKImB4RHyMdlEdIP427Kk+pTAt6WKaSX5Bq19Mi4o3iAkkfJn14jyI10wwhtUeqVPRO8uwsvZTvmaSflgtz/j0SEc8BNwD/mpOOJZ28PZD0BbJNaZd5/dkRMY7UPPVbVn0pPANcmAN+aVonIq7tadkK+R7SLt9BEbEmx2wxqdbd/r1ZynMRqRmjZMvOMoqIZyPilIjYglTr/omk4RWU4Rlgq+6ebIuIxcCpwDcllQJQudd+EdAiSYVs2j+f9u+1sscyIq6JiH1Ir1+QmkGIiMcj4hjSe+Ni4NeS3tMu74XARpKKwbRHn8FcISs1PZbcBny83Anz/Bn9Nekk4PHA5IhY1m61HUhNTQ2nbGCOiJdJJ7l+LOkISetIWkvSIZK+l1e7FviGpE0lbZLX/2UPy3Mv8JHcJ3ID4LzSAklDJY3Lb4K3SSdb3ukgj2nA+yQdK2mApKNJP5V+38MyARART5F+5n29g8XrkZoOXgAGSPqfwPqF5c8B23Sn54Wk95FOcnya9Mb6qqSde1L23KPk46STLKXyvk2qeaxDrknndQfmfuEbRMQ/SO36pdf5Z8DpkvbMPUneI+mwdh/AnvgpcKGkrXMZNpXUrV4/kgYVp1zmKTnf9XLeX2LVe3MKcLakFklDSCe7Osv7SEmlIL6UFKhKr8lzwHs72fQuUtC8KL9WgyTtXcnziYhHSecySl/I5V77O0i/MM7K7/lxpJN05XSan6TtJO0vaW1SW/Cbpecr6dOSNo2Id1hV21ztcxgRz5DONX03P+cPkmraPY0L00ifvZIfkD5fVxfeMy2SfpD3VXI16Vf2J/N8ex8ldf1tOF0Gioj436Q39DdIgecZ0k/63+ZVvkM6S3s/6UzpPTmt2yJiBuls8v2kng3FYNovl2Mhqd31o8DnOsjjRdLZ5i+TAs9XgbG5FrJGIuL2iOjo18B04A+kk0vzSG/m4k/JUrenFyXd09V+cg3rl8DFEXFfRDwOfA34Rf6woNSZ/8Nlstkrr/MaqWngBeDzedmkXM4FpBNgs9ptezzwdG7mOJ3Upk5EzCG1y/2IFKDaSO3Za+pS0hn8WyW9msuzZ2lhbj44rsz2LaTgUZy2JT3f10nNQbcD1wAT8zY/I/XIuZ/Um2Ea6ct1RQf57w7cmV/LqcDZEfFkXvZNUoB4SdJRxY0iYgXpV8pwUhvnfFKgqNT3gVMlbVbutc81wU+Qgt9LpC/z35O+fDvUxbFcm1UnH58l1Y5LlaQxwNz8WlxKOk/wZge7OIb0S2whqenv/Ii4rRvPvegK4LjSL4KIWEI6If8P0nF5ldRj6eX8PEr+ktPmR8TsYoaSdgdei9RtruFo9WYps75J6Sqwn0ZE+2awpiTpTtLz+c96l6UaJF1DaoP/bZcrV5bfDcCVETGtGvlVmwOz9UmSBpNO5N5KOtl2A+mE6RfrWrAekvRRUq+KxaRfOD8F3hsRi+paMOsRD2JkfZVI3b6WkpoyHiadH2lW25FOZL1Easb7lINy83KN2cyswbjGbGbWYJptMJMOacDg0MA17bFljWKXHbaqdxGsSubNe5rFixer6zUr13/9rSOWd9QRpGPx5gvTI2JMNctQa70jMA9cj7W3O6rrFa0p/PXOH9W7CFYle+9Z/cHbYvmb3fq8v3Xvj7u6ArfhuCnDzJqMQP0qn7rKTfq33Ff+QaUB+QdJGibpTkltkq6TNDCvu3Z+3JaXb1PI57yc/qikgwvpY3Jam6RzK3mGDsxm1lwESJVP5bKSWoAvkIYdfT9p8KvxpMvNL4mI4aSeOyfnTU4Glub0S/J6SBqZt9uRdBHOT5TGpOkP/Bg4hHQF8jF53bIcmM2s+VSxxkxq0h2cr7hdh3QZ/f6ksTYgXc5dGi9oHKsu7/41cEC+InEcaTyOt/PwDW2ky+L3ANoi4sl8heZkKrjJiAOzmTUZQb/+lU9phMw5henUUk55oKz/RbpkfhHpEu67gZcKowLOZ9XIeC3k4Rby8pdJ47CvTG+3TWfpZfWKk39m1sd00UTRzuKI6PAspKQNSTXYYaSLc64nNUXUlQOzmTUXUWkTRSUOBJ6KiBcAJP0G2BsYImlArhW3smrI0gWkIVXn56aPDUiDpZXSS4rbdJbeKTdlmFmT6caJv65r1n8HRisNaSzgANKIi38CPpXXmUC6RRmk0QUn5PlPke6AEjl9fO61MQwYQRr2dTYwIvfyGEg6QTi1q0K5xmxmzadKNeaIuFPSr0nDFS8njZtyBeluJ5MlfSenXZk3uZI0/G4bafjh8TmfuZKmkIL6ctLNm1cASDqLNDRwf2BiRJTGRe+UA7OZNZ/utTGXFRHnA+e3S36SDm42kG/PdWQn+VwIXNhB+jTSeN8Vc2A2syajarYxNyQHZjNrLqULTHoxB2Yzaz6uMZuZNRJB//71LkRNOTCbWXOpbj/mhuTAbGbNx23MZmaNxL0yzMwaj2vMZmYNxjVmM7MGUtkYGE3NgdnMmo9rzGZmDcY1ZjOzRuJeGWZmjUWUbhnVazkwm1mTcY3ZzKzxuI3ZzKzBuMZsZtZgXGM2M2sgchuzmVnj6eU15t79tWNmvZKkiqcu8tlO0r2F6RVJX5S0kaQZkh7P/zfM60vSZZLaJN0vaddCXhPy+o9LmlBIHyXpgbzNZeqqUDgwm1mTSbf8q05gjohHI2LniNgZGAW8AdwInAvMjIgRwMz8GOAQYESeTgUuJ5VnI9Kdtvck3V37/FIwz+ucUthuTFfP0YHZzJqLhPpVPnXDAcATETEPGAdcndOvBo7I8+OASZHMAoZI2hw4GJgREUsiYikwAxiTl60fEbMiIoBJhbw65TZmM2s6FbQGFG0iaU7h8RURcUUH640Hrs3zQyNiUZ5/Fhia51uAZwrbzM9p5dLnd5BelgOzmTWdbgbmxRGxWxf5DQQOB85rvywiQlJ0r4Rrxk0ZZtZ0qtXGXHAIcE9EPJcfP5ebIcj/n8/pC4AtC9u15rRy6a0dpJflwGxmzUXdnCpzDKuaMQCmAqWeFROAmwrpJ+TeGaOBl3OTx3TgIEkb5pN+BwHT87JXJI3OvTFOKOTVKTdlmFlTEd2qCXedn/Qe4GPAaYXki4Apkk4G5gFH5fRpwKFAG6kHx2cAImKJpG8Ds/N6F0TEkjx/BnAVMBi4JU9lOTCbWdOpZmCOiNeBjdulvUjqpdF+3QDO7CSficDEDtLnAO/vTpkcmM2s6VQzMDciB2YzazoOzGZmjaR7J/WakgOzmTUVIfr1690dyhyYzazpuCnDzKzR9O647MBsZk1GrjGbmTUcB2YzswbjwGxm1kCqfUl2I3JgNrPm07vjsgPzu+3zx+3HiR//EBHB3LaFnHr+L/nMxz/EWcfux7ZbbUrrfufw4kuvA/BvJxzA0YfuDsCA/v3Yfti/sOX+57L0lTd45OZv8errb7PinXdYvuId9jnue6vt5+zj9+eiL31itfysMdw6/Q985Utns2LFCk486bP8+1fP7XojW8Un/6yatth0A8445qPs8skLeevtf/DLi0/iyINHcce9TzLtLw9y68/PXm39SybN5JJJMwE49CPv5/PH7cfSV95YuXzMqZd2GHRbhw7hgNE78PdFS/5pmdXXihUr+OIXzuTmW2bQ0trKPqN3Z+zYw9lh5Mh6F62p9PbA3Lsvn2lAA/r3Z/Daa9G/fz8GDxrIohde5r5H53cZRI8asxtT/nB3Rfv43lc+ydcv/S1pICxrJLPvuotttx3OsPe+l4EDB3Lk0eP5/e+6HJ7X2qnRPf8ahgPzu2jhCy/zw0kzeeyWb/PUjAt55bU3mTnrkS63GzxoLT72oR347cx7V6ZFBL/7yVn89Vdf5aRP7L0yfey+H2Dh8y/xwGNd3iTB6mDhwgW0tq660UVLSysLFvhYdVcN7mDSUBoyMEsaI+lRSW2Sek0D3JD1BjN23w+ww9jzee9BX+c9gwcyPrchl3PYRz7AHfc+uVozxgGfuYQPHXsxR5z1E047+sPsveu2DB60Fl896WAuuPzmWj4Ns7rqTlB2YK4SSf2BH5PuwTUSOEZSr2iA23/P7Xl64YssXvoay5e/w2//eB+jdxrW5XZHHjyK69s1Yyx84WUAXlj6GlP/eD+777gN723dlK1bNuau687jkZu/RctmQ7jjmnMYuvF6NXk+1n1bbNHC/Pmrbqa8YMF8Wlq6vGmytePA/O7bA2iLiCcjYhkwGRhX5zJVxTPPLmGPDwxj8KC1ANhvj+149Knnym6z/rqD2GfUcH735/tXpq0zaCDrrrP2yvkD99qeuU8sZG7bQrY+4Dy2P+x8tj/sfBY8/xJ7HXsxz734au2elHXLbrvvTlvb4zz91FMsW7aM66+bzGFjD693sZpObw/MjdgrowV4pvB4PrBn+5UknQqcCsBa674rBVtTsx+cx423/Y07rjmH5Sve4b5H5nPlDX/ljGM+ypcmHMjQjddn9pSv8Yfb53LGBdcAcPh+OzFz1iO88daylflstvF6XPeDU4B0MvG6W+Yw4/89XJfnZN0zYMAALrn0R/zrYQezYsUKJpx4EiN33LHexWo+zRlvK6ZGO3Mv6VPAmIj4bH58PLBnRJzV2Tb91tks1t7uqM4WW5NZOvtH9S6CVcnee+7G3XfPqWoYXXvoiGg57tKK13/qksPujojdOlsuaQjwc9J9+QI4CXgUuA7YBngaOCoiluY7XV9KuiHrG8CJEXFPzmcC8I2c7Xci4uqcPopVN2OdBpwdXQTeRmzKWABsWXjcmtPMzFZeYFLFpoxLgT9ExPbATsDDwLnAzIgYAczMjyGd+xqRp1OBywEkbQScT/p1vwdwvqQN8zaXA6cUthvTVYEaMTDPBkZIGiZpIDAemFrnMplZgxAgVT6VzUvaAPgIcCVARCyLiJdI57WuzqtdDRyR58cBkyKZBQyRtDlwMDAjIpZExFJgBjAmL1s/ImblWvKkQl6darg25ohYLuksYDrQH5gYEXPrXCwzaxiiX/cuHNlE0pzC4ysi4oo8Pwx4AfhPSTsBdwNnA0MjYlFe51lgaJ7v6BxYSxfp8ztIL6vhAjNAREwjtcWYmf2Tbva2WFymjXkAsCvw+Yi4U9KlrGq2ACAiQtK7ejKuEZsyzMw6141mjAri93xgfkTcmR//mhSon8vNEOT/z+flnZ0DK5fe2kF6WQ7MZtZUBPTrp4qnciLiWeAZSdvlpAOAh0jntSbktAlAaUCTqcAJSkYDL+cmj+nAQZI2zCf9DgKm52WvSBqde3ScUMirUw3ZlGFmVk6Vrxv5PPCr3NngSeAzpErrFEknA/OAUn/caaSucm2k7nKfAYiIJZK+Teq8AHBBRJRGJjuDVd3lbslTWQ7MZtZ0qnlFX0TcC3TUBn1AB+sGcGYn+UwEJnaQPofUR7piDsxm1lwqaztuag7MZtZUUj/m3h2ZHZjNrMk07+BElXJgNrOm08vjsgOzmTUZ0d0r/5qOA7OZNRW3MZuZNaBeHpcdmM2s+bjGbGbWYHp5XHZgNrMmI9eYzcwaSmmg/N7MgdnMmowvMDEzazi9PC47MJtZk/EFJmZmjcUXmJiZNSAHZjOzBtPL47IDs5k1H9eYzcwaSR+4g4nvkm1mTUW5H3OlU5f5SU9LekDSvZLm5LSNJM2Q9Hj+v2FOl6TLJLVJul/SroV8JuT1H5c0oZA+KufflrftslAOzGbWdKTKpwrtFxE7R0TppqznAjMjYgQwMz8GOAQYkadTgctTebQRcD6wJ7AHcH4pmOd1TilsN6arwjgwm1nT6SdVPPXQOODqPH81cEQhfVIks4AhkjYHDgZmRMSSiFgKzADG5GXrR8SsfIftSYW8On9+PS21mVm9dLPGvImkOYXp1HbZBXCrpLsLy4ZGxKI8/ywwNM+3AM8Utp2f08qlz+8gvSyf/DOzpiJB/+5d+be40ETRkX0iYoGkzYAZkh4pLoyIkBQ9KWtPucZsZk2nmif/ImJB/v88cCOpjfi53AxB/v98Xn0BsGVh89acVi69tYP0sqoemCWtX26q9v7MrO+p1sk/Se+RtF5pHjgIeBCYCpR6VkwAbsrzU4ETcu+M0cDLucljOnCQpA3zSb+DgOl52SuSRufeGCcU8upULZoy5pLabIovSelxAFvVYJ9m1keI1GWuSoYCN+aa9QDgmoj4g6TZwBRJJwPzgKPy+tOAQ4E24A3gMwARsUTSt4HZeb0LImJJnj8DuAoYDNySp7KqHpgjYsuu1zIz67lqDS4XEU8CO3WQ/iJwQAfpAZzZSV4TgYkdpM8B3t+dctW0jVnSeElfy/OtkkbVcn9m1gd0o325WS/drllglvQjYD/g+Jz0BvDTWu3PzPqOGlxg0lBq2V3uQxGxq6S/wco2mIE13J+Z9QGCNblwpCnUMjD/Q1I/0gk/JG0MvFPD/ZlZH9HL43JN25h/DNwAbCrpW8DtwMU13J+Z9RG9vY25ZjXmiJgk6W7gwJx0ZEQ8WKv9mVnf0IMr/5pOrS/J7g/8g9Sc4asMzawqendYrm2vjK8D1wJbkC5DvEbSebXan5n1HW7K6LkTgF0i4g0ASRcCfwO+W8N9mlkvl3pl1LsUtVXLwLyoXf4DcpqZWc81cU24UlUPzJIuIbUpLwHmSpqeHx/EquvIzcx6rJfH5ZrUmEs9L+YCNxfSZ9VgX2bWB7nG3E0RcWW18zQzK3Eb8xqQtC1wITASGFRKj4j31WqfZtY39PYacy37Fl8F/CfpC+4QYApwXQ33Z2Z9gAT9pYqnZlTLwLxOREwHiIgnIuIbpABtZrZGPLpcz72dBzF6QtLppPtcrVfD/ZlZH9HbmzJqGZj/DXgP8AVSW/MGwEk13J+Z9RG9PC7XdBCjO/Psq6waLN/MbI0IeTzm7pJ0I3kM5o5ExCeqvU8z60OauO24UrWoMf+oBnmWtcsOW/HXO9/13ZpZnVS7jVlSf2AOsCAixkoaBkwGNgbuBo6PiGWS1gYmAaOAF4GjI+LpnMd5wMnACuALpc4PksYAl5JG2/x5RFzUVXlqcYHJzGrnaWZWVIPuZGcDDwPr58cXA5dExGRJPyUF3Mvz/6URMVzS+Lze0ZJGAuOBHUkjat4mqXTNxo+BjwHzgdmSpkbEQ+UK4zGSzaypiOoO+ympFTgM+Hl+LGB/4Nd5lauBI/L8uPyYvPyAvP44YHJEvB0RTwFtwB55aouIJyNiGakWPq6rMjkwm1nT6afKJ2ATSXMK06ntsvsh8FVW3ZN0Y+CliFieH88HWvJ8C/AMQF7+cl5/ZXq7bTpLL6vWdzBB0toR8Xat92NmfUMPbi21OCJ26zgvjQWej4i7Je1bjfJVQy3vYLKHpAeAx/PjnST9n1rtz8z6jm7WmMvZGzhc0tOkZob9SSfqhkgqVVxbSRfIkf9vCZCXb0A6Cbgyvd02naWXf35dFrvnLgPGkgpNRNwH7FfD/ZlZH1GtS7Ij4ryIaI2IbUgn7/4YEccBfwI+lVebANyU56fmx+Tlf4yIyOnjJa2de3SMAO4ijUE/QtIwSQPzPqZ29fxq2ZTRLyLmtWt8X1HD/ZlZH5CG/ax5R+ZzgMmSvkO6JV5pOOMrgV9IaiPdDGQ8QETMlTQFeAhYDpwZESsAJJ0FTCd1l5sYEXO72nktA/MzkvYAIvcR/DzwWA33Z2Z9RC1+6kfEn4E/5/knST0q2q/zFnBkJ9tfSBp+on36NGBad8pSy8D8OVJzxlbAc8BtOc3MbI34yr8eiojnydV8M7NqkTxWRo9J+hkdjJkREe37EJqZdUsvj8s1bcq4rTA/CPg4q3e0NjPrEd/zr4ciYrXbSEn6BXB7rfZnZn2D6PYFJk2n5lf+FQwDhr6L+zOz3qiyC0eaWi3bmJeyqo25H6nP37m12p+Z9R2id0fmmgTmPNrSTqy69PCdfHWMmdkaSReY1LsUtVWTS7JzEJ4WESvy5KBsZlVTxbEyGlItx8q4V9IuNczfzPqoao7H3Ihqcc+/AXmc0l1Io/U/AbxO+gUSEbFrtfdpZn1HX2jKqEUb813ArsDhNcjbzPo634y1RwQQEU/UIG8zM1+S3QObSvpSZwsj4gc12KeZ9RFuyuiZ/sC60Ms7GppZnYj+rjF326KIuKAG+ZqZ5btk17sUtVWzNmYzs5po4v7JlapFYD6gBnmama3kk3/dFBFLqp2nmVlJX2jKqOWVf2ZmNdEv38WkkqkcSYMk3SXpPklzJX0rpw+TdKekNknX5Ttck++CfV1Ov1PSNoW8zsvpj0o6uJA+Jqe1SapoIDcHZjNrOlLlUxfeBvaPiJ2AnYExkkYDFwOXRMRwYClwcl7/ZGCnjhkfAAAPFElEQVRpTr8kr4ekkaRb6e0IjAF+Iql/vhH1j4FDgJHAMXndshyYzaypiBS4Kp3KieS1/HCtPAWwP/DrnH41cESeH5cfk5cfkEfTHAdMjoi3I+IpoI10l+09gLaIeDIilgGT87plOTCbWXNRtwcx2kTSnMK02n1Hc832XuB5YAbwBPBSHvMHYD7QkudbyLfIy8tfBjYuprfbprP0st7NO5iYmVVFN8/9LY6I3TpbGBErgJ0lDQFuBLZfo8JVgQOzmTUVQU2u/IuIlyT9CdgLGFIYKbOVVTf9WABsCcyXNADYAHixkF5S3Kaz9E65KcPMmk61Tv5J2jTXlJE0GPgY8DDwJ+BTebUJwE15fmp+TF7+x3wjkKnA+NxrYxgwgjTS5mxgRO7lMZB0gnBqV8/PNWYzazJVHQB/c+Dq3HuiHzAlIn4v6SFgsqTvAH8DrszrXwn8QlIb6T6m4wEiYq6kKcBDwHLgzNxEgqSzgOmkcYQmRsTcrgrlwGxmTaXUK6MaIuJ+0k092qc/SepR0T79LeDITvK6ELiwg/RpwLTulMuB2cyaTrPeMqpSDsxm1nR6d1h2YDazZiPXmM3MGko125gblQOzmTUd15jNzBqMB8o3M2sgqSmjd0dmB2Yzazq9vCXDgdnMmo2Qa8xmZo3FNWYzswbiNmYzs0ZT2S2jmpoDs5k1HQdmM7MG09tP/vX2KxubxmmfPYmtttiMUTu/f2Xap489mj1H7cyeo3Zmu+HbsOeonVcu+/7F32XH7YfzwR23Y8at0+tRZOuhW6f/gQ/uuB07bj+c73/vonoXp+mIdIFJpVMzco25QRw/4UROP+MsPnvSCSvTfnnNdSvnz/n3L7PBBhsA8PBDD3H9dZO55765LFq4kEPHHMgDDz1G//793/VyW/esWLGCL37hTG6+ZQYtra3sM3p3xo49nB1GdnlHeyvo18vbMlxjbhD7fPgjbLTRRh0uiwhu+PUUjjr6GAB+/7ubOPLo8ay99tpsM2wY2247nNl33fVuFtd6aPZdd7HttsMZ9t73MnDgQI48ejy//91NXW9oq1E3/pqRA3MT+Ovt/83QzYYyfMQIABYsWEBr66r7O7a0tLJwYZf3d7QGsHDhPx+7BQt87LqjLzRlNFxgljRR0vOSHqx3WRrFlMnXcuT4Y+pdDLMG0Z36cnNG5oYLzMBVwJh6F6JRLF++nJt++xs+deTRK9NaWlqYP/+ZlY8XLJjPFlu01KN41k1bbPHPx66lxceuW7pxh+wK7pK9paQ/SXpI0lxJZ+f0jSTNkPR4/r9hTpekyyS1Sbpf0q6FvCbk9R+XNKGQPkrSA3mby1TBmKUNF5gj4i+ku88a8MeZt/G+7bantbV1ZdphYw/n+usm8/bbb/P0U0/R1vY4u+/xT/eNtAa02+6709b2OE8/9RTLli3j+usmc9jYw+tdrKajbkxdWA58OSJGAqOBMyWNBM4FZkbECGBmfgxwCDAiT6cCl0MK5MD5wJ6km7ieXwrmeZ1TCtt1WfFsuMBcKUmnSpojac4Li1+od3HW2AmfPoZ9P7wXjz36KNtu08pVE9Pd0q+/bvLKk34lI3fckU8eeRS7fHAkh48dww8v+7F7ZDSJAQMGcMmlP+JfDzuYnT+wA5888ihG7rhjvYvVVFIbsyqeyomIRRFxT55/FXgYaAHGAVfn1a4Gjsjz44BJkcwChkjaHDgYmBERSyJiKTADGJOXrR8RsyIigEmFvDrVtN3lIuIK4AqAUaN2izoXZ41N+uW1Hab/bOJVHaafc97XOee8r9ewRFYrYw45lDGHHFrvYjS1WrQcS9oG2AW4ExgaEYvyomeBoXm+BXimsNn8nFYufX4H6WU1bWA2sz6se5F5E0lzCo+vyBW7VdlJ6wI3AF+MiFeKzcAREZLe1cqfA7OZNZ1uXmCyOCJ262yhpLVIQflXEfGbnPycpM0jYlFujng+py8Atixs3prTFgD7tkv/c05v7WD9shqujVnStcAdwHaS5ks6ud5lMrPGUq2Tf7mHxJXAwxHxg8KiqUCpZ8UE4KZC+gm5d8Zo4OXc5DEdOEjShvmk30HA9LzsFUmj875OKOTVqYarMUeEO+yaWXnVa2TeGzgeeEDSvTnta8BFwJRcMZwHHJWXTQMOBdqAN4DPAETEEknfBmbn9S6IiFLvsjNI3YAHA7fkqayGC8xmZuWkmnB1InNE3E7nYf6ADtYP4MxO8poITOwgfQ7w/n/eonMOzGbWXDxQvplZ4+nlcdmB2cyaUC+PzA7MZtZkmndwoko5MJtZ03Ebs5lZA6lwcKKm5sBsZk2ngpEzm5oDs5k1nV4elx2Yzaz59PK47MBsZk2mDzQyOzCbWdNxdzkzswYi3MZsZtZwenlcdmA2sybUyyOzA7OZNR23MZuZNZh+vTsuOzCbWRNyYDYzaxzVvINJo3JgNrPm4juYmJk1nl4el+lX7wKYmXWbujF1lZU0UdLzkh4spG0kaYakx/P/DXO6JF0mqU3S/ZJ2LWwzIa//uKQJhfRRkh7I21ymCobGc2A2syajbv1V4CpgTLu0c4GZETECmJkfAxwCjMjTqcDlkAI5cD6wJ7AHcH4pmOd1Tils135f/8SB2cyajlT51JWI+AuwpF3yOODqPH81cEQhfVIks4AhkjYHDgZmRMSSiFgKzADG5GXrR8SsiAhgUiGvTrmN2cyaSg8Gl9tE0pzC4ysi4oouthkaEYvy/LPA0DzfAjxTWG9+TiuXPr+D9LIcmM2s+XQvMi+OiN16uquICEnR0+17wk0ZZtZ0+kkVTz30XG6GIP9/PqcvALYsrNea08qlt3aQXv759bTUZmb1UsVOGZ2ZCpR6VkwAbiqkn5B7Z4wGXs5NHtOBgyRtmE/6HQRMz8tekTQ698Y4oZBXp9yUYWbNpcoXmEi6FtiX1BY9n9S74iJgiqSTgXnAUXn1acChQBvwBvAZgIhYIunbwOy83gURUTqheAap58dg4JY8leXAbGZNqHqROSKO6WTRAR2sG8CZneQzEZjYQfoc4P3dKZMDs5k1Fd/BxMysAfXyuOzAbGbNxzVmM7MG42E/zcwaTe+Oyw7MZtZ8enlcdmA2s+YisSZX9DUFB2Yzaz69Oy47MJtZ8+nlcdmB2cyaTy9vyXBgNrNmU/GdSZqWA7OZNZW+cEm2h/00M2swrjGbWdPp7TVmB2YzazpuYzYzayDpApN6l6K2HJjNrPk4MJuZNRY3ZZiZNZjefvLP3eXMrOlU8y7ZksZIelRSm6Rza1TkbnFgNrPmU6XILKk/8GPgEGAkcIykkbUqdqUcmM2s6agbf13YA2iLiCcjYhkwGRhX8yfQhV7RxnzPPXcvHryW5tW7HO+CTYDF9S6EVU1fOJ5bVzvDv91z9/R1BmqTbmwySNKcwuMrIuKKPN8CPFNYNh/Yc03LuKZ6RWCOiE3rXYZ3g6Q5EbFbvcth1eHj2TMRMabeZag1N2WYWV+2ANiy8Lg1p9WVA7OZ9WWzgRGShkkaCIwHpta5TL2jKaMPuaLrVayJ+HjWWUQsl3QWMB3oD0yMiLl1LhaKiHqXwczMCtyUYWbWYByYzcwajAOzmVmDcWBuYJK2k7SXpLXypaPWC/hYWld88q9BSfoE8B+kPpULgDnAVRHxSl0LZj0m6X0R8Vie7x8RK+pdJmtMrjE3IElrAUcDJ0fEAcBNpE7w50hav66Fsx6RNBa4V9I1ABGxwjVn64wDc+NaHxiR528Efg+sBRwr9fbRaHsXSe8BzgK+CCyT9EtwcLbOOTA3oIj4B/AD4BOSPhwR7wC3A/cC+9S1cNZtEfE6cBJwDfAV0qA6K4NzPctmjcmBuXH9N3ArcLykj0TEioi4BtgC2Km+RbPuioiFEfFaRCwGTgMGl4KzpF0lbV/fEloj8SXZDSoi3pL0KyCA8/IH921gKLCoroWzNRIRL0o6Dfi+pEdIlwLvV+diWQNxYG5gEbFU0s+Ah0i1rLeAT0fEc/Utma2piFgs6X7SnTM+FhHz610maxzuLtck8kmiyO3N1uQkbQhMAb4cEffXuzzWWByYzepE0qCIeKve5bDG48BsZtZg3CvDzKzBODCbmTUYB2YzswbjwGxm1mAcmPs4SSsk3SvpQUnXS1pnDfLaV9Lv8/zhks4ts+4QSWf0YB/flPSVStPbrXOVpE91Y1/bSHqwu2U0W1MOzPZmROwcEe8HlgGnFxcq6fb7JCKmRsRFZVYZAnQ7MJv1BQ7MVvTfwPBcU3xU0iTgQWBLSQdJukPSPblmvS6ApDGSHpF0D/CJUkaSTpT0ozw/VNKNku7L04eAi4Btc239+3m9f5c0W9L9kr5VyOvrkh6TdDuwXVdPQtIpOZ/7JN3Q7lfAgZLm5PzG5vX7S/p+Yd+nrekLabYmHJgNAEkDSJcHP5CTRgA/iYgdgdeBbwAHRsSupEH7vyRpEPAz4F+BUcC/dJL9ZcB/RcROwK7AXOBc4IlcW/93SQflfe4B7AyMkvQRSaOA8TntUGD3Cp7ObyJi97y/h4GTC8u2yfs4DPhpfg4nAy9HxO45/1MkDatgP2Y14bEybLCke/P8fwNXkkawmxcRs3L6aGAk8Nc8FPRA4A5ge+CpiHgcII+WdmoH+9gfOAFWDnP5cr4kueigPP0tP16XFKjXA26MiDfyPqZW8JzeL+k7pOaSdYHphWVT8mXtj0t6Mj+Hg4APFtqfN8j7fqyCfZlVnQOzvRkROxcTcvB9vZgEzIiIY9qtt9p2a0jAdyPi/7bbxxd7kNdVwBERcZ+kE4F9C8vaX+oaed+fj4hiAEfSNj3Yt9kac1OGVWIWsLek4ZDuyCHpfcAjwDaSts3rHdPJ9jOBz+Vt+0vaAHiVVBsumQ6cVGi7bpG0GfAX4AhJgyWtR2o26cp6wKJ8i67j2i07UlK/XOb3Ao/mfX8ur4+k9+W7jpjVhWvM1qWIeCHXPK+VtHZO/kZEPCbpVOBmSW+QmkLW6yCLs4ErJJ0MrAA+FxF3SPpr7o52S25n3gG4I9fYXyMNcXqPpOuA+4DngdkVFPl/AHcCL+T/xTL9HbiLdOuu0/O41z8ntT3fo7TzF4AjKnt1zKrPgxiZmTUYN2WYmTUYB2YzswbjwGxm1mAcmM3MGowDs5lZg3FgNjNrMA7MZmYN5v8Df2QtGiq2kEkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All data confusion matrix\n",
      "[[1. 0.]\n",
      " [1. 0.]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVYAAAEmCAYAAAA5jbhCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xu8FXW9//HXG7YgJgKJl9ioGKgIZl4Q8pZWVF4Qy7uiydFSO9mxU50y9aemlZqnix3teLQ6mqbgJQOVRLPMY4mAdwEvqBAXFUEuKghCn98fMwuGxd57rQ2zWGu53k8e82DPzHe+852ZtT7zne98Z5YiAjMzy0+HahfAzOyDxoHVzCxnDqxmZjlzYDUzy5kDq5lZzhxYzcxyVtXAKqmLpLslLZZ0+wbkM0LS/XmWrRok/VHSqdUux/qSNEPS0PTv8yT9qtplWl/reywkHSjphUqUqZZJ2l7SO5I6Vij/WyV9Icf87pR0aF75rSMiSg7AScBk4B3gNeCPwAHlLFsi31OAiUDThuZViQE4GAjgrqLpH0+nP1RmPhcDN2/Eco8EVqXH6x3gFeCrG2G9M4ChVThGs6v4GQmgXw75XAy8nx6vRcDfgX2rtV21NAC7A1MBZaZ9BPh1Go/eBp4Hvg98KP37tBbyOQeYnP49GHi8UmUuWWOV9E3g58CPgG2A7YFfAkeWWrYMOwAvRsTKHPKqlDeBfSVtmZl2KvBiXitQIu+rh0cjYvOI2Bw4GvixpD1zXofla3R6vHoCfwHW+yquLZKaKpFvBZ0J/C7SiCjpw8CjQBeSk09X4LNAd6AvcCPwpRbyOSWdR0RMBLaQNKgiJS5xpuhGcgY9to00nUkC79x0+DnQOZ13MDAb+BYwj+Ts8i/pvO8DK1hzlj6dopod0IekRtCUjo8kqX29DbwKjMhMfySz3H7AJGBx+v9+mXkPAZcCf0vzuR/o2cq2Fcp/LfC1dFpHYA5wIZkaK3AVMAtYAjwOHJhOP6RoO5/OlOOHaTmWAf3SaV9O5/83cGcm/yuAB8mctds4Jmvtj3TaROCkzPjtwOvpPnoYGJiZdxhJDeHtdFu/nZk3DHiKNbWq3TPzZpDWWLPHMnMcTwX+AcwHzs8s1wE4F3gZWADcBny4zNrMwbRSYyX5/P6W5OQ4E7gA6JA5jj9Jy/IqcDZrf9ayx6If8Nd0X80nCYCk+y2Ad9Nje3xxeYDtgN+nZVgAXN1KWVfvr3R8QJr3VmXu+72AJ9NjdjswGvhB0ef4u+kxv6mM/L6bHvu3gReAz2RqepNJPudvAD9t5bvaCxgLvAVMB75StK23pcfmbWAKMKiNY/wKmStk4AfAs4Vj2UL63sBKYIei/bmCzHcduB64qJzPWXuHUh/aQ9ICtnqpDlwCTAC2BrZKD9ClmQO6Mk2zCckXdinQo5UPU/H46oNFUsVfAuySzvsIaTAgE0iADwMLSc5OTcCJ6fiWmS/My8DOJGe8h4DL2/rSkgTqxzJBZzzwZdYOrCcDW6br/BbJB3jTlrYrU45/AAPTZTZh7S/zZiS14pHAgSRf6N6Z5RfRSnMM655o9knT75yZdhrQlTUnxqcy815jzYmhB7BX+veeJCfIISSB6VSSYFo4kc6g7cB6fbrPPw4sB3ZN559D8hnqnZbnf4BbM+V5hsxJoaVj1Mq83wJj0u3sk+7P09N5Z5GcPHqn2/gnWg+stwLnk5wANmXtL/laTQHZ8qT76GngZySf37WWLSprdn91Ai5Pj3lTqX2fpp+Z7sdNgKNIgkg2sK4kOTl3To9BW/ntQlJJ6JU5fn3Tvx8FTkn/3hz4RPF3NR1/mOTKdlNgD5ITy6cz2/oeyXepI3AZMKGV/fIh1j3BTAC+XyJ2PQBckBm/DPhDUZpvAr/PO6hGlA6sI4DXS6R5GTgsM/55YEbmgC4jE5jTg1k4GKs/TK2Mrz5Y6Q5eRHJZ26W1QELabls0/1FgZOYLk93h/wrcV+pLC7yUfuBGpftlrcDawrILgY+3tF2ZclzSwrQvZ8aHkJzxZwInln1Qk/2xMt1fb6f78L9opbZLcgkVQLd0/B8kl19bFKX7b9KTZmbaC8BB6d8zaDuwZk8ME4ET0r+nkdaI0vGPkNTwS7a900pgJfnCrgAGZKadWThmwJ+BMzPzhtJ6YP0tcF22/Jnl2gqs+5IElHK24+K0vItI2scXAAeXs++BT5LULrNtkI+wdmBdQXqiLyO/fiTf06HAJkVpHia52uxZNL1wjJtIaumrgK6Z+ZcBN2S29U+ZeQOAZa3sl+Y032zZXwLOKrE/TwZeSP/uQPKZ/mJRmq8Afy73e9WeoVS73gKgZ4k2mV4kX/yCmem01XnE2m2oS0nOdO0SEe+SXGqdBbwm6V5J/csoT6FMzZnx19ejPDeRXC5+CrireKakb0ualvZwWERyGdqzRJ6z2poZEY+RXAaJ5NKpPSZERPdI2p+2JakZ/ygta0dJl0t6WdISkoBIprxHk9QmZkr6q6R90+k7AN+StKgwkHyJsse7La3t9x2AuzJ5TiP5Ym7Tzm3O6klSeyv+bBY+B71Ye/+3dSy+Q3IMJkqaIum0MsuwHTAzyr+HcFtEdCfZ7ueAvTPz2tr3vYA5kUaLVrbnzYh4r5z8ImI68A2SADhP0ihJhWN8OsnV3vOSJkka1sJ29ALeioi3M9NKfQc3bSXOLEr/75qZtoDk5NuW3wMfkfQJkhPLZsC9RWm6ZvLPVanA+ijJJVtb3Rzmkhykgu3TaevjXZIdULBtdmZEjI+Iz5Ls1OdJLi1LladQpjnrWaaCm0hqt+MiYml2hqQDSb58x5E0c3QnaY9Toeit5Nna9EK+XyO5NJub5r9eIuIN4E7giHTSSSQ3H4eSnAD6FFaZpp8UEUeSNO/8gTVBfRbwwzRgF4bNIuLW9S1bJt9Di/LdNCI25JjNJ6n1Fn82C3m+RtIMULBdaxlFxOsR8ZWI6EVS6/2lpH5llGEWsH17bxZFxHzgDOBiSYUA0ta+fw1olqRMNsXbU/xZa/NYRsQtEXEAyf4LkmYEIuKliDiR5LNxBXCHpA8V5T0X+LCkbDBcr+9gWqEqNN0V/An4Yls3fNPv6B0kN7FOAUZFxIqiZLuSNNXkrs3AGhGLSW7SXCPpC5I2k7SJpEMl/ThNditwgaStJPVM09+8nuV5Cvhk2ieuG/C9wgxJ20g6Mj2Iy0luFvyzhTzGATtLOklSk6TjSS417lnPMgEQEa+SXCad38LsriSX3m8CTZIuBLbIzH8D6NOeO/+SdiZppD+Z5IPxHUl7rE/Z0x4NXyS5SVAo73KSM/9mpDXZNG2ntF9wt4h4n6Rdu7CfrwfOkjQk7cnwIUmHF32B1se1wA8l7ZCWYStJ7ep1ImnT7JCW+bY0365p3t9kzWfzNuAcSc2SupPcrGkt72MlFYLwQpJAU9gnbwAfbWXRiSRB7/J0X20qaf9yticiXiBpyy+cUNva94+S1PDPTj/zR5LcZGpLq/lJ2kXSpyV1JmkLXVbYXkknS9oqIv7JmtreWt/DiJhFcq/lsnSbdyep6a5vXBhH8t0r+CnJ9+vGzGemWdJP03UV3EhylXt0+nexg0i6juau5Bc9In5C8oG8gCRwzCK5JP5DmuQHJHcJnyG5U/dEOq3dIuIBkruZz5DcWc8Gww5pOeaStDseBHy1hTwWkNzt/BZJ4PgOMCytBWyQiHgkIlqqjY8H7iO5OTKT5MOYvRQrdJtZIOmJUutJazg3A1dExNMR8RJwHnBT+mFHSWfsA9vIZt80zTskl9ZvAl9P5/02Lecckhs4E4qWPQWYkTYTnEXSpkxETCZpl7qaJMBMJ2nP3VBXkdxBvl/S22l5hhRmppffI9pYvpnky58d+pJs77skzSmPALcAv0mXuZ6kR8gzJHfTx5GcHFe1kP8+wGPpvhwLnBMRr6TzLib5gi+SdFx2oYhYRXKV0I+kjW82yRe9XFcCZ0jauq19n9bEjiIJXotITsb3kJw8W1TiWHZmzc2z10lqp4VKziHAlHRfXEXSTr6shVWcSHIlNJek6eyiiPhTO7Y96zpgRKFGHhFvkdxQfp/kuLxN0mNmcbodBQ+n02ZHxKRshpL2Ad6JpNtV7rR2s4xZY1LyFM61EVHcjFSXJD1Gsj3/W+2y5EHSLSRt0H8ombi8/O4Efh0R4/LIb538HVitEUnqQnIj8n6Sm0V3ktzw+0ZVC7aeJB1Ecld/PskVxrXARyPitaoWrEH5JSzWqETSbWghSVPANJL7A/VqF5IbMYtImsGOcVAtj6TfSJon6blW5kvSLyRNl/SMpL1K5ukaq5k1MkmfJLkZ/tuI2K2F+YeRtNcfRtL2f1VEDClOl+Uaq5k1tIh4mOSGeGuOJAm6ERETgO6ZbnAtqreXMbRITV1CnTa0x4/Vij133b7aRbCczJw5g/nz56t0yvJ13GKHiJUtdURoWSx7cwpJT52C6yLiunasspm1e/nMTqe12tTywQisnbrSeZfjSie0uvC3x66udhEsJ/sPyf/lUbFyWbu+7+89dc17EVGZt1i14gMRWM2skQhyf8tmm+aw9pNsvSnxFJnbWM2svgiQyh823FjgS2nvgE8Ai0v1uHCN1czqT441Vkm3kryopaek2cBFJC/wISKuJXkq7zCSp7qWAv9SKk8HVjOrM4IO+f20VvpSmbbmB/C19uTpwGpm9SefS/yKcWA1s/oiNvbNq3ZzYDWzOpPbTamKcWA1s/rjGquZWc5cYzUzy9NGf0Cg3RxYzay+FB4QqGEOrGZWf1xjNTPLk6Bjfg8IVIIDq5nVF/djNTOrALexmpnlyb0CzMzy5xqrmVnOXGM1M8tRfi+wrhgHVjOrP66xmpnlzDVWM7M8uVeAmVm+RK4/zVIJDqxmVmdcYzUzy5/bWM3McuYaq5lZzlxjNTPLkdzGamaWP9dYzczyJQdWM7P8JD955cBqZpYfCXVwYDUzy5VrrGZmOXNgNTPLmQOrmVmelA41zIHVzOqKkGusZmZ5c2A1M8uZA6uZWc4cWM3M8uSbV2Zm+RKiQ4fafrtVbZfOzKwFksoeysjrEEkvSJou6dwW5m8v6S+SnpT0jKTDSuXpwGpm9UftGNrKRuoIXAMcCgwATpQ0oCjZBcBtEbEncALwy1LFc2A1s/qiXGusg4HpEfFKRKwARgFHFqUJYIv0727A3FKZuo3VzOpOO3sF9JQ0OTN+XURcl/7dDMzKzJsNDCla/mLgfklfBz4EDC21QgdWM6s77Qys8yNi0Aas7kTghoj4iaR9gZsk7RYR/2xtAQdWM6srOT/SOgfYLjPeO52WdTpwCEBEPCppU6AnMK+1TN3Gamb1J6ebV8AkYCdJO0rqRHJzamxRmn8AnwGQtCuwKfBmW5k6sNaAay8awcwHL2Py7ee1muYn3zmG58ZcxMTR32OP/r1XTx9xxBCeHXMhz465kBFHFDcNWTXcP/4+dh+4CwP79+PKH1++zvzly5dz8knHM7B/Pw7cbwgzZ8xYPe/KKy5jYP9+7D5wFx64f/xGLHUdyfHmVUSsBM4GxgPTSO7+T5F0iaThabJvAV+R9DRwKzAyIqKtfN0UUANuunsC147+K7+69Estzv/8AQPou/1W7Hbk9xn8sT784rwT+OSX/pMeW2zG+Wccyv4jfkxE8Pdbvsu9Dz3DoreXbeQtsIJVq1bxjX/7Gvf+8QGae/fmgE/sw7Bhw9l1wJoePDf85tf06N6DKc9P57bRozj/vO9y8y2jmTZ1KrePHsUTT0/htblzOeyQoTw79UU6duxYxS2qTXk+0hoR44BxRdMuzPw9Fdi/PXm6xloD/vbEy7y1eGmr84cdtDu33DMRgInPzqBb1y5s23MLPrvfrjw44XkWLlnKoreX8eCE5/nc/sVd8GxjmjRxIn379mPHj36UTp06cezxJ3DP3WPWSnPP3WMYccqpABx19DE89OcHiQjuuXsMxx5/Ap07d6bPjjvSt28/Jk2cWI3NqHnqoLKHanBgrQO9tu7O7NcXrh6f88Yiem3dnV5bdWf2G5np8xbRa6vu1SiipebOnUPv3mvuhTQ392bOnDnrptkuSdPU1MQW3bqxYMEC5sxZd9m5c4vvoxjk++RVJdRkYC31iJmZNa72BFUH1lSZj5g1lLnzFtF72x6rx5u36c7ceYuY++Yiem+Tmb51d+a+uagaRbRUr17NzJ69pr/5nDmzaW5uXjfNrCTNypUrWbJ4MVtuuSXNzesu26vX2stawoG1/cp5xKyh3PvXZzlp2GAABn+sD0veWcbr85fwwN+nMXTf/nTv2oXuXbswdN/+PPD3aVUubWMbtM8+TJ/+EjNefZUVK1Zw++hRHD5s+FppDh82nN/ddCMAv7/zDg761KeRxOHDhnP76FEsX76cGa++yvTpL7HP4MHV2IyaV+uBtRZ7BZTziBmSzgDOAGCTzTdKwSrlxstGcuDeO9Gz++ZMv+9SLr12HJs0JXeCf3XHI9z3yBQ+f8BApoy9iKXvvc+ZF98MwMIlS7ns+vt45ObvAPCj6+5j4ZLWb4JZ5TU1NfGzq67miMM/z6pVqzh15GkMGDiQSy6+kL32HsSwI4Yz8rTTOW3kKQzs348ePT7MTb8bBcCAgQM5+tjj2HP3ATQ1NfHzX1zjHgGtqfH3sapEd6yNTtIxwCER8eV0/BRgSESc3doyHTbbOjrvctzGKqJV2MJJV1e7CJaT/YcM4vHHJ+caBjtvs1M0j7iq7PSv/uzwxzfwkdZ2q8UaazmPmJlZo1Lt/zRLLbaxlvOImZk1KAFS+UM11FyNNSJWSio8YtYR+E1ETKlyscysZogOVer4X66aC6zQ8iNmZmYFtd4UUJOB1cysVVW8xC+XA6uZ1RWBmwLMzPLmGquZWc7cxmpmlie3sZqZ5Svpx1rbkdWB1czqTPVerlIuB1Yzqzs1HlcdWM2szsjdrczMcuU2VjOzCqjxuOrAamb1xzVWM7Oc1XhcdWA1szpTBy+6dmA1s7pSeNF1LXNgNbM64wcEzMxyV+Nx1YHVzOqMHxAwM8uXHxAwM6sAB1Yzs5zVeFx1YDWz+uMaq5lZnvwLAmZm+ZL7sZqZ5a/G46oDq5nVnw41Hlk7VLsAZmbtJZU/lM5Lh0h6QdJ0See2kuY4SVMlTZF0S6k8XWM1s7oiQcecnryS1BG4BvgsMBuYJGlsREzNpNkJ+B6wf0QslLR1qXxdYzWzuiOp7KGEwcD0iHglIlYAo4Aji9J8BbgmIhYCRMS8UpnmXmOVtEVb8yNiSd7rNLPG0s4m1p6SJmfGr4uI69K/m4FZmXmzgSFFy++crFN/AzoCF0fEfW2tsBJNAVOAIHmkt6AwHsD2FVinmTUIkXS5aof5ETFoA1bZBOwEHAz0Bh6W9LGIWNTWArmKiO3yztPMLCvHl1vNAbIxq3c6LWs28FhEvA+8KulFkkA7qdXy5Va8Fkg6QdJ56d+9Je1dyfWZWQNoR/tqGW2sk4CdJO0oqRNwAjC2KM0fSGqrSOpJ0jTwSluZViywSroa+BRwSjppKXBtpdZnZo0jr+5WEbESOBsYD0wDbouIKZIukTQ8TTYeWCBpKvAX4D8iYkFb+Vayu9V+EbGXpCcBIuKt9IxgZrbeRL4PCETEOGBc0bQLM38H8M10KEslA+v7kjqQ3LBC0pbAPyu4PjNrEDX+4FVF21ivAe4EtpL0feAR4IoKrs/MGkSObawVUbEaa0T8VtLjwNB00rER8Vyl1mdmjSHPJ68qpdKPtHYE3idpDvBTXmaWi9oOq5XtFXA+cCvQi6Rv2C2Svlep9ZlZ42jYpgDgS8CeEbEUQNIPgSeByyq4TjP7gEt6BVS7FG2rZGB9rSj/pnSamdn6q2JNtFyVeAnLz0jaVN8Cpkgan45/jjYeATMzK1eNx9WK1FgLd/6nAPdmpk+owLrMrAE1XI01In6dd55mZgUN3cYqqS/wQ2AAsGlhekTsXKl1mlljqPUaayX7lt4A/C/JCeZQ4DZgdAXXZ2YNQIKOUtlDNVQysG4WEeMBIuLliLiAJMCamW2QPH9MsBIq2d1qefoSlpclnUXy8tiuFVyfmTWIWm8KqGRg/XfgQ8C/kbS1dgNOq+D6zKxB1HhcrehLWB5L/3ybNS+7NjPbIEK5vo+1EirxgMBdpO9gbUlEHJX3Os2sgVSx7bRclaixXl2BPM3MVmu4NtaIeDDvPM3Msmr9HaSVfh+rmVmuRAPWWM3MKq1hH2ktkNQ5IpZXej1m1hjq4adZKvkLAoMlPQu8lI5/XNJ/VWp9ZtY4Oqj8oSrlq2DevwCGAQsAIuJp4FMVXJ+ZNYhGfqS1Q0TMLGpkXlXB9ZlZA0heG1jbTQGVDKyzJA0GQlJH4OvAixVcn5k1iEbubvVVkuaA7YE3gD+l08zMNkiNV1gr+q6AecAJlcrfzBqT1IDvCiiQdD0tvDMgIs6o1DrNrDHUeFytaFPAnzJ/bwp8EZhVwfWZWYOo8W6sFW0KWOtnWCTdBDxSqfWZWWMQtf+AwMZ8pHVHYJuNuD4z+yCqYsf/clWyjXUha9pYOwBvAedWan1m1jhEbUfWigRWJU8FfJzkd64A/hkRrb782sysXMkDAtUuRdsq0s82DaLjImJVOjiomlluGvldAU9J2rOC+ZtZg5JU9lANlfjNq6aIWAnsCUyS9DLwLkkNPiJir7zXaWaNox6aAirRxjoR2AsYXoG8zazR5fzWKkmHAFcBHYFfRcTlraQ7GrgD2CciJreVZyUCqwAi4uUK5G1mltsjrekLoq4BPgvMJrnKHhsRU4vSdQXOAR4rJ99KBNatJH2ztZkR8dMKrNPMGkTOTQGDgekR8QqApFHAkcDUonSXAlcA/1FOppW4edUR2Bzo2spgZrYBREeVPwA9JU3ODNn3lTSz9qP2s9Npa9Ym7QVsFxH3llvCStRYX4uISyqQr5lZ+iut7VpkfkQMWq91SR2AnwIj27NcxdpYzcwqIt/+qXOA7TLjvVnzYBMkV9m7AQ+lXbe2BcZKGt7WDaxKBNbPVCBPM7PVcnwf6yRgJ0k7kgTUE4CTCjMjYjHQszAu6SHg26V6BeTexhoRb+Wdp5lZQaEpII8fE0z73J8NjAemAbdFxBRJl0ha7y6jG/PtVmZmucjzFwQiYhwwrmjaha2kPbicPB1YzazuNPIvCJiZ5U409q+0mpnlT1Tt5SrlcmA1s7pT22HVgdXM6oyg8ERVzXJgNbO6U+Nx1YHVzOpN9V5gXS4HVjOrK+4VYGZWAa6xmpnlrLbDqgOrmdUb92M1M8uX21jNzCrANVYzs5w14s9fm5lVTNIUUNuR1YHVzOpOjbcEOLCaWb0Rco3VzCxfrrGameXIbaxmZnkr40cCq82B1czqjgOrmVnOav3mVa0/GdYQrr1oBDMfvIzJt5/XapqffOcYnhtzERNHf489+vdePX3EEUN4dsyFPDvmQkYcMWRjFNdKuH/8few+cBcG9u/HlT++fJ35y5cv5+STjmdg/34cuN8QZs6YsXrelVdcxsD+/dh94C48cP/4jVjq+iGSBwTKHarBgbUG3HT3BI782jWtzv/8AQPou/1W7Hbk9zn7B7fyi/NOAKDHFptx/hmH8slT/pMDT76S8884lO5du2ysYlsLVq1axTf+7WuMufuPPPnMVG4fdSvTpk5dK80Nv/k1Pbr3YMrz0/n6Of/O+ed9F4BpU6dy++hRPPH0FMbecx/nfP1fWbVqVTU2o+Z1kMoeqlK+qqzV1vK3J17mrcVLW50/7KDdueWeiQBMfHYG3bp2YdueW/DZ/XblwQnPs3DJUha9vYwHJzzP5/YfsLGKbS2YNHEiffv2Y8ePfpROnTpx7PEncM/dY9ZKc8/dYxhxyqkAHHX0MTz05weJCO65ewzHHn8CnTt3ps+OO9K3bz8mTZxYjc2oeWrHv2pwYK0DvbbuzuzXF64en/PGInpt3Z1eW3Vn9huZ6fMW0Wur7tUooqXmzp1D797brR5vbu7NnDlz1k2zXZKmqamJLbp1Y8GCBcyZs+6yc+euvay5KWC9SPqNpHmSnqt2WcysFrWnvuoaa8ENwCHVLkQtmTtvEb237bF6vHmb7sydt4i5by6i9zaZ6Vt3Z+6bi6pRREv16tXM7NmzVo/PmTOb5ubmddPMStKsXLmSJYsXs+WWW9LcvO6yvXqtvayxuh9ruUM11FxgjYiHgbeqXY5acu9fn+WkYYMBGPyxPix5Zxmvz1/CA3+fxtB9+9O9axe6d+3C0H3788Dfp1W5tI1t0D77MH36S8x49VVWrFjB7aNHcfiw4WulOXzYcH53040A/P7OOzjoU59GEocPG87to0exfPlyZrz6KtOnv8Q+gwdXYzNqntoxVEPd9mOVdAZwBgCbbF7dwmygGy8byYF770TP7psz/b5LufTacWzS1BGAX93xCPc9MoXPHzCQKWMvYul773PmxTcDsHDJUi67/j4eufk7APzouvtYuKT1m2BWeU1NTfzsqqs54vDPs2rVKk4deRoDBg7kkosvZK+9BzHsiOGMPO10Tht5CgP796NHjw9z0+9GATBg4ECOPvY49tx9AE1NTfz8F9fQsWPHKm9R7UnaWGu7H6siotplWIekPsA9EbFbOek7bLZ1dN7luIqWyTaehZOurnYRLCf7DxnE449PzjUK7vqxPeN/7/pL2en33anH4xExKM8ylFK3NVYza2C1XWF1YDWz+lPrTQE1d/NK0q3Ao8AukmZLOr3aZTKz2uKbV+0UESdWuwxmVuNqu8Jae4HVzKwtSU20tiOrA6uZ1Re/6NrMLH81Hldr7+aVmVlJOd69knSIpBckTZd0bgvzvylpqqRnJD0oaYdSeTqwmlmdye8lLJI6AtcAhwIDgBMlFb9780lgUETsDtwB/LhUCR1Yzazu5PgSlsHA9Ih4JSJWAKOAI7MJIuIvEVF4VnwC0JsSHFjNrK60pxUgjas9JU3ODGdksmsGZmXGZ6fTWnM68MdSZfTNKzOrO2pft4D5ebwrQNLJwCDgoFJpHVjNrO7k2N1qDrBdZrx3Oq1ofRoKnA8cFBHLS2XqpgAzqzs5dgqYBOwkaUdJnYATgLFrrUvaE/gfYHhEzCunfA7jL5JlAAAHSklEQVSsZlZf1qORtTURsRI4GxgPTANui4gpki6RVHhD+ZXA5sDtkp6SNLaV7FZzU4CZ1Z08H2mNiHHAuKJpF2b+HtrePB1YzayuCD/SamaWuxqPqw6sZlaHajyyOrCaWd3xawPNzHLWobbjqgOrmdUhB1Yzs/z4FwTMzPLmXxAwM8tfjcdVB1Yzq0M1HlkdWM2szpT+ZYBqc2A1s7rjNlYzsxyV+TrAqnJgNbP6U+OR1YHVzOpOhxpvC3BgNbO6U9th1YHVzOqNHxAwM6uE2o6sDqxmVlf8CwJmZhVQ43HVgdXM6o9rrGZmOfMjrWZmeavtuOrAamb1p8bjqgOrmdUXyU9emZnlr7bjqgOrmdWfGo+rDqxmVn9qvCXAgdXM6o1/QcDMLFf18Ehrh2oXwMzsg8Y1VjOrO7VeY3VgNbO64zZWM7McJQ8IVLsUbXNgNbP648BqZpYvNwWYmeWs1m9eubuVmdUdtWMomZd0iKQXJE2XdG4L8ztLGp3Of0xSn1J5OrCaWf3JKbJK6ghcAxwKDABOlDSgKNnpwMKI6Af8DLiiVPEcWM2s7qgd/0oYDEyPiFciYgUwCjiyKM2RwI3p33cAn5Haboz4QLSxxrI357/31DUzq12OjaAnML/ahai0LptcU+0ibCyNcDx3yDvDJ594fPxmndSzHYtsKmlyZvy6iLgu/bsZmJWZNxsYUrT86jQRsVLSYmBL2jh2H4zAGrFVtcuwMUiaHBGDql0Oy4eP5/qJiEOqXYZS3BRgZo1sDrBdZrx3Oq3FNJKagG7AgrYydWA1s0Y2CdhJ0o6SOgEnAGOL0owFTk3/Pgb4c0REW5l+IJoCGsh1pZNYHfHxrLK0zfRsYDzQEfhNREyRdAkwOSLGAr8GbpI0HXiLJPi2SSUCr5mZtZObAszMcubAamaWMwdWM7OcObDWMEm7SNpX0ibpo3f2AeBj+cHnm1c1StJRwI9I+tDNASYDN0TEkqoWzNabpJ0j4sX0744RsaraZbLKcI21BknaBDgeOD0iPgOMIemg/F1JW1S1cLZeJA0DnpJ0C0BErHLN9YPLgbV2bQHslP59F3APsAlwUqkXQFhtkfQh4GzgG8AKSTeDg+sHmQNrDYqI94GfAkdJOjAi/gk8AjwFHFDVwlm7RcS7wGnALcC3SV4Ksjq4VrNsVhkOrLXr/4D7gVMkfTIiVkXELUAv4OPVLZq1V0TMjYh3ImI+cCbQpRBcJe0lqX91S2h58iOtNSoi3pP0OyCA76VfvOXANsBrVS2cbZCIWCDpTOBKSc+TPEr5qSoXy3LkwFrDImKhpOuBqSS1nPeAkyPijeqWzDZURMyX9AzJm+s/GxGzq10my4+7W9WJ9CZHpO2tVuck9QBuA74VEc9UuzyWLwdWsyqRtGlEvFftclj+HFjNzHLmXgFmZjlzYDUzy5kDq5lZzhxYzcxy5sDa4CStkvSUpOck3S5psw3I62BJ96R/D5d0bhtpu0v61/VYx8WSvl3u9KI0N0g6ph3r6iPpufaW0cyB1ZZFxB4RsRuwAjgrO1OJdn9OImJsRFzeRpLuQLsDq1k9cGC1rP8D+qU1tRck/RZ4DthO0uckPSrpibRmuzmApEMkPS/pCeCoQkaSRkq6Ov17G0l3SXo6HfYDLgf6prXlK9N0/yFpkqRnJH0/k9f5kl6U9AiwS6mNkPSVNJ+nJd1ZVAsfKmlymt+wNH1HSVdm1n3mhu5Ia2wOrAaApCaSxyufTSftBPwyIgYC7wIXAEMjYi+Sl25/U9KmwPXAEcDewLatZP8L4K8R8XFgL2AKcC7wclpb/g9Jn0vXORjYA9hb0icl7U3yc8N7AIcB+5SxOb+PiH3S9U0DTs/M65Ou43Dg2nQbTgcWR8Q+af5fkbRjGesxa5HfFWBdJD2V/v1/JL+h3guYGRET0umfAAYAf0tfBdsJeBToD7waES8BpG9rOqOFdXwa+BKsfk3e4vSRzqzPpcOT6fjmJIG2K3BXRCxN1zG2jG3aTdIPSJobNif5zfiC29LHgl+S9Eq6DZ8Dds+0v3ZL1/1iGesyW4cDqy2LiD2yE9Lg+W52EvBARJxYlG6t5TaQgMsi4n+K1vGN9cjrBuALEfG0pJHAwZl5xY8aRrrur0dENgAjqc96rNvMTQFWlgnA/pL6QfJGfEk7A88DfST1TdOd2MryDwJfTZftKKkb8DZJbbRgPHBapu22WdLWwMPAFyR1kdSVpNmhlK7Aa+lP3IwomnespA5pmT8KvJCu+6tpeiTtnL7132y9uMZqJUXEm2nN71ZJndPJF0TEi5LOAO6VtJSkKaFrC1mcA1wn6XRgFfDViHhU0t/S7kx/TNtZdwUeTWvM75C8IvEJSaOBp4F5wKQyivz/gMeAN9P/s2X6BzCR5Kdvzkrfe/srkrbXJ5Ss/E3gC+XtHbN1+SUsZmY5c1OAmVnOHFjNzHLmwGpmljMHVjOznDmwmpnlzIHVzCxnDqxmZjn7/0aVaG4nkxPKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "specs = 'Baseline: Logistic Regression (CV)'\n",
    "\n",
    "# X_new = test_data\n",
    "# y_true = test_bins\n",
    "# y_pred = model.predict(X_new)\n",
    "\n",
    "#y_true_labels = np.argmax(y_true, axis=1)\n",
    "#y_pred_labels = np.argmax(y_pred, axis=1) \n",
    "from sklearn.metrics import roc_auc_score\n",
    "y_true_labels= np.array(test_df[output_bin_col_name].astype(np.float32))\n",
    "y_pred_labels = preds\n",
    "print('AUC:', roc_auc_score(y_true_labels, y_pred_labels))\n",
    "\n",
    "print('Accuracy:', sklearn.metrics.accuracy_score(y_true_labels, y_pred_labels), '\\n')\n",
    "print('F1:', sklearn.metrics.f1_score(y_true_labels,y_pred_labels, average='macro'),'\\n')\n",
    "\n",
    "#accuracy_plot(model, 'Accuracy: ' + specs)\n",
    "#loss_plot(model, 'Loss: ' + specs)\n",
    "\n",
    "class_names = ['0', '1']\n",
    "confusion_plot(y_true_labels, y_pred_labels, 'Confusion Matrix: ' + specs, class_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average precision-recall score: 0.00\n"
     ]
    }
   ],
   "source": [
    "# from TAs on piazza: use precision-recall, ROC can be misleading \n",
    "# modified code from sklearn documentation: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_curve.html\n",
    "# and referenced: https://machinelearningmastery.com/roc-curves-and-precision-recall-curves-for-classification-in-python/\n",
    "from sklearn.metrics import precision_recall_curve, average_precision_score\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.utils.fixes import signature\n",
    "\n",
    "average_precision = average_precision_score(y_true, y_pred_labels)\n",
    "\n",
    "print('Average precision-recall score: {0:0.2f}'.format(\n",
    "      average_precision))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall: 0.0\n",
      "Precision: 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/kcollins/anaconda/envs/deeptoehold/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import recall_score, precision_score\n",
    "\n",
    "print('Recall:', recall_score(y_true_labels, y_pred_labels))\n",
    "print('Precision:', precision_score(y_true_labels, y_pred_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}