classifier_raw.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train-Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from pathlib import Path\n",
    "from collections import defaultdict\n",
    "from sklearn.metrics import roc_auc_score, f1_score\n",
    "from fastai.text import *\n",
    "\n",
    "seed = 41\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('/home/paperspace/suicides')\n",
    "lm_path = Path('data/lm/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(path/'data/std_format_raw_data.csv',index_col=0)\n",
    "labels = df['labels'].values.copy()\n",
    "tokens = np.load(lm_path/'tmp'/'tokens.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle tokens and labels together\n",
    "idx = np.random.permutation(len(labels))\n",
    "labels = labels[idx]\n",
    "tokens = tokens[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_labels_1 = np.sum(labels==1)\n",
    "ratio = 1 # ratio of 0 labels (non-suicide) to 1 labels (suicide)\n",
    "tokens_label_1 = list(tokens[labels==1])\n",
    "labels_label_1 = list(labels[labels==1])\n",
    "tokens_label_0 = list(tokens[labels==0][:ratio*(num_labels_1+1)])\n",
    "labels_label_0 = list(labels[labels==0][:ratio*(num_labels_1+1)])\n",
    "tokens_balanced = tokens_label_1+tokens_label_0\n",
    "labels_balanced = np.array(labels_label_1+labels_label_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "int_to_str = pickle.load(open(lm_path/'tmp'/'int_to_str.pkl', 'rb'))\n",
    "str_to_int = defaultdict(lambda:0, {v:k for k,v in enumerate(int_to_str)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([[str_to_int[o] for o in p] for p in tokens_balanced])\n",
    "y = labels_balanced\n",
    "\n",
    "X_train,X_test,y_train,y_test = sklearn.model_selection.train_test_split(X, y, test_size=0.2, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "if ratio>1:\n",
    "    # naively random oversampling positive labels in training set (not test set!!!) to make data balanced\n",
    "    is_0 = np.where(y_train == 0)[0]\n",
    "    is_1 = np.where(y_train == 1)[0]\n",
    "    is_1_up = np.random.choice(is_1, size=len(is_0), replace=True)\n",
    "    X_train_up = np.concatenate((X_train[is_1_up], X_train[is_0]))\n",
    "    y_train_up = np.concatenate((y_train[is_1_up], y_train[is_0]))\n",
    "\n",
    "    # reshuffle training data\n",
    "    idx = np.random.permutation(len(y_train_up))\n",
    "    X_train_up = X_train_up[idx]\n",
    "    y_train_up = y_train_up[idx]\n",
    "else:\n",
    "    X_train_up = X_train\n",
    "    y_train_up = y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "bptt = 70\n",
    "embedding_size = 400\n",
    "num_hidden = 1150\n",
    "num_layers = 3\n",
    "vocab_size = len(int_to_str)\n",
    "bs = 48\n",
    "dropouts = np.array([0.4,0.5,0.05,0.3,0.4])*0.7\n",
    "num_labels = y_test.max()+1\n",
    "opt = partial(optim.Adam, betas=(0.7, 0.99))\n",
    "lr = 3e-3\n",
    "lrm = 2.6\n",
    "lrs = np.array([lr/(lrm**4), lr/(lrm**3), lr/(lrm**2), lr/lrm, lr])\n",
    "wd = 1e-7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_train = TextDataset(X_train_up, y_train_up)\n",
    "dataset_test = TextDataset(X_test, y_test)\n",
    "\n",
    "sampler_train = SortishSampler(X_train_up, key=lambda x: len(X_train_up[x]), bs=bs//2)\n",
    "sampler_test = SortSampler(X_test, key=lambda x: len(X_test[x]))\n",
    "\n",
    "dl_train = DataLoader(dataset_train, bs//2, transpose=True, num_workers=1, pad_idx=1, sampler=sampler_train)\n",
    "#dl_test = DataLoader(dataset_test, bs, transpose=True, num_workers=1, pad_idx=1, sampler=sampler_test)\n",
    "dl_test = DataLoader(dataset_test, bs, transpose=True, num_workers=1, pad_idx=1, sampler=None, shuffle=False)\n",
    "\n",
    "model_data = ModelData(path, dl_train, dl_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = get_rnn_classifier(bptt, 20*70, num_labels, vocab_size, emb_sz=embedding_size, n_hid=num_hidden, \n",
    "                           n_layers=num_layers, pad_token=1, layers=[embedding_size*3, 50, 2], \n",
    "                           drops=[dropouts[4], 0.1],dropouti=dropouts[0], wdrop=dropouts[1], \n",
    "                           dropoute=dropouts[2], dropouth=dropouts[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def auc(preds,targs):\n",
    "    try:\n",
    "        preds = preds.cpu().numpy()\n",
    "        preds = np.argmax(preds,axis=1)\n",
    "        targs = targs.cpu().numpy().flatten()\n",
    "        roc = roc_auc_score(targs,preds)\n",
    "    except:\n",
    "        print('Value error occured, setting auc=-1')\n",
    "        roc = -1\n",
    "    return roc\n",
    "def F1(preds,targs):\n",
    "    try:\n",
    "        preds = preds.cpu().numpy()\n",
    "        preds = np.argmax(preds,axis=1)\n",
    "        targs = targs.cpu().numpy().flatten()\n",
    "        f1 = f1_score(targs,preds)\n",
    "    except:\n",
    "        print('Value error occured, setting F1=-1')\n",
    "        f1 = -1\n",
    "    return f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner = RNN_Learner(model_data, TextModel(to_gpu(model)), opt_fn=opt, metrics=[accuracy,F1,auc])\n",
    "learner.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)\n",
    "learner.clip = 25.\n",
    "learner.load_encoder('lm_last_fit_full_encoder')\n",
    "learner.freeze_to(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a321ccf90d4f4a1abcd3d81b9020ce35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/62 [00:00<?, ?it/s]                     \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-4:\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/site-packages/tqdm/_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"/home/paperspace/anaconda3/envs/fastai/lib/python3.6/_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 52/62 [01:04<00:12,  1.24s/it, loss=1.58] \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEOCAYAAAB4nTvgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xd8VVW2wPHfSockhJLQEiCU0KtEqqI4FpgZQR1UsIE64OjgWJ68wfG9p6MzozOWGccyFsQyiIioYMeGqEhJ6FJCCS3U0ANJSFvvj3twLiHJvYHcnCR3fT+f87n37LPPuese+NyVc/Y+e4uqYowxxlQkxO0AjDHG1HyWLIwxxvhkycIYY4xPliyMMcb4ZMnCGGOMT5YsjDHG+GTJwhhjjE+WLIwxxvhkycIYY4xPliyMMcb4FOZ2AFUlPj5ek5OT3Q7DGGNqlaVLl+5X1QRf9epMskhOTiY9Pd3tMIwxplYRkW3+1LPbUMYYY3yyZGGMMcYnSxbGGGN8smRhjDHGJ0sWxhhjfLJkYYwxxidLFtUoY08OuQVFbodhjDGVZsmimry2YAuX/eNbbn41jcLiErfDMcaYSrFkEWCqylNfbOChD9fSIzGOxVsO8scP17gdljHGVIoliwAqLlH+d86P/POrjVyTmsT7dwzitgvaMW3Rdt5c7NdDk8YYUyPUmeE+apqCohLunbmCj1bt5rYL2jF5WGdEhP++rDMb9uTw4Jw1dEiIoX+7Jm6HaowxPtmVRQAcP1HEra+n8dGq3dw/vDP3D++CiAAQGiI8PaYPrZvU5/Y3l5F1KNflaI0xxjdLFlXs0PECrp+ymAWb9vO3UT257YL2p9VpEBXOlJtSKSwuYfwbS62HlDGmxrNkUYV2H8nj6hcXsnb3UV64oS/XpLYqt267hBieGdOHjD1Hue+dlahqNUZqjDGVY8miimzOPsaofy1k75F83rilH5d2a+5znws7NWXy8M58snoPz3y9qRqiNMaYM2MN3FUg61Au1728iOIS5a0JA+ieGOf3vuPPb8e63Tk89cUGOjWP5TI/kowxxlQ3u7I4S4eOF3DT1CXkFhQz7df9K5UoAESER6/qQa+kOO59ewUZe3ICFKkxxpy5gCYLERkmIhkisklEJpex/e8issJZNojI4VLbG4jIThF5NpBxnqm8gmJueT2NrEN5TLkplc7NG5zRcaLCQ3nxxlSiI8O45bU0Hp+7nplpO1iceYA9R/IpKbH2DGOMuwJ2G0pEQoHngEuALCBNRD5Q1bUn66jqPV717wT6lDrMI8D8QMV4NoqKS5g4fRkrdxzm+evPOevnJZrHRfHSTalMemclL87PpMgrQUSFh9C6cX3aNIkmuUl9BrWPZ2jnpmf7FYwxxm+BbLPoB2xS1UwAEZkBjATWllN/DPDgyRUR6Qs0Az4DUgMYZ6WpKn94fzVfrd/Hn67ozrDuLarkuL1bNeSLey+gqLiEXYfz2XrgONsO5rJtv+d1+4FcvtuYzcvfbeEPP+/MhCGnd8s1xphACGSySAR2eK1nAf3LqigibYC2wNfOegjwJHAj8LMAxnhGnvg8g5npWfzuZyncMKBNlR8/LDSE1k3q07pJ/dO2FRaXcM/bK/jLJ+s5fqKYuy9O+emBP2OMCZRAJouyfsHKu/k+GpilqsXO+h3AJ6q6o6IfQhGZAEwAaN269VmE6r/XFmzhuXmbGdOvNfdcnFItn+ktPDSEp0f3oX5EKE9/tZHjJ4p44BddLGEYYwIqkMkiC/B+Ki0J2FVO3dHAb73WBwLni8gdQAwQISLHVPWURnJVfQl4CSA1NTXgrcAfrdrFHz9ayyVdm/HIyG6u/UCHhgiPXdWT+hFhTPl+C8cLivjTFT0IDbGEYYwJjEAmizQgRUTaAjvxJITrSlcSkU5AI2DhyTJVvd5r+zggtXSiqG4/bN7PvW+vJLVNI54Z04ewUHd7HYeECA9e3pWYyDCenbeJ3IJinri6F+Eux2WMqZsClixUtUhEJgJzgVBgqqquEZGHgXRV/cCpOgaYoTV4vIsfdx5hwhtLSY6vz5SbziUqPNTtkADPMxr3XdaJ6Mgw/vrZenILinn2uj5EhtWM+IwxdYfU4N/oSklNTdX09PQqP+6qrMPcMGUxsVHhzLp9IC3i6lX5Z1SFNxZu5f/mrOH8lHhevLEv9SPs4XxjjG8islRVffY4tXsWFVix4zDXT1lMg3rhzJgwoMYmCoCbBibzxNW9WLBpPze9soSdh/NscEJjTJWxPz/LsWz7Ica+soRG0RG8NWEAiQ1rbqI4aVTfJOpHhHLXjOUMfuxrYiLDaJ8QTfuEGNo3jfnpfZsm0USE2d8Jxhj/WbIow9JtBxk7NY34mAimjx9Ay1qQKE76eY8WtE+IYcmWA2zad4zN2cdZmHmA95bv/KlOaIjQPiGaJ6/uTY+kyo1lZYwJTpYsSknbepBxU5fQrEEU08cPoHlclNshVVqn5rF0ah57StmxE0VsyT7O5uxjbM4+xqylWUz4dzpzJg6maWzt+47GmOpl9yK8LM48wNipS2gWF8WMCbUzUZQnJjKMHklxXNEnkf+6tBNTxqZyOLeQ26ct40RRse8DGGOCmiULx8LNBxj3ahotG9ZjxoQBNG1QdxJFWbq1jOOJq3uxdNshHpyzxhrDjTEVsmQBLNi0n5tfW0Krxk6iCJLbMr/o2YKJQzswI20H0xZtczscY0wNFvTJYnP2MW55LY3kJtG8NX4A8TGRbodUre69pCMXd2nKHz9cy8LNB9wOxxhTQwV9smgXH819l3Zi+vgBNAmyRAGeYUP+fm1vkuOj+e30Zew4mOt2SMaYGijok4WIMH5IOxpHR7gdimtio8J5+aZUiopLmPDvpeQWFLkdkjGmhgn6ZGE82sZH88x155Cx5yiT3lllDd7GmFNYsjA/uaBjAvcP78LHq3fz/Deb3Q7HGFOD2EN55hS/Pr8ta3cf5YnPM2jTpD7npyQQGxlGiM2VYUxQs2RhTiEiPHpVDzZnH2Pi9OU/lcdEhhEbdXIJ/+m1X3Ijrjm3lQ2LbkwdZ0OUmzIdyS1k7to9HM0r5Gh+ETn5heSc8lrEweMF7DycR2LDetx9cQpX9kl0fVIoY0zl+DtEuSULc8ZUle827ueJzzNYlXWE9gnR/NelnRjWrbndtjKmlrD5LEzAiQhDOiYw57eDeeGGvoSIcMebyxjx3Pd8k7HPelQZU4dYsjBnTUQY1r05n909hKeu6cWRvELGvZrGtS8uYsmWg26HZ4ypApYsTJUJDRGuOieJr+69kEeu6M7WA8e55sWFTP1+i9uhGWPOkiULU+UiwkK4cUAb5k8ayiVdm/HnT9axKNPGnTKmNrNkYQKmXkQoT13TizZN6jNx+jL2HMl3OyRjzBmyZGECKjYqnBdv6EteQTG3v7mUgqISt0MyxpyBgCYLERkmIhkisklEJpex/e8issJZNojIYae8t4gsFJE1IrJKRK4NZJwmsFKaxfL41b1Yvv0wj3y01u1wjDFnIGBPcItIKPAccAmQBaSJyAeq+tOvhare41X/TqCPs5oL3KSqG0WkJbBUROaq6uFAxWsC6+c9WnDbkHa8+G0mvVo1ZFTfJLdDMsZUQiCvLPoBm1Q1U1ULgBnAyArqjwHeAlDVDaq60Xm/C9gHJAQwVlMNJl3WiYHtmvDA+6v5cecRt8MxxlRCIJNFIrDDaz3LKTuNiLQB2gJfl7GtHxAB2DCotVxYaAjPXNeHxtER/GbaUg4dL3A7JGOMnwKZLMoa76G8R3pHA7NUtfiUA4i0AP4N3Kyqp7WMisgEEUkXkfTs7OyzDtgEXnxMJP+6oS/7jp7grrdXUFxiT3kbUxsEMllkAa281pOAXeXUHY1zC+okEWkAfAz8j6ouKmsnVX1JVVNVNTUhwe5S1Ra9WzXkoRHd+HZDNv/4coPb4Rhj/BDIZJEGpIhIWxGJwJMQPihdSUQ6AY2AhV5lEcD7wBuq+k4AYzQuGdOvFdekJvHM15v4Yu1et8MxxvgQsGShqkXARGAusA6YqaprRORhERnhVXUMMENPHXXuGmAIMM6ra23vQMVqqp+I8PDI7vRIjOPuGcv5eNVut0MyxlTAhig3rtp9JI/bpy1jxY7DXJvaigdHdKV+hM3JZUx1sSHKTa3QIq4e7/xmIHdc2J6ZS3fwy2e+t261xtRAliyM68JDQ/jvYZ1589b+HD9RxFXP/8Ar32+x+TCMqUEsWZgaY1CHeD69awhDOibwyEdrufm1NLJzTrgdljEGSxamhmkcHcHLN/XlkZHd+GHzAYY//R3zN9gzNMa4zZKFqXFEhBsHJvPhxPNoHB3O2KlLeOarjW6HZUxQs2RhaqxOzWP5YOJ5jOzdkie/2MA3GfvcDsmYoGXJwtRoUeGh/PVXPenULJb73lnF/mPWhmGMGyxZmBovKjyUf47pw9H8Qia9s9J6SRnjAksWplbo1DyWPwzvzLyMbN5YuM3tcIwJOpYsTK0xdlAyQzsl8OdP1pGxJ8ftcIwJKpYsTK0hIjx+dS8aRIXzu7eWk19Y7HsnY0yVsGRhapX4mEieuLonGXtzeOzT9W6HY0zQsGRhap0LOzXl5sHJvPbDVuZZd1pjqoUlC1Mr/X5YZzo3j2XSOyttSBBjqoHPZCEi0SIS4rzvKCIjRCQ88KEZU76T3Wlz8ov471nWndaYQPPnyuJbIEpEEoGvgJuB1wIZlDH+6Ngslgd+0YV5Gdm8/sNWt8Mxpk7zZ5YZUdVcEbkVeEZV/yYiywMdmDH+uHFAG+ZnZPOXT9ezL+cEcfXCiYkKIzYqnNioMBp4vW9UP4Ko8FC3QzamVvIrWYjIQOB64NZK7GdMwIkIfxvVk+unLOaF+ZspqeBuVGxUGLN+M4hOzWOrL0Bj6gh/fvTvBu4H3nfm0G4HzAtsWMb4r0lMJJ/dPQRVJbegmJz8InLyCzmaX8SxE877vCIe+3Qdf/lkHa/f0s/tkI2pdXwmC1WdD8wHcBq696vq7wIdmDGVJSJER4YRHRlG87io07bnFhTxp4/X8e2GbIZ0THAhQmNqL396Q00XkQYiEg2sBTJEZFLgQzOmat04sA2tGtfjL5+so7ii+1XGmNP40xuqq6oeBa4APgFaAzcGNCpjAiAyLJTfD+vM+j05vLssy+1wjKlV/EkW4c5zFVcAc1S1EPDrzzIRGSYiGSKySUQml7H97yKywlk2iMhhr21jRWSjs4z19wsZU5Ff9GhB71YNefLzDHILitwOx5haw59k8SKwFYgGvhWRNsBRXzuJSCjwHDAc6AqMEZGu3nVU9R5V7a2qvYFngPecfRsDDwL9gX7AgyLSyN8vZUx5RIQHftGFvUdPMOW7LW6HY0yt4TNZqOo/VTVRVX+uHtuAoX4cux+wSVUzVbUAmAGMrKD+GOAt5/1lwBeqelBVDwFfAMP8+ExjfDo3uTHDujXnhfmb2ZeT73Y4xtQK/jRwx4nIUyKS7ixP4rnK8CUR2OG1nuWUlfUZbYC2wNeV3deYM/H74Z0pKCrh719sdDsUY2oFf25DTQVygGuc5Sjwqh/7SRll5bV1jAZmqerJCQr82ldEJpxMYtnZ2X6EZIxH2/hobhjQhrfTtrNhr02kZIwv/iSL9qr6oHM7KVNV/wi082O/LKCV13oSsKucuqP5zy0ov/dV1ZdUNVVVUxMSrN+8qZzf/SyF6MgwHv1knduhGFPj+ZMs8kTkvJMrIjIYyPNjvzQgRUTaikgEnoTwQelKItIJaAQs9CqeC1wqIo2chu1LnTJjqkzj6AgmDu3AvIxsvt+43+1wjKnR/EkWtwPPichWEdkGPAv8xtdOqloETMTzI78OmOkMF/KwiIzwqjoGmKFeY0yr6kHgETwJJw142CkzpkqNHZRMYsN6/Nke1DOmQuLvPAAi0gDAeUCvxklNTdX09HS3wzC10JwVO7lrxgqeuLoXo/omuR2OMdVKRJaqaqqveuWODSUi95ZTDoCqPnXG0RlTg1zesyVTv9/CE3Mz+EWPFtSLsGHMjSmtottQsT4WY+qEkBDhDz/vwp6j+bzyfabb4RhTI5V7ZeH0ejImKPRv14SLuzTjpW8zufW8dnZ1YUwp/jRwGxMUbj2vLUfzi/h49W63QzGmxrFkYYxjQLvGtEuIZvribW6HYkyNY8nCGIeIcF2/1izbfpj1e2pkpz9jXOPP2FCRInKdiPxBRP7v5FIdwRlT3X51ThIRYSFMX7zd7VCMqVH8ubKYg2e02CLguNdiTJ3TKDqCn3dvzvvLdtp8F8Z48TkHN5CkqjY8uAka1w9ow+wVu/hw5S6uPbe12+EYUyP4c2Xxg4j0CHgkxtQQqW0akdI0xm5FGePFn2RxHrDUmR51lYisFpFVgQ7MGLeICNf1b83KrCP8uPOI2+EYUyP4kyyGAyl4Rn69HPil82pMnXVVnyQiw0KYvsSuLowB/6ZV3QY0xJMgLgcaOmXG1Flx9cP5Zc+WzFm+k2MnrKHbGH+6zt4FvAk0dZZpInJnoAMzxm3XD2jN8YJi5qzY6XYoxrjOn9tQtwL9VfX/VPX/gAHA+MCGZYz7+rRqSOfmsUxfvB1/h/I3pq7yJ1kIUOy1XkzZc2QbU6eICNf3b82aXUdZlWUN3Sa4+ZMsXgUWi8hDIvIQsAh4JaBRGVNDjOyTSL3wUOtGa4KePw3cTwE3AweBQ8DNqvqPQAdmTE3QICqcEb1a8sHKXRzNL3Q7HGNcU26yODmNqog0BrYC04B/A9ucMmOCwnX9W5NXWMyc5dbQbYJXRVcW053XpUC613Jy3Zig0DMpju6JDXjTGrpNECs3WajqL53Xtqrazmtpq6rtqi9EY9zlGbq8Dev35LBs+2G3wzHGFf48Z/GVP2XG1GUjerckOsIauk3wqqjNIsppm4gXkUYi0thZkoGW/hxcRIY5Y0ptEpHJ5dS5RkTWisgaEZnuVf43p2ydiPxTRKy7rnFNTGQYI/sk8tGqXRzJtYZuE3wqurK4DU/7RGfn9eQyB3jO14FFJNSpNxzoCowRka6l6qQA9wODVbUbcLdTPggYDPQEugPnAhdU5osZU9Wu69eaE0UlPP75ek4UFfvewZg6pKI2i6dVtS1wn1dbRVtV7aWqz/px7H7AJlXNVNUCYAaeSZS8jQeeU9VDzmfuO/nxQBQQAUQC4cDeSn0zY6pY98Q4xvRrzbRF2/nlP79n6baDbodkTLXx5zmLZ0Sku3O76KaTix/HTgR2eK1nOWXeOgIdRWSBiCwSkWHOZy4E5gG7nWWuqq7z5wsZE0iPXtWDV8edS25BMaNeWMj/zv6RHHv+wgQBfxq4HwSecZahwN+AEX4cu6w2htL9DsPwDH9+ITAGmCIiDUWkA9AFSMKTYC4SkSFlxDZBRNJFJD07O9uPkIw5e0M7N+Xze4YwblAy0xZv45KnvuWLtXbha+o2f4b7GAX8DNijqjcDvfDcGvIlC2jltZ4E7CqjzhxVLVTVLUAGnuRxJbBIVY+p6jHgUzwDGJ5CVV9S1VRVTU1ISPAjJGOqRnRkGA9e3o33bh9Ew/rhjH8jnTveXMq+nHy3QzMmIPxJFnmqWgIUOU917wP8ec4iDUgRkbYiEgGMBj4oVWc2nqsVRCQez22pTGA7cIGIhIlIOJ7GbbsNZWqcPq0b8eGd5zHpsk58uW4fFz85nxlL7OE9U/f4kyzSRaQh8DKe3lDLgCW+dlLVImAiMBfPD/1MVV0jIg+LyMnbWHOBAyKyFk8bxSRVPQDMAjYDq4GVwEpV/bByX82Y6hEeGsJvh3bgs7vOp0uLBkx+bzXPfr3J7bCMqVJSmb+AnGcsGqhqjZuDOzU1VdPTbRQS466SEuW+d1by3vKdPD26NyN7l+7TYUzNIiJLVTXVV72wCg5wTkXbVHXZmQZnTF0VEiI8+qseZB3OY9KsVSQ1qkffNjbupqn9yr2yEJF5ztsoIBXP7SDB86DcYlU9r1oi9JNdWZia5NDxAq761w8cySvk/TsG0aZJtNshGVMmf68sKnoob6iqDgW2Aec4vY76An0AuyFrTAUaRUcwddy5lKhy82tpNkSIqfX8aeDurKqrT66o6o9A78CFZEzd0DY+mpduTCXrYB63TUunoKjE7ZCMOWP+JIt1IjJFRC4UkQtE5GWsG6sxfunXtjF/HdWDRZkH+cP7q61Lram1ym3g9nIzcDtwl7P+LfCvgEVkTB1zZZ8ktu7P5emvNtI2PprfDu3gdkjGVJrPZKGq+cDfncUYcwbuvjiFbQeO8/jcDFo3rs/lvfwa5d+YGqOirrMzVfUaEVnN6WM6oao9AxqZMXWIiPDXUT3ZeTiP/3pnJS0b1qNvm0Zuh2WM3yrqOttCVXeLSJuytqvqtoBGVknWddbUBgePF3Dl8ws4nFvIc9edw3kp8W6HZIJcVXSd3e28bitrqcpgjQkWjaMjmHZrf5o1iOSmqYuZ8l2mNXqbWqGiaVVzRORoGUuOiBytziCNqUtaNa7Pe3cM5tKuzfnTx+u45+0V5BXYzHumZqvoyiJWVRuUscSqaoPqDNKYuiYmMox/3XAO913akTkrdzHqhR/IOpTrdljGlMuf5ywAEJGmItL65BLIoIwJBiLCxItSeGVsKtsP5DLi2QUs3HzA7bCMKZM/M+WNEJGNwBZgPrAVz2RExpgqcFHnZsyeOJhG9cO54ZXFvLZgi7VjmBrHnyuLR/DMUrdBVdvimTVvQUCjMibItE+IYfZvBzO0U1Me+nAtk2atIr/Q2jFMzeFPsih0JiQKEZEQVZ2HjQ1lTJWLjQrnpRv7ctfPUpi1NIt73l7hdkjG/MSf4T4Oi0gMnmE+3hSRfUBRYMMyJjiFhAj3XNKRiLAQHp+bwZdr93Jx12Zuh2WMX1cWI4Fc4B7gMzzTnV4eyKCMCXYThrSjY7MYHvxgDbkF9reZcZ8/yWIC0FJVi1T1dVX9p3NbyhgTIOGhIfzlyh7sPJzHP77c6HY4xviVLBoAc0XkOxH5rYjYNbEx1SA1uTGjz23FK99vYe0uew7WuMtnslDVP6pqN+C3QEtgvoh8GfDIjDFMHt6ZhvXCeWD2akpKrDutcY/fD+UB+4A9wAGgaWDCMcZ4a1g/ggd+0YXl2w8zfcl2t8MxQcyfh/JuF5FvgK+AeGC8v8OTi8gwEckQkU0iMrmcOteIyFoRWSMi073KW4vI5yKyztme7M9nGlPXXNknkUHtm/DXz9azLyff7XBMkPLnyqINcLeqdlPVB1V1rT8HFpFQ4DlgONAVGCMiXUvVSQHuBwY7t7ru9tr8BvC4qnYB+uG5sjEm6IgIj1zRnROFJfz5Y5vR2LjDnzaLyap6Jk8H9QM2qWqmqhYAM/B0w/U2HnhOVQ85n7UPwEkqYar6hVN+TFVtlDUTtNonxHD7he2Zs2IX323MdjscE4Qq02ZRWYnADq/1LKfMW0ego4gsEJFFIjLMq/ywiLwnIstF5HHnSsWYoHX7he1pGx/N/87+0YYCMdUukMlCyigr3Z0jDEgBLgTGAFNEpKFTfj5wH3Au0A4Yd9oHiEwQkXQRSc/Otr+2TN0WFR7Kn6/oztYDuTw/b5Pb4ZggE8hkkQW08lpPAnaVUWeOqhaq6hYgA0/yyAKWO7ewioDZwDmlP0BVX1LVVFVNTUhICMiXMKYmGdQhniv7JPKv+ZvZtO+Y2+GYIBLIZJEGpIhIWxGJAEYDH5SqMxsYCiAi8XhuP2U6+zYSkZMZ4CLAr4Z1Y+q6B37RhfoRYTzw/mobytxUm4AlC+eKYCIwF1gHzFTVNSLysIiMcKrNBQ6IyFpgHjBJVQ+oajGeW1BfichqPLe0Xg5UrMbUJvExkUwe3pnFWw7y6Y973A7HBAmpK3+ZpKamanp6utthGFMtSkqUoU9+Q3xMJO/ePsjtcEwtJiJLVTXVV71A3oYyxgRISIgwdmAyS7cdYlXWYbfDMUHAkoUxtdTVqUnERIbx6oKtbodigoAlC2NqqdiocEb1TeKjVbvYd9SGATGBZcnCmFps3KBkikqUaYttkMHaZl9OPje+sphFmbVjeiBLFsbUYsnx0VzUqSnTF2/jRJE91V1bqCqT313Ndxv3M2nWSvIKav6/nSULY2q5mwe3Zf+xAj5cudvtUIyf3lqyg6/X7+PKPonsOJjHs/Nq/myIliyMqeUGd2hCx2YxvLpgiz2kVwts3X+cRz5ay/kp8Tx5dS+uOieRl77NZOPeHLdDq5AlC2NqORFh3KC2rNl1lLSth9wOx1SgqLiEu99eQURYCI+P6kVIiPCHnztP5M/+sUYne0sWxtQBV/ZJpGH9cF5dsMXtUEwFnv9mMyt2HOZPV3SneVwU8J8n8pdsOci7y3a6HGH5LFkYUwfUiwhl9LmtmbtmD1mHbOqXmmjljsM8/dVGRvZuyeW9Wp6y7drUVvRt04i/fLKOQ8cLXIqwYpYsjKkjbhrYBhHh3wu3uR2KKSWvoJh7Zq6gaWwkD4/oftr2kBDhz1d250heIX/9bL0LEfpmycKYOqJlw3oM69act5ZsJ7egyO1wjJfHPl1HZvZxnri6F3H1w8us07l5A359XltmpO0gfevBao7QN0sWxtQhNw9O5mh+Ee/V4HvfwWb+hmxeX7iNW89ry+AO8RXWveviFBIb1uOB93+ksLikmiL0jyULY+qQvm0a0SMxjtd+2Fqje9YEi0PHC5j0zkpSmsYw6bJOPuvXjwjjoRHdyNibw9Tva1ZnBUsWxtQhIsLNg5PZtO8Y323c73Y4QU1V+Z/ZP3Iot4C/X9ubqPBQv/a7pGszLunajH98ubFGdVawZGFMHfOLni2Ij4m0brRnoaDo7G8BvbM0i49X7+aeSzrSPTGuUvs+NKIbAA/OWVNjrhDD3A7AGFO1IsNCuWFAa/7x5UYys4/RLiHG7ZBqrLyCYjbuy2H9nhw27MkhY28OGXty2Jdzgou7NGPiRR3o3aphpY65KuswT36+gfkbsumX3JjbhrSvdFyJDetxzyUp/OWT9Xy+di+XdWte6WNUNZspz5g6KDvnBIMf+5ox/Vrxx5Gnd9UMVqrKm4u38+2GbDL25rD9YC4nfwIjw0JIaRZDp2YNiKsXzrvLsjiSV8j5KfGr8HtTAAATj0lEQVRMHNqB/u2aVHjsjD05PPVFBnPX7KVh/XBuv6A9Nw1Mpl6Ef7efSissLuHyZ77ncG4hs24fSFKj+md0HF/8nSnPkoUxddS9M1fw2Y97+P73F9E4OsLtcFyXV1DMfbNW8vGq3bSNj6ZLi1g6Noulc3PPa5sm0YSGyE/1j50oYtqibUz5LpP9xwrol9yYiRd14PyUeET+U2/L/uP848sNfLByFzERYfz6/Hbccl4ysVFld5GtjNVZR7huyiIiw0KZMja10lc5/rBkYUyQ27Qvh0v//i1jByXz4OXd3A7HVbuP5DH+jXTW7DrK5GGdmTCk3Sk/+BXJKyjm7bTtvDA/kz1H8+mVFMfEi1Lo0iKWZ77axKxlWYSHesbnum1IOxpVcWLeuDeHm19LY/+xE/zj2t4M696iSo9vycIYw+R3V/Husiy+/q8LadU4MLcxqsuxE0Us3XaIc5MbUT/C/+bW5dsPMeHfS8krKObp0b35WZdmZ/T5J4qKeW/ZTp7/ZhM7DuYBEBEawnX9W3PH0PY0jY06o+P6Y/+xE4x/I50VOw5z//DOjD/f/2TniyULYwx7j+ZzwePzuKxbc54e3cftcM7IkbxCXv9hK1MXbOFwbiEN64dz08Bkxg5sQ5OYyAr3fX95Fr9/dzXNG0QxZWwqHZvFnnU8RcUlfLhqF5v3HWdM/9YkNqx31sf0R35hMf81cyUfr97NmH6teXhkN8JDz75Da41IFiIyDHgaCAWmqOpjZdS5BngIUGClql7nta0BsA54X1UnVvRZliyMKdvjc9fz3LzNfHTneZXuwummw7kFTF2wlVcXbCEnv4iLuzRlZO9E5qzYxZfr9hIZFsLVqUmMP78dbZpEn7JvcYny+NwMXpi/mQHtGvP89X3rRLtNSYnyxOcZPP/NZs5Piee568+hwVm2jbieLEQkFNgAXAJkAWnAGFVd61UnBZgJXKSqh0Skqaru89r+NJAAHLRkYcyZOZpfyAV/m0e3lnFM+3V/t8Px6eDxAqZ8l8kbC7dx7EQRw7o1Z+JFHU5JdJv25fDyt1t4f/lOikpKGNa9ObcNaU+vVg05dqKIu2cs58t1+7iuf2v+OKJq/gKvSWam7eAP76+mXUI0U8ede1Y9pWpCshgIPKSqlznr9wOo6qNedf4GbFDVKWXs3xeYBHwGpFqyMObMTf1+Cw9/tJY3bunHkI4JbodTpn05+Uz5bgvTFm0jr7CYn/dowZ0XdaBz8wbl73M0n1d/2Mq0RdvIyS+if9vGHMotYHP2cR68vCs3DmhTZff2a5oFm/bzm2lLiQwL5ZWxqfQ6w55SNSFZjAKGqeqvnfUbgf7eP/oiMhvP1cdgPLeqHlLVz0QkBPgauBH4GZYsjDkrJ4qKufip+cRGhvPRnecREuLuD2huQRFrdh1lVdYRVmUdZlXWEbbsP06IwIheLZl4UQc6NPW/fSEnv5AZS3YwdcEWcguKef76c3wO2lcXnOwpVT8ilE/vGnJK119/+ZssAvkEd1lRl85MYUAKcCGQBHwnIt2BG4BPVHVHRX8ViMgEYAJA69atqyBkY+qmyLBQ7ru0E3fNWMGclTu5sk9StX5+ds4JPluzh9VOYtiwN4cS59egRVwUPRLjGNU3ieHdm5/RE+exUeGMH9KOcYOTKSgqIToyOAanSGkWy+zfDuZoXuEZJYrKCOQZzQJaea0nAbvKqLNIVQuBLSKSgSd5DATOF5E7gBggQkSOqepk751V9SXgJfBcWQTmaxhTN1zesyUvf5fJE3M3MLx7C78HtjtbR/MLGfXCD2w7kEvj6Ah6JsVxabfm9EyMo2eruCrtchoeGlLn2id8iY+JJN5Hr7CqEMhkkQakiEhbYCcwGriuVJ3ZwBjgNRGJBzoCmap6/ckKIjIOz22oyRhjzlhIiDB5WBdueGUx0xZt49fntwv4Z6oqv5+1iqxDebz56/4Mat+kzrYh1HUBS8GqWgRMBObi6f46U1XXiMjDIjLCqTYXOCAia4F5wCRVPRComIwJduelxHN+SjzPztvEkbzCgH/eqwu28umPe5g8rDODO8RboqjF7KE8Y4LMml1H+OUz33PbkPZMHt45YJ+zbPshrnlhIUM7N+WlG/taoqih/G3gDq6be8YYurWM44reiby6YAu7j+QF5DMOHS9g4pvLaB4XxROjelmiqAMsWRgThO69pCOq8PcvNlT5sUtKlHtnrmD/sQKev/4c4uqf/eirxn2WLIwJQq0a1+fGgW2YtTSLDXtzqvTY/5q/mXkZ2fzvL7vQM6nqh9Q27rBkYUyQmji0A9GRYTwxN6PKjrko8wBPfp7B5b1acsOANlV2XOM+SxbGBKlG0RGMHZjMF+v2suNg7lkfb19OPne+tZzkJtE8elUPa6eoYyxZGBPErh/QmhAR3li49ayOU1yi3PXWCnLyC3n+hnOICZInqIOJJQtjgliLuHoM69act9N2kFtQdMbH+ceXG1iYeYBHRnavcOA/U3tZsjAmyI0bnMzR/CJmLy89Go9/FmUe4Nl5m7i6bxJXp7byvYOplSxZGBPkUts0omuLBrz+w1Yq+5BuSYny54/X0TKuHg+P7B6gCE1NYMnCmCAnIowblEzG3hwWZlZutJ2PV+9m9c4j3HtJR+pFVM/AhMYdliyMMYzo3ZJG9cN5/Yetfu9TWFzCk59n0KlZLFf0SQxccKZGsGRhjCEqPJTR/Vrzxdq9ZB3yrxvtjLQdbD2Qy++Hdwr4XArGfZYsjDEAPz1EN23Rdp91j58o4ukvN9IvuTFDOzUNdGimBrBkYYwBILFhPS7r1pwZadvJLyyusO7U77ew/9gJfj+8sz18FyQsWRhjfjJ2UDKHcwuZs2JnuXUOHi/gxW8zuaRrM/q2aVSN0Rk3WbIwxvykf9vGdG4ey2s/bCu3G+1z8zaRW1DEf1/WqZqjM26yZGGM+YmIMHZQMut2H2XJloOnbc86lMu/F25jVN8kUprFuhChcYslC2PMKa7onUhcvXBeX7j1tG1PfbEBBO6+uGO1x2XcZcnCGHOKehGhjD63FXPX7GXX4f/MpLd+z1HeX76TcYOSadmwnosRGjdYsjDGnOaGAW1QVd5cvO2nsr99lkFMZBh3XNjexciMWyxZGGNO06pxfS7u0oy3luwgv7CYJVsO8vX6fdx+YXsa1o9wOzzjAksWxpgyjRuUzMHjBXy4chePfbqOZg0iuXlQW7fDMi6xGUqMMWUa2L4JHZvF8KeP13Ekr5C/XNnDBgsMYgG9shCRYSKSISKbRGRyOXWuEZG1IrJGRKY7Zb1FZKFTtkpErg1knMaY053sRnskr5B28dFck5rkdkjGRQG7shCRUOA54BIgC0gTkQ9Uda1XnRTgfmCwqh4SkZODzOQCN6nqRhFpCSwVkbmqejhQ8RpjTndln0Q+XrWb2y9sT1io3bUOZoG8DdUP2KSqmQAiMgMYCaz1qjMeeE5VDwGo6j7ndcPJCqq6S0T2AQmAJQtjqlH9iDCmjx/gdhimBgjknwqJwA6v9SynzFtHoKOILBCRRSIyrPRBRKQfEAFsLmPbBBFJF5H07OzsKgzdGGOMt0Ami7KGoiw92EwYkAJcCIwBpohIw58OINIC+Ddws6qWnHYw1ZdUNVVVUxMSEqoscGOMMacKZLLIArxnb08CSs8InwXMUdVCVd0CZOBJHohIA+Bj4H9UdVEA4zTGGONDIJNFGpAiIm1FJAIYDXxQqs5sYCiAiMTjuS2V6dR/H3hDVd8JYIzGGGP8ELBkoapFwERgLrAOmKmqa0TkYREZ4VSbCxwQkbXAPGCSqh4ArgGGAONEZIWz9A5UrMYYYyom5Y1ZX9ukpqZqenq622EYY0ytIiJLVTXVVz3rOG2MMcYnSxbGGGN8qjO3oUQkG9hWqjgOOOKjrKw68cD+Kg2wfGV9fqD29VW/ou2V3ebrPNeWc3wm+wfiPPtbHizn2Z+6VXWe/Smrzee5jar6fvZAVevsArzkq6ycOuluxhiofX3Vr2h7Zbf5Os+15RzXlPPsb3mwnGd/6lbVefbz/3edPM/eS12/DfWhH2Vl1alOZ/P5ld3XV/2Ktld2W006z2f72TXhPPtbHizn2Z+6VXWe/S2rLtX9/xmoQ7ehqpKIpKsfvQPMmbNzXD3sPFePYDjPdf3K4ky95HYAQcDOcfWw81w96vx5tisLY4wxPtmVhTHGGJ8sWRhjjPHJkoUxxhifLFlUkohEi8hSEfml27HUVSLSRUReEJFZInK72/HUVSJyhYi8LCJzRORSt+Opq0SknYi8IiKz3I7lbARNshCRqSKyT0R+LFU+TEQyRGSTiEz241C/B2YGJsraryrOs6quU9Xf4Bl9uE53RzxTVXSeZ6vqeGAccG0Aw621qug8Z6rqrYGNNPCCpjeUiAwBjuGZI6O7UxYKbAAuwTMRUxqeGftCgUdLHeIWoCeex/qjgP2q+lH1RF97VMV5VtV9zjD2k4FnVXV6dcVfW1TVeXb2exJ4U1WXVVP4tUYVn+dZqjqqumKvamFuB1BdVPVbEUkuVdwP2KSqmQAiMgMYqaqPAqfdZhKRoUA00BXIE5FPtIzpXoNZVZxn5zgfAB+IyMeAJYtSquj/swCPAZ9aoihbVf1/rguCJlmUIxHY4bWeBfQvr7KqPgAgIuPwXFlYovBPpc6ziFwIXAVEAp8ENLK6pVLnGbgTuBiIE5EOqvpCIIOrQyr7/7kJ8Gegj4jc7ySVWifYk4WUUebzvpyqvlb1odRplTrPqvoN8E2ggqnDKnue/wn8M3Dh1FmVPc8HgN8ELpzqETQN3OXIAlp5rScBu1yKpS6z81w97DxXj6A8z8GeLNKAFBFpKyIRwGjgA5djqovsPFcPO8/VIyjPc9AkCxF5C1gIdBKRLBG5VVWLgInAXGAdMFNV17gZZ21n57l62HmuHnae/yNous4aY4w5c0FzZWGMMebMWbIwxhjjkyULY4wxPlmyMMYY45MlC2OMMT5ZsjDGGOOTJQvjGhE5Vg2fMcLPoeer8jMvFJFBZ7BfHxGZ4rwfJyLPVn10lSciyaWH6C6jToKIfFZdMZnqZ8nC1HrOkNFlUtUPVPWxAHxmReOqXQhUOlkAfwCeOaOAXKaq2cBuERnsdiwmMCxZmBpBRCaJSJqIrBKRP3qVz3ZmJlwjIhO8yo+JyMMishgYKCJbReSPIrJMRFaLSGen3k9/oYvIayLyTxH5QUQyRWSUUx4iIs87n/GRiHxyclupGL8Rkb+IyHzgLhG5XEQWi8hyEflSRJo5w1n/BrhHRFaIyPnOX93vOt8vrawfVBGJBXqq6soytrURka+cc/OViLR2ytuLyCLnmA+XdaUmnpkdPxaRlSLyo4hc65Sf65yHlSKyRERinSuI75xzuKysqyMRCRWRx73+rW7z2jwbuL7Mf2BT+6mqLba4sgDHnNdLgZfwjOYZAnwEDHG2NXZe6wE/Ak2cdQWu8TrWVuBO5/0dwBTn/Tg8EygBvAa843xGVzxzEgCMwjMUegjQHDgEjCoj3m+A573WG/GfURB+DTzpvH8IuM+r3nTgPOd9a2BdGcceCrzrte4d94fAWOf9LcBs5/1HwBjn/W9Ons9Sx/0V8LLXehwQAWQC5zplDfCMQF0fiHLKUoB0530y8KPzfgLwP877SCAdaOusJwKr3f5/ZUtglmAfotzUDJc6y3JnPQbPj9W3wO9E5EqnvJVTfgAoBt4tdZz3nNeleObDKMts9cxDslZEmjll5wHvOOV7RGReBbG+7fU+CXhbRFrg+QHeUs4+FwNdRX4a2bqBiMSqao5XnRZAdjn7D/T6Pv8G/uZVfoXzfjrwRBn7rgaeEJG/Ah+p6nci0gPYrappAKp6FDxXIcCzItIbz/ntWMbxLgV6el15xeH5N9kC7ANalvMdTC1nycLUBAI8qqovnlLomQTpYmCgquaKyDd4prQFyFfV4lLHOeG8FlP+/+0TXu+l1Ks/jnu9fwZ4SlU/cGJ9qJx9QvB8h7wKjpvHf76bL34P6KaqG0SkL/Bz4FER+RzP7aKyjnEPsBfo5cScX0YdwXMFN7eMbVF4voepg6zNwtQEc4FbRCQGQEQSRaQpnr9aDzmJojMwIECf/z3wK6ftohmeBmp/xAE7nfdjvcpzgFiv9c/xjFIKgPOXe2nrgA7lfM4PeIbBBk+bwPfO+0V4bjPhtf0UItISyFXVaXiuPM4B1gMtReRcp06s02Afh+eKowS4Ec+c0qXNBW4XkXBn347OFQl4rkQq7DVlai9LFsZ1qvo5ntsoC0VkNTALz4/tZ0CYiKwCHsHz4xgI7+KZ0OZH4EVgMXDEj/0eAt4Rke+A/V7lHwJXnmzgBn4HpDoNwmspY9Y0VV2PZ3rT2NLbnP1vds7DjcBdTvndwL0isgTPbayyYu4BLBGRFcADwJ9UtQC4FnhGRFYCX+C5KngeGCsii/D88B8v43hTgLXAMqc77Yv85ypuKPBxGfuYOsCGKDcGEJEYVT0mnvmSlwCDVXVPNcdwD5CjqlP8rF8fyFNVFZHReBq7RwY0yIrj+RYYqaqH3IrBBI61WRjj8ZGINMTTUP1IdScKx7+AqytRvy+eBmkBDuPpKeUKEUnA035jiaKOsisLY4wxPlmbhTHGGJ8sWRhjjPHJkoUxxhifLFkYY4zxyZKFMcYYnyxZGGOM8en/AV1b4aQLhjlFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f290262f160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.lr_find(lrs/1000)\n",
    "learner.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b42db378379445dbf2614b5ece6d5c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy   F1         auc        \n",
      "    0      0.559315   0.621766   0.675749   0.61645    0.66775   \n",
      "    1      0.505593   0.591342   0.746594   0.644038   0.726606  \n",
      "    2      0.496297   0.513562   0.757493   0.666804   0.73991   \n",
      "    3      0.504554   0.493959   0.754768   0.674524   0.738005  \n",
      "    4      0.48473    0.533696   0.743869   0.636558   0.725782  \n",
      "    5      0.488727   0.470733   0.754768   0.660888   0.737675  \n",
      "    6      0.464358   0.489378   0.762943   0.672949   0.745439  \n",
      "    7      0.469297   0.49725    0.746594   0.644673   0.727852  \n",
      "    8      0.43013    0.488344   0.752044   0.647589   0.731818  \n",
      "    9      0.465323   0.518435   0.749319   0.638491   0.728348  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([ 0.51844]),\n",
       " 0.74931879670483537,\n",
       " 0.63849082702437776,\n",
       " 0.72834813975593005]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(lrs, 1, wds=wd, cycle_len=10, use_clr=(8,3))\n",
    "#learner.save('clf_tmp1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ef8f53be2ac46f9924ddbeab025cd80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=10), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy   F1         auc        \n",
      "    0      0.429842   0.508345   0.73842    0.612211   0.717031  \n",
      "    1      0.48086    0.546719   0.743869   0.649245   0.724528  \n",
      "    2      0.445901   0.515684   0.760218   0.659899   0.740114  \n",
      "    3      0.459717   0.585774   0.73842    0.603009   0.71423   \n",
      "    4      0.440359   0.602096   0.749319   0.739099   0.751444  \n",
      "    5      0.422634   0.508611   0.771117   0.672519   0.750724  \n",
      "    6      0.411085   0.495621   0.79564    0.730642   0.781097  \n",
      "    7      0.398806   0.671865   0.754768   0.63705    0.734853  \n",
      "    8      0.381756   0.450636   0.79564    0.733339   0.781765  \n",
      "    9      0.372775   0.419201   0.809264   0.752968   0.796199  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([ 0.4192]),\n",
       " 0.80926430322818599,\n",
       " 0.75296757699470684,\n",
       " 0.79619898970514624]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.freeze_to(-2)\n",
    "learner.fit(lrs, 1, wds=wd, cycle_len=10, use_clr=(8,3))\n",
    "#learner.save('clf_tmp2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7d70e56bdbf4bf6ac0882698d676184",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy   F1         auc        \n",
      "    0      0.35443    0.469472   0.80654    0.759192   0.795974  \n",
      "    1      0.394834   0.612359   0.798365   0.730233   0.783135  \n",
      "    2      0.359095   0.393794   0.836512   0.791179   0.826786  \n",
      "    3      0.328546   0.35958    0.841962   0.812326   0.836945  \n",
      "    4      0.297317   0.328974   0.850136   0.817019   0.843027  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[array([ 0.32897]),\n",
       " 0.85013623507211256,\n",
       " 0.81701852731820823,\n",
       " 0.84302684206460721]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.unfreeze()\n",
    "learner.fit(lrs, 1, wds=wd, cycle_len=5, use_clr=(8,3))\n",
    "#learner.save('clf_tmp3')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#learner.fit(lrs, 1, wds=wd, cycle_len=1, use_clr=(8,3))\n",
    "#learner.load('clf_tmp2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds = learner.predict()\n",
    "preds.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "yhat = np.argmax(np.exp(preds),axis=1)\n",
    "np.sum(yhat==y_test)/len(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.max(np.exp(preds[:,-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get predictions\n",
    "preds_dist, preds = learner.predict_with_targs()\n",
    "\n",
    "# prepare for accuracy measurement\n",
    "preds = preds.flatten()\n",
    "golds = learner.data.val_y\n",
    "\n",
    "print(np.sum(preds==golds)/len(golds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds[:10],golds[:10],y_test[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(np.sum(np.argmax(preds_dist[:,1],axis=1)==golds)/len(golds))\n",
    "preds_dist[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "??predict_with_targs_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tmp = predict_with_targs_(learner.model,dl_test)\n",
    "p,t = tmp\n",
    "ntmp = np.concatenate(p), np.concatenate(t)\n",
    "ntmp[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=0\n",
    "for *x,y in iter(dl_test):\n",
    "    print(to_np(learner.model(*VV(x))),y)\n",
    "    i+=1\n",
    "    if i>10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tup = learner.model(*VV(x))#.data).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tup[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}