final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "808a6ae2",
   "metadata": {},
   "source": [
    "# Final - James Guo (zg268)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7208c49",
   "metadata": {},
   "source": [
    "## Training/test split (5pts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c3e929a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ae95f36a",
   "metadata": {},
   "outputs": [],
   "source": [
    "Labels = pd.read_csv('Labels.txt', sep='\\t')\n",
    "UniProt2Seq = pd.read_csv('UniProt2Seq.txt', sep='\\t')\n",
    "Expasy_AA_Scales = pd.read_csv('Expasy_AA_Scales.txt', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e228c25d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "U        1\n",
       "X        1\n",
       "W     6477\n",
       "C     9068\n",
       "M    12094\n",
       "H    12681\n",
       "Y    16637\n",
       "F    20696\n",
       "Q    23294\n",
       "N    23354\n",
       "R    28514\n",
       "I    28634\n",
       "D    29907\n",
       "P    29911\n",
       "T    30581\n",
       "K    33709\n",
       "V    35199\n",
       "G    37443\n",
       "E    38569\n",
       "S    40325\n",
       "A    40816\n",
       "L    51044\n",
       "Name: AA, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "UniProt2Seq.AA.value_counts(ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "73b2890c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['A', 'C', 'E', 'D', 'G', 'F', 'I', 'H', 'K', 'M', 'L', 'N', 'Q',\n",
       "       'P', 'S', 'R', 'T', 'W', 'V', 'Y'], dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Expasy_AA_Scales['Amino Acid'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6edcd5da",
   "metadata": {},
   "source": [
    "**I noticed 'X' and 'U' are not in `Expasy_AA_Scales` column `Amino Acid`. I will drop these 2 observations.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1bad578f",
   "metadata": {},
   "outputs": [],
   "source": [
    "index_to_drop = [UniProt2Seq.loc[UniProt2Seq['AA'] == 'U'].index, UniProt2Seq.loc[UniProt2Seq['AA'] == 'X'].index]\n",
    "UniProt2Seq.drop([index_to_drop[0][0], index_to_drop[1][0]], axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "93bfc5f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    296315\n",
       "1     81999\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Labels.Label.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a13d9a6",
   "metadata": {},
   "source": [
    "**I noticed that the number of `0` is almost threefold the number of `1`, leading to class imbalance. I will down-sample class `0`.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a281c720",
   "metadata": {},
   "outputs": [],
   "source": [
    "Labels_major= Labels[Labels.Label==0]\n",
    "Labels_minor = Labels[Labels.Label==1]\n",
    "Labels_major_downsampled = Labels_major.sample(len(Labels_minor), replace=False)\n",
    "df = Labels_major_downsampled.append(Labels_minor)\n",
    "df.sort_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eb165b70",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train, df_test = train_test_split(df, test_size=0.3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65825a50",
   "metadata": {},
   "source": [
    "## Feature engineering (20pts)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "642b201c",
   "metadata": {},
   "source": [
    "**I will set the index of `Labels` and `UniProt2Seq` to be (`UniProt`, `Position`) , and the index of `Expasy_AA_Scales` to be `Amino Acid`.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6b51fcc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "Labels.set_index(['UniProt', 'Position'], inplace=True)\n",
    "UniProt2Seq.set_index(['UniProt', 'Position'], inplace=True)\n",
    "Expasy_AA_Scales.set_index('Amino Acid', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "24f8beca",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_new = df_train.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acf3c2ae",
   "metadata": {},
   "source": [
    "**I will create an empty column called `AA` in `df_train_new`. I will set the index of `df_train_new` to be (`UniProt`, `Position`), and fill the corresponding `AA` values from `UniProt2Seq` into column `AA` in `df_train_new`.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dca64e05",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_train_new['AA'] = ''\n",
    "df_train_new.set_index(['UniProt', 'Position'], inplace=True)\n",
    "df_train_new.sort_index(inplace=True)\n",
    "for i in df_train_new.index:\n",
    "    df_train_new.loc[i, 'AA'] = UniProt2Seq.loc[i].AA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "03d0020c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "      <th>AA</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UniProt</th>\n",
       "      <th>Position</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>P19367</th>\n",
       "      <th>474</th>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P42227</th>\n",
       "      <th>382</th>\n",
       "      <td>0</td>\n",
       "      <td>R</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P23919</th>\n",
       "      <th>57</th>\n",
       "      <td>1</td>\n",
       "      <td>L</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>P30429</th>\n",
       "      <th>185</th>\n",
       "      <td>0</td>\n",
       "      <td>D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Q96QU1</th>\n",
       "      <th>944</th>\n",
       "      <td>0</td>\n",
       "      <td>P</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Label AA\n",
       "UniProt Position          \n",
       "P19367  474           1  L\n",
       "P42227  382           0  R\n",
       "P23919  57            1  L\n",
       "P30429  185           0  D\n",
       "Q96QU1  944           0  P"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_new.sample(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba00f3bc",
   "metadata": {},
   "source": [
    "**`get_neighbor(ID)`:** return a list of the neighboring AA letters within a 5-window with the given ID as the center.<br><br>\n",
    "**`get_ave(result)`:** return a pandas Series that contains the average of each feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d1f84571",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_neighbor(ID):\n",
    "    UniProt = ID[0]\n",
    "    pos = ID[1]\n",
    "    length = len(UniProt2Seq.loc[UniProt])\n",
    "    if pos in [1, 2]:\n",
    "        temp = UniProt2Seq.loc[UniProt][:5]\n",
    "    elif pos in [length, length-1]:\n",
    "        temp = UniProt2Seq.loc[UniProt][length-4:]\n",
    "    else:\n",
    "        temp = UniProt2Seq.loc[UniProt][pos-3:pos+2]\n",
    "    return list(temp.AA)\n",
    "\n",
    "\n",
    "def get_ave(result):\n",
    "    return Expasy_AA_Scales.loc[result].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab891bd5",
   "metadata": {},
   "source": [
    "I will call `get_ave(result)` on the returned list after calling `get_neighbor(ID)` for each index in the training set, and append the returned Series one by one to an empty DataFrame called `df_avg`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f158943d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_avg = pd.DataFrame()\n",
    "for i in df_train_new.index:\n",
    "    df_avg = df_avg.append(get_ave(get_neighbor(i)), ignore_index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb90adf8",
   "metadata": {},
   "source": [
    "The correlation coefficient measures the strength of the relationship between two features. If the absolute value of the correlation coefficient is very high, that means the relationship between these two features is very high, **i.e., they are redundant features**. So, I will not include features whose absolute values of correlation coefficients are bigger than 0.8, "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b31461e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selected Features: \n",
      "\n",
      "['%_accessible_residues', 'Average_flexibility', 'Bulkiness', 'Hphob__Welling_&_al', 'Number_of_codons', 'Ratio_hetero_endside', 'Recognition_factors', 'Relative_mutability']\n"
     ]
    }
   ],
   "source": [
    "corr_matrix = df_avg.corr()\n",
    "for i in range(len(corr_matrix)):\n",
    "    for x in range(len(corr_matrix.columns)):\n",
    "        if corr_matrix.iloc[i, x] != float(1) and abs(corr_matrix.iloc[i, x]) > 0.8:\n",
    "            corr_matrix.iloc[i, x] = np.nan\n",
    "corr_matrix.dropna(inplace=True)\n",
    "features = list(corr_matrix.index)\n",
    "print('Selected Features: \\n\\n' + str(features))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a72d69f9",
   "metadata": {},
   "source": [
    "## Model selection (10pts)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1776a2e3",
   "metadata": {},
   "source": [
    "Because `Label` is categorical variable, I will use three classifiers as models: Logistic Regression, Decission Tree Regressor, Random Forest. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "05c43cda",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4ca4d7b",
   "metadata": {},
   "source": [
    "- Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a77e6811",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression accuracy: 0.594\n"
     ]
    }
   ],
   "source": [
    "log_model = LogisticRegression().fit(df_avg[features], df_train_new['Label'])\n",
    "log_model_accuracy = log_model.score(df_avg[features], df_train_new['Label'])\n",
    "print('Logistic Regression accuracy: {:.3f}'.format(log_model_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4642cc6a",
   "metadata": {},
   "source": [
    "- Decission Tree Regressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "348a3bea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decission Tree Regressor accuracy: 0.367\n"
     ]
    }
   ],
   "source": [
    "tree_reg = DecisionTreeRegressor().fit(df_avg[features], df_train_new['Label'])\n",
    "tree_reg_accuracy = tree_reg.score(df_avg[features], df_train_new['Label'])\n",
    "print('Decission Tree Regressor accuracy: {:.3f}'.format(tree_reg_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6044ea57",
   "metadata": {},
   "source": [
    "- Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c9941900",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest accuracy: 0.354\n"
     ]
    }
   ],
   "source": [
    "forest_reg = RandomForestRegressor().fit(df_avg[features], df_train_new['Label'])\n",
    "forest_reg_accuracy = forest_reg.score(df_avg[features], df_train_new['Label'])\n",
    "print('Random Forest accuracy: {:.3f}'.format(forest_reg_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0beaa7b",
   "metadata": {},
   "source": [
    "Among the accuracy scores of the three models, I observed the Logistic Regression model has the highest accuracy score. Then, I will use cross-validation to compare the cross-validation scores of these three models."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89de5bed",
   "metadata": {},
   "source": [
    "## Cross-validation (5pts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1e1da5ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_validation(reg):\n",
    "    from sklearn.model_selection import cross_val_score\n",
    "    scores = cross_val_score(reg, df_avg[features], df_train_new['Label'], scoring = \"neg_mean_squared_error\", cv = 10)\n",
    "    rmse_scores = np.sqrt(-scores)\n",
    "    return rmse_scores\n",
    "\n",
    "\n",
    "def display_scores(scores):\n",
    "    print(\"Scores:\", scores)\n",
    "    print(\"Mean:\", scores.mean())\n",
    "    print(\"Standard deviation:\", scores.std())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fa9083a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression: \n",
      "\n",
      "Scores: [0.6281714  0.63671084 0.63492983 0.63732619 0.6400539  0.64951067\n",
      " 0.63506701 0.63835044 0.62875321 0.62757441]\n",
      "Mean: 0.6356447896397599\n",
      "Standard deviation: 0.0062634533959885315\n",
      "\n",
      "\n",
      "Decission Tree Regressor: \n",
      "\n",
      "Scores: [0.53651783 0.54900119 0.53781788 0.54060151 0.54109064 0.55310241\n",
      " 0.54373735 0.53764469 0.53749411 0.54154192]\n",
      "Mean: 0.5418549528161053\n",
      "Standard deviation: 0.005146814388476923\n",
      "\n",
      "\n",
      "Random Forest: \n",
      "\n",
      "Scores: [0.50642343 0.51371954 0.51064647 0.51148453 0.51309082 0.52505745\n",
      " 0.51344774 0.50939456 0.50706136 0.51094705]\n",
      "Mean: 0.5121272957233012\n",
      "Standard deviation: 0.004919966700742121\n"
     ]
    }
   ],
   "source": [
    "print('Logistic Regression: \\n')\n",
    "display_scores(cross_validation(log_model))\n",
    "print('\\n')\n",
    "print('Decission Tree Regressor: \\n')\n",
    "display_scores(cross_validation(tree_reg))\n",
    "print('\\n')\n",
    "print('Random Forest: \\n')\n",
    "display_scores(cross_validation(forest_reg))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a51ef2c3",
   "metadata": {},
   "source": [
    "My cross-validation verifies that the Logistic Regression model has the highest scores and performance. So, I will keep this model as my final model."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f903137",
   "metadata": {},
   "source": [
    "## Hyperparameter tuning (10pts)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d34aadc5",
   "metadata": {},
   "source": [
    "I will use Random Search, where random combinations of the hyperparameters are used to find the best solution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cc5d87e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, estimator=LogisticRegression(),\n",
       "                   param_distributions={'max_iter': <scipy.stats._distn_infrastructure.rv_frozen object at 0x7f9cb9735040>},\n",
       "                   scoring='neg_mean_squared_error')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import randint\n",
    "\n",
    "param_distribs = {'max_iter': randint(low=1, high=100),}\n",
    "rnd_search = RandomizedSearchCV(log_model, param_distributions=param_distribs, n_iter=10, cv=5, \\\n",
    "                                scoring='neg_mean_squared_error')\n",
    "rnd_search.fit(df_avg[features], df_train_new['Label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9dcc593b",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_iter = rnd_search.best_params_['max_iter']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a5e60014",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_model = LogisticRegression(max_iter=max_iter).fit(df_avg[features], df_train_new['Label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a735adb",
   "metadata": {},
   "source": [
    "## Test set 1 (10pts)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd216d06",
   "metadata": {},
   "source": [
    "- ROC curve with AUC value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "14ff76b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, roc_auc_score, precision_recall_curve, PrecisionRecallDisplay\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "869e3471",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.set_index(['UniProt', 'Position'], inplace=True)\n",
    "df_test.sort_index(inplace=True)\n",
    "df_test_avg = pd.DataFrame()\n",
    "for i in df_test.index:\n",
    "    df_test_avg = df_test_avg.append(get_ave(get_neighbor(i)), ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c06145c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC score: \t 0.6247531019464968\n"
     ]
    }
   ],
   "source": [
    "prob = log_model.predict_proba(df_test_avg[features])[:, 1]\n",
    "auc_score = roc_auc_score(df_test.Label, prob)\n",
    "print('AUC score: \\t', auc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "47361bda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAGHCAYAAAC9N1U6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABMHklEQVR4nO3dd3gU5drH8e+dQCih995D7x1EARtNadI7oohirxz1eI561ONRj10Rkd57BxVUEJBeQ++9lwDpyT7vH7O8J8aUDdndye7en+vaK9mZ2Zl7Uva388zM84gxBqWUUoEryO4ClFJK2UuDQCmlApwGgVJKBTgNAqWUCnAaBEopFeA0CJRSKsBpECilVIDTIFAAiEhBEYkRESMiA9JYzojIb2nM/01EUrw5RUSqisg3IrJfRCJFJFpEDorIGBFp4obdUEmISA4ReUdEjolIrIgcEZE3RSR7BteTW0TeEpE9zt/ZVRH5Q0S6JVuus4iMT/L7PSsiK0WkfSrrPe78e0rpUSQz+64yJpvdBagsoz8QAhwDhgFT3LlyERkGfAvEANOBHUACUBV4BHhcRGoZY/a6c7sBbibQBRgH/AG0AN4FqgBDXFmBiBQEVgFhwHjgv0AoUAMon2zxMcANYCFwACgEDAWWi8ibxpj3UtjEfiCl6TddqU+5iTFGH/oA2I71D/8s4AAqp7KcAX5LYz2/WX9Wf5p2P5AI7AZKpfCabMALQE27fw4Z+HkJkMfuOtKor6Pzd/VJsumfOKe3dHE9k7He3NP93QD3pjAtN1YoxAEFk807ntbfkj6899CmIYWINATqAxOBqUA81ic5d/kQ642ztzHmbPKZxpgEY8ynJp2jAREpJCKfOps4YkTkiohsFZFXUlj2ERH5VUSui0iUiBwQkS9EJCTJMqEi8oFzfbEicl5EJolI+WTrauNsrhgiIiNFZC/Wkc3LSZbpLSJrReSmc3sbRaRHxn9UbtPP+fWzZNNvP0+1+e82EangXM/3xpi9IhIsInlSW94Y80sK06KAJUB2oFoq28kmIvnSq0d5jgaBAqspKBKYa4y5AiwFBotIpv8+RKQi0BBYm94bvQtmA08Dy4FngHeATUCbZNt8D5gDFAM+BZ4HFmB9Ss7tXCYb8CMwCtiGdUQyHegFbBSRMils/3nn8jOc29/oXNe/nNNuAn93LhMFzBaRka7smIgUycDDlTb+JsAZY8yppBOdz88656enPdZ7xF4Rmezcp5siclpEXnBlv5xu/ywvpjCvmXO9Ec7QnigipTKwbuUOdh+S6MPeB5ATuApMSDKtC1bzQYcUls9Q0xDwsPM1X2SyzvzO9XyTznJNncv9AuRMNk8AcX7/uHO5/yRbppNz+uQk09o4p10FiiVbvqFz3vsp1LIAq1klrwv7ZzLwaOPC+m4CG1OZtwk468I6PnVu7yKwF+sosT/wu3P62y6sox7WEeaaFOYtBd4EegB9gNFY541Ok0IToj4899CTxao7UBCrWei2pVj//I9iffrOjNuH/DcyuZ5oIBZoJiIVjDHHU1muv/Pr34wxMUlnGOe7j1M3rHMhHyRbZqmI7AC6iEiQMcaRZPYkY0zyT7X9sd4UJ6ZwpcsirFBtAfyU1s4BD6QzP6mdLiyTG+vnlZIY5/z05HV+DQHuNtbRIiIyCysYXhWRz4wx11J6sYgUBeZh/e4eSz7fGNMp2aQZIrIGq3nybaywVl6gQaCGAZeA0yJSJcn0n4GeIlLEGHM5E+u/HQB501wqHcaYOBF5HvgcOOZsp/8FWGCMWZVk0TCsN+b03iwrYn0qTulNbA/WOZMi/Lk542AKy9bAOtLYn8a2iqdTC8aYlektk0FRQI5U5uV0zk9PtPPrktshAGCMiReRacBbQHNS+LAgIoWw/oZKAZ2MMSn97P7CGDPN2bSXPCSUB2kQBDBn+31brDey1P5RB/DnE47pfZoM5X9vIADhzq8N7qzK/zHGjBaRhVhvEq2xmhSeFpGZxpg+zsUEKwjSI3dQQkpvnre31wHryqiU7Em3GJESGajjqjEmLp1lzgKlU5lXGjjjwnZOO7+eT2HeOefXgslnOENgJVAd6GpSOImcjuPAXRl8jcoEDYLANhTrjexx4HoK8/+FdcTwWZJpx4AwEQk2xvzpjc95AjYMOHp7mjHmmIhsB+4SkerGmLQ+OafLGHMOGAuMFZFgrMsb+4rIJ8aYzViXKrYH6mK1hafmCNBeRAoYY64nm1cT60jGlSOhQ87tnTTG7MvQzvzZufQX+X9tsc7FpGUz0F9EypokJ4xFpCzWp/RFLmzn9s8vpRPnKZ4Adt538DNQCysEVriwneSqABfu4HXqTtl9kkIf9jywrgY5CexKY5l/YH3abZJk2vvOacNTWP4J57z3kk1/AOvT8g6gRAqvC8a6IifVa9WxjkJypzB9lHOb7ZzPb58sXgnkSGH55CeL/51sfgdSP1k8JIX1NXHOmw8EpzC/WGr7lGy5+zPwKOjC+m6f9E7tPoJWyaaXw/oEnz3Z7+U41hVlpZNMD3X+7VwDQpNMLwhsxTo38VA69RVKZfpIXLgoQB/ufdz+p1ABxnnb/3Lgn8aYt1NZpjbWTWDfGWNGOKflx7pLtQbWnat/OBdvAfQG9gHNjTE3kq3r9p3F0fz5zuIqWHcWVwZqm1QuMRWR+sBqrDfccKw3oRrAk1ifHmsb65p1ROTfwGtYTTIzsZo2KmI1JTU1xlx3XoL5K1YTxExgjbOWp7COjpoY5ydpEWnjXHaoMWZCCrW9hXVyMxzrEtezQEmgEdDRGBOS/DXeICKLgYeAH/jfncXDgCnGmIHJlv0Nq7mtoklyIl5EOgCLsfbpG6wbw4YCtYFhxphxSZbdgrXP04FlKZS03hhz1Lns885aVmCFTTaswO2KdbTWwhhz6Y53XmWM3UmkD3seWG9YBqiTznIHsN4YcyWZlh/rapt9WG/s0c7vPwDyp7GualhhcBCrvT3Guf7vgAbp1FEY63LGHc56ooHDWM1WJVNYvi+wDusyykisk7mfASFJlgl11nwU6w3uIlZTU/lk62pDKkcESZbphHVfwlWsT8SnsIL2SRt/xzmxmveOO2s6inWfQ/YUlv3NuY8VUpjXGuvE/E3n720d8HAKy6V32euQJMvehdU8ddL5u4xx/g39Gyhg9/9HoD30iEAppQKc3lmslFIBToNAKaUCnAaBUkoFOA0CpZQKcBoESikV4HzuzuIiRYqYChUq2F2GUkr5lK1bt142xhRNaZ7PBUGFChXYsmWL3WUopZRPEZETqc3TpiGllApwGgRKKRXgNAiUUirAaRAopVSA0yBQSqkAp0GglFIBToNAKaUCnAaBUkoFOA0CpZQKcB4LAhEZJyIXRSQ8lfkiIl+IyGER2SUiDT1Vi1JKqdR58ohgAtA+jfkdgDDnYzjWEIZKKaW8zGN9DRlj1ohIhTQW6QJMMtZYmRtEpICIlDTGnPNEPWvWrCEuLo7cuXN7YvVKKeUx8YmGuJuXCS1UnJbNm7t9/XZ2Olcaa4Dv2047p/0lCERkONZRA+XKlbujjcXGxpKYmHhHr1VKKW+KTXBwPSqOGzEJXI2MozARVJazxF4/Cn4WBJLCNJPSgsaYMcAYgMaNG6e4THpCQ0MBaNmy5Z28XCmlPObSzVjWH7nM9pPX2XD0CvvP3wSgRL6cPFlgFwMufsSNYo2J6jHdI9u3MwhOA2WTPC8DnLWpFqWU8qp9526wcMdZftp7nmOXIzEGcmQLomG5grzSrhrtapWg8qm5yOL/QKXWFOgznQIhnmnatjMIFgFPi8gMoBkQ4anzA0oplRWcvBLFnK2n+HHPBQ5cuElwkNCiUmG61S9N62pFqV4iHyHZnNfwbPoelr0MVR6A3lMge06P1eWxIBCR6UAboIiInAb+AWQHMMaMBpYBHYHDQBQw1FO1KKWUXa5HxbHh6BWWh59n4Q6r0aNmyXy826UWHeuUpHCeHH990R9fw4+vQ7WO0HMCZEthGTfy5FVDfdOZb4CRntq+UkrZJSougZ/3XmDB9jP8euASAKEhwfRtWpbH765EpaJ5Un/x2k9h5T+hRmd45AfIFuLxen1uqEqllMqK4hIcrD54iUU7z7Jy7wWi4xMpkS8nj95VkbuqFObusKL/a/ZJzer/wK/vQe0e0O07CPbOW7QGgVJKZUJEdDw/rD3G7C2nOBcRQ8Hc2enesDSd65WiSYVCBAWldIFkMsbAL/+C3z+Gen2hy9cQFOz54p00CJRS6g6cvR7NZysPsnDHWeISHTStUIjXO9agfe0SZA/OQKcNxsDPf4f1X0LDQfDQ5xDk3W7gNAiUUioDLt2MZezao4xfdxyAXo3L0LdpOWqVyp/xlRkDK0bBxtHQ5DHo8JHXQwA0CJRSyiUXb8TwzW9HmLn5FNHxiXRvUJoXH6xKmYJ3eG2/wwHLXoIt46D5SGj3HogLzUgeoEGglFJpuBWbwHtL9zJ36xkcxtC5XimealuFKsXSuPInPY5EWPwsbJ8CrV6A+/5hWwiABoFSSqXopz3nGb36CHvO3iA2wUGX+qV48YGqlC8cmrkVJybAwqdg10xo/Rq0+ZutIQAaBEop9f9i4hOZsuEEUzac4PiVKHJmD6JUgVy826U2rcKKZH4DifEwbzjsmQf3vgn3vJL5dbqBBoFSKuBdjYxj+qaTfLf6CDdiEiiZPyfvdKlF36blMnYFUFoS4mDOUNi/BB54F+561j3rdQMNAqVUwDp7PZoJ648z+Y8TRMcn0rJyYUa2rULLyoURdzbXxMfA7MFwcAW0/xCaj3Dfut1Ag0ApFXAOnL/J6NVHWLrrHAkOB/fVKM7LD1ajWom87t9YfDTM6AdHfoGHPoXGj7p/G5mkQaCUCggx8YmsCD/P1I0n2Hz8GjmyBdGudgmeuy8sc1cApSUuEqb1huNrofNX0HCgZ7aTSRoESim/duZ6NJPWH2fqxpPcik2gTMFcPH9/GINbVKBgqAc7dIu9CVN7wakNVr9B9Xp7bluZpEGglPJLRy7d4oNl+1i57yIAHWqXoHeTstwTVtS1/n8yIyYCpvSAM1vhkbFQ+xHPbi+TNAiUUn4lIiqeb1Yf5offj5ErJJjH765I94ZlqFEyn3cKiLoKU7rD+XDoNRFqPOyd7WaCBoFSyi9ERMWzcOcZPlt5iKuRcXSpX4o3OtWgWF7Pjez1F5FXYHIXuHTAGlWsWnvvbTsTNAiUUj4tIiqeb347zPj1x4lLcNCgXAHGDm5Mw3IFvVvIrYswsTNcOwZ9p0OV+727/UzQIFBK+aTI2ATGrzvGd2uOcis2ga71SzP0rgrUKZ3fvfcAuOLGOZjUGSJOQ79ZUKm1d7efSRoESimfcvFGDFM2nGDqxpNciYzj/hrFeenBqt47B5BcxGmY+LB1RDBgLpRvaU8dmaBBoJTyCcYYlu0+z2tzd3ErNoE21Yry3H1hNPB2E1BS105YIRB9DQbOh7JN7aslEzQIlFJZ3uVbsby1MJxlu89TqWgo03o3o26ZAvYWdeWIdU4g7iYMWgilG9pbTyZoECilsqyERAc/7rnA3xeGcysmgRcfqMpTbSqTzV0dwd2py4esI4GEWBi8BErWtbeeTNIgUEplObEJiczYdIr/rNhPZFwi1YrnZeLQptQpcwfDQbrbxX3WkQAGhiyF4jXtrijTNAiUUllGZGwCo1cfYdzaY0TGJVIoNIR/dq5Ftwal7T8KADi/GyZ1gaDsMHgxFK1qd0VuoUGglMoSft1/kVHzdnHhRixNKxaiX9NyPFyvFMGe7g7CVWe3w6SuEBJqhUDhynZX5DYaBEopW528EsU7S/ayct8FKhTOzXcDG/FgzeLevxcgLac2w5RHIGd+GLIYClawuyK30iBQStnixJVIPv35IAt2nCU4SBjZtjLP3hdGjmzBdpf2Zyf+gKk9IbSIdSRQoKzdFbmdBoFSyqui4xL5+tfDfPXrYUKyBTGkZQUevasi5Qrntru0vzr2uzWeQL5SMHiR9dUPaRAopbzCGMNPey/wzuK9nLkezUN1S/Ja++qULZQFAwCsEcWm94OC5WHQIshb3O6KPEaDQCnlcWeuR/PanF2sPXyZsGJ5mPpYM+6qUsTuslJ38CeYOQCKhFk3i4Vm4VrdQINAKeUxDodh/vYzfLB8P1cjY3m9Y3WGtKxISLYscCloavYvhVmDrfsDBi6A3IXsrsjjNAiUUh5xLiKaV+fs4vdDl6lRMh9THmtK9RI2dQznqj3zYe5jULK+1YFcrgJ2V+QVGgRKKbfbcPQKT03dRkx8Im93rsXA5uU9PzxkZu2aDfOHQ9lmVlfSObN4aLmRBoFSyq3mbTvNqLm7KVsoF98NbE6VYnntLil926fCwpFQoRX0nQE58thdkVdpECil3MLhMLy3bB8/rD1G04qFGDOwEQVyh9hdVvq2jIclz0OlttBnGoRk0auYPEiDQCmVabdiE3hp1g5+3HOBRxqW4YPudbL2CeHbNo6B5a9A2IPQazJk9+L4xlmIBoFS6o45HIZ528/w6c8HOX8jhlEdqvPEPZWyVvcQqVn/Ffz0BlTrBD3HQ7YcdldkGw0CpdQdOXLpFi/O3MHO0xHUK5Ofj3vWo0XlwnaX5ZrfP4FV70DNrvDIWAjObndFttIgUEpl2O7TEQz4YSOxCYn8t1c9ujUo7RtHAcbA6g/htw+gTk/oOhqC9W1QfwJKKZcZY5i5+RRvLggnV0gwC0e2oloJH7gqCKwQ+OVd62igfn/o/CUEZbEO7myiQaCUconDYXh17i7mbD1NqypF+KhnXUrmz2V3Wa4xBn56E/74ChoNgU6fQpAPnMz2Eg0CpVS6LtyI4c0F4fy89wIDmpfj7c61s86AMekxBpa/Bpu+g6bDocN/wBeasbxIg0Aplab1hy/Tb+xGgoOE19pXZ0RrH7kqCMDhgKUvwNYJ0OJpePBfGgIp0CBQSqXI4TCMXnOEj388QOkCufiqXwMalCtod1mucyTComdgx1Ro9SLc95aGQCo0CJRSf3H6WhQvztzJpuNXubd6MT7oXofi+XzoZqvEBFgwAnbPhjZ/g9avaQikQYNAKfUn+87dYMDYjcQmOPhPj7r0bFTGd5qCABLjrR5E9y6wjgLufsnuirI8DQKlFGBdGrpq30X+vjAcEWHOky2yfrfRySXEwpxHYf8SePA9aPm03RX5BA0CpRTRcYm8NHsHy3afp3SBXIwf0tj3QiA+BmYNhEM/QYePoNlwuyvyGRoESgW4q5FxDJ2wmZ2nrvNKu2o8fncl3+gwLqm4KJjRD47+Cg99Bo2H2l2RT9EgUCqAhZ+J4Nnp2zl9LZrvBjaiXa0SdpeUcbG3YHofOL4WunwNDQbYXZHP8Wjsi0h7ETkgIodFZFQK8/OLyGIR2Skie0REY1wpL9l56jpDJ2zmbEQ044Y08c0QiLkBU3vAiXXQfYyGwB3yWBCISDDwNdABqAn0FZGayRYbCew1xtQD2gCfiIgPjGShlG/beuIqvcf8gQBTH2tGq7AidpeUcdHXYXI3OL0ZeoyDur3srshnebJpqClw2BhzFEBEZgBdgL1JljFAXrGuTcsDXAUSPFiTUgFv6sYTvL1oL6UL5mLC0CaULxxqd0kZF3XVCoELe6DnRKjxkN0V+TRPBkFp4FSS56eBZsmW+QpYBJwF8gK9jTEOD9akVMByOAwfrtjPd2uO0qpKET7rU58ieXxwMJbIyzCpK1w+CH2mQtV2dlfk8zwZBCndgWKSPW8H7ADuBSoDP4vI78aYG39akchwYDhAuXLl3F+pUn7u4o0Yun2znjPXo2leqRBjBzcmZ3Yf7IL55gWY1AWuHYO+06HKfXZX5Bc8ebL4NFA2yfMyWJ/8kxoKzDOWw8AxoHryFRljxhhjGhtjGhctWtRjBSvljw6cv0mXr9dx6WYs/+pam+mPN/fNELhxFiZ0gusnoP9sDQE38mQQbAbCRKSi8wRwH6xmoKROAvcBiEhxoBpw1IM1KRVQlu8+R7vP1pDoMMx7qiUDmpf3re4ibrt+CsZ3hJvnYcA8qHiP3RX5FY81DRljEkTkaeBHIBgYZ4zZIyIjnPNHA+8CE0RkN1ZT0mvGmMueqkmpQBGX4OCD5fsYv+44tUrl49v+jShXOLfdZd2Za8dh4sMQHQED50PZJnZX5Hc8ekOZMWYZsCzZtNFJvj8LPOjJGpQKNKevRTFy2nbrPoG7KvC3DjV8707h264csUIgLhIGL4RSDeyuyC/pncVK+QljDIt3nePvC8JxOAzf9m9Ihzol7S7rzl06aIWAIx6GLIESdeyuyG9pECjlB+ITHfx9QTgzNp+iVql8fN2vIRWK+OD9Abdd2AuTOgMCQ5ZCsRp2V+TXNAiU8nHXIuN4YdYOfjtwiZFtK/PiA9V8ZzzhlJzbZV0imi0HDF4MRcLsrsjvaRAo5cOuR8XR4fPfOX8jhve71aFfMx+/z+bMNuuO4ZA8MHgRFK5sd0UBQYNAKR91PiKGxyZt5mpkHBOGNqFNtWJ2l5Q5pzbDlO6QqwAMXgIFy9tdUcDQIFDKB207eY0h4zYRl+jgy34NfD8ETqyHqT0hTzGrOSh/GbsrCigaBEr5mCW7zvLirJ3kzZGNeU+1pEqxvHaXlDlHV1vjCeQvA4MWQT4fvtLJR2kQKOUj4hIcvDBrB0t3naNmyXx8N7ARZQv56E1itx1eCTP6Q8GK1jmBPD5+ZOOjNAiU8gFHLt3itTm72HLiGu1rleDT3vXJFeKD/QUldWCFNcZwkWowaAGE+uCYCH5Cg0CpLO6PI1d4fNIW4hIcfN6nPl3ql7a7pMzbtxhmD4USta2+g3IXsruigKZBoFQWtmTXWV6cuZPyhXMz8dGmlCqQy+6SMi98Hsx9DEo3hAFzIWd+uysKeBoESmVBiQ7DWwvDmbrxJHXL5GfSo00pkNsPRnHdORMWjICyzayupHP4+IluP6FBoFQWE5/oYOAPG9lw9CpD76rACw9UJV/O7HaXlXnbJsOiZ6BCK+g3E0J8uAsMP6NBoFQWYozh5dk72XD0Km90rMHj91SyuyT32DIOlrwAle+F3lMhxMevdvIzPto3rVL+Jzoukedm7GDhjrMMaVnBf0Jg43dWCIS1gz7TNQSyID0iUCoL2HM2gpdm7eTAhZs8d18Yz9/vJx2trfsCfv47VH8IeoyHbH5wnsMPaRAoZbPlu8/xwqwd5M2ZnfFD/KDPoNvWfAS//AtqdYPu30OwH5zn8FMaBErZxBjDD2uP8a+l+2hQrgBjBjamaN4cdpeVecbAbx/A6g+hbm/o8g0E61tNVqa/HaVskOgwvDJnJ/O2neH+GsX5ql8Dcmb38TuFwQqBVW/D2k+h/gDo/AUE+cF++TkNAqW8LDI2gVfn7mLprnOMbFuZF+6vSrZgP7huwxj48Q3Y8DU0fhQ6fgJBfrBfAUCDQCkvunwrlscmbmHHqeuM6lCdEa39ZOAVhwOWvwqbv4dmI6D9v0F8eJS0AKNBoJSXRETHM2DsRo5fieSTnvV4pJGf9LnvcMCS52DbJGj5DDzwroaAj9EgUMoLzkfEMHjcJo5evsXYwU1oXbWo3SW5hyMRFj4NO6fB3S/DvW9qCPggDQKlPOzElUj6j93IpZuxTBjalLuq+El3y4kJVr9Bu2dD2zeg9at2V6TukAaBUh60+fhVHp+0hajYRKY+3owmFfyku+XEeJg7DPYuhPv/Ca1esLsilQkaBEp5yPrDl+n/w0bKFcrNnBEtqVIsj90luUdCLMweAgeWQbv3ocVIuytSmaRBoJQHbHEeCYQVy8OUx5pRLG9Ou0tyj/gYmDkADv8MHT+Gpo/bXZFyAw0Cpdxs4Y4zvDZ3F8Xz5WTSo34UAnFRMKOvNdj8w59DoyF2V6TcRINAKTcxxvDFqsN8uvIgdUrnZ9yQJv7RZQRA7C2Y3gdOrIOu30D9fnZXpNxIg0ApN7h4I4YXZu1g/ZErdKhdgs/61CdHNj/pWiHmBkztCac3W53H1elhd0XKzTQIlMqkgxduMmDsRq5ExvFO51r0b1aeoCA/uZY++hpMeQTO7YQe46BWV7srUh6gQaBUJmw/eY2hEzaTPTiIBU/dRZ0yfjQQe9RVmNwVLuyFXpOgeie7K1IeokGg1B1auuscL83eQbG8OZkyrBnlCvvRyFu3LlkhcPkQ9J0OYQ/YXZHyIA0CpTIoJj6Rl2bvZOmuc9QomY+JQ5tQLJ+fXBkEcPM8TOoC105Yg8xXbmt3RcrDNAiUyoDrUXEMGreJXacj6Nu0LG93rk1INj/qavnGWZj4MNw4BwPmQIVWdlekvECDQCkXHbxwk+GTtnA2IoZPe9ejWwM/6T30tusnrRCIvAID50G55nZXpLxEg0ApF5yLiKbf9xsAmPqYH/UZdNvVYzCxM8REwKAFUKax3RUpL9IgUCodxhhem7uby7fiWPx0K/+6MgjgyhHrSCA+CgYvglL17a5IeZkGgVJpSHQYBo3byLrDV3ilXTX/C4FLB6wQcCTC4CVQorbdFSkbaBAolQpjDK/O2cW6w1d4onUlnvSXYSVvu7DHag6SIBiyFIpVt7siZRMNAqVSMXr1UeZuO83jd1dkVPvqiD+NvHVuJ0zqCtlywODFUCTM7oqUjVwOAhEJNcZEerIYpbKCRIfh81WH+GLVIR6qW5JRHWr4Vwic2QqTu0GOfNY5gUKV7K5I2SzdC6BFpKWI7AX2OZ/XE5FvPF6ZUjaIiU/kmenb+GLVIdpWK8p/e9Un2F/6DQI4udE6EshZAIYu0xBQgGtHBJ8C7YBFAMaYnSJyj0erUsoGVyPjGDxuE7vPRPBmpxoMa1XRv44Ejq+zehHNW8JqDspf2u6KVBbhUtOQMeZUsn+IRM+Uo5Q9rkbG0XP0ek5ejeKjHnXp2bis3SW519HfYFofKFDWCoG8JeyuSGUhrgTBKRFpCRgRCQGexdlMpJQ/OHElkqembuP0tWgmD2tG80qF7S7JvQ6thJn9rWagQQshTzG7K1JZjCtBMAL4HCgNnAZ+Ap7yZFFKecvxy5H0GL2eW7EJjB7QyP9C4MBymDUIilaDgQsh1M/2T7mFK0FQzRjTP+kEEbkLWOeZkpTyjk3HrvLE5C2ICEueuZsqxfLYXZJ77V0Ec4ZCibpW30G5CtpdkcqiXOk28UsXpynlMxbtPMugcRspmDuEWU80978Q2D0HZg+BUg2tvoM0BFQaUj0iEJEWQEugqIi8mGRWPsBPBmNVgeiHtcd4d8leapXKxw+Dm1Aivx+NJQCwcwYseBLKtbDGE8iR1+6KVBaXVtNQCJDHuUzSv6QbgI5erXzS6NVH+Pfy/dxXvRijBzYie7AfjSUAsG0yLHoGKt4NfWdASKjdFSkfkGoQGGNWA6tFZIIx5sSdrFxE2mOdaA4Gxhpj/p3CMm2Az4DswGVjTOs72ZZS6Rm39hj/Xr6fTnVL8nnv+mTztxDYPBaWvgRV7ofeUyB7LrsrUj7ClZPFUSLyEVAL+P9jaGPMvWm9SESCga+BB7CuNtosIouMMXuTLFMA+AZob4w5KSJ6XZvyiBXh53lnyV5aVSnCZ/4YAhu+hRWjoGoH6DXR6kNIKRe58t8wFdgPVATeBo4Dm114XVPgsDHmqDEmDpgBdEm2TD9gnjHmJIAx5qKLdSvlsu0nr/H8zO3ULJnPP5uD1n5mhUCNh6HXJA0BlWGu/EcUNsb8AMQbY1YbYx4FXBnDrjRwKsnz085pSVUFCorIbyKyVUQGpbQiERkuIltEZMulS5dc2LRSlrWHLjNg7EaK5MnBpGFNyZPDzzrcXf0RrPwH1H4EeoyHbCF2V6R8kCv/FfHOr+dEpBNwFnBlsNaUOmkxKWy/EXAfkAv4Q0Q2GGMO/ulFxowBxgA0btw4+TqUStGC7Wd4cdYOKhQJ5et+DSmSx48+KRsDv74Pa/4DdftA128gSC/mU3fGlSD4l4jkB17Cun8gH/C8C687DSTtsKUMVogkX+ays3vrSBFZA9QDDqJUJqw/fJnX5++mcflCjBvaxL+OBIyxjgLWfQ4NBsLDn2sIqExJt2nIGLPEGBNhjAk3xrQ1xjQCrrqw7s1AmIhUdPZR1AdnD6ZJLATuFpFsIpIbaIb2Y6QyadW+CwybuIVCoSGMHtjI/0Jgxd+sEGg8DB7+QkNAZVpaN5QFA72w2vVXGGPCReQh4HWsZpwGaa3YGJMgIk8DP2JdPjrOGLNHREY45482xuwTkRXALsCBdYlpuDt2TAWmwxdv8vyMHeQKCWbqY80oFOpHbeYOByx7Gbb8AM2ehPYfgD91k61sk9ZHpR+wmnY2AV+IyAmgBTDKGLPAlZUbY5YBy5JNG53s+UfARxmoWakUnb0eTZ8xG8mRPZj5T7WkbKHcdpfkPg4HLHkOtk2Cu56D+9/WEFBuk1YQNAbqGmMcIpITuAxUMcac905pSrnu2OVIhk3YzPWoOGaNaOFnIZAIC0fCzulwz6vQ9nUNAeVWaQVBnDHGAWCMiRGRgxoCKis6eOEmfcZsQIDJw5rRsJwfdbCWGA/zn4DwudD2TWj9it0VKT+UVhBUF5Fdzu8FqOx8LoAxxtT1eHVKpePklSge+XY9QSLMGdGCsOJ+1MFaQhzMfRT2Lbaaglo9b3dFyk+lFQQ1vFaFUncgNiGR52du52ZMAsuevdvPQiAWZg2Gg8uh3QfQQseCUp6TVqdzd9TRnFLeYIzh7wvC2XbyOp/3qU/NUvnsLsl94qNh5gA4vBI6fQJNHrO7IuXn/OgCaxVIvlh1mFlbTvPMvVXoUj95zyU+LC4SpveFY2ug85fQMMVeV5RyKw0C5XPWHrrM56sO0qV+KV58oKrd5bhP7E2Y1htO/gHdRkO9PnZXpAKES90wikguEanm6WKUSs/RS7d4YvIWqhbPyzudayP+chllTARM7g4nN0D37zUElFelGwQi8jCwA1jhfF5fRJJ3FaGUx8UlOHh2xnayBQfxw5Am5M+d3e6S3CP6GkzqCme3Qc/xUEcHAFTe5coRwT+xxha4DmCM2QFU8FRBSqUkIjqeJ6dsJfzMDd7vVofSBfxk9K3IKzCxM1wIt0YVq5l8yA6lPM+VcwQJxpgIvzkEVz4nJj6RxyduYdPxq/ytQ3U61S1pd0nucesSTOoCV49An+kQdr/dFakA5UoQhItIPyBYRMKAZ4H1ni1LKUuiw/Ds9O1sPnGVL/s24OF6pewuyT1unreOBK6fhH4zoVIbuytSAcyVpqFnsMYrjgWmARG4Nh6BUpmSkOjguRnb+WnvBd7sVNN/QiDiDIzvCBGnYcAcDQFlO1eOCKoZY94A3vB0MUrdFhOfyJDxm9hw9Cp9m5ZjWKuKdpfkHtdPwsSHIeoqDJwP5ZrZXZFSLgXBf0WkJDAbmGGM2ePhmpTi/WX72HD0Ku90qcXA5uXtLsc9rh61moNib8CgBVC6kd0VKQW4NkJZW6ANcAkYIyK7ReRNTxemApPDYRiz5giT/jjBwOblGdSign/cK3D5EIzvBHG3YPBiDQGVpbh0Q5kx5rwx5gtgBNY9BW95sigVmIwxfLhiP+8v2889VYvyRic/6ffw4n7rnEBiHAxZCiXr2V2RUn+SbtOQiNQAegM9gCvADKyB7JVym0SH4d0le5mw/jhd6pfi0171CQrygyOB8+HWJaJBwTB4KRSrbndFSv2FK+cIxgPTgQeNMWc9XI8KUF//epgJ648zrFVF3uhYwz9C4OwOmNwVsuWymoOKVLG7IqVSlG4QGGOae6MQFbimbDjBf38+yEN1S/Jmpxr+cU7g9FaY0g1y5IfBi6CQn1z1pPxSqkEgIrOMMb1EZDdgks5CRyhTbrLt5DXeWhhOzZL5+KRXPf8IgZMbYEoPCC1sHQkUKGd3RUqlKa0jguecXx/yRiEq8By5dIthEzZTMn8uJg1rSo5swXaXlHnH18LUXpCvJAxaBPn9aKwE5bdSvWrIGHPO+e1TxpgTSR+AjpunMuXklSju+2Q1wUHCtMebUSRPDrtLyrwjv1pHAgXKWlcHaQgoH+HK5aMPpDCtg7sLUYEjLsHBk1O3AvBN/0aULxxqc0VucOhna1CZQpVg8BLIW8LuipRyWVrnCJ7E+uRfSUR2JZmVF1jn6cKUf3I4DG8tDGfP2Rt8N7ARTSsWsrukzNu/DGYPhmI1YOACyO0H+6QCSlrnCKYBy4EPgFFJpt80xlz1aFXKLxljeGLKVn7ee4EnWleiXS0/+NS8ZwHMHWbdJDZgHuQqYHdFSmVYWkFgjDHHRWRk8hkiUkjDQGXU978f5ee9F3j2vjBeuD/M7nIyb/ccmDccyjSG/nMgZz67K1LqjqR3RPAQsBXr8tGk1/UZoJIH61J+5pf9F/hwxQEerFmcF+4P8/3LRHdMg4UjoVxLazyBHHnsrkipO5ZqEBhjHnJ+1TthVKbsOHWdp6ZuI6xYHv7bu77vh8DWibD4OajU2hpZLCS33RUplSmuDF5/l4iEOr8fICL/FRG9Q0a5JPxMBIN+2EixvDmZ+GhT8uRwpVeTLGzT97D4WahyP/SdqSGg/IIrl49+C0SJSD3gVeAEMNmjVSm/cOpqFP2+30DenNmZ+lgziufLaXdJmfPH17DsZajWEfpMhew+vj9KObkSBAnGGAN0AT43xnyOdQmpUqm6FhnHsImbMQYmPtqEsoV8/JPz2k/hx9ehRmfoORGy+cENcEo5uXKcflNE/gYMBO4WkWAgu2fLUr4sMjaBIRM2c/xKFOOHNKFKMR//3LD6P/Dre1C7B3T7DoJ9vHlLqWRcOSLojTVw/aPGmPNAaeAjj1alfJbDYXhzQTg7T13nq74NuKtKEbtLunPGwKp3rRCo1w+6j9EQUH7JlaEqzwNTgfwi8hAQY4yZ5PHKlE96de4u5m8/w0sPVOVBX75hzBj4+e/w+8fQcBB0+doaXEYpP+TKVUO9gE1AT6AXsFFEeni6MOV7Zm85xZytpxnUojxP3+vDg7AYAytGwfovoclj8NDnEOTSqK5K+SRXjnPfAJoYYy4CiEhRYCUwx5OFKd8yd+tpXpmzixL5cvJyu2q+e6+AwwHLXoIt46D5SGj3HvjqvijlIleCIOh2CDhdwcVB71VgmLn5JKPm7aZemfxMGtaMfDl99FoCR6J1j8D2KdDqBbjvHxoCKiC4EgQrRORHrHGLwTp5vMxzJSlfMnvLKd6YH07tUvmZMbwFuUJ8tB09MQEWPgW7ZkLr16DN3zQEVMBwZcziV0SkO9AKq7+hMcaY+R6vTGV5G49e4bW5u2hWsTDf9G/owyEQD/Mehz3z4d434Z5X7K5IKa9KazyCMOBjoDKwG3jZGHPGW4WprC0iKp4np26jfOFQvh/c2He7jkiIgzlDYf8SeOBduOtZuytSyuvSausfBywBHsHqgfRLr1SkfMK/V+znamQcn/Sq57shEB8DswZaIdD+Qw0BFbDS+g/Oa4z53vn9ARHZ5o2CVNa39cRVZmw+ybBWFWlYrqDd5dyZ+GiY0Q+O/AIPfQqNH7W7IqVsk1YQ5BSRBvxvHIJcSZ8bYzQYAtCRS7cYOn4zxfLmYGRbH71XIC7SGl/4+Fro/BU0HGh3RUrZKq0gOAf8N8nz80meG+BeTxWlsq63FoYDMGdESwqFhthczR2IvQlTe8GpDVa/QfV6212RUrZLa2Catt4sRGVtxhi+XX2EdYev8Gr7ar7Zm2hMBEzpAWe2wiNjofYjdlekVJbgo2f5lLd9tvIQn686RKc6JXninsp2l5NxUVdhSnc4Hw69JkKNh+2uSKksQ4NApWvd4ct8/ethutQvxX971Sc4yMdutIq8ApO7wKUD0HsKVGtvd0VKZSkaBCpNN2PieWrqNgqFhvBO59q+FwK3LsLEznDtGPSdbg0xqZT6E1d6HxXnWMVvOZ+XE5Gmni9N2S02IZHhk7ZyKzaBD3vUJX9uH+tD6MY5mNAJrp+AfrM0BJRKhSudx30DtAD6Op/fBL52ZeUi0l5EDojIYREZlcZyTUQkUbu3zlremB/OH0ev8FGPurStVszucjIm4jRM6Ag3zsKAuVCptd0VKZVluRIEzYwxI4EYAGPMNSDd6wadQ1p+DXQAagJ9RaRmKst9CPyYgbqVh32/5ihztp6md+OydG9Yxu5yMubaCRjfESIvw8D5UL6l3RUplaW5EgTxzjdrA/8/HoHDhdc1BQ4bY44aY+KAGUCXFJZ7BpgLXExhnrLBtI0neW/ZPjrVKcn73evYXU7GXDlihUDMdRi0EMpqK6ZS6XElCL4A5gPFROQ9YC3wvguvKw2cSvL8tHPa/xOR0kA3YHRaKxKR4SKyRUS2XLp0yYVNqzu1aOdZXp+/mzql8/NJr3q+dXL48iHrnEB8FAxeAqUb2l2RUj7BlW6op4rIVuA+rO4luhpj9rmw7pTeQUyy558BrxljEtMa0coYMwYYA9C4cePk61BucvJKFG8tDCc4SPhhcGNyZvehbqUv7rOuDsLAkKVQ/C+tkEqpVKQbBCJSDogCFiedZow5mc5LTwNlkzwvA5xNtkxjYIYzBIoAHUUkwRizIP3SlTs5HIbhk7dwKyaBn164h2L5ctpdkuvO74ZJXSAoOwxeDEWr2l2RUj7FlfsIlmJ9khcgJ1AROADUSud1m4EwEakInAH6AP2SLmCMqXj7exGZACzRELDHhyv2s//8Tb7s24DKRfPYXY7rzm6HSV0hJNQKgcI+eNezUjZzpWnoT2cLRaQh8IQLr0sQkaexrgYKBsYZY/aIyAjn/DTPCyjviYiO57s1R6lRMh8P1S1pdzmuO7UZpjwCOfPDkMVQsILdFSnlkzJ8Z7ExZpuINHFx2WUkG984tQAwxgzJaC3KPd5evAeA97rVJq1zNVnKiT9gak8ILWIdCRQom/5rlFIpcuUcwYtJngYBDQG9dMdP7Dh1nXnbzlCvTH7fGWTm2O/WeAL5SsHgRdZXpdQdc+WIIG+S7xOwzhnM9Uw5ypuuR8Xx/IztFMubg0nDmtldjmuO/ALT+0HB8jBoEeQtbndFSvm8NIPAeSNZHmPMK16qR3nRy7N3cvZ6DNOHNyN/Lh/oR+jgTzBzABQJs24WCy1id0VK+YVUbygTkWzGmESspiDlZ1buvcDKfRd58cGqNCpfyO5y0rdviTXGcLHq1jkBDQGl3CatI4JNWCGwQ0QWAbOByNszjTHzPFyb8pCERAfvLt1LlWJ5GNaqYvovsNue+TD3MShZ3+pALlcBuytSyq+4co6gEHAFa4zi2/cTGECDwEe9v2w/J65E8f2gxmQPdqWXERvtmg3zh0PZZlZX0jnz2V2RUn4nrSAo5rxiKJz/BcBt2s2Dj9p+8hrj1h3joboleaBmFj/Run0qLBwJFVpB3xmQw4dudFPKh6QVBMFAHlzrM0j5gIREB39fGE7h0BA+yOq9im4ZD0ueh0ptoc80CMltd0VK+a20guCcMeYdr1WiPO6xSVsIP3OD0QMakTdnFr5KaOMYWP4KhD0IvSZDdh/q90gpH5RWA7GP3GKqXHHsciS/HbhEl/qlaF+7hN3lpG79V1YIVOtkDTSvIaCUx6V1RHCf16pQHhWbkMjT07aRI1sQLz9Yze5yUvf7J7DqHajZFR4ZC8FZ+KhFKT+SahAYY656sxDlOR8uP8Ceszf4vE99yhbKgm3txsDqD+G3D6BOL+j6LQRnuBsspdQd0v82P7f1hHWV0MDm5elSv3T6L/A2Y6yjgLX/hfr9ofOXEORDA+Io5Qc0CPzYlVuxvLkgnBL5cvJah+p2l/NXxsBPb8IfX0GjIdDpUwjK4vc1KOWHNAj8VGRsAoPHb2LfuRtMfLQpeXJksV+1MbD8Ndj0HTQdDh3+A77SBbZSfiaLvTsod5my4QThZ24wZmAjWlctanc5f+ZwwNIXYOsEaPE0PPgvDQGlbKRB4IeuR8Uxft1xmlQoyIO1stiloo5EWPQM7JgKrV6E+97SEFDKZhoEfiYqLoHB4zZxNTKOr/tnsY5jExNgwQjYPRva/A1av6YhoFQWoEHgZz5beYidpyP4tHc9GpXPQiOOJcZbPYjuXWAdBdz9kt0VKaWcNAj8yNFLtxi/7hg9G5WhW4MydpfzPwmxMOdR2L8EHnwPWj5td0VKqSQ0CPyEw2F4cdZOBOGVdlno7uH4GJg1EA79BB0+gmbD7a5IKZWMBoGfGPP7UXacus7rHatTLF8W6Z8nLsoaVezor/DQZ9B4qN0VKaVSoEHgB2LiExm/7hg1S+ZjWKtKdpdjib0F0/vA8bXQ5WtoMMDuipRSqdDbOP3AF6sOceFGLKM6VCc4KAtchRNzA6b2gBProPsYDQGlsjg9IvBxhy/eZPTqI3SqW5K7w7LAgO7R12HKI3BuB/QYB7W62V2RUiodGgQ+zBjDxz8eJFf2YN7pXAux+5r8qKswuRtc2AM9J0KNh+ytRynlEm0a8mGf/nyQFXvOM6J1ZQrnyWFvMZGXYeLDcHEf9JmqIaCUD9EjAh915no03605SpMKBRnZtoq9xdy8AJM6w7Xj0Hc6VNExjZTyJRoEPuqdxXsQgc/6NCDIzhPEN87CxM5w4wz0nw0V77GvFqXUHdGmIR+09cQ1ftxzgRGtK1O6QC77Crl+CsZ3hJvnYcA8DQGlfJQeEfiYCzdieHb6dkrmz8nQlhXtK+TaceucQHQEDJwPZZvYV4tSKlM0CHzMPxft4dKtWKY/3pz8uW0a3P3KESsE4iJh8EIo1cCeOpRSbqFNQz7k1/0XWR5uXSVkW8+ilw5azUEJMTBkiYaAUn5Ajwh8xIUbMbw4awcVi4TyVJvKNhWx17o6CIEhS6FYDXvqUEq5lQaBj3h/2T4iYxOZPaIRObMHe7+Ac7tgUhfIlgMGL4YiYd6vQSnlEdo05APCz0SwcMdZht9TiSrF8nq/gDPbrHMC2XNbRwIaAkr5FT0i8AHfrj5CzuxBPH63DT2Lntpk9R2UqwAMXgIFy3u/BqWUR+kRQRYXfiaCZbvPMfSuit6/SujEeqvvoNAiMHS5hoBSfkqDIAszxvDmgnAK5g5hRGsvnyA+uto6EshXCoYsg/xZaOhLpZRbaRBkYcvDz7Pj1HWebF2Z/Lm8eDRweCVM6wUFylvnBPKV9N62lVJep0GQRYWfieDFWTuoWTIfg1p6sUnmwAqY3hcKh1n3CeQp5r1tK6VsoUGQBTkchr/N202BXCGMH9qEHNm8dLnovsUwcwAUrwWDF1nnBpRSfk+DIAuatukku89E8Eq7ahT31kD04fNg1mAoVR8GLYTchbyzXaWU7TQIspiIqHjeXBBOndL56d6wtHc2unMmzB0GZZtZHcjlzO+d7SqlsgQNgixmwvrjALzavpp3hp7cNhnmPwHl74IBcyCHDTesKaVspUGQhVyLjGP8+mM0LFeAVlW80D6/ZRwsehoqt4V+syAk1PPbVEplORoEWch7y/ZxMyaBD7rX9fzRwMbvYMkLENYO+kyHkNye3Z5SKsvSIMgi9p69wZytp+nXtBzVSni4eWbdF7D8Vaj+EPSeAtm9dEJaKZUlaV9DWUB8ooOnp22jUGgIz9/v4Q7d1nwEv/wLanWD7t9DsE2D2yilsgwNgixg+qaTHL0cyegBDSmcJ4dnNmIM/PYBrP4Q6vaGLt9AsP76lVIebhoSkfYickBEDovIqBTm9xeRXc7HehGp58l6sqLouES+WHWIsoVy0a5WCc9sxBhY9bYVAvUHQNdvNQSUUv/PY0EgIsHA10AHoCbQV0RqJlvsGNDaGFMXeBcY46l6sqp3luzl8q04RrWv4ZkTxMbAj2/A2k+h8aPQ+UsIsmFgG6VUluXJI4KmwGFjzFFjTBwwA+iSdAFjzHpjzDXn0w1AQHVxuXz3OaZvOsnQuyrQqa4HOnZzOGDZK7Dha2g2Ajr9F4L0+gCl1J958l2hNHAqyfPTzmmpGQYs92A9WUqiw/Dq3F0Uy5uDUR2qu38DDgcseQ42fw8tn4H2/wZv3KCmlPI5nmwoTuldx6S4oEhbrCBolcr84cBwgHLlyrmrPlt9/NMB5z0DddzfqZwjERY+DTunwd0vw71vaggopVLlySOC00DZJM/LAGeTLyQidYGxQBdjzJWUVmSMGWOMaWyMaVy0aFGPFOtNxy9H8u1vR2hTrSid6ri5SSgxweoyYuc0aPsG3Pd3DQGlVJo8GQSbgTARqSgiIUAfYFHSBUSkHDAPGGiMOejBWrKU//y4H4A3O7n5BHFiPMx9FHbPhvv/Ca1fdd+6lVJ+y2NNQ8aYBBF5GvgRCAbGGWP2iMgI5/zRwFtAYeAb5xtigjGmsadqygp+2nOeZbvP8+y9VahSzI13ECfEwuwhcGAZtHsfWox037qVUn7NoxeTG2OWAcuSTRud5PvHgMc8WUNWcjMmnqenbadGyXw8fa8b7yCOj4aZA+Hwz9DxY2j6uPvWrZTye3pXkRd9/OMB4hId/OPhmoRkc1OrXFwUzOhrDTb/8OfQaIh71quUChgaBF4yft0xJv5xgiEtK9C8UmH3rDT2FkzvAyfWQddvoH4/96xXKRVQNAi8YP7207y9eC93hxXh9Y413LPSmBswtSec3mx1Hlenh3vWq5QKOBoEHmaM4YWZO6lZMh8ThjYlOMgNVwlFX4Mpj8C5ndBjHNTqmvl1KqUClgaBhy3dfQ6AQS3KuycEoq7C5K5wYS/0mgTVO2V+nUqpgKZB4EHnI2J4d8leqpfIyyON3NCN0q1LVghcPgR9p0PYA5lfp1Iq4GkQeEhcgoNhEzdzLTKe8UOakj04k1cJ3TwPk7rAtRPQb6Y1zrBSSrmBBoGH/Pfng+w5e4N3u9SiZql8mVvZjbMw8WG4cQ4GzIEKKXbJpJRSd0SDwANW7bvA6NVH6NOkLANbVMjcyq6ftEIg8goMnAflmrulRqWUuk2DwM2MMXy+6hAl8uXMfPfSV4/BxM4QEwGDFkAZv+59QyllEx2lxM3mbTvDrtMRjLy3CgVyh9z5iq4cgQmdIO4mDF6kIaCU8hg9InCjuAQHH67YT7lCuenTpGz6L0jNpQNWc5AjEQYvgRK13VekUkolo0HgRpP+OM7Fm7GMH9rkzq8SurDHag6SIBiyFIp5YPQypZRKQpuG3OTs9Wg+W3mINtWK0rZasTtbybmdMOEhCM4OQ5dpCCilvEKDwE3+viCcRIfh3S532IxzZqvVHBQSaoVAETd2U62UUmnQIHCDVfsusGr/RZ65rwplC+XO+ApOboRJXSFnASsEClVyd4lKKZUqDYJMik1I5J+L91CteF6GtaqY8RUcXweTu0FoURi6HAqUc3+RSimVBg2CTJq5+RSnrkYzqmN1cmQLztiLj/5m9SKav7R1JJC/tEdqVEqptGgQZIIxhvHrjlOjZL6MnyA+tBKm9YZCFa2rg/KW8EyRSimVDg2CTJi26STHLkcyonUG2/QPLLeGlywSZt0nkOcOrzJSSik30CC4Q9FxiXz680Hqly3AQ3VLuf7CvYtg5gAoXhsGL4ZQNw1bqZRSd0iD4A59+cshLt+K428dqrs+4MzuOTB7CJRqaPUdlKugJ0tUSimXaBDcgYioeMauPUaX+qVo5upA9Dumw7zHrd5DB86DnPk9W6RSSrlIgyCDjDE8O2M7cQkOHr/bxXMD2ybBgietcQT6z4YceT1bpFJKZYAGQQZtOHqV1Qcv8eIDVald2oVP9ZvHwqJnoMp90G+WdeewUkplIRoEGZCQ6ODtxXsomDs7g1tWSP8FG76FpS9B1Q7QZxpkz+XxGpVSKqM0CDJg5pZT7D9/kw+61yF/ruxpL7z2M1gxCmo8DL0mQbYcXqlRKaUySruhdtGt2AQ+/fkQTSoUpF2tdG7+Wv0R/PovqP0IdPvO6k1UKaWyKA0CF8QnOnhu+nYu34rl+0GNEEnlclFj4Nf3Yc1/oG4f6PoNBGWw2wmllPIyDQIX/HPRHlbtv8jbnWvRoFwq1/4bAyv/Aes+hwYD4eHPNQSUUj5BgyAdW09cZerGkwxuUT71E8TGwIq/wcZvofEw6PgxBOnpF6WUb9AgSEN8ooPX54VTIl9OXm2fymhhDgcsexm2/ADNnoT2H0BqTUdKKZUF6cfWNCzYfoYDF27y0oNVCc2RQmY6HLD4WSsE7npOQ0Ap5ZP0iCAV+87d4M0F4dQrW4DuDcv8dQFHIiwcCTunwz2vQtvXNQSUUj5JgyAVH/94gFwhwfwwuPFfO5VLjIf5T0D4XGj7JrR+xZ4ilVLKDbRpKAVnrkfzy4GLDGxeniJ5kt0IlhAHc4ZaIXD/2xoCSimfp0cEKXh/2T6yBwfRq3HZP89IiIVZg+Hgcmj3AbR4yp4ClVLKjTQIkvll/wWW7jrHC/dXpWyh3P+bER9tDShzeCV0+gSaPGZfkUop5UYaBEkkJDp4f9l+KhYJ5ck2lf83Iy4SpveFY2ug85fQcJB9RSqllJvpOYIkxq07xuGLtxjVoToh2Zw/mtibMLUnHP8duo3WEFBK+R09InA6FxHN+8v207h8QR6sWdyaGBMBU3rAma3Q/Xuo08PeIpVSygM0CJw+WnGAkOAgPuhex+pULvoaTO4O53dBz/FQs4vdJSqllEdoEABXbsWydPc5ejYuQ1jxvBB5BSZ3hUv7ofcUqNbB7hKVUspjNAiA95ftJ8FhGNKyAty6BJO6wNUj0Gc6hN1vd3lKKeVRAR8E+8/fYO620wy/pxJhuSNhQme4fhL6zYRKbewuTymlPC6ggyA6LpGnp20nJFsQj9fLAeM7ws3zMGAOVGhld3lKKeUVAR0E7y3by5FLt5jYrThFZ3e1ThAPnA/lmtldmlJKeU3ABsH2k9eYsuEkz9QP5p51QyD2BgxaAKUb2V2aUkp5VUAGQUx8Im8v3kv9XBd5/vSHkBgLgxdDyXp2l6aUUl4XkEHw+apD3DodzpJ8HxLsEBiyBIrXsrsspZSyRcAFwa8HLrJ6za/Mz/1vcmbPAYMWQbFUhqFUSqkA4NG+hkSkvYgcEJHDIjIqhfkiIl845+8SkYaerAfgh9kLmJHjPUJz5YIhyzQElFIBz2NBICLBwNdAB6Am0FdEaiZbrAMQ5nwMB771VD0A169f5ev4f5AtV16CHl0ORap4cnNKKeUTPHlE0BQ4bIw5aoyJA2YAyTvs6QJMMpYNQAERKemJYqJuXiPv9QNEBecj52MroFBFT2xGKaV8jieDoDRwKsnz085pGV0GERkuIltEZMulS5fuqJhcOUOQ2OskDl5CUKHyd7QOpZTyR548WSwpTDN3sAzGmDHAGIDGjRv/Zb4r7mr9AObu+5AgHYJBKaWS8uS74mkg6aC/ZYCzd7CM22gIKKXUX3nynXEzECYiFUUkBOgDLEq2zCJgkPPqoeZAhDHmnAdrUkoplYzHmoaMMQki8jTwIxAMjDPG7BGREc75o4FlQEfgMBAFDPVUPUoppVLm0RvKjDHLsN7sk04bneR7A4z0ZA1KKaXSpo3mSikV4DQIlFIqwGkQKKVUgNMgUEqpAKdBoJRSAU6DQCmlApwGgVJKBTgNAqWUCnAaBEopFeDEurnXd4jIJeDEHb68CHDZjeX4At3nwKD7HBgys8/ljTFFU5rhc0GQGSKyxRjT2O46vEn3OTDoPgcGT+2zNg0ppVSA0yBQSqkAF2hBMMbuAmyg+xwYdJ8Dg0f2OaDOESillPqrQDsiUEoplYxfBoGItBeRAyJyWERGpTBfROQL5/xdItLQjjrdyYV97u/c110isl5E6tlRpzult89JlmsiIoki0sOb9XmCK/ssIm1EZIeI7BGR1d6u0d1c+NvOLyKLRWSnc599eqRDERknIhdFJDyV+e5//zLG+NUDa1jMI0AlIATYCdRMtkxHYDkgQHNgo911e2GfWwIFnd93CIR9TrLcL1gj5fWwu24v/J4LAHuBcs7nxeyu2wv7/DrwofP7osBVIMTu2jOxz/cADYHwVOa7/f3LH48ImgKHjTFHjTFxwAygS7JlugCTjGUDUEBESnq7UDdKd5+NMeuNMdecTzcAZbxco7u58nsGeAaYC1z0ZnEe4so+9wPmGWNOAhhjfH2/XdlnA+QVEQHyYAVBgnfLdB9jzBqsfUiN29+//DEISgOnkjw/7ZyW0WV8SUb3ZxjWJwpflu4+i0hpoBswGv/gyu+5KlBQRH4Tka0iMshr1XmGK/v8FVADOAvsBp4zxji8U54t3P7+5dHB620iKUxLfmmUK8v4Epf3R0TaYgVBK49W5Hmu7PNnwGvGmETrw6LPc2WfswGNgPuAXMAfIrLBGHPQ08V5iCv73A7YAdwLVAZ+FpHfjTE3PFybXdz+/uWPQXAaKJvkeRmsTwoZXcaXuLQ/IlIXGAt0MMZc8VJtnuLKPjcGZjhDoAjQUUQSjDELvFKh+7n6t33ZGBMJRIrIGqAe4KtB4Mo+DwX+bawG9MMicgyoDmzyTole5/b3L39sGtoMhIlIRREJAfoAi5ItswgY5Dz73hyIMMac83ahbpTuPotIOWAeMNCHPx0mle4+G2MqGmMqGGMqAHOAp3w4BMC1v+2FwN0ikk1EcgPNgH1ertOdXNnnk1hHQIhIcaAacNSrVXqX29+//O6IwBiTICJPAz9iXXEwzhizR0RGOOePxrqCpCNwGIjC+kThs1zc57eAwsA3zk/ICcaHO+xycZ/9iiv7bIzZJyIrgF2AAxhrjEnxMkRf4OLv+V1ggojsxmo2ec0Y47O9korIdKANUERETgP/ALKD596/9M5ipZQKcP7YNKSUUioDNAiUUirAaRAopVSA0yBQSqkAp0GglFIBToNAZUnO3kJ3JHlUSGPZW27Y3gQROebc1jYRaXEH6xgrIjWd37+ebN76zNboXM/tn0u4s8fNAuksX19EOrpj28p/6eWjKksSkVvGmDzuXjaNdUwAlhhj5ojIg8DHxpi6mVhfpmtKb70iMhE4aIx5L43lhwCNjTFPu7sW5T/0iED5BBHJIyKrnJ/Wd4vIX3oaFZGSIrImySfmu53THxSRP5yvnS0i6b1BrwGqOF/7onNd4SLyvHNaqIgsdfZ/Hy4ivZ3TfxORxiLybyCXs46pznm3nF9nJv2E7jwSeUREgkXkIxHZLFYf80+48GP5A2dnYyLSVKxxJrY7v1Zz3on7DtDbWUtvZ+3jnNvZntLPUQUgu/ve1oc+UnoAiVgdie0A5mPdBZ/POa8I1l2Vt49obzm/vgS84fw+GMjrXHYNEOqc/hrwVgrbm4BzvAKgJ7ARq/O23UAoVvfGe4AGwCPA90lem9/59TesT9//X1OSZW7X2A2Y6Pw+BKsXyVzAcOBN5/QcwBagYgp13kqyf7OB9s7n+YBszu/vB+Y6vx8CfJXk9e8DA5zfF8DqgyjU7t+3Pux9+F0XE8pvRBtj6t9+IiLZgfdF5B6srhNKA8WB80lesxkY51x2gTFmh4i0BmoC65xda4RgfZJOyUci8iZwCauH1vuA+cbqwA0RmQfcDawAPhaRD7Gak37PwH4tB74QkRxAe2CNMSba2RxVV/43ilp+IAw4luz1uURkB1AB2Ar8nGT5iSIShtUTZfZUtv8g0FlEXnY+zwmUw7f7I1KZpEGgfEV/rNGnGhlj4kXkONab2P8zxqxxBkUnYLKIfARcA342xvR1YRuvGGPm3H4iIventJAx5qCINMLq7+UDEfnJGPOOKzthjIkRkd+wuk7uDUy/vTngGWPMj+msItoYU19E8gNLgJHAF1j97fxqjOnmPLH+WyqvF+ARY8wBV+pVgUHPEShfkR+46AyBtkD55AuISHnnMt8DP2AN97cBuEtEbrf55xaRqi5ucw3Q1fmaUKxmnd9FpBQQZYyZAnzs3E5y8c4jk5TMwOoo7G6sztRwfn3y9mtEpKpzmykyxkQAzwIvO1+THzjjnD0kyaI3sZrIbvsReEach0ci0iC1bajAoUGgfMVUoLGIbME6OtifwjJtgB0ish2rHf9zY8wlrDfG6SKyCysYqruyQWPMNqxzB5uwzhmMNcZsB+oAm5xNNG8A/0rh5WOAXbdPFifzE9a4tCuNNfwiWONE7AW2iTVo+Xekc8TurGUnVtfM/8E6OlmHdf7gtl+BmrdPFmMdOWR31hbufK4CnF4+qpRSAU6PCJRSKsBpECilVIDTIFBKqQCnQaCUUgFOg0AppQKcBoFSSgU4DQKllApwGgRKKRXg/g/8v0IR+DqqFgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pr, tpr, thresholds = roc_curve(df_test.Label, log_model.predict_proba(df_test_avg[features])[:, 1])\n",
    "plt.subplots(1, figsize=(6,6))\n",
    "plt.title('AUC score = ' + str(round(float(auc_score), 3)), fontsize=18)\n",
    "plt.plot(pr, tpr)\n",
    "plt.plot([0, 1])\n",
    "plt.plot([0, 0], [1, 0] , c=\".7\"), plt.plot([1, 1] , c=\".7\")\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "690e5a16",
   "metadata": {},
   "source": [
    "- P-R curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ead70270",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAGDCAYAAAAmphcsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAm0klEQVR4nO3debxVdb3/8ddbBkFRMSEHQCDUBP054piVmSKYhlbmUJma14tDeX+363Cta7drXYd+pZV00ZJsMNEciIr0OuSQpYGKAw55lJTjkDgAigMCn98f33U8+2z2OWzOOescDt/38/E4j7PXsNf+LIb13mt91/e7FBGYmVm+1unuAszMrHs5CMzMMucgMDPLnIPAzCxzDgIzs8w5CMzMMucgsB5D0uck/W8d602R9B9dUVNXkPR3SfsXr/9T0i+7uyZbuzgIrFMUB6u3JL0h6R+SfippQGd+RkRcGRHj6lhvUkSc25mf3URSSFpS7Odzkr4nqVcZn9UekjaUdLGkZ4saG4rpQd1dm625HATWmQ6JiAHALsBuwNerV5DUu8ur6nw7Fvv5UeAI4PhurgcASX2BW4HtgPHAhsDewCvA7u3Y3trwd2V1cBBYp4uI54A/ANvDe9+iT5H0JPBkMe9gSXMkLZT0Z0k7NL1f0jBJ10taIOkVSZcU84+V9KfitSRdJOklSYskPSSp6fOukPStiu39U/HN+FVJMyRtUbEsJE2S9KSk1yRNlqQ697MBuBvYqWJ77dmvUZJuK+a9LOlKSQNX848d4BhgS+CwiHg0IlZExEsRcW5EzKzY360qanrvz0rSvpIaJZ0p6UXgp5Iek3Rwxfq9ixp3Kab3LPZzoaQHJe3bjrqtmzkIrNNJGgYcBDxQMftQYA9gTHEQmQr8M7AJcCkwQ9K6xWWW3wHPACOAIcC0Gh8zDvgIsA0wkPTN/JUatewHnAd8Fti82G719g4mncHsWKx3YJ37uS3wYaChmG7vfqmocQtgNDAM+M96aqiyP3BjRLzRjvc22Qx4HzAcOBG4CjiqYvmBwMsRcb+kIcDvgW8V7/k34DpJgzvw+dYNHATWmaZLWgj8CbgD+O+KZedFxKsR8RbwT8ClEXFvRCyPiJ8B7wB7ki5hbAGcHhFLIuLtiPhTjc96F9gA2BZQRDwWES/UWO9zwNSIuD8i3gH+HdhL0oiKdc6PiIUR8SzwRyq+4bfifklLgMeA24EfFfPbtV8R0RARN0fEOxGxAPge6bLT6toEqPVnsDpWAN8oankL+BXwSUnrFcuPLuYBfB6YGREzi7OPm4HZpC8B1oM4CKwzHRoRAyNieEScXBxImsyveD0c+GpxOWFhER7DSAfKYcAzEbGsrQ+KiNuAS4DJwD8kXSZpwxqrbkH6Ft70vjdIZw5DKtZ5seL1m8AAAElziwbXNyR9uGKdXYp1jiCd5azfkf2S9H5J04rG58XAL4H2NO6+Qjrr6YgFEfF200Rx+esx4JAiDD5JcxAMBw6v2t99OqEG62IOAusqlcPczge+XYRG0896EXFVsWzLehoqI+IHEbErqXF0G+D0Gqs9TzpgASBpfdI35+fq2P52ETGg+LmrallExDXAX4BzOrhf55H+fHaIiA1J37TraqeocgtwYLGPrXkTWK9ierOq5bWGI266PDQReLQIB0j79Iuq/V0/Is5vR+3WjRwE1h1+DEyStEfR6Lu+pE9I2gD4K+nyxvnF/H6SPlS9AUm7Fe/vAywB3gaW1/isXwHHSdpJ0rqky1X3RsTfO2lfzgdOlLRZB/ZrA+ANYGFx3b1WoNXjF6SD83WStpW0jqRNJJ0tqelyzRzgaEm9JI2nvktQ00htMifRfDYA6czlEEkHFtvrVzQ4D21n/dZNHATW5SJiNul6+iXAa6TG1mOLZcuBQ4CtgGeBRtIlmGobkg68r5Eu/bwC/L8an3Ur8B/AdaQD8SjgyE7cl4dJ7SGnd2C/vkm63LSI1Ph6fTtreYfUYPw4cDOwmBRAg4B7i9VOK+pYSGo/mV7Hdl8gnfnsDVxdMX8+6SzhbGABKYROx8eVHkd+MI2ZWd6c3GZmmXMQmJllzkFgZpY5B4GZWeYcBGZmmetxowsOGjQoRowY0d1lmJn1KPfdd9/LEVFzHKgeFwQjRoxg9uzZ3V2GmVmPIumZ1pb50pCZWeYcBGZmmXMQmJllzkFgZpY5B4GZWeYcBGZmmXMQmJllzkFgZpY5B4GZWeZKCwJJUyW9JOmRVpZL0g8kNUh6SNIuZdViZmatK/OM4ApgfBvLJwBbFz8nAv9TYi1mZtaK0oIgIu4EXm1jlYnAzyO5BxgoafOy6uGFF+Cii+D110v7CDOznqg7B50bQnrYdZPGYt4L1StKOpF01sCWW27Zvk87/HC4+27YYAM44YT2bcPMbC3UnY3FqjEvaq0YEZdFxNiIGDt4cM1RVFdt1qz0+6mn2vd+M7O1VHcGQSMwrGJ6KPB8N9ViZpat7gyCGcAxxd1DewKLImKly0JmZlau0toIJF0F7AsMktQIfAPoAxARU4CZwEFAA/AmcFxZtZiZWetKC4KIOGoVywM4pazPNzOz+rhnsZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmlrn8giBqPvvGzCxb+QSBaj0QzczM8gkCMzOryUFgZpY5B4GZWeYcBGZmmXMQmJllzkFgZpY5B4GZWeYcBGZmmXMQmJllzkFgZpY5B4GZWeYcBGZmmXMQmJllzkFgZpY5B4GZWebyCwI/mMbMrIV8gsAPpjEzq6nUIJA0XtITkhoknVVj+caSbpD0kKS/Stq+zHrMzGxlpQWBpF7AZGACMAY4StKYqtXOBuZExA7AMcD3y6rHzMxqK/OMYHegISKejoilwDRgYtU6Y4BbASLicWCEpE1LrMnMzKqUGQRDgPkV043FvEoPAp8CkLQ7MBwYWr0hSSdKmi1p9oIFC0oq18wsT2UGQa3W2epbds4HNpY0B/gy8ACwbKU3RVwWEWMjYuzgwYM7vVAzs5z1LnHbjcCwiumhwPOVK0TEYuA4AEkC5hU/ZmbWRco8I5gFbC1ppKS+wJHAjMoVJA0slgGcANxZhIOZmXWR0s4IImKZpFOBm4BewNSImCtpUrF8CjAa+Lmk5cCjwJfKqsfMzGor89IQETETmFk1b0rF678AW5dZg5mZtS2fnsVmZlaTg8DMLHMOAjOzzDkIzMwy5yAwM8ucg8DMLHMOAjOzzOUXBH5CmZlZC/kEgZ9QZmZWUz5BYGZmNTkIzMwy5yAwM8ucg8DMLHMOAjOzzDkIzMwy5yAwM8ucg8DMLHMOAjOzzDkIzMwy5yAwM8ucg8DMLHMOAjOzzDkIzMwy5yAwM8ucg8DMLHP5BYGfUGZm1kI+QeAnlJmZ1ZRPEJiZWU0OAjOzzDkIzMwyV2oQSBov6QlJDZLOqrF8I0m/lfSgpLmSjiuzHjMzW1lpQSCpFzAZmACMAY6SNKZqtVOARyNiR2Bf4LuS+pZVk5mZrazMM4LdgYaIeDoilgLTgIlV6wSwgSQBA4BXgWUl1mRmZlXKDIIhwPyK6cZiXqVLgNHA88DDwGkRsaLEmszMrEqZQVDrxv3q3lwHAnOALYCdgEskbbjShqQTJc2WNHvBggWdXaeZWdbKDIJGYFjF9FDSN/9KxwHXR9IAzAO2rd5QRFwWEWMjYuzgwYNLK9jMLEdlBsEsYGtJI4sG4COBGVXrPAt8HEDSpsAHgadLrMnMzKr0LmvDEbFM0qnATUAvYGpEzJU0qVg+BTgXuELSw6RLSWdGxMtl1WRmZisrLQgAImImMLNq3pSK188D48qswczM2uaexWZmmXMQmJllzkFgZpY5B4GZWebyCwI/oczMrIV8gsBPKDMzqymfIDAzs5ocBGZmmXMQmJllzkFgZpY5B4GZWeYcBGZmmXMQmJllLr8gcIcyM7MW8gkCdygzM6spnyAwM7OaHARmZplzEJiZZc5BYGaWOQeBmVnmHARmZpnLLwjcj8DMrIV8gsD9CMzMasonCMzMrCYHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWWu1CCQNF7SE5IaJJ1VY/npkuYUP49IWi7pfWXWZGZmLZUWBJJ6AZOBCcAY4ChJYyrXiYjvRMROEbET8O/AHRHxalk1mZnZyso8I9gdaIiIpyNiKTANmNjG+kcBV5VYj5mZ1VBmEAwB5ldMNxbzViJpPWA8cF0ry0+UNFvS7AULFnR6oWZmOSszCGqN8tba0J+HAHe3dlkoIi6LiLERMXbw4MGdVqCZmZUbBI3AsIrpocDzrax7JL4sZGbWLcoMglnA1pJGSupLOtjPqF5J0kbAR4HflFiLmZm1ondZG46IZZJOBW4CegFTI2KupEnF8inFqocB/xsRS8qqxczMWldaEABExExgZtW8KVXTVwBXlFlHVVFd9lFmZj1BPj2L/YQyM7Oa8gkCMzOryUFgZpY5B4GZWeYcBGZmmavrriFJHwL+ExhevEdARMQHyivNzMy6Qr23j14O/F/gPmB5eeWYmVlXqzcIFkXEH0qtxMzMukW9QfBHSd8BrgfeaZoZEfeXUpWZmXWZeoNgj+L32Ip5AezXueWYmVlXqysIIuJjZRdiZmbdo67bRyVtJOl7TQ+HkfTdYtRQMzPr4ertRzAVeB34bPGzGPhpWUWZmVnXqbeNYFREfLpi+puS5pRQj5mZdbF6zwjekrRP00TRweytckoyM7OuVO8ZwUnAz4p2AQGvAseWVZSZmXWdeu8amgPsKGnDYnpxmUWZmVnXaTMIJH0+In4p6V+r5gMQEd8rsbZy+AllZmYtrOqMYP3i9wZlF1I6P6HMzKymNoMgIi4tfn+za8rpIuedB2efDStWOCDMLHv1dii7UNKGkvpIulXSy5I+X3ZxpTn77PR7xYqObafMy0zvFEM6zZ8Pr75a3ueYWfbqvX10XNFAfDDQCGwDnF5aVV3lnXdWvU5rzjoL1lknnVG88w488wx8+9urv52PfCRto/qnX7/0e8stYZNNmuc/+mh6XwTccAM8/nj798HMjPqDoE/x+yDgqojouV9Rl1c8TuG7323fNl57DS64oHm6Xz8YMQK+/vV0sN5nn1bfyiOPwDXXwFe/mta9667V++zttkvvW2cd+NSnYPRoWLKkXbvxngi49FK4/fb00xQ2TZYvhwsvhJ13Tvt5zTXwwAOwYEHzOi+91PEzLDPrFvX2I/itpMdJnchOljQYeLu8skpUefB69tm21x06FJ57Lh0oly6Fddet7zPuvjudJfTrV9/699wDAwak9/Ttmy4FffjD8PrrsOGGaZ2IdPCvZcCAleettx6MGwfTp6fpr30N5s2DX/2qvpracsQRHd/Gt78NixfD3nvDJz/Z8e2ZWbvV24/gLEkXAIsjYrmkJcDEcksrybvvNr+udQBtsnx5CgFovUF5xYp0cN58c/j1r2GXXdIBGOoLgZNOgksuaf0A3xQCTTVEwC9+kYLic5+DjTZKwVHLm282hwCs+rLVBhuk4Nl333RWUO3cc2HUqBQmTz0F06alz6jX6NHwvvelkIQUTJV694Zly2DXXeGFF+D552H48BQUBx4If/sb/OhH8M//DOPHw7BhMGRIqmHu3HSWtvPOab6ZrZZV9SPYLyJuk/SpinmVq1xfVmGlqWzgHT269fXOOKPt7Xzta80H59Xx1lv1nynU8oUvtJyOSAfmxx6DD30InngCJk9Ol41+8AOYOBF23z29vvpqmDMHdtghXZIaMwZefBG2337lz1m+HHr1ar2Oyy9v/vzKfxNNfx5t3Y11993NZz0HHQR//zsMHJj24b77mtd75pn0c9VVzfMuuKDlZbnWTJ8OEya0HpRm9h5FGwcySd+MiG9IqjXSaETE8eWVVtvYsWNj9uzZq//GgQNh0aJ0cPhD8dTNq6+Gz3629vr77NP87RXStfkZM+ADH0iXNCq/rVd64w34+MfT5Y4zzoDGxnTmMXjw6teco8WL0yW4ddaBl19ObQ9LlqTXy5bByJEwa1YKjRtugJNPTqFyyy0p5JbXeKT2OeekBvddd00/HQlisx5K0n0RMbbWslX1I/hG8fu4MgrrFm9XNG00HTSWLoWttoKLLoJPF4OsbrFFmnfrrelb7vDhze9rLQQgHfTvvbd5euTIzqs9B5V/tptvnn6q7bxz+n3RRbW38frrKRiOOSYF83/9V8vlH/tYuoS09dbp8tJf/pLOSg44AE49Ffr375RdMesp2jwjeG8l6b+BCyNiYTG9MfDViPh6ueWtrMNnBHvtlf7jQzrQr78+XHttOihAy0sbw4atukHZ1mwLFqQ2hIcfTl8CJk9Of6ebbJLOMmq54IJ0yarpDi2ztUBbZwT1BsEDEbFz1bz7I2KXTqqxbh0Ogp13Trc+tmb6dDj00OZpj0209mn6O3388XSg79s33bn14x/DP/6RLjs1mTAh/ZtparTuXe+NdmZrlnZfGqrQS9K6EfFOscH+wCrvpZQ0Hvg+0Av4SUScX2OdfYGLSX0VXo6Ij9ZZU/u8vYq7XitD4FvfKrUU6yZN3/Irbxb4wAfg6KPT64aGdIY4enRqT2pqU4J0KWn//VNAjBsHG2/cdXWblaTeM4IzgE+SHk8ZwPHAjIi4sI339AL+BhxA6o08CzgqIh6tWGcg8GdgfEQ8K+n9EfFSW7V0+Ixg5Mh0C2Q9PBaRRcDvfpcaphcuhN//PrUpVdt/fzj2WNhjj3Sbrf/d2Bqmw2cEEXGhpIeA/UkPpjk3Im5axdt2Bxoi4umiiGmkvgeV3VaPBq6PiGeLz2kzBDrFqs4Imixf7v/Mlv4NHHJI+mkybx6cckoaB+qRR9K8W25JP5V23DGdZQwfDgcfnNqjzNZAq3PB8zFgWUTcImk9SRtExOttrD8EmF8x3QjsUbXONkAfSbeThrr+fkT8fDVqWn0vvFDfeq118jIbORJmzmyeXrEidXi7994UDNdfnzq4Pfhg+mmy5ZZpwMPhw1PHPd/GamuIekcf/SfgWuDSYtYQYPqq3lZjXvV1qN7ArsAngAOB/5C0TY3PP1HSbEmzF1QOEdERlb1uAf74x+bXlb2PzVZlnXVg223hi1+E73wndfB79dUUEE89BXvumdZ79lmYNCm1L2ywQbpd9StfgZ/+tGMDIJp1UL1nBKeQLvXcCxART0p6/yre0whU9vcfCjxfY52XI2IJsETSncCOpLaF90TEZcBlkNoI6qy5bfvvD3femUb/vO++NDzEAw/A/ff7zhDrHFJqhG66XXnhwtTB8NZb4fzz0++my0nHF30zR45MITJ4cPq3OWZMapD+yEfS7c5mJaj3iPdORCxtGl5CUm9W/nZfbRawtaSRwHPAkaQ2gUq/AS4ptteXdOmolV5CHVR9vb9PnzTEQWVj+U47pR+zMgwcmH623x5OOy3Ne+qpNGpt//4pJPr3T30eGhpSG0T1mcLYsemOps02S73fx49vHt/KrJ3qDYI7JJ0N9Jd0AHAy8Nu23hARyySdCtxEun10akTMlTSpWD4lIh6TdCPwELCCdIvpI+3dmdXSp8+q1zEr26hRLcdSqhSRzg6eeAJ+8pN0Cen++6HprrmLLkqXpQ4+OHWOO/ZY2G23FBRumLbVUO/towJOAMaRrv3fRDpod3lvq3bfPrrxxunUvIk7illPNW8eTJ2azhYWLUohUf0siGHD4POfTz3pJ0zw5U7rWM9iSesAD0VEjSEqu56DwKxKRLqM9Prr6Yxh7tw0bMpTT6XlAwakkWm32goOOyzdsdTWyLK2VuqMISauBP696X7/7uQgMKvTvHmpV3RTMLxU0U1nv/3gox9Nw5AffXS6WWLp0hQaPntYK3VGENwG7Ab8FXjvuYgR0eWPlnIQmLXTCy/AjTfCCSekM4LWbpMeNSoN1/2d76S+D7ZW6IwgqDn+T0Tc0cHaVpuDwKwTLFuWGqHfeCM9d6OxMQXFwoWpT03T3UqjR6d+EAcdlB5y5Jsseqx2DzEhqR8wCdgKeBi4PCKWdX6JZtalevdOw2xDGh+pUgT88Ifw6KMpIH72s3TH0oYbpmEzhg5Nl5M++MHm4dutR1vVxcCfAe8CdwETgDHAaWUXZWbdSEo9npssXQq/+Q1ceWX6Dc23vG6+eeoXMW5cer3zzukswuN09SirCoIxEfF/ACRdTmojMLOc9O0Lhx+efpYvT43Pb70Ff/1remzoTTfBzTc3r7/xxqlH9YABcNxx6U6ltp7qZ91uVUHwXmtS0UGs5HLMbI3WqxfssEN6XXlJ6cUX4ZVX0gN+Zs9Ow7XccUf6OeGE1CbRq1e6rPTFL8JXv5rGW/IxZY2wqofXL6f5LiEB/YE3i9cREV0e824sNushXn8dLr00XVKaM6f2OocfnhqhJ05MZxBWmg7fNbQmcRCY9WBvvgk//3nq7Hb55Wm47iZbbAEf/3h6fcABqWF6u+3c+a2TOAigZRAccgjMmNGpdZlZO7zzTnoC3LXXpmdFP/poy/4Nm2ySOrsdfHAaJHLnnVvflrWpM55ZvHZxz0mzNcO668KnP51+mrz7bnrQz/33p4H1/vSn5sZoKZ0lTJyYHvBz0EHpbiU/SKpD8jwi7r9/d1dgZq3p0ycd7LfbDr7whXQZ9+ab4eqr4emn4fbbmx8RCulJb2PGwOLFafqss+CYY9z5bTXkeWnotdfSuPBm1vNEwEMPpceAPvdculvp/vtbPmUQUpvDyJEwYkS6c2mffbK+GuBLQ9DyNjU3Ppn1XFJqSN5xx5WXLVmSekI3NqY+Dn/+cxpC48or0/KBA9M4SoMGpY5v55zjW1jJKQgqZfytwGyttv76cPLJzdMrVqQzhWuvTZeT+vVLj6hdujQt//GPU9+GceNSD+nddktnEZnJ84joMwKzPKyzTroltem2VEjhsHBh6uPwwx/CP/4B3/pW8y3lw4fD3nunO5SOPTY9P3otl08bwfve13zPclMvRzMzSP0bnngiXUq65RaYPr152QEHwIEHwr/8S48+brgfAbQMghUrfF3QzFoXkZ7dcMUVcM01ad77358anE86qUfeedhWEOR5861DwMzaIqVnPV99dWpPuPhi2HRTuP76dIaw665pJNbly7u70k6RZxCYmdWrTx847bR0y+qbb8LXvpZuVz300DQy6+abw3nnpTuWeigHgZlZvfr3Tw3Lr76axkr60IfSGcPZZ6dB83bZBR5/vLurXG0OAjOz1bXxxnD88elW1FdegWnT0uWiBx5I/ROGD4dLLmk5qN4azEFgZtZRRxyRnsMwb17qi/Dss/DlL6ebVE49Fd5+u7srbJODwMyss4wYkZ7ctmIF3HVX6r8weTJss03qs7CGBoKDwMyss0npVtNbboGZM2GzzdJzoPv3T6Hw3HPdXWELDgIzszJNmAD33pvOCACefDINa/Gv/wrPP9+9tRUcBGZmZZNSW0FE6r184IGpb8KwYSkovv99WLSo28pzEJiZdaW99kq9lh9/HM44A267LQ1fsemm8KUvwYIFXV6Sg8DMrDtss03qiLZoUXpc57hxMHVqak+48MIuLcVBYGbWnfr1g098Ij1H/de/Ts9MOPPM1GO5izqnlRoEksZLekJSg6SzaizfV9IiSXOKn3PKrMfMbI32mc+kBuShQ+HFF1PntAsuKP1jSwsCSb2AycAEYAxwlKQxNVa9KyJ2Kn7+q6x6PNCcmfUI664L8+dDQ0N6CttZZ6VnN5c4wF2ZZwS7Aw0R8XRELAWmARNL/Ly2LVvWbR9tZrbaRo1Kt53uuy/88pew3nrw7rulfFSZQTAEmF8x3VjMq7aXpAcl/UHSdrU2JOlESbMlzV7Q3hb1xYvb9z4zs+6y7rrprqJDDkmD251+eikfU2YQ1LoWU/0UnPuB4RGxI/BDYHqtDUXEZRExNiLGDs7gsXFmZu+RUkPyaaelR2eWoMwgaASGVUwPBVp0o4uIxRHxRvF6JtBH0qASazIz65kuvhh22qmUTZcZBLOArSWNlNQXOBKYUbmCpM2k1IorafeinldKrMnMzKr0LmvDEbFM0qnATUAvYGpEzJU0qVg+BfgMcJKkZcBbwJHR0x6ibGbWw+Xz8PrK20d72D6bmXWUH15vZmatchCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWWu1CCQNF7SE5IaJJ3Vxnq7SVou6TNl1mNmZisrLQgk9QImAxOAMcBRksa0st4FwE1l1WJmZq0r84xgd6AhIp6OiKXANGBijfW+DFwHvFRiLWZm1ooyg2AIML9iurGY9x5JQ4DDgCkl1mFmZm0oMwhUY15UTV8MnBkRy9vckHSipNmSZi9YsKBjVfXv37H3m5mtZXqXuO1GYFjF9FDg+ap1xgLTJAEMAg6StCwipleuFBGXAZcBjB07tjpMVk/fvh16u5nZ2qbMIJgFbC1pJPAccCRwdOUKETGy6bWkK4DfVYeAmZmVq7QgiIhlkk4l3Q3UC5gaEXMlTSqWu13AzGwNUOYZARExE5hZNa9mAETEsWXW8h7VarowM8uXexabmWUuvyDwGYGZWQv5BYGZmbXgIDAzy1x+QeBLQ2ZmLeQXBGZm1kJ+QeAzAjOzFvILAjMza8FBYGaWufyCwJeGzMxayCcINtss/d5zz+6tw8xsDZNPEOy3X/p95JHdW4eZ2RomnyAwM7OaHARmZplzEJiZZc5BYGaWOQeBmVnmHARmZplzEJiZZc5BYGaWOQeBmVnmHARmZplzEJiZZc5BYGaWOQeBmVnmHARmZplzEJiZZc5BYGaWOQeBmVnmHARmZplzEJiZZa7UIJA0XtITkhoknVVj+URJD0maI2m2pH3KrMfMzFbWu6wNS+oFTAYOABqBWZJmRMSjFavdCsyIiJC0A3ANsG1ZNZmZ2crKPCPYHWiIiKcjYikwDZhYuUJEvBERUUyuDwRmZtalygyCIcD8iunGYl4Lkg6T9Djwe+D4WhuSdGJx6Wj2ggULSinWzCxXZQaBasxb6Rt/RNwQEdsChwLn1tpQRFwWEWMjYuzgwYM7t0ozs8yVGQSNwLCK6aHA862tHBF3AqMkDSqxJjMzq1JmEMwCtpY0UlJf4EhgRuUKkraSpOL1LkBf4JUSazIzsyql3TUUEcsknQrcBPQCpkbEXEmTiuVTgE8Dx0h6F3gLOKKi8djMzLpAaUEAEBEzgZlV86ZUvL4AuKDMGszMrG3uWWxmljkHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgZlZ5hwEZmaZcxCYmWXOQWBmlrl8gmCdfHbVzGx1lDoM9Rrle9+DAQPg8MO7uxIzszVKPkEweDD8z/90dxVmZmscXy8xM8ucg8DMLHMOAjOzzDkIzMwy5yAwM8ucg8DMLHMOAjOzzDkIzMwy5yAwM8ucg8DMLHMOAjOzzDkIzMwy5yAwM8ucIqK7a1gtkhYAz7Tz7YOAlzuxnJ7A+5wH73MeOrLPwyNicK0FPS4IOkLS7IgY2911dCXvcx68z3koa599acjMLHMOAjOzzOUWBJd1dwHdwPucB+9zHkrZ56zaCMzMbGW5nRGYmVmVtTIIJI2X9ISkBkln1VguST8olj8kaZfuqLMz1bHPnyv29SFJf5a0Y3fU2ZlWtc8V6+0mabmkz3RlfWWoZ58l7StpjqS5ku7o6ho7Wx3/tjeS9FtJDxb7fFx31NlZJE2V9JKkR1pZ3vnHr4hYq36AXsBTwAeAvsCDwJiqdQ4C/gAI2BO4t7vr7oJ93hvYuHg9IYd9rljvNmAm8JnurrsL/p4HAo8CWxbT7+/uurtgn88GLiheDwZeBfp2d+0d2OePALsAj7SyvNOPX2vjGcHuQENEPB0RS4FpwMSqdSYCP4/kHmCgpM27utBOtMp9jog/R8RrxeQ9wNAurrGz1fP3DPBl4Drgpa4sriT17PPRwPUR8SxARPT0/a5nnwPYQJKAAaQgWNa1ZXaeiLiTtA+t6fTj19oYBEOA+RXTjcW81V2nJ1nd/fkS6RtFT7bKfZY0BDgMmNKFdZWpnr/nbYCNJd0u6T5Jx3RZdeWoZ58vAUYDzwMPA6dFxIquKa9bdPrxq3eHylkzqca86luj6lmnJ6l7fyR9jBQE+5RaUfnq2eeLgTMjYnn6stjj1bPPvYFdgY8D/YG/SLonIv5WdnElqWefDwTmAPsBo4CbJd0VEYtLrq27dPrxa20MgkZgWMX0UNI3hdVdpyepa38k7QD8BJgQEa90UW1lqWefxwLTihAYBBwkaVlETO+SCjtfvf+2X46IJcASSXcCOwI9NQjq2efjgPMjXUBvkDQP2Bb4a9eU2OU6/fi1Nl4amgVsLWmkpL7AkcCMqnVmAMcUre97Aosi4oWuLrQTrXKfJW0JXA98oQd/O6y0yn2OiJERMSIiRgDXAif34BCA+v5t/wb4sKTektYD9gAe6+I6O1M9+/ws6QwISZsCHwSe7tIqu1anH7/WujOCiFgm6VTgJtIdB1MjYq6kScXyKaQ7SA4CGoA3Sd8oeqw69/kcYBPgR8U35GXRgwfsqnOf1yr17HNEPCbpRuAhYAXwk4ioeRtiT1Dn3/O5wBWSHiZdNjkzInrsqKSSrgL2BQZJagS+AfSB8o5f7llsZpa5tfHSkJmZrQYHgZlZ5hwEZmaZcxCYmWXOQWBmljkHgVkNxWilcyQ9UoxsObCTt/93SYOK12905rbNVpeDwKy2tyJip4jYnjQA2CndXZBZWRwEZqv2F4pBvSSNknRjMaDbXZK2LeZvKumGYkz8ByXtXcyfXqw7V9KJ3bgPZq1a63oWm3UmSb1IwxdcXsy6DJgUEU9K2gP4EWmwsx8Ad0TEYcV7BhTrHx8Rr0rqD8ySdN1aMM6TrWUcBGa19Zc0BxgB3Eca0XIA6QE/v64YzXTd4vd+wDEAEbEcWFTM/4qkw4rXw4CtAQeBrVEcBGa1vRURO0naCPgdqY3gCmBhROxUzwYk7QvsD+wVEW9Kuh3oV0axZh3hNgKzNkTEIuArwL8BbwHzJB0O7z07tunZz7cCJxXze0naENgIeK0IgW1JjxU0W+M4CMxWISIeID0r90jgc8CXJD0IzKX5sYmnAR8rRsC8D9gOuBHoLekh0giZ93R17Wb18OijZmaZ8xmBmVnmHARmZplzEJiZZc5BYGaWOQeBmVnmHARmZplzEJiZZc5BYGaWuf8PqYdQnoTEwcYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(df_test.Label, \\\n",
    "                                                       log_model.predict_proba(df_test_avg[features])[:, 1])\n",
    "plt.subplots(1, figsize=(6,6))\n",
    "plt.plot(recall, precision, color='red')\n",
    "plt.title('Precision-Recall Curve')\n",
    "plt.ylabel('Precision')\n",
    "plt.xlabel('Recall')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4e3d530",
   "metadata": {},
   "source": [
    "## Model persistence (5pts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c50cb457",
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "joblib.dump(final_model, \"final_model.pkl\")\n",
    "final_model = joblib.load(\"final_model.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfbc24c7",
   "metadata": {},
   "source": [
    "### Output as a text file for website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3ec09a95",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_avg = pd.DataFrame()\n",
    "for i in Labels.index:\n",
    "    label_avg = label_avg.append(get_ave(get_neighbor(i)).loc[features], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c90942b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.DataFrame(final_model.predict(label_avg))\n",
    "pred.columns = ['pred']\n",
    "pred.index = Labels.index\n",
    "result = Labels.join(pred)\n",
    "result.to_csv('result.txt', sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77b0d4de",
   "metadata": {},
   "source": [
    "## Test set 2 (20pts)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddeaf405",
   "metadata": {},
   "source": [
    "### Please run the entire cell when you are grading! Thank you!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "742278ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Please run the entire cell when you are grading! Thank you!\n",
    "\n",
    "def get_neighbor(ID):\n",
    "    UniProt = ID[0]\n",
    "    pos = ID[1]\n",
    "    length = len(UniProt2Seq.loc[UniProt])\n",
    "    if pos in [1, 2]:\n",
    "        temp = UniProt2Seq.loc[UniProt][:5]\n",
    "    elif pos in [length, length-1]:\n",
    "        temp = UniProt2Seq.loc[UniProt][length-4:]\n",
    "    else:\n",
    "        temp = UniProt2Seq.loc[UniProt][pos-3:pos+2]\n",
    "    return list(temp.AA)\n",
    "\n",
    "\n",
    "def get_ave(result):\n",
    "    return Expasy_AA_Scales.loc[result].mean()\n",
    "\n",
    "\n",
    "def final_function(testset2):\n",
    "    import pandas as pd\n",
    "    from sklearn.metrics import roc_auc_score\n",
    "    import joblib\n",
    "    test = pd.read_csv(testset2, sep='\\t')\n",
    "    UniProt2Seq = pd.read_csv('UniProt2Seq.txt', sep='\\t')\n",
    "    Expasy_AA_Scales = pd.read_csv('Expasy_AA_Scales.txt', sep='\\t')\n",
    "    \n",
    "    UniProt2Seq.set_index(['UniProt', 'Position'], inplace=True)\n",
    "    index_to_drop = [UniProt2Seq.loc[UniProt2Seq['AA'] == 'U'].index, UniProt2Seq.loc[UniProt2Seq['AA'] == 'X'].index]\n",
    "    UniProt2Seq.drop([index_to_drop[0][0], index_to_drop[1][0]], axis=0, inplace=True)\n",
    "    test.set_index(['UniProt', 'Position'], inplace=True)\n",
    "    Expasy_AA_Scales.set_index('Amino Acid', inplace=True)\n",
    "    final_model = joblib.load(\"final_model.pkl\")\n",
    "    \n",
    "    test_avg = pd.DataFrame()\n",
    "    for i in test.index:\n",
    "        test_avg = test_avg.append(get_ave(get_neighbor(i)), ignore_index=True)\n",
    "    prob = final_model.predict_proba(test_avg[features])[:, 1]\n",
    "    auc_score = roc_auc_score(test.Label, prob)\n",
    "    return auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e914215",
   "metadata": {},
   "source": [
    "**Thank you very much for your reading and grading my final exam!**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}