scikit-learn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ***Multi Label Text Classification with Scikit-Learn*** <br/>\n",
    "https://towardsdatascience.com/multi-label-text-classification-with-scikit-learn-30714b7819c5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /home/anto/.local/lib/python3.8/site-packages (3.6.2)\n",
      "Requirement already satisfied: tqdm in /home/anto/.local/lib/python3.8/site-packages (from nltk) (4.54.0)\n",
      "Requirement already satisfied: click in /home/anto/.local/lib/python3.8/site-packages (from nltk) (7.1.2)\n",
      "Requirement already satisfied: regex in /home/anto/.local/lib/python3.8/site-packages (from nltk) (2021.8.21)\n",
      "Requirement already satisfied: joblib in /home/anto/.local/lib/python3.8/site-packages (from nltk) (0.17.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to /home/anto/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import re\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import accuracy_score, roc_auc_score, precision_score, recall_score, f1_score, classification_report\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "import seaborn as sns\n",
    "\n",
    "import warnings\n",
    "from sklearn.exceptions import UndefinedMetricWarning\n",
    "warnings.filterwarnings(action='ignore', category=UndefinedMetricWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000997932d777bf</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000103f0d9cfb60f</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000113f07ec002fd</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0001b41b1c6bb37e</td>\n",
       "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0001d958c54c6e35</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  toxic  \\\n",
       "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
       "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
       "2  000113f07ec002fd  Hey man, I'm really not trying to edit war. It...      0   \n",
       "3  0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...      0   \n",
       "4  0001d958c54c6e35  You, sir, are my hero. Any chance you remember...      0   \n",
       "\n",
       "   severe_toxic  obscene  threat  insult  identity_hate  \n",
       "0             0        0       0       0              0  \n",
       "1             0        0       0       0              0  \n",
       "2             0        0       0       0              0  \n",
       "3             0        0       0       0              0  \n",
       "4             0        0       0       0              0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/train.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Null values:  False\n",
      "Same columns between train and test:  False\n",
      "Number of rows original: 153164\n",
      "Number of rows after: 63978\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "      <th>one_hot_labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0001ea8717f6de06</td>\n",
       "      <td>Thank you for understanding. I think very high...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>000247e83dcc1211</td>\n",
       "      <td>:Dear god this site is horrible.</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0002f87b16116a7f</td>\n",
       "      <td>\"::: Somebody will invariably try to add Relig...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0003e1cccfd5a40a</td>\n",
       "      <td>\" \\n\\n It says it right there that it IS a typ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>00059ace3e3e9a53</td>\n",
       "      <td>\" \\n\\n == Before adding a new product to the l...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  id                                       comment_text  \\\n",
       "5   0001ea8717f6de06  Thank you for understanding. I think very high...   \n",
       "7   000247e83dcc1211                   :Dear god this site is horrible.   \n",
       "11  0002f87b16116a7f  \"::: Somebody will invariably try to add Relig...   \n",
       "13  0003e1cccfd5a40a  \" \\n\\n It says it right there that it IS a typ...   \n",
       "14  00059ace3e3e9a53  \" \\n\\n == Before adding a new product to the l...   \n",
       "\n",
       "    toxic  severe_toxic  obscene  threat  insult  identity_hate  \\\n",
       "5       0             0        0       0       0              0   \n",
       "7       0             0        0       0       0              0   \n",
       "11      0             0        0       0       0              0   \n",
       "13      0             0        0       0       0              0   \n",
       "14      0             0        0       0       0              0   \n",
       "\n",
       "        one_hot_labels  \n",
       "5   [0, 0, 0, 0, 0, 0]  \n",
       "7   [0, 0, 0, 0, 0, 0]  \n",
       "11  [0, 0, 0, 0, 0, 0]  \n",
       "13  [0, 0, 0, 0, 0, 0]  \n",
       "14  [0, 0, 0, 0, 0, 0]  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = pd.read_csv('data/test.csv')\n",
    "cols = df_test.columns\n",
    "label_cols = list(cols[2:])\n",
    "\n",
    "test_labels_df = pd.read_csv('data/test_labels.csv')\n",
    "df_test = df_test.merge(test_labels_df, on='id', how='left')\n",
    "test_label_cols = list(df_test.columns[2:])\n",
    "print('Null values: ', df_test.isnull().values.any()) #should not be any null sentences or labels\n",
    "print('Same columns between train and test: ', label_cols == test_label_cols) #columns should be the same\n",
    "print(f\"Number of rows original: {df_test.shape[0]}\")\n",
    "\n",
    "df_test = df_test[~df_test[test_label_cols].eq(-1).any(axis=1)] #remove irrelevant rows/comments with -1 values\n",
    "df_test['one_hot_labels'] = list(df_test[test_label_cols].values)\n",
    "print(f\"Number of rows after: {df_test.shape[0]}\")\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of comments in each category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        category  number_of_comments\n",
      "0          toxic               15294\n",
      "1   severe_toxic                1595\n",
      "2        obscene                8449\n",
      "3         threat                 478\n",
      "4         insult                7877\n",
      "5  identity_hate                1405\n",
      "\n",
      "- Percentage of comments that are not labelled: 0.8983211235124177\n",
      "- Number of missing comments in comment text: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anto/.local/lib/python3.8/site-packages/seaborn/_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explanation\n",
      "Why the edits made under my username Hardcore Metallica Fan were reverted? They weren't vandalisms, just closure on some GAs after I voted at New York Dolls FAC. And please don't remove the template from the talk page since I'm retired now.89.205.38.27\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAGHCAYAAADPzkp9AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2DUlEQVR4nO3deZxcVZn/8c+XhCUQMWw2kAQSFlEkoBA2Re2AQtiM44CAyCaSGUVBjUDEBYZlBAUZ+CloRpCwSEBUYFjECLSIyBa2sEoMgSTshEDCHnh+f5zTcNN0d27S1X2rKt/361Wvqjp3qefeqq56+tyzKCIwMzOzpdsyVQdgZmZm1XNCYGZmZk4IzMzMzAmBmZmZ4YTAzMzMcEJgZmZmOCEw6zFJ50o6oaLXlqTfSHpB0m1VxGBmzcEJgTUdSTMkPSNppULZVyW1VRhWb9kO+CwwJCK2qjqY3iZpmKSQ1L/qWGpFUqukWVXHYeaEwJpVP+DwqoNYXJL6LeYm6wIzIuLl3ojHOtcsCUmzHIfVhhMCa1Y/Bb4raVDHBZ39lympTdJX8+MDJf1d0mmS5kqaLunjuXxmrn04oMNuV5c0WdI8SX+VtG5h3x/Ky+ZIeljSFwvLzpV0lqSrJb0MjOok3rUlXZG3nybpkFx+MPBrYFtJ8yX9V2cnQtIhkh7MsT0gafNc/uF83HMl3S/pcx3iOlPSNXnff5e0pqT/yZcnHpL0scL6MyQdIeleSS9LOltSS95+nqS/SFqlsP42km7Or32PpNYO78Xx+TXnSfqzpNXz4hvz/dwc17aSNsjn/EVJz0m6uIvz0P6+j5X0hKQnJX23sHwZSeMl/UvS85IukbRqh20PlvQ4cH0XrzFG0t2SXsr7GZ3LDyq8B9Ml/UcuXwm4Blg7H8/8/H53GUvebn9Jj+VlP8zn/zN52fL5fXoi3/5H0vJ5WaukWZKOkvQU8BtJ90navbDvZfN5/Bi2dIkI33xrqhswA/gM8AfghFz2VaAtPx4GBNC/sE0b8NX8+EBgAXAQqabhBOBx4BfA8sCOwDxgYF7/3Pz8U3n56cBNedlKwMy8r/7Ax4DngI0L274IfIKUoK/QyfHcCJwJrAB8FHgW2L4Q603dnIs9gdnAloCADUi1CssC04CjgeWA7fMxbFSI6zlgi/y61wOPAvsXzskNHc75LUALMBh4BrgzH2/79sfkdQcDzwO75GP+bH6+RuG9+BfwQWBAfn5SN+/dRcD3288fsF0X56J924vy+zIin8vP5OWH52MYkt/HXwEXddj2vLztgE72v1V+Lz+bYxkMfCgv2xVYP78HnwZeATbPy1qBWR321V0sGwPzSZeLlgNOAd4sHMdxedsPAGsANwPHF15rAXBy3u8A4Ejg4sJrjwGmVv137Fvf3yoPwDffan3j3YRgk/wFvQaLnxA8Ulg2Iq/fUih7HvhofnwuMKmwbCDwFjAU2Av4W4f4fsW7P47nAud1cyxD877eVyj7MXBuIdbuEoJrgcM7Kf8k8BSwTKHsIuDYQlz/W1j2TeDBDudkbodzvm/h+e+Bszpsf1l+fBRwfidxHlB4L35QWPZ14E/dvHfnARNI7Si6+1y0b/uhQtlPgLPz4weBHQrL1iL90PYvbLteN/v/FXBayc/oZe3vC50nBN3F8iNycpCXrQi8wbsJwb+AXQrLdyJdVmp/rTcoJJ7A2qRkcOX8/FLgyN7+O/Wt/m6+ZGBNKyLuA64Exi/B5k8XHr+a99exbGDh+czC684H5pC+aNcFts5V43MlzQX2BdbsbNtOrA3MiYh5hbLHSP99ljGU9APR2X5nRsTb3ey34/F2d/yLs/66wJ4dzsl2pB+9dk8VHr/SyWsVHUn6z/u2fOnjK92sCwuf78dI56I9rj8WYnqQlIy1dLFtR12dayTtLOmWfNlnLql2ZPXO1i0Ry9os/Hl7hZSgtls7H1dnxwjwbES8Vtj+CeDvwL8rXWLbGbiwm9isSblBiTW7Y0hV16cWytob4K0IvJQfF3+gl8TQ9geSBgKrAk+Qvrj/GhGf7Wbb7qYcfQJYVdL7CknBOqTLAGXMJFVVd7bfoZKWKSQF6wD/LLnfnphJqiE4ZAm2fc+5ioingPZ2FdsBf5F0Y0RM62IfQ4GH8uN1SOeiPa6vRMTfO24gaVhXr1/Q6bnO1+9/T7rccnlEvCnpMlIS09U+u4vlSWCjwvMBwGqFVZ4gJRT35+fFY+zq9SaSatH6A/+IiLKfL2siriGwppZ/FC4GDiuUPUv6Qf2ypH75P8rOfjQXxy6StpO0HHA8cEtEzCTVUHxQ0n65sdaykraU9OGS8c8kXQP+saQVJG0KHAxcUDKuX5MaV26hZAOlBo+3kv7zPjLH1ArsDkxajGNeUhcAu0vaKZ//FXJjtyEltn0WeBtYr71A0p6FbV8g/eC93cm27X4oaUVJHyG17WhvhPhL4MR8fpC0hqQxi3FcZwMHSdohNwocLOlDpOv8y+fYF0jamdQOpd3TwGqS3l8o6y6WS0nn7+P583Ys7yYXkC79/CBvszrpEsOiPi+XAZuT2i6ctxjHbE3ECYEtDY4jNQQrOgQ4glTV+hHSj25P/JZUGzGH1BDvywD5v/odgb1J/6U9xbsNusrah3QN+wngj6T2B38ps2FE/A44Mcc3j/TFv2pEvEFKAHYmNR48E9g/Ih7qYlc1k5OcMaQGjc+S/hs+ghLfR7l6/ETg77k6fRtSg8lbJc0HriBdm5/ezW7+SmpQeR1wSkT8OZefnrf/s6R5pIZ5Wy/Gcd1GSjBOI7Vd+Suwbv4MHAZcQkpYvpRfp327h0g/4tPzMa3dXSwRcT+pTcYk4ElSA8NngNfzLk8A7gDuBaaSasi6HTgrIl4l1WIMJzXGtaWQIrqrATMzaw652v9RYNmIWFBxODWTL1HNBTaMiEd7sJ8fAR+MiC/XKjZrLK4hMDNrMJJ2z5c9ViJ1O5xK6umxpPtblXQpakJtIrRG5ITAzKzxjCFdQnoC2BDYO5awuldpoKuZwDURceOi1rfm1SeXDCSdA+wGPBMRmxTKvwkcSupOc1VEHJnLv0fKVt8CDouIa3P5aNK1tX7AryPipFw+nHQ9bTVgCrBfvkZqZmZmJfRVDcG5wOhigaRRpCx3s4j4CKnaC0kbkxpgfSRvc2ZuidyPNFLczqSRuvbJ60JqpHVaRGxAarRzcK8fkZmZWRPpk3EIIuLGQj/edl8jDUf6el7nmVw+hjTq2+vAo5KmkYYEBZjW3npY0iRgjKQHScOufimvM5HUDeesRcW1+uqrx7BhHcOq1ssvv8xKK3VsEG8d+TyV53NVjs9TeT5X5dTjeZoyZcpzEbFGZ8uqHJjog8AnJZ0IvAZ8NyJuJ42UdkthvVm8O3razA7lW5MuE8wttBourv8eksYCYwFaWlo45ZRTanAotTN//nwGDuxuUDYDn6fF4XNVjs9TeT5X5dTjeRo1atRjXS2rMiHoTxrNrb0f8SWS1ut+k56LiAnklrQjR46M1tbW3n7JxdLW1ka9xVSPfJ7K87kqx+epPJ+rchrtPFWZEMwC/pBbxt4m6W3S2N6zKQwDS5rtq30Yzc7KnwcGSeqfawmK65uZmVkJVXY7vIw897ukD5KG93yONDrX3nlO7+GkLjW3AbcDG0oanofr3Bu4IicUNwB75P0eAFzelwdiZmbW6PqkhkDSRaRpN1eXNIs0xOs5wDmS7iNNx3lA/nG/X9IlwAOkebsPjYi38n6+QZomtR9wTh7CE9J0qpMknQDcRRpT3MzMzErqq14G+3SxqNMhMiPiRNJ45R3Lrwau7qR8Ou/2RDAzM7PF5JEKzczMzAmBmZmZOSEwMzMznBCYmZkZTgjMzMwMJwRmZmZGtSMVNo1h46+q2b7GjVjAgTXa34yTdq3JfszMrPm5hsDMzMycEJiZmZkTAjMzM8MJgZmZmeGEwMzMzHBCYGZmZjghMDMzM5wQmJmZGU4IzMzMDCcEZmZmhhMCMzMzwwmBmZmZ4YTAzMzMcEJgZmZmOCEwMzMznBCYmZkZTgjMzMwMJwRmZmZGHyUEks6R9Iyk+zpZNk5SSFo9P5ekMyRNk3SvpM0L6x4g6ZF8O6BQvoWkqXmbMySpL47LzMysWfRVDcG5wOiOhZKGAjsCjxeKdwY2zLexwFl53VWBY4Ctga2AYyStkrc5CziksN17XsvMzMy61icJQUTcCMzpZNFpwJFAFMrGAOdFcgswSNJawE7A5IiYExEvAJOB0XnZyhFxS0QEcB7w+V48HDMzs6bTv6oXljQGmB0R93So4R8MzCw8n5XLuiuf1Ul5V687llTzQEtLC21tbUt+ENm4EQt6vI92LQNqt79aHFu9mj9/flMfXy35XJXj81Sez1U5jXaeKkkIJK0IHE26XNCnImICMAFg5MiR0dra2uN9Hjj+qh7vo924EQs4dWpt3pYZ+7bWZD/1qK2tjVq8d0sDn6tyfJ7K87kqp9HOU1W9DNYHhgP3SJoBDAHulLQmMBsYWlh3SC7rrnxIJ+VmZmZWUiUJQURMjYgPRMSwiBhGqubfPCKeAq4A9s+9DbYBXoyIJ4FrgR0lrZIbE+4IXJuXvSRpm9y7YH/g8iqOy8zMrFH1VbfDi4B/ABtJmiXp4G5WvxqYDkwD/hf4OkBEzAGOB27Pt+NyGXmdX+dt/gVc0xvHYWZm1qz6pA1BROyziOXDCo8DOLSL9c4Bzumk/A5gk55FaWZmtvTySIVmZmbmhMDMzMycEJiZmRlOCMzMzAwnBGZmZoYTAjMzM8MJgZmZmeGEwMzMzHBCYGZmZjghMDMzM5wQmJmZGU4IzMzMDCcEZmZmhhMCMzMzwwmBmZmZ4YTAzMzMcEJgZmZmOCEwMzMznBCYmZkZTgjMzMwMJwRmZmaGEwIzMzPDCYGZmZnhhMDMzMxwQmBmZmb0UUIg6RxJz0i6r1D2U0kPSbpX0h8lDSos+56kaZIelrRToXx0LpsmaXyhfLikW3P5xZKW64vjMjMzaxZ9VUNwLjC6Q9lkYJOI2BT4J/A9AEkbA3sDH8nbnCmpn6R+wC+AnYGNgX3yugAnA6dFxAbAC8DBvXs4ZmZmzaVPEoKIuBGY06HszxGxID+9BRiSH48BJkXE6xHxKDAN2CrfpkXE9Ih4A5gEjJEkYHvg0rz9RODzvXk8ZmZmzaZ/1QFkXwEuzo8HkxKEdrNyGcDMDuVbA6sBcwvJRXH995A0FhgL0NLSQltbW09jZ9yIBYteqaSWAbXbXy2OrV7Nnz+/qY+vlnyuyvF5Ks/nqpxGO0+VJwSSvg8sAC7si9eLiAnABICRI0dGa2trj/d54PireryPduNGLODUqbV5W2bs21qT/dSjtrY2avHeLQ18rsrxeSrP56qcRjtPlSYEkg4EdgN2iIjIxbOBoYXVhuQyuih/HhgkqX+uJSiub2ZmZiVU1u1Q0mjgSOBzEfFKYdEVwN6Slpc0HNgQuA24Hdgw9yhYjtTw8IqcSNwA7JG3PwC4vK+Ow8zMrBn0VbfDi4B/ABtJmiXpYODnwPuAyZLulvRLgIi4H7gEeAD4E3BoRLyV//v/BnAt8CBwSV4X4CjgO5KmkdoUnN0Xx2VmZtYs+uSSQUTs00lxlz/aEXEicGIn5VcDV3dSPp3UC8HMzMyWgEcqNDMzMycEZmZm5oTAzMzMcEJgZmZmOCEwMzMznBCYmZkZTgjMzMwMJwRmZmaGEwIzMzPDCYGZmZnhhMDMzMxYwoRA0ihJn651MGZmZlaNUgmBpL9K+kR+fBQwCfitpKN7MzgzMzPrG2VrCDYBbsmPDwFGAdsA/9kbQZmZmVnfKjv98TJASFofUEQ8ACBplV6LzMzMzPpM2YTgJuDnwFrAHwFycvBcL8VlZmZmfajsJYMDgbnAvcCxuexDwOk1j8jMzMz6XKkagoh4Hji6Q9lVvRKRmZmZ9bmyvQyWl3SipOmSXsxlO0r6Ru+GZ2ZmZn2h7CWD00g9DfYFIpfdD3ytN4IyMzOzvlW2UeG/ARtExMuS3gaIiNmSBvdeaGZmZtZXytYQvEGH5EHSGsDzNY/IzMzM+lzZhOB3wERJwwEkrUXqhjiptwIzMzOzvlM2ITgaeBSYCgwCHgGeAI7rnbDMzMysL5XtdvgG8G3g2/lSwXMREYvYzMzMzBpE2W6H+0vaFCAino2IkLSZpP1Kbn+OpGck3VcoW1XSZEmP5PtVcrkknSFpmqR7JW1e2OaAvP4jkg4olG8haWre5gxJKnsCzMzMrPwlg+OBmR3KZgInlNz+XGB0h7LxwHURsSFwXX4OsDOwYb6NBc6ClEAAxwBbA1sBxxTmUjiLNOlS+3YdX8vMzMy6UTYhWBl4qUPZi6T2BIsUETcCczoUjwEm5scTgc8Xys+L5BZgUG7EuBMwOSLmRMQLwGRgdF62ckTcki9jnFfYl5mZmZVQdhyCB4B/By4plP0b8GAPXrslIp7Mj58CWvLjwSxcGzErl3VXPquT8k5JGkuqeaClpYW2trYlP4Js3IgFPd5Hu5YBtdtfLY6tXs2fP7+pj6+WfK7K8Xkqz+eqnEY7T2UTgqOAqyXtBfwL2ADYAdilFkHkNgl90kgxIiYAEwBGjhwZra2tPd7ngeNrN63DuBELOHVq2belezP2ba3JfupRW1sbtXjvlgY+V+X4PJXnc1VOo52nUpcMIuImYARwO7AScBuwSUT8vQev/XSu7m8f1+CZXD4bGFpYb0gu6658SCflZmZmVlLZNgRExGMRcVJEHJrvOzYyXFxXAO09BQ4ALi+U7597G2wDvJgvLVwL7ChpldyYcEfg2rzsJUnb5N4F+xf2ZWZmZiWUqpvOLfy/C3wUGFhcFhGfKrH9RUArsLqkWaTeAicBl0g6GHgM+GJe/WrSpYhpwCvAQfl15kg6nlRLAXBcRLQ3VPw6qSfDAOCafDMzM7OSyl6s/i2wPKlR4SuL+yIRsU8Xi3boZN0ADu1iP+cA53RSfgdpNkYzMzNbAmUTgo8Da0TE670ZjJmZmVWjbBuCe1m44Z6ZmZk1kbI1BNcDf5L0G9KYAe/I1fhmZmbWwMomBJ8kDfjz2Q7lQSfX9M3MzKyxlJ3tcFRvB2JmZmbVKT0OgaTVJO0n6Yj8fG1JbldgZmbWBMpOf/xp4GFgX+BHuXhD8kyEZmZm1tjK1hD8D7BXRIwG2mfeuZU0DbGZmZk1uLIJwbCIuC4/bp+E6A3KN0o0MzOzOlY2IXhA0k4dyj4DTK1xPGZmZlaBsv/hjwOulHQVMEDSr4DdgTG9FpmZmZn1mbI1BLcBmwL3k8YdeBTYKiJu73YrMzMzawiLrCGQ1A+YDwyKiJ/0fkhmZmbW1xZZQxARbwH/BFbr/XDMzMysCmXbEFxIakNwOmkI4/aeBkTE9b0RmJmZmfWdsgnB1/L9sR3KA1ivZtGYmZlZJcq0IVgG+CpwU0S83vshmZmZWV8r04bgbeAyJwNmZmbNq2y3wxslbdOrkZiZmVllyrYheAy4RtLlwEwWblT4oy63MjMzs4ZQNiEYAFyWH3vKY7NeNmz8VTXb17gRCziwBvubcdKuNYjGzOpVqYQgIg7q7UDMzMysOqUSAklddi2MiOm1C8fMzMyqUPaSwTRSuwEVytrbEfSraURmZmbW58peMlioN4KkNYFjgL/1RlBmZmbWt8p2O1xIRDwFfAv4cU8DkPRtSfdLuk/SRZJWkDRc0q2Spkm6WNJyed3l8/Npefmwwn6+l8sflrRTT+MyMzNbmixRQpBtBKzYkxeXNBg4DBgZEZuQLj/sDZwMnBYRGwAvAAfnTQ4GXsjlp+X1kLRx3u4jwGjgzDxLo5mZmZVQKiGQ9DdJNxZudwC3Aj+rQQz9gQGS+pMSjCeB7YFL8/KJwOfz4zH5OXn5DpKUyydFxOsR8SipzcNWNYjNzMxsqVC2UeGvOzx/GbgnIh7pyYtHxGxJpwCPA68CfwamAHMjYkFebRYwOD8eTBoYiYhYIOlF0rTMg4FbCrsubrMQSWOBsQAtLS20tbX15BCA1M+7VloG1G5/tTi2ejV//vymPr56/Ew18/mG5v9M1ZLPVTmNdp7KNiqcuOi1Fp+kVUj/3Q8H5gK/I1X595qImABMABg5cmS0trb2eJ+1GPSl3bgRCzh1atk8rXsz9m2tyX7qUVtbG7V47+pVPX6mmvnzBM3/maoln6tyGu08lR2H4A+ka/p/K5R9Ejg8Ivbowet/Bng0Ip4tvM4ngEGS+udagiHA7Lz+bGAoMCtfYng/8HyhvF1xGzOzpZ5Hv7RFKduo8NPAzR3K/gGM6uHrPw5sI2nF3BZgB+AB4AagPdE4ALg8P74iPycvvz4iIpfvnXshDAc2BG7rYWxmZmZLjbL1iK8BKwEvFcoGAm/25MUj4lZJlwJ3AguAu0jV+VcBkySdkMvOzpucDZwvaRowh9SzgIi4X9IlpGRiAXBoRLzVk9jMzMyWJmUTgmuBX0n6j4h4SdLKwM+BP/U0gIg4hjTIUdF0OuklEBGvAXt2sZ8TgRN7Go+ZmdnSqOwlg3HAysAcSc+Q/jt/P2lwIjMzM2twZXsZvADsmocsHgrMzKMVmpmZWRMo28tgR2BGRPwTeCqXbQSsExGTezE+MzMz6wNlLxn8ApjXoWxeLjczM7MGVzYh+EBEPNmh7ElgzRrHY2ZmZhUomxBMl7R9h7JW4NHahmNmZmZVKNvt8FjgD5LOBv4FrA8clG9mZmbW4ErVEETE5cCOpMGJds33O+VyMzMza3ClZzyJiNvwcMBmZmZNaZE1BJKGSTpX0mxJr+f7iZLW64sAzczMrPd1mxBI+jBpnoEPAN8HPpfv1wDuyMvNzMyswS3qksFJwC8i4ocdys/NEw/9BNi9VyIzMzOzPrOohOBTvDvdcEen4m6HZmZmTWFRbQj60fUUx2/m5WZmZtbgFpUQ3E7XYw0cCNxR02jMzMysEou6ZPBD4No8kdGlpOGK1wL2JF1K2Kl3wzMzM7O+0G0NQUTcTBqQaDPgOuChfL8ZMDovNzMzswa3yIGJIuIfwKckDQBWBV6IiFd6PTIzMzPrM4szUuGrwOxejMXMzMwqUna2QzMzM2tiTgjMzMys64RA0k8Lj7fvm3DMzMysCt3VEIwtPL6sl+MwMzOzCnXXqPAeSZcCDwDLSzqus5Ui4ke9EpmZmZn1me5qCPYA7iYNRCRgaCe3IT0NQNIgSZdKekjSg5K2lbSqpMmSHsn3q+R1JekMSdMk3Stp88J+DsjrPyKpq/kXzMzMrBNd1hBExDPACQCS+kdEV0MY99TpwJ8iYg9JywErAkcD10XESZLGA+OBo4CdgQ3zbWvgLGBrSasCxwAjgQCmSLoiIl7opZjNzMyaSqleBhFxkKRVJO0v6Xv5ftWevrik95NmVDw7v84bETEXGANMzKtNBD6fH48BzovkFmCQpLVIQyhPjog5OQmYDIzuaXxmZmZLC0XEoleStgWuIg1d/BiwDvBhYNc8kuGSvbj0UWACqZ3CZsAU4HBgdkQMyuuINDriIElXAidFxE152XWkmoNWYIWIaK/R+CHwakSc0slrjiU3mGxpadli0qRJSxr+O6bOfrHH+2jXMgCefrU2+xox+P212VEdmj9/PgMHDqw6jF5Tj5+pZv48gT9Ti8OfqXLq8TM1atSoKRExsrNlZUcq/B/g6xHxzq+npL2AM4AtexBbf2Bz4JsRcauk00mXB94RESFp0VlLSRExgZSEMHLkyGhtbe3xPg8cf1WP99Fu3IgFnDq19ACS3Zqxb2tN9lOP2traqMV7V6/q8TPVzJ8n8GdqcfgzVU6jfabKDkz0QeCSDmWXAhv08PVnAbMi4tbCPjcHns6XAsj3z+Tls0mNGdsNyWVdlZuZmVkJZROCR4C9O5TtCfyrJy8eEU8BM/P0ygA7kC4fXEGaXpl8f3l+fAWwf+5tsA3wYkQ8CVwL7JjbOaxCmqHx2p7EZmZmtjQpW+fzLeBKSYeR2hAMI7X0360GMXwTuDD3MJgOHERKVC6RdHB+vS/mda8GdgGmAa/kdYmIOZKOB27P6x0XEXNqEJuZmdlSoVRCEBE3S1of2BVYG/g/4Opa/OhGxN2k7oId7dDJugEc2sV+zgHO6Wk8ZmZmS6PFmf74BeCCXozFzMzMKuLZDs3MzMwJgZmZmTkhMDMzMxYjIZC0bm8GYmZmZtVZnBqCuwBy10MzMzNrIt32MpA0hTS/wF1Av1x8LGnIYjMzM2sSi6oh2AP4M7AusKKkO4HlJY3KMxWamZlZE1hUQtAvIi6NiPHAPNL0wyKNLni3pEd6O0AzMzPrfYsamOhCSeuQ5hdYAVgFeC0ivgAgadVejs/MzMz6QLcJQURsLak/MAK4Cfg58D5JZwF35pvnDDAzM2twi+xlEBELIuIu4I2I+BTwMtBGmtzo5N4Nz8zMzPpC6bkMgG/n+4iIi4GLeyEeMzMzq0DpcQgi4tz8cL3eCcXMzMyqsthDF+dZD83MzKyJeC4DMzMzc0JgZmZmTgjMzMwMJwRmZmaGEwIzMzPDCYGZmZnhhMDMzMxwQmBmZmY4ITAzMzOcEJiZmRl1khBI6ifpLklX5ufDJd0qaZqkiyUtl8uXz8+n5eXDCvv4Xi5/WNJOFR2KmZlZQ6qLhAA4HHiw8Pxk4LSI2AB4ATg4lx8MvJDLT8vrIWljYG/gI8Bo4ExJ/foodjMzs4ZXeUIgaQiwK/Dr/FzA9sCleZWJwOfz4zH5OXn5Dnn9McCkiHg9Ih4FpgFb9ckBmJmZNQFFRLUBSJcCPwbeB3wXOBC4JdcCIGkocE1EbCLpPmB0RMzKy/4FbA0cm7e5IJefnbe5tMPLIWksMBagpaVli0mTJvX4GKbOfrHH+2jXMgCefrU2+xox+P212VEdmj9/PgMHDqw6jF5Tj5+pZv48gT9Ti8OfqXLq8TM1atSoKRExsrNl/fs6mCJJuwHPRMQUSa198ZoRMQGYADBy5Mhobe35yx44/qoe76PduBELOHVqbd6WGfu21mQ/9aitrY1avHf1qh4/U838eQJ/phaHP1PlNNpnqtKEAPgE8DlJuwArACsDpwODJPWPiAXAEGB2Xn82MBSYJak/8H7g+UJ5u+I2ZmZmtgiVtiGIiO9FxJCIGEZqFHh9ROwL3ADskVc7ALg8P74iPycvvz7SNY8rgL1zL4ThwIbAbX10GGZmZg2v6hqCrhwFTJJ0AnAXcHYuPxs4X9I0YA4piSAi7pd0CfAAsAA4NCLe6vuwzczMGlPdJAQR0Qa05cfT6aSXQES8BuzZxfYnAif2XoRmZmbNq/Juh2ZmZlY9JwRmZmbmhMDMzMycEJiZmRlOCMzMzAwnBGZmZoYTAjMzM8MJgZmZmeGEwMzMzHBCYGZmZjghMDMzM5wQmJmZGU4IzMzMDCcEZmZmhhMCMzMzwwmBmZmZ4YTAzMzMcEJgZmZmOCEwMzMznBCYmZkZTgjMzMwMJwRmZmaGEwIzMzPDCYGZmZnhhMDMzMyoOCGQNFTSDZIekHS/pMNz+aqSJkt6JN+vkssl6QxJ0yTdK2nzwr4OyOs/IumAqo7JzMysEVVdQ7AAGBcRGwPbAIdK2hgYD1wXERsC1+XnADsDG+bbWOAsSAkEcAywNbAVcEx7EmFmZmaLVmlCEBFPRsSd+fE84EFgMDAGmJhXmwh8Pj8eA5wXyS3AIElrATsBkyNiTkS8AEwGRvfdkZiZmTU2RUTVMQAgaRhwI7AJ8HhEDMrlAl6IiEGSrgROioib8rLrgKOAVmCFiDghl/8QeDUiTunkdcaSahdoaWnZYtKkST2OfersF3u8j3YtA+DpV2uzrxGD31+bHdWh+fPnM3DgwKrD6DX1+Jlq5s8T+DO1OPyZKqceP1OjRo2aEhEjO1vWv6+D6YykgcDvgW9FxEspB0giIiTVLGuJiAnABICRI0dGa2trj/d54PireryPduNGLODUqbV5W2bs21qT/dSjtrY2avHe1at6/Ew18+cJ/JlaHP5MldNon6mq2xAgaVlSMnBhRPwhFz+dLwWQ75/J5bOBoYXNh+SyrsrNzMyshKp7GQg4G3gwIn5WWHQF0N5T4ADg8kL5/rm3wTbAixHxJHAtsKOkVXJjwh1zmZmZmZVQ9SWDTwD7AVMl3Z3LjgZOAi6RdDDwGPDFvOxqYBdgGvAKcBBARMyRdDxwe17vuIiY0ydHYGZm1gQqTQhy40B1sXiHTtYP4NAu9nUOcE7tojMzM1t6VN6GwMzMzKpX9SUDW8oMq1FL53EjFtSs1fSMk3atyX7MzBqZawjMzMzMCYGZmZn5koGZmdk7anVZE2p3abOvLmu6hsDMzMycEJiZmZkTAjMzM8MJgZmZmeGEwMzMzHBCYGZmZjghMDMzM5wQmJmZGU4IzMzMDCcEZmZmhhMCMzMzwwmBmZmZ4YTAzMzMcEJgZmZmOCEwMzMzoH/VAZiZ9USt5q+v1dz10Hfz15vVkmsIzMzMzAmBmZmZOSEwMzMznBCYmZkZTZYQSBot6WFJ0ySNrzoeMzOzRtE0CYGkfsAvgJ2BjYF9JG1cbVRmZmaNoWkSAmArYFpETI+IN4BJwJiKYzIzM2sIioiqY6gJSXsAoyPiq/n5fsDWEfGNDuuNBcbmpxsBD/dpoIu2OvBc1UE0AJ+n8nyuyvF5Ks/nqpx6PE/rRsQanS1Y6gYmiogJwISq4+iKpDsiYmTVcdQ7n6fyfK7K8Xkqz+eqnEY7T810yWA2MLTwfEguMzMzs0VopoTgdmBDScMlLQfsDVxRcUxmZmYNoWkuGUTEAknfAK4F+gHnRMT9FYe1JOr2ckad8Xkqz+eqHJ+n8nyuymmo89Q0jQrNzMxsyTXTJQMzMzNbQk4IzMzMzAmBmZmZOSEws6WUpOXLlJktLZwQVCx3k1yh8HyApGEVhlS3JB0qaVDh+SqSvl5hSHVN0opVx1Dn/lGyzABJe5Yps3e+xzeqOo7F5YSger8D3i48fyuX2XsdEhFz259ExAvAIdWFU58kfVzSA8BD+flmks6sOKy6IWlNSVsAAyR9TNLm+dYKOInq2vdKli3VJO0O3A38KT//qKSGGBOnacYhaGD982RMAETEG3lgJXuvfpIUua9snuHS5+q9TgN2Ig/MFRH3SPpUtSHVlZ2AA0mjmf6sUD4POLqKgOqZpJ2BXYDBks4oLFoZWFBNVHXtWNJke20AEXG3pOFVBlSWE4LqPSvpcxFxBYCkMdTfZBj14k/AxZJ+lZ//Ry6zDiJipqRi0VtVxVJvImIiMFHSv0fE76uOpwE8AUwBPpfv280Dvl1JRPXtzYh4scPfX0MM+OOEoHr/CVwo6eeAgJnA/tWGVLeOIiUBX8vPJwO/ri6cujVT0seBkLQscDjwYMUx1Z2I+L2kXYGPACsUyo+rLqr6ExH3APdIuiAiXCOwaPdL+hKpRnND4DDg5opjKsUjFdYJSQMBImJ+1bFYY5O0OnA68BlSkvln4PCIeL7SwOqMpF+S2gyMIiWWewC3RcTBlQZWZyRNpZv/cCNi0z4Mp+7lxrzfB3bMRdcCx0fE69VFVY4TgopI+nJEXCDpO50tj4ifdVa+NJJ0SUR8sasvJn8h2ZKQdG9EbFq4HwhcExGfrDq2eiJp3e6WR8RjfRVLI5C0Z0T8blFl9ciXDKqzUr5/X6VRNIbD8/1ulUbRICStQep9MYzC33hEfKWqmOrUq/n+FUlrA88Da1UYT13yD/5i+x7v7SnWWVndcUJQkYj4Vb7/r47L3MtgYRHxZH64UkQ8UFyWu4r5C2thlwN/A/6CGxN258o8rsVPgTtJtU9uk9IFSfN4t4ZuOWBZ4OWIWLm6qOpHM/TG8CWDiklqAw6MiBn5+ZbAryNisyrjqkeS7gPOB35CagT2E2BkRGxbaWB1RtLdEfHRquNoJHmEwhUi4sWqY2kESk3oxwDbRMT4quOpB5I2Az4KHAf8qLBoHnBDHjelrjkhqJiknUgNwM4ABgM7A1+NiDsrDawOSVoJOBnYgnSp5ULg5Ih4u9sNlzKSTgBujoirq46lnuXGX+OAdSLikNwifKOIuLLi0BqGpLsi4mNVx1FPJC0bEW9WHceS8CWDikXEtZL+k9SF7jngYxHxVMVh1as3Sdd9B5BqCB51MtCpw4GjJb0BvEHqaRCu2n2P35D61bfXMM0mXed1QtAJSV8oPF0GGAm8VlE49WyYpB8DG7Nwd9b1qgupHCcEFZP0Q+CLwKeATYE2SeMi4qpqI6tLt5Ouj28JrA78Mg8u4/HUCyLCDVXLWT8i9pK0D0BEvKIOo8nYQnYvPF4AzCBdNrCF/QY4hjRi6CjgIBpkmgAnBNVbDdgqIl4F/iHpT6SGTU4I3uvgiLgjP34SGCNpvyoDqkf5R21fYHhEHC9pKLBWRNxWcWj15g1JA8gN5SStD9R9X/GqRMRBVcfQIAZExHV5mPXHgGMlTWHhdgV1qSGylmYWEd8CVpa0m6TdgFcj4rMVh1Wv7pF0mKRL8+0bwKSqg6pDZ5Kqwb+Un88HflFdOHXrGNLQ10MlXQhcBxxZbUj1S9JPJK0saVlJ10l6VtKXq46rDr0uaRngEUnfkPRvwMCqgyrDCUHF8vShtwF7ki4d3Cppj2qjqltnkRoUnplv7Y9tYVtHxKHk67u5dbO7shbkL+xVgC+QJjq6iNRjpa3CsOrdjhHxEmk8kBnABsARlUZUnw4njYB5GOk7aj/ggEojKsmXDKr3A2DLiHgG3hlU5i/ApZVGVZ+27NAd83pJ91QWTf16M88E2V4VvgYLT7G91IuItyUdGRGX4MtzZbX/XuwK/K6TCXwMiIjb88P5pPYDDcMJQfWWaU8GsudxzU1X3pK0fkT8C0DSenjgnc6cAfwR+ICkE0lj9P+g2pDq0l8kfRe4GHi5vTAi5lQXUl27UtJDpJ4+X8uJpnsZdCDpg6Sak3VZeKTQ7SsLqiSPQ1AxST8BNiNVWQLsBdwbEUdVF1V9krQDqQXvdFJXunWBr0TE9ZUGVockfQjYgXSerosIz3bYgaRHOymORugeVhVJqwIvRsRbeRyHld1NemG51vKXpC6t7/zDEhFTutyoTjghqJikk4Fbge1y0d9Io385IeggjyYHsFG+fxigEWYR60uStgHuj4h5+fnKwIcj4tZqI7NGl6fVHsbC//meV1lAdUjSlIjYouo4loQTgopJujMiNu9Qdq9n8HuvLs7Ve8qWdpLuAjaP/MedG9Dd4fP0Xv6BK0/S+cD6wN28+59vRMRhlQVVR3LtCaTGhM+QLtu9889KI1yKchuCikj6GvB1YD1J9xYWvQ/4ezVR1SdJa5KGdR4g6WOkanBIk4asWFlg9UtRyPRzAzr/rXfQ1Q8c4ISgcyOBjcP/RXZlCunz0/79VOyBEUDdX4ryl0R1fgtcA/wYKE4OMq8RMsk+thOpa9gQ4FTe/YN7CTi6opjq2XRJh5G6aUJKPKdXGE+98g/c4rkPWJM0KJh1EBHDy6wn6bMRMbm341kSvmRgDSMPU/z7bpYfEBET+zKmeiTpA6SeBtuT/jO5DvhWh94sSz1JvwMOK0yvbd2QdANpNr/bWLgq/HNVxdSI6vkypxMCaxr1/Idm9UPS/5ESpffhH7jSJH26s/KI+Gtfx9LI6nmGSF8ysGbiUVJ4pyvrCaT+4n8iTZr17Yi4oNLA6scppM/KycDnC+XtZdYJ//DXTN3+F+6EwJpJ3f6h9bEdI+LIPIb6DNLwvDcCTgh494ctz1u/0I9cnuzICiTdFBHbSZrHwn9jnla7yTghsGbiGoLEQ8x2wz18Fk9EbJfvPa12bcyoOoCuOCGwZuIv88RDzHbPPXys1+Spjs8BfpsnFltIRHyh76Mqx40KrWFIagH+G1g7InaWtDGwbUScXXFodcdDzJpVQ9IGpEmN9gLuIA23/udG6N7qhMAahqRrSH9c34+IzfJgO3dFxIiKQ6srklYgVYlvR7rmexNwVkS4lsCsj+QRQncjjQfyFum76/R6roXyrHrWSFbP09W+DRARC/Bsh505D/gI8P+AnwMbA+dXGpHZUkTSpqRB1H4K/B7YkzSQWl1PxOY2BNZIXpa0Grmlc57E58VqQ6pLm0TExoXnN0h6oLJozJYiuQ3BXOBsYHxh8rVbJX2issBKcEJgjeQ7wBXA+pL+DqwB7FFtSHXpTknbRMQtAJK2Jl3LNLPet2dELDRUuKThEfFoPTcoBCcE1iAk9QM+nW8bkboYPhwRb1YaWB2RNJVUe7IscLOkx/PzdYGHqozNbClyKdBxxNRLgbqfEtkJgTWE3Fp+n4g4Dbi/6njq1G6Fx6sAn8yPbyRVYZpZL5H0IVLbnfdLKtYErAysUE1Ui8cJgTWSv0v6OXAx8HJ7YUTcWV1I9SMiHgOQdDjwVeAPpJqU84H/JTUyNLPesREpKR8E7F4onwccUkVAi8vdDq1h5NnWOoqI2L7Pg6ljefS9bSPi5fx8JeAfEbFptZGZNT9J20bEP6qOY0m4hsAaRkSMqjqGBiEW7o75Fh7W2axXSToyIn4CfEnSPh2XR8RhFYS1WJwQWMPwSIWl/YbUxemP+fnnSV2gzKz3PJjvG7ZHjy8ZWMPwSIXlSdqcNFIhwN8i4q4q4zFbWkjaMyJ+t6iyeuSEwBqGpNsjYktJd0XEx3LZ3RHx0YpDMzMDQNKdEbH5osrqkS8ZWCPxSIVmVpck7QzsAgyWdEZh0crAgmqiWjxOCKyRjMMjFZpZfXqC1H7gc8CUQvk84NuVRLSYfMnAGkpuN+CRCs2sLklatlG/lzzboTWM3L/+SOC1iLivUf/ozKypbSVpsqR/Spou6VFJ0xe9WfVcQ2ANQ9K6wF759jZpxMJLIuLxSgMzM8skPUS6RDCFwnggEfF8ZUGV5ITAGpKkDYEfAvtGRL+q4zEzA5B0a0RsXXUcS8KNCq2hdKgleIt0CcHMrF7cIOmnpLlEXm8vbIQ5V1xDYA1D0q2kqX1/B1zccc5xM7OqNfKcK04IrGFI2igiHq46DjOzZuReBtZI5ko6Ow9hjKSNJR1cdVBmZu0ktTTq95QTAmsk5wLXAmvn5/8EvlVVMGZmnTiXBv2eckJgjWT1iLiE1OWQiFjAwtP8mplVrWG/p5wQWCPxXAZmVu8a9nvK3Q6tkXwHz2VgZvWtYb+nnBBYI1kf2BkYCvw7sDX+DJtZHYmIOyV9mgacc8XdDq1hSLo3IjaVtB1wPHAK8KNGHRXMzJqHpC90tzwi/tBXsSwp/3dljaS9Yc6uwP9GxFWSTqgyIDOzbPd8/wHg48D1+fko4GbSyIV1zQmBNZLZkn4FfBY4WdLyuGGsmdWBiDgIQNKfgY0j4sn8fC1SV8S65y9TayRfJPXv3Ski5gKrAkdUGpGZ2cKGticD2dPAOlUFszjchsDMzKxGJP0c2BC4KBftBUyLiG9WF1U5TgjMzMxqKDcw/GR+emNE/LHKeMpyQmBmZmZuVGhmZtZTkm6KiO0kzSOPUti+iDT98coVhVaaawjMzMzMvQzMzMzMCYGZmZnhhMDMzMxwQmBmNSBphqTPVB2HmS05JwRm1rAkuaeUWY04ITCzhUgaKukPkp6V9Lykn0taX9L1+flzki6UNCivfz5paNb/kzRf0pG5fBtJN0uaK+keSa2F1xgu6UZJ8yT9RdIvJF1QWP45SffnbdskfbiwbIakoyTdC7ws6QhJv+9wDGdIOr03z5NZs3FCYGbvkNQPuBJ4DBgGDAYmkfpS/xhYG/gwMBQ4FiAi9gMeB3aPiIER8RNJg4GrgBNIc058F/i9pDXyS/0WuA1YLe9nv0IMHyQN+/otYA3galKysVwh1H1Is14OAi4ARhcSlP7A3sB5NTglZksNJwRmVrQV6Uf/iIh4OSJei4ibImJaREyOiNcj4lngZ8Cnu9nPl4GrI+LqiHg7IiYDdwC7SFoH2BL4UUS8ERE3AVcUtt0LuCq/3pvAKcAA0pSy7c6IiJkR8WqeSOZGYM+8bDTwXERM6fHZMFuKOCEws6KhwGMRsaBYKKlF0iRJsyW9RPqvfPVu9rMusGeu8p8raS6wHbAWKeGYExGvFNafWXi8NqmGAoCIeDsvH9zF+gATSUkI+f787g/TzDpyQmBmRTOBdTpprPffpOFYR+QhWL9MuozQruOQpzOB8yNiUOG2UkScBDwJrCppxcL6QwuPnyAlFABIUl4+u5vXuwzYVNImwG7AhYs+VDMrckJgZkW3kX6wT5K0kqQVJH0CeB8wH3gxtw84osN2TwPrFZ5fAOwuaSdJ/fJ+WiUNiYjHSJcPjpW0nKRtgd0L214C7CppB0nLAuOA14Gbuwo6Il4DLiW3TYiIx3twDsyWSk4IzOwdEfEW6cd5A1JDwVmka/r/BWwOvEhqLPiHDpv+GPhBvjzw3YiYCYwBjgaeJdUYHMG73zn7AtsCz5MaHl5M+tEnIh4m1UD8P+C5HM/uEfHGIsKfCIzAlwvMlognNzKzykm6GHgoIo7pwT7WAR4C1oyIl2oWnNlSwjUEZtbnJG2ZxzZYRtJoUm3CZT3Y3zLAd4BJTgbMloxH+TKzKqxJuuywGumyxNci4q4l2ZGklUhtGB4jdTk0syXgSwZmZmbmSwZmZmbmhMDMzMxwQmBmZmY4ITAzMzOcEJiZmRnw/wHbaUKbXCui9gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgoAAAFQCAYAAADePFZZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAoGUlEQVR4nO3de7RdVX33//eHIIha7hghQYKSapFq1QiojxpFIagYHx8vUCvRUlOr1uqwKtoqFqE/7NMW5fHSoiBgrYh4CSoWqRqtF+4qCF6IIE0iyC1cvAAC398fa6ZujmflbMI52Scn79cYe5y1vmuuOefaYbC/e8651k5VIUmSNJ7NRt0BSZI0fZkoSJKkXiYKkiSpl4mCJEnqZaIgSZJ6mShIkqReJgrSJElSSfZYx/FLkywcsq6fJnnmZPVtukvylCQ/GnU/JP0uEwVt8tqH8h1JdhwT/0778J+3HnWelOSowVhVPaqqlt+33k6NJAuTrBpV+1X1X1X1iFG1L0gyr/33vvmo+6LpxURB6lwJHLJ2J8kfAg8YXXc2HRvDB9PG0EdpqpgoSJ2PAocO7C8BThkskGR5kj8b2H95km+MrSjJUuClwJuT/CLJ51r8f6YTkrwzyelJPpHk1iQXJXnMeB1LslmSw5P8JMkNSU5Lsn3fhSRZnOS7SW5p5yxq8Vck+UFr74okf97iDwS+COzS+vuLJLtM1G6SQ5Nc1Y69fcz1bZnkPUl+1l7vSbJlO7Ywyaokb0lyDfCRsSMarf1PJbkuyZVJXjdwbO8kF7Tr+3mSf+55H9a287Yk17f+vXTg+JZJ/jHJf7d6/iXJVn197GnjlQPv6WVJHtfif9D+e7mpTTk9b+Cck5J8IMkX23v9zSQPae/RmiQ/TPLYgfI/TfKmJBcn+WWSE5LMbuffmuQ/k2w3UH7fJN9qbX8vA9NdrU/vam3emuRL+e1I2tfb35tav5443jVr02OiIHXOAbZu/4OfBRwM/Nv6VFRVxwMfA/6hqh5UVQf1FF0MfBLYHvh34LNJ7jdOub8Eng88DdgFWAO8f7wKk+xNl+C8CdgWeCrw03b4WuC5wNbAK4Bjkzyuqn4JHAj8rPX3QVX1s3W1m2RP4AN0CdHOwDbAnIGu/A2wL/BHwGOAvYG/HTj+kHbduwFLx1zDZsDngO+1OvcDXp/kgFbkvcB7q2pr4OHAaeO9FwPt7NjqWQIcn2TtFMcxwO+3Pu7RyrxjmD62fr4IeCddgrk18DzghvZv+DngS8CD6d7Hjw20C/Di9n7sCNwOfBu4qO2fDoxNfv4P8KzW34PoEru3ATvR/X/8da1Pc4AvAEe1vv818KkkOw3U9cd0//4PBrZoZaD7bwVg2/bfwLfHXrM2TSYK0m+tHVV4FvADYPUUt3dhVZ1eVb+h+2C4P92H61ivAv6mqlZV1e10H04vzPjD4YcBJ1bV2VV1d1WtrqofAlTVF6rqJ9X5Gt0H2VPW0b91tftC4HNV9Y2quoPuA3bwh2NeChxZVddW1XXA3wEvGzh+N3BEVd1eVb8e0+4TgJ2q6siquqOqrgA+RJe8AfwG2CPJjlX1i6o6Zx3XAPD21s7X6D5EX5wkdB/+b6iqG6vqVuDvB9qYqI8Af0aXDJ7f3tMVVXUV3b/hg4BjWv+/Anyegakt4DNVdWFV3QZ8Britqk6pqruATwCPHdPW/6uqn1fVauC/gHOr6jsD568t/yfAmVV1Zvv3Pxu4AHj2QF0fqaoft2s6jS5Rkno57yb91kfphl93Z8y0wxRZuXajqu5uQ++7jFNuN+AzSe4eiN0FzOZ3k5ldgTPHayzJgcARdN9KN6Nbg3HJOvq3rnZ3GdP/XyW5YaDcLsBVA/tXcc9ru659yPW1u0uSmwZis+g+IKFLho4EfpjkSuDvqurzPXWtaSMmY/uxE931X9jlDACktTNMH6F7r38yTnwXYGVVDb5vV3HPEZefD2z/epz9B42pc9jyuwEvSjI4inU/4KsD+9cMbP9qnLakezBRkJqquqp98Dyb7sNorF9yzwWOD1lXdUM0uevajTbcPhf42TjlVgJ/WlXfHKLOlXTD8ffQ1gd8im7EZFlV/SbJZ+k+HPv629tukquBRwzsbwXsMFDkZ3QfWpe2/Ydyz2tb1/uzEriyquaPd7CqLgcOae/ZC4DTk+wwJiFYa7skDxw49lDg+8D1dB+wj2rf0sdtah19XNvP33mv6a5z1ySbDSQLDwV+PEF9k2El8NGqeuV6nOtPCWtcTj1I93QY8IyeD53vAi9I8oB0z0sYL5lY6+fAwyZo6/FJXtCG8l9PN1c93jD6vwBHJ9kNIMlOSRb31HkC8Iok+6VbjDgnySPp5qK3BK4D7myjC/uP6e8OSbYZst3TgYOSPCnJFnTTEhk49+PA37ZzdqSbmhh2zcd5wK1tIeFWSWYl2SvJE1o//iTJTu1D+KZ2zt19lQF/l2SLJE+hW6PxyXbuh+jWaTy41TtnYB3EMD4M/HWSx6ezR3uvzqX7pv7mJPdriwkPAk69F3Wvr3+j+3c5oL1v90+3MHPuEOdeR/c+TvTfrTYxJgrSgDaHf0HP4WOBO+g+VE+mW7DY5wRgz7by/LM9ZZYBL6FbJPgy4AVtvcJY7wXOAL6U5Fa6ZGKfnv6fR1uoCNwMfA3Yrc3Bv45uTnoN3YK2MwbO+yHdh/sVrc+7rKvdqrqUbpHeqcDVwC/oFkve3qo8im5u/GK66Y2LWmxCbZ7+uXRz51fSffv/MN2CSYBFwKVJftH6eHDPGgLohtnX0H3L/xjwqrVrNoC3ACuAc5LcAvwnA6MkQ/Tzk8DRdAtRbwU+C2zf1mwcRLdA9Hq6RZ+HDrQ7ZapqJd0i2bfRffCvpFvYOuH/66vqV3TX883238B462W0CUqVo03ShpbkncAeVfUno+7LZEjyILpv9/Or6soRdwfobnEE/q2qhvk2LamHIwqS1kuSg9o0zAOBf6QbOfjpaHslabKZKEhaX4vphvR/BsynmwJwiFKaYZx6kCRJvRxRkCRJvUwUJElSLx+4NI4dd9yx5s2bN2n13XTTTQBsu+22k1anJEmT5cILL7y+qnYa75iJwjjmzZvHBRf03Up/7y1btgyAxYv7npEjSdLoJLmq75hTD5IkqZeJgiRJ6mWiIEmSepkoSJKkXiYKkiSpl4mCJEnqZaIgSZJ6mShIkqReJgqSJKmXiYIkSeploiBJknqZKEiSpF4mCpIkqdcGSRSSnJjk2iTfH+fYG5NUkh3bfpIcl2RFkouTPG6g7JIkl7fXkoH445Nc0s45LklafPskZ7fyZyfZbkNcryRJM8WGGlE4CVg0NphkV2B/4L8HwgcC89trKfDBVnZ74AhgH2Bv4IiBD/4PAq8cOG9tW4cDX66q+cCX274kSRrSBkkUqurrwI3jHDoWeDNQA7HFwCnVOQfYNsnOwAHA2VV1Y1WtAc4GFrVjW1fVOVVVwCnA8wfqOrltnzwQlyRJQxjZGoUki4HVVfW9MYfmACsH9le12Lriq8aJA8yuqqvb9jXA7MnpvSRJm4bNR9FokgcAb6ObdtggqqqSVN/xJEvppjqYPXs2y5cvn/Q+TEWdkiRNpZEkCsDDgd2B77V1h3OBi5LsDawGdh0oO7fFVgMLx8SXt/jcccoD/DzJzlV1dZuiuLavQ1V1PHA8wIIFC2rhwoV9Re+1ZcuWATCZdUqStCGMZOqhqi6pqgdX1byqmkc3XfC4qroGOAM4tN39sC9wc5s+OAvYP8l2bRHj/sBZ7dgtSfZtdzscCixrTZ0BrL07YslAXJIkDWFD3R75ceDbwCOSrEpy2DqKnwlcAawAPgS8GqCqbgTeBZzfXke2GK3Mh9s5PwG+2OLHAM9KcjnwzLYvSZKGtEGmHqrqkAmOzxvYLuA1PeVOBE4cJ34BsNc48RuA/e5ldyVJUuOTGSVJUi8TBUmS1MtEQZIk9TJRkCRJvUwUJElSLxMFSZLUy0RBkiT1MlGQJEm9TBQkSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8TBUmS1MtEQZIk9TJRkCRJvUwUJElSLxMFSZLUy0RBkiT1MlGQJEm9TBQkSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8TBUmS1MtEQZIk9dogiUKSE5Ncm+T7A7H/m+SHSS5O8pkk2w4ce2uSFUl+lOSAgfiiFluR5PCB+O5Jzm3xTyTZosW3bPsr2vF5G+J6JUmaKTbUiMJJwKIxsbOBvarq0cCPgbcCJNkTOBh4VDvnA0lmJZkFvB84ENgTOKSVBXg3cGxV7QGsAQ5r8cOANS1+bCsnSZKGtEEShar6OnDjmNiXqurOtnsOMLdtLwZOrarbq+pKYAWwd3utqKorquoO4FRgcZIAzwBOb+efDDx/oK6T2/bpwH6tvCRJGsJ0WaPwp8AX2/YcYOXAsVUt1hffAbhpIOlYG79HXe34za28JEkawuaj7kCSvwHuBD424n4sBZYCzJ49m+XLl096G1NRpyRJU2mkiUKSlwPPBfarqmrh1cCuA8Xmthg98RuAbZNs3kYNBsuvrWtVks2BbVr531FVxwPHAyxYsKAWLlx4n65t0LJlywCYzDolSdoQRjb1kGQR8GbgeVX1q4FDZwAHtzsWdgfmA+cB5wPz2x0OW9AteDyjJRhfBV7Yzl8CLBuoa0nbfiHwlYGERJIkTWCDjCgk+TiwENgxySrgCLq7HLYEzm7rC8+pqldV1aVJTgMuo5uSeE1V3dXqeS1wFjALOLGqLm1NvAU4NclRwHeAE1r8BOCjSVbQLaY8eMovVpKkGWSDJApVdcg44RPGia0tfzRw9DjxM4Ezx4lfQXdXxNj4bcCL7lVnJUnS/5gudz1IkqRpyERBkiT1MlGQJEm9TBQkSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8TBUmS1MtEQZIk9TJRkCRJvUwUJElSLxMFSZLUy0RBkiT1MlGQJEm9TBQkSVIvEwVJktTLREGSJPUyUZAkSb1MFCRJUi8TBUmS1MtEQZIk9TJRkCRJvUwUJElSLxMFSZLUa70ShSRPT/K0ye6MJEmaXoZKFJJ8LcmT2/ZbgFOBf0/ytqnsnCRJGq1hRxT2As5p268Eng7sC7xqmJOTnJjk2iTfH4htn+TsJJe3v9u1eJIcl2RFkouTPG7gnCWt/OVJlgzEH5/kknbOcUmyrjYkSdJwhk0UNgMqycOBVNVlVbUSGPaD9yRg0ZjY4cCXq2o+8OW2D3AgML+9lgIfhO5DHzgC2AfYGzhi4IP/g3QJzNrzFk3QhiRJGsKwicI3gPcB/wh8BqAlDdcPc3JVfR24cUx4MXBy2z4ZeP5A/JTqnANsm2Rn4ADg7Kq6sarWAGcDi9qxravqnKoq4JQxdY3XhiRJGsKwicLLgZuAi4F3ttgjgffeh7ZnV9XVbfsaYHbbngOsHCi3qsXWFV81TnxdbUiSpCFsPkyhqroBeNuY2BcmqxNVVUlqsupbnzaSLKWb6mD27NksX7580vswFXVKkjSVhkoUkmwJvAM4BNihqrZJsj/w+1X1vvVs++dJdq6qq9v0wbUtvhrYdaDc3BZbDSwcE1/e4nPHKb+uNn5HVR0PHA+wYMGCWrhwYV/Re23ZsmUATGadkiRtCMNOPRxLd+fDS4G138ovBf7iPrR9BrD2zoUlwLKB+KHt7od9gZvb9MFZwP5JtmuLGPcHzmrHbkmyb7vb4dAxdY3XhiRJGsJQIwrA/wb2qKpfJrkboKpWJ5kzwXkAJPk43WjAjklW0d29cAxwWpLDgKuAF7fiZwLPBlYAvwJe0dq7Mcm7gPNbuSOrau0CyVfT3VmxFfDF9mIdbUiSpCEMmyjcMbZskp2AG4Y5uaoO6Tm03zhlC3hNTz0nAieOE7+AbsRjbPyG8dqQJEnDGXbq4ZPAyUl2B2jz/e+je0KjJEmaoYZNFN4GXAlcAmwLXA78DDhyarolSZKmg2Fvj7wDeAPwhjblcH2bIpAkSTPYsD8KdWiSRwNU1XXtmQSPSfKyqe2eJEkapWGnHt7FPZ+KSNs/anK7I0mSppNhE4WtgVvGxG6mW68gSZJmqGEThcuA/zMm9r+BH0xudyRJ0nQy7HMU3gKcmeQlwE+APeieT/DsqeqYJEkavaFGFKrqG8Af0j0V8YHAecBeVfXNKeybJEkasWFHFKiqq+geiSxJkjYRw/565PbAXwN/BDxo8FhVPXXyuyVJkqaDYUcU/h3YEjiN7oeaJEnSJmDYROFJwE5VdftUdkaSJE0vw94eeTEwdyo7IkmSpp9hRxS+AvxHko8A1wweaD/9LEmSZqBhE4WnAKuAZ42JF2CiIEnSDDXsr0c+fao7IkmSpp9h1yiQZIckL0vypra/SxLXLUiSNIMN+zPTTwN+BLwUeEcLzwc+OEX9kiRJ08CwIwrvAV5SVYuAO1vsXGDvqeiUJEmaHoZNFOZV1ZfbdrW/d3AvHgEtSZI2PkP/zHSSA8bEnglcMsn9kSRJ08iwIwJvBD6f5AvAVkn+FTgIWDxlPZMkSSM37IjCecCjgUvpnptwJbB3VZ0/VR2TJEmjN+GIQpJZwC+AbavqH6a+S5IkabqYcEShqu4CfgzsMPXdkSRJ08mwaxQ+RrdG4b10j3Jee+cDVfWVqeiYJEkavWEThb9of985Jl7AwyatN5IkaVqZcOohyWbAnwGPrKrdx7zuc5KQ5A1JLk3y/SQfT3L/JLsnOTfJiiSfSLJFK7tl21/Rjs8bqOetLf6jwVs5kyxqsRVJDr+v/ZUkaVMyzBqFu4HPVtXtk914kjnA64AFVbUXMAs4GHg3cGxV7QGsAQ5rpxwGrGnxY1s5kuzZznsUsAj4QJJZbSHm+4EDgT2BQ1pZSZI0hGFvj/x6kn2nqA+b0z2bYXPgAcDVwDOA09vxk4Hnt+3FbZ92fL8kafFTq+r2qroSWEH3eOm9gRVVdUVV3QGcis9+kCRpaMOuUbgK+GKSZcBK7rmY8R29Z02gqlYn+Ufgv4FfA18CLgRuqqq1vymxCpjTtue09qmqO5PcTHc3xhzgnIGqB89ZOSa+z3h9SbIUWAowe/Zsli9fvr6X1Wsq6pQkaSoNmyhsBXy2bU/aT0sn2Y7uG/7uwE3AJ+mmDja4qjoeOB5gwYIFtXDhwkmre9myZQBMZp2SJG0IQyUKVfWKKWr/mcCVVXUdQJJPA08Gtk2yeRtVmAusbuVXA7sCq9pUxTbADQPxtQbP6YtLkqQJDLVGIcnD+l73sf3/BvZN8oC21mA/4DLgq8ALW5klwLK2fUbbpx3/SlVVix/c7orYHZhP99jp84H57S6KLegWPJ5xH/ssSdImY9iphxV06xIyEFu7TmHW+jZeVecmOR24CLgT+A7d8P8XgFOTHNViJ7RTTgA+mmQFcCPdBz9VdWmS0+iSjDuB17QnSpLktcBZrZ8nVtWl69tfSZI2NcNOPdxj5CHJQ4AjgP+6rx2oqiNaXYOuoLtjYWzZ24AX9dRzNHD0OPEzgTPvaz8lSdoUDXt75D1U1TXA64H/b1J7I0mSppX1ShSaR9A990CSJM1QQ009JPkvBp6dQJcgPAo4cio6JUmSpodhFzN+eMz+L4HvVdXlk9wfSZI0jQy7mPHkiUtJkqSZZtjnKHw6yVPGxJ7Sbm2UJEkz1LCLGZ8GfGtM7NvA0ye3O5IkaToZNlG4DXjgmNiDgN9MbnckSdJ0MmyicBbwr0m2Bmh/3wf8x1R1TJIkjd6wicIbga2BG5NcS/f45G3oHrokSZJmqGHvelgDPKc9unlXYGV7OqMkSZrBhn3g0v7AT6vqx8A1LfYI4KFVdfYU9k+SJI3QsFMP7wduHRO7tcUlSdIMNWyi8OCqunpM7GrgIZPcH0mSNI0MmyhckeQZY2ILgSsntzuSJGk6Gfa3Ht4JfDrJCcBPgIcDr2gvSZI0Qw01olBVy4D96R669Jz294AWlyRJM9SwIwpU1XnAeVPYF0mSNM1MOKKQZF6Sk5KsTnJ7+3tykodtiA5KkqTRWWeikOQPgIuABwN/Azyv/d0JuKAdlyRJM9REUw/HAO+vqrePiZ+U5CjgH4CDpqRnkiRp5CZKFJ4KLOk59k94e6QkSTPaRGsUZtH/U9K/acclSdIMNVGicD79z0p4OXDBpPZGkiRNKxNNPbwdOKv9ANTpdI9t3hl4Ed2UxAFT2z1JkjRK6xxRqKpv0T1o6THAl4Eftr+PARa145IkaYaa8IFLVfVt4KlJtgK2B9ZU1a+mvGeSJGnkhv1RKKrq11W1erKThCTbJjk9yQ+T/CDJE5Nsn+TsJJe3v9u1sklyXJIVSS5O8riBepa08pcnWTIQf3ySS9o5xyXJZPZfkqSZbOhEYQq9F/iPqnok3ZTGD4DDgS9X1Xy6qY7DW9kDgfnttRT4IECS7YEjgH2AvYEj1iYXrcwrB85btAGuSZKkGWGkiUKSbeie1XACQFXdUVU3AYuBk1uxk4Hnt+3FwCnVOQfYNsnOdIsqz66qG6tqDXA2sKgd27qqzqmqAk4ZqEuSJE2gd41Ckv9bVW9q28+oqq9MQfu7A9cBH0nyGOBC4K+A2VV1dStzDTC7bc8BVg6cv6rF1hVfNU78dyRZSjdKwezZs1m+fPl6X1SfqahTkqSptK7FjEuBN7XtzwJbT1H7jwP+sqrOTfJefjvNAEBVVZKagrbvoaqOB44HWLBgQS1cuHDS6l62rPs17smsU5KkDWFdicL3kpwOXAZsmeTI8QpV1TvuQ/urgFVVdW7bP50uUfh5kp2r6uo2fXBtO74a2HXg/LktthpYOCa+vMXnjlNekiQNYV1rFF4IfJfuAUuh+4Ae+5rbd/IwquoaYGV7oBPAfnSJyRn89jcmlgDL2vYZwKHt7od9gZvbFMVZwP5JtmuLGPcHzmrHbkmyb7vb4dCBuiRJ0gR6RxSq6lrgKIAkm1dV36Oc76u/BD6WZAvgCrpHRm8GnJbkMOAq4MWt7JnAs4EVwK9aWarqxiTvonvkNMCRVXVj2341cBKwFfDF9pIkSUOY8IFLAFX1ivZN/SC6xYCrgc8PfBivt6r6LrBgnEP7jVO2gNf01HMicOI48QuAve5bLyVJ2jQNdXtkkicCPwFeBTwa+HNgRYtLkqQZaqgRBeA9wKur6tS1gSQvAY4DnjAF/ZIkSdPAsA9c+n3gtDGx04E9Jrc7kiRpOhk2UbgcOHhM7EV00xGSJGmGGnbq4fXA55O8ju4uhHl0v5vw3KnpliRJmg6GvevhW0keDjwH2AX4HHDmZNz1IEmSpq9hRxRoP7b0b1PYF0mSNM1Mh5+ZliRJ05SJgiRJ6mWiIEmSeg2dKCTZbSo7IkmSpp97M6LwHYB2i6QkSdoErPOuhyQXAhfSJQmzWviddI9uliRJM9xEIwovBL4E7AY8IMlFwJZJnp5kmynvnSRJGqmJEoVZVXV6VR0O3AosBgL8JfDdJJdPdQclSdLoTPTApY8leShwGXB/YDvgtqp6AUCS7ae4f5IkaYTWmShU1T5JNgf+EPgG8D7g95J8ELiovXyMsyRJM9SEdz1U1Z1V9R3gjqp6KvBLYDndj0K9e2q7J0mSRmno33oA3tD+VlV9AvjEFPRHkiRNI0M/R6GqTmqbD5uarkiSpOnmXj/Cuf2KpCRJ2gT4Ww+SJKmXiYIkSeploiBJknqZKEiSpF4mCpIkqZeJgiRJ6jUtEoUks5J8J8nn2/7uSc5NsiLJJ5Js0eJbtv0V7fi8gTre2uI/SnLAQHxRi61IcvgGvzhJkjZi0yJRAP4K+MHA/ruBY6tqD2ANcFiLHwasafFjWzmS7AkcDDwKWAR8oCUfs4D3AwcCewKHtLKSJGkII08UkswFngN8uO0HeAZweityMvD8tr247dOO79fKLwZOrarbq+pKYAWwd3utqKorquoO4NRWVpIkDWHkiQLwHuDNwN1tfwfgpqq6s+2vAua07TnASuh+rAq4uZX/n/iYc/rikiRpCPfmR6EmXZLnAtdW1YVJFo64L0uBpQCzZ89m+fLlk97GVNQpSdJUGmmiADwZeF6SZwP3B7YG3gtsm2TzNmowF1jdyq8GdgVWJdkc2Aa4YSC+1uA5ffF7qKrjgeMBFixYUAsXLrzPF7fWsmXLAJjMOiVJ2hBGOvVQVW+tqrlVNY9uMeJXquqlwFeBF7ZiS4BlbfuMtk87/pWqqhY/uN0VsTswHzgPOB+Y3+6i2KK1ccYGuDRJkmaEUY8o9HkLcGqSo4DvACe0+AnAR5OsAG6k++Cnqi5NchpwGXAn8JqqugsgyWuBs4BZwIlVdekGvRJJkjZi0yZRqKrlwPK2fQXdHQtjy9wGvKjn/KOBo8eJnwmcOYldlSRpkzEd7nqQJEnTlImCJEnqZaIgSZJ6mShIkqReJgqSJKmXiYIkSeploiBJknqZKEiSpF4mCpIkqZeJgiRJ6mWiIEmSepkoSJKkXiYKkiSpl4mCJEnqZaIgSZJ6mShIkqReJgqSJKmXiYIkSeploiBJknqZKEiSpF4mCpIkqZeJgiRJ6mWiIEmSem0+6g5siuYd/oV77P/0mOeMqCeSJK2bIwqSJKmXiYIkSeploiBJknqNNFFIsmuSrya5LMmlSf6qxbdPcnaSy9vf7Vo8SY5LsiLJxUkeN1DXklb+8iRLBuKPT3JJO+e4JNnwVypJ0sZp1CMKdwJvrKo9gX2B1yTZEzgc+HJVzQe+3PYBDgTmt9dS4IPQJRbAEcA+wN7AEWuTi1bmlQPnLdoA1yVJ0oww0kShqq6uqova9q3AD4A5wGLg5FbsZOD5bXsxcEp1zgG2TbIzcABwdlXdWFVrgLOBRe3Y1lV1TlUVcMpAXZIkaQLT5vbIJPOAxwLnArOr6up26BpgdtueA6wcOG1Vi60rvmqc+LQy9nZJ8JZJSdL0MC0ShSQPAj4FvL6qbhlcRlBVlaQ2QB+W0k1nMHv2bJYvXz7pbayt841/eOfQZSVJGqWRJwpJ7keXJHysqj7dwj9PsnNVXd2mD65t8dXArgOnz22x1cDCMfHlLT53nPK/o6qOB44HWLBgQS1cuHC8Yutl2bJlAKyt8+XjjCCM9dOXTl77kiStr1Hf9RDgBOAHVfXPA4fOANbeubAEWDYQP7Td/bAvcHObojgL2D/Jdm0R4/7AWe3YLUn2bW0dOlCXJEmawKhHFJ4MvAy4JMl3W+xtwDHAaUkOA64CXtyOnQk8G1gB/Ap4BUBV3ZjkXcD5rdyRVXVj2341cBKwFfDF9pIkSUMYaaJQVd8A+p5rsN845Qt4TU9dJwInjhO/ANjrPnRTkqRN1qhHFDYJd1eXDY13d4MkSdOZicI05S9MSpKmg1E/mVGSJE1jJgqSJKmXiYIkSeploiBJknqZKEiSpF4mCpIkqZeJgiRJ6uVzFDYSPldBkjQKjihIkqReJgqSJKmXUw8bKaciJEkbgiMKkiSpl4mCJEnqZaIgSZJ6uUZhhnDNgiRpKjiiIEmSepkoSJKkXk49zFBORUiSJoMjCpIkqZcjCpsIRxgkSevDEQVJktTLEYVNlCMMkqRhmCgIMHGQJI3PREHjMnGQJIGJgoY0NnEAkwdJ2hRsEolCkkXAe4FZwIer6pgRd2lGGC95GGQiIUkbvxmfKCSZBbwfeBawCjg/yRlVddloezbzTZRIjGViIUnTz4xPFIC9gRVVdQVAklOBxYCJwjRzbxOL9WEyIkn3zqaQKMwBVg7srwL2GVFfNGIbIhnZ2Jg8SVqXTSFRGEqSpcDStvuLJD+axOofCdwPuGQS65QmRd496h5ImgZ26zuwKSQKq4FdB/bnttg9VNXxwPFT0YEkdwGbVdWCqahfkqSpsik8wvl8YH6S3ZNsARwMnDHiPkmStFGY8SMKVXVnktcCZ9HdHnliVV064m5JkrRRSFWNug8zXpJzgAdX1cNG3RdJku4NEwVJktRrU1ijIEmS1pOJwhRK8skkNeb1xFH3S5KkYZkoTK2PAwX8Avh0i+03uu5IknTvmChMrVva3wAfBu6me6S0JEkbhRl/e+SI/RFdkvBA4MwWu25kvZEk6V5yRGFqPar9/Wl7ATx9JD2RJGk9OKIwtR7Q/u5Gt06huOfjpCVJmtYcUZhah9OtS7gFWEI3DeEPQ0mSNho+cGkKJbkIeOyY8Euq6rRR9EeSpHvLREGSJPVy6kGSJPUyUZAkSb1MFCRJUi8TBUmS1MtEQZIk9TJRkDS0JLOTfD3JrUn+adT9mSxJ/iXJ20fdD2k68smM0iYgyXnAnwB3AqdX1ePWs6qlwPXA1jWJ91YnOQlYVVV/O1l13htV9apRtCttDBxRkGa4JPeje4z45cDjgYvuQ3W7AZdNZpIwaklmjboP0nRmoiDNfHvx2w/3BUyQKCR5UpLzk9zc/j6pxU+iexT5m5P8Iskzxzl3qyT/lOSqdv43kmzVjn0yyTUt/vUkj2rxpcBLB+r9XIvvkuRTSa5LcmWS141p5+Qka5L8IMmbk6waOP4HSZYnuSnJpUmeN3DspCQfTHJmkl8CT2+xowbKPDfJd9v530ry6IFjb0myuk2//CjJfvfi30La+FSVL1++ZuALeAVwE/Ar4La2fSdwa9vefZxztgfWAC+jm5o8pO3v0I6fBBy1jjbfDywH5gCzgCcBW7Zjfwr8HrAl8B7guwPn3aNeui8xFwLvALYAHgZcARzQjh8DfA3YDpgLXEw3dQFwP2AF8LZ27jPaNT9ioK2bgSe3du4/2D7dY9evBfZp17CE7tdftwQeAawEdmll5wEPH/W/tS9fU/lyREGaoarqI1W1Ld0H7r7Ao4Hv060v2LaqrhzntOcAl1fVR6vqzqr6OPBD4KCJ2kuyGV0y8FdVtbqq7qqqb1XV7a0/J1bVrW3/ncBjkmzTU90TgJ2q6siquqOqrgA+BBzcjr8Y+PuqWlNVq4DjBs7dF3gQcEw79yvA5+mSnrWWVdU3q+ruqrptTNtLgX+tqnPbNZwM3N7qvYsuYdgzyf2q6qdV9ZOJ3htpY2aiIM1ASbZvw+Y3032rXw78iO4b8Zokr+85dRfgqjGxq+hGCCayI92389/54EwyK8kxSX6S5Ba6b+hrzxnPbsAu7RpuSnIT3QjB7IF+rhwoP7i9C7Cyqu5exzUMlh+v7TeOaXtXulGEFcDr6RKda5OcmmSXddQlbfRMFKQZqKpubKMJfw58uG3/B3BQG014T8+pP6P7oBz0UGD1EM1eTzfF8fBxjv0xsBh4JrAN3ZA9dD+9DjB2ceRK4MrW17Wv36uqZ7fjV9NNOay165hr2LWNcPRdw7oWY64Ejh7T9gPa6ApV9e9V9b/o3qcC3r2OuqSNnomCNLMN3uXwWLppiHU5E/j9JH+cZPMkLwH2pBu6X6f2Df5E4J/bQsRZSZ6YZEu6tQm3AzcADwD+fszpP6dbh7DWecCtbeHgVq2uvZI8oR0/DXhrku2SzAFeO3DuuXTrMt6c5H5JFtJNnZw60TU0HwJelWSfdB6Y5DlJfi/JI5I8o13TbcCvgbvXXZ20cTNRkGa2xwMXJdkBuKuq1qyrcFXdADwXeCPdh/qbgedW1fVDtvfXwCXA+cCNdN+2NwNOoRv+Xw1cBpwz5rwT6Ob9b0ry2aq6q/Xjj4Ar6UYrPkw3GgFwJLCqHftP4HS6RISquoMuMTiwnfcB4NCq+uEwF1BVFwCvBN5Ht5BzBfDydnhLuoWU1wPXAA8G3jpMvdLGKlUz5nZoSZuoJH8BHFxVTxt1X6SZxhEFSRudJDsneXKSzZI8gm4E5DOj7pc0E/kIZ0kboy2AfwV2p3smxKl0UwySJplTD5IkqZdTD5IkqZeJgiRJ6mWiIEmSepkoSJKkXiYKkiSpl4mCJEnq9f8DgkKEz+oX82cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_toxic = df.drop(['id', 'comment_text'], axis=1)\n",
    "counts = []\n",
    "categories = list(df_toxic.columns.values)\n",
    "for i in categories:\n",
    "    counts.append((i, df_toxic[i].sum()))\n",
    "df_stats = pd.DataFrame(counts, columns=['category', 'number_of_comments'])\n",
    "print(f\"{df_stats}\\n\")\n",
    "df_stats.plot(x='category', y='number_of_comments', kind='bar', legend=False, grid=True, figsize=(8, 5))\n",
    "plt.title(\"Number of comments per category\")\n",
    "plt.ylabel('# of Occurrences', fontsize=12)\n",
    "plt.xlabel('category', fontsize=12)\n",
    "\n",
    "rowsums = df.iloc[:,2:].sum(axis=1)\n",
    "x=rowsums.value_counts()\n",
    "#plot\n",
    "plt.figure(figsize=(8,5))\n",
    "ax = sns.barplot(x.index, x.values)\n",
    "plt.title(\"Multiple categories per comment\")\n",
    "plt.ylabel('# of Occurrences', fontsize=12)\n",
    "plt.xlabel('# of categories', fontsize=12)\n",
    "\n",
    "print(f\"- Percentage of comments that are not labelled: {len(df[(df['toxic']==0) & (df['severe_toxic']==0) & (df['obscene']==0) & (df['threat']== 0) & (df['insult']==0) & (df['identity_hate']==0)]) / len(df)}\")\n",
    "print(f\"- Number of missing comments in comment text: {df['comment_text'].isnull().sum()}\")\n",
    "\n",
    "lens = df.comment_text.str.len()\n",
    "lens.hist(bins = np.arange(0,5000,50))\n",
    "\n",
    "print(df['comment_text'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"what's\", \"what is \", text)\n",
    "    text = re.sub(r\"\\'s\", \" \", text)\n",
    "    text = re.sub(r\"\\'ve\", \" have \", text)\n",
    "    text = re.sub(r\"can't\", \"can not \", text)\n",
    "    text = re.sub(r\"n't\", \" not \", text)\n",
    "    text = re.sub(r\"i'm\", \"i am \", text)\n",
    "    text = re.sub(r\"\\'re\", \" are \", text)\n",
    "    text = re.sub(r\"\\'d\", \" would \", text)\n",
    "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
    "    text = re.sub(r\"\\'scuse\", \" excuse \", text)\n",
    "    text = re.sub('\\W', ' ', text)\n",
    "    text = re.sub('\\s+', ' ', text)\n",
    "    text = text.strip(' ')\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'explanation why the edits made under my username hardcore metallica fan were reverted they were not vandalisms just closure on some gas after i voted at new york dolls fac and please do not remove the template from the talk page since i am retired now 89 205 38 27'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['comment_text'] = df['comment_text'].map(lambda com : clean_text(com))\n",
    "df['comment_text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(106912,) (106912, 6)\n",
      "(52659,) (52659, 6)\n"
     ]
    }
   ],
   "source": [
    "categories = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
    "train, valid = train_test_split(df, random_state=42, test_size=0.33, shuffle=True)\n",
    "X_train = train.comment_text\n",
    "y_train = train[categories].values\n",
    "X_valid = valid.comment_text\n",
    "y_valid = valid[categories].values\n",
    "print(X_train.shape,y_train.shape)\n",
    "print(X_valid.shape,y_valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5         thank you for understanding i think very highl...\n",
       "7                            dear god this site is horrible\n",
       "11        somebody will invariably try to add religion r...\n",
       "13        it says it right there that it is a type the t...\n",
       "14        before adding a new product to the list make s...\n",
       "                                ...                        \n",
       "153150    jerome i see you never got around to this i m ...\n",
       "153151    lucky bastard http wikimediafoundation org wik...\n",
       "153154    shame on you all you want to speak about gays ...\n",
       "153155    mel gibson is a nazi bitch who makes shitty mo...\n",
       "153156    unicorn lair discovery supposedly a unicorn la...\n",
       "Name: comment_text, Length: 63978, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test['comment_text'] = df_test['comment_text'].map(lambda com : clean_text(com))\n",
    "df_test['comment_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(63978,) (63978, 6)\n"
     ]
    }
   ],
   "source": [
    "X_test = df_test.comment_text.values\n",
    "y_test = df_test[categories].values\n",
    "print(X_test.shape,y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAc+UlEQVR4nO3deZhcZZn+8e9NFgIBEpM0MaQJHYGBYRlZQkRhFEEFUUlQJKASAkHUYRUdiTOo4A+8QFREmZ+CgKBggGEZGEAwIpFBkBD2kIBhCabDlkQStolAeOaP922oNL1Up7ur3k7uz3XV1Wd5zznPOVV17jpLVykiMDMzK8069S7AzMysLQ4oMzMrkgPKzMyK5IAyM7MiOaDMzKxIDigzMyuSA8r6NEmnSloi6dkuTjdT0hG9VVdPUPJLSS9ImlXvetrTlW0paQ9JzVW2nSLp9tWsabWntXI4oKzXSfpx3sneKamxYvjnJP2kG/MdA3wN2CYi3t0TtRZmd+CjQGNEjK93MWa15oCyXiVpPLAz8G7gdmBaHj4E+FfgpG7MfgywNCKe726dhdoMWBARr3R1Qkn9e6Ees5pyQFlvGwvcHhF/B24B3pOHnwacGREvdjSxpCGSfiVpsaSnJJ0kaR1JHwFmAJtIelnSRe1MP0HS/ZJelPS4pH3aaLO5pD9IWppPF14qaWjF+BMlLZL0kqRHJe2Vh4+XNDvP+zlJP6qYZldJd0haJukBSXtUjJsi6Yk8vyclfb6NmqYC5wPvz+t3Sh7+RUmPSfqbpOskbVIxTUg6StJ8YH4726Ojug6TNC/X9YSkL3VhW24m6U952t9JGtHW8tuoZ1qe10uS5kra/51NdI6k5ZIeadn2ecQQSRdIeiY/P6dK6lfNcq2PiAg//Oi1B7Ad6chpPeDM/BgHzKhy+l8B1wIbAk3AX4CpedweQHMH044HlpNOk60DjAa2zuNmAkfk7i1ym3WBBuA24Md53FbAQmCT3N8EbJ677wQOyd0bALvm7tHAUmDfvNyP5v4GYDDwIrBVbjsK2Lad+qeQwr2lf09gCbBTrvWnwG0V44MU2sOA9dqYX7t15fGfADYHBHwIeBXYqcpt+TjwD/l5ngmc3s46rfKcAZ8FNsnznAS8AoyqWP83gK8CA/L45cCwPP4a4Ny8TTcGZgFfamvb+dE3H3UvwI81/5F3MA8Al+ed9B3APwLH5jC4FBjaxnT9gNdI15hahn0JmJm7V9nZtTH9ucBZ7YybSQ6oNsZNBO7L3VsAzwMfAQa0ancbcAowotXwE4Fftxp2M3Bo3pkuAz7TVoi0mmaVnSxwAfD9iv4NgNeBptwfwJ4dzK/dutpp/1/AcVVuy5Mq+v8FuKmdtp09Z/cDEyrW/2lAFeNnAYcAI4G/V25D4GDg1ra2nR998+FTfNbrIuKsiHhvREwCDiTt2NcBjgT2AuaRr021MoL0yfmpimFPkT69V2NT0if7DkkaKemyfJroReCSvGwi4jHgeOBk4PncruW02lTSUcMjku6W9Mk8fDPgs/k02jJJy0g3PIyKdD1pEvBl4BlJN0jausr12YSKbRERL5OOgCq3x8IOpm+3rrwdPi7pz/n04TLSkVbLqbrOtmXlXZSvksKzU5Im59OGLfVsV7FMgEWREyd7irQdNiO9Np6pmPZc0pGUrSEcUFYzkkaSQum7pB3RgxHxOnA38E9tTLKEdISwWcWwMcCiKhe5kHTKqjPfIx19bB8RGwFfIJ3mAiAifhMRu+c6AjgjD58fEQeTdopnAFdKGpyX++uIGFrxGBwRp+fpbo6Ij5KC4RHgF1Wuz9NUbIu8rOGsuj06+nmCduuStC5wFfADYGREDAVurNgO1W7LqknajLTuRwPD8zLnVCwTYLSkyv4xpO2wkHQENaJiXTaKiG17skarLweU1dKPgJMj4lXgSWAXSRuQTvs80bpxRKwErgBOk7Rh3qGdQDrCqcYFwGGS9so3Voxu52hlQ+BlYLmk0aS7CwGQtJWkPfMOfAXwv8CbedwXJDVExJuk03bkcZcAn5K0t6R+kgYp/f9PYz5am5DD5e95uW9WuT7T8/rskOv5HnBXRCyocvp26wIGkq5rLQbekPRx4GMV01a7LbtiMClQF0O6SYP0waXSxsCxkgZI+izp1PCNEfEM8Dvgh5I2yjVtLulD3azJCuKAspqQtCfpOtM1ABExC7iB9En4w8Dp7Ux6DOnC+ROkmy1+A1xYzTLzMg4DziJdXP8jqx6NtTiFdOPB8lzT1RXj1s21LSGdxtoY+GYetw/wsKSXgbOBgyLifyNiITAB+DfSznchKfTWyY8TSEcBfyPdjPCVKtfn98C3SEc6z5COaA6qZto8fbt1RcRLpGuCVwAvAJ8DrquYttptWbWImAv8kHSzyXPA9sCfWjW7C9iStP1PAw6IiKV53GRSsM7NNV9JPl1pawatenrXzMysDD6CMjOzIjmgzMysSA4oMzMrkgPKzMyK1Ke/UHLEiBHR1NRU7zLMzKwb7rnnniUR0dB6eJ8OqKamJmbPnl3vMszMrBskPdXWcJ/iMzOzIjmgzMysSA4oMzMrUp++BmVmZvD666/T3NzMihUr6l1KhwYNGkRjYyMDBgyoqr0Dysysj2tubmbDDTekqamJVb/8vRwRwdKlS2lubmbs2LFVTeNTfGZmfdyKFSsYPnx4seEEIInhw4d36Siv1wJK0oWSnpc0p2LYMEkzJM3Pf9+Vh0vSTyQ9JulBSTv1Vl1mZmuiksOpRVdr7M0jqItIP0dQaRpwS0RsCdzC27+i+nHSV+pvSfpBu5/1Yl1mZtYH9No1qIi4TVJTq8ETSD9OB3AxMBM4MQ//Vf5p5z9LGippVP5RMjMz64KmaTf06PwWnP6JTtscfvjhXH/99Wy88cbMmTOn0/bVqPVNEiMrQudZYGTuHk368bQWzXnYOwJK0pGkoyzGjBnTe5Warc1OHtIL81ze8/O0YkyZMoWjjz6ayZMn99g863aTRD5a6vKvJUbEeRExLiLGNTS846ubzMysDj74wQ8ybNiwHp1nrY+gnms5dSdpFPB8Hr4I2LSiXWMeZqXr6U/a/pRtVp6n72t/3CY79tpia30EdR1waO4+FLi2YvjkfDffrsByX38yM1u79doRlKTppBsiRkhqBr4DnA5cIWkq8BRwYG5+I7Av8BjwKnBYb9VlZmZ9Q2/exXdwO6P2aqNtAEf1Vi1mZtb3+KuOzMzWMO3eFt7RtaRuOvjgg5k5cyZLliyhsbGRU045halTp3Zrng4oMzPrtunTp/f4PP1dfGZmViQHlJmZFckBZWZmRfI1KDNbu/lrnYrlgDKrlr81w6ymfIrPzMyK5CMoM7M1TU8f7R85s6pmN910E8cddxwrV67kiCOOYNq0aZ1P1AEfQZmZWbetXLmSo446it/+9rfMnTuX6dOnM3fu3G7N0wFlZmbdNmvWLLbYYgve8573MHDgQA466CCuvfbazifsgAPKzMy6bdGiRWy66du/mtTY2MiiRd371SQHlJmZFckBZWZm3TZ69GgWLlz4Vn9zczOjR4/u1jwdUGZm1m277LIL8+fP58knn+S1117jsssuY7/99uvWPH2beW/wP3SaWT21t8/oxZ/b6N+/P+eccw577703K1eu5PDDD2fbbbft3jx7qDYzM1vL7bvvvuy77749Nj+f4jMzsyI5oMzMrEgOKDOzNUBE1LuETnW1RgeUmVkfN2jQIJYuXVp0SEUES5cuZdCgQVVP45skzMz6uMbGRpqbm1m8eHHHDZc93/MLXz6v6qaDBg2isbGx6vYOKDOzPm7AgAGMHTu284Yn79rzC+/Ff4PxKT4zMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMiuSAMjOzIjmgzMysSA4oMzMrkgPKzMyK5IAyM7MiOaDMzKxIdQkoSV+V9LCkOZKmSxokaaykuyQ9JulySQPrUZuZmZWh5gElaTRwLDAuIrYD+gEHAWcAZ0XEFsALwNRa12ZmZuWo1ym+/sB6kvoD6wPPAHsCV+bxFwMT61OamZmVoOYBFRGLgB8AfyUF03LgHmBZRLyRmzUDo9uaXtKRkmZLmt3pj3OZmVmfVY9TfO8CJgBjgU2AwcA+1U4fEedFxLiIGNfQ0NBLVZqZWb3V4xTfR4AnI2JxRLwOXA3sBgzNp/wAGoFFdajNzMwKUY+A+iuwq6T1JQnYC5gL3AockNscClxbh9rMzKwQ9bgGdRfpZoh7gYdyDecBJwInSHoMGA5cUOvazMysHP07b9LzIuI7wHdaDX4CGF+HcszMrED+JgkzMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMiuSAMjOzIjmgzMysSA4oMzMrkgPKzMyK5IAyM7MiOaDMzKxIDigzMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMiuSAMjOzIjmgzMysSA4oMzMrkgPKzMyK5IAyM7MiOaDMzKxIDigzMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMiuSAMjOzIjmgzMysSA4oMzMrkgPKzMyK5IAyM7Mi1SWgJA2VdKWkRyTNk/R+ScMkzZA0P/99Vz1qMzOzMtTrCOps4KaI2Bp4LzAPmAbcEhFbArfkfjMzW0vVPKAkDQE+CFwAEBGvRcQyYAJwcW52MTCx1rWZmVk56nEENRZYDPxS0n2Szpc0GBgZEc/kNs8CI+tQm5mZFaIeAdUf2An4WUTsCLxCq9N5ERFAtDWxpCMlzZY0e/Hixb1erJmZ1Uc9AqoZaI6Iu3L/laTAek7SKID89/m2Jo6I8yJiXESMa2hoqEnBZmZWezUPqIh4Flgoaas8aC9gLnAdcGgedihwba1rMzOzcvSv03KPAS6VNBB4AjiMFJZXSJoKPAUcWKfazMysAHUJqIi4HxjXxqi9alyKmZkVyt8kYWZmRXJAmZlZkRxQZmZWJAeUmZkVyQFlZmZFckCZmVmRHFBmZlYkB5SZmRXJAWVmZkXqUkBJ2lXSTZJmSprYSzWZmZl1/FVHkt6dv9y1xQnA/oCAu4D/6r3SzMxsbdbZd/H9XNK9wPcjYgWwDDgAeBN4sZdrMzOztViHp/giYiJwH3C9pMnA8cC6wHD8k+xmZtaLOr0GFRH/DewNDAGuAf4SET+JCP+crZmZ9ZoOA0rSfpJuBW4C5gCTgAmSLpO0eS0KNDOztVNn16BOBcYD6wE3R8R44GuStgROAw7q5frMzGwt1VlALQc+DawPPN8yMCLm43AyM7Ne1Nk1qP1JN0T0Bz7X++WYmZklHR5BRcQS4Kc1qsXMzOwt/qojMzMrkgPKzMyK5IAyM7MiOaDMzKxIDigzMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMiuSAMjOzIjmgzMysSA4oMzMrkgPKzMyK5IAyM7MiOaDMzKxIDigzMytS3QJKUj9J90m6PvePlXSXpMckXS5pYL1qMzOz+qvnEdRxwLyK/jOAsyJiC+AFYGpdqjIzsyLUJaAkNQKfAM7P/QL2BK7MTS4GJtajNjMzK0O9jqB+DHwDeDP3DweWRcQbub8ZGN3WhJKOlDRb0uzFixf3eqFmZlYfNQ8oSZ8Eno+Ie1Zn+og4LyLGRcS4hoaGHq7OzMxK0b8Oy9wN2E/SvsAgYCPgbGCopP75KKoRWFSH2szMrBA1P4KKiG9GRGNENAEHAX+IiM8DtwIH5GaHAtfWujYzMytHSf8HdSJwgqTHSNekLqhzPWZmVkf1OMX3loiYCczM3U8A4+tZj5mZlaOkIygzM7O3OKDMzKxIDigzMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMiuSAMjOzIjmgzMysSA4oMzMrkgPKzMyK5IAyM7MiOaDMzKxIDigzMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMiuSAMjOzIjmgzMysSA4oMzMrkgPKzMyK5IAyM7MiOaDMzKxIDigzMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMiuSAMjOzIjmgzMysSA4oMzMrUs0DStKmkm6VNFfSw5KOy8OHSZohaX7++65a12ZmZuWoxxHUG8DXImIbYFfgKEnbANOAWyJiS+CW3G9mZmupmgdURDwTEffm7peAecBoYAJwcW52MTCx1rWZmVk56noNSlITsCNwFzAyIp7Jo54FRrYzzZGSZkuavXjx4toUamZmNVe3gJK0AXAVcHxEvFg5LiICiLami4jzImJcRIxraGioQaVmZlYPdQkoSQNI4XRpRFydBz8naVQePwp4vh61mZlZGepxF5+AC4B5EfGjilHXAYfm7kOBa2tdm5mZlaN/HZa5G3AI8JCk+/OwfwNOB66QNBV4CjiwDrWZmVkhah5QEXE7oHZG71XLWszMrFz+JgkzMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMiuSAMjOzIjmgzMysSA4oMzMrkgPKzMyK5IAyM7MiOaDMzKxIDigzMyuSA8rMzIrkgDIzsyI5oMzMrEgOKDMzK5IDyszMilTzn3wvTdO0G3p8ngsG9fgszczWOj6CMjOzIjmgzMysSA4oMzMrkgPKzMyK5IAyM7MiOaDMzKxIDigzMyuSA8rMzIq01v+jrq2Z/A/YZn2fA8rM+gx/8Fi7+BSfmZkVyQFlZmZFckCZmVmRHFBmZlYkB5SZmRXJAWVmZkVyQJmZWZGKCihJ+0h6VNJjkqbVux4zM6ufYv5RV1I/4D+AjwLNwN2SrouIufWtbM3hf3I06zv8fi0ooIDxwGMR8QSApMuACYADyqwD3pHZmkoRUe8aAJB0ALBPRByR+w8B3hcRR7dqdyRwZO7dCni0poVWZwSwpN5F1IjXdc3kdV0zlbqum0VEQ+uBJR1BVSUizgPOq3cdHZE0OyLG1buOWvC6rpm8rmumvrauJd0ksQjYtKK/MQ8zM7O1UEkBdTewpaSxkgYCBwHX1bkmMzOrk2JO8UXEG5KOBm4G+gEXRsTDdS5rdRV9CrKHeV3XTF7XNVOfWtdibpIwMzOrVNIpPjMzs7c4oMzMrEgOqCpIGirpX1Zz2i9LmtzTNVnXSWqSNKfeddRK5etW0h6Sru+l5UyRtElvzLurJN3Rw/N76zUjaQdJ+/bk/K1jDqjqDAVWK6Ai4ucR8aueLadv6M6OS9Imkq7s6ZrWMkPp4us2f+VYV00BigioiPhAL85+B6BbAdVegEq6KH9ZwerMc5XglLRfy3eZSpooaZvVnO8CSSNWt46e4ICqzunA5pLul3RmfsyR9JCkSQCSzpb07dy9t6TbJK0j6WRJX8/Dt5D0e0kPSLpX0uZ1XKcuk9TVuz6nsJo7roh4OiJW6w3bQtIJ+XmaI+n4PLi/pEslzZN0paT1c9vTJc2V9KCkH+RhIyVdk5+vByR9IA//gqRZ+fVwbstOXdLLkk7Lbf8saWQe3iDpKkl358du3VmvLnjrdQucCWyQ1/mRvA2U61sg6QxJ9wKflfQxSXfm1+h/Stogt/t2rn+OpPOUHACMAy7N22O9Gq1bmyS9nP/uIWlmO+vb1nO9SkC0zKeifyDwXWBSXs9Jq1NfLwXoDlQEZ0RcFxGn596JwGoFVHfr6BER4UcnD6AJmJO7PwPMIN0KPxL4KzAKWB94GPgw6euXNs/tTwa+nrvvAvbP3YOA9XuovsHADcADwBxgErAz8EfgHtKt+6OArYFZrdbrodz9jvZ5+Ezgx8Bs4GvttWujpgOAl/O2uB9YD9gLuA94CLgQWBfYBXgwb4/BeRtu12qb9wN+kNftQeCYKrbJznk5g4EN8nx3BALYLbe5EPg6MDzX2XJX69D893Lg+IoahgD/CPw3MCAP///A5NwdwKdy9/eBk3L3b4Ddc/cYYF4dXrd7AMtJ/wC/DnBnRU0LgG/k7hHAbcDg3H8i8O3cPaxi3r+uWNeZwLh6v09zLS93tL4dPNcXAQe0MZ/KbTgFOKeH6hNwTq7l98CNLcun4/fiGcAs4C/APwMDSfugxaT32aSWOoEPAH8DnszjNgfurahly8r+NmpdAJwC3Et6L22dh4/P2/M+4A7SV861Vcdg0ntsVm47oavby0dQXbc7MD0iVkbEc6QX0i4R8SrwRVJ4nRMRj1dOJGlDYHREXAMQESvyND1hH+DpiHhvRGwH3AT8lPSC35n0IjktIh4BBkoam6ebBFwuaUBb7SvmPzDS16P8pJN2b4mIK0mh9vmI2IG0874ImBQR25P+B+8rEXE36R+yTyXt1C+JiNbXiY4k7Sh2iIh/Ai6tYpvsDlwTEa9ExMvA1aQ39MKI+FNuc0lutxxYAVwg6dNAy/OyJ/CzvD4rI2I5KWR3Jn3b/v25/z25/WtAy3Wee3LNAB8BzsntrwM2ajkqqbFZEdEcEW+SdiJNFeMuz393JX3i/lOu91Bgszzuw5LukvQQadtsW4uiu6Gt9W3vua61/Uk79m2AyaQwoYr3Yv+IGA8cD3wnIl4Dvg1cHhE7RETL80hE3EF6vf1rHvc4sFzSDrnJYcAvO6lzSUTsRHoffD0PewT454jYMS/7e+3U8e/AH3K9HwbOlDS4KxupmH/UXUNsDyyl9ufjHwJ+KOkM0g7yBdJRyIx8VqMf8ExuewUpmE7PfyeR3ijttYe3d16dtevIVsCTEfGX3H8xcBTp6Oy7pG8SWQEc28a0HwF+HhFvAETE36pcZlta/+NfRPon8fGksDkAOJq0A26LgIsj4pttjHs98kdMYCVvv7/WAXaNiBXdqLsn/L2iu7I+gFfyXwEzIuLgygklDSIdLY6LiIWSTiYd9ZbsHevbwXP9BvmSh6R1SEcEvemD5A+6wNOS/pCHd/Yeuzr/rfwA1BXnA4dJOoH03h/fSfvK5X06dw8BLpa0Jen9NKCdaT8G7NdyiYP0ehkDzKu2WB9BVeclYMPc/T+k89D9JDWQXmizJG1GOgW2I/BxSe+rnEFEvAQ0S5oIIGndlusf3ZV3+juRgupU0mnIh/MnmR0iYvuI+FhufjlwoKR/SJPGfNJOqb32sOrOq6N2q2s46TTchvTcTu9/gImS1s+f2vbPw8ZIen9u8zng9nw0MyQibgS+Crw3j78F+AqkmwckDcnDDpC0cR4+LD/3HfkdcExLT8Un2N5W+bqt1p+B3SRtASBpcH6ttDwvS/L2qrw+uDrLqYsOnusFpCNjgP1oe6dbi/Xs7D3WErqtP2BU6yrg48AngXsiYmkn7dta3v8Dbs1naz5F++9ZAZ+pWJcxEVF1OIEDqir5SfyT0u2m7yddB3kA+APwDeA54ALStaanganA+flTZ6VDgGMlPUg6d/vunqhP6U65VyPiEtLF8PcBDS07YkkDJG2b1+Vx0ovtW7x9ZPRoe+1bqbZdi8o39KNAU8uOj7Qt/pi7z831XEo6x97aDOBLyjdpSBrWwTLJ63kv6ZTiLNK1v/NJR5aPAkdJmge8i3TqYkPg+vy83A6ckGdzHOm01kOkT5DbRPoBzZOA3+X2M0jX9zpyLDAuX5SfC3y5s/p7QqvX7ZlVTrOYdA1jel6/O0nXHpYBvyBdB7yZdMTb4iLg5yXcJFGF9p7rXwAfkvQA6T3+ShvT3gps052bJCrcxtsfdEeRToFB199j0HFwrjIuH8XfTHrdd3Z6rz1DePuLvKd0UMfNwDEVN6fs2OUldfWilR/lPYC9SaF5P2nHMY50R81tpCB9GPhiRfuvkw7NmyqGtdmeVhfAO5pvG3V9hs5vkpgMXJXb9yOFyZ6senG6P/Aj0o9XPgAcXe9t7ocfq/Og7ZskZrDqTRKdvhdJN7MsyN3D8vv+fipuksjjdsvvm/t4+8atXUm/Wt6vk1oXACNy9zhgZu5+P+kmjftIZ2zaq2M90ofPh/J6XN/V7eXv4jMzW4vka0JDIuJb9a6lM75JwsxsLSHpGtLt5u3dBFQUH0FZt0n6D9KphEpnR8TqnuM2sxrJoTW21eATI+LmetRTyQFlZmZF8l18ZmZWJAeUmZkVyQFlZmZFckCZmVmR/g/EgGfUNZGa/wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average % of class 1 per label: 3.665870782702789\n",
      "Average % of class 0 per label: 96.33412921729722\n"
     ]
    }
   ],
   "source": [
    "ones, zeros = [], []\n",
    "for cat in categories:\n",
    "    zeros.append(df[cat].value_counts(normalize=True)[0] * 100)\n",
    "    ones.append(df[cat].value_counts(normalize=True)[1] * 100)\n",
    "\n",
    "\n",
    "x = np.arange(len(categories))  # the label locations\n",
    "width = 0.35  # the width of the bars\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "rects1 = ax.bar(x - width/2, ones, width, label='1')\n",
    "rects2 = ax.bar(x + width/2, zeros, width, label='0')\n",
    "\n",
    "ax.set_ylabel('%')\n",
    "ax.set_title('% of classes for each label')\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(categories)\n",
    "ax.legend()\n",
    "fig.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(f\"Average % of class 1 per label: {np.mean(ones)}\")\n",
    "print(f\"Average % of class 0 per label: {np.mean(zeros)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(106912, 149457)\n"
     ]
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer(stop_words=stop_words)\n",
    "X = vectorizer.fit_transform(X_train)\n",
    "print(X.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multinomial Naive Bayes <br/>\n",
    "https://scikit-learn.org/stable/modules/naive_bayes.html#multinomial-naive-bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.8986118232400919\n",
      "Test accuracy: 0.9029197536653224\n",
      "Validation roc: 0.8482772767640118\n",
      "Test roc: 0.8426850105845768\n",
      "Validation scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.99      0.17      0.28      5083\n",
      " severe_toxic       0.00      0.00      0.00       526\n",
      "      obscene       0.99      0.10      0.18      2831\n",
      "       threat       1.00      0.01      0.01       152\n",
      "       insult       0.98      0.04      0.08      2643\n",
      "identity_hate       0.00      0.00      0.00       471\n",
      "\n",
      "    micro avg       0.99      0.11      0.19     11706\n",
      "    macro avg       0.66      0.05      0.09     11706\n",
      " weighted avg       0.90      0.11      0.19     11706\n",
      "  samples avg       0.02      0.01      0.01     11706\n",
      "\n",
      "Test scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.91      0.19      0.32      6090\n",
      " severe_toxic       0.00      0.00      0.00       367\n",
      "      obscene       0.96      0.11      0.19      3691\n",
      "       threat       0.00      0.00      0.00       211\n",
      "       insult       0.92      0.04      0.08      3427\n",
      "identity_hate       0.00      0.00      0.00       712\n",
      "\n",
      "    micro avg       0.92      0.12      0.21     14498\n",
      "    macro avg       0.46      0.06      0.10     14498\n",
      " weighted avg       0.84      0.12      0.20     14498\n",
      "  samples avg       0.02      0.01      0.01     14498\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Define a pipeline combining a text feature extractor with multi lable classifier\n",
    "NB_pipeline = Pipeline([\n",
    "        ('tfidf', TfidfVectorizer(stop_words=stop_words)),\n",
    "        ('clf', OneVsRestClassifier(MultinomialNB(fit_prior=True, class_prior=None))),\n",
    "    ])\n",
    "\n",
    "NB_pipeline.fit(X_train, y_train)\n",
    "\n",
    "print(f'Validation accuracy: {NB_pipeline.score(X_valid, y_valid)}')\n",
    "print(f'Test accuracy: {NB_pipeline.score(X_test, y_test)}')\n",
    "\n",
    "y_valid_pred = NB_pipeline.predict(X_valid)\n",
    "y_test_pred = NB_pipeline.predict(X_test)\n",
    "\n",
    "print(f\"Validation roc: {roc_auc_score(y_valid, NB_pipeline.predict_proba(X_valid), multi_class='ovr')}\")\n",
    "print(f\"Test roc: {roc_auc_score(y_test, NB_pipeline.predict_proba(X_test), multi_class='ovr')}\")\n",
    "\n",
    "print(f\"Validation scores:\\n{classification_report(y_valid, y_valid_pred, target_names=categories)}\")\n",
    "print(f\"Test scores:\\n{classification_report(y_test, y_test_pred, target_names=categories)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LinearSVC <br/>\n",
    "https://scikit-learn.org/stable/modules/generated/sklearn.svm.LinearSVC.html#sklearn-svm-linearsvc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/anto/.local/lib/python3.8/site-packages/sklearn/svm/_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "/home/anto/.local/lib/python3.8/site-packages/sklearn/svm/_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "/home/anto/.local/lib/python3.8/site-packages/sklearn/svm/_base.py:985: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.8950226931768549\n",
      "Test accuracy: 0.8325049235674763\n",
      "Validation roc: 0.8263248616507596\n",
      "Test roc: 0.8429049488949042\n",
      "Validation scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.71      0.80      0.75      5083\n",
      " severe_toxic       0.33      0.61      0.43       526\n",
      "      obscene       0.76      0.82      0.79      2831\n",
      "       threat       0.42      0.49      0.45       152\n",
      "       insult       0.62      0.76      0.69      2643\n",
      "identity_hate       0.38      0.53      0.45       471\n",
      "\n",
      "    micro avg       0.66      0.77      0.71     11706\n",
      "    macro avg       0.54      0.67      0.59     11706\n",
      " weighted avg       0.67      0.77      0.72     11706\n",
      "  samples avg       0.07      0.07      0.07     11706\n",
      "\n",
      "Test scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.47      0.87      0.61      6090\n",
      " severe_toxic       0.17      0.71      0.28       367\n",
      "      obscene       0.53      0.82      0.64      3691\n",
      "       threat       0.31      0.59      0.40       211\n",
      "       insult       0.46      0.75      0.57      3427\n",
      "identity_hate       0.36      0.61      0.45       712\n",
      "\n",
      "    micro avg       0.46      0.81      0.58     14498\n",
      "    macro avg       0.38      0.72      0.49     14498\n",
      " weighted avg       0.47      0.81      0.59     14498\n",
      "  samples avg       0.07      0.08      0.07     14498\n",
      "\n"
     ]
    }
   ],
   "source": [
    "SVC_pipeline = Pipeline([\n",
    "                ('tfidf', TfidfVectorizer(stop_words=stop_words)),\n",
    "                ('clf', OneVsRestClassifier(LinearSVC(class_weight='balanced'), n_jobs=1)),\n",
    "            ])\n",
    "\n",
    "SVC_pipeline.fit(X_train, y_train)\n",
    "\n",
    "print(f'Validation accuracy: {SVC_pipeline.score(X_valid, y_valid)}')\n",
    "print(f'Test accuracy: {SVC_pipeline.score(X_test, y_test)}')\n",
    "\n",
    "y_valid_pred = SVC_pipeline.predict(X_valid)\n",
    "y_test_pred = SVC_pipeline.predict(X_test)\n",
    "\n",
    "print(f\"Validation roc: {roc_auc_score(y_valid, SVC_pipeline.predict(X_valid), multi_class='ovr')}\")\n",
    "print(f\"Test roc: {roc_auc_score(y_test, SVC_pipeline.predict(X_test), multi_class='ovr')}\")\n",
    "\n",
    "print(f\"Validation scores:\\n{classification_report(y_valid, y_valid_pred, target_names=categories)}\")\n",
    "print(f\"Test scores:\\n{classification_report(y_test, y_test_pred, target_names=categories)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression <br/>\n",
    "https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn-linear-model-logisticregression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.8800964697392658\n",
      "Test accuracy: 0.8130294788833662\n",
      "Validation roc: 0.978224053405245\n",
      "Test roc: 0.9734228374482244\n",
      "Validation scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.66      0.85      0.74      5083\n",
      " severe_toxic       0.28      0.83      0.42       526\n",
      "      obscene       0.71      0.87      0.78      2831\n",
      "       threat       0.22      0.71      0.34       152\n",
      "       insult       0.57      0.85      0.68      2643\n",
      "identity_hate       0.25      0.74      0.37       471\n",
      "\n",
      "    micro avg       0.57      0.85      0.68     11706\n",
      "    macro avg       0.45      0.81      0.56     11706\n",
      " weighted avg       0.61      0.85      0.70     11706\n",
      "  samples avg       0.06      0.08      0.07     11706\n",
      "\n",
      "Test scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.44      0.90      0.59      6090\n",
      " severe_toxic       0.14      0.89      0.24       367\n",
      "      obscene       0.47      0.87      0.61      3691\n",
      "       threat       0.18      0.79      0.30       211\n",
      "       insult       0.40      0.86      0.54      3427\n",
      "identity_hate       0.23      0.80      0.36       712\n",
      "\n",
      "    micro avg       0.39      0.88      0.54     14498\n",
      "    macro avg       0.31      0.85      0.44     14498\n",
      " weighted avg       0.42      0.88      0.56     14498\n",
      "  samples avg       0.06      0.08      0.07     14498\n",
      "\n"
     ]
    }
   ],
   "source": [
    "LogReg_pipeline = Pipeline([\n",
    "                ('tfidf', TfidfVectorizer(stop_words=stop_words)),\n",
    "                ('clf', OneVsRestClassifier(LogisticRegression(solver='lbfgs', class_weight='balanced'), n_jobs=1)),\n",
    "            ])\n",
    "\n",
    "LogReg_pipeline.fit(X_train, y_train)\n",
    "\n",
    "print(f'Validation accuracy: {LogReg_pipeline.score(X_valid, y_valid)}')\n",
    "print(f'Test accuracy: {LogReg_pipeline.score(X_test, y_test)}')\n",
    "\n",
    "y_valid_pred = LogReg_pipeline.predict(X_valid)\n",
    "y_test_pred = LogReg_pipeline.predict(X_test)\n",
    "\n",
    "print(f\"Validation roc: {roc_auc_score(y_valid, LogReg_pipeline.predict_proba(X_valid), multi_class='ovr')}\")\n",
    "print(f\"Test roc: {roc_auc_score(y_test, LogReg_pipeline.predict_proba(X_test), multi_class='ovr')}\")\n",
    "\n",
    "print(f\"Validation scores:\\n{classification_report(y_valid, y_valid_pred, target_names=categories)}\")\n",
    "print(f\"Test scores:\\n{classification_report(y_test, y_test_pred, target_names=categories)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RandomForestClassifier <br/>\n",
    "https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html#sklearn-ensemble-randomforestclassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.8975293871892744\n",
      "Test accuracy: 0.90241958173122\n",
      "Validation roc: 0.5\n",
      "Test roc: 0.5\n",
      "Validation scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.00      0.00      0.00      5083\n",
      " severe_toxic       0.00      0.00      0.00       526\n",
      "      obscene       0.00      0.00      0.00      2831\n",
      "       threat       0.00      0.00      0.00       152\n",
      "       insult       0.00      0.00      0.00      2643\n",
      "identity_hate       0.00      0.00      0.00       471\n",
      "\n",
      "    micro avg       0.00      0.00      0.00     11706\n",
      "    macro avg       0.00      0.00      0.00     11706\n",
      " weighted avg       0.00      0.00      0.00     11706\n",
      "  samples avg       0.00      0.00      0.00     11706\n",
      "\n",
      "Test scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.00      0.00      0.00      6090\n",
      " severe_toxic       0.00      0.00      0.00       367\n",
      "      obscene       0.00      0.00      0.00      3691\n",
      "       threat       0.00      0.00      0.00       211\n",
      "       insult       0.00      0.00      0.00      3427\n",
      "identity_hate       0.00      0.00      0.00       712\n",
      "\n",
      "    micro avg       0.00      0.00      0.00     14498\n",
      "    macro avg       0.00      0.00      0.00     14498\n",
      " weighted avg       0.00      0.00      0.00     14498\n",
      "  samples avg       0.00      0.00      0.00     14498\n",
      "\n"
     ]
    }
   ],
   "source": [
    "RF_pipeline = Pipeline([\n",
    "                ('tfidf', TfidfVectorizer(stop_words=stop_words)),\n",
    "                ('clf', RandomForestClassifier(n_estimators=25, max_depth=2, random_state=0)),\n",
    "            ])\n",
    "\n",
    "RF_pipeline.fit(X_train, y_train)\n",
    "\n",
    "print(f'Validation accuracy: {RF_pipeline.score(X_valid, y_valid)}')\n",
    "print(f'Test accuracy: {RF_pipeline.score(X_test, y_test)}')\n",
    "\n",
    "y_valid_pred = RF_pipeline.predict(X_valid)\n",
    "y_test_pred = RF_pipeline.predict(X_test)\n",
    "\n",
    "y_valid_pred_p = RF_pipeline.predict_proba(X_valid)\n",
    "y_test_pred_p = RF_pipeline.predict_proba(X_test)\n",
    "\n",
    "print(f\"Validation roc: {roc_auc_score(y_valid, y_valid_pred, multi_class='ovr')}\")\n",
    "print(f\"Test roc: {roc_auc_score(y_test, y_test_pred, multi_class='ovr')}\")\n",
    "\n",
    "print(f\"Validation scores:\\n{classification_report(y_valid, y_valid_pred, target_names=categories)}\")\n",
    "print(f\"Test scores:\\n{classification_report(y_test, y_test_pred, target_names=categories)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KNeighborsClassifier <br/>\n",
    "https://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html#sklearn-neighbors-kneighborsclassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy: 0.8542889154750375\n",
      "Test accuracy: 0.8524336490668667\n",
      "Validation roc: 0.5606735529358775\n",
      "Test roc: 0.5633477786785899\n",
      "Validation scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.29      0.21      0.24      5083\n",
      " severe_toxic       0.47      0.10      0.17       526\n",
      "      obscene       0.39      0.20      0.26      2831\n",
      "       threat       0.57      0.05      0.10       152\n",
      "       insult       0.76      0.15      0.25      2643\n",
      "identity_hate       0.64      0.09      0.16       471\n",
      "\n",
      "    micro avg       0.36      0.18      0.24     11706\n",
      "    macro avg       0.52      0.13      0.20     11706\n",
      " weighted avg       0.45      0.18      0.24     11706\n",
      "  samples avg       0.02      0.02      0.02     11706\n",
      "\n",
      "Test scores:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "        toxic       0.28      0.25      0.26      6090\n",
      " severe_toxic       0.21      0.08      0.11       367\n",
      "      obscene       0.35      0.19      0.25      3691\n",
      "       threat       0.33      0.07      0.12       211\n",
      "       insult       0.63      0.15      0.25      3427\n",
      "identity_hate       0.59      0.12      0.19       712\n",
      "\n",
      "    micro avg       0.34      0.20      0.25     14498\n",
      "    macro avg       0.40      0.14      0.20     14498\n",
      " weighted avg       0.40      0.20      0.25     14498\n",
      "  samples avg       0.02      0.02      0.02     14498\n",
      "\n"
     ]
    }
   ],
   "source": [
    "KN_pipeline = Pipeline([\n",
    "                ('tfidf', TfidfVectorizer(stop_words=stop_words)),\n",
    "                ('clf', KNeighborsClassifier(n_neighbors=5)),\n",
    "            ])\n",
    "\n",
    "KN_pipeline.fit(X_train, y_train)\n",
    "\n",
    "print(f'Validation accuracy: {KN_pipeline.score(X_valid, y_valid)}')\n",
    "print(f'Test accuracy: {KN_pipeline.score(X_test, y_test)}')\n",
    "\n",
    "y_valid_pred = KN_pipeline.predict(X_valid)\n",
    "y_test_pred = KN_pipeline.predict(X_test)\n",
    "\n",
    "print(f\"Validation roc: {roc_auc_score(y_valid, y_valid_pred, multi_class='ovr')}\")\n",
    "print(f\"Test roc: {roc_auc_score(y_test, y_test_pred, multi_class='ovr')}\")\n",
    "\n",
    "print(f\"Validation scores:\\n{classification_report(y_valid, y_valid_pred, target_names=categories)}\")\n",
    "print(f\"Test scores:\\n{classification_report(y_test, y_test_pred, target_names=categories)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLPClassifier <br/>\n",
    "https://scikit-learn.org/stable/modules/generated/sklearn.neural_network.MLPClassifier.html#sklearn-neural-network-mlpclassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MLP_pipeline = Pipeline([\n",
    "                ('tfidf', TfidfVectorizer(stop_words=stop_words)),\n",
    "                ('clf', MLPClassifier(max_iter=100, hidden_layer_sizes=100, activation='relu', solver='adam', verbose=True)),\n",
    "            ])\n",
    "\n",
    "MLP_pipeline.fit(X_train, y_train)\n",
    "\n",
    "print(f'Validation accuracy: {MLP_pipeline.score(X_valid, y_valid)}')\n",
    "print(f'Test accuracy: {MLP_pipeline.score(X_test, y_test)}')\n",
    "\n",
    "y_valid_pred = MLP_pipeline.predict(X_valid)\n",
    "y_test_pred = MLP_pipeline.predict(X_test)\n",
    "\n",
    "print(f\"Validation roc: {roc_auc_score(y_valid, MLP_pipeline.predict_proba(X_valid), multi_class='ovr')}\")\n",
    "print(f\"Test roc: {roc_auc_score(y_test, MLP_pipeline.predict_proba(X_test), multi_class='ovr')}\")\n",
    "\n",
    "print(f\"Validation scores:\\n{classification_report(y_valid, y_valid_pred, target_names=categories)}\")\n",
    "print(f\"Test scores:\\n{classification_report(y_test, y_test_pred, target_names=categories)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "toxic",
   "language": "python",
   "name": "toxic"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}