disaster-tweets.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "id": "5eb60d84",
   "metadata": {
    "papermill": {
     "duration": 0.017179,
     "end_time": "2023-12-26T07:08:50.946595",
     "exception": false,
     "start_time": "2023-12-26T07:08:50.929416",
     "status": "completed"
    },
    "tags": []
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "23c2211b",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:50.982599Z",
     "iopub.status.busy": "2023-12-26T07:08:50.981927Z",
     "iopub.status.idle": "2023-12-26T07:08:51.317539Z",
     "shell.execute_reply": "2023-12-26T07:08:51.316200Z"
    },
    "papermill": {
     "duration": 0.357071,
     "end_time": "2023-12-26T07:08:51.320508",
     "exception": false,
     "start_time": "2023-12-26T07:08:50.963437",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/distil_bert/keras/distil_bert_base_en_uncased/1/config.json\n",
      "/kaggle/input/distil_bert/keras/distil_bert_base_en_uncased/1/tokenizer.json\n",
      "/kaggle/input/distil_bert/keras/distil_bert_base_en_uncased/1/metadata.json\n",
      "/kaggle/input/distil_bert/keras/distil_bert_base_en_uncased/1/model.weights.h5\n",
      "/kaggle/input/distil_bert/keras/distil_bert_base_en_uncased/1/assets/tokenizer/vocabulary.txt\n",
      "/kaggle/input/nlp-getting-started/sample_submission.csv\n",
      "/kaggle/input/nlp-getting-started/train.csv\n",
      "/kaggle/input/nlp-getting-started/test.csv\n",
      "/kaggle/input/bert/tensorflow2/en-cased-preprocess/3/saved_model.pb\n",
      "/kaggle/input/bert/tensorflow2/en-cased-preprocess/3/keras_metadata.pb\n",
      "/kaggle/input/bert/tensorflow2/en-cased-preprocess/3/assets/vocab.txt\n",
      "/kaggle/input/bert/tensorflow2/en-cased-preprocess/3/variables/variables.index\n",
      "/kaggle/input/bert/tensorflow2/en-cased-preprocess/3/variables/variables.data-00000-of-00001\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the read-only \"../input/\" directory\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n",
    "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "183125ac",
   "metadata": {
    "papermill": {
     "duration": 0.016957,
     "end_time": "2023-12-26T07:08:51.356223",
     "exception": false,
     "start_time": "2023-12-26T07:08:51.339266",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Disaster tweets\n",
    "\n",
    "0 = Not a disaster\n",
    "1 = a disaster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "80e2b5e0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:51.396314Z",
     "iopub.status.busy": "2023-12-26T07:08:51.395752Z",
     "iopub.status.idle": "2023-12-26T07:08:51.487931Z",
     "shell.execute_reply": "2023-12-26T07:08:51.486636Z"
    },
    "papermill": {
     "duration": 0.117163,
     "end_time": "2023-12-26T07:08:51.490611",
     "exception": false,
     "start_time": "2023-12-26T07:08:51.373448",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>10869</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Two giant cranes holding a bridge collapse int...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>10870</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>@aria_ahrary @TheTawniest The out of control w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>10871</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>10872</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Police investigating after an e-bike collided ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>10873</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The Latest: More Homes Razed by Northern Calif...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id keyword location  \\\n",
       "0         1     NaN      NaN   \n",
       "1         4     NaN      NaN   \n",
       "2         5     NaN      NaN   \n",
       "3         6     NaN      NaN   \n",
       "4         7     NaN      NaN   \n",
       "...     ...     ...      ...   \n",
       "7608  10869     NaN      NaN   \n",
       "7609  10870     NaN      NaN   \n",
       "7610  10871     NaN      NaN   \n",
       "7611  10872     NaN      NaN   \n",
       "7612  10873     NaN      NaN   \n",
       "\n",
       "                                                   text  target  \n",
       "0     Our Deeds are the Reason of this #earthquake M...       1  \n",
       "1                Forest fire near La Ronge Sask. Canada       1  \n",
       "2     All residents asked to 'shelter in place' are ...       1  \n",
       "3     13,000 people receive #wildfires evacuation or...       1  \n",
       "4     Just got sent this photo from Ruby #Alaska as ...       1  \n",
       "...                                                 ...     ...  \n",
       "7608  Two giant cranes holding a bridge collapse int...       1  \n",
       "7609  @aria_ahrary @TheTawniest The out of control w...       1  \n",
       "7610  M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...       1  \n",
       "7611  Police investigating after an e-bike collided ...       1  \n",
       "7612  The Latest: More Homes Razed by Northern Calif...       1  \n",
       "\n",
       "[7613 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "train_df = pd.read_csv(\"/kaggle/input/nlp-getting-started/train.csv\")\n",
    "test_df = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')\n",
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "005dd7ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:51.526616Z",
     "iopub.status.busy": "2023-12-26T07:08:51.526044Z",
     "iopub.status.idle": "2023-12-26T07:08:51.540503Z",
     "shell.execute_reply": "2023-12-26T07:08:51.539724Z"
    },
    "papermill": {
     "duration": 0.034266,
     "end_time": "2023-12-26T07:08:51.542563",
     "exception": false,
     "start_time": "2023-12-26T07:08:51.508297",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target\n",
       "0    4342\n",
       "1    3271\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['target'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bb63e500",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:51.641351Z",
     "iopub.status.busy": "2023-12-26T07:08:51.640726Z",
     "iopub.status.idle": "2023-12-26T07:08:51.662032Z",
     "shell.execute_reply": "2023-12-26T07:08:51.661224Z"
    },
    "papermill": {
     "duration": 0.044161,
     "end_time": "2023-12-26T07:08:51.663961",
     "exception": false,
     "start_time": "2023-12-26T07:08:51.619800",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7613 entries, 0 to 7612\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   id        7613 non-null   int64 \n",
      " 1   keyword   7552 non-null   object\n",
      " 2   location  5080 non-null   object\n",
      " 3   text      7613 non-null   object\n",
      " 4   target    7613 non-null   int64 \n",
      "dtypes: int64(2), object(3)\n",
      "memory usage: 297.5+ KB\n"
     ]
    }
   ],
   "source": [
    "# check for null data\n",
    "train_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f2326a49",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:51.701251Z",
     "iopub.status.busy": "2023-12-26T07:08:51.700562Z",
     "iopub.status.idle": "2023-12-26T07:08:51.708362Z",
     "shell.execute_reply": "2023-12-26T07:08:51.706976Z"
    },
    "papermill": {
     "duration": 0.028959,
     "end_time": "2023-12-26T07:08:51.710331",
     "exception": false,
     "start_time": "2023-12-26T07:08:51.681372",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       Our Deeds are the Reason of this #earthquake M...\n",
       "1                  Forest fire near La Ronge Sask. Canada\n",
       "2       All residents asked to 'shelter in place' are ...\n",
       "3       13,000 people receive #wildfires evacuation or...\n",
       "4       Just got sent this photo from Ruby #Alaska as ...\n",
       "                              ...                        \n",
       "7608    Two giant cranes holding a bridge collapse int...\n",
       "7609    @aria_ahrary @TheTawniest The out of control w...\n",
       "7610    M1.94 [01:04 UTC]?5km S of Volcano Hawaii. htt...\n",
       "7611    Police investigating after an e-bike collided ...\n",
       "7612    The Latest: More Homes Razed by Northern Calif...\n",
       "Name: text, Length: 7613, dtype: object"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = train_df[\"text\"]\n",
    "train_labels = train_df[\"target\"]\n",
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7d4a890b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:51.747983Z",
     "iopub.status.busy": "2023-12-26T07:08:51.747581Z",
     "iopub.status.idle": "2023-12-26T07:08:53.129737Z",
     "shell.execute_reply": "2023-12-26T07:08:53.128249Z"
    },
    "papermill": {
     "duration": 1.404259,
     "end_time": "2023-12-26T07:08:53.132493",
     "exception": false,
     "start_time": "2023-12-26T07:08:51.728234",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6851, 6851, 762, 762)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# split the train_data into train and valid sets \n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(train_data,\n",
    "                                                     train_labels,\n",
    "                                                     test_size=0.1,\n",
    "                                                     random_state=42,\n",
    "                                                     shuffle=True)\n",
    "len(X_train), len(y_train), len(X_valid), len(y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3adf5cdf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:53.173665Z",
     "iopub.status.busy": "2023-12-26T07:08:53.173231Z",
     "iopub.status.idle": "2023-12-26T07:08:53.179504Z",
     "shell.execute_reply": "2023-12-26T07:08:53.178875Z"
    },
    "papermill": {
     "duration": 0.029627,
     "end_time": "2023-12-26T07:08:53.181699",
     "exception": false,
     "start_time": "2023-12-26T07:08:53.152072",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Forest fire near La Ronge Sask. Canada', 1)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[1], y_train[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccd29d45",
   "metadata": {
    "papermill": {
     "duration": 0.018212,
     "end_time": "2023-12-26T07:08:53.217694",
     "exception": false,
     "start_time": "2023-12-26T07:08:53.199482",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Prepare the data for Neural Networks\n",
    "\n",
    "* Vectorization layer will convert text to numbers\n",
    "* Embedding layers will capture relationships between those numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d44eccb7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:53.255327Z",
     "iopub.status.busy": "2023-12-26T07:08:53.254459Z",
     "iopub.status.idle": "2023-12-26T07:08:53.267453Z",
     "shell.execute_reply": "2023-12-26T07:08:53.266840Z"
    },
    "papermill": {
     "duration": 0.033676,
     "end_time": "2023-12-26T07:08:53.269289",
     "exception": false,
     "start_time": "2023-12-26T07:08:53.235613",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14.87447088016348"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# convert text to numbers\n",
    "import numpy as np \n",
    "\n",
    "tweet_lengths = [len(tweet.split()) for tweet in X_train] # an array of all tweet lengths\n",
    "avg_tweet_length = np.mean(tweet_lengths)\n",
    "avg_tweet_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e7e696b4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:53.307555Z",
     "iopub.status.busy": "2023-12-26T07:08:53.306847Z",
     "iopub.status.idle": "2023-12-26T07:08:53.647535Z",
     "shell.execute_reply": "2023-12-26T07:08:53.646579Z"
    },
    "papermill": {
     "duration": 0.362592,
     "end_time": "2023-12-26T07:08:53.650241",
     "exception": false,
     "start_time": "2023-12-26T07:08:53.287649",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 355.,  974., 1530., 2081., 1242.,  556.,  113.]),\n",
       " array([ 1.        ,  5.28571429,  9.57142857, 13.85714286, 18.14285714,\n",
       "        22.42857143, 26.71428571, 31.        ]),\n",
       " <BarContainer object of 7 artists>)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArEklEQVR4nO3df3AUZZ7H8c8QmOGHyYQQksmcIQQ8wQhEjRqnlCwsuYSYZfVg70RQcM3C6gb3JMqGeC4GvDJUuMMfuxyWtSJbdaDoleAJuxwBJNmVABLMBVBTwgWjRSa4KjMQJCSk7w+LPmcJmmDi5Ener6qumn6ep7u/3XRVPvQ8M+OwLMsSAACAQfqFuwAAAIDOIsAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIzTP9wFdJe2tjYdP35ckZGRcjgc4S4HAAB0gGVZOnXqlLxer/r1u/Rzll4bYI4fP67ExMRwlwEAAC7Dxx9/rCuvvPKS/b02wERGRkr66gJERUWFuRoAANARwWBQiYmJ9t/xS+m1AebC20ZRUVEEGAAADPNt0z+YxAsAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgnP7hLgCAmUYu3hLuEsLq2PLccJcA9Gk8gQEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxulUgCkpKdFNN92kyMhIxcXF6c4771RtbW3ImLNnzyo/P1/Dhg3TFVdcoRkzZqixsTFkTH19vXJzczV48GDFxcVp0aJFam1tDRmza9cu3XDDDXK5XLrqqqu0du3ayztDAADQ63QqwJSXlys/P1979uxRWVmZWlpalJWVpaamJnvMwoUL9eabb+q1115TeXm5jh8/runTp9v958+fV25urs6dO6fdu3fr97//vdauXaslS5bYY+rq6pSbm6vJkyerurpaDz/8sH72s5/pv//7v7vglAEAgOkclmVZl7vxp59+qri4OJWXlysjI0OBQEDDhw/X+vXr9ZOf/ESS9MEHH+iaa65RZWWlbrnlFv3xj3/Uj370Ix0/flzx8fGSpOeff16FhYX69NNP5XQ6VVhYqC1btujQoUP2sWbOnKmTJ09q69atHaotGAzK7XYrEAgoKirqck8RwCXwW0j8FhLQHTr69/s7zYEJBAKSpJiYGElSVVWVWlpalJmZaY8ZO3asRowYocrKSklSZWWlxo8fb4cXScrOzlYwGNThw4ftMV/fx4UxF/bRnubmZgWDwZAFAAD0TpcdYNra2vTwww/r1ltv1bhx4yRJfr9fTqdT0dHRIWPj4+Pl9/vtMV8PLxf6L/R905hgMKgvv/yy3XpKSkrkdrvtJTEx8XJPDQAA9HCXHWDy8/N16NAhvfLKK11Zz2UrKipSIBCwl48//jjcJQEAgG7S/3I2WrBggTZv3qyKigpdeeWVdrvH49G5c+d08uTJkKcwjY2N8ng89ph9+/aF7O/Cp5S+PuavP7nU2NioqKgoDRo0qN2aXC6XXC7X5ZwOAAAwTKeewFiWpQULFmjjxo3auXOnkpOTQ/rT0tI0YMAA7dixw26rra1VfX29fD6fJMnn8+ngwYM6ceKEPaasrExRUVFKSUmxx3x9HxfGXNgHAADo2zr1BCY/P1/r16/XG2+8ocjISHvOitvt1qBBg+R2u5WXl6eCggLFxMQoKipKDz30kHw+n2655RZJUlZWllJSUnTvvfeqtLRUfr9fjz/+uPLz8+0nKA888IB++9vf6le/+pXuv/9+7dy5U6+++qq2bOnbn3oAAABf6dQTmNWrVysQCGjSpElKSEiwlw0bNthjnn76af3oRz/SjBkzlJGRIY/Ho9dff93uj4iI0ObNmxURESGfz6d77rlHc+bM0bJly+wxycnJ2rJli8rKypSamqp/+7d/0+9+9ztlZ2d3wSkDAADTfafvgenJ+B4YoHvxPTB8DwzQHb6X74EBAAAIBwIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxOh1gKioqNG3aNHm9XjkcDm3atCmk3+FwtLusWLHCHjNy5MiL+pcvXx6yn5qaGk2cOFEDBw5UYmKiSktLL+8MAQBAr9PpANPU1KTU1FStWrWq3f6GhoaQZc2aNXI4HJoxY0bIuGXLloWMe+ihh+y+YDCorKwsJSUlqaqqSitWrFBxcbFeeOGFzpYLAAB6of6d3SAnJ0c5OTmX7Pd4PCHrb7zxhiZPnqxRo0aFtEdGRl409oJ169bp3LlzWrNmjZxOp6699lpVV1dr5cqVmj9/fmdLBgAAvUy3zoFpbGzUli1blJeXd1Hf8uXLNWzYMF1//fVasWKFWltb7b7KykplZGTI6XTabdnZ2aqtrdUXX3zR7rGam5sVDAZDFgAA0Dt1+glMZ/z+979XZGSkpk+fHtL+y1/+UjfccINiYmK0e/duFRUVqaGhQStXrpQk+f1+JScnh2wTHx9v9w0dOvSiY5WUlGjp0qXddCYAAKAn6dYAs2bNGs2ePVsDBw4MaS8oKLBfT5gwQU6nUz//+c9VUlIil8t1WccqKioK2W8wGFRiYuLlFQ4AAHq0bgswf/rTn1RbW6sNGzZ869j09HS1trbq2LFjGjNmjDwejxobG0PGXFi/1LwZl8t12eEHAACYpdvmwLz44otKS0tTamrqt46trq5Wv379FBcXJ0ny+XyqqKhQS0uLPaasrExjxoxp9+0jAADQt3Q6wJw+fVrV1dWqrq6WJNXV1am6ulr19fX2mGAwqNdee00/+9nPLtq+srJSzzzzjP7nf/5H//u//6t169Zp4cKFuueee+xwMmvWLDmdTuXl5enw4cPasGGDnn322ZC3iAAAQN/V6beQ9u/fr8mTJ9vrF0LF3LlztXbtWknSK6+8IsuydPfdd1+0vcvl0iuvvKLi4mI1NzcrOTlZCxcuDAknbrdb27ZtU35+vtLS0hQbG6slS5bwEWoAACBJcliWZYW7iO4QDAbldrsVCAQUFRUV7nKAXmfk4i3hLiGsji3PDXcJQK/U0b/f/BYSAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBx+oe7AMBEIxdvCXcJANCndfoJTEVFhaZNmyav1yuHw6FNmzaF9N93331yOBwhy9SpU0PGfP7555o9e7aioqIUHR2tvLw8nT59OmRMTU2NJk6cqIEDByoxMVGlpaWdPzsAANArdTrANDU1KTU1VatWrbrkmKlTp6qhocFeXn755ZD+2bNn6/DhwyorK9PmzZtVUVGh+fPn2/3BYFBZWVlKSkpSVVWVVqxYoeLiYr3wwgudLRcAAPRCnX4LKScnRzk5Od84xuVyyePxtNv3/vvva+vWrXrnnXd04403SpJ+85vf6Pbbb9e//uu/yuv1at26dTp37pzWrFkjp9Opa6+9VtXV1Vq5cmVI0AEAAH1Tt0zi3bVrl+Li4jRmzBg9+OCD+uyzz+y+yspKRUdH2+FFkjIzM9WvXz/t3bvXHpORkSGn02mPyc7OVm1trb744ovuKBkAABikyyfxTp06VdOnT1dycrKOHj2qxx57TDk5OaqsrFRERIT8fr/i4uJCi+jfXzExMfL7/ZIkv9+v5OTkkDHx8fF239ChQy86bnNzs5qbm+31YDDY1acGAAB6iC4PMDNnzrRfjx8/XhMmTNDo0aO1a9cuTZkypasPZyspKdHSpUu7bf8AAKDn6PbvgRk1apRiY2N15MgRSZLH49GJEydCxrS2turzzz+35814PB41NjaGjLmwfqm5NUVFRQoEAvby8ccfd/WpAACAHqLbA8wnn3yizz77TAkJCZIkn8+nkydPqqqqyh6zc+dOtbW1KT093R5TUVGhlpYWe0xZWZnGjBnT7ttH0lcTh6OiokIWAADQO3U6wJw+fVrV1dWqrq6WJNXV1am6ulr19fU6ffq0Fi1apD179ujYsWPasWOH7rjjDl111VXKzs6WJF1zzTWaOnWq5s2bp3379untt9/WggULNHPmTHm9XknSrFmz5HQ6lZeXp8OHD2vDhg169tlnVVBQ0HVnDgAAjNXpALN//35df/31uv766yVJBQUFuv7667VkyRJFRESopqZGP/7xj3X11VcrLy9PaWlp+tOf/iSXy2XvY926dRo7dqymTJmi22+/XbfddlvId7y43W5t27ZNdXV1SktL0yOPPKIlS5bwEWoAACBJcliWZYW7iO4QDAbldrsVCAR4Owldjp8SwLHlueEuAeiVOvr3mx9zBAAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACM0+kAU1FRoWnTpsnr9crhcGjTpk12X0tLiwoLCzV+/HgNGTJEXq9Xc+bM0fHjx0P2MXLkSDkcjpBl+fLlIWNqamo0ceJEDRw4UImJiSotLb28MwQAAL1OpwNMU1OTUlNTtWrVqov6zpw5owMHDujXv/61Dhw4oNdff121tbX68Y9/fNHYZcuWqaGhwV4eeughuy8YDCorK0tJSUmqqqrSihUrVFxcrBdeeKGz5QIAgF6of2c3yMnJUU5OTrt9brdbZWVlIW2//e1vdfPNN6u+vl4jRoyw2yMjI+XxeNrdz7p163Tu3DmtWbNGTqdT1157raqrq7Vy5UrNnz+/syUDAIBeptvnwAQCATkcDkVHR4e0L1++XMOGDdP111+vFStWqLW11e6rrKxURkaGnE6n3Zadna3a2lp98cUX7R6nublZwWAwZAEAAL1Tp5/AdMbZs2dVWFiou+++W1FRUXb7L3/5S91www2KiYnR7t27VVRUpIaGBq1cuVKS5Pf7lZycHLKv+Ph4u2/o0KEXHaukpERLly7txrMBAAA9RbcFmJaWFv3jP/6jLMvS6tWrQ/oKCgrs1xMmTJDT6dTPf/5zlZSUyOVyXdbxioqKQvYbDAaVmJh4ecUDwLcYuXhLuEsIq2PLc8NdAvq4bgkwF8LLRx99pJ07d4Y8fWlPenq6WltbdezYMY0ZM0Yej0eNjY0hYy6sX2rejMvluuzwAwAAzNLlc2AuhJcPP/xQ27dv17Bhw751m+rqavXr109xcXGSJJ/Pp4qKCrW0tNhjysrKNGbMmHbfPgIAAH1Lp5/AnD59WkeOHLHX6+rqVF1drZiYGCUkJOgnP/mJDhw4oM2bN+v8+fPy+/2SpJiYGDmdTlVWVmrv3r2aPHmyIiMjVVlZqYULF+qee+6xw8msWbO0dOlS5eXlqbCwUIcOHdKzzz6rp59+uotOGwAAmMxhWZbVmQ127dqlyZMnX9Q+d+5cFRcXXzT59oK33npLkyZN0oEDB/SLX/xCH3zwgZqbm5WcnKx7771XBQUFIW8B1dTUKD8/X++8845iY2P10EMPqbCwsMN1BoNBud1uBQKBb30LC+isvj7/AWAODLpLR/9+dzrAmIIAg+5EgEFfR4BBd+no329+CwkAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYJxOB5iKigpNmzZNXq9XDodDmzZtCum3LEtLlixRQkKCBg0apMzMTH344YchYz7//HPNnj1bUVFRio6OVl5enk6fPh0ypqamRhMnTtTAgQOVmJio0tLSzp8dAADolTodYJqampSamqpVq1a1219aWqrnnntOzz//vPbu3ashQ4YoOztbZ8+etcfMnj1bhw8fVllZmTZv3qyKigrNnz/f7g8Gg8rKylJSUpKqqqq0YsUKFRcX64UXXriMUwQAAL2Nw7Is67I3dji0ceNG3XnnnZK+evri9Xr1yCOP6NFHH5UkBQIBxcfHa+3atZo5c6bef/99paSk6J133tGNN94oSdq6datuv/12ffLJJ/J6vVq9erX++Z//WX6/X06nU5K0ePFibdq0SR988EGHagsGg3K73QoEAoqKirrcUwTaNXLxlnCXAITVseW54S4BvVRH/3536RyYuro6+f1+ZWZm2m1ut1vp6emqrKyUJFVWVio6OtoOL5KUmZmpfv36ae/evfaYjIwMO7xIUnZ2tmpra/XFF1+0e+zm5mYFg8GQBQAA9E5dGmD8fr8kKT4+PqQ9Pj7e7vP7/YqLiwvp79+/v2JiYkLGtLePrx/jr5WUlMjtdttLYmLidz8hAADQI/WaTyEVFRUpEAjYy8cffxzukgAAQDfp0gDj8XgkSY2NjSHtjY2Ndp/H49GJEydC+ltbW/X555+HjGlvH18/xl9zuVyKiooKWQAAQO/UpQEmOTlZHo9HO3bssNuCwaD27t0rn88nSfL5fDp58qSqqqrsMTt37lRbW5vS09PtMRUVFWppabHHlJWVacyYMRo6dGhXlgwAAAzU6QBz+vRpVVdXq7q6WtJXE3erq6tVX18vh8Ohhx9+WP/yL/+i//qv/9LBgwc1Z84ceb1e+5NK11xzjaZOnap58+Zp3759evvtt7VgwQLNnDlTXq9XkjRr1iw5nU7l5eXp8OHD2rBhg5599lkVFBR02YkDAABz9e/sBvv379fkyZPt9QuhYu7cuVq7dq1+9atfqampSfPnz9fJkyd12223aevWrRo4cKC9zbp167RgwQJNmTJF/fr104wZM/Tcc8/Z/W63W9u2bVN+fr7S0tIUGxurJUuWhHxXDMKLjxEDAMLpO30PTE/G98B0LwIM0LfxPTDoLmH5HhgAAIDvAwEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4XR5gRo4cKYfDcdGSn58vSZo0adJFfQ888EDIPurr65Wbm6vBgwcrLi5OixYtUmtra1eXCgAADNW/q3f4zjvv6Pz58/b6oUOH9Hd/93f6h3/4B7tt3rx5WrZsmb0+ePBg+/X58+eVm5srj8ej3bt3q6GhQXPmzNGAAQP01FNPdXW5AADAQF0eYIYPHx6yvnz5co0ePVo/+MEP7LbBgwfL4/G0u/22bdv03nvvafv27YqPj9d1112nJ598UoWFhSouLpbT6ezqkgEAgGG6dQ7MuXPn9B//8R+6//775XA47PZ169YpNjZW48aNU1FRkc6cOWP3VVZWavz48YqPj7fbsrOzFQwGdfjw4Useq7m5WcFgMGQBAAC9U5c/gfm6TZs26eTJk7rvvvvstlmzZikpKUler1c1NTUqLCxUbW2tXn/9dUmS3+8PCS+S7HW/33/JY5WUlGjp0qVdfxIAAKDH6dYA8+KLLyonJ0der9dumz9/vv16/PjxSkhI0JQpU3T06FGNHj36so9VVFSkgoICez0YDCoxMfGy9wcAAHqubgswH330kbZv324/WbmU9PR0SdKRI0c0evRoeTwe7du3L2RMY2OjJF1y3owkuVwuuVyu71g1AAAwQbfNgXnppZcUFxen3NzcbxxXXV0tSUpISJAk+Xw+HTx4UCdOnLDHlJWVKSoqSikpKd1VLgAAMEi3PIFpa2vTSy+9pLlz56p///8/xNGjR7V+/XrdfvvtGjZsmGpqarRw4UJlZGRowoQJkqSsrCylpKTo3nvvVWlpqfx+vx5//HHl5+fzhAUAAEjqpgCzfft21dfX6/777w9pdzqd2r59u5555hk1NTUpMTFRM2bM0OOPP26PiYiI0ObNm/Xggw/K5/NpyJAhmjt3bsj3xgAAgL6tWwJMVlaWLMu6qD0xMVHl5eXfun1SUpL+8Ic/dEdpAACgF+C3kAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDj9w10AAMA8IxdvCXcJYXdseW64S+jTeAIDAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4XR5giouL5XA4QpaxY8fa/WfPnlV+fr6GDRumK664QjNmzFBjY2PIPurr65Wbm6vBgwcrLi5OixYtUmtra1eXCgAADNUtH6O+9tprtX379v8/SP//P8zChQu1ZcsWvfbaa3K73VqwYIGmT5+ut99+W5J0/vx55ebmyuPxaPfu3WpoaNCcOXM0YMAAPfXUU91RLgAAMEy3BJj+/fvL4/Fc1B4IBPTiiy9q/fr1+uEPfyhJeumll3TNNddoz549uuWWW7Rt2za999572r59u+Lj43XdddfpySefVGFhoYqLi+V0OrujZAAAYJBumQPz4Ycfyuv1atSoUZo9e7bq6+slSVVVVWppaVFmZqY9duzYsRoxYoQqKyslSZWVlRo/frzi4+PtMdnZ2QoGgzp8+PAlj9nc3KxgMBiyAACA3qnLA0x6errWrl2rrVu3avXq1aqrq9PEiRN16tQp+f1+OZ1ORUdHh2wTHx8vv98vSfL7/SHh5UL/hb5LKSkpkdvttpfExMSuPTEAANBjdPlbSDk5OfbrCRMmKD09XUlJSXr11Vc1aNCgrj6craioSAUFBfZ6MBgkxAAA0Et1+8eoo6OjdfXVV+vIkSPyeDw6d+6cTp48GTKmsbHRnjPj8Xgu+lTShfX25tVc4HK5FBUVFbIAAIDeqdsDzOnTp3X06FElJCQoLS1NAwYM0I4dO+z+2tpa1dfXy+fzSZJ8Pp8OHjyoEydO2GPKysoUFRWllJSU7i4XAAAYoMvfQnr00Uc1bdo0JSUl6fjx43riiScUERGhu+++W263W3l5eSooKFBMTIyioqL00EMPyefz6ZZbbpEkZWVlKSUlRffee69KS0vl9/v1+OOPKz8/Xy6Xq6vLBQAABuryAPPJJ5/o7rvv1meffabhw4frtttu0549ezR8+HBJ0tNPP61+/fppxowZam5uVnZ2tv793//d3j4iIkKbN2/Wgw8+KJ/PpyFDhmju3LlatmxZV5cKAAAM5bAsywp3Ed0hGAzK7XYrEAgwH6YbjFy8JdwlAEBYHVueG+4SeqWO/v3mt5AAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcfqHuwATjVy8JdwlAADQp/EEBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxujzAlJSU6KabblJkZKTi4uJ05513qra2NmTMpEmT5HA4QpYHHnggZEx9fb1yc3M1ePBgxcXFadGiRWptbe3qcgEAgIH6d/UOy8vLlZ+fr5tuukmtra167LHHlJWVpffee09Dhgyxx82bN0/Lli2z1wcPHmy/Pn/+vHJzc+XxeLR79241NDRozpw5GjBggJ566qmuLhkAABimywPM1q1bQ9bXrl2ruLg4VVVVKSMjw24fPHiwPB5Pu/vYtm2b3nvvPW3fvl3x8fG67rrr9OSTT6qwsFDFxcVyOp1dXTYAADBIt8+BCQQCkqSYmJiQ9nXr1ik2Nlbjxo1TUVGRzpw5Y/dVVlZq/Pjxio+Pt9uys7MVDAZ1+PDhdo/T3NysYDAYsgAAgN6py5/AfF1bW5sefvhh3XrrrRo3bpzdPmvWLCUlJcnr9aqmpkaFhYWqra3V66+/Lkny+/0h4UWSve73+9s9VklJiZYuXdpNZwIAAHqSbg0w+fn5OnTokP785z+HtM+fP99+PX78eCUkJGjKlCk6evSoRo8efVnHKioqUkFBgb0eDAaVmJh4eYUDAIAerdveQlqwYIE2b96st956S1deeeU3jk1PT5ckHTlyRJLk8XjU2NgYMubC+qXmzbhcLkVFRYUsAACgd+ryAGNZlhYsWKCNGzdq586dSk5O/tZtqqurJUkJCQmSJJ/Pp4MHD+rEiRP2mLKyMkVFRSklJaWrSwYAAIbp8reQ8vPztX79er3xxhuKjIy056y43W4NGjRIR48e1fr163X77bdr2LBhqqmp0cKFC5WRkaEJEyZIkrKyspSSkqJ7771XpaWl8vv9evzxx5Wfny+Xy9XVJQMAAMN0+ROY1atXKxAIaNKkSUpISLCXDRs2SJKcTqe2b9+urKwsjR07Vo888ohmzJihN998095HRESENm/erIiICPl8Pt1zzz2aM2dOyPfGAACAvqvLn8BYlvWN/YmJiSovL//W/SQlJekPf/hDV5UFAAB6EX4LCQAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADBO/3AXAACAiUYu3hLuEsLq2PLcsB6fJzAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxunRAWbVqlUaOXKkBg4cqPT0dO3bty/cJQEAgB6gxwaYDRs2qKCgQE888YQOHDig1NRUZWdn68SJE+EuDQAAhFmPDTArV67UvHnz9NOf/lQpKSl6/vnnNXjwYK1ZsybcpQEAgDDrH+4C2nPu3DlVVVWpqKjIbuvXr58yMzNVWVnZ7jbNzc1qbm621wOBgCQpGAx2eX1tzWe6fJ8AAJikO/6+fn2/lmV947geGWD+8pe/6Pz584qPjw9pj4+P1wcffNDuNiUlJVq6dOlF7YmJid1SIwAAfZn7me7d/6lTp+R2uy/Z3yMDzOUoKipSQUGBvd7W1qbPP/9cw4YNk8PhCBkbDAaVmJiojz/+WFFRUd93qcbhenUc16rjuFYdx7XqOK5Vx/XUa2VZlk6dOiWv1/uN43pkgImNjVVERIQaGxtD2hsbG+XxeNrdxuVyyeVyhbRFR0d/43GioqJ61D9aT8f16jiuVcdxrTqOa9VxXKuO64nX6puevFzQIyfxOp1OpaWlaceOHXZbW1ubduzYIZ/PF8bKAABAT9Ajn8BIUkFBgebOnasbb7xRN998s5555hk1NTXppz/9abhLAwAAYdZjA8xdd92lTz/9VEuWLJHf79d1112nrVu3XjSx93K4XC498cQTF73lhPZxvTqOa9VxXKuO41p1HNeq40y/Vg7r2z6nBAAA0MP0yDkwAAAA34QAAwAAjEOAAQAAxiHAAAAA4/TJALNq1SqNHDlSAwcOVHp6uvbt2xfuknqc4uJiORyOkGXs2LHhLqtHqKio0LRp0+T1euVwOLRp06aQfsuytGTJEiUkJGjQoEHKzMzUhx9+GJ5ie4Bvu1733XffRffa1KlTw1NsGJWUlOimm25SZGSk4uLidOedd6q2tjZkzNmzZ5Wfn69hw4bpiiuu0IwZMy76ws++oCPXatKkSRfdVw888ECYKg6v1atXa8KECfYX1vl8Pv3xj3+0+029r/pcgNmwYYMKCgr0xBNP6MCBA0pNTVV2drZOnDgR7tJ6nGuvvVYNDQ328uc//zncJfUITU1NSk1N1apVq9rtLy0t1XPPPafnn39ee/fu1ZAhQ5Sdna2zZ89+z5X2DN92vSRp6tSpIffayy+//D1W2DOUl5crPz9fe/bsUVlZmVpaWpSVlaWmpiZ7zMKFC/Xmm2/qtddeU3l5uY4fP67p06eHserw6Mi1kqR58+aF3FelpaVhqji8rrzySi1fvlxVVVXav3+/fvjDH+qOO+7Q4cOHJRl8X1l9zM0332zl5+fb6+fPn7e8Xq9VUlISxqp6nieeeMJKTU0Ndxk9niRr48aN9npbW5vl8XisFStW2G0nT560XC6X9fLLL4ehwp7lr6+XZVnW3LlzrTvuuCMs9fRkJ06csCRZ5eXllmV9dR8NGDDAeu211+wx77//viXJqqysDFeZPcJfXyvLsqwf/OAH1j/90z+Fr6gebujQodbvfvc7o++rPvUE5ty5c6qqqlJmZqbd1q9fP2VmZqqysjKMlfVMH374obxer0aNGqXZs2ervr4+3CX1eHV1dfL7/SH3mNvtVnp6OvfYN9i1a5fi4uI0ZswYPfjgg/rss8/CXVLYBQIBSVJMTIwkqaqqSi0tLSH31tixYzVixIg+f2/99bW6YN26dYqNjdW4ceNUVFSkM2fOhKO8HuX8+fN65ZVX1NTUJJ/PZ/R91WO/ibc7/OUvf9H58+cv+jbf+Ph4ffDBB2GqqmdKT0/X2rVrNWbMGDU0NGjp0qWaOHGiDh06pMjIyHCX12P5/X5Javceu9CHUFOnTtX06dOVnJyso0eP6rHHHlNOTo4qKysVERER7vLCoq2tTQ8//LBuvfVWjRs3TtJX95bT6bzoR2r7+r3V3rWSpFmzZikpKUler1c1NTUqLCxUbW2tXn/99TBWGz4HDx6Uz+fT2bNndcUVV2jjxo1KSUlRdXW1sfdVnwow6LicnBz79YQJE5Senq6kpCS9+uqrysvLC2Nl6G1mzpxpvx4/frwmTJig0aNHa9euXZoyZUoYKwuf/Px8HTp0iHlnHXCpazV//nz79fjx45WQkKApU6bo6NGjGj169PddZtiNGTNG1dXVCgQC+s///E/NnTtX5eXl4S7rO+lTbyHFxsYqIiLiotnVjY2N8ng8YarKDNHR0br66qt15MiRcJfSo124j7jHLt+oUaMUGxvbZ++1BQsWaPPmzXrrrbd05ZVX2u0ej0fnzp3TyZMnQ8b35XvrUteqPenp6ZLUZ+8rp9Opq666SmlpaSopKVFqaqqeffZZo++rPhVgnE6n0tLStGPHDrutra1NO3bskM/nC2NlPd/p06d19OhRJSQkhLuUHi05OVkejyfkHgsGg9q7dy/3WAd98skn+uyzz/rcvWZZlhYsWKCNGzdq586dSk5ODulPS0vTgAEDQu6t2tpa1dfX97l769uuVXuqq6slqc/dV5fS1tam5uZms++rcM8i/r698sorlsvlstauXWu999571vz5863o6GjL7/eHu7Qe5ZFHHrF27dpl1dXVWW+//baVmZlpxcbGWidOnAh3aWF36tQp691337XeffddS5K1cuVK691337U++ugjy7Isa/ny5VZ0dLT1xhtvWDU1NdYdd9xhJScnW19++WWYKw+Pb7pep06dsh599FGrsrLSqqurs7Zv327dcMMN1t/+7d9aZ8+eDXfp36sHH3zQcrvd1q5du6yGhgZ7OXPmjD3mgQcesEaMGGHt3LnT2r9/v+Xz+SyfzxfGqsPj267VkSNHrGXLlln79++36urqrDfeeMMaNWqUlZGREebKw2Px4sVWeXm5VVdXZ9XU1FiLFy+2HA6HtW3bNsuyzL2v+lyAsSzL+s1vfmONGDHCcjqd1s0332zt2bMn3CX1OHfddZeVkJBgOZ1O62/+5m+su+66yzpy5Ei4y+oR3nrrLUvSRcvcuXMty/rqo9S//vWvrfj4eMvlcllTpkyxamtrw1t0GH3T9Tpz5oyVlZVlDR8+3BowYICVlJRkzZs3r0/+h6K9ayTJeumll+wxX375pfWLX/zCGjp0qDV48GDr7//+762GhobwFR0m33at6uvrrYyMDCsmJsZyuVzWVVddZS1atMgKBALhLTxM7r//fispKclyOp3W8OHDrSlTptjhxbLMva8clmVZ39/zHgAAgO+uT82BAQAAvQMBBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADG+T+YWd1Y8A9tDwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.hist(tweet_lengths, bins=7)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c5b28e1",
   "metadata": {
    "papermill": {
     "duration": 0.017747,
     "end_time": "2023-12-26T07:08:53.689408",
     "exception": false,
     "start_time": "2023-12-26T07:08:53.671661",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Looks like most sentences are between 5 and 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d758a34c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:53.727441Z",
     "iopub.status.busy": "2023-12-26T07:08:53.726016Z",
     "iopub.status.idle": "2023-12-26T07:08:53.738389Z",
     "shell.execute_reply": "2023-12-26T07:08:53.737467Z"
    },
    "papermill": {
     "duration": 0.033072,
     "end_time": "2023-12-26T07:08:53.740199",
     "exception": false,
     "start_time": "2023-12-26T07:08:53.707127",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# how long of a tweet covers 95% of the length?\n",
    "output_seq_len = int(np.percentile(tweet_lengths, 95))\n",
    "output_seq_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4acf1ca6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:08:53.777738Z",
     "iopub.status.busy": "2023-12-26T07:08:53.777146Z",
     "iopub.status.idle": "2023-12-26T07:09:07.596626Z",
     "shell.execute_reply": "2023-12-26T07:09:07.595585Z"
    },
    "papermill": {
     "duration": 13.841121,
     "end_time": "2023-12-26T07:09:07.599180",
     "exception": false,
     "start_time": "2023-12-26T07:08:53.758059",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "max_vocab_length = 10000\n",
    "\n",
    "text_vectorizer = tf.keras.layers.TextVectorization(max_tokens=max_vocab_length,\n",
    "                                                   pad_to_max_tokens=True,\n",
    "                                                   output_sequence_length=24,\n",
    "                                                   split=\"whitespace\",\n",
    "                                                   output_mode=\"int\",\n",
    "                                                   standardize=\"lower_and_strip_punctuation\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f554d834",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:07.638753Z",
     "iopub.status.busy": "2023-12-26T07:09:07.638391Z",
     "iopub.status.idle": "2023-12-26T07:09:08.498859Z",
     "shell.execute_reply": "2023-12-26T07:09:08.497368Z"
    },
    "papermill": {
     "duration": 0.883464,
     "end_time": "2023-12-26T07:09:08.501741",
     "exception": false,
     "start_time": "2023-12-26T07:09:07.618277",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "text_vectorizer.adapt(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ab53c987",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:08.541808Z",
     "iopub.status.busy": "2023-12-26T07:09:08.541359Z",
     "iopub.status.idle": "2023-12-26T07:09:08.610714Z",
     "shell.execute_reply": "2023-12-26T07:09:08.608981Z"
    },
    "papermill": {
     "duration": 0.092602,
     "end_time": "2023-12-26T07:09:08.613135",
     "exception": false,
     "start_time": "2023-12-26T07:09:08.520533",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text:\n",
      "#News: 'Many deaths' in shipwreck: Rescuers are trying to save hundreds of migrants after the... http://t.co/tX51oYbrN6 via @TheNewsHype\n",
      "\n",
      "Length of text: 18\n",
      "\n",
      "Vectorized text:\n",
      "[[  57  134  331    4 3423  321   21  640    5  338  416    6  341   38\n",
      "     2    1   51 7606    0    0    0    0    0    0]]\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "target_sentence = random.choice(X_train)\n",
    "print(f\"Text:\\n{target_sentence}\")\n",
    "print(f\"\\nLength of text: {len(target_sentence.split())}\")\n",
    "print(f\"\\nVectorized text:\\n{text_vectorizer([target_sentence])}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "791308c2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:08.652110Z",
     "iopub.status.busy": "2023-12-26T07:09:08.651520Z",
     "iopub.status.idle": "2023-12-26T07:09:08.679368Z",
     "shell.execute_reply": "2023-12-26T07:09:08.677225Z"
    },
    "papermill": {
     "duration": 0.0503,
     "end_time": "2023-12-26T07:09:08.682244",
     "exception": false,
     "start_time": "2023-12-26T07:09:08.631944",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words in vocabulary: 10000\n",
      "Most common words in the vocabulary: ['', '[UNK]', 'the', 'a', 'in']\n",
      "Least common words in the vocabulary: ['pakthey', 'pakistan\\x89Ûªs', 'pakistans', 'pajamas', 'paints']\n"
     ]
    }
   ],
   "source": [
    "# How many words in our training vocabulary?\n",
    "tweets_text_vocab = text_vectorizer.get_vocabulary()\n",
    "print(f\"Number of words in vocabulary: {len(tweets_text_vocab)}\"),\n",
    "print(f\"Most common words in the vocabulary: {tweets_text_vocab[:5]}\")\n",
    "print(f\"Least common words in the vocabulary: {tweets_text_vocab[-5:]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27d0dd27",
   "metadata": {
    "papermill": {
     "duration": 0.018123,
     "end_time": "2023-12-26T07:09:08.719022",
     "exception": false,
     "start_time": "2023-12-26T07:09:08.700899",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Create the token embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "49962dc0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:08.758068Z",
     "iopub.status.busy": "2023-12-26T07:09:08.757434Z",
     "iopub.status.idle": "2023-12-26T07:09:08.764398Z",
     "shell.execute_reply": "2023-12-26T07:09:08.763301Z"
    },
    "papermill": {
     "duration": 0.029089,
     "end_time": "2023-12-26T07:09:08.766481",
     "exception": false,
     "start_time": "2023-12-26T07:09:08.737392",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "token_embedding = tf.keras.layers.Embedding(input_dim=len(tweets_text_vocab),\n",
    "                                           output_dim=128, # size of output embedding vector. Ie: a value of 100 outputs a feature vector of size 100 for each word\n",
    "                                           mask_zero=True,\n",
    "                                           name=\"token_embed\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a3804ed1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:08.807909Z",
     "iopub.status.busy": "2023-12-26T07:09:08.806800Z",
     "iopub.status.idle": "2023-12-26T07:09:08.860660Z",
     "shell.execute_reply": "2023-12-26T07:09:08.859156Z"
    },
    "papermill": {
     "duration": 0.077958,
     "end_time": "2023-12-26T07:09:08.863598",
     "exception": false,
     "start_time": "2023-12-26T07:09:08.785640",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence before vectorization:\n",
      "#News: 'Many deaths' in shipwreck: Rescuers are trying to save hundreds of migrants after the... http://t.co/tX51oYbrN6 via @TheNewsHype\n",
      "\n",
      "Sentence after vectorization (before embedding):\n",
      "[[  57  134  331    4 3423  321   21  640    5  338  416    6  341   38\n",
      "     2    1   51 7606    0    0    0    0    0    0]]\n",
      "\n",
      "Sentence after embedding:\n",
      "[[[ 0.03019544 -0.02418411  0.01751139 ... -0.01936489 -0.01953087\n",
      "    0.03324485]\n",
      "  [-0.00149969  0.04522185  0.04490285 ... -0.04235858  0.00419889\n",
      "    0.01360964]\n",
      "  [-0.04690814 -0.02690545 -0.01564493 ...  0.02323444  0.04659337\n",
      "   -0.04844758]\n",
      "  ...\n",
      "  [-0.01573888  0.00268257 -0.0046014  ...  0.02686039  0.04894122\n",
      "    0.00162994]\n",
      "  [-0.01573888  0.00268257 -0.0046014  ...  0.02686039  0.04894122\n",
      "    0.00162994]\n",
      "  [-0.01573888  0.00268257 -0.0046014  ...  0.02686039  0.04894122\n",
      "    0.00162994]]]\n",
      "\n",
      "Embedded sentence shape: (1, 24, 128)\n"
     ]
    }
   ],
   "source": [
    "# Show example embedding\n",
    "print(f\"Sentence before vectorization:\\n{target_sentence}\\n\")\n",
    "vectorized_sentence = text_vectorizer([target_sentence])\n",
    "print(f\"Sentence after vectorization (before embedding):\\n{vectorized_sentence}\\n\")\n",
    "embedded_sentence = token_embedding(vectorized_sentence)\n",
    "print(f\"Sentence after embedding:\\n{embedded_sentence}\\n\")\n",
    "print(f\"Embedded sentence shape: {embedded_sentence.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8fc29d92",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:08.905526Z",
     "iopub.status.busy": "2023-12-26T07:09:08.905128Z",
     "iopub.status.idle": "2023-12-26T07:09:08.920164Z",
     "shell.execute_reply": "2023-12-26T07:09:08.919271Z"
    },
    "papermill": {
     "duration": 0.038859,
     "end_time": "2023-12-26T07:09:08.922256",
     "exception": false,
     "start_time": "2023-12-26T07:09:08.883397",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original text:\n",
      "@random_tourist it rained. Some guys tree hit by lightning and some jackholes drove onto flooded streets.      \n",
      "\n",
      "Embedded version:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 24, 128), dtype=float32, numpy=\n",
       "array([[[-0.01000402,  0.02437745,  0.03948537, ..., -0.03636986,\n",
       "          0.00658759, -0.00348108],\n",
       "        [-0.03506844,  0.0406344 ,  0.01081747, ..., -0.02909932,\n",
       "          0.02497986,  0.01631084],\n",
       "        [ 0.00942886, -0.01196678,  0.01535243, ..., -0.03290317,\n",
       "          0.02591038, -0.02769855],\n",
       "        ...,\n",
       "        [-0.01573888,  0.00268257, -0.0046014 , ...,  0.02686039,\n",
       "          0.04894122,  0.00162994],\n",
       "        [-0.01573888,  0.00268257, -0.0046014 , ...,  0.02686039,\n",
       "          0.04894122,  0.00162994],\n",
       "        [-0.01573888,  0.00268257, -0.0046014 , ...,  0.02686039,\n",
       "          0.04894122,  0.00162994]]], dtype=float32)>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get a random sentence from training set\n",
    "random_sentence = random.choice(X_train)\n",
    "print(f\"Original text:\\n{random_sentence}\\\n",
    "      \\n\\nEmbedded version:\")\n",
    "\n",
    "# Embed the random sentence (turn it into numerical representation)\n",
    "sample_embed = token_embedding(text_vectorizer([random_sentence]))\n",
    "sample_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1d13c351",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:08.961636Z",
     "iopub.status.busy": "2023-12-26T07:09:08.960768Z",
     "iopub.status.idle": "2023-12-26T07:09:08.974996Z",
     "shell.execute_reply": "2023-12-26T07:09:08.973622Z"
    },
    "papermill": {
     "duration": 0.03662,
     "end_time": "2023-12-26T07:09:08.977595",
     "exception": false,
     "start_time": "2023-12-26T07:09:08.940975",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(128,), dtype=float32, numpy=\n",
       "array([-1.00040212e-02,  2.43774541e-02,  3.94853689e-02, -2.78397929e-02,\n",
       "       -4.59815264e-02, -4.17415127e-02,  3.27600501e-02,  4.49448265e-02,\n",
       "       -1.63150206e-02, -4.95513082e-02,  1.43349878e-02,  1.84648111e-03,\n",
       "        4.50806059e-02, -1.33390315e-02, -3.85915861e-02,  1.37749650e-02,\n",
       "       -4.08224352e-02, -4.14562225e-03, -3.50546129e-02, -2.99784895e-02,\n",
       "       -1.71714053e-02,  3.83880772e-02,  4.02889960e-02,  4.04076651e-03,\n",
       "       -3.96116152e-02,  9.44361091e-05,  4.15216759e-03,  1.29116215e-02,\n",
       "        1.43699162e-02,  2.75588892e-02,  1.91606916e-02, -1.96596514e-02,\n",
       "       -1.00344047e-02,  9.19680670e-03,  2.16869265e-03, -4.29504029e-02,\n",
       "        1.14498138e-02, -9.33218002e-03, -1.59054026e-02, -3.22229862e-02,\n",
       "        2.04294212e-02,  3.11428793e-02,  3.95304002e-02, -1.04162842e-03,\n",
       "        2.77494229e-02, -1.29353181e-02, -3.32513936e-02,  2.41281502e-02,\n",
       "       -2.54145153e-02,  2.03067064e-03, -1.78489797e-02, -3.05795800e-02,\n",
       "       -4.35135141e-02, -4.54300046e-02, -4.26415913e-02, -4.25090641e-03,\n",
       "       -2.25202926e-02, -2.23941095e-02,  2.69538499e-02,  2.25461610e-02,\n",
       "       -3.61445546e-02, -4.02220264e-02,  4.05870192e-02,  2.75466591e-03,\n",
       "       -8.70941952e-03,  4.08574603e-02,  4.73958254e-03,  2.41484903e-02,\n",
       "       -4.80813496e-02, -1.12027153e-02,  1.82721950e-02, -2.17017774e-02,\n",
       "        1.80958584e-03,  3.93999852e-02,  1.94556974e-02, -7.10256025e-03,\n",
       "       -1.04433410e-02, -3.55416536e-02, -2.33876463e-02, -4.44896221e-02,\n",
       "        1.26322359e-03, -3.23335528e-02,  2.01779641e-02, -4.39677387e-03,\n",
       "        4.56624106e-03, -1.41355880e-02,  1.92301013e-02, -1.92915797e-02,\n",
       "       -3.54166627e-02, -1.09240301e-02, -2.48361379e-04,  2.96907686e-02,\n",
       "        1.37888081e-02,  2.33972073e-03, -4.49129827e-02,  4.20786776e-02,\n",
       "       -3.42841744e-02, -1.15416870e-02,  3.28807123e-02, -4.62357178e-02,\n",
       "       -2.52305269e-02,  3.39749716e-02, -4.19965871e-02, -3.65551934e-02,\n",
       "        1.53415538e-02, -3.01634911e-02, -1.81924701e-02, -3.34215388e-02,\n",
       "       -2.60928757e-02, -1.66302100e-02, -4.51510549e-02, -8.07894394e-03,\n",
       "       -2.18884703e-02, -3.13725695e-02, -2.41732951e-02, -1.72806010e-02,\n",
       "        1.46382190e-02, -2.78657675e-02,  2.30699293e-02,  1.48687027e-02,\n",
       "        2.54523866e-02,  9.57272947e-04,  1.77407265e-03, -3.23383361e-02,\n",
       "       -4.56932448e-02, -3.63698602e-02,  6.58758730e-03, -3.48107889e-03],\n",
       "      dtype=float32)>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check out a single tokens embedding\n",
    "sample_embed[0][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "485daa04",
   "metadata": {
    "papermill": {
     "duration": 0.019309,
     "end_time": "2023-12-26T07:09:09.016086",
     "exception": false,
     "start_time": "2023-12-26T07:09:08.996777",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Notice its an array of size 128. Each word is represented in an array of 128 values. \n",
    "\n",
    "Its a tensorflow layer, which is important b/c it can be used as part of a model. Its parameters(word representations) can be updated and improved as the model learns. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2edea16b",
   "metadata": {
    "papermill": {
     "duration": 0.018732,
     "end_time": "2023-12-26T07:09:09.053767",
     "exception": false,
     "start_time": "2023-12-26T07:09:09.035035",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "b41c60b6",
   "metadata": {
    "papermill": {
     "duration": 0.01994,
     "end_time": "2023-12-26T07:09:09.093098",
     "exception": false,
     "start_time": "2023-12-26T07:09:09.073158",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Load in datasets faster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7796ae15",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:09.133084Z",
     "iopub.status.busy": "2023-12-26T07:09:09.132409Z",
     "iopub.status.idle": "2023-12-26T07:09:09.151902Z",
     "shell.execute_reply": "2023-12-26T07:09:09.150426Z"
    },
    "papermill": {
     "duration": 0.042043,
     "end_time": "2023-12-26T07:09:09.154296",
     "exception": false,
     "start_time": "2023-12-26T07:09:09.112253",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_dataset = tf.data.Dataset.from_tensor_slices((X_train, y_train))\n",
    "valid_dataset = tf.data.Dataset.from_tensor_slices((X_valid, y_valid))\n",
    "\n",
    "train_dataset = train_dataset.batch(32).prefetch(tf.data.AUTOTUNE)\n",
    "valid_dataset = valid_dataset.batch(32).prefetch(tf.data.AUTOTUNE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "513652da",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:09.196848Z",
     "iopub.status.busy": "2023-12-26T07:09:09.196200Z",
     "iopub.status.idle": "2023-12-26T07:09:09.274714Z",
     "shell.execute_reply": "2023-12-26T07:09:09.273494Z"
    },
    "papermill": {
     "duration": 0.102248,
     "end_time": "2023-12-26T07:09:09.277188",
     "exception": false,
     "start_time": "2023-12-26T07:09:09.174940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = tf.keras.Input(shape=(1,), dtype=\"string\") # 1 dimensional strings\n",
    "\n",
    "x = text_vectorizer(inputs) # turn input text into numbers\n",
    "x = token_embedding(x)\n",
    "\n",
    "x = tf.keras.layers.GlobalAveragePooling1D()(x)\n",
    "\n",
    "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "model_1 = tf.keras.Model(inputs, outputs, name=\"model_1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4223bfdd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:09.317660Z",
     "iopub.status.busy": "2023-12-26T07:09:09.317100Z",
     "iopub.status.idle": "2023-12-26T07:09:09.335378Z",
     "shell.execute_reply": "2023-12-26T07:09:09.334035Z"
    },
    "papermill": {
     "duration": 0.041722,
     "end_time": "2023-12-26T07:09:09.338048",
     "exception": false,
     "start_time": "2023-12-26T07:09:09.296326",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_1.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cfcd6742",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:09.378775Z",
     "iopub.status.busy": "2023-12-26T07:09:09.378154Z",
     "iopub.status.idle": "2023-12-26T07:09:09.407485Z",
     "shell.execute_reply": "2023-12-26T07:09:09.406323Z"
    },
    "papermill": {
     "duration": 0.0519,
     "end_time": "2023-12-26T07:09:09.409632",
     "exception": false,
     "start_time": "2023-12-26T07:09:09.357732",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 1)]               0         \n",
      "                                                                 \n",
      " text_vectorization (TextVe  (None, 24)                0         \n",
      " ctorization)                                                    \n",
      "                                                                 \n",
      " token_embed (Embedding)     (None, 24, 128)           1280000   \n",
      "                                                                 \n",
      " global_average_pooling1d (  (None, 128)               0         \n",
      " GlobalAveragePooling1D)                                         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 1)                 129       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1280129 (4.88 MB)\n",
      "Trainable params: 1280129 (4.88 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b3ca8091",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:09.452344Z",
     "iopub.status.busy": "2023-12-26T07:09:09.451994Z",
     "iopub.status.idle": "2023-12-26T07:09:22.592520Z",
     "shell.execute_reply": "2023-12-26T07:09:22.590803Z"
    },
    "papermill": {
     "duration": 13.165068,
     "end_time": "2023-12-26T07:09:22.595125",
     "exception": false,
     "start_time": "2023-12-26T07:09:09.430057",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "215/215 [==============================] - 3s 11ms/step - loss: 0.6026 - accuracy: 0.7062 - val_loss: 0.5220 - val_accuracy: 0.7848\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 2s 10ms/step - loss: 0.4202 - accuracy: 0.8313 - val_loss: 0.4793 - val_accuracy: 0.7874\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.3265 - accuracy: 0.8721 - val_loss: 0.4876 - val_accuracy: 0.7900\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.2641 - accuracy: 0.8999 - val_loss: 0.5117 - val_accuracy: 0.7874\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.2170 - accuracy: 0.9225 - val_loss: 0.5446 - val_accuracy: 0.7822\n"
     ]
    }
   ],
   "source": [
    "model_1_history = model_1.fit(train_dataset,\n",
    "                             epochs=5,\n",
    "                             validation_data=(valid_dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8c859ce",
   "metadata": {
    "papermill": {
     "duration": 0.033399,
     "end_time": "2023-12-26T07:09:22.662627",
     "exception": false,
     "start_time": "2023-12-26T07:09:22.629228",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model 2: Transfer Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3feaf95a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:22.729208Z",
     "iopub.status.busy": "2023-12-26T07:09:22.728891Z",
     "iopub.status.idle": "2023-12-26T07:09:23.527075Z",
     "shell.execute_reply": "2023-12-26T07:09:23.525388Z"
    },
    "papermill": {
     "duration": 0.834441,
     "end_time": "2023-12-26T07:09:23.529604",
     "exception": false,
     "start_time": "2023-12-26T07:09:22.695163",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tensorflow_text as text\n",
    "import tensorflow_hub as hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ba7aa40c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:23.598960Z",
     "iopub.status.busy": "2023-12-26T07:09:23.598544Z",
     "iopub.status.idle": "2023-12-26T07:09:36.533021Z",
     "shell.execute_reply": "2023-12-26T07:09:36.531039Z"
    },
    "papermill": {
     "duration": 12.971188,
     "end_time": "2023-12-26T07:09:36.535548",
     "exception": false,
     "start_time": "2023-12-26T07:09:23.564360",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sentence_encoder_layer = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder/4\",\n",
    "                                        input_shape=[], # shape of inputs coming to our model \n",
    "                                        dtype=tf.string, # data type of inputs coming to the USE layer\n",
    "                                        trainable=False, # keep the pretrained weights (we'll create a feature extractor)\n",
    "                                        name=\"USE\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c0603aae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:36.604477Z",
     "iopub.status.busy": "2023-12-26T07:09:36.604110Z",
     "iopub.status.idle": "2023-12-26T07:09:37.508591Z",
     "shell.execute_reply": "2023-12-26T07:09:37.506734Z"
    },
    "papermill": {
     "duration": 0.942986,
     "end_time": "2023-12-26T07:09:37.511473",
     "exception": false,
     "start_time": "2023-12-26T07:09:36.568487",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Create model using the Sequential API\n",
    "model_2 = tf.keras.Sequential([\n",
    "  sentence_encoder_layer, # take in sentences and then encode them into an embedding\n",
    "  tf.keras.layers.Dense(64, activation=\"relu\"),\n",
    "  tf.keras.layers.Dense(1, activation=\"sigmoid\")\n",
    "], name=\"model_2_USE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3e7f1389",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:37.580988Z",
     "iopub.status.busy": "2023-12-26T07:09:37.580474Z",
     "iopub.status.idle": "2023-12-26T07:09:37.584134Z",
     "shell.execute_reply": "2023-12-26T07:09:37.583346Z"
    },
    "papermill": {
     "duration": 0.040627,
     "end_time": "2023-12-26T07:09:37.585993",
     "exception": false,
     "start_time": "2023-12-26T07:09:37.545366",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import tensorflow_hub as hub\n",
    "# text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
    "# preprocessing_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3\", name='preprocessing')\n",
    "# encoder_inputs = preprocessing_layer(text_input)\n",
    "# encoder = hub.KerasLayer(\"https://tfhub.dev/google/distilbert_base_uncased/1\", trainable=False, name='BERT_encoder')\n",
    "# outputs = encoder(encoder_inputs)\n",
    "# x = outputs['pooled_output']\n",
    "# x = tf.keras.layers.Dropout(0.1)(x)\n",
    "# x = tf.keras.layers.Dense(1, activation=None, name='classifier')(x)\n",
    "# model_2 = tf.keras.Model(text_input, x, name=\"model_2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a18f6659",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:37.659761Z",
     "iopub.status.busy": "2023-12-26T07:09:37.658935Z",
     "iopub.status.idle": "2023-12-26T07:09:37.675700Z",
     "shell.execute_reply": "2023-12-26T07:09:37.673652Z"
    },
    "papermill": {
     "duration": 0.057941,
     "end_time": "2023-12-26T07:09:37.678421",
     "exception": false,
     "start_time": "2023-12-26T07:09:37.620480",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_2.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "46fa30b0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:37.750845Z",
     "iopub.status.busy": "2023-12-26T07:09:37.749476Z",
     "iopub.status.idle": "2023-12-26T07:09:37.779258Z",
     "shell.execute_reply": "2023-12-26T07:09:37.777512Z"
    },
    "papermill": {
     "duration": 0.073904,
     "end_time": "2023-12-26T07:09:37.787414",
     "exception": false,
     "start_time": "2023-12-26T07:09:37.713510",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2_USE\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " USE (KerasLayer)            (None, 512)               256797824 \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 64)                32832     \n",
      "                                                                 \n",
      " dense_2 (Dense)             (None, 1)                 65        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 256830721 (979.73 MB)\n",
      "Trainable params: 32897 (128.50 KB)\n",
      "Non-trainable params: 256797824 (979.61 MB)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1abb14d5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:37.865830Z",
     "iopub.status.busy": "2023-12-26T07:09:37.865476Z",
     "iopub.status.idle": "2023-12-26T07:09:38.140696Z",
     "shell.execute_reply": "2023-12-26T07:09:38.139775Z"
    },
    "papermill": {
     "duration": 0.315424,
     "end_time": "2023-12-26T07:09:38.143109",
     "exception": false,
     "start_time": "2023-12-26T07:09:37.827685",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQQAAAFgCAIAAADSD5ARAAAABmJLR0QA/wD/AP+gvaeTAAAgAElEQVR4nO3deUBM+/8/8NcsTUkM7WnTShRS160rJmuRbK1K4V7cLJHl2inLtV1k+15L9uVSubgXF1myRVSixScqSiNESaR1Zn5/vM/n/M5nSgbVye31+GvO+7znvF9nmmdnmTNnODKZDBBCAFy2C0CoqcAwIETBMCBEwTAgROEzJ27durVhwwa2SkGokTk6Os6cOZOe/J8tQ15e3rFjxxq9pGbt2LFjYrGY7Sqao/j4+Fu3bjFb+DU7RUdHN1Y9CDgczowZM7y9vdkupNnx8vKSa8FjBoQoGAaEKBgGhCgYBoQoGAaEKBgGhCgYBoQoGAaEKBgGhCgYBoQoGAaEKBgGhCgYBoQoGAaEKLVcwl23V69eyV37umLFCicnJ/J49+7dBw8epGcdOHDAyMiIPE5PTz98+HB6evqHDx+0tLS6des2ePBga2truvOgQYPKysrqGHrHjh0dOnT43IL/BebPn8+88t7R0XHVqlUs1kNrsoV9mc8OQ+vWrcPCwi5fvrx8+fJRo0ZNnDixY8eO9Ny+ffuamZmFhIQMGDDAzc1NU1OTtG/ZsmXGjBkuLi6jRo3S1NR89uzZvn375s6d26VLl/v375M+CxYsePjw4YQJE3r27LlixQq5cYcMGfLu3btPllddXT1p0iRTU9P58+d/7qo1WX5+fi4uLkOGDAGA06dPa2hosF0RpckW9oVkDJGRkXItH3PkyBEAmDt3bq1zRSJRREQEPfnkyRM+n+/q6srsI5VKBwwYoKGhwWxMTk4GgGHDhtVcplAoTEhI+GRhBQUFSkpK5ubmiqxFQ3B1dZ06dari/QEgMjJSkZ5CoVAoFH5pXV9r3rx5IpGourq65ix2C/tinp6enp6ezJbGOGa4fv16dXV1//79mY0cDmfq1KmKL+T06dOK7CNpaWllZGTExcV9dpX15Pr16yTS/zKpqalXr16V/atvOffZu0lfQFVVFQAyMjLk2vv37//XX3998unBwcFubm6urq4KDmdqavq5FSIEjRMGBwcHZWXlvXv3mpubT5kyRU1NjbSrqqr27Nnzk09PTk62s7NTZKAbN24sWrSIPI6JiREIBMyWs2fP7tu37/z58zwez9HRMTg4WFlZGQAiIyO3bdsGAFpaWhEREVu2bLlz545EIvn++++Dg4PV1dUB4ODBg7t37wYAGxubLVu2AMCdO3fmzJkDAEKhkESaLKesrCw1NdXZ2RkAeDzepUuXPuelUtT79+/JnjoArF27tqio6MCBAyUlJVZWVrNmzdLV1QWAhw8f/vzzz6TPgQMHYmJizp8/X1JSYmlpOXnyZCsrKwBITU0NDg4mfS5dusTj8SorKwcOHEha9u7da2JiQpaTmpoKAP369eNwOACwZcsWGxsbxas9fPhwfHx8fn6+np7esGHDRowYQWaNHz8+KyuLPLazs1u/fj0ABAcHk+Ho1xYAEhISDh48mJmZyefzu3btOnHiRHJuRu6lKCgoiIyMzM/Pl0gkS5cuFYlEn/GyMveZGuiYQSaTbd26lcfjAUDLli09PT0jIiLEYnHNJ5IdDA0NDRFD69at9+7dq0hVr169io2NJQErKyuTawkMDFy5cuXp06cXLlzI4XB8fX3Js54+fRobG2tmZqaurv7dd9+RPr/99lvr1q2NjIxyc3NlMllOTk5sbKyKikrPnj3JswoLC2NjYy0tLenDHrIcFRUVa2vr2NjY2NjYK1eufLJm+KJjhqqqqtjY2ClTpgCAh4dHUFDQ33//vW3bNnV19fbt25N1LykpiY2N9fDwAICePXtOnjz55MmTu3fv7tChg4qKyrlz52QyWXFxcWxsrIODAwBUVVXJZDKJRBIbG0tuUJCamkovh/S5ePEiWbXi4uJaC6vpw4cPQqHQyspq//79p06dmj17tpKSUlBQEJmbkJDg7+8PAEuXLr137x5pvHv37oQJE/r06XPjxg3SsmLFCg6HM3LkyD///PPQoUPfffddy5YtL1++LPdS9OnTZ+zYsSdOnDh06JC6unp0dHQdr2fNY4ZGCoNMJktJSRk3bhz5RwsAHA5n0KBBGRkZzD4kDD179oxlsLa2VjAMxLBhw+gwMFu2b99Ot/Tr14/8U6FbunbtCgAnT56kW06fPg0Abm5udEvLli3pMBB2dnZy5wBq9qnbl4WBCA8PBwA61TKZbPny5QBw+vRpumX69OkAEBISQrcUFBSoqqpqaWmVlpaSFjc3NzoMxKxZs+gwfKxPHYUxvXv3jsvlPn36lG4JCwsDgKSkJDKZmZnJ5XJdXFzoDhKJxNzcPCYmhkxevnwZAPr37093KC0t1dfX19PTKy8v/9hLkZqaWlBQ8LGqZPV4AE22lbKPHE7JZDLSgcnGxmbPnj2vXr2Kj49fvHixoaHh2bNnf/jhh5cvX8r11NTUdGYQCoVfVqScQYMG0Y/J6eDc3FxmByUlJXd3d3rSzc1NXV397Nmzb9++rZcCGsgn1wsAyPaB0NLScnFxefXqVQPtwslp0aJFXFycoaEh3TJgwAAAuHr1Kpk0Nzd3d3c/f/482TUCgJMnT7Zo0YJ0A4C9e/cCwLhx4+glqKqqDhky5Pnz5zExMcyx6P0lALC2ttbS0vqsUr8wDOSY+MOHD7XOff/+fcuWLWsfj8v9/vvvly1blpmZ6e7uXlRUtGfPnrrH2rJli+JHz3XQ1tamH5PjlsrKSmYHLS0tLvd/XhADAwOpVPr06dOvH73hfHK9AEBHR4c5Sd6aOTk5DV4cAI/HEwqFixYt8vb27tevn7Oz87Rp0wCgqKiI7kNua0cOGABg3bp1M2bMoOdmZ2eTucx/kefPnweAlJQU5lhf+UHHFx5Ak2MXUqUcmUz25MkT5n+CtLS05cuXk30wmkAgCAoKOnXqVM1/Y3JsbW2/rMjPVVpaKtdCPuajg83hcKRSKbND3R+ZNx1yq1ZzvQCAuWr1uF6RkZF+fn5dunSZNm2aoaEhn8/PysqaMGECc7eid+/e9vb2R44cWbly5ePHjx8/fuzn50fPJQkfO3ZszUN2+vqGevGFW4bOnTtraWldu3at5i4E+VSB+Q4uLi6Oiop69uyZXM8XL14AgImJiSIj/vzzz2TfseG8ffuWuRF4+fKlWCzW09OjK2zbtu3r16/pDuXl5TWTLBAIJBIJeUwO5hq0ZgXJ/QdNSkoCgB9++IFMtm3bFgCYq1bzPLhAIAAAsmobN24MDQ395KCzZs0ih/VSqfTQoUPjxo3r37+/s7Nz9+7da3aeMWNGZWXl5s2b161bN3nyZHKij+jTpw8AKCsrM7cMpqamS5curaqqUmj9FfOFYeDz+XPnzi0tLR09enRBQQHdnp6ePmHChJCQELIfxeTv78986yQmJi5ZskRdXX3MmDGKjJiUlMQcqCGoqKjMmTOH/FOsrKwMCQmpqqpasmQJffxjb2+fmZmZmJhIJpctW0ZOkTFZWFjk5eVJJJLCwsKoqCj6hAG7tm7dSud8y5YtKSkpPj4+9HU09vb2AEBOigDAmTNn7t27J7cECwsLAMjJyZHJZEePHpXbn6zV/fv38/PzyRPpnXuJRMK8eo3m7e1tYGDw+++/X7hwYdKkScxZkydPNjc3X7FixYMHD0jLmzdvxo8fX11dXc/XqjGPphU/m0Rs3rzZwMCAz+fb2Ng4OTlZWFjo6OisWrVKIpEwuxUVFa1YscLe3l5JScnCwsLBwcHY2JjL5Q4cODA9PZ3u5urqSj5PkDu1SqipqR05cqTueq5fvy4SiciOY69evVasWJGens5s2bVrl0wmE4lEZC/Ozs5u3rx55Lldu3bV19c/evSosbGxvb1927ZtVVVV165dy1x+RkZGx44dlZWV7ezsOnTosGPHDjs7Oz6fLxKJduzYQfqcOnVKTU3NwsJCT0/Px8fnk68hKHA2iVwKwePxeDyeSCQiNQ8YMMDMzAwAbGxsyAUg06dPJzsSZmZm9MUv5GzS8ePHbWxsunXrZmRkxOFwAgICPnz4QC+/rKyMHIV36tTJ2tp64sSJISEhAGBvbz9p0iTSJzs729TUVFtbu2PHjp06dXr9+nXNwuS0adNm27ZthYWF5IMLY2NjR0dHS0vLoKAgADAyMhKJRMwy1qxZAwBkD0pOfn7+iBEjlJSUOnbsaG9v37p1a19f3zdv3pC5zJdCJBIlJyd/8mWX1XY2iSNj7LpFRUWRv99nxSk3N/f58+cymUxbW9vU1LTmeSRaWVmZWCwuKCho0aKFhYVFq1atmHOvX79O72DUqlOnTsyDxZpev36dlpZGT+ro6Ojr69+9e5duMTY2NjExuXLlCt2ioaFB3kDdunV7/fq1WCyuqKhIT0+vrq62trauuX2TSCTZ2dnFxcVWVlatWrVKSkoi+99GRkb0J98lJSWPHj1SV1dX5LNwDocTGRlZ942HU1NTCwsL5Wq+du0avZcvFAptbW3v3btXXFxMWng8Xq9evQAgJCRk06ZNmZmZZmZm//nPf96+fWtubl7raRaxWCwWi42NjfX09LKzs/Py8gCgdevW9F5NZWVlRkYGl8vt2LEjn8+vWVhNlpaW7dq1A4AXL17k5uYKBIJOnTpVVVXRW9devXrRW1exWGxkZJSenk4+EKzpzZs3jx49EggEpqamzHOMzJcCALp169amTZs6qiLIxdf/c5ttZjI+d8vwb0K2DI0/Lij8OcOXIVuGzMzMhhuivmzevFnuas4Gxc6Fegh9zLNnzwIDAwHg3bt369atmzt3LovFNMa1SfWLXPZTq7Fjx44dO/ZzF0iuKcrKyqqqqnJ2dg4ICPjpp5++psImglxTRK788ff379GjB7mqqkkpKys7ePBgcnLyixcvPDw86vjjNoJvLwzMPf564ePj4+PjU7/LbAo6dOhQ769VvSPfPGG7CgruJiFEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIESp5UI9uTvOo4YWHh7+P18xQY0iPj6e3BmN9j9bBkNDQ09Pz8Ytqbnz9PSUSCTkG/qoMTk4ODg6OjJbOE3nAtpmKywsLDo6Oj09ne1Cmjs8ZkCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgr/PwIKbN28uXrxYIpGQyby8vNevX9va2pJJDoczcODA+fPns1dgM4VhYMH79+81NTUrKio+1uHo0aM+Pj6NWRIC3E1ihZqa2tChQ5WUlGqdq6KiMmTIkEYuCQGGgS3+/v7V1dU125WUlEaOHNmyZcvGLwlhGNgxaNAgNTW1mu1VVVX+/v6NXw8CDANbBAKBt7d3zT0loVA4YMAAVkpCGAbW+Pn5VVVVMVuUlJT8/Pw+diyBGhqeTWKNVCrV1dV99eoVs/HatWu9evViq6RmDrcMrOFyuaNHj2ZuB3R1dXv27MliSc0choFNo0aNoveUBAJBYGAgl4t/EdbgbhLLTExMcnJyyOPk5ORu3bqxWk6zhv+HWBYQEED2lExNTTEJ7MIwsIzsKXE4nDFjxrBdS3OHu0nss7GxSUtLe/jwoaWlJdu1NGu4ZWBfYGCgnZ0dJoF1/IYeQCwW37x5s6FH+aapqalZW1tHRUWxXUiTZmho6Ojo2LBjyBpYZGRkw64Aah48PT0b+r3a4FsGQvYtH5lwOJzIyEhvb2+2C2m+vLy8GmEUPGZAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIiCYUCIgmFAiIJhQIjSSJdw162kpGTDhg3MFl9f344dO5LHly9fvnbtGj1r6tSpmpqa5HFVVVVsbGx6evqHDx+0tLS6detmb2/PvNvKypUrKysr6xh64sSJ7dq1q7c1Qd+yJrRlSEtLW7p06cWLF2udu2/fvrt37zJbbty4YWZmFhAQkJ6e/u7du+vXrw8YMEBXV3fy5MnMbi9evFi6dOnRo0drLnP16tX5+fn1Uvwff/wRFhamoqKioqISFhb2xx9/kPbTp08PHz6cy+VOmjQpLCxMKpXWy3D1WCH6/xr620Pkm26K9Dxy5AgAzJ07t9a5IpEoIiKCnnzz5o26unrnzp3fvXtHN7569crMzExDQ4P5xOTkZAAYNmxYzWUKhcKEhIRPFgYAkZGRiqyCUCgUCoXMlpiYGA0NjcOHDyvy9EZQs8JvgqenZyN8060JbRk+y8WLF4uKivz9/Zk3dtfU1FyyZIniC5k3b16D7iMdOHDAz88vKirKz8+v4UZB9aVJHDN8AfIbUO/fv5drd3FxmTNnziefvmfPnu7du8+bN69BigMAgFWrVv3++++XL1+2sbFpuFFQPfpWw9C1a1cA2LZtm7u7u4ODA92uo6OjYBi4XG4D3cFOIpEEBwdfu3bt5s2bhoaGNTt8+PDh3LlzmZmZfD6/a9euffv2JQf95eXlq1evJn1Gjx6tpqZ25syZ/Px8iUTi7e3dqVMnAMjIyLh9+3Z+fr6enp6rq6uuri5zydXV1TExMZmZmRUVFV27du3Xrx+f/9l/4o8NsX379hcvXpDHpqamgYGBALBnz56nT58CgKqqKv3Kf80KsuhbDYO1tXVgYOCBAwccHR1FIpGXl5eLi4u5ufnH+mdkZISFhdGT5O/XECQSiYeHR15e3o0bN9q0aVOzw4ULF0aPHq2qquru7l5WVrZ8+fL27dv//fffRkZGpEN8fPz58+ffvHkTFxfn5ORUWlq6d+9ea2trc3PzwYMH37p1y8fHR11d/cSJE0FBQREREQEBAeSJYrG4T58+UqnU3d2dx+PNmDGjpKSE/L9QsPjKysq6h/jnn38SEhICAgJMTU3pZyUmJj58+HDChAlfuYKsh+FbPYCWyWTV1dX/93//x/yrWFparlmzpqysjNmNHEB36NAhlMHQ0HDv3r0KrgJ8zgE0h8PhcDiqqqqxsbE1O2RmZqqqqlpYWJSUlNAtLVq06NGjB90nPDwcAHr06FFRUUFaDhw4QM6YAcDVq1dJo1Qq9ff3FwgEubm5pGXmzJlcLrewsJBMVlZW2trahoaGylVYxwH0J4e4cuUKAIwfP575rN69e+/evfvrV/BjVcka6wC6CYWBnP2sIwy7du2qdVZycvLq1at79erF4XAAwM7Ojn6VZR85m9SzZ88GCoNQKNy6dSsAtGzZkn5X0aZNmwYAv//+O7OR/IjbjRs3yCR5r8glXyaTVVRUrF27ltlCTkNv27aNTE6fPh0Azp8/T3eIiYmRy2TdYfjkEDKZzM7OTllZ+cWLF2Tyzp07Ojo65eXlX7+CdWicMDSh3SRlZWX475FxTWVlZSoqKrXO6tatW7du3ebOnfuf//zHw8MjKSlpz549QUFBdYz1448/Ntwtr6dMmSKRSKZPn+7m5nb27FknJyd61r179wDgzp07L1++pBvz8vIAID4+nvlLJfr6+nKLFQgEM2fOvHjx4qNHjwoLCyUSCdmDf/78OemwYMGCtLQ0FxcXOzu7gQMH9uzZc8CAAQKBQPHKPzkEAMyYMWP06NFbt25dvnw5APz222+TJ08mf7uvXEHWNaFTq9ra2gDw7NmzWuc+e/aMdCDy8vK2bNki18fKyoocGKSmptY9VoOGAQCmTZsWHh7+/v37QYMGxcXFyc2VO6jt06dPaGioXD1kK8eUnJzcvn17Dw+P27dvM/9lSCQS8kBbW/vixYsZGRnjxo1LT0/39PQ0MDA4e/as4mV/cggA8PHxMTAw2LZt24cPH548eXL27NlJkybVywqyr6E3PYrvJr1//15ZWVlPT08ikcjNyszM5PF4r169oluuX7/O5/M/fPgg1zM6OhoAFixYQLfU8aHbjh07UlNTP1kYfOmHbuvWrQOAVq1a3bx5k7SEhIQAQHR0NPNZ7969CwsLe/z4MZkkexFnz56VW/jgwYMB4PLly3KrtnDhQjJ55MgR5uqIxWI9PT1NTc06KqTt378/ISHhk0MQ5IzQ1q1bp06dOmHCBOasr1nBOjS7D91atmwZGBj4/Plzsv2lVVRUhISEeHp60pckEdXV1XPnzmVe4FBWVrZp0yYul6vgrSB37tyZlpZWL8XXatasWWvWrHn37p2rq2t8fDwABAcHt2zZMjw8vLy8nO62dOnSnTt3fnK3gezwMH9KPSkpidnh6NGjJH6Ejo6OUChk/lOvw4EDBxITEz85BPHzzz+3bNly3bp1+/btmzFjBnPW16wg65rQMQMAbNiwobCwMCws7Pjx4yKRSCgUvnz58uzZs507d/7999+ZPQ0MDPr37799+/aYmJjevXtraGg8f/78zJkz1dXVBw8eJJ9CAMDKlSvJWVS5U6tEfV2YBAB//PHHo0ePyDsgLCzM0tKSfOo8Z84ciUSyYMECFxeXadOmWVhYnD592s/Pz8rKytXVVU1NLS4uTiwW//PPP+SNuGLFihs3bgDAoUOH4uPjx44d2759ezJEaGjotWvXRo4cOXLkSH19/czMTLJLee3atbCwsAULFohEorCwsKysLPLBy6VLl/Ly8iIiImqtUK7+x48fKzIEKbJNmzbjxo3bunWrq6urlZUVczmmpqZfvIKsa/AfK4mKivLx8fmsUbKysuLi4p4/fy6TybS1tR0dHT92Brq4uDg+Pj4vL6+goKBFixYdOnTo06ePqqoq3eGTV616enpaW1vXXY8iNx4mbzV6kg4DcfDgwezsbPjvZ1Xl5eXnz5/PyMgQCAQWFhaurq70TvaKFSuY/5jl3iulpaWnT5/OyckRCASOjo7t27ffvn07mUXeqe/fv79y5Up2dnZ1dbWhoeGgQYNatWpVa4U1DRkyxN7e/pNDkMePHj3q0KFDTExMrT/h/sUr+DHkxsNkH7gBNfR+mOLHDE0WKHzM0HwcP37cxsam0YZrdscMqIkrKioiO10ymWzt2rWzZs1iu6J61rSOGVBTVlRUNGnSpPv372dlZUkkktGjR7NdUT3DMCBFqaurL1q0iMvl2traent783g8tiuqZxgGpCh1dfWap6H+TfCYASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhSiNdmxQVFdU4AzWQW7dusV1CsyYWiw0MDBp8mIb+wgT5cg9CX6kRvtzT4F/7RJ8UFhYWHR2dnp7OdiHNHR4zIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIETBMCBEwTAgRMEwIERppF/uQUyPHj26cuUKPZmUlPTmzZudO3fSLUZGRq6urixU1rzhj5Ww4MmTJ2ZmZhwOh8fjAQD5E3A4HPK4urp67dq1v/zyC8tVNj8YBnZ89913SUlJtb74HA7nyZMnxsbGjV9VM4fHDOwIDAwkmwU5XC7X0dERk8AKDAM7fH19a90scLncwMDAxq8HAYaBLVpaWiKRqObGQSaTeXh4sFISwjCwJiAgQG7jwOPx+vfvr6mpyVZJzRyGgTUjR47k8//n1LZMJgsICGCrHoRhYE3r1q3d3NyYeVBSUho2bBiLJTVzGAY2+fv7SyQS8pjP5w8bNkxNTY3dkpozDAOb3NzcVFVVyWOJROLv789uPc0choFNKioqnp6eSkpKAKCmpubi4sJ2Rc0ahoFlfn5+VVVVPB7Py8tLWVmZ7XKaNbwcg2USiURbW7uoqOjSpUt9+/Zlu5xmDbcMLOPxeH5+fjo6OiKRiO1amju8hJt9o0aNUlJSqvVSJdSoZCyxsrJie9VRE5WWlsbKe5LNLYOnp6eXlxeLBTR90dHRt27d2rBhA9uFNBKxWDxr1iy2RmczDJ07d/b29maxgKbvwYMH6enpzedVSk9PZzEMeACNEAXDgBAFw4AQBcOAEAXDgBAFw4AQBcOAEAXDgBAFw4AQBcOAEAXDgBAFw4AQBcOAEAXDgBDlGwtDx44dORwOh8NZvXo127V8nqKioilTpvD5/BUrVtTvkhMTEzn/S0lJSVtb28nJacmSJc+ePavf4f7FvrEwZGRkXL9+ne0qPo9UKt2xY0fnzp2TkpLoW4bVI3t7e5lMFh4eDgDh4eEymez9+/e3b98eM2bMkSNHTE1NN27cWO+D/it9Y2H4FqWkpOzbt+/ChQvjx49vnBGVlZVNTEwmTJiQnJzcs2fPGTNmrF+/vnGG/qZhGBqclZVVXFyctbV14w+tpqZ29OhRNTW1BQsW5OTkNH4B3xYMQ4NTVlbmcll7nbW1tX18fCorK3fv3s1WDd+Kph4GmUy2YcOGDh06KCsrGxgYzJkzp7y8vGa3v/76y9nZuXXr1i1atLC2tl69enV1dTWZ1a1bN3JYOW/evP3791tZWbVo0cLCwmLTpk3MJcTFxQ0ePFhHR0dZWbljx46LFy/Oy8tTcIgmzsnJCQCYx1opKSleXl5aWloCgcDY2HjKlCkFBQVkVkhICHm5HBwcbt++7eTkpKqqamBgMHny5NLSUnoJT58+/fHHH42MjMjf5aeffkpMTGQOWscQTRcr9+SQyWRWVlahoaGf7DZlyhQAmDx5cm5ubmFh4bZt27p27QoAq1atovusXLkSAMaNG/f06dOioqKdO3cKBILhw4dLpVLSITk5GQA6deq0ePHigoKCly9fkq/Ynz59mnTIzMxs0aJFYGCgWCwuKyu7du2aiYmJSCRSfAhFREREAMDy5csVf0poaKiVlZUiPZkH0HLOnj0LAJaWlmTy8uXLKioqPXr0uHfv3rt37y5cuGBoaGhiYlbgIaQAABFfSURBVFJYWEg/RSgUGhoa+vr6Zmdnl5SUrFu3DgBmz55N5kqlUgsLC1tb27S0tPLy8ocPHw4cOJD5XlJkiFqlpaUBe7eKadJhIC+NtbU1szE4OJgZhqysLB6P17Fjx+rqarrP9OnTAeDUqVNkkoTBzs5ObslTpkwhk/v37weAmJgYusPhw4fpMCgyhCLYCsM///xDh0EikZiYmPD5/JycHLrDiRMnAGDWrFl0i1AoVFZWfv36NZmUSqXa2tqdO3cmk7m5uQCwYMECuj85gUseKzhErdgNQ5PeTTp//jwAyN2b2tnZmTl56tQpiUTi5ubGvCNdv379AODkyZPMnt999x392NDQEADy8/PJpJOTk5qaWlBQ0J49e169egUAfn5+9O+WKz5E0/T8+XMAaNeuHQCkpKQ8efKka9euzB8UJfd4lVsXExMTDQ0N8pjD4ejr69Mvl76+vo2NzebNm3/99dfMzEyycNl/b9qr+BBNTZMOA3lfamlpMRu1tbWZky9fvgSA9evXMz91Gjp0KACQvxNNKBTSjwUCAQBIpVIyaWpqmpKS4ubm9uuvv2pra5ubm8+ePZvex1V8iKbpxo0bANC7d2/477okJSUx14W8Mk+ePGF+DMJ8uQBAIBDQLxePx4uLi1uwYMHx48etrKy0tLQCAwPJ5vezhmhqmnQYSAxIJGhFRUXMSV1dXQCodY/r6tWrio9lYmKyefPm7Ozs3NzcoKCgiIgIOzu7kpKSehyCFQUFBVFRUcrKyuRTDrIuzMMhmkQiUfx+r61atZo/f35SUlJhYeHGjRtv377do0ePO3fu1OMQja9Jh8HV1ZXD4ZCdJRr5P0dzd3fn8/lyb8rS0lJNTc2dO3cqONDGjRt9fX3JYyMjo9mzZ0+aNEksFt+/f7++hmBFaWmpr69vaWnpmjVryJ5hly5dzM3N79+///btW2bPQYMG+fn5KbjYe/fu6enpkf0ioVDo7++/adOm6upq8peqlyFY0aTD0KlTp+Dg4LS0tClTpuTl5b1582bv3r1//vkns4+pqenq1auvXLkSEhLy5MmT8vLy+/fvDx061MTE5LN+OfPPP/+MiIgoKiqqqKi4c+fOiRMnNDU1bWxs6nGIRlNZWZmTk7Nr1y5bW9ubN29u3ryZHO4DAIfD2bNnT2VlpaenZ3JycllZ2dOnT6dNm5aQkLBkyRLFh3jx4kVwcHBeXl5lZWV2dvaOHTu4XC45nKuvIVjwNUffX0PBU6tSqTQ8PNzS0lJJSUlXVzcoKOj06dN08Xl5eaTbmTNn+vXr17p1axUVFUtLy/nz5xcXF5NZcsffVVVVoaGhzJaEhITS0tKIiAhnZ2ddXV0VFRVjY+OffvopMzOTWUkdQ3ySnZ2d3Muur6+vyBMVOZuUkJAgt3Aej6ehofHDDz8sXrxYLBbXfEp6evqoUaO0tbWVlJQMDQ0DAwMfPnxIZq1atYq5qAsXLsTGxjJbfvvtN6lU+s8//4wcOdLExERZWVlXV3fIkCGXLl1ScIg64KlV9FGKn1r9d8BTqwg1CRgGhCgYhq8VFhbG+Ti2q0OfAcPwtcLCwurYDWW7OvQZMAwIUTAMCFEwDAhRMAwIUTAMCFEwDAhRMAwIUTAMCFEwDAhRMAwIUTAMCFEwDAj9V6N9c0KOlZUV26uOmii2vtzDZ2uFw8PD5b4w3mxFR0fHx8fjjbJp5N4FjY+1MMh9Nbk5e/DgwYMHD8gdLxGL8JgBIQqGASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhCoYBIQqGASEKhgEhCmu3pG/OKioqPnz4QE+Wl5dLJJI3b97QLXw+v1WrVmyU1qxxZDIZ2zU0O9euXROJRHV0CA4O3rx5c6PVgwgMAwukUqm+vv6LFy8+1uHmzZuOjo6NWRICPGZgBZfLDQgIEAgEtc5t166dg4NDI5eEAMPAllGjRlVWVtZsFwgEY8eO5XA4jV8Swt0k1lhYWGRlZdVsT01Ntba2bvx6EG4ZWBMQEKCkpCTX2KFDB0wCWzAMrPH396+urma2KCkpjRkzhq16EO4mscnW1vb+/fv0n4DD4WRnZ5uYmLBbVbOFWwY2BQYG8ng88pjD4djb22MSWIRhYJOvr69UKiWPeTxeYGAgu/U0cxgGNunp6Tk5OXG5XACQSqVeXl5sV9SsYRhYFhAQwOFwuFyus7Ozjo4O2+U0axgGlnl4eHC5XKlUGhAQwHYtzR2GgWVt27YdOHCgQCAYMWIE27U0d3gJN/tGjx6toqIiFArZLqS5Y+1zhpkzZ+bl5bEydFMjkUhevXqlq6vLdiFNRXh4uIGBQeOPy9pu0rlz5x48eMDW6E0Kj8f7WBIePHgQExPTyPWwqKSk5NixY2/fvmVldDZ3k7y8vMLCwlgsoOkLCwuLioqKjo5mu5BGkp6ezuKlWXgAjRAFw4AQBcOAEAXDgBAFw4AQBcOAEAXDgBAFw4AQBcOAEAXDgBAFw4AQBcOAEAXDgBAFw4AQ5Rv7pltgYODTp08BYOLEiX5+fmyXo5Dy8vK///774sWLubm5rVu3NjExCQgIsLGxqa/lP3z48Oeff2a2KCkptW3b1tLSsm/fvn369MHbGCvoG9syTJ061cPD4+rVqyQSTR/5Ctu0adPMzc2nT58+YsSIxMTErl27zpgxo76GaNeuXVhYmLW19dWrV62trcPCwmbPnj1o0KAnT564uLjY2NjcuXOnvsb6d/vGtgw9evSo9U7uTVZFRcXbt29jYmJ69OhBWnx8fBwcHDZu3Oju7t63b9+vH6JVq1bOzs737t0DAHNzc2dnZ9I+bty4+fPnu7m5OTk5nTt3rl7G+nf7xrYM3xwtLa2rV6/SSQAAHo83dOhQALhx40ZDj25tbX3ixAmJROLr6/vu3buGHu5bh2FoWMrKyr1795ZrJDffbteuXSMU0L17dzc3t1evXu3du7cRhvumfQO7SQkJCbt27crJydHS0vLz82vdunXNPmKxeNeuXXfv3q2qqjIzM/P396d/E238+PHkN0H8/PycnJy2bNmSk5NjYGAQFBRkZ2dHL6GwsHD37t1JSUnFxcUmJiZeXl59+/ZlHnrWMcTnunjxorq6ure395c9/XMNHDjw1KlTMTEx06ZNIy2lpaX79u27cuXK27dv9fT0hgwZQu5lBgBbt249duwYAHTq1OnXX39dv3793bt31dXVhw4dyiy4srLy6NGjFy9efPnypZaWlouLi5eXl4qKCt2hjiGaLhlLrKysQkNDP9lt//79XC7X3t5+//79f/75p6+vL7nZ1qpVq+g+58+fV1NT69Kly4EDB44fP07OMq1Zs4bMTUhIiIiIAABXV1d3d/eoqKjIyMguXbrw+fy0tDTS5/Xr1yYmJt27dz906NDZs2eXLVumqqrq4eGh4BCf5fjx4xwO5+TJk4p0Dg0NtbKyUqRneHg4AISHh9ecderUKQDo3LkzmczNzTU3N9fQ0Ni4ceOpU6cWLlwoEAjc3d2rq6tlMllmZmZsbGzLli07d+48aNCg3bt3//33356engCwfft2epnu7u5aWlpbt249d+7cjh072rdvr6GhQc+te4g6pKWlAQD9d2lkTToML168aNGihba2dmlpKd3o7u7ODENRUVGbNm00NTXfvHlD9xk+fDiXy01OTiaTycnJAGBoaFhZWUlarl+/DgCzZ88mk3/88QcAXLp0iV7C+vXrRSKR4kMoKDExsU2bNlu2bFGwf72EgdxsxtTUlEyS3bbr16/THTZu3AgAmzZtoluEQiGHw3nw4AGZ/PDhg5qamr29PZnMz88HgMWLF9P9k5KSmP9YFRmiVuyGoUlvtv7666+ysjJPT09VVVW60dfXl9nnxIkTxcXFI0aMaNOmDd3o4+MjlUoPHjzI7Nm/f3/6Z6M6duwIALm5uWSyffv2ABAeHv748WPS8uOPP5K/32cNUbeUlJQBAwasXLly6tSpij/r6xUWFgKAlpYWADx+/PjatWsWFhZOTk50Bx8fHwDYv38/81nt27e3srIij1u0aGFsbEy/XEKhsG3btpGRkXFxcaSle/fuV65cIY8VH6KpadLHDDk5OQBgaGjIbJS711p2djYAXLp0iT6lCADkLlQpKSnMntra2vRjNTU1AKDP0jo6OkZFRa1YscLMzKxTp06Ojo7Dhw93c3P73CHqkJeXN3jw4EWLFk2aNEnBp9QX8m+bnNEi61JYWMhcFyI1NZU5yXy5AEBNTU0sFpPHqqqqly5dmjdvXu/evfX09BwcHAYPHkze7p81RFPTpMNANghy5wQrKiqYk+Rt7ezsXPMu1rUean+Ml5eXl5dXXl7elStXIiMj3d3dBw4ceObMGT6f//VDFBcXDxo06Keffpo5cyZpuXDhwoULF9auXat4hV+mqqrqyJEjAODv7w//fbnat29f6+3bpFKpgse4tra258+fLy4ujouLO3ny5JQpU5YtW3bjxg0DA4P6GqLxNekw9OzZEwASExOZjWS3ktanTx8AKCsrY/4fkslk5M3dvXt3RQY6duxYVlbWvHnzDA0NAwICAgICgoKCduzYkZiY6ODg8JVDVFRUDB8+vH///kuXLqUbX758qfhW5WvMmTPn2bNngYGB33//PQDY2toKhcKCgoJevXrRv6AFAMuWLZNbwTpkZWUtWrTo6NGjbdq0cXNzc3Nz69ev36hRow4dOjRv3rx6GYIVTTSjRJ8+fUQiUUxMTFRUFGl5+PDh5s2bmX0cHBy8vb2joqJOnDhBWqqqqhYuXHjlyhWRSKTgQGKxeP369RkZGWSyoqIiOzubz+eTjwK+ZgiZTDZmzJjr168nJyc7M/z6668K1vbFsrKy/Pz8Nm7c6OHhsXPnTtKooqKyatUqsVi8cOFC+rdGT5w4sWrVKsXfpu/fv4+MjGTe9JLcNtfIyKi+hmAHK4ftMoVPrRYVFQ0bNozD4Ziamnbp0qVLly7btm0DABMTE5FIVFBQIJPJKioqfvnlF1VVVSMjo++//15LS8vR0ZH8iqZMJvvll1/I5wmGhoYikai6unrv3r29evUCAA0NDZFIlJGR8fDhQ39/f1VVVRsbGwcHh7Zt21pYWBw7dowuo+4h6kD212vl4uLyyacrcjYpIyNDJBKZmZkBgJmZmUgkcnR0NDY2VlJS6tWrV3R0dM2n7N27V19fX0NDo0ePHiYmJiYmJlFRUWTW4cOHRSIRj8dr1aqVSCRKTExMTk4WiUStWrXi8Xgikejo0aMlJSVz5szR1NQ0NTV1dHQ0NDRUV1cPDQ2VSqWKDFEHds8msXZL+k6dOnl7eyt44+H8/PycnBwNDY0OHTq8ffuWnCoFAEdHR2VlZfK4oqLiwYMHlZWVRkZGenp69HNTUlKKioroSZFIlJubSw7NCTs7u1atWgFAZWXlkydPiouLdXR0jI2Na17s+bEh6vDu3buP5UFdXb1Lly51P53ceLju25XXHILP57dp08bU1JR5Fk6OTCbLyMh48+aNrq5u+/bt6f34p0+f0qfUAKBr164cDodc+ESYmZmRUxpSqVQsFufn57dt29bExEQgECg4RB3IjYfT0tI6d+78yc71j5UIyhTeMjRzin/O8O+AnzMg1CQ06bNJ34R9+/bt27fvY3Ppj6JQ04dh+Fpjx44dO3Ys21WgeoC7SQhRMAwIUTAMCFEwDAhRMAwIUTAMCFEwDAhRMAwIUTAMCFEwDAhRMAwIUTAMCFHYvFAvOjo6PT2dxQKavgcPHjx79szLy4vtQhpJSUkJi6Oz9k23mTNn5uXlsTI0auLCw8PlbgjUOFgLA0JNDR4zIETBMCBEwTAgRMEwIET5fyK/xRNzNfL0AAAAAElFTkSuQmCC",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.keras.utils.plot_model(model_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d06bfea0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:38.226146Z",
     "iopub.status.busy": "2023-12-26T07:09:38.225419Z",
     "iopub.status.idle": "2023-12-26T07:09:38.230533Z",
     "shell.execute_reply": "2023-12-26T07:09:38.229713Z"
    },
    "papermill": {
     "duration": 0.048439,
     "end_time": "2023-12-26T07:09:38.233060",
     "exception": false,
     "start_time": "2023-12-26T07:09:38.184621",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# early_stopping = tf.keras.callbacks.EarlyStopping(\n",
    "#     monitor='val_loss',  # Metric to monitor\n",
    "#     patience=3,  # Number of epochs with no improvement after which training will be stopped\n",
    "#     verbose=1,  # Logs when training is stopped\n",
    "#     restore_best_weights=True  # Restores model weights from the epoch with the best value of the monitored quantity\n",
    "# )\n",
    "early_stopping_callback = tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\",\n",
    "                                                          patience=5,\n",
    "                                                          restore_best_weights=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "f8d530e8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:09:38.314485Z",
     "iopub.status.busy": "2023-12-26T07:09:38.313975Z",
     "iopub.status.idle": "2023-12-26T07:10:21.428775Z",
     "shell.execute_reply": "2023-12-26T07:10:21.427117Z"
    },
    "papermill": {
     "duration": 43.161432,
     "end_time": "2023-12-26T07:10:21.431387",
     "exception": false,
     "start_time": "2023-12-26T07:09:38.269955",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "215/215 [==============================] - 6s 18ms/step - loss: 0.5112 - accuracy: 0.7714 - val_loss: 0.4233 - val_accuracy: 0.8110\n",
      "Epoch 2/100\n",
      "215/215 [==============================] - 3s 14ms/step - loss: 0.4191 - accuracy: 0.8146 - val_loss: 0.4068 - val_accuracy: 0.8189\n",
      "Epoch 3/100\n",
      "215/215 [==============================] - 3s 14ms/step - loss: 0.4050 - accuracy: 0.8205 - val_loss: 0.4026 - val_accuracy: 0.8228\n",
      "Epoch 4/100\n",
      "215/215 [==============================] - 3s 14ms/step - loss: 0.3958 - accuracy: 0.8259 - val_loss: 0.4008 - val_accuracy: 0.8228\n",
      "Epoch 5/100\n",
      "215/215 [==============================] - 3s 14ms/step - loss: 0.3881 - accuracy: 0.8300 - val_loss: 0.4003 - val_accuracy: 0.8268\n",
      "Epoch 6/100\n",
      "215/215 [==============================] - 3s 15ms/step - loss: 0.3808 - accuracy: 0.8323 - val_loss: 0.3999 - val_accuracy: 0.8268\n",
      "Epoch 7/100\n",
      "215/215 [==============================] - 3s 14ms/step - loss: 0.3738 - accuracy: 0.8361 - val_loss: 0.3999 - val_accuracy: 0.8255\n",
      "Epoch 8/100\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.3667 - accuracy: 0.8381 - val_loss: 0.4000 - val_accuracy: 0.8320\n",
      "Epoch 9/100\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.3597 - accuracy: 0.8409 - val_loss: 0.4004 - val_accuracy: 0.8320\n",
      "Epoch 10/100\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.3526 - accuracy: 0.8432 - val_loss: 0.4013 - val_accuracy: 0.8294\n",
      "Epoch 11/100\n",
      "215/215 [==============================] - 2s 10ms/step - loss: 0.3454 - accuracy: 0.8481 - val_loss: 0.4024 - val_accuracy: 0.8307\n",
      "Epoch 12/100\n",
      "215/215 [==============================] - 3s 12ms/step - loss: 0.3383 - accuracy: 0.8511 - val_loss: 0.4035 - val_accuracy: 0.8294\n"
     ]
    }
   ],
   "source": [
    "model_2_history = model_2.fit(train_dataset,\n",
    "                             epochs=100,\n",
    "                             validation_data=(valid_dataset),\n",
    "                             callbacks=[early_stopping_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "4c6dfa45",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:10:21.561083Z",
     "iopub.status.busy": "2023-12-26T07:10:21.560716Z",
     "iopub.status.idle": "2023-12-26T07:10:21.764865Z",
     "shell.execute_reply": "2023-12-26T07:10:21.763751Z"
    },
    "papermill": {
     "duration": 0.272693,
     "end_time": "2023-12-26T07:10:21.767319",
     "exception": false,
     "start_time": "2023-12-26T07:10:21.494626",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 8ms/step - loss: 0.3999 - accuracy: 0.8255\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.399919718503952, 0.8254593014717102]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_2.evaluate(valid_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "71d16629",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:10:21.893616Z",
     "iopub.status.busy": "2023-12-26T07:10:21.893218Z",
     "iopub.status.idle": "2023-12-26T07:10:25.787501Z",
     "shell.execute_reply": "2023-12-26T07:10:25.786176Z"
    },
    "papermill": {
     "duration": 3.959549,
     "end_time": "2023-12-26T07:10:25.789512",
     "exception": false,
     "start_time": "2023-12-26T07:10:21.829963",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 512), dtype=float32, numpy=\n",
       "array([[ 3.59669104e-02, -8.57946873e-02, -1.15274275e-02,\n",
       "         5.25982305e-03, -1.85217243e-02, -5.04201166e-02,\n",
       "        -3.61694135e-02,  5.34677319e-03,  4.80591655e-02,\n",
       "         4.69074361e-02, -3.72333042e-02, -1.14954598e-02,\n",
       "         4.35241461e-02,  7.05099106e-02,  7.09376037e-02,\n",
       "        -8.18043128e-02,  8.71716719e-03, -4.65412140e-02,\n",
       "        -2.24577561e-02,  4.68687192e-02,  2.02255719e-03,\n",
       "         3.09907217e-02,  2.04356313e-02,  6.39216825e-02,\n",
       "        -7.64108598e-02,  8.42117891e-02, -4.57604155e-02,\n",
       "        -1.06165453e-03, -2.05941647e-02,  1.24110589e-02,\n",
       "         5.72753921e-02,  3.81562002e-02, -2.74211802e-02,\n",
       "        -3.54347494e-03, -9.83258560e-02, -1.24485437e-02,\n",
       "         3.86562981e-02,  5.03195338e-02, -2.36250591e-02,\n",
       "         3.21848248e-03,  3.22520882e-02,  7.38095073e-03,\n",
       "         4.47310396e-02, -4.12235968e-03,  1.15160132e-02,\n",
       "         2.83772387e-02,  6.01386302e-04, -5.90335354e-02,\n",
       "        -4.94347280e-03, -6.88527431e-03, -4.12424169e-02,\n",
       "        -2.07700115e-03, -1.77201983e-02, -2.27782112e-02,\n",
       "        -2.42898297e-02,  1.39937820e-02, -5.17674796e-02,\n",
       "         5.97374998e-02,  3.81307378e-02, -3.59316505e-02,\n",
       "         3.09860446e-02,  6.09485433e-03, -4.37486842e-02,\n",
       "        -5.38942926e-02, -4.32431735e-02, -3.92870679e-02,\n",
       "         5.61528504e-02,  3.82246706e-03,  6.57869549e-03,\n",
       "        -4.53822017e-02,  3.96313481e-02, -5.69341145e-03,\n",
       "        -2.00726055e-02,  8.17294195e-02,  4.15083915e-02,\n",
       "         3.72894853e-02, -6.70029521e-02, -5.10405190e-02,\n",
       "        -2.56260186e-02,  1.60032213e-02,  3.81825268e-02,\n",
       "        -8.70015658e-03, -2.34965291e-02,  1.55745624e-02,\n",
       "         5.66765331e-02,  7.38508403e-02,  1.52101796e-02,\n",
       "        -8.41723289e-03, -6.44073635e-02, -5.55147976e-02,\n",
       "         7.97353312e-02,  3.61875142e-03,  6.64433539e-02,\n",
       "         2.87908632e-02, -2.44696643e-02,  2.12148149e-02,\n",
       "        -1.03562616e-01, -4.12506908e-02,  2.98409052e-02,\n",
       "        -9.24953595e-02,  4.00588615e-03,  6.01527244e-02,\n",
       "        -3.41755003e-02,  3.29850353e-02, -4.83456142e-02,\n",
       "        -3.69678251e-02,  1.21584861e-02, -2.31851619e-02,\n",
       "        -4.88560051e-02,  3.28469612e-02, -4.62380089e-02,\n",
       "         3.91756278e-03, -3.57808359e-02, -1.83102041e-02,\n",
       "        -3.70730087e-02,  3.35950032e-02,  5.02075367e-02,\n",
       "        -5.49601465e-02,  4.69865017e-02, -3.31131369e-02,\n",
       "         1.66334976e-02, -3.02660111e-02,  6.20632172e-02,\n",
       "        -4.43871841e-02,  6.67187944e-02, -6.10228628e-03,\n",
       "         3.40605080e-02, -5.00078686e-02,  2.04585260e-03,\n",
       "         3.39132398e-02, -3.10870782e-02, -1.93445925e-02,\n",
       "         7.54030049e-02, -6.52688220e-02,  5.25344461e-02,\n",
       "        -6.12028502e-02, -3.02657727e-02,  4.56587551e-03,\n",
       "        -8.89162868e-02, -4.83766720e-02, -2.00605262e-02,\n",
       "        -3.86419035e-02,  1.66885965e-02, -1.47099299e-02,\n",
       "         1.01379864e-02, -8.22198540e-02,  2.74562407e-02,\n",
       "        -3.86098213e-03, -1.09152887e-02, -5.79520538e-02,\n",
       "         2.82120109e-02, -1.71919707e-02,  3.00881453e-02,\n",
       "         2.37060748e-02, -2.33154069e-03, -2.97773238e-02,\n",
       "         1.91768687e-02,  5.16294092e-02, -1.44551247e-02,\n",
       "         1.86847970e-02, -2.49356478e-02,  3.22919488e-02,\n",
       "         2.82293409e-02, -2.51184143e-02, -1.08434986e-02,\n",
       "         3.01642809e-02,  5.04591502e-04, -2.27955673e-02,\n",
       "         2.47641988e-02,  6.97232559e-02,  6.99917274e-03,\n",
       "        -7.34951161e-03,  9.45000499e-02, -6.13124780e-02,\n",
       "         2.62528770e-02,  3.58930118e-02,  6.17794041e-03,\n",
       "         3.43658775e-02,  1.17269838e-02,  4.01629470e-02,\n",
       "         1.53442472e-02,  4.48108371e-03,  5.02520911e-02,\n",
       "         1.88765489e-02,  3.31166796e-02, -2.70847231e-02,\n",
       "        -6.57292223e-03,  8.24265555e-02,  1.05237104e-02,\n",
       "         3.99747752e-02,  2.82466621e-03, -1.24388905e-02,\n",
       "        -3.28871747e-03, -7.26752877e-02, -1.81361043e-03,\n",
       "        -2.76129469e-02,  6.45669401e-02,  5.02010472e-02,\n",
       "         6.75971806e-02,  9.59104113e-03, -8.90628770e-02,\n",
       "         1.17860874e-02,  2.72054579e-02, -3.41815525e-03,\n",
       "        -8.05912465e-02, -5.22511080e-02, -3.81539552e-03,\n",
       "        -4.17377092e-02, -9.68295336e-03,  3.78751308e-02,\n",
       "        -5.67596816e-02, -2.05952711e-02, -1.14134215e-02,\n",
       "        -2.83810738e-02,  9.09044072e-02,  3.93447524e-04,\n",
       "        -2.67902762e-03,  1.53436465e-03,  6.49558231e-02,\n",
       "         5.22872619e-02,  1.80281438e-02,  1.84829552e-02,\n",
       "         1.61396414e-02,  5.27072288e-02, -4.12999056e-02,\n",
       "        -4.80413213e-02, -2.36369204e-02, -2.67371200e-02,\n",
       "         3.93583998e-02,  7.55005376e-03, -6.21337853e-02,\n",
       "        -1.67872924e-02,  1.47809135e-03, -5.64959720e-02,\n",
       "         5.52060865e-02, -1.27272941e-02,  1.03164591e-01,\n",
       "        -1.69769656e-02, -2.17627510e-02, -1.17612286e-02,\n",
       "         1.40766259e-02, -1.40489535e-02,  6.19004555e-02,\n",
       "        -3.57006304e-02,  1.31614264e-02, -3.44647504e-02,\n",
       "        -7.06140092e-03,  1.24527169e-02,  8.55674818e-02,\n",
       "         5.42758591e-02, -3.89169194e-02, -7.57954717e-02,\n",
       "        -1.40139572e-02,  1.48370443e-02,  9.68214497e-02,\n",
       "        -1.03704758e-01,  3.29137146e-02, -6.21125400e-02,\n",
       "         1.86114740e-02, -2.82176267e-02, -7.13014323e-03,\n",
       "         4.11865069e-03, -8.93324912e-02, -5.72975539e-02,\n",
       "         4.44728173e-02,  2.29126122e-02,  4.30734223e-03,\n",
       "        -6.96439147e-02, -6.36359677e-02, -2.61848159e-02,\n",
       "        -6.48371652e-02, -9.65941474e-02,  5.09348139e-02,\n",
       "        -8.90865326e-02, -3.94751914e-02, -2.23109536e-02,\n",
       "        -2.93314215e-02,  3.75236943e-02,  4.67260703e-02,\n",
       "         8.62412080e-02, -2.14184057e-02, -5.29951602e-02,\n",
       "         5.89277968e-02,  1.65255722e-02, -1.04241502e-02,\n",
       "        -2.38159727e-02,  1.38605097e-02,  2.62649842e-02,\n",
       "         5.47960103e-02, -4.24158275e-02,  3.17361066e-03,\n",
       "        -3.96554358e-02, -7.35218003e-02,  2.73186504e-03,\n",
       "        -2.72395392e-03, -3.98468450e-02, -4.29545268e-02,\n",
       "         2.14427803e-02,  3.84629294e-02,  3.88851911e-02,\n",
       "         5.13683306e-03, -4.95571718e-02, -3.24903652e-02,\n",
       "         2.83539910e-02, -4.67896536e-02,  1.97179299e-02,\n",
       "        -4.20575328e-02, -7.54898638e-02, -1.03863399e-03,\n",
       "        -6.73990846e-02,  3.91278006e-02,  4.40352261e-02,\n",
       "        -8.06924552e-02, -2.36365721e-02,  5.99047504e-02,\n",
       "         2.12464621e-03,  2.21580118e-02,  4.36153412e-02,\n",
       "         2.49224412e-03,  4.98775207e-03,  7.48325605e-03,\n",
       "        -2.41463929e-02,  4.05475833e-02,  2.91890893e-02,\n",
       "        -1.10162618e-02, -5.65643162e-02, -6.73807636e-02,\n",
       "         4.05558571e-02, -8.50009471e-02, -5.40723391e-02,\n",
       "         7.34482110e-02, -7.35008046e-02,  7.54501158e-03,\n",
       "        -7.00687692e-02,  4.39716466e-02, -2.71129999e-02,\n",
       "        -6.92854971e-02, -2.58241743e-02,  3.92786302e-02,\n",
       "         5.67648746e-02,  1.59088895e-02,  2.35410146e-02,\n",
       "        -5.06520681e-02,  1.17191877e-02, -6.92344531e-02,\n",
       "         1.38218021e-02,  1.09325098e-02,  7.00728893e-02,\n",
       "        -1.26283066e-02,  7.97541440e-02, -3.13802809e-02,\n",
       "        -4.11442146e-02, -3.50070745e-02, -4.26069349e-02,\n",
       "         4.05677110e-02,  7.93018416e-02, -9.75148305e-02,\n",
       "        -1.42815765e-02,  2.42777336e-02, -7.38642067e-02,\n",
       "        -1.56750157e-02, -3.48734483e-02, -5.93835339e-02,\n",
       "        -4.12432365e-02, -4.60711075e-03,  3.71533111e-02,\n",
       "         2.14900598e-02,  5.58095276e-02,  2.14638896e-02,\n",
       "        -6.75328374e-02,  1.01270585e-03,  5.95025197e-02,\n",
       "         1.24676106e-03,  3.76901664e-02,  4.18491513e-02,\n",
       "        -2.79387459e-02, -4.01746631e-02, -8.21424834e-03,\n",
       "        -5.58206625e-02, -1.60544906e-02, -4.75718491e-02,\n",
       "        -5.95801920e-02, -1.01037407e-02, -7.33452067e-02,\n",
       "        -1.84457712e-02,  3.76496129e-02, -8.28180313e-02,\n",
       "         7.54492171e-03, -2.76729763e-02, -3.70695889e-02,\n",
       "        -4.07462008e-02,  7.34208757e-03, -3.86688090e-03,\n",
       "        -1.10412011e-05,  3.76898386e-02,  1.45606184e-02,\n",
       "         2.78660888e-03, -6.57309070e-02, -2.55980715e-02,\n",
       "        -4.81109843e-02,  1.18657360e-02, -2.13191174e-02,\n",
       "         3.31167621e-03,  4.66567352e-02,  4.63188067e-02,\n",
       "        -3.60700563e-02, -7.05618635e-02, -7.14360178e-02,\n",
       "        -9.04082786e-03,  2.11611893e-02, -4.92307954e-02,\n",
       "         3.87155823e-02,  7.73286894e-02,  1.89449191e-02,\n",
       "         3.85826752e-02, -6.83923960e-02, -5.28842211e-02,\n",
       "         8.54228660e-02,  1.28640411e-02,  2.20266078e-02,\n",
       "         2.16055308e-02,  5.56477928e-04,  1.64826363e-02,\n",
       "        -3.31564397e-02,  8.13280344e-02, -2.25724392e-02,\n",
       "        -4.43336144e-02,  7.10498765e-02, -8.87572840e-02,\n",
       "         5.36288284e-02, -6.93188459e-02, -2.98249465e-03,\n",
       "         4.52212580e-02,  4.61279564e-02,  3.65344770e-02,\n",
       "         4.84847240e-02, -2.92588528e-02,  7.36933053e-02,\n",
       "         3.66693325e-02, -6.80679232e-02,  3.96359526e-03,\n",
       "        -9.04875919e-02, -6.32659942e-02, -5.38307987e-02,\n",
       "         6.08242899e-02, -1.84412934e-02,  4.89365384e-02,\n",
       "        -5.12656756e-02, -1.50806140e-02, -5.71213290e-03,\n",
       "        -5.10706715e-02, -6.49115443e-02, -1.13350083e-03,\n",
       "         1.27703352e-02,  1.82183105e-02, -8.28226283e-02,\n",
       "         2.10518437e-03, -3.31831463e-02,  5.95652163e-02,\n",
       "         3.61991227e-02,  5.35384640e-02, -2.25340743e-02,\n",
       "         2.64133215e-02, -3.91572677e-02, -5.87077700e-02,\n",
       "        -1.03312604e-01,  7.44982332e-04,  1.97312124e-02,\n",
       "         3.94937135e-02,  9.22513306e-02, -2.05244888e-02,\n",
       "         2.21597645e-02, -3.13704535e-02, -1.32244062e-02,\n",
       "         4.00840454e-02, -1.50230555e-02,  6.54290244e-02,\n",
       "        -6.08944334e-02,  7.63139571e-04, -3.62863131e-02,\n",
       "         4.91616279e-02, -6.04683384e-02, -1.77060775e-02,\n",
       "         2.45514186e-03, -4.52068225e-02,  3.63887697e-02,\n",
       "        -9.97238085e-02, -2.95221955e-02,  8.41336604e-03,\n",
       "        -8.18829536e-02,  1.82565190e-02,  1.11088259e-02,\n",
       "        -3.46704014e-02, -1.07370624e-02, -4.20162492e-02,\n",
       "        -2.58049741e-02,  3.06181330e-02,  1.86228398e-02,\n",
       "         2.78460630e-03, -3.67320632e-03,  5.00444323e-02,\n",
       "        -2.75492594e-02,  4.84464467e-02,  7.04697818e-02,\n",
       "         9.36839804e-02,  2.49330979e-02, -4.27237116e-02,\n",
       "        -3.15812379e-02, -4.44082879e-02, -3.41433547e-02,\n",
       "         2.81602386e-02, -8.78943969e-03]], dtype=float32)>"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow_hub as hub\n",
    "embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\") # load Universal Sentence Encoder\n",
    "embed_samples = embed([\"When you call the universal sentence encoder on a sentence, it turns it into numbers.\"])\n",
    "embed_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "1fb2be69",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:10:25.916322Z",
     "iopub.status.busy": "2023-12-26T07:10:25.915999Z",
     "iopub.status.idle": "2023-12-26T07:10:25.921064Z",
     "shell.execute_reply": "2023-12-26T07:10:25.920006Z"
    },
    "papermill": {
     "duration": 0.070406,
     "end_time": "2023-12-26T07:10:25.922775",
     "exception": false,
     "start_time": "2023-12-26T07:10:25.852369",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import tensorflow_hub as hub\n",
    "# embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\") # load Universal Sentence Encoder\n",
    "# embed_samples = embed([sample_sentence,\n",
    "#                       \"When you call the universal sentence encoder on a sentence, it turns it into numbers.\"])\n",
    "\n",
    "# print(embed_samples[0][:50])\n",
    "\n",
    "# We can use this encoding layer in place of our text_vectorizer and embedding layer\n",
    "# sentence_encoder_layer = hub.KerasLayer(\"https://tfhub.dev/google/universal-sentence-encoder/4\",\n",
    "#                                         input_shape=[], # shape of inputs coming to our model \n",
    "#                                         dtype=tf.string, # data type of inputs coming to the USE layer\n",
    "#                                         trainable=False, # keep the pretrained weights (we'll create a feature extractor)\n",
    "#                                         name=\"USE\") \n",
    "\n",
    "# Create model using the Sequential API\n",
    "# model_6 = tf.keras.Sequential([\n",
    "#   sentence_encoder_layer, # take in sentences and then encode them into an embedding\n",
    "#   layers.Dense(64, activation=\"relu\"),\n",
    "#   layers.Dense(1, activation=\"sigmoid\")\n",
    "# ], name=\"model_6_USE\")\n",
    "\n",
    "# # Compile model\n",
    "# model_6.compile(loss=\"binary_crossentropy\",\n",
    "#                 optimizer=tf.keras.optimizers.Adam(),\n",
    "#                 metrics=[\"accuracy\"])\n",
    "\n",
    "# model_6.summary()\n",
    "\n",
    "# text_input = tf.keras.layers.Input(shape=(), dtype=tf.string)\n",
    "# preprocessor = hub.KerasLayer(\n",
    "#     \"https://kaggle.com/models/tensorflow/bert/frameworks/TensorFlow2/variations/en-uncased-preprocess/versions/3\")\n",
    "# encoder_inputs = preprocessor(text_input)\n",
    "# encoder = hub.KerasLayer(\n",
    "#     \"https://www.kaggle.com/models/tensorflow/bert/frameworks/TensorFlow2/variations/bert-en-uncased-l-10-h-128-a-2/versions/2\",\n",
    "#     trainable=True)\n",
    "# outputs = encoder(encoder_inputs)\n",
    "# pooled_output = outputs[\"pooled_output\"]      # [batch_size, 128].\n",
    "# sequence_output = outputs[\"sequence_output\"]  # [batch_size, seq_length, 128].\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d76d4dac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:10:26.048462Z",
     "iopub.status.busy": "2023-12-26T07:10:26.047535Z",
     "iopub.status.idle": "2023-12-26T07:10:26.053201Z",
     "shell.execute_reply": "2023-12-26T07:10:26.051883Z"
    },
    "papermill": {
     "duration": 0.071358,
     "end_time": "2023-12-26T07:10:26.055325",
     "exception": false,
     "start_time": "2023-12-26T07:10:25.983967",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# # Code below is adapted from: https://www.tensorflow.org/tutorials/text/word_embeddings#retrieve_the_trained_word_embeddings_and_save_them_to_disk\n",
    "# import io\n",
    "\n",
    "# # Create output writers\n",
    "# out_v = io.open(\"embedding_vectors.tsv\", \"w\", encoding=\"utf-8\")\n",
    "# out_m = io.open(\"embedding_metadata.tsv\", \"w\", encoding=\"utf-8\")\n",
    "\n",
    "# # Write embedding vectors and words to file\n",
    "# for num, word in enumerate(words_in_vocab):\n",
    "#   if num == 0: \n",
    "#      continue # skip padding token\n",
    "#   vec = embed_weights[num]\n",
    "#   out_m.write(word + \"\\n\") # write words to file\n",
    "#   out_v.write(\"\\t\".join([str(x) for x in vec]) + \"\\n\") # write corresponding word vector to file\n",
    "# out_v.close()\n",
    "# out_m.close()\n",
    "\n",
    "# # Download files locally to upload to Embedding Projector\n",
    "# try:\n",
    "#   from google.colab import files\n",
    "# except ImportError:\n",
    "#   pass\n",
    "# else:\n",
    "#   files.download(\"embedding_vectors.tsv\")\n",
    "#   files.download(\"embedding_metadata.tsv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddac81f0",
   "metadata": {
    "papermill": {
     "duration": 0.064159,
     "end_time": "2023-12-26T07:10:26.182875",
     "exception": false,
     "start_time": "2023-12-26T07:10:26.118716",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model 3: LSTM "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "900cdeb4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:10:26.319622Z",
     "iopub.status.busy": "2023-12-26T07:10:26.319111Z",
     "iopub.status.idle": "2023-12-26T07:10:27.264214Z",
     "shell.execute_reply": "2023-12-26T07:10:27.262674Z"
    },
    "papermill": {
     "duration": 1.019704,
     "end_time": "2023-12-26T07:10:27.266810",
     "exception": false,
     "start_time": "2023-12-26T07:10:26.247106",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = tf.keras.layers.Input(shape=(), dtype=\"string\")\n",
    "x = text_vectorizer(inputs)\n",
    "x = token_embedding(x)\n",
    "x = tf.keras.layers.LSTM(64)(x)\n",
    "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "model_3 = tf.keras.Model(inputs, outputs, name=\"model_3_lstm\")\n",
    "\n",
    "# inputs = tf.keras.Input(shape=(1,), dtype=\"string\") # 1 dimensional strings\n",
    "\n",
    "# x = text_vectorizer(inputs) # turn input text into numbers\n",
    "# x = token_embedding(x)\n",
    "\n",
    "# x = tf.keras.layers.GlobalAveragePooling1D()(x)\n",
    "\n",
    "# outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "# model_1 = tf.keras.Model(inputs, outputs, name=\"model_1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "a5b1e04e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:10:27.395336Z",
     "iopub.status.busy": "2023-12-26T07:10:27.394764Z",
     "iopub.status.idle": "2023-12-26T07:10:27.404724Z",
     "shell.execute_reply": "2023-12-26T07:10:27.404074Z"
    },
    "papermill": {
     "duration": 0.075992,
     "end_time": "2023-12-26T07:10:27.406796",
     "exception": false,
     "start_time": "2023-12-26T07:10:27.330804",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_3.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e4ebde22",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:10:27.536085Z",
     "iopub.status.busy": "2023-12-26T07:10:27.535712Z",
     "iopub.status.idle": "2023-12-26T07:10:27.563443Z",
     "shell.execute_reply": "2023-12-26T07:10:27.561565Z"
    },
    "papermill": {
     "duration": 0.09679,
     "end_time": "2023-12-26T07:10:27.566153",
     "exception": false,
     "start_time": "2023-12-26T07:10:27.469363",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_3_lstm\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_2 (InputLayer)        [(None,)]                 0         \n",
      "                                                                 \n",
      " text_vectorization (TextVe  (None, 24)                0         \n",
      " ctorization)                                                    \n",
      "                                                                 \n",
      " token_embed (Embedding)     (None, 24, 128)           1280000   \n",
      "                                                                 \n",
      " lstm (LSTM)                 (None, 64)                49408     \n",
      "                                                                 \n",
      " dense_3 (Dense)             (None, 1)                 65        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1329473 (5.07 MB)\n",
      "Trainable params: 1329473 (5.07 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_3.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "630b569d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:10:27.697472Z",
     "iopub.status.busy": "2023-12-26T07:10:27.697083Z",
     "iopub.status.idle": "2023-12-26T07:11:07.561984Z",
     "shell.execute_reply": "2023-12-26T07:11:07.561140Z"
    },
    "papermill": {
     "duration": 39.932164,
     "end_time": "2023-12-26T07:11:07.564635",
     "exception": false,
     "start_time": "2023-12-26T07:10:27.632471",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "215/215 [==============================] - 10s 29ms/step - loss: 0.2038 - accuracy: 0.9355 - val_loss: 0.6440 - val_accuracy: 0.7533\n",
      "Epoch 2/100\n",
      "215/215 [==============================] - 5s 24ms/step - loss: 0.1408 - accuracy: 0.9479 - val_loss: 0.7683 - val_accuracy: 0.7598\n",
      "Epoch 3/100\n",
      "215/215 [==============================] - 5s 23ms/step - loss: 0.1099 - accuracy: 0.9588 - val_loss: 0.9327 - val_accuracy: 0.7546\n",
      "Epoch 4/100\n",
      "215/215 [==============================] - 5s 23ms/step - loss: 0.0874 - accuracy: 0.9664 - val_loss: 1.0929 - val_accuracy: 0.7415\n",
      "Epoch 5/100\n",
      "215/215 [==============================] - 5s 25ms/step - loss: 0.0734 - accuracy: 0.9711 - val_loss: 1.1178 - val_accuracy: 0.7402\n",
      "Epoch 6/100\n",
      "215/215 [==============================] - 5s 25ms/step - loss: 0.0657 - accuracy: 0.9720 - val_loss: 1.0928 - val_accuracy: 0.7454\n"
     ]
    }
   ],
   "source": [
    "model_3_history = model_3.fit(train_dataset,\n",
    "                             epochs=100,\n",
    "                             validation_data=(valid_dataset),\n",
    "                             callbacks=[early_stopping_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "52699d70",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:07.819070Z",
     "iopub.status.busy": "2023-12-26T07:11:07.818666Z",
     "iopub.status.idle": "2023-12-26T07:11:08.136134Z",
     "shell.execute_reply": "2023-12-26T07:11:08.134405Z"
    },
    "papermill": {
     "duration": 0.47971,
     "end_time": "2023-12-26T07:11:08.138947",
     "exception": false,
     "start_time": "2023-12-26T07:11:07.659237",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 7ms/step - loss: 0.6440 - accuracy: 0.7533\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.6439670324325562, 0.7532808184623718]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_3.evaluate(valid_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f4ecf141",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:08.328191Z",
     "iopub.status.busy": "2023-12-26T07:11:08.327841Z",
     "iopub.status.idle": "2023-12-26T07:11:08.645376Z",
     "shell.execute_reply": "2023-12-26T07:11:08.643991Z"
    },
    "papermill": {
     "duration": 0.415199,
     "end_time": "2023-12-26T07:11:08.647951",
     "exception": false,
     "start_time": "2023-12-26T07:11:08.232752",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 8ms/step - loss: 0.3999 - accuracy: 0.8255\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.399919718503952, 0.8254593014717102]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_2.evaluate(valid_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "39a5a872",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:08.838064Z",
     "iopub.status.busy": "2023-12-26T07:11:08.836876Z",
     "iopub.status.idle": "2023-12-26T07:11:08.992476Z",
     "shell.execute_reply": "2023-12-26T07:11:08.991217Z"
    },
    "papermill": {
     "duration": 0.253181,
     "end_time": "2023-12-26T07:11:08.995308",
     "exception": false,
     "start_time": "2023-12-26T07:11:08.742127",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 3ms/step - loss: 0.5557 - accuracy: 0.7651\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.5556552410125732, 0.7650918364524841]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_1.evaluate(valid_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0efe4b42",
   "metadata": {
    "papermill": {
     "duration": 0.095983,
     "end_time": "2023-12-26T07:11:09.182890",
     "exception": false,
     "start_time": "2023-12-26T07:11:09.086907",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model 4: GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "1eb6ace2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:09.372950Z",
     "iopub.status.busy": "2023-12-26T07:11:09.372503Z",
     "iopub.status.idle": "2023-12-26T07:11:10.209251Z",
     "shell.execute_reply": "2023-12-26T07:11:10.207778Z"
    },
    "papermill": {
     "duration": 0.933136,
     "end_time": "2023-12-26T07:11:10.211510",
     "exception": false,
     "start_time": "2023-12-26T07:11:09.278374",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = tf.keras.layers.Input(shape=(), dtype=\"string\")\n",
    "x = text_vectorizer(inputs)\n",
    "x = token_embedding(x)\n",
    "x = tf.keras.layers.GRU(64)(x)\n",
    "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "model_4 = tf.keras.Model(inputs, outputs, name=\"model_4_gru\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "f9de1a03",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:10.401773Z",
     "iopub.status.busy": "2023-12-26T07:11:10.401292Z",
     "iopub.status.idle": "2023-12-26T07:11:10.414111Z",
     "shell.execute_reply": "2023-12-26T07:11:10.412748Z"
    },
    "papermill": {
     "duration": 0.110791,
     "end_time": "2023-12-26T07:11:10.416454",
     "exception": false,
     "start_time": "2023-12-26T07:11:10.305663",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_4.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "95c91004",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:10.611306Z",
     "iopub.status.busy": "2023-12-26T07:11:10.609990Z",
     "iopub.status.idle": "2023-12-26T07:11:10.642313Z",
     "shell.execute_reply": "2023-12-26T07:11:10.640502Z"
    },
    "papermill": {
     "duration": 0.1334,
     "end_time": "2023-12-26T07:11:10.645045",
     "exception": false,
     "start_time": "2023-12-26T07:11:10.511645",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_4_gru\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_3 (InputLayer)        [(None,)]                 0         \n",
      "                                                                 \n",
      " text_vectorization (TextVe  (None, 24)                0         \n",
      " ctorization)                                                    \n",
      "                                                                 \n",
      " token_embed (Embedding)     (None, 24, 128)           1280000   \n",
      "                                                                 \n",
      " gru (GRU)                   (None, 64)                37248     \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 1)                 65        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1317313 (5.03 MB)\n",
      "Trainable params: 1317313 (5.03 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_4.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "e3d7d761",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:10.836905Z",
     "iopub.status.busy": "2023-12-26T07:11:10.836525Z",
     "iopub.status.idle": "2023-12-26T07:11:49.316437Z",
     "shell.execute_reply": "2023-12-26T07:11:49.314740Z"
    },
    "papermill": {
     "duration": 38.579168,
     "end_time": "2023-12-26T07:11:49.319901",
     "exception": false,
     "start_time": "2023-12-26T07:11:10.740733",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "215/215 [==============================] - 10s 28ms/step - loss: 0.1958 - accuracy: 0.9307 - val_loss: 0.7300 - val_accuracy: 0.7559\n",
      "Epoch 2/100\n",
      "215/215 [==============================] - 5s 22ms/step - loss: 0.1275 - accuracy: 0.9555 - val_loss: 0.8393 - val_accuracy: 0.7454\n",
      "Epoch 3/100\n",
      "215/215 [==============================] - 5s 22ms/step - loss: 0.1036 - accuracy: 0.9642 - val_loss: 0.9693 - val_accuracy: 0.7362\n",
      "Epoch 4/100\n",
      "215/215 [==============================] - 5s 22ms/step - loss: 0.0871 - accuracy: 0.9695 - val_loss: 1.0267 - val_accuracy: 0.7323\n",
      "Epoch 5/100\n",
      "215/215 [==============================] - 5s 22ms/step - loss: 0.0775 - accuracy: 0.9724 - val_loss: 1.0742 - val_accuracy: 0.7388\n",
      "Epoch 6/100\n",
      "215/215 [==============================] - 5s 22ms/step - loss: 0.0685 - accuracy: 0.9743 - val_loss: 1.0965 - val_accuracy: 0.7454\n"
     ]
    }
   ],
   "source": [
    "model_4_history = model_4.fit(train_dataset,\n",
    "                             epochs=100,\n",
    "                             validation_data=(valid_dataset),\n",
    "                             callbacks=[early_stopping_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "58a89540",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:49.563192Z",
     "iopub.status.busy": "2023-12-26T07:11:49.562080Z",
     "iopub.status.idle": "2023-12-26T07:11:49.727323Z",
     "shell.execute_reply": "2023-12-26T07:11:49.726180Z"
    },
    "papermill": {
     "duration": 0.290747,
     "end_time": "2023-12-26T07:11:49.729298",
     "exception": false,
     "start_time": "2023-12-26T07:11:49.438551",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 6ms/step - loss: 0.7300 - accuracy: 0.7559\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.7300063967704773, 0.7559055089950562]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_4.evaluate(valid_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9045c65c",
   "metadata": {
    "papermill": {
     "duration": 0.120687,
     "end_time": "2023-12-26T07:11:49.973866",
     "exception": false,
     "start_time": "2023-12-26T07:11:49.853179",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model 5: LSTM and GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "82ee4387",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:50.221267Z",
     "iopub.status.busy": "2023-12-26T07:11:50.220038Z",
     "iopub.status.idle": "2023-12-26T07:11:56.402153Z",
     "shell.execute_reply": "2023-12-26T07:11:56.400548Z"
    },
    "papermill": {
     "duration": 6.309302,
     "end_time": "2023-12-26T07:11:56.404760",
     "exception": false,
     "start_time": "2023-12-26T07:11:50.095458",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = tf.keras.layers.Input(shape=(), dtype=\"string\")\n",
    "x = text_vectorizer(inputs)\n",
    "x = token_embedding(x)\n",
    "x = tf.keras.layers.LSTM(64, return_sequences=True)(x)\n",
    "x = tf.keras.layers.GRU(64)(x)\n",
    "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "model_5 = tf.keras.Model(inputs, outputs, name=\"model_5_lstm_and_gru\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "9c820c0a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:56.656059Z",
     "iopub.status.busy": "2023-12-26T07:11:56.655683Z",
     "iopub.status.idle": "2023-12-26T07:11:56.667237Z",
     "shell.execute_reply": "2023-12-26T07:11:56.666356Z"
    },
    "papermill": {
     "duration": 0.139206,
     "end_time": "2023-12-26T07:11:56.669571",
     "exception": false,
     "start_time": "2023-12-26T07:11:56.530365",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_5.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "a2333f41",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:11:56.914086Z",
     "iopub.status.busy": "2023-12-26T07:11:56.913450Z",
     "iopub.status.idle": "2023-12-26T07:13:07.949259Z",
     "shell.execute_reply": "2023-12-26T07:13:07.947274Z"
    },
    "papermill": {
     "duration": 71.16251,
     "end_time": "2023-12-26T07:13:07.952224",
     "exception": false,
     "start_time": "2023-12-26T07:11:56.789714",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "215/215 [==============================] - 18s 48ms/step - loss: 0.1599 - accuracy: 0.9477 - val_loss: 0.7985 - val_accuracy: 0.7559\n",
      "Epoch 2/100\n",
      "215/215 [==============================] - 8s 36ms/step - loss: 0.1170 - accuracy: 0.9594 - val_loss: 0.9330 - val_accuracy: 0.7467\n",
      "Epoch 3/100\n",
      "215/215 [==============================] - 8s 36ms/step - loss: 0.0956 - accuracy: 0.9653 - val_loss: 1.0365 - val_accuracy: 0.7375\n",
      "Epoch 4/100\n",
      "215/215 [==============================] - 8s 36ms/step - loss: 0.0774 - accuracy: 0.9710 - val_loss: 1.1701 - val_accuracy: 0.7270\n",
      "Epoch 5/100\n",
      "215/215 [==============================] - 8s 35ms/step - loss: 0.0694 - accuracy: 0.9720 - val_loss: 1.3131 - val_accuracy: 0.7310\n",
      "Epoch 6/100\n",
      "215/215 [==============================] - 8s 36ms/step - loss: 0.0614 - accuracy: 0.9739 - val_loss: 1.4176 - val_accuracy: 0.7402\n"
     ]
    }
   ],
   "source": [
    "model_5_history = model_5.fit(train_dataset,\n",
    "                             epochs=100,\n",
    "                             validation_data=(valid_dataset),\n",
    "                             callbacks=[early_stopping_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "527b0e9e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:13:08.290047Z",
     "iopub.status.busy": "2023-12-26T07:13:08.289526Z",
     "iopub.status.idle": "2023-12-26T07:13:08.562376Z",
     "shell.execute_reply": "2023-12-26T07:13:08.561536Z"
    },
    "papermill": {
     "duration": 0.441415,
     "end_time": "2023-12-26T07:13:08.564726",
     "exception": false,
     "start_time": "2023-12-26T07:13:08.123311",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 10ms/step - loss: 0.7985 - accuracy: 0.7559\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.7984885573387146, 0.7559055089950562]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_5.evaluate(valid_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "031c2b2f",
   "metadata": {
    "papermill": {
     "duration": 0.163064,
     "end_time": "2023-12-26T07:13:08.892158",
     "exception": false,
     "start_time": "2023-12-26T07:13:08.729094",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model 6: LSTM and GRU multiple layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1c2b7e5d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:13:09.218801Z",
     "iopub.status.busy": "2023-12-26T07:13:09.217804Z",
     "iopub.status.idle": "2023-12-26T07:13:13.582157Z",
     "shell.execute_reply": "2023-12-26T07:13:13.580275Z"
    },
    "papermill": {
     "duration": 4.533141,
     "end_time": "2023-12-26T07:13:13.585408",
     "exception": false,
     "start_time": "2023-12-26T07:13:09.052267",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = tf.keras.layers.Input(shape=(), dtype=\"string\")\n",
    "x = text_vectorizer(inputs)\n",
    "x = token_embedding(x)\n",
    "x = tf.keras.layers.LSTM(64, return_sequences=True)(x)\n",
    "x = tf.keras.layers.GRU(32, return_sequences=True)(x)\n",
    "x = tf.keras.layers.GRU(16, return_sequences=True)(x)\n",
    "x = tf.keras.layers.LSTM(8)(x)\n",
    "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "model_6 = tf.keras.Model(inputs, outputs, name=\"model_6_mix_rnn\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "35050bce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:13:13.911240Z",
     "iopub.status.busy": "2023-12-26T07:13:13.910577Z",
     "iopub.status.idle": "2023-12-26T07:13:13.945743Z",
     "shell.execute_reply": "2023-12-26T07:13:13.943847Z"
    },
    "papermill": {
     "duration": 0.200127,
     "end_time": "2023-12-26T07:13:13.948268",
     "exception": false,
     "start_time": "2023-12-26T07:13:13.748141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_6_mix_rnn\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_5 (InputLayer)        [(None,)]                 0         \n",
      "                                                                 \n",
      " text_vectorization (TextVe  (None, 24)                0         \n",
      " ctorization)                                                    \n",
      "                                                                 \n",
      " token_embed (Embedding)     (None, 24, 128)           1280000   \n",
      "                                                                 \n",
      " lstm_2 (LSTM)               (None, 24, 64)            49408     \n",
      "                                                                 \n",
      " gru_2 (GRU)                 (None, 24, 32)            9408      \n",
      "                                                                 \n",
      " gru_3 (GRU)                 (None, 24, 16)            2400      \n",
      "                                                                 \n",
      " lstm_3 (LSTM)               (None, 8)                 800       \n",
      "                                                                 \n",
      " dense_6 (Dense)             (None, 1)                 9         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1342025 (5.12 MB)\n",
      "Trainable params: 1342025 (5.12 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_6.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "7f96925d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:13:14.282999Z",
     "iopub.status.busy": "2023-12-26T07:13:14.282551Z",
     "iopub.status.idle": "2023-12-26T07:15:02.711085Z",
     "shell.execute_reply": "2023-12-26T07:15:02.708956Z"
    },
    "papermill": {
     "duration": 108.598447,
     "end_time": "2023-12-26T07:15:02.713590",
     "exception": false,
     "start_time": "2023-12-26T07:13:14.115143",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "215/215 [==============================] - 30s 79ms/step - loss: 0.2317 - accuracy: 0.9372 - val_loss: 0.6676 - val_accuracy: 0.7533\n",
      "Epoch 2/100\n",
      "215/215 [==============================] - 12s 57ms/step - loss: 0.1342 - accuracy: 0.9588 - val_loss: 0.7385 - val_accuracy: 0.7520\n",
      "Epoch 3/100\n",
      "215/215 [==============================] - 12s 57ms/step - loss: 0.1122 - accuracy: 0.9664 - val_loss: 0.8285 - val_accuracy: 0.7454\n",
      "Epoch 4/100\n",
      "215/215 [==============================] - 12s 57ms/step - loss: 0.0893 - accuracy: 0.9721 - val_loss: 0.9407 - val_accuracy: 0.7349\n",
      "Epoch 5/100\n",
      "215/215 [==============================] - 12s 56ms/step - loss: 0.0751 - accuracy: 0.9756 - val_loss: 0.9715 - val_accuracy: 0.7336\n",
      "Epoch 6/100\n",
      "215/215 [==============================] - 12s 58ms/step - loss: 0.0639 - accuracy: 0.9759 - val_loss: 1.0080 - val_accuracy: 0.7415\n"
     ]
    }
   ],
   "source": [
    "model_6.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])\n",
    "model_6_history = model_6.fit(train_dataset,\n",
    "                             epochs=100,\n",
    "                             validation_data=(valid_dataset),\n",
    "                             callbacks=[early_stopping_callback])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c207488f",
   "metadata": {
    "papermill": {
     "duration": 0.232615,
     "end_time": "2023-12-26T07:15:03.179604",
     "exception": false,
     "start_time": "2023-12-26T07:15:02.946989",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model 7: Bidirectional\n",
    "\n",
    "* a normal RNN will process a sequence from left to right, where a bidirectional RNN will process the sequence from left to right and then right to left. \n",
    "* Many models see improvement in performance when using bidirectional RNNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "9e42c3cf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:15:03.650011Z",
     "iopub.status.busy": "2023-12-26T07:15:03.649105Z",
     "iopub.status.idle": "2023-12-26T07:15:05.565668Z",
     "shell.execute_reply": "2023-12-26T07:15:05.564004Z"
    },
    "papermill": {
     "duration": 2.15982,
     "end_time": "2023-12-26T07:15:05.568860",
     "exception": false,
     "start_time": "2023-12-26T07:15:03.409040",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tf.random.set_seed(42)\n",
    "from tensorflow.keras import layers\n",
    "# model_4_embedding = layers.Embedding(input_dim=max_vocab_length,\n",
    "#                                      output_dim=128,\n",
    "#                                      embeddings_initializer=\"uniform\",\n",
    "#                                      input_length=max_length,\n",
    "#                                      name=\"embedding_4\")\n",
    "\n",
    "# Build a Bidirectional RNN in TensorFlow\n",
    "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
    "x = text_vectorizer(inputs)\n",
    "x = token_embedding(x)\n",
    "# x = layers.Bidirectional(layers.LSTM(64, return_sequences=True))(x) # stacking RNN layers requires return_sequences=True\n",
    "x = layers.Bidirectional(layers.LSTM(64))(x) # bidirectional goes both ways so has double the parameters of a regular LSTM layer\n",
    "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "model_7 = tf.keras.Model(inputs, outputs, name=\"model_7_Bidirectional\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "46069704",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:15:06.128333Z",
     "iopub.status.busy": "2023-12-26T07:15:06.127931Z",
     "iopub.status.idle": "2023-12-26T07:15:06.164680Z",
     "shell.execute_reply": "2023-12-26T07:15:06.161996Z"
    },
    "papermill": {
     "duration": 0.282583,
     "end_time": "2023-12-26T07:15:06.169662",
     "exception": false,
     "start_time": "2023-12-26T07:15:05.887079",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_7_Bidirectional\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_6 (InputLayer)        [(None, 1)]               0         \n",
      "                                                                 \n",
      " text_vectorization (TextVe  (None, 24)                0         \n",
      " ctorization)                                                    \n",
      "                                                                 \n",
      " token_embed (Embedding)     (None, 24, 128)           1280000   \n",
      "                                                                 \n",
      " bidirectional (Bidirection  (None, 128)               98816     \n",
      " al)                                                             \n",
      "                                                                 \n",
      " dense_7 (Dense)             (None, 1)                 129       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1378945 (5.26 MB)\n",
      "Trainable params: 1378945 (5.26 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_7.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "62a1daa6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:15:06.658704Z",
     "iopub.status.busy": "2023-12-26T07:15:06.658372Z",
     "iopub.status.idle": "2023-12-26T07:15:59.524096Z",
     "shell.execute_reply": "2023-12-26T07:15:59.521771Z"
    },
    "papermill": {
     "duration": 53.110281,
     "end_time": "2023-12-26T07:15:59.527480",
     "exception": false,
     "start_time": "2023-12-26T07:15:06.417199",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "215/215 [==============================] - 15s 38ms/step - loss: 0.1496 - accuracy: 0.9549 - val_loss: 0.7673 - val_accuracy: 0.7546\n",
      "Epoch 2/100\n",
      "215/215 [==============================] - 6s 28ms/step - loss: 0.1101 - accuracy: 0.9632 - val_loss: 0.8662 - val_accuracy: 0.7507\n",
      "Epoch 3/100\n",
      "215/215 [==============================] - 6s 28ms/step - loss: 0.0915 - accuracy: 0.9680 - val_loss: 0.9441 - val_accuracy: 0.7454\n",
      "Epoch 4/100\n",
      "215/215 [==============================] - 6s 28ms/step - loss: 0.0785 - accuracy: 0.9705 - val_loss: 1.0356 - val_accuracy: 0.7323\n",
      "Epoch 5/100\n",
      "215/215 [==============================] - 6s 27ms/step - loss: 0.0679 - accuracy: 0.9739 - val_loss: 1.0582 - val_accuracy: 0.7087\n",
      "Epoch 6/100\n",
      "215/215 [==============================] - 6s 28ms/step - loss: 0.0582 - accuracy: 0.9761 - val_loss: 1.6568 - val_accuracy: 0.6995\n"
     ]
    }
   ],
   "source": [
    "model_7.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])\n",
    "model_7_history = model_7.fit(train_dataset,\n",
    "                             epochs=100,\n",
    "                             validation_data=(valid_dataset),\n",
    "                             callbacks=[early_stopping_callback])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00a60dfa",
   "metadata": {
    "papermill": {
     "duration": 0.270605,
     "end_time": "2023-12-26T07:16:00.074366",
     "exception": false,
     "start_time": "2023-12-26T07:15:59.803761",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model 8: Conv1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "a70d1e39",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:00.687326Z",
     "iopub.status.busy": "2023-12-26T07:16:00.686375Z",
     "iopub.status.idle": "2023-12-26T07:16:03.887161Z",
     "shell.execute_reply": "2023-12-26T07:16:03.886005Z"
    },
    "papermill": {
     "duration": 3.542513,
     "end_time": "2023-12-26T07:16:03.889491",
     "exception": false,
     "start_time": "2023-12-26T07:16:00.346978",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inputs = layers.Input(shape=(1,), dtype=\"string\")\n",
    "x = text_vectorizer(inputs)\n",
    "x = token_embedding(x)\n",
    "x = layers.Conv1D(filters=32, kernel_size=5, activation=\"relu\")(x)\n",
    "x = layers.GlobalMaxPool1D()(x)\n",
    "# x = layers.Dense(64, activation=\"relu\")(x) # optional dense layer\n",
    "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "model_8 = tf.keras.Model(inputs, outputs, name=\"model_8_Conv1D\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "70d1d021",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:04.430158Z",
     "iopub.status.busy": "2023-12-26T07:16:04.429775Z",
     "iopub.status.idle": "2023-12-26T07:16:23.234568Z",
     "shell.execute_reply": "2023-12-26T07:16:23.233686Z"
    },
    "papermill": {
     "duration": 19.078152,
     "end_time": "2023-12-26T07:16:23.237185",
     "exception": false,
     "start_time": "2023-12-26T07:16:04.159033",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "215/215 [==============================] - 3s 13ms/step - loss: 0.1635 - accuracy: 0.9472 - val_loss: 0.8631 - val_accuracy: 0.7402\n",
      "Epoch 2/100\n",
      "215/215 [==============================] - 3s 12ms/step - loss: 0.1132 - accuracy: 0.9597 - val_loss: 0.9698 - val_accuracy: 0.7428\n",
      "Epoch 3/100\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.0885 - accuracy: 0.9645 - val_loss: 1.0629 - val_accuracy: 0.7415\n",
      "Epoch 4/100\n",
      "215/215 [==============================] - 2s 12ms/step - loss: 0.0724 - accuracy: 0.9708 - val_loss: 1.1489 - val_accuracy: 0.7415\n",
      "Epoch 5/100\n",
      "215/215 [==============================] - 3s 12ms/step - loss: 0.0628 - accuracy: 0.9730 - val_loss: 1.2154 - val_accuracy: 0.7336\n",
      "Epoch 6/100\n",
      "215/215 [==============================] - 3s 12ms/step - loss: 0.0565 - accuracy: 0.9749 - val_loss: 1.2521 - val_accuracy: 0.7402\n"
     ]
    }
   ],
   "source": [
    "model_8.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])\n",
    "model_8_history = model_8.fit(train_dataset,\n",
    "                             epochs=100,\n",
    "                             validation_data=(valid_dataset),\n",
    "                             callbacks=[early_stopping_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "82e77932",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:23.819040Z",
     "iopub.status.busy": "2023-12-26T07:16:23.817268Z",
     "iopub.status.idle": "2023-12-26T07:16:23.922491Z",
     "shell.execute_reply": "2023-12-26T07:16:23.920699Z"
    },
    "papermill": {
     "duration": 0.396774,
     "end_time": "2023-12-26T07:16:23.925065",
     "exception": false,
     "start_time": "2023-12-26T07:16:23.528291",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 4ms/step - loss: 0.8631 - accuracy: 0.7402\n"
     ]
    }
   ],
   "source": [
    " i = model_8.evaluate(valid_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "23b8ea38",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:24.578413Z",
     "iopub.status.busy": "2023-12-26T07:16:24.578090Z",
     "iopub.status.idle": "2023-12-26T07:16:24.585183Z",
     "shell.execute_reply": "2023-12-26T07:16:24.583892Z"
    },
    "papermill": {
     "duration": 0.37568,
     "end_time": "2023-12-26T07:16:24.587390",
     "exception": false,
     "start_time": "2023-12-26T07:16:24.211710",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def plot_models(models, val_data=valid_dataset):\n",
    "    accuracies = []\n",
    "    names = []\n",
    "    for m in models:\n",
    "        _, accuracy = m.evaluate(val_data)\n",
    "        accuracies.append(accuracy)\n",
    "        names.append(m.name)\n",
    "    \n",
    "    # Plotting the accuracies\n",
    "    plt.figure()\n",
    "    plt.bar(names, accuracies, color='skyblue')\n",
    "    plt.xlabel('Models')\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.title('Model Accuracies')\n",
    "    plt.xticks(rotation=45, fontsize=8)\n",
    "    plt.ylim([min(accuracies) - 0.05, max(accuracies) + 0.05])  # Adjust y-axis limits for better visualization\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "580cfabf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:25.166566Z",
     "iopub.status.busy": "2023-12-26T07:16:25.165176Z",
     "iopub.status.idle": "2023-12-26T07:16:29.727726Z",
     "shell.execute_reply": "2023-12-26T07:16:29.726158Z"
    },
    "papermill": {
     "duration": 4.854413,
     "end_time": "2023-12-26T07:16:29.730305",
     "exception": false,
     "start_time": "2023-12-26T07:16:24.875892",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 3ms/step - loss: 0.6052 - accuracy: 0.7651\n",
      "24/24 [==============================] - 0s 9ms/step - loss: 0.3999 - accuracy: 0.8255\n",
      "24/24 [==============================] - 0s 8ms/step - loss: 0.6789 - accuracy: 0.7651\n",
      "24/24 [==============================] - 0s 7ms/step - loss: 0.7567 - accuracy: 0.7612\n",
      "24/24 [==============================] - 1s 10ms/step - loss: 0.8114 - accuracy: 0.7612\n",
      "24/24 [==============================] - 1s 13ms/step - loss: 0.6487 - accuracy: 0.7664\n",
      "24/24 [==============================] - 0s 9ms/step - loss: 0.7423 - accuracy: 0.7625\n",
      "24/24 [==============================] - 0s 4ms/step - loss: 0.8631 - accuracy: 0.7402\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkgAAAIbCAYAAAAQBGH2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8WgzjOAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB9t0lEQVR4nO3deXhMZ/8/8PckZKFCbYloCGqvhIQERS0httQSxBbEvlUlrVqKoLaHFu1je9oG1SKqtccW0VBbQmy1JLUH2UMWIfvn94ffnK+ZJCqamEnyfl3XXK1z7jnzuWeSOe+c+z7nqEREQEREREQKA10XQERERKRvGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCI6K1TqVSYN29evp937949qFQqbNq0qcBrKo4CAwOhUqkQGBio61KIihwGJKISatOmTVCpVFCpVDh58mSO9SICKysrqFQq9OzZUwcVFowDBw5ApVLB0tIS2dnZui6HiIoIBiSiEs7ExARbt27Nsfz48eN4+PAhjI2NdVBVwdmyZQusra0RGRmJY8eO6bqct6pdu3Z4/vw52rVrp+tSiIocBiSiEq579+7YsWMHMjMzNZZv3boV9vb2sLCw0FFl/15KSgr27NkDLy8vNGvWDFu2bNF1SXlKSUkp8G0aGBjAxMQEBgb8qifKL/7WEJVwgwYNQnx8PPz9/ZVl6enp+O233zB48OBcn5OSkoLPPvsMVlZWMDY2Rv369fH1119DRDTapaWlwdPTE1WqVEG5cuXw8ccf4+HDh7lu89GjRxg5ciTMzc1hbGyMxo0bY8OGDf+qb7t27cLz58/Rv39/DBw4EDt37kRqamqOdqmpqZg3bx7q1asHExMTVKtWDX379sXt27eVNtnZ2fj222/RpEkTmJiYoEqVKujatSvOnz8P4NXzo7TnXM2bNw8qlQrXr1/H4MGD8e6776JNmzYAgCtXrmDEiBGoXbs2TExMYGFhgZEjRyI+Pj7X92zUqFGwtLSEsbExatWqhQkTJiA9PR1A3nOQgoKC0LVrV5QvXx5lypTBRx99hFOnTmm0SU5OxtSpU2FtbQ1jY2NUrVoVnTt3xoULF17rvScq6krpugAi0i1ra2u0atUK27ZtQ7du3QAABw8eRGJiIgYOHIjvvvtOo72I4OOPP8Yff/yBUaNGoWnTpjh8+DCmTZuGR48eYeXKlUrb0aNH45dffsHgwYPRunVrHDt2DD169MhRQ3R0NFq2bAmVSoXJkyejSpUqOHjwIEaNGoWkpCRMnTr1jfq2ZcsWdOjQARYWFhg4cCBmzJiBffv2oX///kqbrKws9OzZEwEBARg4cCA+/fRTJCcnw9/fH1evXkWdOnUAAKNGjcKmTZvQrVs3jB49GpmZmfjzzz9x9uxZNG/e/I3q69+/P+rWrYvFixcr4dLf3x937tyBh4cHLCwscO3aNXz//fe4du0azp49C5VKBQCIiIiAg4MDEhISMHbsWDRo0ACPHj3Cb7/9hmfPnsHIyCjX1zx27Bi6desGe3t7eHt7w8DAABs3bkTHjh3x559/wsHBAQAwfvx4/Pbbb5g8eTIaNWqE+Ph4nDx5Ejdu3ICdnd0b9ZeoSBEiKpE2btwoAOTcuXOyevVqKVeunDx79kxERPr37y8dOnQQEZGaNWtKjx49lOft3r1bAMjChQs1ttevXz9RqVRy69YtERG5dOmSAJCJEydqtBs8eLAAEG9vb2XZqFGjpFq1ahIXF6fRduDAgVK+fHmlrrt37woA2bhx4z/2Lzo6WkqVKiU//PCDsqx169bSq1cvjXYbNmwQALJixYoc28jOzhYRkWPHjgkAmTJlSp5tXlWbdn+9vb0FgAwaNChHW3VfX7Zt2zYBICdOnFCWDRs2TAwMDOTcuXN51vTHH38IAPnjjz+U5XXr1hVnZ2eljfo1a9WqJZ07d1aWlS9fXiZNmpRj20QlBYfYiAgDBgzA8+fPsX//fiQnJ2P//v15Dq8dOHAAhoaGmDJlisbyzz77DCKCgwcPKu0A5GinfTRIRPD777/DxcUFIoK4uDjl4ezsjMTExDca1vH19YWBgQFcXV2VZYMGDcLBgwfx5MkTZdnvv/+OypUr45NPPsmxDfXRmt9//x0qlQre3t55tnkT48ePz7HM1NRU+f/U1FTExcWhZcuWAKC8D9nZ2di9ezdcXFxyPXqVV02XLl3CzZs3MXjwYMTHxyvvc0pKCjp16oQTJ04oZ/pVqFABQUFBiIiIeOP+ERVlHGIjIlSpUgVOTk7YunUrnj17hqysLPTr1y/Xtvfv34elpSXKlSunsbxhw4bKevV/DQwMlCEqtfr162v8OzY2FgkJCfj+++/x/fff5/qaMTEx+e7TL7/8AgcHB8THxyvzd5o1a4b09HTs2LEDY8eOBQDcvn0b9evXR6lSeX8d3r59G5aWlqhYsWK+63iVWrVq5Vj2+PFjzJ8/H76+vjn6nZiYCODFe5aUlIQPPvggX6938+ZNAMDw4cPzbJOYmIh3330Xy5Ytw/Dhw2FlZQV7e3t0794dw4YNQ+3atfP1mkRFFQMSEQEABg8ejDFjxiAqKgrdunVDhQoV3srrqo9YDB06NM8dt42NTb62efPmTZw7dw4AULdu3Rzrt2zZogSkgpLXUZusrKw8n/Py0SK1AQMG4PTp05g2bRqaNm2Kd955B9nZ2ejateu/vo6T+vnLly9H06ZNc23zzjvvKHW0bdsWu3btwpEjR7B8+XL85z//wc6dO5W5akTFGQMSEQEA+vTpg3HjxuHs2bPYvn17nu1q1qyJo0ePIjk5WeMoUmhoqLJe/d/s7GzlCI1aWFiYxvbUZ7hlZWXBycmpQPqyZcsWlC5dGj///DMMDQ011p08eRLfffcdwsPDUaNGDdSpUwdBQUHIyMhA6dKlc91enTp1cPjwYTx+/DjPo0jvvvsuACAhIUFjufqI2ut48uQJAgICMH/+fMydO1dZrj7yo1alShWYmZnh6tWrr71tAMrRPDMzs9d6r6tVq4aJEydi4sSJiImJgZ2dHRYtWsSARCUC5yAREYAXRw7WrVuHefPmwcXFJc923bt3R1ZWFlavXq2xfOXKlVCpVMrOU/1f7bPgVq1apfFvQ0NDuLq64vfff891hx8bG5vvvmzZsgVt27aFm5sb+vXrp/GYNm0aAGDbtm0AAFdXV8TFxeXoDwDlzDJXV1eICObPn59nGzMzM1SuXBknTpzQWL927drXrlsd5kTrcgna75mBgQF69+6Nffv2KZcZyK0mbfb29qhTpw6+/vprPH36NMd69XudlZWlDOepVa1aFZaWlkhLS3vt/hAVZTyCRESKV81NUXNxcUGHDh3w5Zdf4t69e7C1tcWRI0ewZ88eTJ06VTlK0bRpUwwaNAhr165FYmIiWrdujYCAANy6dSvHNpcuXYo//vgDjo6OGDNmDBo1aoTHjx/jwoULOHr0KB4/fvzafQgKCsKtW7cwefLkXNdXr14ddnZ22LJlC6ZPn45hw4Zh8+bN8PLyQnBwMNq2bYuUlBQcPXoUEydORK9evdChQwe4u7vju+++w82bN5Xhrj///BMdOnRQXmv06NFYunQpRo8ejebNm+PEiRP4+++/X7t2MzMztGvXDsuWLUNGRgaqV6+OI0eO4O7duznaLl68GEeOHMFHH32EsWPHomHDhoiMjMSOHTtw8uTJXIdIDQwM8OOPP6Jbt25o3LgxPDw8UL16dTx69Ah//PEHzMzMsG/fPiQnJ+O9995Dv379YGtri3feeQdHjx7FuXPn8M0337x2f4iKNN2dQEdEuvTyaf6von2av4hIcnKyeHp6iqWlpZQuXVrq1q0ry5cv1zh1XETk+fPnMmXKFKlUqZKULVtWXFxc5MGDBzlOexd5cVr+pEmTxMrKSkqXLi0WFhbSqVMn+f7775U2r3Oa/yeffCIA5Pbt23m2mTdvngCQy5cvi8iL09y//PJLqVWrlvLa/fr109hGZmamLF++XBo0aCBGRkZSpUoV6datm4SEhChtnj17JqNGjZLy5ctLuXLlZMCAARITE5Pnaf6xsbE5anv48KH06dNHKlSoIOXLl5f+/ftLREREru/Z/fv3ZdiwYVKlShUxNjaW2rVry6RJkyQtLU1Ecp7mr3bx4kXp27evVKpUSYyNjaVmzZoyYMAACQgIEBGRtLQ0mTZtmtja2kq5cuWkbNmyYmtrK2vXrs3zPSUqblQieRyLJSIiIiqhOAeJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFp0HpDWrFkDa2trmJiYwNHREcHBwa9sv2rVKtSvXx+mpqawsrKCp6cnUlNTlfXW1tZQqVQ5HpMmTVLatG/fPsf63O6JRERERCWTTq+DtH37dnh5eWH9+vVwdHTEqlWr4OzsjLCwMFStWjVH+61bt2LGjBnYsGEDWrdujb///hsjRoyASqXCihUrAADnzp3TuLT/1atX0blzZ/Tv319jW2PGjMGCBQuUf5cpU6aQeklERERFjU4D0ooVKzBmzBh4eHgAANavXw8/Pz9s2LABM2bMyNH+9OnT+PDDD5W7jFtbW2PQoEEICgpS2lSpUkXjOUuXLkWdOnXw0UcfaSwvU6YMLCws3rj27OxsREREoFy5cv/qbt5ERET09ogIkpOTYWlpCQODVwyk6eoCTGlpaWJoaCi7du3SWD5s2DD5+OOPc33Oli1bpHz58hIUFCQiIrdv35YGDRrIokWL8nyNSpUq5Vj/0UcfSeXKlaVSpUrSuHFjmTFjhqSkpLyy3tTUVElMTFQe169fFwB88MEHH3zwwUcRfDx48OCV+32dHUGKi4tDVlYWzM3NNZabm5srN73UNnjwYMTFxaFNmzYQEWRmZmL8+PGYNWtWru13796NhIQEjBgxIsd2atasCUtLS1y5cgXTp09HWFgYdu7cmWe9S5YsyfU+TA8ePICZmdk/9JaIiIj0QVJSEqysrDRutp2bInUvtsDAQCxevBhr166Fo6Mjbt26hU8//RRfffUV5syZk6O9j48PunXrBktLS43lY8eOVf6/SZMmqFatGjp16oTbt28r95HSNnPmTHh5eSn/Vr/BZmZmDEhERERFzD9Nj9FZQKpcuTIMDQ0RHR2tsTw6OjrPuUFz5syBu7s7Ro8eDeBFuElJScHYsWPx5Zdfaowl3r9/H0ePHn3lUSE1R0dHAMCtW7fyDEjGxsYwNjZ+rb4RERFR0aaz0/yNjIxgb2+PgIAAZVl2djYCAgLQqlWrXJ/z7NmzHBOqDA0NAQCidUu5jRs3omrVqujRo8c/1nLp0iUAQLVq1fLTBSIiIiqmdDrE5uXlheHDh6N58+ZwcHDAqlWrkJKSopzVNmzYMFSvXh1LliwBALi4uGDFihVo1qyZMsQ2Z84cuLi4KEEJeBG0Nm7ciOHDh6NUKc0u3r59G1u3bkX37t1RqVIlXLlyBZ6enmjXrh1sbGzeXueJiIhIb+k0ILm5uSE2NhZz585FVFQUmjZtikOHDikTt8PDwzWOGM2ePRsqlQqzZ8/Go0ePUKVKFbi4uGDRokUa2z169CjCw8MxcuTIHK9pZGSEo0ePKmHMysoKrq6umD17duF2loiIiIoMlWiPTdFrSUpKQvny5ZGYmMhJ2kREREXE6+6/dX6rESIiIiJ9w4BEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItOg8IK1ZswbW1tYwMTGBo6MjgoODX9l+1apVqF+/PkxNTWFlZQVPT0+kpqYq6+fNmweVSqXxaNCggcY2UlNTMWnSJFSqVAnvvPMOXF1dER0dXSj9IyIioqJHpwFp+/bt8PLygre3Ny5cuABbW1s4OzsjJiYm1/Zbt27FjBkz4O3tjRs3bsDHxwfbt2/HrFmzNNo1btwYkZGRyuPkyZMa6z09PbFv3z7s2LEDx48fR0REBPr27Vto/SQiIqKipZQuX3zFihUYM2YMPDw8AADr16+Hn58fNmzYgBkzZuRof/r0aXz44YcYPHgwAMDa2hqDBg1CUFCQRrtSpUrBwsIi19dMTEyEj48Ptm7dio4dOwIANm7ciIYNG+Ls2bNo2bJlrs9LS0tDWlqa8u+kpKT8d5iIiIiKBJ0dQUpPT0dISAicnJz+rxgDAzg5OeHMmTO5Pqd169YICQlRhuHu3LmDAwcOoHv37hrtbt68CUtLS9SuXRtDhgxBeHi4si4kJAQZGRkar9ugQQPUqFEjz9cFgCVLlqB8+fLKw8rK6o36TURERPpPZwEpLi4OWVlZMDc311hubm6OqKioXJ8zePBgLFiwAG3atEHp0qVRp04dtG/fXmOIzdHREZs2bcKhQ4ewbt063L17F23btkVycjIAICoqCkZGRqhQocJrvy4AzJw5E4mJicrjwYMHb9hzIiIi0nc6n6SdH4GBgVi8eDHWrl2LCxcuYOfOnfDz88NXX32ltOnWrRv69+8PGxsbODs748CBA0hISMCvv/76r17b2NgYZmZmGg8iIiIqnnQ2B6ly5cowNDTMcfZYdHR0nvOH5syZA3d3d4wePRoA0KRJE6SkpGDs2LH48ssvYWCQM+9VqFAB9erVw61btwAAFhYWSE9PR0JCgsZRpFe9LhEREZUsOjuCZGRkBHt7ewQEBCjLsrOzERAQgFatWuX6nGfPnuUIQYaGhgAAEcn1OU+fPsXt27dRrVo1AIC9vT1Kly6t8bphYWEIDw/P83WJiIioZNHpWWxeXl4YPnw4mjdvDgcHB6xatQopKSnKWW3Dhg1D9erVsWTJEgCAi4sLVqxYgWbNmsHR0RG3bt3CnDlz4OLiogSlzz//HC4uLqhZsyYiIiLg7e0NQ0NDDBo0CABQvnx5jBo1Cl5eXqhYsSLMzMzwySefoFWrVnmewUZEREQli04DkpubG2JjYzF37lxERUWhadOmOHTokDJxOzw8XOOI0ezZs6FSqTB79mw8evQIVapUgYuLCxYtWqS0efjwIQYNGoT4+HhUqVIFbdq0wdmzZ1GlShWlzcqVK2FgYABXV1ekpaXB2dkZa9eufXsdJyIiIr2mkrzGpuiVkpKSUL58eSQmJnLCNhERURHxuvvvInUWGxEREdHbwIBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERaGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRERGRFgYkIiIiIi0MSERERERadB6Q1qxZA2tra5iYmMDR0RHBwcGvbL9q1SrUr18fpqamsLKygqenJ1JTU5X1S5YsQYsWLVCuXDlUrVoVvXv3RlhYmMY22rdvD5VKpfEYP358ofSPiIiIih6dBqTt27fDy8sL3t7euHDhAmxtbeHs7IyYmJhc22/duhUzZsyAt7c3bty4AR8fH2zfvh2zZs1S2hw/fhyTJk3C2bNn4e/vj4yMDHTp0gUpKSka2xozZgwiIyOVx7Jlywq1r0RERFR0lNLli69YsQJjxoyBh4cHAGD9+vXw8/PDhg0bMGPGjBztT58+jQ8//BCDBw8GAFhbW2PQoEEICgpS2hw6dEjjOZs2bULVqlUREhKCdu3aKcvLlCkDCwuLwugWERERFXE6C0jp6ekICQnBzJkzlWUGBgZwcnLCmTNncn1O69at8csvvyA4OBgODg64c+cODhw4AHd39zxfJzExEQBQsWJFjeVbtmzBL7/8AgsLC7i4uGDOnDkoU6ZMnttJS0tDWlqa8u+kpKTX6iflbenFOF2X8EZmNKus6xKIiKiQ6SwgxcXFISsrC+bm5hrLzc3NERoamutzBg8ejLi4OLRp0wYigszMTIwfP15jiO1l2dnZmDp1Kj788EN88MEHGtupWbMmLC0tceXKFUyfPh1hYWHYuXNnnvUuWbIE8+fPf4OeEhERUVGj0yG2/AoMDMTixYuxdu1aODo64tatW/j000/x1VdfYc6cOTnaT5o0CVevXsXJkyc1lo8dO1b5/yZNmqBatWro1KkTbt++jTp16uT62jNnzoSXl5fy76SkJFhZWRVQz4iIiEif6CwgVa5cGYaGhoiOjtZYHh0dnefcoDlz5sDd3R2jR48G8CLcpKSkYOzYsfjyyy9hYPB/c84nT56M/fv348SJE3jvvfdeWYujoyMA4NatW3kGJGNjYxgbG792/4iIiKjo0tlZbEZGRrC3t0dAQICyLDs7GwEBAWjVqlWuz3n27JlGCAIAQ0NDAICIKP+dPHkydu3ahWPHjqFWrVr/WMulS5cAANWqVXuTrhAREVExo9MhNi8vLwwfPhzNmzeHg4MDVq1ahZSUFOWstmHDhqF69epYsmQJAMDFxQUrVqxAs2bNlCG2OXPmwMXFRQlKkyZNwtatW7Fnzx6UK1cOUVFRAIDy5cvD1NQUt2/fxtatW9G9e3dUqlQJV65cgaenJ9q1awcbGxvdvBFERESkV3QakNzc3BAbG4u5c+ciKioKTZs2xaFDh5SJ2+Hh4RpHjGbPng2VSoXZs2fj0aNHqFKlClxcXLBo0SKlzbp16wC8uBjkyzZu3IgRI0bAyMgIR48eVcKYlZUVXF1dMXv27MLvMBERERUJKlGPTVG+JCUloXz58khMTISZmZmuyymSeJo/ERG9ba+7/9b5rUaIiIiI9A0DEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItKi84C0Zs0aWFtbw8TEBI6OjggODn5l+1WrVqF+/fowNTWFlZUVPD09kZqamq9tpqamYtKkSahUqRLeeecduLq6Ijo6usD7RkREREWTTgPS9u3b4eXlBW9vb1y4cAG2trZwdnZGTExMru23bt2KGTNmwNvbGzdu3ICPjw+2b9+OWbNm5Wubnp6e2LdvH3bs2IHjx48jIiICffv2LfT+EhERUdGgEhHR1Ys7OjqiRYsWWL16NQAgOzsbVlZW+OSTTzBjxowc7SdPnowbN24gICBAWfbZZ58hKCgIJ0+efK1tJiYmokqVKti6dSv69esHAAgNDUXDhg1x5swZtGzZ8rVqT0pKQvny5ZGYmAgzM7N/9T6UVEsvxum6hDcyo1llXZdARERv6HX33zo7gpSeno6QkBA4OTn9XzEGBnBycsKZM2dyfU7r1q0REhKiDJnduXMHBw4cQPfu3V97myEhIcjIyNBo06BBA9SoUSPP1wWAtLQ0JCUlaTyIiIioeCqlqxeOi4tDVlYWzM3NNZabm5sjNDQ01+cMHjwYcXFxaNOmDUQEmZmZGD9+vDLE9jrbjIqKgpGRESpUqJCjTVRUVJ71LlmyBPPnz89vN4mIiKgI0vkk7fwIDAzE4sWLsXbtWly4cAE7d+6En58fvvrqq0J/7ZkzZyIxMVF5PHjwoNBfk4iIiHRDZ0eQKleuDENDwxxnj0VHR8PCwiLX58yZMwfu7u4YPXo0AKBJkyZISUnB2LFj8eWXX77WNi0sLJCeno6EhASNo0ivel0AMDY2hrGx8Zt0lYiIiIoYnR1BMjIygr29vcaE6+zsbAQEBKBVq1a5PufZs2cwMNAs2dDQEAAgIq+1TXt7e5QuXVqjTVhYGMLDw/N8XSIiIipZdHYECQC8vLwwfPhwNG/eHA4ODli1ahVSUlLg4eEBABg2bBiqV6+OJUuWAABcXFywYsUKNGvWDI6Ojrh16xbmzJkDFxcXJSj90zbLly+PUaNGwcvLCxUrVoSZmRk++eQTtGrV6rXPYCMiIqLiTacByc3NDbGxsZg7dy6ioqLQtGlTHDp0SJlkHR4ernHEaPbs2VCpVJg9ezYePXqEKlWqwMXFBYsWLXrtbQLAypUrYWBgAFdXV6SlpcHZ2Rlr1659ex0nIiIivZbv6yBZW1tj5MiRGDFiBGrUqFFYdek9Xgfp3+N1kIiI6G0rtOsgTZ06FTt37kTt2rXRuXNn+Pr6Ii0t7V8VS0RERKRP3iggXbp0CcHBwWjYsCE++eQTVKtWDZMnT8aFCxcKo0YiIiKit+qNz2Kzs7PDd999h4iICHh7e+PHH39EixYt0LRpU2zYsAE6vIMJERER0b/yxpO0MzIysGvXLmzcuBH+/v5o2bIlRo0ahYcPH2LWrFk4evQotm7dWpC1EhEREb0V+Q5IFy5cwMaNG7Ft2zYYGBhg2LBhWLlyJRo0aKC06dOnD1q0aFGghRIRERG9LfkOSC1atEDnzp2xbt069O7dG6VLl87RplatWhg4cGCBFEhERET0tuU7IN25cwc1a9Z8ZZuyZcti48aNb1wUERERkS7le5J2TEwMgoKCciwPCgrC+fPnC6QoIiIiIl3Kd0CaNGlSrneyf/ToESZNmlQgRRERERHpUr4D0vXr12FnZ5djebNmzXD9+vUCKYqIiIhIl/IdkIyNjREdHZ1jeWRkJEqV0umt3YiIiIgKRL4DUpcuXTBz5kwkJiYqyxISEjBr1ix07ty5QIsjIiIi0oV8H/L5+uuv0a5dO9SsWRPNmjUDAFy6dAnm5ub4+eefC7xAIiIiorct3wGpevXquHLlCrZs2YLLly/D1NQUHh4eGDRoUK7XRCIiIiIqat5o0lDZsmUxduzYgq6FiIiISC+88azq69evIzw8HOnp6RrLP/74439dFBEREZEuvdGVtPv06YO//voLKpUKIgIAUKlUAICsrKyCrZCIiIjoLcv3WWyffvopatWqhZiYGJQpUwbXrl3DiRMn0Lx5cwQGBhZCiURERERvV76PIJ05cwbHjh1D5cqVYWBgAAMDA7Rp0wZLlizBlClTcPHixcKok4iIiOityfcRpKysLJQrVw4AULlyZURERAAAatasibCwsIKtjoiIiEgH8n0E6YMPPsDly5dRq1YtODo6YtmyZTAyMsL333+P2rVrF0aNRERERG9VvgPS7NmzkZKSAgBYsGABevbsibZt26JSpUrYvn17gRdIRERE9LblOyA5Ozsr///+++8jNDQUjx8/xrvvvqucyUZERERUlOVrDlJGRgZKlSqFq1evaiyvWLEiwxEREREVG/kKSKVLl0aNGjV4rSMiIiIq1vJ9FtuXX36JWbNm4fHjx4VRDxEREZHO5XsO0urVq3Hr1i1YWlqiZs2aKFu2rMb6CxcuFFhxRERERLqQ74DUu3fvQiiDiIiISH/kOyB5e3sXRh1EREREeiPfc5CIiIiIirt8H0EyMDB45Sn9PMONiIiIirp8B6Rdu3Zp/DsjIwMXL17ETz/9hPnz5xdYYURERES6ku+A1KtXrxzL+vXrh8aNG2P79u0YNWpUgRRGREREpCsFNgepZcuWCAgIKKjNEREREelMgQSk58+f47vvvkP16tULYnNEREREOpXvITbtm9KKCJKTk1GmTBn88ssvBVocERERkS7kOyCtXLlSIyAZGBigSpUqcHR0xLvvvlugxRERERHpQr4D0ogRIwqhDCIiIiL9ke85SBs3bsSOHTtyLN+xYwd++umnAimKiIiISJfyHZCWLFmCypUr51hetWpVLF68+I2KWLNmDaytrWFiYgJHR0cEBwfn2bZ9+/ZQqVQ5Hj169FDa5LZepVJh+fLlShtra+sc65cuXfpG9RMREVHxku8htvDwcNSqVSvH8po1ayI8PDzfBWzfvh1eXl5Yv349HB0dsWrVKjg7OyMsLAxVq1bN0X7nzp1IT09X/h0fHw9bW1v0799fWRYZGanxnIMHD2LUqFFwdXXVWL5gwQKMGTNG+Xe5cuXyXT8REREVP/kOSFWrVsWVK1dgbW2tsfzy5cuoVKlSvgtYsWIFxowZAw8PDwDA+vXr4efnhw0bNmDGjBk52lesWFHj376+vihTpoxGQLKwsNBos2fPHnTo0AG1a9fWWF6uXLkcbYmIiIjyPcQ2aNAgTJkyBX/88QeysrKQlZWFY8eO4dNPP8XAgQPzta309HSEhITAycnp/woyMICTkxPOnDnzWtvw8fHBwIEDUbZs2VzXR0dHw8/PL9crfC9duhSVKlVCs2bNsHz5cmRmZub5OmlpaUhKStJ4EBERUfGU7yNIX331Fe7du4dOnTqhVKkXT8/OzsawYcPyPQcpLi4OWVlZMDc311hubm6O0NDQf3x+cHAwrl69Ch8fnzzb/PTTTyhXrhz69u2rsXzKlCmws7NDxYoVcfr0acycORORkZFYsWJFrttZsmQJ7zVHRERUQuQ7IBkZGWH79u1YuHAhLl26BFNTUzRp0gQ1a9YsjPpeycfHB02aNIGDg0OebTZs2IAhQ4bAxMREY7mXl5fy/zY2NjAyMsK4ceOwZMkSGBsb59jOzJkzNZ6TlJQEKyurAugFERER6Zt8ByS1unXrom7duv/qxStXrgxDQ0NER0drLI+Ojv7HuUEpKSnw9fXFggUL8mzz559/IiwsDNu3b//HWhwdHZGZmYl79+6hfv36OdYbGxvnGpyIiIio+Mn3HCRXV1f85z//ybF82bJlGhOlX4eRkRHs7e01bnKbnZ2NgIAAtGrV6pXP3bFjB9LS0jB06NA82/j4+MDe3h62trb/WMulS5dgYGCQ65lzREREVLLkOyCdOHEC3bt3z7G8W7duOHHiRL4L8PLywg8//ICffvoJN27cwIQJE5CSkqKc1TZs2DDMnDkzx/N8fHzQu3fvPM+cS0pKwo4dOzB69Ogc686cOYNVq1bh8uXLuHPnDrZs2QJPT08MHTqUt0shIiKi/A+xPX36FEZGRjmWly5d+o3O7HJzc0NsbCzmzp2LqKgoNG3aFIcOHVImboeHh8PAQDPHhYWF4eTJkzhy5Eie2/X19YWIYNCgQTnWGRsbw9fXF/PmzUNaWhpq1aoFT09PjTlGREREVHKpRETy8wQHBwf07NkTc+fO1Vg+b9487Nu3DyEhIQVaoL5KSkpC+fLlkZiYCDMzM12XUyQtvRin6xLeyIxmOa8kT0RERcPr7r/zfQRpzpw56Nu3L27fvo2OHTsCAAICArB161b89ttvb14xERERkZ7Id0BycXHB7t27sXjxYvz2228wNTWFra0tjh07luMq10RERERF0Rud5t+jRw/l5rBJSUnYtm0bPv/8c4SEhCArK6tACyQiIiJ62/J9FpvaiRMnMHz4cFhaWuKbb75Bx44dcfbs2YKsjYiIiEgn8nUEKSoqCps2bYKPjw+SkpIwYMAApKWlYffu3WjUqFFh1UhERET0Vr32ESQXFxfUr18fV65cwapVqxAREYH//ve/hVkbERERkU689hGkgwcPYsqUKZgwYcK/vsUIERERkT577SNIJ0+eRHJyMuzt7eHo6IjVq1cjLq5oXseGiIiI6FVeOyC1bNkSP/zwAyIjIzFu3Dj4+vrC0tIS2dnZ8Pf3R3JycmHWSURERPTW5PsstrJly2LkyJE4efIk/vrrL3z22WdYunQpqlatio8//rgwaiQiIiJ6q97oOkhq9evXx7Jly7BkyRLs27cPGzZsKKi6iIhIR4rqbYAA3gqICs4bXwfpZYaGhujduzf27t1bEJsjIiIi0qkCCUhERERExcm/GmKjwsHD28VLUf08+VkSUUnGI0hEREREWhiQiIiIiLQwIBERERFpYUAiIiIi0sJJ2kREVCLxBAp6FR5BIiIiItLCgERERESkhUNsRFQgSspwRUnpJ1FJxyNIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFr0IiCtWbMG1tbWMDExgaOjI4KDg/Ns2759e6hUqhyPHj16KG1GjBiRY33Xrl01tvP48WMMGTIEZmZmqFChAkaNGoWnT58WWh+JiIio6NB5QNq+fTu8vLzg7e2NCxcuwNbWFs7OzoiJicm1/c6dOxEZGak8rl69CkNDQ/Tv31+jXdeuXTXabdu2TWP9kCFDcO3aNfj7+2P//v04ceIExo4dW2j9JCIioqJD5wFpxYoVGDNmDDw8PNCoUSOsX78eZcqUwYYNG3JtX7FiRVhYWCgPf39/lClTJkdAMjY21mj37rvvKutu3LiBQ4cO4ccff4SjoyPatGmD//73v/D19UVERESh9peIiIj0n04DUnp6OkJCQuDk5KQsMzAwgJOTE86cOfNa2/Dx8cHAgQNRtmxZjeWBgYGoWrUq6tevjwkTJiA+Pl5Zd+bMGVSoUAHNmzdXljk5OcHAwABBQUG5vk5aWhqSkpI0HkRERFQ86TQgxcXFISsrC+bm5hrLzc3NERUV9Y/PDw4OxtWrVzF69GiN5V27dsXmzZsREBCA//znPzh+/Di6deuGrKwsAEBUVBSqVq2q8ZxSpUqhYsWKeb7ukiVLUL58eeVhZWWVn64SERFREVJK1wX8Gz4+PmjSpAkcHBw0lg8cOFD5/yZNmsDGxgZ16tRBYGAgOnXq9EavNXPmTHh5eSn/TkpKYkgiIiIqpnR6BKly5cowNDREdHS0xvLo6GhYWFi88rkpKSnw9fXFqFGj/vF1ateujcqVK+PWrVsAAAsLixyTwDMzM/H48eM8X9fY2BhmZmYaDyIiIiqedBqQjIyMYG9vj4CAAGVZdnY2AgIC0KpVq1c+d8eOHUhLS8PQoUP/8XUePnyI+Ph4VKtWDQDQqlUrJCQkICQkRGlz7NgxZGdnw9HR8Q17Q0RERMWFzs9i8/Lywg8//ICffvoJN27cwIQJE5CSkgIPDw8AwLBhwzBz5swcz/Px8UHv3r1RqVIljeVPnz7FtGnTcPbsWdy7dw8BAQHo1asX3n//fTg7OwMAGjZsiK5du2LMmDEIDg7GqVOnMHnyZAwcOBCWlpaF32kiIiLSazqfg+Tm5obY2FjMnTsXUVFRaNq0KQ4dOqRM3A4PD4eBgWaOCwsLw8mTJ3HkyJEc2zM0NMSVK1fw008/ISEhAZaWlujSpQu++uorGBsbK+22bNmCyZMno1OnTjAwMICrqyu+++67wu0sERERFQk6D0gAMHnyZEyePDnXdYGBgTmW1a9fHyKSa3tTU1McPnz4H1+zYsWK2Lp1a77qJCIiopJB50NsRERERPqGAYmIiIhICwMSERERkRYGJCIiIiItDEhEREREWhiQiIiIiLQwIBERERFpYUAiIiIi0sKARERERKSFAYmIiIhICwMSERERkRYGJCIiIiItDEhEREREWkrpugAiIiIqPEsvxum6hDcyo1llnb4+jyARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERadGLgLRmzRpYW1vDxMQEjo6OCA4OzrNt+/btoVKpcjx69OgBAMjIyMD06dPRpEkTlC1bFpaWlhg2bBgiIiI0tmNtbZ1jG0uXLi3UfhIREVHRoPOAtH37dnh5ecHb2xsXLlyAra0tnJ2dERMTk2v7nTt3IjIyUnlcvXoVhoaG6N+/PwDg2bNnuHDhAubMmYMLFy5g586dCAsLw8cff5xjWwsWLNDY1ieffFKofSUiIqKioZSuC1ixYgXGjBkDDw8PAMD69evh5+eHDRs2YMaMGTnaV6xYUePfvr6+KFOmjBKQypcvD39/f402q1evhoODA8LDw1GjRg1lebly5WBhYVHQXSIiIqIiTqdHkNLT0xESEgInJydlmYGBAZycnHDmzJnX2oaPjw8GDhyIsmXL5tkmMTERKpUKFSpU0Fi+dOlSVKpUCc2aNcPy5cuRmZmZ5zbS0tKQlJSk8SAiIqLiSadHkOLi4pCVlQVzc3ON5ebm5ggNDf3H5wcHB+Pq1avw8fHJs01qaiqmT5+OQYMGwczMTFk+ZcoU2NnZoWLFijh9+jRmzpyJyMhIrFixItftLFmyBPPnz3/NnhEREVFRpvMhtn/Dx8cHTZo0gYODQ67rMzIyMGDAAIgI1q1bp7HOy8tL+X8bGxsYGRlh3LhxWLJkCYyNjXNsa+bMmRrPSUpKgpWVVQH1hIiIiPSJTofYKleuDENDQ0RHR2ssj46O/se5QSkpKfD19cWoUaNyXa8OR/fv34e/v7/G0aPcODo6IjMzE/fu3ct1vbGxMczMzDQeREREVDzpNCAZGRnB3t4eAQEByrLs7GwEBASgVatWr3zujh07kJaWhqFDh+ZYpw5HN2/exNGjR1GpUqV/rOXSpUswMDBA1apV898RIiIiKlZ0PsTm5eWF4cOHo3nz5nBwcMCqVauQkpKinNU2bNgwVK9eHUuWLNF4no+PD3r37p0j/GRkZKBfv364cOEC9u/fj6ysLERFRQF4cQackZERzpw5g6CgIHTo0AHlypXDmTNn4OnpiaFDh+Ldd999Ox0nIiIivaXzgOTm5obY2FjMnTsXUVFRaNq0KQ4dOqRM3A4PD4eBgeaBrrCwMJw8eRJHjhzJsb1Hjx5h7969AICmTZtqrPvjjz/Qvn17GBsbw9fXF/PmzUNaWhpq1aoFT09PjTlGREREVHLpPCABwOTJkzF58uRc1wUGBuZYVr9+fYhIru2tra3zXKdmZ2eHs2fP5rtOIiIiKhl0fiVtIiIiIn3DgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi06EVAWrNmDaytrWFiYgJHR0cEBwfn2bZ9+/ZQqVQ5Hj169FDaiAjmzp2LatWqwdTUFE5OTrh586bGdh4/fowhQ4bAzMwMFSpUwKhRo/D06dNC6yMREREVHToPSNu3b4eXlxe8vb1x4cIF2NrawtnZGTExMbm237lzJyIjI5XH1atXYWhoiP79+yttli1bhu+++w7r169HUFAQypYtC2dnZ6SmpipthgwZgmvXrsHf3x/79+/HiRMnMHbs2ELvLxEREek/nQekFStWYMyYMfDw8ECjRo2wfv16lClTBhs2bMi1fcWKFWFhYaE8/P39UaZMGSUgiQhWrVqF2bNno1evXrCxscHmzZsRERGB3bt3AwBu3LiBQ4cO4ccff4SjoyPatGmD//73v/D19UVERMTb6joRERHpKZ0GpPT0dISEhMDJyUlZZmBgACcnJ5w5c+a1tuHj44OBAweibNmyAIC7d+8iKipKY5vly5eHo6Ojss0zZ86gQoUKaN68udLGyckJBgYGCAoKyvV10tLSkJSUpPEgIiKi4kmnASkuLg5ZWVkwNzfXWG5ubo6oqKh/fH5wcDCuXr2K0aNHK8vUz3vVNqOiolC1alWN9aVKlULFihXzfN0lS5agfPnyysPKyuqfO0hERERFks6H2P4NHx8fNGnSBA4ODoX+WjNnzkRiYqLyePDgQaG/JhEREemGTgNS5cqVYWhoiOjoaI3l0dHRsLCweOVzU1JS4Ovri1GjRmksVz/vVdu0sLDIMQk8MzMTjx8/zvN1jY2NYWZmpvEgIiKi4kmnAcnIyAj29vYICAhQlmVnZyMgIACtWrV65XN37NiBtLQ0DB06VGN5rVq1YGFhobHNpKQkBAUFKdts1aoVEhISEBISorQ5duwYsrOz4ejoWBBdIyIioiKslK4L8PLywvDhw9G8eXM4ODhg1apVSElJgYeHBwBg2LBhqF69OpYsWaLxPB8fH/Tu3RuVKlXSWK5SqTB16lQsXLgQdevWRa1atTBnzhxYWlqid+/eAICGDRuia9euGDNmDNavX4+MjAxMnjwZAwcOhKWl5VvpNxEREekvnQckNzc3xMbGYu7cuYiKikLTpk1x6NAhZZJ1eHg4DAw0D3SFhYXh5MmTOHLkSK7b/OKLL5CSkoKxY8ciISEBbdq0waFDh2BiYqK02bJlCyZPnoxOnTrBwMAArq6u+O677wqvo0RERFRk6DwgAcDkyZMxefLkXNcFBgbmWFa/fn2ISJ7bU6lUWLBgARYsWJBnm4oVK2Lr1q35rpWIiIiKvyJ9FhsRERFRYWBAIiIiItLCgERERESkhQGJiIiISAsDEhEREZEWBiQiIiIiLQxIRERERFoYkIiIiIi0MCARERERaWFAIiIiItLCgERERESkRS/uxVYUqe8Fl5SUVODbTn2aXODbfFuSkoxeu21R7Wd++giwn/qO/cypqPYRKBn95M/sv93ui/32q+7pCgAq+acWlKuHDx/CyspK12UQERHRG3jw4AHee++9PNczIL2h7OxsREREoFy5clCpVLou57UkJSXBysoKDx48gJmZma7LKTTsZ/HCfhYfJaGPAPup70QEycnJsLS0hIFB3jONOMT2hgwMDF6ZPPWZmZlZkfphflPsZ/HCfhYfJaGPAPupz8qXL/+PbThJm4iIiEgLAxIRERGRFgakEsTY2Bje3t4wNjbWdSmFiv0sXtjP4qMk9BFgP4sLTtImIiIi0sIjSERERERaGJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiSgfsrKydF0CEdG/EhcXp+sSigQGJHot2dnZui5B527duoXvvvsOz58/13Up9C9pX92EVzvJH34fFF3Pnz/HiBEjMHnyZF2XUmAK6/eXAYnydP/+fezduxfAi3vPleSdSFhYGPr16wdzc3OYmprqupxCp/6sMzIydFxJ4VCpVDh27BgmTZqk/Lsk/3znx+XLl/Hbb7/h6dOnui6F3oCRkRE+/fRTXL9+HTNmzNB1Ofl2584drFixAl5eXvjkk08QGxtbaDeMZ0CiXIWGhsLOzg7ffvstfvrpJwAldydy48YNeHh4YOrUqRg8eDCys7Nx5MgRpKWl6bq0QqNSqXDw4EF8+umn+OKLL4plUKpVqxZ+/PFHTJ06FUDJ/fnOr71792Lz5s04fPgwUlJSdF3Oa1F/riEhITh9+jSuXbum44p0x9DQEO3bt4e3tzdOnTpVpEJSaGgoXFxckJSUhNKlS+PWrVuws7PDsWPHABTCkSQh0pKamirTp0+XKVOmyNq1a8Xd3V02btyorM/KytJdcW9ZTEyMNGnSRMaPH68s69ChgyxdulSHVRW+y5cvi7W1tWzatEkaN24sAwYMkAcPHui6rAJ3584dqVy5skyePFlZlp2drcOK9I/6/UhLS1OWLVq0SAYMGCC//vqrPH/+XFelvRZ1/f7+/mJubi6DBw+Wxo0by969e3Vc2duV2891YGCgfPjhhzJ9+nQdVJQ/N27cEAcHB9myZYvG8ilTpsh7770njx49EpGC/f1lQKIcsrKy5NKlS/L8+XOJjY2VpUuXiru7u2zYsEFpo+9figXl7t27MmHCBPHy8pJDhw5J37595fPPP9do8/KOoyhTf7HEx8fLwYMHlR1IamqqdO7cWQYOHCjh4eG6LPFfu3r1qtjY2EhGRoay7M6dO1K2bFmZOnWqDivTb8HBwbJ69WplJyQismDBAmnRooX4+vpKcnKyDqv7Z8HBwTJp0iQ5d+6cZGZmyoYNG8Ta2rrEhKSXQ6KXl5esX79eLly4ICIvQtJHH30kn376qQ4rfLUnT55Iu3btpF+/fsqy1NRU5f979eolgwcPLvDX5RAb5WBgYAAbGxuYmJigcuXKGDlyJBo3boxjx47Bz88Pp0+fxtq1a5GamqrrUgtNeHg4rl69Cmtra0yZMgWlSpXC1KlTkZmZieXLlyvtgoKC4O3tXSwmbqtUKuzevRsDBgzAuHHj8OeffyIpKQnGxsbYt28fwsPD8emnnxbZocVTp06hXLlyqF69Otq1a4fMzEwAL4baRowYgW+//RahoaEcZvv/1O/D/fv3cfHiRWzcuBF79uxBZGQkAGDOnDnIysrC7t27kZycrMtSX+nJkyeYOXMmjh8/jubNm8PQ0BDu7u6YO3cuRo4cid27d+u6xEKnHjL/7LPP0KRJE2zevBkzZ87E0aNH8dFHH2HOnDk4e/YswsLCdF1qrkqVKoWuXbvi3Xffxffffw/gxY1y1d9FHTp0KJR5SAxIlCv1D5uIoEqVKhg+fDjatWuHZcuWoWPHjqhduzZMTEx0XGXhCA0NhbOzM86cOYPk5GQ0aNAAo0aNQo8ePWBhYYE//vgDwItwNGbMGLRv375YTNy+fv06/vOf/2D+/PkYN24cTp06hZMnTyIlJQXGxsY4duwYZsyYUaTu3K3eyZ85cwYeHh5ISEjArl27ULVqVbRq1Qrp6ek4d+4csrKycPPmTTRo0KDQJnwWNSqVCoGBgfj4448xfPhwfPnll/D19cXu3bsRExOD0NBQ1KtXD59//jmqVaum63LzVLZsWUyfPh2pqamYPn06gBc7XHd3dyxZsgTly5fXcYWF78aNG5g1axZ2796NatWq4enTp2jSpAmWL1+O48ePo1OnTjhw4ADq16+v61Jz9c4772DChAlo0KAB/vzzT42QpF5fo0YNiEjBnmFZ4MekqNhRH5799ddf5Z133pH9+/drLC9OQkNDpUWLFrJp06Yc627fvi3Tpk2TSZMmybJly8TBwUH8/Px0UGXBUX+G586dEzc3N5k7d66ybvny5dK2bVvZtWuX3g+hvMq5c+dk8uTJGnMXnj9/Lj179pTGjRvL+++/L7///ruyrjj+XL+JS5cuSY8ePSQwMFBZ9ttvv0mXLl2kd+/e8t5778nBgwd1WGHu1J9fSEiInD9/Xv7++28RETl27Ji0bt1aZs2aledzirMrV67I6dOnpXnz5hITEyMhISFSo0YN6dKli8TGxuq6vBzu378v+/bt01j25MkT+eabb2To0KGybt06EXnx+12vXj0JCAgo8BoYkEq49PR0EfnnL4jk5GQZO3asbN++XWlfHL9UpkyZIosXL1b+feHCBVm8eLGsXr1abt68KdHR0fLJJ5+IlZVVkZ6/8PIcskePHsnDhw+lVatW0qNHDwkLC1PWLVy4UFq0aCExMTG6KLNAfPXVV2JpaSmenp45gt7Vq1fl7t27IlIydpKvKyMjQxYvXiwmJiayfv16jXXXrl2T4OBgCQ4O1lF1/+zIkSPy3nvvSe/evaVLly7KjjYwMFBsbGxk2rRpOq6wcKl/lhMTEzV+5n/44QcZPny4iLwIwEOGDNH4fdcXN27ckIoVK0qnTp3kp59+0lj35MkTWbFihYwdO1amTJkidnZ2OYJUQWFAKsFiYmJkxIgRcvnyZRH55x1EYmKi0q647UzUZ+ZNmjRJVq1aJc+ePZOpU6fKoEGD5IMPPpBBgwZJjx495OnTpxIWFibXr18XkaK5U83KypLNmzfLV199JSdPnpS6detKWlqaXL58WTp27CizZ8+WmzdvKu3v3Lmjw2rf3I0bN+TevXsiIvL1119L586dxc/PT549e6bjyvTbtWvXJCIiQpKTk8Xb21ucnJz08khRXs6fPy+DBg2Sv/76S6KiomTt2rXStm1b5ch3QECAnD59WsdVFr69e/eKg4ODDBw4UL755hsReXEUrVWrVjJq1Chp0KCB8p7ok+fPn8sXX3whn3zyiaxZs0bc3d1zDUlLly6VJk2aKH+oFsZ3MQNSCfbXX3/JuHHjxM3NTa5duyYief+QFcUg8LrCw8PF19dXsrKyZP/+/fLuu+/K+++/L87OzrJ3715JSUmRy5cvS/fu3SU+Pl7X5RaIp0+fSsWKFcXc3Fw5m0VE5OLFi9KlSxfx8vJShiaK0mevrvXixYvywQcfSIcOHSQiIkJERObNmyfdu3eXXbt2MSRpUb9vf/31lwwaNEg8PDwkJiZGEhMTZdGiRfLxxx/r9RFTdf0PHz6UUaNGiZ2dnbIuKipK1q1bJ3Z2drJ7925dlfhWXbp0Sbp37y6//PKL7NixQxo3bizLly8XEZH9+/fLrFmz5MiRIzquMndZWVly8eJFefbsmURHR8vixYvF3d09x7SHlJQU5Q+gwvqOYkAq4S5duiReXl7i6uqaZ0jKzMwUkRfDbEePHi1210HavHmz2NjYKL+Ad+/elTNnzojI//X9zJkz0qpVK7l//77O6iwI6s82PT1dnJycpEaNGjJp0iSNNpcvX5b27dtLaGioLkr81/bu3SvdunWTb775RurVqyd9+/ZVruE0a9Ys6dSpU5EeMiws+/fvlw8//FDc3NykefPmMm7cOImIiJCEhASZO3euODs7S0xMjN4G5n379snQoUNl8+bNYmtrKwsWLFDWRUREyH//+1/l97o4u3btmtja2srmzZtF5MVwaWBgoDRu3FgWLVqk0VbfPsvc9i1RUVGyZMkSGTp0qBw4cEBOnTol33zzjTI9pDAxIJVA2r8Ud+/elSlTpuQaktQB4cmTJ9KyZUsJCQl5u8W+JT/++KO0bdtWNm7cKElJSRrrTp8+Lba2tnr9F/TrUH+mfn5+cvz4ccnMzJQnT55IgwYNxMPDQ0RETp06JSdOnCiy13Z6/PixtGjRQg4fPiwiL66V4uzsLE5OThIZGSkiUmSDX2G6deuW2NjYyJUrV0TkxfVyxo0bJxMmTJC4uDiJj4/XGHbVN3///bd07txZqd/X11f69u2rEQiK6s/061JPfbhy5Yo0b95cWrRoobHO399f3n//fbl586Ze/5F75syZHNeai4iIkP/973/Srl07MTY2ll27dr2VWhiQShj1TvLQoUMyf/588fT0lPj4eLl165Z4eXmJm5ub/PXXXyIiysX0EhISpFOnTnLixAmd1V3QcrvQ5ffffy/t27eXn3/+WVJTUyUxMVGWL18ujRo1KtRx7rdp37590rBhQ9mzZ4+y7O+//xZra2txcnKS2rVrv7Uvn8Lw5MkTadOmjcYFLcPCwqRy5coyceJEZVlR/xwLWkREhHTr1k1u376tLFuxYoU0aNBAvvjiC2X+oT66fv26tG/fXj7++GONycm//vqrODs7y/z583VcYeFS9zk5OVk5qnL79m1xcXGRAQMGKGEoOzu7SEwRCAwMlP79+0tiYqJkZ2crf6Rv3779rZ9FzYBUAu3bt0/s7e0lICBA6tSpIwMHDhSRF0MrEyZMkN69eytnPjx58kQ++uijYhWOYmNjpW3btrn26YcffpAWLVrI+fPnReTFWU7nzp0TkaK5U1V/uYi8mLRsa2srDx8+FJEXk1m///57efLkiSQkJMjy5cvl1KlTIlI0+6o2ZMgQ6dy5sxLwz549K+7u7mJlZSU//vijjqvTT+Hh4dKlSxfZtm2bpKSkiIjIyZMnZcCAAdKtWzflyLK+Up/N5O/vr1xhOTExUbZu3Vpsj3qLaP7B27FjR+nSpYuMHDlSnj9/Ljdu3JC+fftKz549i9Tv8507d8Te3l45Y1pEJCkpSUaNGvXWz6JmQCoB1DvJzMxMiY+PF1dXV4mJiZFdu3ZJ27ZtNSa6vXyGVmZmpsybN0/++OMPXZVeaCZNmiR2dna5zkmYNGmS9O/fX68PQ7+OyMhI+eCDD5T5NyEhIeLo6CibNm2Szz//XAYPHiw1atSQL7/8UseVFgz15xURESFubm5Sr149Wbx4sVhZWUloaKj85z//0fjSJU2//PKL2NjYiJeXl8yfP18aNWokly9floEDB8qBAwd0XV6uXv4DYNKkSdK3b18JDAxUQtLL64sbdd8CAwOlQYMGsm/fPrl8+bI0b95cuSXH9evXxcXFReNEDH10+fJlCQwMlISEBBER2bp1q3Tp0kX57srOzlbWvc2zqHkl7WIuJiYGw4cPx8OHD2FoaAgAMDMzw8qVK7Fs2TJs3LgRNWvWhK+vL9atW4d69eqhYcOGAF7ccsTLywvt27fXYQ8Klvoqq6tXr4aTkxPGjRuHs2fPaqz78MMPUbduXRgYFO1fDwsLC9StWxdt2rRBREQE7Ozs4OzsjLVr1+LDDz/Eli1b8L///Q+3bt3Cs2fPCvYKtIVMcrkdiPrzqlatGnx9fTFx4kRUrVoVhw4dQnR0NDZv3gxbW9u3XareUd9iRU39Xg4ZMgTLly+Hubk54uLisHXrVqSkpODKlSto3LixLkr9R4aGhhq/0++99x6WLFmC06dPQ0SU77zi5OHDhwCg9C0sLAzjx49Hz549YWNjg3PnzuHChQvYvHkzGjZsiK1bt6JZs2a6LDlP2dnZEBGsXr0aq1evRufOnfHnn3+iWrVqaNSoEeLi4gC8uKq7+ornKpXq7V3t/q3EMNKZsLAw8fDwkD59+ig3mvT09JQGDRrIn3/+KSIiJ06ckAYNGsjRo0d1Wepb8/KRoS+++EKaNm0qx44dk8ePH8tff/0lzZo1k0OHDumwwn/v5b+chw0bJtWrV1c+f/Up7sePHxcbG5sidTXwiIgIZYjwdf+KVF89WH29r5LowYMHygUCRXIeWcntvTxy5IjUrVtXmfisD/L6zF/uz/jx44vtsFpmZqYMHTpU4zNZuHChfPDBB8rQqIjIzJkz5bffftNFia9F/TlGR0drLJs3b564urpKv379RKVSadycVhcYkEoAf39/6du3rwwYMEDi4+OVC6m1b99epk2bJo0aNdLLC4YVppdD0oIFC8TZ2Vl69uwpLVu2LPJnq6m9vNNwd3eXatWqKSHp7NmzYmNjU6T6GhkZKVWrVpXu3bu/1jWa1P0PCwvTuAt9SfTgwQNp2bKluLm5KcvyGn5SL79165beXSRUfSZiborzcNrL0tPT5c6dO9KjRw/l30OHDhUPDw+Jjo6W4OBg+eCDD5Q/gPWVn5+fNG7cWNzd3WXevHnK8ps3b8qff/4pHTp00DiZRBdUIrx1dXG2f/9+LFy4EI0bN8bFixdhbW2NjRs3Ii0tDQEBATAwMECtWrXg4OAAESlRN+rMzs5WhmVCQ0OVQ9Z169YtNu9FVlaW0q/hw4fjxIkTOHXqFCwtLXHz5s0i1dfs7Gw4OzujSZMmePDgAebPn49GjRrl2lbd7+TkZNy4cQMODg5vuVr9ExwcjLFjx8LOzg4bNmwAoPnz8fK/nz59itDQUDRv3lxX5eZw584djBkzBvv27YOxsXGuw2eZmZkoVaoUMjMzkZ2dDSMjIx1UWvDS0tKQnZ0NU1NT3L9/H6VKlYKTkxNsbGywfft2XL9+HfPmzcOtW7dgYmKCGTNm4OOPP9Z12Xk6f/48vL29MXLkSGRkZODXX39FjRo1sGrVKqWN+rPU6feTLtMZFa579+6Jvb29ctr+pUuXZNy4cTJo0KBX/iVWXKSlpSmTNfNSEq4c/vJf1v379xdzc/Mid02YrKwsefbsmXTt2lV8fHzE09NT3N3dJTAwULZu3arRVt3fhIQEad68eYm4rcQ/2bVrl3Tt2lVGjBghtWrVEldXV2XdyydxiLw4c7VDhw7K94auaP8OXrhwQRo0aJBne3X9jx8/Fg8PjyJxSvvrSE9Pl8OHD8umTZtk+/bt4urqKrGxsRIZGSl2dnYyYMAApW14eLjExcWJiP5+h928eVPat28vK1euFJEXV/UPDAyUvn37yoQJE5R2+lA/A1IxFhkZKW3atFHGq7OysuSHH34QS0tL6dmzp6Smphb5M7Xycu3aNRk4cKA4OTnJ2rVrc1z8UVtRPjyfkpKiMf8gNy/379KlS4VdUoFT/5wuWbJEQkJC5MmTJzJy5EgxMzNTzsJ7+ZopT548kY4dO+r9MMPbcPfuXbG1tVVO1b9586Y4OTnJiBEjlDbq6+c8efJEOnXqJMePH9dJrbkJCQmR2NhYuXPnjgwdOlQ5m+ll6ks6qOs/duzY2y6zUAUEBEjLli2lWrVqGjdmjYyMFEdHR+nUqZMOq3s96sBz+vRp6dGjh9jY2ChnUD9//lz8/f2lR48eylnU+oABqRjRTtxpaWkyaNAg+f7775X7UR0/flwmT56sVxMvC9r169fFzs5OfHx8xNfXVxo3biw7d+7Ms716p5qYmChnz559W2UWiGvXrsnHH38szs7O8vXXX78y8GqHwKIYCufPny+bNm2Shw8fSp06daR9+/bSr18/jTuSJyQk5Hmdq5IoNDRUmjVrptxeJTMzU9avXy+mpqYaRx8eP34sTk5OOn/fwsLCZOPGjSLyIvy3bt1aunfvLq1btxaVSiVDhgyRkSNHyv/+9z/ZsWOH8rwnT57oRf0FSf37/OTJE3FxcREnJyf5+uuvNW6V8+jRI2nVqpVyvTZ9o94vxcXFKf25f/++jB49WgYPHqyEpGfPnundUT/OQSom5P+P0wYGBuLYsWMQEbi7uyM6OhoLFy6Eubk56tSpg59//hlr1qyBs7OzrksuFE+fPsW4cePQrl07jBs3DgDwzTff4Pr16/Dx8ckxnq2ec5GYmAgnJyd89913aNWqla7Kz5cbN25g+PDh+OSTT1CpUiWMGzcOp0+fhpWVVa7t1X1NSkrC48ePYW1t/XYLLgAHDx7Etm3bcO3aNQwePBiurq5YuXIlhg8fDjs7O2RnZ2POnDno1q0b2rRpo+tydUL7ZzwtLQ0jRoxA27Zt4ebmhkqVKuHgwYM4deoUunfvjtatWyMrKwujR4+Gu7s7OnbsqMPqgQsXLiAlJQUNGjRAlSpVkJ6eDgC4f/8+3Nzc0KtXL5QvXx6JiYlo1aoVunTpgmfPnqFHjx6YN28ePvroI53WX1DUn+OxY8dw7do1DB8+HIGBgdi5cyesra2VOUdXrlxBjx49YGxsrOuSc1D34fDhw1i0aBGqVq2K58+fY9euXXjw4AG++eYbREZGYuXKlfr5faS7bEYFbf/+/WJjYyNfffWVTJw4UUxMTOTy5cvy999/y5o1a2T69OnF/lT+jIwM2blzpzx+/Fi5oNiGDRuke/fuGjdqFdH866yoDcckJibKoEGD5IcfflCWOTo6ymeffSYbNmzIcZrzy/Ny8rpAZlEQGxsrdevWlaVLlyrLnjx5otFGn2+L8bYEBATIypUrZfv27ZKdnS2+vr4yZMgQ6devn6xdu1Zq1KghgYGBSvvMzMxch650JSUlRSpVqiRz5szRWD579mxZv359jvYXL17U+6t9v4n9+/dLkyZNNC7U+csvv8iwYcOkT58+Ur9+fY3PUR+dOnVK6tatKwcPHpS//vpL+vTpI40aNZLMzEy5ePGijBs3Ti5evKjrMnPFgFRMPH/+XIYMGaKxk1+9erXUrVtXYmNjdVjZ26N9g121P//8U4YOHSoiL35Z161bp9yLragOx2RkZCjX9cnKypKPP/5YevXqJb/++qt4eHjI559/rgTBojQvR11zbhM0Xx4KzY0+TOrUB8ePH5f69evLyJEjpVevXjJ58mTJzs6WixcvyoIFC8TT01O5ma8+0f78Dh8+LJaWlrJkyRJlmaenp4wePTrX9sVNeHi4tGnTRoKDg0VE83vtzJkzsmrVKjly5IiuystTVFSU3Lp1S5n36ePjI97e3hptOnbsqNz2R/sPHH3CgFRMpKenS9u2bZUzA7KysiQhIUH69esnUVFRui1OR9RfoAEBAeLl5SXnz5+X+vXrK3+NZWZmyqxZs/Q6MORGe8fw8OFDjetYnTp1Spo3by6PHz9Wlj158kTat2+v10EwJiZGRowYoQS/f9oBFtcTDP6Nixcviqurq9y4cUNEXlzvys3NTSZOnKhMZFb/V5+oP+vz58/L8ePHlYm6AQEBUqVKFSUkHTlypFje+ig30dHR0qtXL4mIiJCMjAwlIIWGhmq006egeOPGDWnfvr0sW7ZMOaK3ZcsWady4sdy6dUtp5+XlJT4+Proq87UxIBVR6l+K58+fK1dG3rhxo7i7uytXgQ4KChJ7e3u9u9jb23bw4EExMzMTW1vbHPeUKo7DMcHBwdKlSxfldN+MjAwZNWqUXocjEZG//vpLxo0bJ25ubsqXa15f/gxHOSUmJsrMmTPlnXfekYCAABF5caJGUFCQ9OrVS9zd3UVEf9+7o0ePStWqVWXIkCFiaWkp/v7+IiJy7NgxMTExkWXLlum4wsKl/bMeExMjtra2snv3bmVZUFCQdOjQQblQqj65du2a2NraypYtWzT68vDhQ5kyZYp88sknEhQUJFeuXBFbW1u9/z4SYUAq0vbs2SPOzs4yYMAAOXLkiKSmpoq3t7c0aNBAhgwZIvXq1dM4JbS4edVwzMtOnz4tFStWVL5wRfR3J/FvnTlzRpo1a5bjCrTqsKTvLl26JF5eXuLq6ppnSFL/JZ2UlCQBAQHF9rN8E6GhoTJ27Fjp1auXcjmH9PR0OXPmjF7O81B/tsnJyTJt2jQ5deqUiIj897//lXLlyim/s/7+/no5nFRQ1O/D0aNH5YsvvpDly5dLWFiY7NmzR959912ZNm2aLFq0SBo1aqSX3+nPnj2T3r17a8yJfNnx48dl1qxZ0rhxY+nUqZPs2rXr7Rb4hhiQiqgbN25Iu3btZPPmzbJy5UopX7688mVy8eJF2b9/v/IFqU+HYAtKfoZjnj59qozjF8X34nWD4NGjRzVulZKdnV0kwoN2v+7evStTpkzJNSS9PJ+qZcuWxfaeW69D/Z6EhITI/v37JSwsTLKysuTWrVvy2WefSf/+/YvE++Pv7y9r1qyRjz/+WGNYePXq1aJSqTTmSxXF39/X5efnJ82aNZNt27bJhx9+qNwW5tChQ/LVV1+Jt7e3Mryob+9DamqqdO7cWQnh6vrU/71//748e/ZM4uLilDlH+taH3DAgFSEv3+Bv7969yiQ3kRcT4SpWrCi//vqrrsp7q153OOblgFAUfiG15ScIPnv2TJm7UVT6qq7z0KFDMn/+fPH09JT4+Hi5deuWeHl5iZubm3JFZ/XcmYSEBOnUqVOROERf2NRnOY0aNUrat28vq1evVu7VNXHiRPn4448lKSlJ734e1PWcO3dOqlWrJnPmzJF3331XmYCt9t133xX5G0e/jri4OOnVq5c8fPhQ/P39xdHRUe7fvy8i/3dzaX2VnZ0tMTEx0rp1a7lw4YKI5DxT+Pvvv9erC0C+LgakImb37t1Sp04dcXR0lBYtWmisW7dunZiYmEhUVFSROHLwb+VnOCY5OVmOHj1a5N6XNwmCRc2+ffvE3t5eAgICpE6dOjJw4EAREbl8+bJMmDBBevfuLcnJySLy4sjRRx99xHAkIn/88YfY2tpKdHS0bNu2TbmFyH//+1/JyMiQ27dv6/VOyd/fX7y9vZWrdj948EAsLCxk3LhxOdrqW8ArCOo+hYaGSnp6uixcuFCmT58uDg4OyrzR/fv3y48//ihpaWl6/x4MHz5cGjVqpNzGSP2ddPbsWWnRokWRvDgxA1IRcuvWLRk3bpwEBATI3bt3pWbNmjJkyBCNNg8ePNBRdW9HSRyOKW5B8OV7f8XHx4urq6vExMTIrl27pG3btsqVdbOzsyUsLEzZyWdmZsq8efNKzFlM/+TUqVNy/vx5OXDggLRo0UL+/vtvmTJlilhbW8vSpUv1foc6duxYUalUGle5f/jwoZQtW1Y8PDx0WFnhU382f/zxh9jZ2cnZs2eVeaPqI6anT5+WevXqKRPu9dXL15NzdnYWGxsbuX37tty9e1dOnDghjRs3zjEnsqhgQCoiYmNjxdDQUD799FNlWVxcnNSuXVt69+6do72+fzm+iZI0HFNcg2B0dLQMGTJECfLx8fHi4eEhM2fOlFatWimnAm/btk3WrFmj8dzs7Ox/vKdeSZKVlSWZmZkyatQo+fnnn0VE5Ndff5XBgwfL1atXdVzd65kwYYJYWlpq3DxbPcxU3AUHB8v06dOVyedBQUHStGlTGT58uIwZM0YaN26slxOyXyU6Olrc3NzE1tZWPvzwQ+nUqZNyFl5R3CcxIBUhfn5+8s4772hcTyI2NlaqVasmFy9eLJI/gPlVEoZjinMQDAsLEw8PD+nTp488evRIRF5c/K9BgwbK9ahOnDghDRo0KPZXfS8o06dPlzFjxsgPP/wgLVq00PsrK4toXvRw2LBhUrNmTXn48KFGm+L2fXb9+nXlHnPZ2dni5OQkFSpUkO3btyvvx9WrV2X79u3yyy+/KCeW6JvX+Vxu374tsbGxyj3jiupnyYBUxOzfv1+qVKmicW0j9YS44qikDscU5yDo7+8vffv2lQEDBkh8fLycP39eBg0aJO3bt5dp06ZJo0aNNC58Sa+2Z88e8fT0lA8//FA5g7EoeDkk9e/fX6pWrSqpqak6rKhwXbx4UY4ePSrR0dEi8uLMr549e0rfvn01jqDpq5iYGAkPD39lrXkN7TMg0Vvj5+cnxsbGcvv2bV2XUqhK0nBMSQmC+/btE0dHRxk5cqQ0a9ZM+vTpIwkJCRIdHS1bt24VX19fCQoKEpGi+6VaWP7p/Xj69OlrtXtbwsLCxN/f/5X1vByS1GdqFjdhYWHKsHdSUpIYGBjIrFmzRORFSProo4/Ezc1NOaKqj65duyYtW7YUZ2dnGT9+/D8GOu3bPRVVDEhF1N69e4v9OH1JGY4pKUHw3r17Ym9vrwwPXrp0ScaNGyeDBg0qEn9Bv23aN1d+FX3bIV2/fl0aNGig8fOqfW0cNe3a9a0v/9aKFStEpVIpp8AHBgZKmTJl5KuvvhKRFyGpRYsW0qdPH+UMMH1y584dad68ufz6668SGhoqbdu21ZjmIaL5mb58z8SzZ8++1VoLGgNSEacvfy0WlpIwHFNSgmBkZKS0adNGOd03KytLfvjhB7G0tJSePXtKamqqXp99pwsHDhyQCRMmyLRp0/IMSi/vkPThjKfk5GRp3bq1Mt9G5EUIeHn4LK8d6t27d99WmYUuOztb6efMmTOlbNmySkg6efKklC5dWhYtWiQiL96fM2fO6KzWV9m6dasMGzZM+bednZ0MHDhQJk2aJFu3btVoq/4sExISpHnz5nL69Om3WmtBMwAVaSqVStclFJr9+/dj9uzZqFChAm7evInRo0fj/fffx6pVqzB27FjY29tj48aN6NGjB0RE1+W+sXr16mHw4MFQqVTw9PTE48ePMWTIEDRr1gxz5szBF198gfHjx+Prr79Gp06ddF3ua9P+TCpWrAgrKyucPXsWkZGRMDAwQL169dC3b18sXrwYxsbGMDDgV5LalStXMHHiRDg6OuLAgQMYOnQoHj58qNEmKysLhoaGSEhIQJ8+ffDOO+/oqNr/k5qaijJlymDEiBEAgJEjR2Lw4MGYMmUKtm3bBuD/vrfU9ScmJqJDhw6IiorSVdmFQqVS4dChQzAyMkL16tXRtm1bBAcH48MPP0RgYCBmz56N+fPnw9jYGC1bttR1ubmqUKECYmNjsWDBAjg4OMDGxgaTJk1CjRo18Ntvv+HOnTsANH8W+/bti5UrV6JVq1Y6rv5f0nVCI8pNSRqOKY7zcl6+zsucOXNk9uzZEhYWJidOnJAuXbqIu7u7zJs3T+rUqVMirpT8utTvW3x8vBw8eFCZdK2+lcPAgQMlPDxcRDQv79CxY0e9mqQ/YMAAOXjwoDJUfPLkSfnPf/4j/fv3V44SadevPlpaHKg/x/Pnz0u1atXk5MmTcvv2bZk/f74YGxsrc5JOnDiR4wba+ubp06fyzTffyPLly6Vdu3bK8ocPH0qHDh0kLCxMWZaQkCBt27bVq5/Ff4MBifRSSRmOKc5BcP/+/WJjYyNfffWVTJw4UUxMTOTy5cvy999/y5o1a2T69OlFesiwsOzatUs6deokNWrUkGnTpkliYqKIvAhJrVu3lj59+sjz589FROTx48fSuXNnvdohZWVliaenpwwcOFDGjRun/J4+ePBAnJycNOavPHnyRNq3b69X9f8bd+7cUX6XRUS2b9+uDE+p3wcXFxdRqVRy7tw5pZ2+/uHz8ndscnKyjBo1Sjk5KCwsTJo1ayY3btwQkReBd9asWcUq6DIgkV7Q/oJIS0uTQYMGyffffy8REREi8uKO0JMnTy6Sl6zPS3ENgs+fP5chQ4ZofFmuXr1a6tatK7GxsTqsTL+pzxY6efKkLFq0SFq3bi1+fn7KGWqpqakaRxMHDhyoF/OOtD158kR69uwp7733nhw8eFBERG7evCkODg7KEYeMjAwZNWpUsQlHmZmZ8sMPP0hQUJBy/7SzZ89KgwYN5NixY0q7//3vf9KpUyflfSkqUlNTpUePHtKrVy/x8vISGxsb5SKQauowX1yoRIrw5A0qFkQEKpUKgYGBOHbsGEQE7u7uiI6OxsKFC2Fubo46derg559/xpo1a+Ds7Kzrkt+Yuq9q6enpGDFiBDp06ICePXuiWrVqOHHiBHbs2IGxY8eiSZMmOqz2zWVkZKBTp07o27cvpk6diuzsbCQnJ2P06NFYvXo1zM3NdV2i3lD/TJw/fx5ff/016tevj/nz5wMAvv76a+zduxdeXl5wcnJS5hipn5OWlgZjY2Ndlp+Dei5KUlISJk2ahFKlSiE9PR1XrlzBggUL0KdPH6VtfHw8KlWqpMNqC15kZCS6deuG1atXo02bNvjyyy8RERGBNm3aoHHjxpgwYQJ8fX1Rv379HN8HupSRkYHSpUvnWpN62ZMnT7Bx40aICOzs7NChQwfIiwMtxXL+IAMS6QU/Pz/MmjUL/fv3R2RkJDZs2ICgoCCYmprC398f4eHh6Ny5c5GapKytOAdBdd9SU1MhIjA1NcWmTZtw7NgxDBkyBM7OzggODsbEiROxY8cO1KpVS9cl61xqaipMTEwAABERERAR9O/fHxUrVsSKFStQr149AMCiRYuwZ88e+Pn5oUqVKsrzdb1zfdUOVR2SUlNTcf/+fTx48ACVK1dG06ZNi90O9e7duzh16hSqVKmCdu3aQUSwcOFCBAQE4Mcff0T16tXx66+/YsOGDTA3N8fw4cPRr18/XZetITY2Fl988QU8PT1hY2Pzys/0Zbr+GSx0b+lIFVGeStJwTHGel7Nnzx5xdnaWAQMGyJEjRyQ1NVW8vb2lQYMGyo04i9q9pQpLVlaWbN68Wb766is5efKk1K1bV9LS0uTy5cvSsWNHmT17tty8eVNp//KV8/VBTEyMjBgxQrm4Y25zaPKaV6Ov823exI0bN6Rp06bi5uYm3bt3l19++UVEXkxWnjNnjtjZ2Smn9j99+lS5Zpm+vQd//fWXjBs3Ttzc3PK8IbZacbtO1aswIJHOpaenS9u2bWXlypUi8mLnkZCQIP369ZOoqCjdFleAinMQvHHjhrRr1042b94sK1eulPLlyysXMr148aLs379fLl26JCL6t3PQladPn0rFihXF3Nxc2YmKvHi/unTpIl5eXvL333+LiP69Z9yhioSGhoq9vb34+vqKiMgXX3whS5culRs3bii3AfL29pZ69erJqVOndFnqa7l06ZJ4eXnlekNsNfVnmZSUJEePHi2S8yPzo3gc46QiRf7/qG5qaiqeP3+O0qVLY+TIkbhw4QIOHz4MAwMDhIWF4e7du3j27JmOqy04hoaGCA8Px/nz5wEA2dnZGDp0KGxtbZGVlaXj6vJP/TnGxMTg5s2bGDZsGNzd3TF16lSsWLECbm5u2LFjB5o2bYoePXrA1tYWQPG+dtfrUL9vRkZGsLOzg7GxMXx8fJT1TZs2xfLly3HhwgVkZ2cD0L/37IMPPsCECRNQvXp1zJ07F9evX4dKpcpx7Sv1sExycjKOHTum9KeoS0pKwrBhw9CkSRO4ubkBAI4cOQI/Pz94eXnB3t4ejx8/xqxZszB48GC97ffLn5etrS0++eSTPD/Tl69z1KVLF7z77rvFZpg0T7pMZ1RylYThGPVfX8+fP1fOatm4caO4u7sr1/4JCgoSe3t7vRtCeV27d++WOnXqiKOjo7Ro0UJj3bp168TExESioqKK/V+ar0v9M+Hn5yfHjx+XzMxMefLkiTRo0EA8PDxEROTUqVNy4sQJvbzthPYRhbt378qUKVNyPerw8nWOWrZsqVz7p7j4+uuvpUePHrJ161bp1q2bfP755yLy4kyuHj16yLRp00REf28mrv6cDh06JPPnzxdPT0+Jj4+XW7duiZeXl7i5uSmXLMjIyBCRF0OHnTp1KjZnHv4TBiR660rScExxDoK3bt2ScePGSUBAgNy9e1dq1qwpQ4YM0Wijvr8c/Z99+/ZJw4YNZc+ePcqyv//+W6ytrcXJyUlq164tu3bt0l2BeeAO9YWXw/7KlSulUaNG0qNHD43vqgULFoi3t7cOqsufffv2ib29vQQEBEidOnVk4MCBIvLixsETJkyQ3r17K8OFT548kY8++qhYfZb/hAGJ3gr1l0d0dLTs3btXfvzxR2Wdj4+PVKxYUX799VddlVcoinMQjI2NFUNDQ/n000+VZXFxcVK7dm3p3bt3jvZFrX8F6eU5ODdu3BBbW1t5+PChiLy40vL3338vT548kYSEBFm+fLkyX0Uf3zPuUF94OSStX79enJyc5LfffhORF++FjY2NHDlyRFfl5Un9s5iZmSnx8fHi6uoqMTExsmvXLmnbtq3cu3dPRF787IWFhcn169eV9vPmzZM//vhDV6XrBAMSvTUlYTimJAVBPz8/eeeddzSujBwbGyvVqlWTixcv6uUO/m2LjIyUDz74QDmSFhISIo6OjrJp0yb5/PPPZfDgwVKjRg358ssvdVxp7rhDzdvL31MrVqyQrl27ypw5c6RJkybi5+enw8pyFx0dLUOGDFF+FuPj45VbwbRq1Ur5Pd62bZusWbNG47nZ2dnKGXglCQMSvRUlaTimJARBtf3790uVKlU05lDp65wLXenTp4/UrFlTHj16JCIic+fOFQcHB2UY7eDBg+Lm5iYpKSl69TPBHeoL6tu65Oblz+vrr7+W999/Xy/DkciLW4N4eHhInz59lJ9FT09PadCggXJm7YkTJ6RBgwZF9lIjBY0BiQpdSRqOKUlBUM3Pz0+MjY2VezTRCy8PrQ0bNkyqV6+u7JjUk/aPHz8uNjY2erlT5Q71xe1RvLy85OnTp3l+L70cktRDp9nZ2Xr5Pebv7y99+/aVAQMGSHx8vJw/f14GDRok7du3l2nTpkmjRo1k//79ui5TbzAg0VtREoZjSlIQ1LZ3715lfhX9n5dDkru7u1SrVk0JG2fPnhUbGxvZu3evrsr7RyV5hxoaGir16tUTlUr1j/OosrKy9P73ed++feLo6CgjR46UZs2aSZ8+fSQhIUGio6Nl69at4uvrq3GfP+K92Ogt8vPzg4eHB4KCgpRbTahvV1BcHDhwAG5ubrh06RLq1KkDAIiLi4ONjQ0OHDgAW1tbvbumTUGS4n7rgTfw8i0ahg8fjhMnTuDUqVOwtLTEzZs3UbduXb183/bv34+FCxeicePGuHjxIqytrbFx40akpaUhICAABgYGqFWrFhwcHPSy/n8jNDQUw4YNw9y5cxEeHo4///wT33//PcqVK5dr+5fvP3fjxg04Ojq+5Ypf7f79+3B1dcWmTZvwwQcf4PLly1i3bh2SkpKwYsUKWFhY6LpE/aTTeEYlTkkYjuG8nJLrda4m3b9/fzE3N9fL6xyp3bt3T+zt7ZXT9i9duiTjxo2TQYMGSWRkpI6rK1zR0dHSqVMn2bhxo4iIHD58WNq0aaMMn2nPE1N/tgkJCdK8eXM5ffr0W633dURGRkqbNm3kypUrIvKiDz/88INYWlpKz549JTU1Va/mv+mLYn4ZTNI33bt3x44dO3Dnzh1dl1JoevTogU2bNqFhw4ZKP4vTUTLK6cGDBwgPD8/zKIqhoaFytfRff/0Vhw8fhpGR0dssMV+MjY1hamqqXEW5SZMmaN68OY4fP44xY8YgLS1Nb68O/W+ZmZlh8eLFGDFiBACgS5cueOeddzBjxgwA0Lh69MtXl+7bty9WrlyJVq1a6aJsDaI1MFSxYkVYWVnh7NmziIyMhIGBAerVq4e+ffti8eLFMDY2Lv5XxX4DHGIjnZFidlhe2759+2BqagonJyddl0KFKCwsDMOGDcP8+fPRtWvXV7bVviN6bndI1wXt38X09HSMGDECHTp0QM+ePVGtWjWcOHECO3bswNixY9GkSRMdVlt4srOzNYKCegrAn3/+iRUrVuDbb79FjRo1NN6vxMREuLi4YNGiRWjbtq2uSleoawsMDMSxY8cgInB3d0d0dDQWLlwIc3Nz1KlTBz///DPWrFkDZ2dnXZestxiQiApZcQ+CJVloaCjc3d3h6emJwYMH/2P7l+eqPH78GNbW1oVf5D/gDvWfRUVFwdnZGcOGDcNnn32mLM/KysLcuXPRrVs3tGnTRocVavLz88OsWbPQv39/REZGYsOGDQgKCoKpqSn8/f0RHh6Ozp07o1OnTrouVa8xIBERvYGkpCQMGDAABgYGOHDgAADgP//5D6Kjo2FnZ4cOHTqgevXqSnt1OEpMTETHjh2xZs0atGzZUlfla+AONW/qAPnzzz9j9erVOHz4MMqXL6/80ZOUlAQzMzMdV/l/UlNTMXr0aIwfP14JbWvWrMG3336L06dPo3LlyjqusOgopesCiIiKIlNTU/To0QOhoaFYvnw5AgICULt2bbz33nvw8fFBbGwsPD09AWjOVXF1dcW3336rN+EoNTUV27Ztw5o1a5QdaqNGjdCvXz+cPn0aEydO1HGFhUc9hPaqo7zq5Y6OjrCzs0OFChU01utTOAJezHcLDw/H+fPn0aZNG2RnZ2Po0KEIDAxU5sHR6+GsLCKifBIRlC5dGmPHjoWNjQ1+/fVX1K9fH2vXrsWsWbPg5eWFX375BU+fPgUAJRz16dMH8+bN06vhmJd3qACUHaqtrW2x3qHGxsZi7NixuHLlClQqVY6Jzdrq1auHxo0bv6XqXp+67tTUVDx//hylS5fGyJEjceHCBRw+fBgGBgYICwvD3bt38ezZMx1XW7TwCBIRUT6pd6jGxsYYMWIE3n//fbRv315Zb2ZmBmtra2XCb2ZmJj7//HMsWLBA5xN51UdLUlNTISIwNTXFyJEjcezYMRw+fBjOzs4lYocaHR0NY2NjLF68GHPnzkWjRo3yPJKkL5Ppc6NSqbB3716sXbsW5cuXx+jRozFo0CDcu3cPU6dOhb29Pc6dO4dvvvlGuf4cvR7OQSIiekO57VDPnj2LiRMnYv78+XBxcVGWx8fHo1KlSm+7xFxp71DbtWuHJUuWYPv27Ro71J49e+q61EJ1+fJlbN68Gffv38eCBQtyDUnqcJScnIxz586hffv2enVKfGhoKMaNG4fRo0cjPj4e8+bNw2+//QYnJydcunQJjx49wnvvvQdbW1ueMJJPDEhERP/gdeaqAMCpU6fg6emJuXPnomfPnpAXt3PiDlWPaPfp3r17WLlyJR49epQjJL08d6xbt25Ys2YN7OzsdFj9C+r6YmJiEBQUhJiYGIwaNQoAsGHDBkybNg3r169H//79dVxp0caARET0CrGxsfjiiy/g6ekJGxubV4aGZ8+e4fbt22jSpIlehQvuUF9Qvw+HDx9GUFAQEhISMHv2bDx58gRr167Fo0ePMHv2bHzwwQfIzMxEqVKlkJiYCFdXV3h7e+t8ePRle/bswWeffYbKlSsjOzsbwcHByrr169fD09MT9+7dQ5UqVfQqoBcphXJ9biKiYuKvv/6ScePGiZubm1y7dk1Ecr+liL7fqmH37t1Sp04dcXR0lBYtWmisW7dunZiYmEhUVJTe9+Pf2rdvn9jb20tAQIDUqVNHBg4cKCIily9flgkTJkjv3r0lOTlZRESePHkiH3300T/erPZtu3XrlowbN04CAgLk7t27UrNmTRkyZIhGmwcPHuiouuKDsZKI6BU++OADTJgwAdWrV8fcuXNx/fr1XM96Uv/76dOnCAgI0Ktbcdy+fRsHDx7E999/D19fX8TExGDo0KHK+vHjx+PmzZswNzcvdkcb1GfiZWVl4fHjx9i0aRMOHjyIpKQkWFpaYunSpQBe3E5l6tSpWLx4Md555x1kZWXh22+/xbx58/TqyFFcXBzq168PExMTdOzYEdbW1ggJCcGZM2fQp08fpd17770HIOdtR+j1Fa/fBCKiAvLyjsXW1haffPJJniHp5bkqnTt3xrvvvqs3QaMk71BjYmIwfPhwPHz4UDkLzczMDCtXrsSyZcuwceNG1KxZE76+vli3bh3q1auHhg0bAnhxzzUvLy+NsxP1QeXKlbF37174+Pjg9u3bAIBKlSohKCgIQUFBuHTpksZnqC/DvEWRfvwGExHpEXlprsqCBQvg5eUFMzMzTJkyBTVr1sSCBQtw9epVqFQqZGZmKlfI7tevH5YtW6YXE3nVSvIONSEhAUZGRpgyZQoiIiJQsWJFVKhQAbt27cKyZctQp04d/Pnnn5g/fz7q16+v8VyVSoVy5crpqPJX6969O3x9fdGqVSvcvXsXwIvP+f79+2jatGmx+gx1SmeDe0REeqw4zFV52f79+6VKlSpy584dZVl6eroOK3o7/P39pW/fvjJgwACJj4+X8+fPy6BBg6R9+/Yybdo0adSokezfv1/XZb4RPz8/MTY2ltu3b+u6lGKJZ7EREeH/hsmysrKQmJiIsWPHYt26dTh16hRWrFiBn3/+GTVr1oSI4ObNm8jKykLDhg2RlZWFhQsX4qOPPtK74RhtBw4cQN++fXH9+nXUrl1b1+UUuv3792PhwoVo3LgxLl68CGtra2zcuBFpaWkICAiAgYEBatWqBQcHB7066zA/9u3bB1NTUzg5Oem6lGKHAYmISryYmBh4eXlh6dKleO+99/D48WN8/vnnsLCwQGBgIH7++WfUqVMHvr6+ePz4scb9yUQET58+1dvhGG0lZYd6//59uLq6YtOmTfjggw9w+fJlrFu3DklJSVixYgUsLCx0XWKBKqoBT59xDhIRlXjFda5KblxcXODk5FSsJmPnxtjYGKampko/mzRpgubNm+P48eMYM2YM0tLS9OpMw3+L4ajgMSARUYlXr149DB48GCqVCp6ennj8+DGGDBmCZs2aYc6cOfjiiy8wfvx4fP311+jUqZOuyy0QxW2Hqh34KlasCCsrK5w9exaRkZEwMDBAvXr10LdvXyxevBjGxsZ6c6Yh6ScOsRFRiVcS5qoUZ+rPJDAwEMeOHYOIwN3dHdHR0Vi4cCHMzc1Rp04d/Pzzz1izZg2cnZ11XTIVAQxIRFSilbS5KsWVn58fZs2ahf79+yMyMhIbNmxAUFAQTE1N4e/vj/DwcHTu3LnYHAGkwldK1wUQEelSXnNVvL29MWbMGPz2228oXbo0h2P0WGpqKrZt24Y1a9agTZs2AIBGjRqhX79+OH36tMakeqLXxd94IipROFel+DE0NER4eDjOnz8PAMjOzsbQoUNha2ur3GqEKL94BImISoy85qpMmDABCxcuxJ9//qkxV6VJkya6Lplyof4cU1NTISIwNTXFyJEjcezYMRw+fBjOzs4ICwvD3bt38ezZM12XS0UU/ywiohJDpVLBz88Pn376KYyMjPD48WPY2tqifPnyWL16NVq2bInnz5/jf//7Hyfy6jGVSoW9e/eid+/eGDFiBPz9/TFo0CDUrl0bU6dOxdChQ+Hu7o558+ahVq1aui6XiihO0iaiEiM1NRWjR4/G+PHjlbkqa9aswbfffovTp0+jcuXKOq6QXkdoaCjGjRuH0aNHIz4+HvPmzcNvv/0GJycnXLp0CY8ePcJ7770HW1tbnnVIb4xDbERUYrw8V6VNmzbKXJXAwEDOVdFz6qATExODmzdvYtiwYXB3dwcAmJmZwc3NDevXr0f//v3RtGlT5XkMR/SmGJCIqNjiXJXiQ6VSYc+ePfjss89QuXJlZGdnY9SoUQCAkSNHIj09HcOGDUO7du1QpUoVTqynf41DbERUrO3duxdr165F+fLlMXr0aLRr1w5LlizB9u3bYW9vj3PnzuGbb75Bz549dV0qvcLt27exfPlyDBgwALVr10b79u3Rpk0b/PLLL0qbhw8f4r333tNhlVScMCARUbHFuSrFQ1xcHCwsLDB58mSsWrUKABAfHw8HBwfY2Nhg165dGu35WVJBYEAiomLl5bkqQUFBiImJUYZiNmzYgGnTpilzVajoOHDgANzc3HDp0iXUqVMHwIvgZGNjgwMHDsDW1pahiAoUAxIRFTvac1WCg4OVdevXr4enpyfu3bvHuSpFjJ+fHzw8PBAUFKScvp+RkYHSpUvruDIqjhiQiKhY4VyV4u3AgQPo27cvrl+/jtq1a+u6HCrG+KcTERUbcXFxqF+/PkxMTNCxY0dYW1sjJCQEZ86cQZ8+fZR26nDEvw+Lnu7du2PHjh24c+eOrkuhYo5HkIioWOFclZKDk7GpMDEgEVGxw7kqRPRvMSARUbHEuSpE9G8wIBFRsbVv3z6YmprCyclJ16UQURHDgERExR7nqhBRfvEsNiIq9hiOiCi/GJCIiIiItDAgEREREWlhQCIiIiLSwoBEREREpIUBiYiIiEgLAxIRUR4CAwOhUqmQkJDw2s+xtrbGqlWrCq0mIno7GJCIqMgaMWIEVCoVxo8fn2PdpEmToFKpMGLEiLdfGBEVeQxIRFSkWVlZwdfXF8+fP1eWpaamYuvWrahRo4YOKyOioowBiYiKNDs7O1hZWWHnzp3Ksp07d6JGjRpo1qyZsiwtLQ1TpkxB1apVYWJigjZt2uDcuXMa2zpw4ADq1asHU1NTdOjQAffu3cvxeidPnkTbtm1hamoKKysrTJkyBSkpKbnWJiKYN28eatSoAWNjY1haWmLKlCkF03EiKlQMSERU5I0cORIbN25U/r1hwwZ4eHhotPniiy/w+++/46effsKFCxfw/vvvw9nZGY8fPwYAPHjwAH379oWLiwsuXbqE0aNHY8aMGRrbuH37Nrp27QpXV1dcuXIF27dvx8mTJzF58uRc6/r999+xcuVK/O9//8PNmzexe/duNGnSpIB7T0SFgQGJiIq8oUOH4uTJk7h//z7u37+PU6dOYejQocr6lJQUrFu3DsuXL0e3bt3QqFEj/PDDDzA1NYWPjw8AYN26dahTpw6++eYb1K9fH0OGDMkxf2nJkiUYMmQIpk6dirp166J169b47rvvsHnzZqSmpuaoKzw8HBYWFnByckKNGjXg4OCAMWPGFOp7QUQFgwGJiIq8KlWqoEePHti0aRM2btyIHj16oHLlysr627dvIyMjAx9++KGyrHTp0nBwcMCNGzcAADdu3ICjo6PGdlu1aqXx78uXL2PTpk145513lIezszOys7Nx9+7dHHX1798fz58/R+3atTFmzBjs2rULmZmZBdl1IiokpXRdABFRQRg5cqQy1LVmzZpCeY2nT59i3Lhxuc4jym1CuJWVFcLCwnD06FH4+/tj4sSJWL58OY4fP47SpUsXSo1EVDB4BImIioWuXbsiPT0dGRkZcHZ21lhXp04dGBkZ4dSpU8qyjIwMnDt3Do0aNQIANGzYEMHBwRrPO3v2rMa/7ezscP36dbz//vs5HkZGRrnWZWpqChcXF3z33XcIDAzEmTNn8NdffxVEl4moEPEIEhEVC4aGhspwmaGhoca6smXLYsKECZg2bRoqVqyIGjVqYNmyZXj27BlGjRoFABg/fjy++eYbTJs2DaNHj0ZISAg2bdqksZ3p06ejZcuWmDx5MkaPHo2yZcvi+vXr8Pf3x+rVq3PUtGnTJmRlZcHR0RFlypTBL7/8AlNTU9SsWbNw3gQiKjA8gkRExYaZmRnMzMxyXbd06VK4urrC3d0ddnZ2uHXrFg4fPox3330XwIshst9//x27d++Gra0t1q9fj8WLF2tsw8bGBsePH8fff/+Ntm3bolmzZpg7dy4sLS1zfc0KFSrghx9+wIcffggbGxscPXoU+/btQ6VKlQq240RU4FQiIrougoiIiEif8AgSERERkRYGJCIiIiItDEhEREREWhiQiIiIiLQwIBERERFpYUAiIiIi0sKARERERKSFAYmIiIhICwMSERERkRYGJCIiIiItDEhEREREWv4fKONnHURsFQ4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_models([model_1, model_2, model_3, model_4, model_5, model_6, model_7, model_8])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "182f4b18",
   "metadata": {
    "papermill": {
     "duration": 0.292552,
     "end_time": "2023-12-26T07:16:30.321482",
     "exception": false,
     "start_time": "2023-12-26T07:16:30.028930",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model Ensembling\n",
    "\n",
    "* if several uncorrelated models agree on a prediction, then the prediction must be more robust than a prediction made by a single model\n",
    "* The models should be uncorrelated...different types of models. \n",
    "* Ie: using lstm, bidirectional and tensorflow USE model. All these models are trained on the same training data, but they have different ways of finding patterns\n",
    "* If we were to use 3 similarly trained models such as 3 LSTM models, the predictions they output will be similar. \n",
    "* **Averaging Ensemble** - take the output prediction probabilities of each model for each sample. Combine them then average them. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "bcebf113",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:30.914951Z",
     "iopub.status.busy": "2023-12-26T07:16:30.913537Z",
     "iopub.status.idle": "2023-12-26T07:16:34.193519Z",
     "shell.execute_reply": "2023-12-26T07:16:34.191734Z"
    },
    "papermill": {
     "duration": 3.579945,
     "end_time": "2023-12-26T07:16:34.196479",
     "exception": false,
     "start_time": "2023-12-26T07:16:30.616534",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 2ms/step\n",
      "24/24 [==============================] - 1s 9ms/step\n",
      "24/24 [==============================] - 2s 8ms/step\n"
     ]
    }
   ],
   "source": [
    "# use the results from model_1, model_2, model_7 \n",
    "model_1_preds = model_1.predict(valid_dataset)\n",
    "model_2_preds = model_2.predict(valid_dataset)\n",
    "model_7_preds = model_7.predict(valid_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "1cfc8678",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:34.846844Z",
     "iopub.status.busy": "2023-12-26T07:16:34.846177Z",
     "iopub.status.idle": "2023-12-26T07:16:34.855384Z",
     "shell.execute_reply": "2023-12-26T07:16:34.853705Z"
    },
    "papermill": {
     "duration": 0.304291,
     "end_time": "2023-12-26T07:16:34.857873",
     "exception": false,
     "start_time": "2023-12-26T07:16:34.553582",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(762, 1)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_7_preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "afa568f2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:35.462800Z",
     "iopub.status.busy": "2023-12-26T07:16:35.461798Z",
     "iopub.status.idle": "2023-12-26T07:16:35.478112Z",
     "shell.execute_reply": "2023-12-26T07:16:35.477136Z"
    },
    "papermill": {
     "duration": 0.318454,
     "end_time": "2023-12-26T07:16:35.480862",
     "exception": false,
     "start_time": "2023-12-26T07:16:35.162408",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(762,), dtype=float32, numpy=\n",
       "array([0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0., 0., 0., 1.,\n",
       "       0., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 0., 1., 0., 0., 1.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 1., 0., 1., 1., 0.,\n",
       "       1., 0., 0., 1., 0., 1., 0., 0., 0., 1., 0., 1., 0., 0., 1., 1., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 1., 0., 0., 0., 1., 0.,\n",
       "       0., 1., 0., 1., 0., 1., 1., 0., 0., 0., 1., 0., 0., 1., 1., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 1., 0., 1., 1., 1., 1., 0., 1., 0., 1., 0.,\n",
       "       0., 1., 1., 0., 1., 1., 0., 0., 1., 1., 0., 0., 0., 0., 0., 1., 0.,\n",
       "       0., 0., 0., 1., 0., 0., 1., 1., 1., 0., 1., 1., 1., 0., 1., 0., 1.,\n",
       "       1., 1., 0., 0., 1., 1., 0., 0., 0., 1., 1., 0., 1., 0., 0., 1., 0.,\n",
       "       0., 1., 0., 0., 0., 0., 0., 1., 1., 1., 1., 0., 0., 0., 1., 1., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0.,\n",
       "       1., 0., 0., 0., 1., 0., 1., 1., 1., 1., 0., 0., 1., 1., 1., 0., 0.,\n",
       "       1., 1., 1., 1., 1., 0., 0., 0., 1., 0., 0., 1., 0., 0., 0., 1., 0.,\n",
       "       1., 0., 0., 1., 0., 0., 0., 0., 1., 1., 0., 1., 0., 1., 0., 1., 0.,\n",
       "       0., 0., 1., 0., 1., 0., 1., 1., 0., 1., 0., 0., 0., 1., 0., 0., 1.,\n",
       "       0., 0., 1., 0., 1., 1., 0., 1., 1., 1., 0., 0., 1., 1., 1., 0., 0.,\n",
       "       0., 1., 1., 0., 0., 1., 1., 0., 1., 1., 0., 1., 0., 1., 1., 0., 1.,\n",
       "       0., 0., 0., 0., 1., 1., 0., 1., 0., 1., 0., 0., 1., 1., 0., 1., 1.,\n",
       "       0., 0., 1., 0., 0., 0., 1., 0., 1., 1., 0., 1., 1., 1., 1., 1., 1.,\n",
       "       0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 0., 1., 0., 0., 1., 0., 1.,\n",
       "       0., 0., 0., 0., 1., 0., 0., 1., 1., 0., 1., 0., 1., 0., 1., 0., 0.,\n",
       "       0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 1., 0., 1., 1., 1., 0., 0., 0., 0., 1., 0., 0., 1., 1., 1.,\n",
       "       0., 1., 1., 1., 0., 1., 0., 0., 1., 0., 1., 0., 0., 1., 0., 0., 0.,\n",
       "       0., 0., 0., 1., 1., 0., 0., 0., 0., 1., 0., 1., 1., 0., 0., 1., 0.,\n",
       "       1., 1., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 1., 1., 0., 1., 1.,\n",
       "       0., 1., 1., 0., 1., 1., 1., 1., 0., 1., 0., 1., 1., 1., 0., 1., 1.,\n",
       "       0., 0., 0., 0., 0., 1., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
       "       0., 1., 0., 0., 0., 0., 0., 1., 0., 0., 1., 1., 0., 0., 0., 0., 0.,\n",
       "       1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 1., 1., 0., 0.,\n",
       "       1., 1., 0., 0., 0., 0., 0., 0., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 1., 1., 0., 1., 0., 0., 0., 1., 0., 1., 0., 1., 0., 1., 0., 1.,\n",
       "       1., 1., 0., 0., 1., 0., 0., 0., 1., 1., 1., 1., 1., 1., 0., 0., 0.,\n",
       "       0., 1., 0., 1., 0., 0., 0., 0., 1., 0., 1., 1., 1., 1., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 0., 1., 1., 0., 0., 1., 0.,\n",
       "       0., 1., 1., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 0., 1., 0.,\n",
       "       0., 1., 1., 1., 0., 0., 1., 1., 0., 0., 0., 0., 1., 1., 1., 0., 0.,\n",
       "       0., 1., 1., 0., 0., 0., 0., 1., 0., 0., 1., 1., 0., 1., 0., 0., 0.,\n",
       "       0., 1., 0., 1., 1., 1., 0., 0., 1., 0., 1., 0., 1., 0., 0., 0., 0.,\n",
       "       0., 0., 1., 0., 0., 0., 1., 1., 1., 1., 0., 1., 0., 0., 1., 1., 1.,\n",
       "       1., 1., 0., 1., 0., 1., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1.,\n",
       "       0., 0., 0., 0., 1., 1., 0., 0., 0., 1., 0., 0., 1., 0., 1., 1., 1.,\n",
       "       0., 1., 1., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 1., 0.,\n",
       "       1., 0., 0., 1., 0., 0., 1., 1., 1., 0., 0., 1., 0., 0.],\n",
       "      dtype=float32)>"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get mean pred probs for 3 models\n",
    "combined_pred_probs = tf.squeeze(model_1_preds, axis=1) + tf.squeeze(model_2_preds, axis=1) + tf.squeeze(model_7_preds, axis=1)\n",
    "combined_preds = tf.round(combined_pred_probs/3)\n",
    "combined_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "8887cf2f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:36.067982Z",
     "iopub.status.busy": "2023-12-26T07:16:36.067602Z",
     "iopub.status.idle": "2023-12-26T07:16:36.074955Z",
     "shell.execute_reply": "2023-12-26T07:16:36.072798Z"
    },
    "papermill": {
     "duration": 0.305796,
     "end_time": "2023-12-26T07:16:36.077710",
     "exception": false,
     "start_time": "2023-12-26T07:16:35.771914",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
    "\n",
    "def calculate_results(y_true, y_pred):\n",
    "  \"\"\"\n",
    "  Calculates model accuracy, precision, recall and f1 score of a binary classification model.\n",
    "\n",
    "  Args:\n",
    "  -----\n",
    "  y_true = true labels in the form of a 1D array\n",
    "  y_pred = predicted labels in the form of a 1D array\n",
    "\n",
    "  Returns a dictionary of accuracy, precision, recall, f1-score.\n",
    "  \"\"\"\n",
    "  # Calculate model accuracy\n",
    "  model_accuracy = accuracy_score(y_true, y_pred) * 100\n",
    "  # Calculate model precision, recall and f1 score using \"weighted\" average\n",
    "  model_precision, model_recall, model_f1, _ = precision_recall_fscore_support(y_true, y_pred, average=\"weighted\")\n",
    "  model_results = {\"accuracy\": model_accuracy,\n",
    "                  \"precision\": model_precision,\n",
    "                  \"recall\": model_recall,\n",
    "                  \"f1\": model_f1}\n",
    "  return model_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "c5c3fd97",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:36.730938Z",
     "iopub.status.busy": "2023-12-26T07:16:36.729727Z",
     "iopub.status.idle": "2023-12-26T07:16:36.742318Z",
     "shell.execute_reply": "2023-12-26T07:16:36.740894Z"
    },
    "papermill": {
     "duration": 0.308608,
     "end_time": "2023-12-26T07:16:36.745012",
     "exception": false,
     "start_time": "2023-12-26T07:16:36.436404",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 79.00262467191601,\n",
       " 'precision': 0.7895463977631777,\n",
       " 'recall': 0.7900262467191601,\n",
       " 'f1': 0.7891155349992447}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ensemble_results = calculate_results(y_valid, combined_preds)\n",
    "ensemble_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "d1ceb7f4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:37.356796Z",
     "iopub.status.busy": "2023-12-26T07:16:37.356100Z",
     "iopub.status.idle": "2023-12-26T07:16:37.366070Z",
     "shell.execute_reply": "2023-12-26T07:16:37.365303Z"
    },
    "papermill": {
     "duration": 0.319025,
     "end_time": "2023-12-26T07:16:37.368077",
     "exception": false,
     "start_time": "2023-12-26T07:16:37.049052",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 76.24671916010499,\n",
       " 'precision': 0.7623943018867197,\n",
       " 'recall': 0.7624671916010499,\n",
       " 'f1': 0.762429426257091}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_7_results = calculate_results(y_valid, tf.squeeze(tf.round(model_7_preds), axis=1))\n",
    "model_7_results"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90ce8ac2",
   "metadata": {
    "papermill": {
     "duration": 0.288822,
     "end_time": "2023-12-26T07:16:37.953185",
     "exception": false,
     "start_time": "2023-12-26T07:16:37.664363",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Try submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "6a785a6f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:38.625710Z",
     "iopub.status.busy": "2023-12-26T07:16:38.625114Z",
     "iopub.status.idle": "2023-12-26T07:16:41.715184Z",
     "shell.execute_reply": "2023-12-26T07:16:41.713307Z"
    },
    "papermill": {
     "duration": 3.4063,
     "end_time": "2023-12-26T07:16:41.717948",
     "exception": false,
     "start_time": "2023-12-26T07:16:38.311648",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 3s 3s/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.98391265]], dtype=float32)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_7.predict([\"There is a tornado in my area\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "a2f6dcf1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:42.312079Z",
     "iopub.status.busy": "2023-12-26T07:16:42.311673Z",
     "iopub.status.idle": "2023-12-26T07:16:42.320530Z",
     "shell.execute_reply": "2023-12-26T07:16:42.318892Z"
    },
    "papermill": {
     "duration": 0.310096,
     "end_time": "2023-12-26T07:16:42.322582",
     "exception": false,
     "start_time": "2023-12-26T07:16:42.012486",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                      Just happened a terrible car crash\n",
       "1       Heard about #earthquake is different cities, s...\n",
       "2       there is a forest fire at spot pond, geese are...\n",
       "3                Apocalypse lighting. #Spokane #wildfires\n",
       "4           Typhoon Soudelor kills 28 in China and Taiwan\n",
       "                              ...                        \n",
       "3258    EARTHQUAKE SAFETY LOS ANGELES ÛÒ SAFETY FASTE...\n",
       "3259    Storm in RI worse than last hurricane. My city...\n",
       "3260    Green Line derailment in Chicago http://t.co/U...\n",
       "3261    MEG issues Hazardous Weather Outlook (HWO) htt...\n",
       "3262    #CityofCalgary has activated its Municipal Eme...\n",
       "Name: text, Length: 3263, dtype: object"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_text_df = test_df[\"text\"]\n",
    "test_text_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "28723bbd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T06:55:49.002280Z",
     "iopub.status.busy": "2023-12-26T06:55:49.001879Z",
     "iopub.status.idle": "2023-12-26T06:55:51.624073Z",
     "shell.execute_reply": "2023-12-26T06:55:51.622646Z",
     "shell.execute_reply.started": "2023-12-26T06:55:49.002249Z"
    },
    "papermill": {
     "duration": 0.291002,
     "end_time": "2023-12-26T07:16:42.909254",
     "exception": false,
     "start_time": "2023-12-26T07:16:42.618252",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "d0fd97d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:43.560757Z",
     "iopub.status.busy": "2023-12-26T07:16:43.560374Z",
     "iopub.status.idle": "2023-12-26T07:16:46.996098Z",
     "shell.execute_reply": "2023-12-26T07:16:46.994423Z"
    },
    "papermill": {
     "duration": 3.799133,
     "end_time": "2023-12-26T07:16:46.999246",
     "exception": false,
     "start_time": "2023-12-26T07:16:43.200113",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "102/102 [==============================] - 0s 2ms/step\n",
      "102/102 [==============================] - 1s 7ms/step\n",
      "102/102 [==============================] - 2s 7ms/step\n"
     ]
    }
   ],
   "source": [
    "# use the results from model_1, model_2, model_7 \n",
    "model_1_preds_test = model_1.predict(test_text_df)\n",
    "model_2_preds_test = model_2.predict(test_text_df)\n",
    "model_3_preds_test = model_3.predict(test_text_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "0d10ce6f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:47.601532Z",
     "iopub.status.busy": "2023-12-26T07:16:47.601202Z",
     "iopub.status.idle": "2023-12-26T07:16:47.612851Z",
     "shell.execute_reply": "2023-12-26T07:16:47.611363Z"
    },
    "papermill": {
     "duration": 0.308293,
     "end_time": "2023-12-26T07:16:47.615588",
     "exception": false,
     "start_time": "2023-12-26T07:16:47.307295",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3263,), dtype=float32, numpy=array([1., 0., 1., ..., 1., 1., 1.], dtype=float32)>"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get mean pred probs for 3 models\n",
    "combined_pred_probs = tf.squeeze(model_1_preds_test, axis=1) + tf.squeeze(model_2_preds_test, axis=1) + tf.squeeze(model_3_preds_test, axis=1)\n",
    "combined_preds = tf.round(combined_pred_probs/3)\n",
    "combined_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "be6e0909",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:48.209785Z",
     "iopub.status.busy": "2023-12-26T07:16:48.209245Z",
     "iopub.status.idle": "2023-12-26T07:16:48.214958Z",
     "shell.execute_reply": "2023-12-26T07:16:48.213395Z"
    },
    "papermill": {
     "duration": 0.306289,
     "end_time": "2023-12-26T07:16:48.217834",
     "exception": false,
     "start_time": "2023-12-26T07:16:47.911545",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission_df = pd.DataFrame({\n",
    "    \"id\": test_df[\"id\"],\n",
    "    \"target\": combined_preds\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "62b4afbf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:48.883559Z",
     "iopub.status.busy": "2023-12-26T07:16:48.883149Z",
     "iopub.status.idle": "2023-12-26T07:16:48.897767Z",
     "shell.execute_reply": "2023-12-26T07:16:48.896131Z"
    },
    "papermill": {
     "duration": 0.386553,
     "end_time": "2023-12-26T07:16:48.900412",
     "exception": false,
     "start_time": "2023-12-26T07:16:48.513859",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3258</th>\n",
       "      <td>10861</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3259</th>\n",
       "      <td>10865</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3260</th>\n",
       "      <td>10868</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3261</th>\n",
       "      <td>10874</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3262</th>\n",
       "      <td>10875</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3263 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id  target\n",
       "0         0     1.0\n",
       "1         2     0.0\n",
       "2         3     1.0\n",
       "3         9     1.0\n",
       "4        11     1.0\n",
       "...     ...     ...\n",
       "3258  10861     1.0\n",
       "3259  10865     1.0\n",
       "3260  10868     1.0\n",
       "3261  10874     1.0\n",
       "3262  10875     1.0\n",
       "\n",
       "[3263 rows x 2 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "009bdc11",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-26T07:16:49.488669Z",
     "iopub.status.busy": "2023-12-26T07:16:49.487196Z",
     "iopub.status.idle": "2023-12-26T07:16:49.504330Z",
     "shell.execute_reply": "2023-12-26T07:16:49.503179Z"
    },
    "papermill": {
     "duration": 0.313075,
     "end_time": "2023-12-26T07:16:49.506612",
     "exception": false,
     "start_time": "2023-12-26T07:16:49.193537",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission_df.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "databundleVersionId": 869809,
     "sourceId": 17777,
     "sourceType": "competition"
    },
    {
     "isSourceIdPinned": true,
     "modelInstanceId": 1844,
     "sourceId": 2529,
     "sourceType": "modelInstanceVersion"
    },
    {
     "isSourceIdPinned": true,
     "modelInstanceId": 4689,
     "sourceId": 5916,
     "sourceType": "modelInstanceVersion"
    }
   ],
   "dockerImageVersionId": 30626,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 484.677632,
   "end_time": "2023-12-26T07:16:52.851681",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-12-26T07:08:48.174049",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}