fine_tune_LM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[WindowsPath('.ipynb_checkpoints'),\n",
       " WindowsPath('build_arabic_language_model.ipynb'),\n",
       " WindowsPath('data'),\n",
       " WindowsPath('fine_tune_LM.ipynb'),\n",
       " WindowsPath('models')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = Path('.')\n",
    "path.ls()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## prepare data for MADAMIRA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(؟:\\]÷»“>\\-؛×\\`$–~\"…=¦%'&;|,?@*«<_!.#}،”\\[)\\^/ـ{\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "\n",
    "#preparing punctuations list\n",
    "arabic_punctuations = '''`÷×؛<>_()*&^%][ـ،/:\"؟.,'{}~¦+|!”…“–ـ«»'''\n",
    "english_punctuations = string.punctuation\n",
    "punc_to_remove = ''.join(set(arabic_punctuations + english_punctuations))\n",
    "punc_to_keep = '+'\n",
    "punc_to_escape = '''[]-^'''\n",
    "for p in punc_to_keep: punc_to_remove = punc_to_remove.replace(p, '')\n",
    "for p in punc_to_escape: punc_to_remove = punc_to_remove.replace(p, '\\\\{}'.format(p))\n",
    "print(punc_to_remove)\n",
    "\n",
    "def pre_process(text):\n",
    "    text = text.replace('\\\\n', ' ').replace('\\n', ' ')\n",
    "    text = text.replace('؛', '،')\n",
    "    text = re.sub(r'\\([^)]+\\)', '', text)  # remove parentheses and everything in between\n",
    "    text = re.sub(r'[a-zA-Z]', '', text)  # remove non-arabic characters\n",
    "    text = re.sub(r'\\d+(\\.\\d+)?', ' رقم ', text)  # replace numbers by special token\n",
    "    for p in punc_to_remove: text = text.replace(p, '')  # remove punctuations\n",
    "    text = re.sub(r'(.)\\1{2,}', r'\\1', text)  # remove repeated chars\n",
    "    text = re.sub(r'\\s+', r' ', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__AJGT__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Feed</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>اربد فيها جامعات اكثر من عمان ... وفيها قد عم...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>الحلو انكم بتحكوا على اساس انو الاردن ما فيه ...</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>كله رائع بجد ربنا يكرمك</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>لسانك قذر يا قمامه</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>​انا داشره وغير متزوجه ولدي علاقات مشبوه واحشش...</td>\n",
       "      <td>Negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID                                               Feed Sentiment\n",
       "0   1   اربد فيها جامعات اكثر من عمان ... وفيها قد عم...  Positive\n",
       "1   2   الحلو انكم بتحكوا على اساس انو الاردن ما فيه ...  Negative\n",
       "2   3                            كله رائع بجد ربنا يكرمك  Positive\n",
       "3   4                                 لسانك قذر يا قمامه  Negative\n",
       "4   5  ​انا داشره وغير متزوجه ولدي علاقات مشبوه واحشش...  Negative"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_excel(path/'data'/'AJGT.xlsx')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.Feed.to_csv(path/'data'/'AJGT_text_only.txt', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MADAMIRA:\n",
    "`\n",
    "C:\\Users\\oae15\\Downloads\\JDK64\\bin\\java.exe -Xmx3000m -Xms3000m -XX:NewRatio=3 -jar MADAMIRA-release-20170403-2.1.jar -rawinput \"C:\\Python\\Python36\\coding4fun\\Notebooks\\fastai\\for research\\Ar_LM\\data\\AJGT_text_only.txt\" -rawoutdir .\\samples\\output -rawconfig .\\samples\\sampleConfigFile.xml -msaonly\n",
    "`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>اربد  في +ها جامعات  اكثر  من  عمان  .  .  . ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ال+ حلو  ان +كم بتحكوا  علي  اساس  انو  ال+ ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>كل +ه راءع  ب+ جد  رب +نا يكرم +ك</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>لسان +ك قذر  يا  قمامه</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>انا  داشره  و+ غير  متزوجه  و+ لدي  علاقات  م...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      processed_text\n",
       "0   اربد  في +ها جامعات  اكثر  من  عمان  .  .  . ...\n",
       "1   ال+ حلو  ان +كم بتحكوا  علي  اساس  انو  ال+ ا...\n",
       "2                  كل +ه راءع  ب+ جد  رب +نا يكرم +ك\n",
       "3                             لسان +ك قذر  يا  قمامه\n",
       "4   انا  داشره  و+ غير  متزوجه  و+ لدي  علاقات  م..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mada = pd.read_csv(path/'data'/'AJGT_text_only_processed.txt', sep='\\t', header=None, names=['processed_text'])\n",
    "df_mada.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(columns=['ID'], inplace=True)\n",
    "df.Feed = df_mada.processed_text\n",
    "df.to_csv(path/'data'/'AJGT_final.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__ArTwitter__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>txt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>انشاء الله هنعمل حاجه</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>اقسم باللله ان العرب اكثر الشعوب تخلفاا</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>﻿هات ناس تفهم .. و المثل بحكي اسأل مجرب و لا ت...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>صرماتي براس اهلك</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>حرام السخرية من الناس</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                                txt\n",
       "0          1                              انشاء الله هنعمل حاجه\n",
       "1          0            اقسم باللله ان العرب اكثر الشعوب تخلفاا\n",
       "2          0  ﻿هات ناس تفهم .. و المثل بحكي اسأل مجرب و لا ت...\n",
       "3          0                                   صرماتي براس اهلك\n",
       "4          0                              حرام السخرية من الناس"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(path/'data'/'artwitter.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    1771\n",
       "0    1771\n",
       "Name: sentiment, dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sentiment.value_counts() # balanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.txt.to_csv(path/'data'/'artwitter_text_only.txt', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MADAMIRA..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>انشاء  الله  هنعمل  حاج +ه</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>اقسم  بال +ه ان  ال+ عرب  اكثر  ال+ شعوب  تخلفا</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>هات  ناس  تفهم  .  .  و  ال+ مثل  ب+ حك +ي اس...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>صرماتي  ب+ راس  اهل +ك</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>حرام  ال+ سخريه  من  ال+ ناس</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      processed_text\n",
       "0                         انشاء  الله  هنعمل  حاج +ه\n",
       "1    اقسم  بال +ه ان  ال+ عرب  اكثر  ال+ شعوب  تخلفا\n",
       "2   هات  ناس  تفهم  .  .  و  ال+ مثل  ب+ حك +ي اس...\n",
       "3                             صرماتي  ب+ راس  اهل +ك\n",
       "4                       حرام  ال+ سخريه  من  ال+ ناس"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mada = pd.read_csv(path/'data'/'artwitter_text_only_processed.txt', sep='\\t', header=None, names=['processed_text'])\n",
    "df_mada.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.txt = df_mada.processed_text\n",
    "df.to_csv(path/'data'/'ArTwitter_final.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>txt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>انشاء  الله  هنعمل  حاج +ه</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>اقسم  بال +ه ان  ال+ عرب  اكثر  ال+ شعوب  تخلفا</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>هات  ناس  تفهم  .  .  و  ال+ مثل  ب+ حك +ي اس...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>صرماتي  ب+ راس  اهل +ك</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>حرام  ال+ سخريه  من  ال+ ناس</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                                txt\n",
       "0          1                         انشاء  الله  هنعمل  حاج +ه\n",
       "1          0    اقسم  بال +ه ان  ال+ عرب  اكثر  ال+ شعوب  تخلفا\n",
       "2          0   هات  ناس  تفهم  .  .  و  ال+ مثل  ب+ حك +ي اس...\n",
       "3          0                             صرماتي  ب+ راس  اهل +ك\n",
       "4          0                       حرام  ال+ سخريه  من  ال+ ناس"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__ASTD__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>OBJ</td>\n",
       "      <td>بعد استقالة رئيس #المحكمة_الدستورية ننتظر استق...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>POS</td>\n",
       "      <td>أهنئ الدكتور أحمد جمال الدين، القيادي بحزب مصر...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NEG</td>\n",
       "      <td>البرادعي يستقوى بامريكا مرةاخرى و يرسل عصام ال...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OBJ</td>\n",
       "      <td>#الحرية_والعدالة | شاهد الآن: #ليلة_الاتحادية ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NEUTRAL</td>\n",
       "      <td>الوالدة لو اقولها بخاطري حشيشة تضحك بس من اقول...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  category                                               text\n",
       "0      OBJ  بعد استقالة رئيس #المحكمة_الدستورية ننتظر استق...\n",
       "1      POS  أهنئ الدكتور أحمد جمال الدين، القيادي بحزب مصر...\n",
       "2      NEG  البرادعي يستقوى بامريكا مرةاخرى و يرسل عصام ال...\n",
       "3      OBJ  #الحرية_والعدالة | شاهد الآن: #ليلة_الاتحادية ...\n",
       "4  NEUTRAL  الوالدة لو اقولها بخاطري حشيشة تضحك بس من اقول..."
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(path/'data'/'ASTD.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OBJ        6691\n",
       "NEG        1684\n",
       "NEUTRAL     832\n",
       "POS         799\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.category.value_counts() # unbalanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-2</td>\n",
       "      <td>بعد استقالة رئيس المحكمة الدستورية ننتظر استقا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>أهنئ الدكتور أحمد جمال الدين القيادي بحزب مصر ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-1</td>\n",
       "      <td>البرادعي يستقوى بامريكا مرةاخرى و يرسل عصام ال...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-2</td>\n",
       "      <td>الحرية والعدالة شاهد الآن ليلة الاتحادية أول ف...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>الوالدة لو اقولها بخاطري حشيشة تضحك بس من اقول...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   category                                               text\n",
       "0        -2  بعد استقالة رئيس المحكمة الدستورية ننتظر استقا...\n",
       "1         1  أهنئ الدكتور أحمد جمال الدين القيادي بحزب مصر ...\n",
       "2        -1  البرادعي يستقوى بامريكا مرةاخرى و يرسل عصام ال...\n",
       "3        -2  الحرية والعدالة شاهد الآن ليلة الاتحادية أول ف...\n",
       "4         0  الوالدة لو اقولها بخاطري حشيشة تضحك بس من اقول..."
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.category = df.category.replace({'NEUTRAL':0, 'POS':1, 'NEG':-1, 'OBJ':-2})\n",
    "df.text = df.text.apply(lambda x: pre_process(x.replace('_',' ')))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.text.to_csv(path/'data'/'ASTD_text_only.txt', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MADAMIRA..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>بعد  استقاله  رءيس  ال+ محكمه  ال+ دستوريه  ن...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>اهنء  ال+ دكتور  احمد  جمال  ال+ دين  ال+ قيا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ال+ برادعي  يستقوي  ب+ امريكا  مره  اخري  و  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ال+ حريه  و+ ال+ عداله  شاهد  ال+ ان  ليله  ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ال+ والده  لو  اقول +ها ب+ خاطري  حشيشه  تضحك...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      processed_text\n",
       "0   بعد  استقاله  رءيس  ال+ محكمه  ال+ دستوريه  ن...\n",
       "1   اهنء  ال+ دكتور  احمد  جمال  ال+ دين  ال+ قيا...\n",
       "2   ال+ برادعي  يستقوي  ب+ امريكا  مره  اخري  و  ...\n",
       "3   ال+ حريه  و+ ال+ عداله  شاهد  ال+ ان  ليله  ا...\n",
       "4   ال+ والده  لو  اقول +ها ب+ خاطري  حشيشه  تضحك..."
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mada = pd.read_csv(path/'data'/'ASTD_text_only_processed.txt', sep='\\t', header=None, names=['processed_text'])\n",
    "df_mada.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.text = df_mada.processed_text\n",
    "df.to_csv(path/'data'/'ASTD_final.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__ASTD-B__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>اهنئ الدكتور احمد جمال الدين القيادي بحزب مصر ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>امير عيد هو اللي فعلا يتقال عليه ستريكر صريح ك...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>الصداقه تزرع الحياه ازهارا مي زياده</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>دضياء رشوان ان الدكتور عبد المنعم ابو الفتوح ا...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>الى زملائي المحامين الراغبين في الانضمام لمباد...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  polarity\n",
       "0  اهنئ الدكتور احمد جمال الدين القيادي بحزب مصر ...         1\n",
       "1  امير عيد هو اللي فعلا يتقال عليه ستريكر صريح ك...         1\n",
       "2                الصداقه تزرع الحياه ازهارا مي زياده         1\n",
       "3  دضياء رشوان ان الدكتور عبد المنعم ابو الفتوح ا...         1\n",
       "4  الى زملائي المحامين الراغبين في الانضمام لمباد...         1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(path/'data'/'ASTD-B-utf8.csv', encoding='Windows-1256')\n",
    "df.head()\n",
    "\n",
    "#with open(path/'data'/'ASTD-B.csv', encoding='Windows-1256') as file:\n",
    "#    print(file.readlines()[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 1    665\n",
       "-1    665\n",
       "Name: polarity, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.polarity.value_counts() # balanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.text.to_csv(path/'data'/'ASTD-B_text_only.txt', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MADAMIRA..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>اهنء  ال+ دكتور  احمد  جمال  ال+ دين  ال+ قيا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>امير  عيد  هو  ال+ لي  فعلا  يتقال  علي +ه ست...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ال+ صداقه  تزرع  ال+ حياه  ازهارا  مي  زياده</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>دضياء  رشوان  ان  ال+ دكتور  عبد  ال+ منعم  ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>الي  زملاء +ي ال+ محامين  ال+ راغبين  في  ال+...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      processed_text\n",
       "0   اهنء  ال+ دكتور  احمد  جمال  ال+ دين  ال+ قيا...\n",
       "1   امير  عيد  هو  ال+ لي  فعلا  يتقال  علي +ه ست...\n",
       "2       ال+ صداقه  تزرع  ال+ حياه  ازهارا  مي  زياده\n",
       "3   دضياء  رشوان  ان  ال+ دكتور  عبد  ال+ منعم  ا...\n",
       "4   الي  زملاء +ي ال+ محامين  ال+ راغبين  في  ال+..."
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mada = pd.read_csv(path/'data'/'ASTD-B_text_only_processed.txt', sep='\\t', header=None, names=['processed_text'])\n",
    "df_mada.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.text = df_mada.processed_text\n",
    "df.to_csv(path/'data'/'ASTD-B_final.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__ArSenTD-LEV__\n",
    " - remove punctuations (except +)\n",
    " - replace numbers by رقم\n",
    " - remove consecutive spaces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tweet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>انا اؤمن بان الانسان ينطفئ جماله عند ابتعاد م...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>من الذاكره. . عندما اعتقد كريستيانو انه ثاني ا...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>لا نخلو من ضغوطات الحياه. فنحن نعيش علي ارض اع...</td>\n",
       "      <td>neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>مصطلحات لبنانيه حيرت البشريه بتوصل عالبيت ، بن...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>نصمت !! لتسير حياتنا علي م يرام فالناس لم تعد ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Tweet     label\n",
       "0   انا اؤمن بان الانسان ينطفئ جماله عند ابتعاد م...  negative\n",
       "1  من الذاكره. . عندما اعتقد كريستيانو انه ثاني ا...  positive\n",
       "2  لا نخلو من ضغوطات الحياه. فنحن نعيش علي ارض اع...   neutral\n",
       "3  مصطلحات لبنانيه حيرت البشريه بتوصل عالبيت ، بن...  negative\n",
       "4  نصمت !! لتسير حياتنا علي م يرام فالناس لم تعد ...  negative"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(path/'data'/'ArSenTD-LEV-processed-no-emojis2.csv')  # , encoding='Windows-1256'\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "negative         1253\n",
       "neutral           885\n",
       "positive          835\n",
       "very_negative     630\n",
       "very_positive     397\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.label.value_counts() # unbalanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.Tweet.to_csv(path/'data'/'ArSenTD-LEV-no-emojis-text-only.txt', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MADAMIRA..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>انا  اءمن  ب+ ان  ال+ انسان  ينطفء  جمال +ه ع...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>من  ال+ ذاكره  .  .  عندما +ما اعتقد  كريستيا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>لا  نخلو  من  ضغوطات  ال+ حياه  .  ف+ نحن  نع...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"  مصطلحات  لبناني +ه حيرت  ال+ بشريه  ب+ توص...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>نصمت  !  !  ل+ تسير  حياه +نا علي  م  يرام  ف...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      processed_text\n",
       "0   انا  اءمن  ب+ ان  ال+ انسان  ينطفء  جمال +ه ع...\n",
       "1   من  ال+ ذاكره  .  .  عندما +ما اعتقد  كريستيا...\n",
       "2   لا  نخلو  من  ضغوطات  ال+ حياه  .  ف+ نحن  نع...\n",
       "3   \"  مصطلحات  لبناني +ه حيرت  ال+ بشريه  ب+ توص...\n",
       "4   نصمت  !  !  ل+ تسير  حياه +نا علي  م  يرام  ف..."
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mada = pd.read_csv(path/'data'/'ArSenTD-LEV-no-emojis-text-only-processed.txt', sep='\\t', header=None, names=['processed_text'])\n",
    "df_mada.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.Tweet = df_mada.processed_text\n",
    "df.to_csv(path/'data'/'ArSenTD-LEV-final.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "__HARD__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>no</th>\n",
       "      <th>Hotel name</th>\n",
       "      <th>rating</th>\n",
       "      <th>user type</th>\n",
       "      <th>room type</th>\n",
       "      <th>nights</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>فندق 72</td>\n",
       "      <td>2</td>\n",
       "      <td>مسافر منفرد</td>\n",
       "      <td>غرفة ديلوكس مزدوجة أو توأم</td>\n",
       "      <td>أقمت ليلة واحدة</td>\n",
       "      <td>“ممتاز”. النظافة والطاقم متعاون.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>فندق 72</td>\n",
       "      <td>5</td>\n",
       "      <td>زوج</td>\n",
       "      <td>غرفة ديلوكس مزدوجة أو توأم</td>\n",
       "      <td>أقمت ليلة واحدة</td>\n",
       "      <td>استثنائي. سهولة إنهاء المعاملة في الاستقبال. ل...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16</td>\n",
       "      <td>فندق 72</td>\n",
       "      <td>5</td>\n",
       "      <td>زوج</td>\n",
       "      <td>-</td>\n",
       "      <td>أقمت ليلتين</td>\n",
       "      <td>استثنائي. انصح بأختيار الاسويت و بالاخص غرفه ر...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20</td>\n",
       "      <td>فندق 72</td>\n",
       "      <td>1</td>\n",
       "      <td>زوج</td>\n",
       "      <td>غرفة قياسية مزدوجة</td>\n",
       "      <td>أقمت ليلة واحدة</td>\n",
       "      <td>“استغرب تقييم الفندق كخمس نجوم”. لا شي. يستحق ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23</td>\n",
       "      <td>فندق 72</td>\n",
       "      <td>4</td>\n",
       "      <td>زوج</td>\n",
       "      <td>غرفة ديلوكس مزدوجة أو توأم</td>\n",
       "      <td>أقمت ليلتين</td>\n",
       "      <td>جيد. المكان جميل وهاديء. كل شي جيد ونظيف بس كا...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   no Hotel name  rating    user type                   room type  \\\n",
       "0   2    فندق 72       2  مسافر منفرد  غرفة ديلوكس مزدوجة أو توأم   \n",
       "1   3    فندق 72       5          زوج  غرفة ديلوكس مزدوجة أو توأم   \n",
       "2  16    فندق 72       5          زوج                           -   \n",
       "3  20    فندق 72       1          زوج          غرفة قياسية مزدوجة   \n",
       "4  23    فندق 72       4          زوج  غرفة ديلوكس مزدوجة أو توأم   \n",
       "\n",
       "            nights                                             review  \n",
       "0  أقمت ليلة واحدة                  “ممتاز”. النظافة والطاقم متعاون.   \n",
       "1  أقمت ليلة واحدة  استثنائي. سهولة إنهاء المعاملة في الاستقبال. ل...  \n",
       "2      أقمت ليلتين  استثنائي. انصح بأختيار الاسويت و بالاخص غرفه ر...  \n",
       "3  أقمت ليلة واحدة  “استغرب تقييم الفندق كخمس نجوم”. لا شي. يستحق ...  \n",
       "4      أقمت ليلتين  جيد. المكان جميل وهاديء. كل شي جيد ونظيف بس كا...  "
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(path/'data'/'HARD-balanced-reviews-utf8.tsv', sep='\\t', encoding='utf-8')  # , encoding='Windows-1256'\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-1    52849\n",
      " 1    52849\n",
      "Name: rating, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "      <th>review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1</td>\n",
       "      <td>ممتاز النظافة والطاقم متعاون</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>استثنائي سهولة إنهاء المعاملة في الاستقبال لاشيئ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>استثنائي انصح بأختيار الاسويت و بالاخص غرفه رق...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-1</td>\n",
       "      <td>استغرب تقييم الفندق كخمس نجوم لا شي يستحق رقم ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>جيد المكان جميل وهاديء كل شي جيد ونظيف بس كان ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   rating                                             review\n",
       "0      -1                      ممتاز النظافة والطاقم متعاون \n",
       "1       1   استثنائي سهولة إنهاء المعاملة في الاستقبال لاشيئ\n",
       "2       1  استثنائي انصح بأختيار الاسويت و بالاخص غرفه رق...\n",
       "3      -1  استغرب تقييم الفندق كخمس نجوم لا شي يستحق رقم ...\n",
       "4       1  جيد المكان جميل وهاديء كل شي جيد ونظيف بس كان ..."
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[['rating', 'review']] # we are interested in rating and review only\n",
    "df.rating = df.rating.apply(lambda x: 1 if x>3 else -1)\n",
    "df.review = df.review.apply(lambda x: pre_process(x))\n",
    "print(df.rating.value_counts())\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.review.to_csv(path/'data'/'HARD-text-only.txt', header=False, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "MADAMIRA..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ممتاز  ال+ نظافه  و+ ال+ طاقم  متعاون</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>استثناءي  سهوله  انهاء  ال+ معامله  في  ال+ ا...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>استثناءي  انصح  ب+ اختيار  الاسويت  و  ب+ ال+...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>استغرب  تقييم  ال+ فندق  ك+ خمس  نجوم  لا  شي...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>جيد  ال+ مكان  جميل  وهاديء  كل  شي  جيد  و+ ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      processed_text\n",
       "0              ممتاز  ال+ نظافه  و+ ال+ طاقم  متعاون\n",
       "1   استثناءي  سهوله  انهاء  ال+ معامله  في  ال+ ا...\n",
       "2   استثناءي  انصح  ب+ اختيار  الاسويت  و  ب+ ال+...\n",
       "3   استغرب  تقييم  ال+ فندق  ك+ خمس  نجوم  لا  شي...\n",
       "4   جيد  ال+ مكان  جميل  وهاديء  كل  شي  جيد  و+ ..."
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mada = pd.read_csv(path/'data'/'HARD_text_only_processed.txt', sep='|', header=None, names=['processed_text'])\n",
    "df_mada.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(105698, 1)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mada.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "processed_text    NaN\n",
       "Name: 1012, dtype: object"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mada.iloc[1012]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "processed_text    53\n",
       "dtype: int64"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mada.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.review = df_mada.processed_text\n",
    "df.to_csv(path/'data'/'HARD-final.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine-Tune pre-trained LM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrained_fnames=['Ar-LM-epoch1-acc43','itos']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10006, 2)\n",
      "(8004, 2) (2002, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7862</th>\n",
       "      <td>-2</td>\n",
       "      <td>اجلاء  اكثر  من  الفي  شخص  تحسبا  ل+ اعصار  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1095</th>\n",
       "      <td>-2</td>\n",
       "      <td>ل+ ال+ تخلص  من  مرسي  غاندي  هو  ال+ حل</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3123</th>\n",
       "      <td>0</td>\n",
       "      <td>رقم  يا  اخ +ي اي  منصب  رفضت  ال+ مناصب  ال+...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8503</th>\n",
       "      <td>1</td>\n",
       "      <td>وحدوا  الله  يا  جماعه  و+ بلاش  اشاعات  عن  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1561</th>\n",
       "      <td>-1</td>\n",
       "      <td>ب+ ال+ فيديو  عكاشه  و+ الله  هيفاء  و+ هبي  ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      category                                               text\n",
       "7862        -2   اجلاء  اكثر  من  الفي  شخص  تحسبا  ل+ اعصار  ...\n",
       "1095        -2           ل+ ال+ تخلص  من  مرسي  غاندي  هو  ال+ حل\n",
       "3123         0   رقم  يا  اخ +ي اي  منصب  رفضت  ال+ مناصب  ال+...\n",
       "8503         1   وحدوا  الله  يا  جماعه  و+ بلاش  اشاعات  عن  ...\n",
       "1561        -1   ب+ ال+ فيديو  عكاشه  و+ الله  هيفاء  و+ هبي  ..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "df = pd.read_csv(path/'data'/'ASTD_final.csv')\n",
    "print(df.shape)\n",
    "#df.dropna(inplace=True)\n",
    "#print(df.shape)\n",
    "#df.Tweet = df.Tweet.apply(lambda x: pre_process(x))\n",
    "#df.label.replace({'very_positive':2, 'positive':1, 'neutral':0, 'negative':-1, 'very_negative':-2}, inplace=True)\n",
    "np.random.seed(42)\n",
    "df_train, df_val = train_test_split(df, test_size=0.2)\n",
    "print(df_train.shape, df_val.shape)\n",
    "#df.columns\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-2    5362\n",
       "-1    1340\n",
       " 0     670\n",
       " 1     632\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.category.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-2    1329\n",
       "-1     344\n",
       " 1     167\n",
       " 0     162\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.category.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 8004 entries, 7862 to 7270\n",
      "Data columns (total 2 columns):\n",
      "category    8004 non-null int64\n",
      "text        8004 non-null object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 187.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "category    0\n",
       "text        0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "ar_tok = Tokenizer(lang='ar')\n",
    "#data_lm = TextLMDataBunch.from_df(path, train_df=df_train, valid_df=df_val, text_cols=0, label_cols=None, tokenizer=ar_tok, bs=32)\n",
    "data_lm = TextLMDataBunch.from_df(path, train_df=df_train, valid_df=df_val, text_cols=1, label_cols=None, tokenizer=ar_tok, bs=64, include_bos=False, min_freq=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LMTextList (1197 items)\n",
       "  برلمان مشكوك في شرعيه + ه س+ ينتخب لجن + ه نصف + ها من + ه اي غير محايد + ه ل+ وضع دستور ل+ مصر و+ ليس ل+ ال+ اغلبيه ال+ دستور مصدر ال+ سلطات و+ ليس ال+ عكس المتاهه مستمره,  ال+ اعلام ال+ فاسد و+ فءران + ه اميركا تساند ال+ اخوان و+ ال+ يوم وفد من ال+ كونجرس يلتقي سيسي و+ ال+ طرطور و+ ب+ لاوي مجانين رسمي,  تدعون ل+ ال+ دين و+ تنشرون ال+ اكاذيب لم اكن ابدا عضوا ب+ ال+ وطني انتم تتاجرون ب+ ال+ دين,  انا ب+ حب ال+ رقاصات و+ هما فوق راس + ي انت نجس اصلا يا انتخبوا ال+ عرص,  مرسي محبوس وخاين و+ مش هيرجع و+ ال+ تصالح مرفوض ال+ ساده ال+ محترمون\n",
       "Path: ."
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_lm.train_ds.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>idx</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>انت ل+ ال+ دنيا ب+ ذات + ك فوق مالا يعقلون   اقبال ال+ مصريين علي ال+ تصويت في ال+ انتخابات ضمان + ه ل+ برلمان يعبر عن ال+ شعب و+ انتخاب ممثلين ل+ شباب ال+ ثوره ضمان + ه ل+ برلمان يستكمل مهام ثوره + نا   ميثاق ال+ شرف ال+ اعلامي الذي اتخذوا + ه ذريع + ه ل+ اغلاق الفراعين برء من ال+ انقلاب قتلوا</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>او استثناء دع ال+ كلاب تنبح   كرن ? ال دولي ب+ حضور ال+ ملك مبروك ل ال+ شباب بطل كاس ال+ ملك ول ال+ شعب ال+ سعودي ال+ شقيق ملعب ال+ جوهره نحب + ك ياملكنا عبدالله   نعم حضرت جلسه استماع واحد + ه ل+ ال+ دستور استمعوا ل + نا ب+ كل اهتمام ثم ضربوا ب+ ملاحظات + نا عرض ال+ حاءط لا ل+ ال+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>مصر اخر عاما و+ يطالب ب+ خضوع مصر ل+ ال+ جيش و+ استشاره + ه في اي حاج + ه تخص + ه   عبدالله شعب + ك يحبك ال+ ملك اسال من هو في ملك + ه مقيم و+ في جلال + ه عظيم و+ علي عباد + ه رحيم و+ ب+ كل شء عليم ان يشفي + ك   الي شريك شفيق في مشروع + هما</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>ه جنات و+ اي حاج + ه من صافيناز   مداخل + ه نجيب ساويرس مع الابراشيي و+ تعليق + ه علي استقاله دالبرادعي و+ احداث مصر   جيد ان تتنوع ال+ اراء حول ال+ سياسه و+ قضايا + ها و+ ضروري ان يتم ال+ تعبير عن هذا ال+ تنوع في ال+ اعلام ال+ تقليدي و+ شب   صور + ه ال+ شهيده رضا رمضان سن +</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>+ ني الي مهوس + ه ب + ها حولت ال+ بيت الي معرض ال+ كتاب ممتنه ل + ك جدا استاذي   ال+ انقلاب يريد قمع مليون مصري لو حاولت امريكا ب+ جبروت + ها قمع + هم ما نجحت شعب مصر ال+ عظيم س+ ينتصر و+ يدفع ال+ عالم ل+ احترام + ه   منع حازم عبد ال+ عظيم و+ ال+ سيد ال+ بدوي اكبر انصار</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_lm.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.2, pretrained_fnames=pretrained_fnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl03HW9//HnO5Nksm9tmqY7lLZQZCuBgkgBERQuiOCGekXQn1yuXNTr0eu9x3P0CsfrdhVwRUDxosK9yqKCgq0LUDYxLS3U0o1utEnbJM2+z8z798dM0hCSNm3znSV5Pc6ZMzPf72fm+850mnc+38/38/6YuyMiIgKQleoAREQkfSgpiIjIICUFEREZpKQgIiKDlBRERGSQkoKIiAxSUhARkUFKCiIiMkhJQUREBmWnOoDDNXXqVJ83b16qwxARySirVq1qdPfKQ7XLuKQwb948amtrUx2GiEhGMbMdY2mn00ciIjJISUFERAYpKYiIyCAlBRERGaSkICIig5QURERkkJKCiIgMUlIQEckAt/1xEys3NwR+HCUFEZE05+58989b+OvW/YEfS0lBRCTNdfVFicac4rzgi1AoKYiIpLm2nn4ASvJzAj+WkoKISJpr74kAqKcgIiLQ1p3oKeSppyAiMumppyAiIoMGxhSK1VMQEZG2RE+hJF89BRGRSa+9R2MKIiKS0N4TISdkhLOD/5WtpCAikubauvspycvBzAI/VqBJwczKzOwBM9tgZq+Y2dnD9puZfcfMtpjZS2a2JMh4REQyUXtPJClXHgEEfZTbgcfd/T1mlgsUDNt/CbAgcVsK/DBxLyIiCW09/UmZzQwB9hTMrARYBvwYwN373L1lWLMrgHs97nmgzMyqg4pJRCQTJbOnEOTpo2OBBuAeM3vRzO42s8JhbWYCrw15viux7XXM7HozqzWz2oaG4EvHioikk/aeforDGd5TIH5qagnwQ3c/DegE/n1Ym5FGTfwNG9zvdPcad6+prKwc/0hFRNJYW3ckKXMUINiksAvY5e5/TTx/gHiSGN5m9pDns4C6AGMSEck47T39SZnNDAEmBXffA7xmZosSmy4E1g9r9lvgmsRVSGcBre5eH1RMIiKZJhKN0dkXTcrENQj+6qObgF8krjzaClxnZjcAuPsdwO+BS4EtQBdwXcDxiIhklI7e5BXDg4CTgruvAWqGbb5jyH4HbgwyBhGRTJbMCqmgGc0iImmttTt5q66BkoKISFpTT0FERAa1JbFCKigpiIiktYGegpKCiIgMrqWg00ciIkJbt8YUREQkob2nn4LcENmh5Py6VlIQEUljyayQCkoKIiJpra2nP2mDzKCkICKS1tRTEBGRQclcdQ2UFERE0lq8p6CkICIiDKyloNNHIiJCYtU19RRERKSnP0pfNKaegoiIDC2Gp6QgIjLpDRbD09VHIiKS7LUUQElBRCRttXUndy0FUFIQEUlbB3oKSgoiIpNeW5LXUgAlBRGRtDWwwI4GmkVEhPaeCFkGhbmhpB1TSUFEJE21dfdTnJeDmSXtmEoKIiJpKtlls0FJQUQkbbUluUIqKCmIiKSt+Kpr6imIiAjJX0sBAk4KZrbdzF42szVmVjvC/lIze8TM1prZ383suiDjERHJJG3dye8pJONoF7h74yj7bgTWu/vlZlYJbDSzX7h7XxLiEhFJa+1JXooTUn/6yIFii19vVQTsByKpDUlEJPViMaejd+JdfeTAcjNbZWbXj7D/e8AJQB3wMvApd48Nb2Rm15tZrZnVNjQ0BBuxiEga6OyLEPPkFsOD4JPCOe6+BLgEuNHMlg3b/3ZgDTADOBX4npmVDH8Td7/T3WvcvaaysjLgkEVEUi8VZbMh4KTg7nWJ+33Aw8CZw5pcBzzkcVuAbcDxQcYkIpIJDhTDmyA9BTMrNLPigcfAxcC6Yc12Ahcm2lQBi4CtQcUkIpIpDqy6NnGuPqoCHk7U7MgG7nP3x83sBgB3vwO4Bfipmb0MGPD5g1ypJCIyabSnqKcQWFJw963AKSNsv2PI4zriPQgRERmirTvRU5hIYwoiInJkUtVTUFIQEUlDbRPx6iMRETkybT395GZnkZeTvAV2QElBRCQttXVHkj6eAEoKIiJpqbmzj/KC3KQfV0lBRCQN7e/so6JQSUFERICmzl6mFCkpiIgI0NzVr9NHIiIC0ZjT3NXHFJ0+EhGRlq4+3NGYgoiIQHNXfPHJciUFERFp6ognhSmF4aQfW0lBRCTN7O+MJwWdPhIREfZ3KSmIiEjC/o6BMYXkVkgFJQURkbTT1NlHcTibcHZyi+GBkoKISNpp7upLyZVHoKQgIpJ2UlX3CJQURETSTlNHamYzg5KCiEja0ekjEREBwN1p6lRPQUREgM6+KH2RmMYUREQkvuIapGbiGigpiIiklSYlBRERGbC/sxdQUhAREWB/Zz+QmgqpoKQgIpJWBnoKqah7BEoKIiJppamzj9xQFkXh7JQcP9Cjmtl2oB2IAhF3rxmhzfnAbUAO0Oju5wUZk4hIOmtOlLgws5QcPxmp6AJ3bxxph5mVAT8A3uHuO81sWhLiERFJW/s7UzebGVJ/+uiDwEPuvhPA3felOB4RkZRK5WxmCD4pOLDczFaZ2fUj7F8IlJvZE4k21wQcj4hIWmtOYYVUCP700TnuXpc4LbTCzDa4+1PDjn86cCGQDzxnZs+7+6ahb5JIKNcDzJkzJ+CQRURSpynFSWFMPQUzm29m4cTj883sk4nxgINy97rE/T7gYeDMYU12AY+7e2di3OEp4JQR3udOd69x95rKysqxhCwiknH6IjHaeyLpnxSAB4GomR0H/Bg4BrjvYC8ws0IzKx54DFwMrBvW7DfAuWaWbWYFwFLglcOIX0RkwmjpSm2JCxj76aOYu0fM7ErgNnf/rpm9eIjXVAEPJy6rygbuc/fHzewGAHe/w91fMbPHgZeAGHC3uw9PHCIik0Kq6x7B2JNCv5l9APgIcHli20Gn27n7VkY+FXTHsOffBL45xjhERCas/WmQFMZ6+ug64GzgK+6+zcyOAX4eXFgiIpPPQFJI5SWpY+opuPt64JMAZlYOFLv714IMTERkshlICmk/eS0xj6DEzCqAtcA9ZvbtYEMTEZlcmjr7MIPygjRPCkCpu7cBVwH3uPvpwNuCC0tEZPJp7uyjLD+HUFZq6h7B2JNCtplVA+8DHg0wHhGRSSvVdY9g7EnhZuAPwKvu/jczOxbYHFxYIiKTT1Nnb0oHmWHsA82/An415PlW4N1BBSUiMhk1d/Yzb2pBSmMY60DzLDN72Mz2mdleM3vQzGYFHZyIyGSS6rpHMPbTR/cAvwVmADOBRxLbRERkHMRiTnNX5iSFSne/x90jidtPAVWmExEZJ+09EaIxp6IwnNI4xpoUGs3sH80slLj9I9AUZGAiIpNJU2cvABWFB60gFLixJoWPEr8cdQ9QD7yHeOkLEREZB40dAyUuMqCn4O473f2d7l7p7tPc/V3EJ7KJiMg4qG/tBqC6NC+lcRzNcpyfGbcoREQmufrWHgCqy/JTGsfRJIXUzcMWEZlg6lu6Kc7Lpigc9CrJB3c0ScHHLQoRkUmurrWHGaWp7SXAIWY0m1k7I//yNyD10YuITBD1rd1Ul6V2PAEOkRTcvThZgYiITGb1LT2cNLMs1WEc1ekjEREZBz39UZo6+5iR4iuPQElBRCTl9qTJlUegpCAiknJ1iTkK6imIiAj1LeopiIhIQrrMZgYlBRGRlKtr7aGiMJe8nFCqQ1FSEBFJtfqW7rToJYCSgohIytW39lCdBrOZQUlBRCTl6lq6mZEGs5lBSUFEJKU6eyO09UQmR0/BzLab2ctmtsbMag/S7gwzi5rZe4KMR0Qk3QxceZQuPYVk1Gi9wN0bR9tpZiHg68AfkhCLiEhaqRuYozAZegpjdBPwILAv1YGIiCRbOs1RgOCTggPLzWyVmV0/fKeZzQSuBO4IOA4RkbRU19KDGUxPk6QQ9Omjc9y9zsymASvMbIO7PzVk/23A5909ajb6Qm6JhHI9wJw5cwINWEQkmepbu6ksCpMTSocTNwH3FNy9LnG/D3gYOHNYkxrgf81sO/Ae4Adm9q4R3udOd69x95rKysogQxYRSar61p60qHk0ILCkYGaFZlY88Bi4GFg3tI27H+Pu89x9HvAA8Al3/3VQMYmIpJu6lm6qS9Lj1BEE21OoAp42s7XAC8Dv3P1xM7vBzG4I8LgiIhnB3RM9hfRJCoGNKbj7VuCUEbaPOKjs7tcGFYuISDpq647Q1RdlRppcjgrpcUmqiMikNLC4Tjr1FJQURERSZHAZTvUURESkLs1KXICSgohIytS39BDKMqYVKymIiEx6da3dVBWHCWWNPnk32ZQURERSpL4lvSaugZKCiEjKvNbcxUwlBRER6eiNsKu5m4VVRakO5XWSsZ5CWnh2SyPfXrGJjt7I4G1qUZgLT5jGxYurOHV2eVqd1xORiW3jnnYAjp9ekuJIXm/SJIVQlpGbncXswgKKw9kUhrPZ1tjJj1du40dPbqWiMJeyghyiMScSdQpyQ7xtcRX/cFI1J84o4WBVXEVEDtdAUlg0vTjFkbzepEkKS4+dwn3HTnnD9raefp7Y2MBTmxrojcTIzjJCWcbeth7ufGorP3ziVeZOKeCq02bxgaWzD3rpWHtPPys3N1LX0k1bT4T2nn76ozGqS/OZVR6/lRXkkmVGlkGWGeWFuRTmhpR0RCaZDXvaKApnM6s8vcYUJk1SGE1JXg7vPGUG7zxlxhv27e/sY/nf9/DoS/Xc+sdNfO8vm7n0pGo+tHQuFYW59Edj9EdjbNrbwePr6nlqcyN9kdjg64vC2WSHjJau/oPGkJ8TorI4TFVJmLlTCpk3pYB5UwuZWhQmPydEfm5osE1eTmjcP4OhBgp0ra9rY8f+Lo6bVsSps8sozc8J9Lgik82GPe0sml6cdn8QTvqkcDAVhblcfeYcrj5zDtsaO7n3ue08ULuL36ype0PbGaV5/OPSuVxy0nQWVhVTFM4eHKPo7ouyu6WL1/Z309bTjzvEPH6aqrmrj8aOXhrae6lr7WHl5gYeWNU7akwledlUleRRVpBDdlYW2SEjO8s4trKIM+ZVcMa8cqYUhXF3mrv6qW/tJpwd4piphaOOmXT2Rlixfi+PrK2jdkczrd1vTGILphUxv7Jo8HihrCyKwiFK8nMoycuhND+HkvxsSvJyKMnPoaWrn7/XtfL3uja27OugrCCHORUFzJlSwLwphRw/vZi5U0aPaTh359WGTsLZWcwsyydrhNd19EZ4dksjT25qoHZ7M1WleZw4o4QTZ5Rw/PRiZpTlU5Crr7yknruzob6Ny0b4YzTVzN1THcNhqamp8dra2pQdv6M3wlObGojGnJxQFjmh+GzEN80cv3GHrr4I2xu7aOnqoycSpbsvRmdvhIaOXva19bC3rZeW7j5iMYjEYvRGYmzZ10FvopdSVRKmuav/db2WwtwQJ84oZfGMEorCB34x7tjfxR/X76W7P8rMsnyWLZzK4hmlLK4uYU5FAZv2trN6RzOrdzbzWnM3sZgTiTmRaIzOvuhgkhvNjNI8FlQV09bTz86mLpo6+wb35eVksbCqmPycEO098cH//miMJXPKOW9RJectrMSAX6/ZzYOrdrNxb/vg646dWsTM8nx6+qN09Ebo7I2wrbGT/qhTmBvi9HkVNLT3snlvO5HYgQBL83OoLs2jJD+HvJwQ4ews8nJCFIVDFIWzKQrnUBgOJf5ts8jNzmLelAJOqC6hMKyEIuOjvrWbs7/6Z26+4kSuOXteUo5pZqvcveZQ7fQtP0xF4WwuPak60GMU5GazeMbhXZHQG4mybncrL2xrZvO+diqLwkwvzWN6SR4dvRHW7W7l5d2t/N/fXqMveiBZlObncNWSmVxx6kxq5pa/4S/wyuIw5xw3ddTjxmJOZ1+E1u5+2rrj963d/RSFszlxRgnlhbmva9/RG2FbQycb9rSxYU87G/e00x+NMaMsj6JwNg48v7WJ371cD4AZuMNpc8q4+YoTyQllsWVfB682dLCzqYv83Pgv82nFYS5YNI3zFlVSM7eC3Oyswc9l894ONu9rp761h/qWHupbe+jo7aetu5/eSGwwsXT0ROjuj474c5rBsVMLOX56CZXFYaYW5VJRGKYwHEqMERnZIeP46cXMqShIu1MCkl42pOmVR6CkMGGEs0OcPreC0+dWjLj/vTWzAzluVpZRnJdDcV4OlB+6fVE4m5NmlXLSrNJR27g7G/a088TGBnojUS4/ZQbzK4/sWu5wdog3zSzlTTNHP95QkWiMrv4okajTl0gYrzZ0sG53G+vqWllf30bj5l7aeyKjvsfUolyWzClnQVUR0RiDY08leTnMLM9nZlk+M8rymVKYS2l+zoinwmRi21CfuPKoKr2uPAIlBUlDZsYJ1SWcUJ38v6KyQ1mUDFtAfd7UQi48oep123ojUZo6+ujuj+LuxDw+drSurpVVO5pZvaOZFa/sjZ+CCmURyjLae/qJDTvVlmVQVpBLVUkeC6uKWFhVzMKqYmrmlr+hlyUTx8Y9bcwozaO0IP0u4FBSEDkC4ewQM0YoT3DK7DI+tHTuiK+JRGPsaeuhrqWH+tZu9nf20dzZx/6uPnY1d1O7vXnwIoYsi7/X+Qvjp8ROnFFCTkgFCCaKgSuP0pGSgkiSZIeymFVewKzyglHbtPf0s3FPO09vaeQvGxu47U+buPWPm8jPCXHK7FJOn1vO6XPLOW22ehKZqi8S49WGDs5fNC3VoYxISUEkjRTn5VAzr4KaeRV8+m0Laero5bmtTaza0cyqHc3c8eRWoolzUMdOLeS0OeVctHga5y+aFvgcFhkfWxs76I86J1SrpyAih2lKUZjLTp7BZSfHr2fv6ovw8q5WVu9sYfXOZv68YS8Prt5FUTibixdX8Q8nV/Pm+VPJz1WCSFfpWt5igJKCSAYpyM1m6bFTWJoo2RKJxnhuaxOPrK3j8XV7eOjF3YSzszh7/hQuWDSNt584neml6bOql8THE7KzjGOnpld11AGavCYyQfRGovx1637+snEfT2xsYFtjJ1kG5y+axvvPmM1bj5+mweo0cN09L1Df2sPjn16W1ONq8prIJBPODrFsYSXLFlbypcvh1YYOHlq9iwdW7eKffraPyuIwnzh/Ph9cOodwtk4vpcrGPe2ccczI84nSgf5sEJmg5lcW8bm3H88zn38rP/5IDcdVFvHlR9Zz4bee5OEXdxEbPmlCAtfa1U9da09azmQeoJ6CyASXHcriwhOqeOvx01i5uZGvPbaBf/2/tXx7xSbOW1jJuQsqOXv+FEry0m8i1UQzUL/r+DQdZAYlBZFJw8xYtrCStxw3lUdfruc3L+7m4dW7+fnzOwllGZedXM2n37aQY6YWpjrUCeuV+jYgfa88AiUFkUknK8sG1xDpi8R4cWczy9fv5b6/7uTRl+q56rSZfPLCBcyuGH2SnRyZ57c2UV2aR3UaXxEWaFIws+1AOxAFIsNHvs3sQ8DnE087gH9297VBxiQiB+RmZw1e4nrDefP54ROv8vO/7uA3a+q44fz5fOL8+ZoUN06iMeeZLY28403T07qKbjIGmi9w91NHuRRqG3Ceu58M3ALcmYR4RGQElcVhvnj5Yp783PlcctJ0vvOnzVxy+0qe2dKY6tAmhJd2tdDWE+HcBZWpDuWgUnr1kbs/6+7NiafPA7NSGY+IQHVpPrdffRo/+9iZuDsfuvuvfO5Xa+nsHb1cuBzays2NmHHQ9UnSQdBJwYHlZrbKzK4/RNuPAY8FHI+IjNG5Cyp5/NPLuPGC+Ty4eheXf/dp1u1uTXVYGWvl5gZOmllKRZoXMgw6KZzj7kuAS4AbzWzEKXxmdgHxpPD5UfZfb2a1Zlbb0NAQXLQi8jp5OSE+9/bj+cX/O4vOvghX/eBZfvL0NjKtEkKqtff0s3pnC+cuSO9eAgScFNy9LnG/D3gYOHN4GzM7GbgbuMLdm0Z5nzvdvcbdayor0/t8nMhEdPb8KTz2qWWcu2AqNz+6nmt+8gL1rd2pDitjPPdqE9GY85bj0v/3V2BJwcwKzax44DFwMbBuWJs5wEPAh919U1CxiMjRqyjM5e6P1HDLFSdSu72Zi299iodW71KvYQxWbm6kIDfEkrllqQ7lkILsKVQBT5vZWuAF4Hfu/riZ3WBmNyTafBGYAvzAzNaYmSrdiaQxM+PDZ8/jsU+dy6KqYj7zy7Xc8PNVNHf2pTq0tLZycwNnHTslI2pOqUqqiByRaMy5e+VW/nv5RqYWhbnt/acOlvSWA17b38W53/gLX7p8Mdedc0zK4hhrlVQVxBORIxLKMv7pvPk89M/nEM7O4gN3Pc9tf9w0uDKcxK3cHJ/nke7zEwYoKYjIUTlpVimPfvJc3nXqTG7742auvecF2nr6Ux1W2li5uYEZpXnMr8yMmlJKCiJy1IrC2Xz7/afy1atO4rlXm3jvD59jd4uuTopEYzyzpZFzF1SmdWmLoZQURGTcfODMOfz0ujOpa+nmXd9/hpd3Te7Jbqt2NNPWE2HZwsw4dQRKCiIyzt6yYCoPfuLN5IayeN+PnuNPr+xNdUgps2L9XnJDWZy3SElBRCaxhVXF/PrGc1hQVcTH763l/hd2pjqkpHN3Vryyl7PnT6EonDmrFCgpiEggKovD3P/xs1i2sJL/eOhlbl2xaVJNdNu8r4MdTV1cfGJVqkM5LEoKIhKYwnA2d11Tw3tPn8Xtf9rMvz/4MpFoLNVhJcXyv+8B4G0nZFZSyJw+jYhkpJxQFt94z8lUl+bxnT9vYV97D9//0BIKcif2r58V6/dyyuwyqkrSd5W1kainICKBMzM+c/EivnLlm3hyUwNX3/k8jR29qQ4rMHvbeli7q5WLF2dWLwGUFEQkiT60dC53friGTXvbueoHz/JqQ0eqQwrEivXxK66UFEREDuFti6u4/+Nn0dEb4V3fe2bw3PtEsmL9XuZNKeC4aUWpDuWwKSmISNKdNqecR256C8dUFnL9z1bxreUbJ0zNpPaefp57tYmLFldlzCzmoZQURCQlZpbl88t/Opv31cziu3/ewrX3vMD2xs5Uh3XUntzUQF80xkWLp6c6lCOipCAiKZOXE+Lr7z6Z/7ryJFbtaOaiW5/kP3/7d/Zn8PoMv3upnorCXE6fW57qUI6IkoKIpJSZ8cGlc3jis+fzntNnc+9z2znvG3/hnmcyby3o1TubeWzdHj5w5mxCWZl36giUFEQkTUwryeOrV53E8n9dxpK55Xz5kfV8/N5VtHRlRq8hFnO+/Mh6phWH+cT5x6U6nCOmpCAiaeW4acX89Loz+OJli3ly0z7+4TtPs2pHc6rDOqSHX9zN2tda+Pw7jqcwg2odDaekICJpx8z46FuO4YEb3kxWFrz/R89x8yPr07bX0Nkb4euPb+CU2WVcedrMVIdzVJQURCRtnTK7jN998lzeWzOLnz67jfO++QQ/fnobfZH0qp/0gye2sK+9ly9dvpisDB1LGKCkICJprSQvh69edTK//9S5nDyrlFseXc9Ftz7JI2vriKXB3IZtjZ3ctXIbV542kyVzMvOKo6GUFEQkIxw/vYR7P3om91x3Bvk5IW66/0Xe+f2nWbm5IWUxNXX08tGf/o2C3BCff8fxKYtjPCkpiEjGMDMuWDSN333yXL79vlNo7uznwz9+gQ/e9TxrXmtJaizdfVE+9j+11LV0c/c1NUwvzaxqqKNRUhCRjBPKMq5aMos/f/Y8vnjZYjbuaedd33+G6++tZdPe9sCPH4nGuOn+F1m7q4Xbrz6NmnkVgR8zWSzTJofU1NR4bW1tqsMQkTTS0RvhJ09v466nttLeG+GsYyt495JZXHJS9bgvhRmLOV/49Truf2EnN19xItecPW9c3z8oZrbK3WsO2U5JQUQmiubOPn7+/A4eXL2L7U1d5OeEePuJVVx6UjXLFlaSlxM6qvfv6Y/ymV+u4fcv7+ET58/n3zJoHEFJQUQmLXdn9c5mHli1m8fW1dPS1U9hbogLT6jig0vnsPSYisOuYNrY0cvH761lzWstfOHSE/jYW47JqCqoSgoiIkB/NMZzrzbx2Lp6Hlu3h5aufk6dXcYN5x3LRYunj6lG0drXWviX+1ezr62X268+lXe8qToJkY8vJQURkWF6+qP8atUu7npqKzv3dzGzLJ+lx1Rw2pwyTp1dzjGVhRTmhjAzojFnxfo9/OTp7bywfT9Ti3K565oaTsvQuQhpkRTMbDvQDkSByPCALN73uh24FOgCrnX31Qd7TyUFETla0Zjz2Lp6fv1iHWtea3ndetHZWUZZQQ7u0NTZx6zyfK598zzed8ZsSvJyUhj10RlrUkhG1aYL3L1xlH2XAAsSt6XADxP3IiKBCWUZl508g8tOnoG7s6u5mzWvtVDf2k1LVz8t3f309EW5+MTpXLS4KmPLYB+JVJfyuwK41+PdlefNrMzMqt29PsVxicgkYWbMrihgdkVBqkNJC0FPXnNguZmtMrPrR9g/E3htyPNdiW2vY2bXm1mtmdU2NKRuSruIyEQXdFI4x92XED9NdKOZLRu2f6Q+2RsGOdz9TnevcfeaysrKIOIUERECTgruXpe43wc8DJw5rMkuYPaQ57OAuiBjEhGR0QWWFMys0MyKBx4DFwPrhjX7LXCNxZ0FtGo8QUQkdYIcaK4CHk7M+MsG7nP3x83sBgB3vwP4PfHLUbcQvyT1ugDjERGRQwgsKbj7VuCUEbbfMeSxAzcGFYOIiBwelc4WEZFBSgoiIjIo42ofmVkD0AK0DttVeohth3o8cD8VGG0G9sGMdPyx7B++/WDPh8c6dNuRxJ3MmIc+TsVnre+Hvh8H25+J34/DiRlggbuXHjISd8+4G3Dn4W471OMh97XjFdNY9g/ffrDnw2M92riTGXOqP2t9P/T9mGjfj8OJeSzHGLhl6umjR45g26Eej/T6o41pLPuHbz/Y85FiPZq4kxnz0Mep+Kz1/Th8+n6M/XG6xzyWYwAZePooaGZW62OoJJhuMjFuxZw8mRi3Yk6NTO0pBOnOVAdwhDIxbsWcPJkYt2JOAfUURERkkHqL9K7qAAAHC0lEQVQKIiIyaEInBTP7iZntM7PhNZfG8trTzexlM9tiZt+xISt0m9lNZrbRzP5uZt8Y36iDidvM/tPMdpvZmsTt0nSPecj+z5qZm9nU8Ys4sM/5FjN7KfEZLzezGRkQ8zfNbEMi7ofNrGw8Yw4w7vcm/g/GzGzczuMfTayjvN9HzGxz4vaRIdsP+r1PmSO5fCpTbsAyYAmw7ghe+wJwNvHy3o8BlyS2XwD8EQgnnk/LkLj/E/hsJn3WiX2zgT8AO4Cp6R4zUDKkzSeBOzIg5ouB7MTjrwNfz4TvB3ACsAh4AqhJdayJOOYN21YBbE3clycelx/s50r1bUL3FNz9KWD/0G1mNt/MHk8s/LPSzI4f/jozqyb+n/s5j//r3Qu8K7H7n4GvuXtv4hj7MiTuQAUY863AvzHCOhvpGLO7tw1pWjjecQcU83J3jySaPk+8hP24CijuV9x9Y7rEOoq3Ayvcfb+7NwMrgHek8v/qoUzopDCKO4Gb3P104LPAD0ZoM5P4Wg8Dhq4ItxA418z+amZPmtkZgUZ7wNHGDfAviVMEPzGz8uBCHXRUMZvZO4Hd7r426ECHOOrP2cy+YmavAR8CvhhgrAPG47sx4KPE/2pNhvGMO2hjiXUko60umS4/1xukeo3mpDKzIuDNwK+GnL4Lj9R0hG0Df/FlE+8GngWcAfzSzI5NZPtAjFPcPwRuSTy/BfgW8V8AgTjamM2sAPgC8VMbSTFOnzPu/gXgC2b2H8C/AF8a51APBDJOMSfe6wtABPjFeMY4kvGMO2gHi9XMrgM+ldh2HPB7M+sDtrn7lYwef8p/rtFMqqRAvGfU4u6nDt1oZiFgVeLpb4n/Ah3ahR66Itwu4KFEEnjBzGLE650EuXj0Ucft7nuHvO4u4NEA44Wjj3k+cAywNvEfcRaw2szOdPc9aRrzcPcBvyPApMA4xZwYAL0MuDDIP3CGGO/POkgjxgrg7vcA9wCY2RPAte6+fUiTXcD5Q57PIj72sIvU/1wjS/WgRtA3YB5DBoyAZ4H3Jh4bcMoor/sb8d7AwCDQpYntNwA3Jx4vJN41tAyIu3pIm38F/jfdYx7WZjvjPNAc0Oe8YEibm4AHMiDmdwDrgcrxjjUZ3w/GeaD5SGNl9IHmbcTPLpQnHleM9XufilvKAwj0h4P7gXqgn3hm/hjxvz4fB9Ym/iN8cZTX1hBfPvRV4HscmOiXC/w8sW818NYMiftnwMvAS8T/AqtO95iHtdnO+F99FMTn/GBi+0vEa83MzICYtxD/42ZN4jauV0wFGPeViffqBfYCf0hlrIyQFBLbP5r4jLcA1x3O9z4VN81oFhGRQZPx6iMRERmFkoKIiAxSUhARkUFKCiIiMkhJQUREBikpyIRgZh1JPt7dZrZ4nN4ravGqquvM7JFDVSk1szIz+8R4HFtkOF2SKhOCmXW4e9E4vl+2HygSF6ihsZvZ/wCb3P0rB2k/D3jU3d+UjPhkclFPQSYsM6s0swfN7G+J2zmJ7Wea2bNm9mLiflFi+7Vm9iszewRYbmbnm9kTZvaAxdcb+MVAzfvE9prE445EEby1Zva8mVUlts9PPP+bmd08xt7McxwoCFhkZn8ys9UWr7t/RaLN14D5id7FNxNtP5c4zktm9uVx/BhlklFSkInsduBWdz8DeDdwd2L7BmCZu59GvIrpfw15zdnAR9z9rYnnpwGfBhYDxwLnjHCcQuB5dz8FeAr4+JDj3544/iHr2iTq/lxIfMY5QA9wpbsvIb6Ox7cSSenfgVfd/VR3/5yZXQwsAM4ETgVON7NlhzqeyEgmW0E8mVzeBiweUtmyxMyKgVLgf8xsAfHKlDlDXrPC3YfW0n/B3XcBmNka4jVxnh52nD4OFBhcBVyUeHw2B2rk3wf89yhx5g9571XEa+5DvCbOfyV+wceI9yCqRnj9xYnbi4nnRcSTxFOjHE9kVEoKMpFlAWe7e/fQjWb2XeAv7n5l4vz8E0N2dw57j94hj6OM/H+m3w8Mzo3W5mC63f1UMyslnlxuBL5DfD2GSuB0d+83s+1A3givN+Cr7v6jwzyuyBvo9JFMZMuJr2cAgJkNlD4uBXYnHl8b4PGfJ37aCuDqQzV291biS3h+1sxyiMe5L5EQLgDmJpq2A8VDXvoH4KOJuv+Y2UwzmzZOP4NMMkoKMlEUmNmuIbfPEP8FW5MYfF1PvOw5wDeAr5rZM0AowJg+DXzGzF4AqoHWQ73A3V8kXonzauKL3dSYWS3xXsOGRJsm4JnEJazfdPflxE9PPWdmLwMP8PqkITJmuiRVJCCJ1eO63d3N7GrgA+5+xaFeJ5JKGlMQCc7pwPcSVwy1EODypyLjRT0FEREZpDEFEREZpKQgIiKDlBRERGSQkoKIiAxSUhARkUFKCiIiMuj/AyrxPQktesJWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.lr_find()\n",
    "learn_lm.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 00:04 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.520447</td>\n",
       "      <td>4.112982</td>\n",
       "      <td>0.285067</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.fit_one_cycle(1, 5e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.save('learn_lm_ASTD_epoch1_acc272')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XuYXXV97/H3d9/mPplcJhcSQkgCIijXEEU8FNBSRQ5qpRVPbRF6yvFSL/WxPaU+9bTam3Jar+eIiAIWUCuWFlAB2x6KBTEkQBIKqLlBJhcy98nsPXv27Xv+WGsmm2GSTDKz9mXm83qe/czea62913d2Mvuzf+v3W79l7o6IiAhArNoFiIhI7VAoiIjIOIWCiIiMUyiIiMg4hYKIiIxTKIiIyDiFgoiIjFMoiIjIOIWCiIiMS1S7gGO1aNEiX7VqVbXLEBGpK5s2bepx986jbVd3obBq1So2btxY7TJEROqKmb0wle10+EhERMYpFEREZJxCQURExikURERknEJBRETGKRRERGScQkFERMYpFERE6sAX/uUX/OSX3ZHvR6EgIlLjSiXnS//6Szbs7It8XwoFEZEaN5TNU3LoaE5Fvi+FgohIjetL5wBY0JKMfF8KBRGRGtefyQMwXy0FERHpD1sKCgUREaE/M3b4qM5Dwcw6zOxuM3vezJ4zswsmrL/YzAbN7Onw9qko6xERqUdjodDRHH2fQtTXU/gi8IC7X2VmKaB5km1+4u5XRFyHiEjd6kvnScaN1oboL4ET2R7MrB24CHgfgLvngFxU+xMRma0GMjk6mlOYWeT7ivLw0WqgG7jVzJ4ys1vMrGWS7S4ws81m9iMzOyPCekRE6lJfOseCCnQyQ7ShkADOBb7q7ucAaeCPJ2zzJHCSu58FfBn4p8leyMyuN7ONZraxuzv607xFRGrJQCbP/AqcowDRhkIX0OXuPwsf300QEuPcfcjdh8P7PwSSZrZo4gu5+83uvs7d13V2HvW60yIis0pfJleR4agQYSi4+35gt5m9Klz0JuDZ8m3MbKmFB8nMbH1YT29UNYmI1KOBTI75FRiOCtGPPvowcGc48mgHcK2ZvR/A3W8CrgI+YGYFYAS42t094ppEROpGqeT0Z/LMr8BwVIg4FNz9aWDdhMU3la3/CvCVKGsQEalnB7MFiiWv/8NHIiIyfZU8mxkUCiIiNa0vU7l5j0ChICJS0wbGQkEtBRER6UuPTZtd/+cpiIjINKmlICIi4/rSORIxo60Ck+GBQkFEpKb1V3AyPFAoiIjUtP50viLXZh6jUBARqWF9YUuhUhQKIiI1bCBTuWmzQaEgIlLT+tKVmzYbFAoiIjXL3YMZUtVSEBGRg6MFCiWv2LxHoFAQEalZ/engxDV1NIuICP2ZYIoLDUkVERG1FERE5JDxaykoFEREpC9d2cnwQKEgIlKzBjJ54jGjvbEyk+GBQkFEpGb1ZXLMb05WbDI8UCiIiNSs/nRl5z0ChYKISM3qr/C8R6BQEBGpWf3pPB0VugznGIWCiEiN6s/kKjrFBSgURERqkrvTn8lVdDgqKBRERGpSOlckX3Tm6/CRiIiMTXFRyWmzIeJQMLMOM7vbzJ43s+fM7IIJ683MvmRm28xsi5mdG2U9IiL1oq9KoRD1aXJfBB5w96vMLAU0T1j/VuCU8PY64KvhTxGROW1s3qNZ06dgZu3ARcA3ANw95+4DEzZ7O/AtDzwOdJjZsqhqEhGpF+OT4c2WUABWA93ArWb2lJndYmYtE7ZZDuwue9wVLhMRmdP608G1FGZTR3MCOBf4qrufA6SBP56wzWQTevjEBWZ2vZltNLON3d3dM1+piEiN6c/kiBm0N86eUOgCutz9Z+HjuwlCYuI2J5Y9XgHsnfhC7n6zu69z93WdnZ2RFCsiUkv6M8G8R7FY5SbDgwhDwd33A7vN7FXhojcBz07Y7F7gd8JRSK8HBt19X1Q1iYjUi/50vuKHjiD60UcfBu4MRx7tAK41s/cDuPtNwA+By4FtQAa4NuJ6RETqQs/waMU7mSHiUHD3p4F1ExbfVLbegQ9FWYOISD16aSjLa1d0VHy/OqNZRKTGuDv7BrMsm9dY8X0rFEREasxAJs9oocTSdoWCiMict28wC6CWgoiIwP6hEQCWKhRERGTvQNBSOKGjqeL7ViiIiNSY/YNZ4jFjUWtDxfetUBARqTH7BrMsaWsgXuGzmUGhICJSc/YPjVSlPwEUCiIiNSc4R6Hy/QmgUBARqSnuzv7BrFoKIiICQ9kCmVyxKucogEJBRKSm7A9PXFNLQURE2DcYnLimloKIiJS1FNTRLCIy5+0bzGIGi9sqf+IaKBRERGrKvsEROlsbSMar8/GsUBARqSHVuo7CGIWCiEgN2V/FE9dAoSAiUlOqeeIaKBRERGrGwWyeg6MFHT4SERF4aai6J66BQkFEpGYcugyn+hREROa8al6beYxCQUSkRoydzby4vTonroFCQUSkZuwbzLKoNUVDIl61GhQKIiI1Yv9g9a64NkahICJSI/YNZlnaXr1OZog4FMxsl5ltNbOnzWzjJOsvNrPBcP3TZvapKOsREall1Z7iAiBRgX1c4u49R1j/E3e/ogJ1iIjUrEyuwOBIXoePRETk0MijarcUog4FBx4ys01mdv1htrnAzDab2Y/M7IyI6xERqUn7a+DENYj+8NGF7r7XzBYDPzaz5939kbL1TwInufuwmV0O/BNwysQXCQPleoCVK1dGXLKISOXVwolrEHFLwd33hj8PAPcA6yesH3L34fD+D4GkmS2a5HVudvd17r6us7MzypJFRKpi7NrMs7ZPwcxazKxt7D5wGfDMhG2WmpmF99eH9fRGVZOISK3a1ZthSXsDjcnqnbgG0R4+WgLcE37mJ4C73P0BM3s/gLvfBFwFfMDMCsAIcLW7e4Q1iYjUpJ09aVYtbKl2GdGFgrvvAM6aZPlNZfe/AnwlqhpEROrFrp40l52xpNplaEiqiEi1DY7k6U3naqKloFAQEamyXT1pAFYtUiiIiMx5O8NQWF0voWBma8ysIbx/sZl9xMw6oi1NRGRu2NmTxgxOXNBc7VKm3FL4PlA0s7XAN4CTgbsiq0pEZA7Z1ZvmhHlNVR+OClMPhZK7F4B3Al9w9z8AlkVXlojI3LGzJ83qzuofOoKph0LezN4DXAPcHy5LRlOSiMjc4e41c44CTD0UrgUuAP7S3Xea2cnAHdGVJSIyN/SmcxzMFji5BjqZYYonr7n7s8BHAMxsPtDm7n8TZWEiInPB2HDUWgmFqY4+etjM2s1sAbAZuNXM/i7a0kREZr+dNXSOAkz98NE8dx8Cfh241d3PA94cXVkiInPDzp40iZixYn51r6MwZqqhkDCzZcBvcqijWUREpmlXb5oTFzSTjNfGucRTreLTwIPAdnd/wsxWA7+MriwRkblhR3e6ZvoTYOodzd8Dvlf2eAfwrqiKEhGZC9ydF3ozvGHNK64tVjVT7WheYWb3mNkBM3vJzL5vZiuiLk5EZDZ7aWiUkXyRkxdVf3qLMVM9fHQrcC9wArAcuC9cJiIix2lHzzAAJy9qrXIlh0w1FDrd/VZ3L4S32wBdLFlEZBp29WQAWFWHLYUeM3uvmcXD23vRtZRFRKZlZ88wqUSME+bVxnBUmHooXEcwHHU/sI/g2srXRlWUiMhcsLMnw6qFzcRiVu1Sxk0pFNz9RXe/0t073X2xu7+D4EQ2ERE5Tjt7hmtmIrwx0zlb4uMzVoWIyBxTLDm7+0Y4uUamzB4znVConfaOiEid2TswQq5Y4uRZ1FLwGatCRGSO+eWBgwCs7qyd4ahwlDOazewgk3/4G1A73eUiInVmS9cgZnD6Ce3VLuVljhgK7t5WqUJEROaSrV2DrO1spbVhSrMNVUxtTMsnIjKHuDtb9gzy2hXzql3KKygUREQqbP9Qlu6Do5y5fI6FgpntMrOtZva0mW2cZL2Z2ZfMbJuZbTGzc6OsR0SkFmzpGgTgtSs6qlzJK1XiYNYl7t5zmHVvBU4Jb68Dvhr+FBGZtbZ2DRKPGWfUWCczVP/w0duBb3ngcaAjvMKbiMistWXPIKcuaaMxGa92Ka8QdSg48JCZbTKz6ydZvxzYXfa4K1z2MmZ2vZltNLON3d3dEZUqIhI9d2dr10BN9idA9KFwobufS3CY6ENmdtGE9ZOdFf2K8yLc/WZ3X+fu6zo7NWO3iNSvrv4R+jP5mhx5BBGHgrvvDX8eAO4B1k/YpAs4sezxCmBvlDWJiFTTWCfzWTXYyQwRhoKZtZhZ29h94DLgmQmb3Qv8TjgK6fXAoLvvi6omEZFq27JngFQ8xqlLa2t6izFRjj5aAtxjZmP7ucvdHzCz9wO4+03AD4HLgW1ABl2jQURmua1dg5y2rI2GRO11MkOEoeDuO4CzJll+U9l9Bz4UVQ0iIrWkVHK27hnkyrNOqHYph1XtIakiInPGrt40B7OFmu1PAIWCiEjFbN0zdiZzbY48AoWCiEjFbOkapCER45TFtdnJDAoFEZGK2do1yBkntJOI1+5Hb+1WJiIyixSKJZ7ZO8iZNdyfAAoFEZGK2Nw1SCZX5PxVC6pdyhEpFEREKuCxbT2YwQVrFla7lCNSKIiIVMCj23s4fVk7C1pS1S7liBQKIiIRG8kVefKFAS5cu6japRyVQkFEJGJP7OojVywpFEREBB7d1kMybpy/an61SzkqhYKISMQe3d7DOSvn05yqxBWQp0ehICISof50jv/cO8Qb6+DQESgUREQi9dMdvbjDhWtreyjqGIWCiEiEHt3WQ2tDoubPZB6jUBARidBj23t53ckLSNbwfEfl6qNKEZE6tGdghJ09ad5QJ/0JoFAQEYnMo9t6AOqmkxkUCiIikXl0Ww+LWhs4dUntXj9hIoWCiEgEhkcL/PjZl7j0tE7MrNrlTJlCQUQkAvdv3ksmV+Td56+sdinHRKEgIhKBbz+xm1OXtHLuyvoYijpGoSAiMsOe2zfE5t0DvPv8lXV16AgUCiIiM+47G14kFY/x6+csr3Ypx0yhICIyg7L5Ivc8tYe3vGYp82v8gjqTUSiIiMygHz2zj6FsgavXn1jtUo6LQkFEZAZ9e8NuTlrYzOtPro8J8CaKPBTMLG5mT5nZ/ZOse5+ZdZvZ0+Htv0ddj4hIVLZ3D7NhZx/vPv9EYrH66mAeU4krPnwUeA5oP8z677r771egDhGRSN3x+AvEY8ZV562odinHLdKWgpmtAN4G3BLlfkREqq0vneM7G3bz9rNOYHFbY7XLOW5RHz76AvBHQOkI27zLzLaY2d1mNmnPjJldb2YbzWxjd3d3JIWKiEzHbY/uZCRf5AMXr6l2KdMSWSiY2RXAAXffdITN7gNWufuZwL8At0+2kbvf7O7r3H1dZ2dnBNWKiBy/4dECtz22i8tOX8IpS9qqXc60RNlSuBC40sx2Ad8BLjWzO8o3cPdedx8NH34dOC/CekREInHn4y8wlC3wwUvWVruUaYssFNz9Bndf4e6rgKuBf3P395ZvY2bLyh5eSdAhLSJSN7L5Irf8x07euHYRZ59YX/McTaYSo49exsw+DWx093uBj5jZlUAB6APeV+l6RESm4+5NXXQfHOWL7z672qXMiIqEgrs/DDwc3v9U2fIbgBsqUYOIyEwrFEt87ZHtnH1iBxesqc+T1SbSGc0iIsfpuxt3s7tvhA9evKbuZkM9HIWCiMhx6EvnuPHBn/O6kxfwq6cvqXY5M0ahICJyHG588HkOZgt85h2vmTWtBFAoiIgcs6de7Oc7T+zmugtXcWqdn5cwkUJBROQYFEvOn/7zMyxua+Cjbz612uXMOIWCiMgxuGvDizyzZ4hPvu10WhsqPqo/cgoFEZEpemkoy40PPM8b1izkv5657OhPqEMKBRGRKXB3/uf3t5ArlviLWda5XE6hICIyBd/esJuHf97NDW99Nas7W6tdTmQUCiIiR/FCb5q/+MGzvHHtIn779SdVu5xIKRRERI6gWHI+8b3NxGPG5646s24vszlVs6/rXERkBn39Jzt4Ylc/f/ebZ3FCR1O1y4mcWgoiIofx+I5ebnzw57z1NUt55znLq11ORSgUREQmsXdghA/d+SQnLWzmc1edOWtHG02kUBARmSCbL/KBOzYxWihx82+vo60xWe2SKkZ9CiIiZdydP/2nZ9jcNcjXfvs81i6evcNPJ6OWgohImdsf28X3NnXxkUvX8mtnLK12ORWnUBARCf1w6z7+/P5nefOrl/CxWTjZ3VQoFEREgJ9u7+Vj33mac1fO58vvOWfWn49wOAoFEZnznts3xPXf2sjKhc1845p1NKXi1S6pahQKIjKn7e7LcM03N9DSkOBb162nozlV7ZKqSqOPRGTOemkoy2/d8jNGCyX+4X9cMCfOWD4atRREZE7qS+d47y0/o3d4lNuuPZ9XLZ1dl9U8XmopiMicM5TNc803N/BiX4bbrl3POSvnV7ukmqGWgojMKSO5Ir972xM8t2+Ir773XC5Ys7DaJdUUhYKIzBm5Qon337GJTS/084Wrz+bS05ZUu6Sao8NHIjInFEvOH3z3af79F9189l2v5YozT6h2STUp8paCmcXN7Ckzu3+SdQ1m9l0z22ZmPzOzVVHXIyJzj7vzJ/+4lR9s3ccnL3817z5/ZbVLqlmVOHz0UeC5w6z7XaDf3dcCnwc+W4F6RGQOyeaL/Ok/P8N3N+7mw5eu5fcuWl3tkmpapKFgZiuAtwG3HGaTtwO3h/fvBt5kc2XSchGJ3JMv9nPFl/+DOx5/kesvWs3Hf3Vuzmd0LKLuU/gC8EfA4QYALwd2A7h7wcwGgYVAT/lGZnY9cD3AypVq9onIkWXzRT7/41/w9Z/sYEl7I7dft55fObWz2mXVhchCwcyuAA64+yYzu/hwm02yzF+xwP1m4GaAdevWvWK9iAgEw03v2vAiX/v37Rw4OMp71q/kTy4/bU5dJGe6omwpXAhcaWaXA41Au5nd4e7vLdumCzgR6DKzBDAP6IuwpsMaLRQZLZSAQ0mVzZcYyRXJ5AtkckVGckXSowVG8kVyhRLJeIxkPEYibrQ2JFjQkmJhS4r5LSmy+SJ96Rx96RxD2QJxM5JxI5mI0ZSMs7A1xYLmFIn4sR/Bc3dyxRLZXImRfJFsvshIvkgmV6Q/naNneJSe4VEGMnmaGxJ0NCXpaE7SkIgzMJKjP52jP5NncCRPerTA8GiB9GiBfNEplJxiqUSx5MTMiJlhBsl4UHdjKk5zMk5zQ5zWhgTNqQStDXHaGpPMa0rS3pSgJZUgkyuG+8ozPFqgWHLcnZJDIm50NCWZ15ykoylFW2PwOi0NcZpScUql4N8jVyhRKDmNyTjNqWBdsejsG8yyf2iEfYNZMqNFnOB13aEpGaOlIUFrQ2J8UjN3cBwzIxX+myXjRiIWw4zg94xBSyp4XmtjguRx/LtI9WRyBe58/EW+9sgOeoZHef3qBXz5PefwutU6B+FYRRYK7n4DcANA2FL4xIRAALgXuAb4KXAV8G/uHklLYLRQZGdPmm0Hhtl+IM227mF292XGP7iHRwtR7PaIzGB+c4r2xgSNyTgNyTiNieCDKviQc4olJ5MrhrcwnPJFpvIuNSZjZPOlSde1pOLMa0oGH6CNwQd5QyJGLGYkYkEYOE6pBEV3CsUggAYzOfaF9aRzh8Jkqr9vzIxiqfYbe43JGAuaUyxoTbGgpYGFLSkWhLdFrSmaUwkSMSMeMxJxo70xyfyWIOjnNSXn7LTLlTaQyXH7Yy9w22M76c/kuXDtQv7PpQqD6aj4eQpm9mlgo7vfC3wD+Hsz20bQQrg6qv3+YMs+Pv4Pm8MaYHlHEyctbOakhR3j3/Abk4emy3UPPhiaUgmawm+qzak4LeE30FQ8Rr4YfJPNFUoMjxboS+foTQffxJuS8eBDoiX4Bl1yyBdK5IolMrkivcOjdA8H3+oPZguM5otkCyWy4Qd+zCAWi9GQMBa2NtCSitPckKA5GXxjbkzGg2/uyThNqeBbfEMyzvzm4ENrUWsDjck4xZJzMJtnIJNnJF9kfnOKjubky37X6RotFBnOFhjKFhgayXMwW6ClIU5Hc4qOpiRtjQniMRu/8HmhWGIoW2Agk2NgJM9wtkAmV2B4tMhIrkA8FiOViNGQiBGPGdmwFZTNFzEzls1rZNm8RpbOa6S1ITHemoGgdTfW+snkipgFLT8zo+ROoRj8e+WKRYolKLnjHvzM5IoczObD3yVPXzpPX3qUvnSO7QeG6UvnGMkXj/p+mEFDIhYEfSJGSyoRtoqSdIRfAprD1kxzKk5LKvg/1dIQpzmVGH9eYzJOSyr4f6SWyyGFYomndg/wo637+e4TL5LOFXnTaYv54CVrOO+kBdUur+5ZRF/MI7Nu3TrfuHHjMT9v3+AIT+zqZ01nC6sXtc7p+dLl+I3kivSmRxnJFcMWVHDIbWgkT38maHX2Z/Jkw8N62XyRdK7I0EgQzP2Z3MsO103VvKYkC1tSLGprCAKxvZEl7Y0sbE3R3pSkPTx819Ec/JxtIbJvcIRHt/Xy8M8P8MgvuhnKFkjEjLeduYwPXLyG05a2V7vEmmdmm9x93dG2mzNnNC+b18SVZ2laXJmeplScFanmGXmtXCFo1WTyRTJhyyadKzCaD1qM2UKR4dEifcM5etOj9A7n6D44ylMvDrB/MEuuOPmhQQgOD3Y0p+hsa2BxWwOd4W1hawOLWlIsbG1gaXvQ2kolaitAhrJ5th8YZnt3ms27B3h0ew87utMAdLY18GtnLOWS0xbzxlMW0a4O5Bk3Z0JBpNakEjFSiRTHMz+nu4+3SoayeYZGgoEDg2GLZHAkT1846OCF3gxP7OqjP5N/xeuYweK2Bk7oaKKztYFFbUFoLGhJ0dYY9DkFAwGCw5ZN4WHLlvDQ17GeVlQqOb3pHC8NZYMBA4Mj7BnIsmdghD39GXb3j9B9cHR8++ZUnNedvID/tn4lF65dxKuWtKm/JmIKBZE6ZBb0NS1sbZjycwrFEn2ZHL1hX9a+wSx7+kfYOzDC3sERXujNsOmFfvoyuSkNZEiGHeztTUmKJX/ZiLGxEV7JeAx3wtFxhUkHPqTiMZZ1NLK8o4mLT+1kzeJW1nS2sqazhZULmo9rhJ4cP4WCyByRiMdY3NbI4rbGI25XLDkDmRzp0SIHR/OkR4Oh2OVDn4dHC+Mtk6GRPPGY0ZAIBggkYjEKpRL5gpMvBSEQDNQIOtEXtqRYOjZYoL2RRa0N+vZfQxQKIvIy8dhYK6TalUg1qF0mIiLjFAoiIjJOoSAiIuMUCiIiMk6hICIi4xQKIiIyTqEgIiLjFAoiIjKu7mZJDS/Z+ctJVs0DBo+wbOL6sceTbbOICZcEnaLJapjK+rlS+9Huq/apr1fttVH7kR5Pdr98WaVrP8Xd5x311d29rm7AzVNdXr5s4vqxx5NtQ3C9hxmrTbVP7b5qV+31VvuRHh+m3vJlVX/fJ7vV4+Gj+45h+X1HWH/fFLY5Vkd7/lyvfSr3j4dqf+Uy1X5kM1X7kR5Pdn+6dU/lNY7lfX+Fujt8VAlmttGncDGKWqTaq0O1V4dqn3n12FKohJurXcA0qPbqUO3VodpnmFoKIiIyTi0FEREZN+tDwcy+aWYHzOyZ43jueWa21cy2mdmXrOzag2b2YTP7uZn9p5l9bmarHt/HjNduZn9mZnvM7OnwdvnMVx7d+x6u/4SZuZktmrmKX/b6UbzvnzGzLeF7/pCZnVAndd9oZs+Htd9jZh0zXXe4nyhq/43w77NkZjN+7H46NR/m9a4xs1+Gt2vKlh/x72HGHc+QqHq6ARcB5wLPHMdzNwAXAAb8CHhruPwS4F+AhvDx4jqq/c+AT9Tj+x6uOxF4EHgBWFQvtQPtZdt8BLipTuq+DEiE9z8LfLaO3vNXA68CHgbW1UrNYT2rJixbAOwIf84P788/0u8X1W3WtxTc/RGgr3yZma0xswfMbJOZ/cTMTpv4PDNbRvCH/FMP/mW+BbwjXP0B4G/cfTTcx4E6qr0iIqz988AfAZF1hkVRu7sPlW3aEkX9EdX9kLsXwk0fB1bMdN0R1v6cu/88inqnU/Nh/BrwY3fvc/d+4MfAW6rxtzzrQ+EwbgY+7O7nAZ8A/u8k2ywHusoed4XLAE4F/ouZ/czM/t3Mzo+02pebbu0Avx8eDvimmc2PrtRXmFbtZnYlsMfdN0dd6CSm/b6b2V+a2W7gt4BPRVhruZn4/zLmOoJvqpUyk7VXylRqnsxyYHfZ47Hfo+K/35y7RrOZtQJvAL5XdmiuYbJNJ1k29u0uQdDEez1wPvAPZrY6TPLIzFDtXwU+Ez7+DPC3BH/skZpu7WbWDHyS4HBGRc3Q+467fxL4pJndAPw+8L9muNSXFzNDdYev9UmgANw5kzUezkzWXilHqtnMrgU+Gi5bC/zQzHLATnd/J4f/PSr++825UCBoHQ24+9nlC80sDmwKH95L8OFZ3lReAewN73cB/xiGwAYzKxHMY9IdZeHMQO3u/lLZ874O3B9lwWWmW/sa4GRgc/gHtwJ40szWu/v+Gq99oruAHxBxKDBDdYednlcAb4r6i0+ZmX7PK2HSmgHc/VbgVgAzexh4n7vvKtukC7i47PEKgr6HLir9+0XZYVErN2AVZZ1BwGPAb4T3DTjrMM97gqA1MNbBc3m4/P3Ap8P7pxI0+6xOal9Wts0fAN+pl/d9wja7iKijOaL3/ZSybT4M3F0ndb8FeBbojOq9jvr/CxF1NB9vzRy+o3knwRGI+eH9BVP5/Wb8d4r6H7raN+DbwD4gT5C6v0vwjfMBYHP4H/5Th3nuOuAZYDvwFQ6d7JcC7gjXPQlcWke1/z2wFdhC8E1rWb3UPmGbXUQ3+iiK9/374fItBHPQLK+TurcRfOl5OrzN+KipCGt/Z/hao8BLwIO1UDOThEK4/Lrw/d4GXHssfw8zedMZzSIiMm6ujj4SEZFJKBRERGScQkFERMYpFEREZJxCQURExikUZFYws+EK7+8WMzt9hl6raMHsqc+Y2X1Hm4nUzDrM7IMzsW+RiTQkVWYFMxt299YZfL2EH5oILlLltZvZ7cAv3P0vj7D9KuB+d39NJeqTuUUllX+mAAACs0lEQVQtBZm1zKzTzL5vZk+EtwvD5evN7DEzeyr8+apw+fvM7Htmdh/wkJldbGYPm9ndFlxT4M6xuezD5evC+8PhZHebzexxM1sSLl8TPn7CzD49xdbMTzk0AWCrmf2rmT1pwXz6bw+3+RtgTdi6uDHc9g/D/Wwxsz+fwbdR5hiFgsxmXwQ+7+7nA+8CbgmXPw9c5O7nEMxW+ldlz7kAuMbdLw0fnwN8DDgdWA1cOMl+WoDH3f0s4BHg98r2/8Vw/0edryac1+dNBGeaA2SBd7r7uQTX8PjbMJT+GNju7me7+x+a2WXAKcB64GzgPDO76Gj7E5nMXJwQT+aONwOnl81Y2W5mbcA84HYzO4Vgxslk2XN+7O7lc+RvcPcuADN7mmCum/+YsJ8chyYW3AT8anj/Ag7NfX8X8L8PU2dT2WtvIphLH4K5bv4q/IAvEbQglkzy/MvC21Ph41aCkHjkMPsTOSyFgsxmMeACdx8pX2hmXwb+n7u/Mzw+/3DZ6vSE1xgtu19k8r+ZvB/qnDvcNkcy4u5nm9k8gnD5EPAlgusudALnuXvezHYBjZM834C/dvevHeN+RV5Bh49kNnuI4LoFAJjZ2JTG84A94f33Rbj/xwkOWwFcfbSN3X2Q4FKdnzCzJEGdB8JAuAQ4Kdz0INBW9tQHgevC+fwxs+VmtniGfgeZYxQKMls0m1lX2e3jBB+w68LO12cJpjwH+Bzw12b2KBCPsKaPAR83sw3AMmDwaE9w96cIZti8muCCNuvMbCNBq+H5cJte4NFwCOuN7v4QweGpn5rZVuBuXh4aIlOmIakiEQmvFjfi7m5mVwPvcfe3H+15ItWkPgWR6JwHfCUcMTRABS57KjJdaimIiMg49SmIiMg4hYKIiIxTKIiIyDiFgoiIjFMoiIjIOIWCiIiM+/9fnou/vnR1yQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.lr_find()\n",
    "learn_lm.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 00:38 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.901358</td>\n",
       "      <td>4.004862</td>\n",
       "      <td>0.300022</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.766921</td>\n",
       "      <td>3.870316</td>\n",
       "      <td>0.315848</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.581739</td>\n",
       "      <td>3.816319</td>\n",
       "      <td>0.324263</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.390822</td>\n",
       "      <td>3.809503</td>\n",
       "      <td>0.326473</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>3.221774</td>\n",
       "      <td>3.827946</td>\n",
       "      <td>0.326362</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>3.119199</td>\n",
       "      <td>3.827676</td>\n",
       "      <td>0.326339</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.fit_one_cycle(6, slice(5e-3/(2.6**4),5e-3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.save_encoder('fine_tuned_encoder_ASTD_acc326_min4')\n",
    "# 3.935220\t4.281686\t0.295004"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'الله يبارك في +ك  ما شاء الله في هجوم ثالث من اقوي ال+ رب   حمايه مصر و+ ال+ مجتمع ال+ عربي و+ حمايه حقوق + نا بقيت ال+ ارهاب يقدر فقط ل+ وحد'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_lm.predict('الله يبارك في +ك ', n_words=30)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "ar_tok = Tokenizer(lang='ar')\n",
    "data_clas = TextClasDataBunch.from_df(path, train_df=df_train, valid_df=df_val, text_cols=1, label_cols=0, tokenizer=ar_tok, bs=32, vocab=data_lm.train_ds.vocab, include_bos=False, min_freq=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas.save(path/'data'/'data_clas_ASTD_min4.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas = load_data(path, path/'data'/'data_clas_HARD.pkl', bs=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>الناصبي مرسي ف اخر ليل + ه ل + ه استخدم مفرد + ه ال+ شرعيه عشرات ال+ مرات لان عقد + ه ال+ شرعيه عند + ه وصلت حد + ها و+ هذا عقد + ه نقص + ه شرعي + ه قياد + ه ال+ بحارنه من زمان</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>و+ من يردد + ها هم من لا نريد + هم في مصر لا في ال+ سياسه و+ لا في ال+ برلمان ال+ قادم و+ ك+ فاهم ال+ ضرر الذي احدثوا + ه ل+ ثقافه + نا و+ وعي + نا ال+ سياسي بلغه + هم و+ اساليب + هم</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ال+ مصري ال+ حقيقي ك+ ال+ هرم ال+ شامخ رفقه + ه شرف و ال+ تواصل مع + ه حق و نسيان + ه محال و ال+ دعاء ل + ه واجبكل عام و انتم ب+ خير و مصر طيب + ه و حره امن + ه</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>من قام ليل + ه ال+ قدر ايمانا و+ احتسابا غفر ل + ه ما تقدم من ذنب + ه قف بين يدي ال+ رحمن و+ اطل ال+ وقوف و+ قل يا رب اتيه + ك ب+ ذنوب كثير + ه و انت ال+ عفو ال+ كريم</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>ابراهيم عيسي قلم شجاع و+ ضمير يقظ و+ علام + ه بارز + ه في تاريخ ال+ مهنه تحي + ه ل + ه و+ ل+ فريق ال+ دستور و+ س+ تبقي حريه ال+ صحافه في مصر ثمر + ه مستحق + ه ل+ نضال ال+ صحفيين</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_clas = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.2, metrics=[accuracy,FBeta(average='weighted')])\n",
    "learn_clas.load_encoder('fine_tuned_encoder_ASTD_acc326_min4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8lNW9x/HPL/tCCIEsQAKEIPsOQeGqiEpxqVVxqdq6tV6tXbTr7W2v19rq7WJtr1qXqrVor1q07lsFV4o7O7LLDmFLgCyETMgkOfePGTBAkgkhs+b7fr3mxeR5npn5TkLml3PO85xjzjlERERaExfuACIiEvlULEREJCAVCxERCUjFQkREAlKxEBGRgFQsREQkIBULEREJSMVCREQCUrEQEZGAEsIdoKNkZ2e7wsLCcMcQEYkqCxcu3O2cywl0XMwUi8LCQhYsWBDuGCIiUcXMNrflOHVDiYhIQCoWIiISkIqFiIgEpGIhIiIBqViIiEhAKhYiIhKQioWIiASkYiEiEsWeX1jCzHlbgv46KhYiIlHshcUlPLewJOivo2IhIhLFPHUNpCbGB/11VCxERKJYTV0DqUkqFiIi0gqPVy0LEREJwFPXQJpaFiIi0hqPuqFERKQ1zjlq1A0lIiKt8TY4GhqduqFERKRlnroGAFKTgr+OnYqFiEiUqvHWA6gbSkREWnawZaFuKBERaVHNoW4oFQsREWmBx+svFuqGEhGRlqgbSkREAlI3lIiIBFQbC91QZjbDzErNbHkL+6eYWaWZLfHfftFk39lmtsbM1pnZz4KVUUQkmtUc6oaK7ussHgfODnDM+865Mf7b7QBmFg88AJwDDAOuMLNhQcwpIhKVaupi4DoL59xcYG87HnoisM45t8E5Vwc8DVzQoeFERGLAoW6oTjBmMcnMlprZG2Y23L8tH9ja5JgS/7aI45zj1peW8+9/m4+3oTHccUSkk6mpayAhzkhKCP5HeTiLxSKgn3NuNHAf8JJ/uzVzrGvuCczsBjNbYGYLysrKghSzZQ/OWc8Tn2zm7VWl/PafqwMev3BzOV99+GNeWbo9BOlEJNbVhGhJVYDgj4q0wDlX1eT+P83sQTPLxteS6NPk0AKg2U9X59wjwCMAxcXFzRaUYJm1fAd3zV7DhWN60y0tiRkfbqS4MItzR/Y66tiGRscD763j3nfWEmcwb+NetpV7uPG0Isyaq40iIoHVekOzlgWEsViYWU9gl3POmdmJ+Fo5e4AKYKCZ9Qe2AZcDXwtXzuYs31bJD59Zyti+3fjdxaOIM2NpSQU/fe4zhvTMoCiny6Fjt1V4+OHTS5i3aS8XjOnNrecN41evruTOWaspKa/hV+cPJyE+3L2BIhKNakK0Sh4EsViY2UxgCpBtZiXAbUAigHPuIeAS4NtmVg94gMudcw6oN7PvAbOBeGCGc25FsHIeqy17arjh/xaQlZbIw1eNJ8XfBHzga+M4774P+PaTi/jTFWP5cN1u3l1dyqcb95AUH8fdl41m+tgCAO69bAz53VJ56F/rKSn3cNmEPgzI6UJhdhrJCaH5wUerOWtK+dM7a4mPM7LSkujRJYn0pAT21dZT6fFS4amj0cGAnHQG5HThhNwujC7oRlZ6Uriji3S4mrqGQ59BwWa+z+foV1xc7BYsWBCU566q9TJr2U5eXLyNTzbuITUxnmdvnMTw3pmHHff+2jKunjGPg9/SgbldOGNILl8/qR99e6Qd9bxPfLKZ219dgbfB94A4gyE9u/Lr6SMY2zfrmDO+smQ7nroGuqYmkJGSSFZaEsWFWSRGUculodHx/toy0pMTGNc3i/g4Xzdd9YF6fv36KmbO20JhjzR6ZqZQvt/L3po6qmvryUhJoFtaIpmpiTgH68uqKa/xApCeFM+Ppg3mmkn9jmrF7a4+QEZKgoq0RKUrH/2Umrp6XvjOye1+DjNb6JwrDnicikXLVu2o4rEPN/Lyku0cqG+kf3Y608fmc9G4fAqyjv7wB3jts+3s3neAM4bkNVsgjlRTV8+Gsv2sL6tmfWk1zy/axq6qWn52zhCuO6V/wDGNrXtreOzDTTwzfwv7/RfoNDUorwu3XzCCiUU92vamQ2Dl9iq2ltcwpGcGfbLSiIszaurqeW5hCY++v5Ete2sA6JGexNSheYzt240H5qyjpNzD9acW8aMvDWrTX1N7qg/w+a5qHp67njlryhjeuyu/nj6S/j3SeW3Zdl5ctI0Fm8vJ75bKz84ZwnmjemkMSaLKRQ9+SFpSAk/++0ntfg4Vi3ZyzvHu6lL++sFGPlrva0VMH5fPV4v7MLogM+gfJpU1Xn7y3FLeWrmLqUPz+MOlo+iWdngXinOOhZvLeezDTbyxfAdxZpw3qhfXnVJEYXYaVbX17Kv18vmuau58YzXbKjxMH5vPz88dQm5GSlDzt2RfrZdXlm7nmflb+ayk8tD2LskJDOmZwbqyaipqvIzt243rTy2iodHx5spdzFldyr4D9fTtnsYfvzqaCYXdj/m1nXO8sXwnv3p1BaX7DpAYF0ddQyMn5Hbh3BE9eWtVKat2VDG+Xxa3njeMMX26deRbFwmas++ZS0FWGo9eE/CzvkUqFu2wcPNe/uf1VSzeUkGvzBSunlTIFSf2OerDOticczz24SZ++8YqUhLjKe6Xxfh+WYzv153tFR4e+2gjy7dV0TUlgStO7Mu1JxfSKzO12efy1DXw4Jx1PPyvDSTGG+eN6s0FY3szsX8P4uJC81f0U59u5n9eW4XH28CQnhlcPqEPo/p04/Od+1i5o4qV26vIyUjmulP6U3xEMairb2TljioG5XU57ikN9tV6efhfG/B4G7hwTD4j8rtiZjQ0Op5buJW7Zn/O7uoDfOu0In561pBDXWAikWrKXe8xqqAbf7pibLufQ8XiGGzes5/fz1rD68t2kJuRzI+nDeKicQVh7+tfVlLJk59sZuGWctaVVh/afkJuF679t0IuGpff5g/QDWXVPPDeemYt38H+ugZ6dk3h4vH53HTGwKAOkD0ydz2/+edqJg/K4cdfGsSoELTO2qv6QD2//ecqnvp0C5MH5XDf5WPJTEsMdyyRFp3467c5fXAud14yqt3PoWLRRhvKqjnrnrkkxMXxrdOKuP7UItKTw3ZGcYsqaupYvKWClMR4JhZ1b/cHrqeugbdX7eLlJdt4e1UpQ3t15cGvj6N/dnqH5nXOce87a7nn7bWcN6oXd182JuzFt62enreFW19eTu9uqfzl6mIG5WWEO5JIs0beNpuLxxfwy/OHBz64BSoWbeSc468fbOQro3uT1zU8/fnh8t6aUn74zBLqGxx3XTKKc5q5oLA9Ghodv5+1mofnbuCS8QXcefGoqOvSWbh5Lzc+uYj9B+r5nwtHMH1sfsS2iKTzOuG//skNk4v46dlD2v0cKhbSJtsqPHznqUUs3VrB1KG5pCUlUN/YiLfB0T87nasm9qNP98PP6lq7ax8vLdlGckI8Rf7rGfK6pjBv4x7eWlnKu6t3UV7j5epJ/fjlV4aHbGyko+2qquWmvy9m3qa9nD+6N3dcOILMVHVLSWSoq29k0H+/wY+/NIibzhzY7udpa7GIvP4WCan8bqk8+61J/H7Wamat2ElCnJEQH0dCnPHe6lIefX8DU4fmce3JhezdX8cTH2/m0417iY/zDQwfKTM1kTOG5HLW8J6cNTwvqv8az+uawswbJvLnOeu4++21LNxczq3nDaOmrp7VO/exaodvxprvnn5CRJ2aLJ2DJ4QzzoJaFtKKnZW1PPHJJp76dAsV/gvc+nRP5Wsn9uOrxQWkJSWwcbfvGpHtFR5G9+lGcb+smJy+ZPGWcn7wzBI27/FdA5KUEMfA3C7srj7ArqoDnDkkl/88Z4jGNyRkdlbWMvG37/Cb6SP52kl92/086oaSDuOpa2D2ip1kpiUyeWBO1I0/dJT9B+r5dOMe+nZPo7BHOgnxcdR6G5jx4Ub+/N569tfVc+XEfvzXuUNDNgWDdF4byqo544//OmwqofZQN5R0mNSkeC4cG5FLioRUenICZwzJO2xbSmI835lyAldM6Mu976zl8Y82sWRrBX++cjz53Zq/9kWkIxzqhkoMzcd47PUXiIRBVnoSvzx/OI9cNZ6NZfs570/v88Ha3eGOJTHMc2j97dC0YlUsRDrQtOE9efl7J5PdJZmrZ3zKPW9/Tl29VlGUjldTF9oBbhULkQ5WlNOFl757MueP7u27KPG+91m0pTzcsSTGfNENpWIhErXSkxO45/KxPHp1Mftq67n4zx9x28vL2X+gPtzRJEaEuhtKA9wiQTR1WB4TB/TgD7PX8LePN7FxTw2PXTuh055RJh1H3VAiMaZLcgK/PH84v5k+krmfl3HvO2vDHUliwMFuqDSdDSUSWy6f0IdLxxfwp3fW8t7q0nDHkSjnqfN1aaYkheZjXMVCJETMjDsuHMGwXl35/tOL2eK/GlykPWrqGoiPM5JCNGOCioVICKUkxvPQleMBuPHJhYcGKUWOlcfbQFpifMjmX1OxEAmxvj3SuPuyMazcUcUFD3zA8m2VgR8kcgRPXQMpIRrcBhULkbA4c2gej39jAhU1XqY/+CEPvLeO+gZdvCdt5/E2hOy0WdCpsyJhM2VwLrN/MJn/fnk5d81ewxvLdzAw94tZazNTE7l+cpHmmJJm1dQ1hOyCPFCxEAmrrPQk7r9iLNOG5XH/u+tYuPmLK713VtXy9Pwt3HjaAL41eUDIzqeX6OCpawjp/wkVC5EwMzMuGJPPBWMOn9l3W4WH3/xzFfe8vZZnF5Rw21eGMW14zzCllEgT6m4ojVmIRKj8bqk88LVxPHPDRDJSEvj2U4s0GC6H+LqhQvf3voqFSIQ7qagHT98wkay0RH7+wjINhAvguygvlN1QKhYiUaBbWhK3fWU4y7ZV8vhHm8IdRyLAwessQkXFQiRKnDeqF2cMyeWPb37O1r26+ruzqwnxALeKhUiUODhdSJzBLS8txzkX7kgSRqE+G0rFQiSK5HdL5SdnDWbu52X8fd4WFYxOytvQSH2jUzeUiLTs6kmFjO+XxS0vLmfyXe/xh9lrWFe6L9yxJIRCvZYFqFiIRJ34OOOJ607kj5eOprBHOg/OWcfU/53LzTMXa73vTsIThmKhi/JEolBaUgIXjy/g4vEFlO6r5YmPN3Pfu+uo9Hh56Mrxuto7xh1a+CgWWhZmNsPMSs1seYDjJphZg5ld0mRbg5kt8d9eCVZGkViQm5HCj6cN5s6LR/L+2jKunvEpVbXecMeSIKrxL3wUyrmhgtkN9ThwdmsHmFk8cCcw+4hdHufcGP/t/CDlE4kpl03oy31XjGPJ1gqueOQT9lQfCHckCZJa78FuqBi4gts5NxfYG+Cwm4DnAa0xKdIBvjyqF3+5upj1ZdVc9dd5VHrUwohFBwe4Y6IbKhAzywemAw81szvFzBaY2SdmdmErz3GD/7gFZWVlQcsqEk2mDM7l4auKWVu6j+sen3+oy6I59Q2NzJy3hVteXMbG3ftDmFKOx6GzoWKkGyqQe4D/dM41t65kX+dcMfA14B4zG9DcEzjnHnHOFTvninNycoKZVSSqnDYoh3svH8uiLeXc+OSio86Scs7xzqpdnHPv+/z8hWU8PX8rZ90zl/vfXaszqqLAF91QneNsqGLgaf/6sdnAuWZW75x7yTm3HcA5t8HM5gBjgfVhSyoShc4d2YvfXTSKnz7/GTfPXMxXRvemdF8tpfsOsHBTOfM27aV/djoPXTmesX27cfurK/nDm5/zytLt/PaiUYzvlxXutyAtCEc3VNiKhXOu/8H7ZvY48Jpz7iUzywJqnHMHzCwbOBn4fZhiikS1r07oQ1Wtl/95fRWzVuwEICHOyM9K5Y4LhnP5iX1JjPd1MDzw9XFctGoXt760nMsf+ZgHvjZO62dEqHB0QwWtWJjZTGAKkG1mJcBtQCKAc665cYqDhgIPm1kjvm6y3znnVgYrp0is+/dTi5g8KIdG58jNSKFbaiJxcdbssWcOzaO4X3eufmwe33lqEfddMZZzRvYKcWIJJKa6oZxzVxzDsdc2uf8RMDIYmUQ6q0F5GYEP8stMS+SJ607kG4/N53szF3N3o+P80b2DmE6OVU1dPXEGSfGhG3bWdB8icpSuKYn87ZsnMr5vFj94ejEvL9kW7kjSRE1dA2lJCfjHfENCxUJEmtUlOYHHvzmBCYXd+Y9nP2PxlvJwRxK/Wm9opycHFQsRaUVaUgIPXzWevMxkvv3kIsr26arwSOBbf1vFQkQiSLe0JB6+spgKTx3ffWoRXq0BHna+bigVCxGJMMN6d+XOi0cxb9Nefv36qnDH6fTC0Q2lKcpFpE0uGJPP0q2VzPhwIwNy0rlqUmG4I3Va6oYSkYj283OHMGVwDre+vIKfv/DZofP9JbQ86oYSkUiWGB/Ho1cX8+0pA5g5byuXPvQxJeU14Y7V6Xi8DSGdnhxULETkGCXEx/GfZw/hkavGs2n3fs677wPmbQy0GoF0pJq6elITQ/vxrWIhIu0ybXhPXr3pFHqkJ/GNx+axSNdhhIzHf1FeKKlYiEi7FWan8/frJ5Kdkcy1M+axYntluCN1Ch5dlCci0SavawpP/ftJdElO4Kq/zmPtrn3hjhTTvA2NeBuczoYSkehTkJXGU9dPJD7O+Pqjn7Kj0hPuSDHL4w39WhagYiEiHaR/djpPXncS5TV1PDRHa5UFi8e/lkWKWhYiEq0G98zggjH5/GNBCeX768IdJyaFY5U8ULEQkQ52w+QiPN4Gnvhkc7ijxCSPioWIxIJBeRmcPjiHv320SVd4B4HHWw+oG0pEYsD1k4vYs7+O5xeVhDtKzPmiG0rXWYhIlJtU1IOR+Zk8+v5GGhpduOPEFHVDiUjMMDNumFzExt37eXvVrnDHiSkHT51VN5SIxIRzRvSkICuVR+ZuCHeUmKKzoUQkpiTEx3HdKf1ZuLlc63d3oIjuhjKzAWaW7L8/xcxuNrNuwY0mItHukvEFpCfF6zTaDhTp3VDPAw1mdgLwV6A/8PegpRKRmJCRksj0cfm89tkOXaTXQTx1DcQZJCdE5hTljc65emA6cI9z7odAr+DFEpFYceXEftTVN/Lswq3hjhIT9tbUkZoYj5mF9HXbWiy8ZnYFcA3wmn9bYnAiiUgsGdKzKxMKs3jq0y006jTa47Jm5z6eW1DClMG5IX/tthaLbwCTgF875zaaWX/gyeDFEpFYcuXEfmzeU8P763aHO0rUqqtv5IfPLKFragK3XzA85K/fpmLhnFvpnLvZOTfTzLKADOfc74KcTURixNkjepLdJYknPtZAd3v96Z21rNxRxW+mj6RHl+SQv35bz4aaY2Zdzaw7sBR4zMz+N7jRRCRWJCfE89XiPry7ehfbKrTWxbFatKWcB+es49LxBUwb3jMsGdraDZXpnKsCLgIec86NB6YGL5aIxJqvndQXB8z8dEu4o0SVmrp6fvyPpfTKTOUXXxkWthxtLRYJZtYL+CpfDHCLiLRZQVYaZw7J5en5W6j0eMMdJ2r8Y/5WNu7ez12XjiIjJXznFbW1WNwOzAbWO+fmm1kRsDZ4sUQkFt10xkAqarz8/IXPcE5nRrXFgs3l5HdL5d8GZIc1R1sHuJ91zo1yzn3b//UG59zFwY0mIrFmdJ9u/MdZg/nnsp08qe6oNlmytYIxfcI/YUZbB7gLzOxFMys1s11m9ryZFQR4zAz/8csDHDfBzBrM7JIm264xs7X+2zVteysiEg2uP7WIKYNzuOO1lazYXhnuOBFtd/UBSso90VMsgMeAV4DeQD7wqn9bax4Hzm7tADOLB+7E18V1cFt34DbgJOBE4Db/6boiEgPi4ow/XjqarLREvvf3xVQfqA93pIi1ZEsFAGP6Rk+xyHHOPeacq/ffHgdyWnuAc24usDfA896Eb96p0ibbzgLecs7tdc6VA28RoOiISHTp0SWZey8fy+Y9+7n1pVY7Hzq1JVsriI8zRvTODHeUNheL3WZ2pZnF+29XAnuO54XNLB/fXFMPHbErH2g6iUyJf5uIxJCJRT343hkDeXHxNt5bUxr4AZ3Qkq0VDM7LIDXE05E3p63F4pv4TpvdCewALsE3BcjxuAf4T+fckSu6Nzc7VrOnTZjZDWa2wMwWlJWVHWccEQm1754+gKKcdG59afmhdRrEp7HRsbSkIiK6oKDtZ0Ntcc6d75zLcc7lOucuxHeB3vEoBp42s034is+DZnYhvpZEnybHFQDbW8j1iHOu2DlXnJPTaq+YiESg5IR4fjN9JCXlHu59R2fjN7Vh93721dZHxOA2HN9KeT86nhd2zvV3zhU65wqB54DvOOdewjfYPc3MsvwD29NoMgAuIrFlYlEPLh1fwKPvb2D1zqpwx4kYS7b6BrfHxkCxaHUydTObCXwMDDazEjO7zsxuNLMbW3ucc24vcAcw33+73b9NRGLUf507lK6pifzXC8s0jbnfkq3lZCQnMCCnS7ijAJBwHI9t9SfqnLuizU/k3LVHfD0DmNG+WCISbbLSk7jl3KH8+NmlzJy/ha+f1C/ckcJuydYKRvXJJC4utIsctaTVloWZ7TOzqmZu+/BdcyEi0iEuGpdPcb8s/jxnfadvXdR6G1i9Y1/EjFdAgGLhnMtwznVt5pbhnDueVomIyGHMjKsm9aOk3MOnGzt3z/PybZXUNzrG9Imc65FDu+K3iEgrpg3rSZfkBJ5bWBLuKGF1cHB7dJ/wX4x3kIqFiESM1KR4zhvVizeW72B/J54GZPHWCvK7pZKbkRLuKIeoWIhIRLlkfAE1dQ28sXxnuKOEzZItkTHTbFMqFiISUcb3y6KwRxrPLdwa+OAYVLbvANsqImOm2aZULEQkopgZF48r4JMNe9m6tybccUJu1Q7fhYkj8iNnvAJULEQkAl00vgAzeGHRtnBHCbnymjoAcjKSw5zkcCoWIhJxfMuI9uC5RVs73TUXVf71yTNTw7fednNULEQkIl08roCtez3M39S5rrmo9BeLrqmRdSmbioWIRKSzR/QkPSmel5Y0O+l0zKr0eElJjCM5IfxrWDSlYiEiESktKYFTB+YwZ00pznWerqhKjzfiuqBAxUJEItiUwTnsqKzl813VR+3bXuHhgvs/YH3Z0fuiWZWnXsVCRORYTBmcC9DssqvPLSxhaUklz8fY1CBqWYiIHKOemSkM7dWV91YfXiycc7yy1DeWMXtFbF3prWIhItIOUwbnsHBzOVW13kPbVu3Yx7rSaob16sr6sv2sK42drqhKj5euKSoWIiLH5PTBudQ3Oj5cu/vQtleWbichzrjr0lFAbLUuqjxeuqplISJybMb17UZGSsKhcQvnHK8u3c4pA7MZ3juT0QWZvLlyV5hTdoyGRse+AxrgFhE5ZgnxcUwemMOcNWU451i0pZxtFR7OH+1brHPa8J4s3VrBzsraMCc9fpF69TaoWIhIFJgyOIfSfQdYuaOKV5ZsJzkhjmnDewJwlv/fN1dGf1fUwXEZFQsRkXY4bXAOAO+sKuX1ZTs4c2guXZJ902GckNuFATnpMTFuUamWhYhI++VmpDAivyt/eX8Du6vrDnVBHTRteE8+2bCXCv+MrdHqULFIU7EQEWmX0wfnsq+2nozkhEMX6x101vCeNDQ63l199MV70eTQJII6dVZEpH2m+Luipg3vSUri4ZPsjcrPpGfXlKjvilI3lIjIcRrTJ4tvnVbEt6cMOGpfXJwxbXge//q8DE9dQxjSdQwVCxGR4xQfZ/z8nKGckNul2f1nDe9JrbeRuWvLQpys41R6vCTFx5GSGHkfzZGXSESkHU7s351uaYnMXh69XVFVnnq6piZiZuGOchQVCxGJCYnxcUwdmsfbq3ZRV98Y7jjtUuXxkhlhK+QdpGIhIjHj7OE9qaqt55MNe8IdpV0qI3ReKFCxEJEYcsrAbNKS4pkVpWdFRer05KBiISIxJCUxntOH5PLmil00NEbfUqwqFiIiIXL28J7srj7Aoi3l4Y5yzFQsRERC5PQhuSQlxDErys6Kamx0VNWqWIiIhESX5AQmD8xm1vKdOBc9XVHVdfU4F5kX5EEQi4WZzTCzUjNb3sL+C8zsMzNbYmYLzOyUJvsa/NuXmNkrwcooIrHprOE92VbhYcX2qnBHabPKGv+8UBFaLIJ5Qu/jwP3A/7Ww/x3gFeecM7NRwD+AIf59HufcmCBmE5EYNnVoHvFxxqzlO8nJSOatlbt4a+Uu+nZP444LR4Q7XrMieRJBCGKxcM7NNbPCVvY3XWE9HYie9qKIRLSs9CQmFnXnkfc3cP976wDISE5g7toybjrjBHK7poQ54dEieZU8CG7LIiAzmw78FsgFvtxkV4qZLQDqgd85514KRz4RiV7XndKfhkbHqQNzmDYsD4Av3T2XWSt2cvWkwvCGa0YkTyIIYS4WzrkXgRfNbDJwBzDVv6uvc267mRUB75rZMufc+iMfb2Y3ADcA9O3bN1SxRSQKnDEkjzOG5B22bWBuF17/bEdkF4sIXPgIIuRsKOfcXGCAmWX7v97u/3cDMAcY28LjHnHOFTvninNyckIVV0Si1LkjezFv015K99WGO8pRIr1lEbZiYWYnmH9qRTMbByQBe8wsy8yS/duzgZOBleHKKSKx48ujeuEcETkzbVWtl/g4Iz0pPvDBYRC0bigzmwlMAbLNrAS4DUgEcM49BFwMXG1mXsADXOY/M2oo8LCZNeIrZr9zzqlYiMhxG5SXwQm5XXh92Q6uirCuqEqPl64pCRE5PTkE92yoKwLsvxO4s5ntHwEjg5VLRDq3c0f24v5311K6r5bcjMg5K6rSUx+xXVAQIWMWIiKh8uWRvWiMwK6oSJ4XClQsRKSTGZTXhQE56by+bEe4oxwmkteyABULEelkzIwvj+zFvI17Kdt3INxxDqlSy0JEJLKcO8rXFRVJiySpG0pEJMIMzsugKCed15ZuD3cUAJxzalmIiEQaM+PCMfl8unEvJeU14Y5DTV0D9Y1OYxYiIpFm+th8AF5eEv7WRaRfvQ0qFiLSSfXpnsaEwixeWFQS9kWSVCxERCLY9LEFrC/bz7JtlWHNoWIhIhLBvjyyF0kJcbywaFtYc6hYiIj66sU5AAAOqElEQVREsMy0RKYOzeXVpdvxNjSGLUekL3wEKhYi0slNH1vAnv11zP28LGwZIn1JVVCxEJFO7rRBOWSlJfLC4vB1RVV5vJhBRkpY16NrlYqFiHRqSQlxfGV0b95auYuqWm9YMlR6vGQkJxAXF5nTk4OKhYgIF40roK6+kdeWhmdywUqPN2KXUz1IxUJEOr3RBZmMKsjkf99aw979dSF//UifFwpULEREMDPuvHgUlR4vt72yokOf2znHK0u3s6yk5Ws5VCxERKLE0F5dufmMgby6dDuzlndMd9SOSg9Xz5jHzTMX898vL2/xuKra+og+EwpULEREDrlxygBG5Hflv19aflzdUc45XlxcwrS757JgUzmTinqwdGsFpVW1zR6vloWISBRJjI/jD5eOptLj5RettAQC+f3sNfzwmaUMysvgje+fym3nDwPgndWlzR6vYiEiEmWG9OzK988cyGuf7eCtlbuO+fFl+w7w1w82cv7o3vzjW5MozE5ncF4GBVmpvN3M89V6G6irb4zo6clBxUJE5Cg3njaAoux0/vjmGhobj21G2r99tAlvQyM//NIg4v3XTZgZU4fm8cG63XjqGg47vrzG192lloWISJRJiI/j+1MHsnrnvmNaenX/gXqe+GQz04bl0T87/bB9XxqWx4H6Rj5Yt/uw7QfX0xhd0O34gweRioWISDPOG9WbE3K7cM/bn7e5dfHM/K1Uerx867QBR+07sX93MlISDuuKqqmr5y9zNzB5UA4jCzI7LHswqFiIiDQjPs64+cyBfL6rmteXBT6Vtr6hkb9+sJEJhVmM65t11P7E+DimDM7lndW7DhWfv3+6hT3767j5jBM6PH9HU7EQEWnBl0f2YmBuF+59Zy0NAVoXry/bwbYKDzdMPrpVcdDUobnsrq5jSUkFtd4GHp67gUlFPSgu7N7R0TucioWISAvi44zvTx3IutJqXvus5bW6nXM8MncDA3LSOXNIbovHTRmUS0Kc8fbKXTwzfytl+w5w05mR36oAFQsRkVadO6IXg/MyWm1dfLhuDyu2V3HD5KJWZ47NTEtkQmF3Zq/YyUP/Wk9xvywmFfUIVvQOpWIhItKKOP/YxYay/Xx4xJlMB72ydBuZqYlcODY/4PNNHZbH+rL97Kis5eYzB2IWudOSN6ViISISwJTBOcTHGfM37W12/4JN5UwozCI5IT7gc00d6uumGt2nG6cOzO7QnMEUucsyiYhEiPTkBIb37tpssdhTfYANu/fz1Ql92vRc/Xqkc8u5Q/m3E3pETasC1LIQEWmT4n7dWbK1grr6xsO2L9xc7t9/9OmyLbl+chHDe0f2dRVHUrEQEWmDCYVZ1HobWbH98HUpFmwuJyk+jhH50fXhf6xULERE2mB8oa/lsGBT+WHbF2zay8iCTFISA49XRLOgFQszm2FmpWbW7Dy/ZnaBmX1mZkvMbIGZndJk3zVmttZ/uyZYGUVE2io3I4XCHmmHjVvUehtYvq3qmLqgolUwWxaPA2e3sv8dYLRzbgzwTeBRADPrDtwGnAScCNxmZrH/kxCRiFdc2J0Fm8txzne9xbJtldQ1NEbFFdjHK2jFwjk3F2j+PDPf/mp38DsO6cDB+2cBbznn9jrnyoG3aL3oiIiExITCLPbur2PD7v0Ah1oZ49WyCC4zm25mq4HX8bUuAPKBrU0OK/Fva+7xN/i7sBaUlZUFN6yIdHoHWxAL/EVi4aZyinLS6Z6eFM5YIRHWYuGce9E5NwS4ELjDv7m5E4+bvcbeOfeIc67YOVeck5MTrJgiIgAUZfsKw/xN5TQ2OhZuKe8U4xUQIWdD+busBphZNr6WRNOrWwqAlmfwEhEJETOjuF8WCzbtZX1ZNRU1Xor7xf54BYSxWJjZCea/fNHMxgFJwB5gNjDNzLL8A9vT/NtERMJuQmF3Nu2pYdZy3wp6xYWdo2URtOk+zGwmMAXINrMSfGc4JQI45x4CLgauNjMv4AEu8w947zWzO4D5/qe63TnX4kC5iEgoHSwOj3+0iR7pSUctnxqrglYsnHNXBNh/J3BnC/tmADOCkUtE5HgM751JSmIce/bX8aVheVE1v9PxiIgxCxGRaJGUEMeYPt2AY5sPKtqpWIiIHKMJ/lNoO8t4BWiKchGRY3bZhD54GxyjC7qFO0rIqFiIiByjgqw0fnbOkHDHCCl1Q4mISEAqFiIiEpCKhYiIBKRiISIiAalYiIhIQCoWIiISkIqFiIgEpGIhIiIB2Rcrm0Y3M6sE1jazKxOoDLCt6dfN3T/4bzawu50Rm8vRlv3Kf/i29r6HQPlbO6a1vEd+Hei+8h/7MYH+D7X0fjoyf2v5Au2P9N/hfs65wKvHOedi4gY80tbtR25r+nVz95v8u6Cj8yl/2/Mfz3sIlP9Y3sOx5u+In4Hyt7ytpffTkfnb8h6i/Xc40C2WuqFePYbtR257NcD9lp77WAR6DuVv+X4o8rd2TGt5j/y6LffbQ/lb3tbS++nI/G15jlj4HWhRzHRDhYKZLXDOFYc7R3tFe36I/veg/OGl/O0XSy2LUHgk3AGOU7Tnh+h/D8ofXsrfTmpZiIhIQGpZiIhIQJ22WJjZDDMrNbPl7XjseDNbZmbrzOxP1mQRXjO7yczWmNkKM/t9x6Y+LEOH5zezX5rZNjNb4r+d2/HJD2UIyvffv/8nZubMLLvjEjebIxg/gzvM7DP/9/9NM+vd8ckPZQhG/rvMbLX/PbxoZkFbHShI+S/1/+42mllQxgaOJ3cLz3eNma31365psr3V35NjdjynUkXzDZgMjAOWt+Ox84BJgAFvAOf4t58OvA0k+7/OjbL8vwR+Eq3ff/++PsBsYDOQHW3vAeja5JibgYeiLP80IMF//07gzijLPxQYDMwBiiMptz9T4RHbugMb/P9m+e9ntfYe23vrtC0L59xcYG/TbWY2wMxmmdlCM3vfzI5aCsvMeuH7hf7Y+X4i/wdc6N/9beB3zrkD/tcojbL8IRPE/HcDPwWCPhgXjPfgnKtqcmg6QXwfQcr/pnOu3n/oJ0BBlOVf5ZxbE6zMx5O7BWcBbznn9jrnyoG3gLOD8XveaYtFCx4BbnLOjQd+AjzYzDH5QEmTr0v82wAGAaea2adm9i8zmxDUtEc73vwA3/N3Icwws1CvRn9c+c3sfGCbc25psIO24rh/Bmb2azPbCnwd+EUQszanI/4PHfRNfH/RhlJH5g+ltuRuTj6wtcnXB99Lh79HrcHtZ2ZdgH8Dnm3StZfc3KHNbDv4118CvqbgRGAC8A8zK/JX9qDqoPx/Bu7wf30H8Ed8v/BBd7z5zSwNuAVfN0hYdNDPAOfcLcAtZvZz4HvAbR0ctVkdld//XLcA9cBTHZmxNR2ZP5Ray21m3wC+7992AvBPM6sDNjrnptPye+nw96hi8YU4oMI5N6bpRjOLBxb6v3wF3wdq06Z1AbDdf78EeMFfHOaZWSO+uVzKghnc77jzO+d2NXncX4DXghn4CMebfwDQH1jq/4UrABaZ2YnOuZ1Bzn5QR/wfaurvwOuEqFjQQfn9g6znAWeG4g+lJjr6+x8qzeYGcM49BjwGYGZzgGudc5uaHFICTGnydQG+sY0SOvo9BmMAJ1puQCFNBpmAj4BL/fcNGN3C4+bjaz0cHDg617/9RuB2//1B+JqHFkX5ezU55ofA09H0/T/imE0EeYA7SD+DgU2OuQl4Lsrynw2sBHKC/b0P5v8hgjjA3d7ctDzAvRFfj0aW/373trzHY84cih9oJN6AmcAOwIuvCl+H7y/TWcBS/3/4X7Tw2GJgObAeuJ8vLm5MAp7071sEnBFl+Z8AlgGf4fsLrFc05T/imE0E/2yoYPwMnvdv/wzfXD75UZZ/Hb4/kpb4b8E8mysY+af7n+sAsAuYHSm5aaZY+Ld/0/99Xwd841h+T47lpiu4RUQkIJ0NJSIiAalYiIhIQCoWIiISkIqFiIgEpGIhIiIBqVhITDOz6hC/3qNmNqyDnqvBfLPPLjezVwPN4Gpm3czsOx3x2iJH0qmzEtPMrNo516UDny/BfTFRXlA1zW5mfwM+d879upXjC4HXnHMjQpFPOhe1LKTTMbMcM3vezOb7byf7t59oZh+Z2WL/v4P92681s2fN7FXgTTObYmZzzOw5863d8NTBtQL824v996v9kwIuNbNPzCzPv32A/+v5ZnZ7G1s/H/PFhIldzOwdM1tkvvUKLvAf8ztggL81cpf/2P/wv85nZvarDvw2SiejYiGd0b3A3c65CcDFwKP+7auByc65sfhme/1Nk8dMAq5xzp3h/3os8ANgGFAEnNzM66QDnzjnRgNzgeubvP69/tcPOF+Pf26jM/FdVQ9QC0x3zo3Dt4bKH/3F6mfAeufcGOfcf5jZNGAgcCIwBhhvZpMDvZ5IczSRoHRGU4FhTWb47GpmGUAm8DczG4hvhs7EJo95yznXdA2Cec65EgAzW4Jvrp8PjnidOr6YjHEh8CX//Ul8sbbA34E/tJAztclzL8S3VgH45vr5jf+DvxFfiyOvmcdP898W+7/ugq94zG3h9URapGIhnVEcMMk552m60czuA95zzk339//PabJ7/xHPcaDJ/Qaa/13yui8GBVs6pjUe59wYM8vEV3S+C/wJ3zoXOcB455zXzDYBKc083oDfOucePsbXFTmKuqGkM3oT3zoRAJjZwamhM4Ft/vvXBvH1P8HX/QVweaCDnXOV+JZY/YmZJeLLWeovFKcD/fyH7gMymjx0NvBN/3oJmFm+meV20HuQTkbFQmJdmpmVNLn9CN8Hb7F/0HclvqnlAX4P/NbMPgTig5jpB8CPzGwe0AuoDPQA59xifDOSXo5vQaFiM1uAr5Wx2n/MHuBD/6m2dznn3sTXzfWxmS0DnuPwYiLSZjp1ViTE/Kv6eZxzzswuB65wzl0Q6HEi4aQxC5HQGw/c7z+DqYIQLV0rcjzUshARkYA0ZiEiIgGpWIiISEAqFiIiEpCKhYiIBKRiISIiAalYiIhIQP8P0HtcEUINF7QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_clas.lr_find()\n",
    "learn_clas.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 01:04 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f_beta</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.851637</td>\n",
       "      <td>0.866255</td>\n",
       "      <td>0.678322</td>\n",
       "      <td>0.641764</td>\n",
       "      <td>01:04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_clas.fit_one_cycle(1, 2e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_clas.save('ASTD_first_acc685_f643')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (8004 items)\n",
       "x: TextList\n",
       "  اجلاء اكثر من xxunk شخص تحسبا ل+ xxunk ال+ xxunk,  ل+ ال+ تخلص من مرسي غاندي هو ال+ حل,  رقم يا اخ + ي اي منصب رفضت ال+ مناصب ال+ لي عرضت عليا xxunk ان + ي ب+ قوم ب+ دوري و+ مش محتاج مناصب ارحمونا يرحم + كم الله xxunk ب+ نزاهه,  xxunk الله يا جماعه و+ بلاش اشاعات عن مفاجات عشان مش عايزين xxunk كل ال+ حكايه ملف معقد xxunk نفهم + ه مع بعض,  ب+ ال+ فيديو عكاشه و+ الله هيفاء و+ هبي ست مش طريه و+ ال+ لي يتجوز + ها يبقي خد خازوق بوابه ال+ قاهره مصر توفيق عكاشه\n",
       "y: CategoryList\n",
       "-2,-2,0,1,-1\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2002 items)\n",
       "x: TextList\n",
       "  في عالم موازي ان + ي مخلص كل حاجه ليله ال+ امتحان و ب+ راجع او احل,  عندما + ما تحتاج ل+ حواءط xxunk ل+ مءسسات ال+ دوله ف+ هذا xxunk ان تلك ال+ مءسسات ليست ل+ خدمه ال+ شعب و+ انما + ما ل+ قهر + ه,  رونالدو xxunk ل+ لقاء البايرن ريال مدريد كريستيانو رونالدو,  رقم انا ب+ رض + ه ال+ لي ب+ تكلم زي عمر سليمان ده شكل + ه xxunk من + كم و+ ب+ كل امانه انا كنت احترم + كم ايضا و+ لكن كفايه و+ هم ل+ حد هنا,  ل+ ابد من اقاله وزير ال+ داخليه و+ مدير امن بورسعيد و+ محافظ + ها فورا xxunk ل+ ال+ دكتور xxunk xxunk ب+ جلسه استثناءيه غدا ل+ مجلس ال+ شعب\n",
       "y: CategoryList\n",
       "-2,-2,-1,0,-1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(8270, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(8270, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=4, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x000001212572AD08>, FBeta(average='weighted', pos_label=1, eps=1e-09, beta=2)], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=WindowsPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (8004 items)\n",
       "x: TextList\n",
       "  اجلاء اكثر من xxunk شخص تحسبا ل+ xxunk ال+ xxunk,  ل+ ال+ تخلص من مرسي غاندي هو ال+ حل,  رقم يا اخ + ي اي منصب رفضت ال+ مناصب ال+ لي عرضت عليا xxunk ان + ي ب+ قوم ب+ دوري و+ مش محتاج مناصب ارحمونا يرحم + كم الله xxunk ب+ نزاهه,  xxunk الله يا جماعه و+ بلاش اشاعات عن مفاجات عشان مش عايزين xxunk كل ال+ حكايه ملف معقد xxunk نفهم + ه مع بعض,  ب+ ال+ فيديو عكاشه و+ الله هيفاء و+ هبي ست مش طريه و+ ال+ لي يتجوز + ها يبقي خد خازوق بوابه ال+ قاهره مصر توفيق عكاشه\n",
       "y: CategoryList\n",
       "-2,-2,0,1,-1\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2002 items)\n",
       "x: TextList\n",
       "  في عالم موازي ان + ي مخلص كل حاجه ليله ال+ امتحان و ب+ راجع او احل,  عندما + ما تحتاج ل+ حواءط xxunk ل+ مءسسات ال+ دوله ف+ هذا xxunk ان تلك ال+ مءسسات ليست ل+ خدمه ال+ شعب و+ انما + ما ل+ قهر + ه,  رونالدو xxunk ل+ لقاء البايرن ريال مدريد كريستيانو رونالدو,  رقم انا ب+ رض + ه ال+ لي ب+ تكلم زي عمر سليمان ده شكل + ه xxunk من + كم و+ ب+ كل امانه انا كنت احترم + كم ايضا و+ لكن كفايه و+ هم ل+ حد هنا,  ل+ ابد من اقاله وزير ال+ داخليه و+ مدير امن بورسعيد و+ محافظ + ها فورا xxunk ل+ ال+ دكتور xxunk xxunk ب+ جلسه استثناءيه غدا ل+ مجلس ال+ شعب\n",
       "y: CategoryList\n",
       "-2,-2,-1,0,-1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(8270, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(8270, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=4, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x000001212572AD08>, FBeta(average='weighted', pos_label=1, eps=1e-09, beta=2)], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=WindowsPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(8270, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(8270, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=4, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): Embedding(8270, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(8270, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=4, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_clas.load('ASTD_first_acc685_f643')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_clas.freeze_to(-2)\n",
    "#learn_clas.lr_find()\n",
    "#learn_clas.recorder.plot()\n",
    "#learn_clas.fit_one_cycle(2, slice(1e-2/(2.6**4),1e-2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 01:07 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f_beta</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.843509</td>\n",
       "      <td>0.848543</td>\n",
       "      <td>0.690310</td>\n",
       "      <td>0.665196</td>\n",
       "      <td>01:07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_clas.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_clas.save('ASTD_second_acc690_f665_min4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (3200 items)\n",
       "x: TextList\n",
       "  ال+ ان اي مواطن يراجع اي جهه حكومي + ه و+ لم تخدم + ه حسب ال+ نظام يرفع جوال + ه و+ لا يرد + ه الا رقم مكافحه ال+ فساد خلو + نا ننظف ال+ بلد من + هم ال+ ملك يحارب ال+ فساد,  كلام ال+ ناس محمد عبيد يعتبر ان ما جري ال+ يوم هو اخطر من رقم ايار قرارات ال+ حكومه ال+ لبنانيه عام رقم حول امن ال+ مطار و+ خط,  ال+ اردنيه ال+ مهندسه هدي ال+ حسيني تحصد ذهبي + ه جاءز + ه ستيفي ك+ افضل موظف + ه في ال+ عالم و+ ذلك عن فءه ال+ مءسسات ال+ حكوميه او غي,  في متل ب+ قول ادفشني و+ شوف ما اجحشني و+ انت يا سعد جحش خريها شخ تحتك و+ لكن اسمع لبنان مقاوم + ه لا تعرف ال+ هوان و+ لا تعر,  رجعت الضرايب ب+ كل دول ال+ عالم ب+ استثناء لبنان مقابل كل ضريبه في خدم + ه عن + نا مقابل كل ضريبه في ضرب + ه بتضربنا ب+ ال+ قلب طيب ليش انا لبناني\n",
       "y: CategoryList\n",
       "1,-1,1,-1,-2\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (800 items)\n",
       "x: TextList\n",
       "  تصل ال+ سحب ال+ ممطره ل+ ال+ منطقه ال+ وسطي سدير صباح غد ال+ ثلاثاء ب+ شكل خفيف و+ تتحسن ب+ اذن الله من بعد ال+ ظهر و+ هي من خفيف + ه الي,  فعلا xxunk ب+ لبنان بدنا نتعلم كيف نكون ايد وحد + ه ب+ غض ال+ نظر عن ال+ سياسه او ال+ مذهب او ال+ دين xxunk بدنا مصير نفكر ب+ طريق + ه اوسع شوي,  نتقدم ب+ ال+ تهاني ل+ سعاده ال+ دكتور فهد العضياني ب+ مناسبه تعيين + ه مشرفا مالي اداري ل+ فرع رابغ و نشكر ل + ه كريم زياره + ه ل+ عما,  نبارك من ال+ قلب حب ل+ منتخب ال+ مغرب ال+ شقيق ال+ تاهل ل+ كاس ال+ عالم في روسيا رقم عشت في ال+ اوطان ل+ ال+ علي عنوان ملء كل جنا,  لبنان بلد فاشل ب+ كفي تعرف ان + ه الضرايب ال+ جديده مش لحتي xxunk كهربا و+ مي لكن بس ل+ تغطي + ه سلسل + ه ال+ اجور ال+ جديده\n",
       "y: CategoryList\n",
       "0,1,2,2,-2\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(11880, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(11880, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x0000029E142FA730>, FBeta(average='weighted', pos_label=1, eps=1e-09, beta=2)], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=WindowsPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (3200 items)\n",
       "x: TextList\n",
       "  ال+ ان اي مواطن يراجع اي جهه حكومي + ه و+ لم تخدم + ه حسب ال+ نظام يرفع جوال + ه و+ لا يرد + ه الا رقم مكافحه ال+ فساد خلو + نا ننظف ال+ بلد من + هم ال+ ملك يحارب ال+ فساد,  كلام ال+ ناس محمد عبيد يعتبر ان ما جري ال+ يوم هو اخطر من رقم ايار قرارات ال+ حكومه ال+ لبنانيه عام رقم حول امن ال+ مطار و+ خط,  ال+ اردنيه ال+ مهندسه هدي ال+ حسيني تحصد ذهبي + ه جاءز + ه ستيفي ك+ افضل موظف + ه في ال+ عالم و+ ذلك عن فءه ال+ مءسسات ال+ حكوميه او غي,  في متل ب+ قول ادفشني و+ شوف ما اجحشني و+ انت يا سعد جحش خريها شخ تحتك و+ لكن اسمع لبنان مقاوم + ه لا تعرف ال+ هوان و+ لا تعر,  رجعت الضرايب ب+ كل دول ال+ عالم ب+ استثناء لبنان مقابل كل ضريبه في خدم + ه عن + نا مقابل كل ضريبه في ضرب + ه بتضربنا ب+ ال+ قلب طيب ليش انا لبناني\n",
       "y: CategoryList\n",
       "1,-1,1,-1,-2\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (800 items)\n",
       "x: TextList\n",
       "  تصل ال+ سحب ال+ ممطره ل+ ال+ منطقه ال+ وسطي سدير صباح غد ال+ ثلاثاء ب+ شكل خفيف و+ تتحسن ب+ اذن الله من بعد ال+ ظهر و+ هي من خفيف + ه الي,  فعلا xxunk ب+ لبنان بدنا نتعلم كيف نكون ايد وحد + ه ب+ غض ال+ نظر عن ال+ سياسه او ال+ مذهب او ال+ دين xxunk بدنا مصير نفكر ب+ طريق + ه اوسع شوي,  نتقدم ب+ ال+ تهاني ل+ سعاده ال+ دكتور فهد العضياني ب+ مناسبه تعيين + ه مشرفا مالي اداري ل+ فرع رابغ و نشكر ل + ه كريم زياره + ه ل+ عما,  نبارك من ال+ قلب حب ل+ منتخب ال+ مغرب ال+ شقيق ال+ تاهل ل+ كاس ال+ عالم في روسيا رقم عشت في ال+ اوطان ل+ ال+ علي عنوان ملء كل جنا,  لبنان بلد فاشل ب+ كفي تعرف ان + ه الضرايب ال+ جديده مش لحتي xxunk كهربا و+ مي لكن بس ل+ تغطي + ه سلسل + ه ال+ اجور ال+ جديده\n",
       "y: CategoryList\n",
       "0,1,2,2,-2\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(11880, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(11880, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x0000029E142FA730>, FBeta(average='weighted', pos_label=1, eps=1e-09, beta=2)], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=WindowsPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(11880, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(11880, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): Embedding(11880, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(11880, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=5, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_clas.load('ArSenTD-LEV_preprocessed_second_acc484_f471')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_clas.freeze_to(-3)\n",
    "#learn_clas.lr_find()\n",
    "#learn_clas.recorder.plot()\n",
    "#learn_clas.fit_one_cycle(1, slice(1e-3/(2.6**4),1e-3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 01:09 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f_beta</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.753973</td>\n",
       "      <td>0.835982</td>\n",
       "      <td>0.692308</td>\n",
       "      <td>0.665192</td>\n",
       "      <td>01:09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# to be deleted\n",
    "learn_clas.fit_one_cycle(1, slice(1e-3/(2.6**4),1e-3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_clas.save('ASTD_third_acc692_f665_min4')\n",
    "# 0.686314\t0.656684"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (8004 items)\n",
       "x: TextList\n",
       "  اجلاء اكثر من xxunk شخص تحسبا ل+ xxunk ال+ xxunk,  ل+ ال+ تخلص من مرسي غاندي هو ال+ حل,  رقم يا اخ + ي اي منصب رفضت ال+ مناصب ال+ لي عرضت عليا xxunk ان + ي ب+ قوم ب+ دوري و+ مش محتاج مناصب ارحمونا يرحم + كم الله xxunk ب+ نزاهه,  xxunk الله يا جماعه و+ بلاش اشاعات عن مفاجات عشان مش عايزين xxunk كل ال+ حكايه ملف معقد xxunk نفهم + ه مع بعض,  ب+ ال+ فيديو عكاشه و+ الله هيفاء و+ هبي ست مش طريه و+ ال+ لي يتجوز + ها يبقي خد خازوق بوابه ال+ قاهره مصر توفيق عكاشه\n",
       "y: CategoryList\n",
       "-2,-2,0,1,-1\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2002 items)\n",
       "x: TextList\n",
       "  في عالم موازي ان + ي مخلص كل حاجه ليله ال+ امتحان و ب+ راجع او احل,  عندما + ما تحتاج ل+ حواءط xxunk ل+ مءسسات ال+ دوله ف+ هذا xxunk ان تلك ال+ مءسسات ليست ل+ خدمه ال+ شعب و+ انما + ما ل+ قهر + ه,  رونالدو xxunk ل+ لقاء البايرن ريال مدريد كريستيانو رونالدو,  رقم انا ب+ رض + ه ال+ لي ب+ تكلم زي عمر سليمان ده شكل + ه xxunk من + كم و+ ب+ كل امانه انا كنت احترم + كم ايضا و+ لكن كفايه و+ هم ل+ حد هنا,  ل+ ابد من اقاله وزير ال+ داخليه و+ مدير امن بورسعيد و+ محافظ + ها فورا xxunk ل+ ال+ دكتور xxunk xxunk ب+ جلسه استثناءيه غدا ل+ مجلس ال+ شعب\n",
       "y: CategoryList\n",
       "-2,-2,-1,0,-1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(8270, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(8270, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=4, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x000001212572AD08>, FBeta(average='weighted', pos_label=1, eps=1e-09, beta=2)], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=WindowsPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True)], callbacks=[RNNTrainer\n",
       "learn: RNNLearner(data=TextClasDataBunch;\n",
       "\n",
       "Train: LabelList (8004 items)\n",
       "x: TextList\n",
       "  اجلاء اكثر من xxunk شخص تحسبا ل+ xxunk ال+ xxunk,  ل+ ال+ تخلص من مرسي غاندي هو ال+ حل,  رقم يا اخ + ي اي منصب رفضت ال+ مناصب ال+ لي عرضت عليا xxunk ان + ي ب+ قوم ب+ دوري و+ مش محتاج مناصب ارحمونا يرحم + كم الله xxunk ب+ نزاهه,  xxunk الله يا جماعه و+ بلاش اشاعات عن مفاجات عشان مش عايزين xxunk كل ال+ حكايه ملف معقد xxunk نفهم + ه مع بعض,  ب+ ال+ فيديو عكاشه و+ الله هيفاء و+ هبي ست مش طريه و+ ال+ لي يتجوز + ها يبقي خد خازوق بوابه ال+ قاهره مصر توفيق عكاشه\n",
       "y: CategoryList\n",
       "-2,-2,0,1,-1\n",
       "Path: .;\n",
       "\n",
       "Valid: LabelList (2002 items)\n",
       "x: TextList\n",
       "  في عالم موازي ان + ي مخلص كل حاجه ليله ال+ امتحان و ب+ راجع او احل,  عندما + ما تحتاج ل+ حواءط xxunk ل+ مءسسات ال+ دوله ف+ هذا xxunk ان تلك ال+ مءسسات ليست ل+ خدمه ال+ شعب و+ انما + ما ل+ قهر + ه,  رونالدو xxunk ل+ لقاء البايرن ريال مدريد كريستيانو رونالدو,  رقم انا ب+ رض + ه ال+ لي ب+ تكلم زي عمر سليمان ده شكل + ه xxunk من + كم و+ ب+ كل امانه انا كنت احترم + كم ايضا و+ لكن كفايه و+ هم ل+ حد هنا,  ل+ ابد من اقاله وزير ال+ داخليه و+ مدير امن بورسعيد و+ محافظ + ها فورا xxunk ل+ ال+ دكتور xxunk xxunk ب+ جلسه استثناءيه غدا ل+ مجلس ال+ شعب\n",
       "y: CategoryList\n",
       "-2,-2,-1,0,-1\n",
       "Path: .;\n",
       "\n",
       "Test: None, model=SequentialRNN(\n",
       "  (0): MultiBatchEncoder(\n",
       "    (module): AWD_LSTM(\n",
       "      (encoder): Embedding(8270, 400, padding_idx=1)\n",
       "      (encoder_dp): EmbeddingDropout(\n",
       "        (emb): Embedding(8270, 400, padding_idx=1)\n",
       "      )\n",
       "      (rnns): ModuleList(\n",
       "        (0): WeightDropout(\n",
       "          (module): LSTM(400, 1150, batch_first=True)\n",
       "        )\n",
       "        (1): WeightDropout(\n",
       "          (module): LSTM(1150, 1150, batch_first=True)\n",
       "        )\n",
       "        (2): WeightDropout(\n",
       "          (module): LSTM(1150, 400, batch_first=True)\n",
       "        )\n",
       "      )\n",
       "      (input_dp): RNNDropout()\n",
       "      (hidden_dps): ModuleList(\n",
       "        (0): RNNDropout()\n",
       "        (1): RNNDropout()\n",
       "        (2): RNNDropout()\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=4, bias=True)\n",
       "    )\n",
       "  )\n",
       "), opt_func=functools.partial(<class 'torch.optim.adam.Adam'>, betas=(0.9, 0.99)), loss_func=FlattenedLoss of CrossEntropyLoss(), metrics=[<function accuracy at 0x000001212572AD08>, FBeta(average='weighted', pos_label=1, eps=1e-09, beta=2)], true_wd=True, bn_wd=True, wd=0.01, train_bn=True, path=WindowsPath('.'), model_dir='models', callback_fns=[functools.partial(<class 'fastai.basic_train.Recorder'>, add_time=True)], callbacks=[...], layer_groups=[Sequential(\n",
       "  (0): Embedding(8270, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(8270, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=4, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True)\n",
       "alpha: 2.0\n",
       "beta: 1.0], layer_groups=[Sequential(\n",
       "  (0): Embedding(8270, 400, padding_idx=1)\n",
       "  (1): EmbeddingDropout(\n",
       "    (emb): Embedding(8270, 400, padding_idx=1)\n",
       "  )\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(400, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 1150, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): WeightDropout(\n",
       "    (module): LSTM(1150, 400, batch_first=True)\n",
       "  )\n",
       "  (1): RNNDropout()\n",
       "), Sequential(\n",
       "  (0): PoolingLinearClassifier(\n",
       "    (layers): Sequential(\n",
       "      (0): BatchNorm1d(1200, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (1): Dropout(p=0.08000000000000002)\n",
       "      (2): Linear(in_features=1200, out_features=50, bias=True)\n",
       "      (3): ReLU(inplace)\n",
       "      (4): BatchNorm1d(50, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (5): Dropout(p=0.1)\n",
       "      (6): Linear(in_features=50, out_features=4, bias=True)\n",
       "    )\n",
       "  )\n",
       ")], add_time=True)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn_clas.load('ASTD_third_acc689_f668')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_clas.unfreeze()\n",
    "#learn_clas.lr_find()\n",
    "#learn_clas.recorder.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 07:12 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f_beta</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.720984</td>\n",
       "      <td>0.832350</td>\n",
       "      <td>0.696304</td>\n",
       "      <td>0.666945</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.708480</td>\n",
       "      <td>0.839034</td>\n",
       "      <td>0.689810</td>\n",
       "      <td>0.669408</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.694973</td>\n",
       "      <td>0.839271</td>\n",
       "      <td>0.686813</td>\n",
       "      <td>0.667135</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.702815</td>\n",
       "      <td>0.832584</td>\n",
       "      <td>0.697802</td>\n",
       "      <td>0.670850</td>\n",
       "      <td>01:11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.702329</td>\n",
       "      <td>0.833517</td>\n",
       "      <td>0.696304</td>\n",
       "      <td>0.671418</td>\n",
       "      <td>01:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.684041</td>\n",
       "      <td>0.837961</td>\n",
       "      <td>0.690809</td>\n",
       "      <td>0.671963</td>\n",
       "      <td>01:12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#learn_clas.unfreeze()\n",
    "learn_clas.fit_one_cycle(6, slice(1e-4/(2.6**4),1e-4))\n",
    "# 0.659341\t0.857126\t0.698801\t0.677395"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXeYFdXZwH/vdraxCyx1QbrSYUUsoIIoCvaSCGrsMWos0TTURI2fRhKNMRo1aiyJUYmxooINsQMC0qRJh6UuS93ezvfHzNyd2+8uO1vg/T3Pfe7MmTNz3zu797xzztvEGIOiKIqiRCKuqQVQFEVRmj+qLBRFUZSoqLJQFEVRoqLKQlEURYmKKgtFURQlKqosFEVRlKioslAURVGiospCURRFiYoqC0VRFCUqCU0tQEPRrl07071796YWQ1EUpUWxYMGCXcaYnGj9Dhll0b17d+bPn9/UYiiKorQoRGRjLP10GUpRFEWJiioLRVEUJSqqLBRFUZSoHDI2C0VRDh0qKyvJz8+nrKysqUU5ZEhJSSE3N5fExMR6na/KQlGUZkd+fj4ZGRl0794dEWlqcVo8xhgKCwvJz8+nR48e9bqGLkMpitLsKCsro23btqooGggRoW3btgc1U1NloShKs0QVRcNysPdTlQXwzdpdrCsoamoxFEVRmi2qLIBLnp3LKX/5vKnFUBSlmVBYWMjQoUMZOnQoHTt2pEuXLr79ioqKmK5x1VVXsWrVKo8lbTzUwK0oihJA27ZtWbRoEQD33nsv6enp/OpXv/LrY4zBGENcXOhn7hdeeMFzORsTnVkoiqLEyJo1axg4cCDXX389eXl5bNu2jeuuu47hw4czYMAA7rvvPl/fUaNGsWjRIqqqqsjKymLy5MkMGTKE448/np07dzbht6gfOrNQFKVZ84d3l7F86/4GvWb/zpncc/aAep27fPlyXnjhBf7xj38AMGXKFNq0aUNVVRVjxozhoosuon///n7n7Nu3j5NPPpkpU6Zw++238/zzzzN58uSD/h6Nic4sFEVR6kCvXr045phjfPuvvvoqeXl55OXlsWLFCpYvXx50TqtWrRg/fjwARx99NBs2bGgscRsMnVkoitKsqe8MwCvS0tJ826tXr+Zvf/sb3377LVlZWVx22WUhYxmSkpJ82/Hx8VRVVTWKrA2JziwURVHqyf79+8nIyCAzM5Nt27bx4YcfNrVInqEzC0VRlHqSl5dH//79GThwID179mTkyJFNLZJniDGmqWVoEIYPH27qW/yo++T3Adgw5cyGFElRlHqyYsUK+vXr19RiHHKEuq8issAYMzzauZ4uQ4nIGSKySkTWiEiQ6V9EjhCRmSKyREQ+E5Fc17FqEVlkv6Z5KaeiKIoSGc+WoUQkHngCOA3IB+aJyDRjjNtV4GHg38aYf4nIKcCDwE/sY6XGmKFeyacoiqLEjpczixHAGmPMOmNMBTAVODegT39gpr09K8RxRVEUpRngpbLoAmx27efbbW4WAxfa2+cDGSLS1t5PEZH5IjJHRM7zUE5FURQlCl4qi1D5cAOt6b8CThaRhcDJwBbAcUDuZhtdLgEeFZFeQR8gcp2tUOYXFBTUS8hDxcCvKIriJV4qi3ygq2s/F9jq7mCM2WqMucAYMwy4y27b5xyz39cBnwHDAj/AGPOMMWa4MWZ4Tk5OvYSsUV2hKIoSFS+VxTygj4j0EJEkYCLg59UkIu1ExJHhDuB5uz1bRJKdPsBIIDiGvgGo0ZmFoigBjB49OijA7tFHH+XGG28Me056ejoAW7du5aKLLgp73Wgu/o8++iglJSW+/QkTJrB3795YRfcMz5SFMaYKuAn4EFgBvGaMWSYi94nIOXa30cAqEfkB6AA8YLf3A+aLyGIsw/eUAC+qBqNapxaKogQwadIkpk6d6tc2depUJk2aFPXczp078/rrr9f7swOVxfTp08nKyqr39RoKT+MsjDHTjTF9jTG9jDEP2G13G2Om2duvG2P62H2uNcaU2+3fGGMGGWOG2O/PeSejV1dWFKWlctFFF/Hee+9RXl4OwIYNG9i6dStDhw5l7Nix5OXlMWjQIN55552gczds2MDAgQMBKC0tZeLEiQwePJiLL76Y0tJSX78bbrjBl9r8nnvuAeCxxx5j69atjBkzhjFjxgDQvXt3du3aBcAjjzzCwIEDGThwII8++qjv8/r168dPf/pTBgwYwLhx4/w+p6E47NN96DKUojRzZkyG7Usb9podB8H4KWEPt23blhEjRvDBBx9w7rnnMnXqVC6++GJatWrFW2+9RWZmJrt27eK4447jnHPOCVvf+qmnniI1NZUlS5awZMkS8vLyfMceeOAB2rRpQ3V1NWPHjmXJkiXccsstPPLII8yaNYt27dr5XWvBggW88MILzJ07F2MMxx57LCeffDLZ2dmsXr2aV199lWeffZYf//jHvPHGG1x22WUNc69sDvtEgtWqLBRFCYF7KcpZgjLGcOeddzJ48GBOPfVUtmzZwo4dO8Je44svvvAN2oMHD2bw4MG+Y6+99hp5eXkMGzaMZcuWhUxt7uarr77i/PPPJy0tjfT0dC644AK+/PJLAHr06MHQoVYMs1cp0A/7mYWpaWoJFEWJSIQZgJecd9553H777Xz33XeUlpaSl5fHiy++SEFBAQsWLCAxMZHu3buHTEnuJtSsY/369Tz88MPMmzeP7OxsrrzyyqjXieTmn5yc7NuOj4/3ZBlKZxY6s1AUJQTp6emMHj2aq6++2mfY3rdvH+3btycxMZFZs2axcePGiNc46aSTePnllwH4/vvvWbJkCWClNk9LS6N169bs2LGDGTNm+M7JyMjgwIEDIa/19ttvU1JSQnFxMW+99RYnnnhiQ33dqBz2Mwu1WSiKEo5JkyZxwQUX+JajLr30Us4++2yGDx/O0KFDOeqooyKef8MNN3DVVVcxePBghg4dyogRIwAYMmQIw4YNY8CAAUGpza+77jrGjx9Pp06dmDVrlq89Ly+PK6+80neNa6+9lmHDhjVa1b3DPkV5eVU1R/7uAzpmpjDnzrEeSKYoSl3RFOXecDApyg/7mUVyQjwn9mnHgbKWV+ZQURSlsTjsbRYAcSJBSasURVGUWlRZAHGiCQUVpbmhv8mG5WDvpyoLrJmFGroVpfmQkpJCYWGhKowGwhhDYWEhKSkp9b7GYW+zABCBGo23UJRmQ25uLvn5+dS39IASTEpKCrm5udE7hkGVBVbQjD6/KErzITExkR49ejS1GIoLXYZCbRaKoijRUGUBCGqzUBRFiYQqCyAuTlOVK4qiREKVBZbNQmcWiqIo4VFlAQg6s1AURYmEKgs0gltRFCUaqiywvKF0GUpRFCU8qizQCG5FUZRoqLIA0AhuRVGUiKiywJpZKIqiKOFRZYHaLBRFUaLhqbIQkTNEZJWIrBGRySGOHyEiM0VkiYh8JiK5rmNXiMhq+3WFp3JqBLeiKEpEPFMWIhIPPAGMB/oDk0Skf0C3h4F/G2MGA/cBD9rntgHuAY4FRgD3iEi2V7JqBLeiKEpkvJxZjADWGGPWGWMqgKnAuQF9+gMz7e1ZruOnAx8bY3YbY/YAHwNneCWoFcHt1dUVRVFaPl4qiy7AZtd+vt3mZjFwob19PpAhIm1jPLfBsCK4VVsoiqKEw0tlEcrFKHBE/hVwsogsBE4GtgBVMZ6LiFwnIvNFZP7BFEnRCG5FUZTIeKks8oGurv1cYKu7gzFmqzHmAmPMMOAuu21fLOfafZ8xxgw3xgzPycmpt6DqDaUoihIZL5XFPKCPiPQQkSRgIjDN3UFE2omII8MdwPP29ofAOBHJtg3b4+w2TxARatRooSiKEhbPlIUxpgq4CWuQXwG8ZoxZJiL3icg5drfRwCoR+QHoADxgn7sb+D8shTMPuM9u8wQR9YZSFEWJhKc1uI0x04HpAW13u7ZfB14Pc+7z1M40PEVtFoqiKJHRCG7UZqEoihINVRZopTxFUZRoqLJAbRaKoijRUGWBbbNQZaEoihIWVRZYEYC6DKUoihIeVRaoN5SiKEo0VFmg3lCKoijRUGUBoDYLRVGUiKiywJpZgGaeVRRFCYcqC2prcGt6KEVRlNCosqB2ZqF2C0VRlNCossCK4AZVFoqiKOFQZYEVwQ0axa0oihIOVRbU2ixUWSiKooRGlQW1NVx1GUpRFCU0qixwzSyaWA5FUZTmiioLam0WOrNQFEUJjSoLar2hTE0TC6IoitJMUWWBK4JbF6IURVFCosoCjeBWFEWJhioL1GahKIoSDVUWaAS3oihKNFRZUGuzqKo2VFSplVtRFCUQT5WFiJwhIqtEZI2ITA5xvJuIzBKRhSKyREQm2O3dRaRURBbZr394Kadjszj/ya/p+7sZXn6UoihKiyTBqwuLSDzwBHAakA/ME5Fpxpjlrm6/A14zxjwlIv2B6UB3+9haY8xQr+Tzk9V+37G/vDE+TlEUpcXh5cxiBLDGGLPOGFMBTAXODehjgEx7uzWw1UN5wuLMLBRFUZTQeKksugCbXfv5dpube4HLRCQfa1Zxs+tYD3t56nMROdFDOVFdoSiKEhkvlUWoITjQ3WgS8KIxJheYALwkInHANqCbMWYYcDvwiohkBpyLiFwnIvNFZH5BQUH9BVVtoSiKEhEvlUU+0NW1n0vwMtM1wGsAxpjZQArQzhhTbowptNsXAGuBvoEfYIx5xhgz3BgzPCcnp96CxqmuUBRFiYiXymIe0EdEeohIEjARmBbQZxMwFkBE+mEpiwIRybEN5IhIT6APsM4rQdVmoSiKEhnPvKGMMVUichPwIRAPPG+MWSYi9wHzjTHTgF8Cz4rIbVhLVFcaY4yInATcJyJVQDVwvTFmt1eyBuqK6hpDvE43FEVRfHimLACMMdOxDNfutrtd28uBkSHOewN4w0vZ3ATaLCqra4iPi2+sj1cURWn2aAQ3wTaLimqN4lYURXHj6cyipRBos6jUlB+NgzFQugdKCqG4wH7tsl8FUFFkF0Y3rgLpprbNuUbU49TjnGifE+maIT4zKR3SciCtnf3KgdR2rrYcSEpTP26l2aLKgmAf38pqTShYL4yxBvjiAih2KYASlwJwHyvZBTVVoa+VkgXJmfYfR1yDqLMdpg0iH/f9setyTgyf4+sS4hyw7sue9dZ9qCgK/Z0TUvyVR6pLsQS+p7aDxJTQ11FaFjXVUH4Ayvdb72X7Xfuh2g5A2T57325r3x+uCPQfalhUWRDaZqHYVJbWDvThZgDuY1Vloa+TlAFpba2BLqsrdB5qD37Oq23tdmpbiE9s3O/ZmLjvafEuW5kG3NeinbBjudVWHSYNTVJGgBIJM2NJa3fo39OmwBioKA4Y0F0DvN8gvy/0oF9+IPzDgxuJg+QMSG5tvadkQnp7aNvb2m/Xx/Ovq8qCw8xmUV0ZMOhHmQFUHAh9ncCn4Pb9/QcsZ9B3+iS2atzv2ZxJbGUpzKyu0fv6zdYC/z4uRbN3E2z5zto21aGv1So7thlLWo7VN+4QMWnW1Fj3pKbKeoo31dZ7TVXAU3pdnuz3W7+NWGoxJ6Vbs2RnkE/JhNa59r49+Cdn+PdJzvTfT0xt8iVKVRYcQjOLffmQPy94QHHvl+0NfW5cgv8TaXb30IOIoxCS0pv8n/ewQKR2MGnTM3r/mhrrbxw0YwmYFe76ATZ+DSW7CU6sgPUk61b2zv9AaltLJveg63uvCdFeE6JfqPaqOvSt42fVl4SUgAE8A9J6Brf59lsHD/pJ6XCIeFaqsiB4ZlFZ1UJtFhu/gTd/au+I/WO3f+gdB0Z+qkzJ0sH/UCAuDlLbWK/gpAfBVFdZTgY+RRJmuXHrIuu9fJ/rZLEGQol3vccF7Edqj3PtJ1jbcQmQkBylb30+K0R7fKI1sAcO+s57QpJXf6UWiSoLgr2hVu88wKDc1k0kzUHQ+1S4YbatFNocMk80iofEJ0B6jvWKhWrbISEuXh8uDjNiWpQUkV4ikmxvjxaRW0Qky1vRGpGA//k73lzaNHIcLKltoEN/64evikLxgvgE66WK4rAjVgvWG0C1iPQGngN6AK94JlUjEzizuOy4I5pIEkVRlOZJrMqixhhTBZwPPGqMuQ3o5J1YjUtCgNEiPVlX5xRFUdzEqiwqRWQScAXwnt12yDhtJyf434aqmhbqDaUoiuIRsSqLq4DjgQeMMetFpAfwH+/EalySE/zX96tqWqg3lKIoikfEpCyMMcuNMbcYY14VkWwgwxgzxWPZGo3kRP/b8MnyHU0kiaIoSvMkVm+oz0QkU0TaAIuBF0TkEW9FazwS4/1vw9qC4iaSRFEUpXkS6zJUa2PMfuAC4AVjzNHAqd6J1bioE6CiKEpkYlUWCSLSCfgxtQbuQ4bUpOCYhLUFRWzZW9oE0iiKojQ/YlUW92GVR11rjJln18Ve7Z1YjUv7zOBUz2P/8jkjp3zaBNIoiqI0P2IKKDDG/A/4n2t/HXChV0I1BenJCRSVh6mtoCiKcpgTq4E7V0TeEpGdIrJDRN4QkVyvhWtM1G6hKIoSnliXoV4ApgGdgS7Au3bboYOtLQZ1aYEJBBVFUTwmVmWRY4x5wRhTZb9eBGJMU9kycGYWY/u1b1I5FEVRmiOxKotdInKZiMTbr8uAQi8Fa2ycAkihPKMURVEOd2JVFldjuc1uB7YBF2GlADnkSElUZaEoihJIrOk+NhljzjHG5Bhj2htjzsMK0IuIiJwhIqtEZI2ITA5xvJuIzBKRhSKyREQmuI7dYZ+3SkROr9O3qgdOlvKk+Fj1p6IoyuHDwYyMt0c6KCLxwBPAeKA/MElE+gd0+x3wmjFmGDAReNI+t7+9PwA4A3jSvp5nDM61ajmlp2h6ckVRlEAOZmSM5m06Alhjx2QgIlOBc4Hlrj4GyLS3WwNb7e1zganGmHJgvYissa83+yDkjciTl+axavsBKqo0PbmiKEogBzOziJbHuwuw2bWfb7e5uRe4TETygenAzXU4FxG5TkTmi8j8goKCOogeTHpyAkcfkR2UgdYYTVeuKIoSUVmIyAER2R/idQAr5iLi6SHaAkfeScCLxphcYALwkojExXguxphnjDHDjTHDc3IaxpM3sBBSZbUqC0VRlIjKwhiTYYzJDPHKMMZEW8LKB7q69nOpXWZyuAZ4zf6s2UAK0C7Gcz0hsBDSRf/4pjE+VlEUpVnjpevPPKCPiPQQkSQsg/W0gD6bgLEAItIPS1kU2P0mikiyXZWvD/Cth7L6SAlYhlqSv68xPlZRFKVZ45nrjzGmSkRuwspWGw88b4xZJiL3AfONMdOAXwLPishtWMtMVxrLSLBMRF7DMoZXAT83xlR7JaubwJmFoiiK4qGyADDGTMcyXLvb7nZtLwdGhjn3AeABL+ULRVKCxlkoiqIEoiNjAIEGbkVRFEWVRRApifG8f8sofnPGkU0tiqIoSrNBlUUIBnRuzXlDg8I6FEVRDltUWYShc1YrbjmlN6CBeYqiKKosIuAYuzUwT1GUwx1VFhFwlEVFteaLUhTl8EaVRQQS7XTlv3/7+yaWRFEUpWlRZRGBOLvIxVsLtzSxJIqiKE2LKgtFURQlKqosIhAfF61khxIrz3yxltU7DjS1GIqi1BNVFhFIjFdl0RBUVNXwx+krOe+Jr5taFEVR6okqiwjkdctuahEOCcqqrByQxRWNkgtSURQPUGURgT4dMvjR0bm0TUtqalFaNGWqJBSlxaPKIgpt0pIorqhqajFaNGWVGqeiKC0dVRZRSE6Mp6yyRlN+HATOMpSiKC0XVRZRaJVoFUMqr9Kn4/pSWFTh216Sv7cJJVEUpb6osoiCU2a1VNfd682q7ft92+f8XT2iFKUlosoiCs7MQpdS6o96QSlKy0eVRRRSHGWhRtp6U1Tu7yCg9h9FaXmosoiCLkMdPMUByuLI333AvtLKJpJGUZT6oMoiCilRlqHKKqs57o8z+XDZ9sYUq0VRXO5/7yqqa/hu054mkkZRlPqgyiIKPmURZmaxZmcR2/eX8eD0FY0pVovije/ySUqI4+Vrj/W1PTlrDZXVNXy2amcTSqYoSqyosoiCY+Ces343VdU11NQY3lqYz74Saxlld7HlFuqkM4+V/WWVdJ/8Pm8syG9YgZsZzvJdRVUNORnJvvZ5G/bw2zeWcOUL85ilCkNRmj2eKgsROUNEVonIGhGZHOL4X0Vkkf36QUT2uo5Vu45N81LOSDgzi8dmrua5r9Yz4/vt3Pbfxfxn7kbAGgSBOkd5f79lHwAvzdnYgNI2P9bsLPJt9+2Q4Xfsze+sOiHrCoobVSZFUeqOZ8pCROKBJ4DxQH9gkoj0d/cxxtxmjBlqjBkKPA686Tpc6hwzxpzjlZzRcGYWAA/OWMnqnVaa7W37SoHakqsl5XUzgDtKpo4TkhaFMYaz//6VX9vxPdsG9dsfg7F76rebePO7Q3sWpijNGS9nFiOANcaYdcaYCmAqcG6E/pOAVz2Up1443lAOj36y2m/fGfRLKmNTFu8t2croh2b5IsIPZS+r2WsLfdv3nzcQgFevO44XrjzGr9/OA+VRrzX5zaXc/trihhVQUZSY8VJZdAE2u/bz7bYgROQIoAfwqas5RUTmi8gcETkvzHnX2X3mFxQUNJTcfqQkxYdsF6wpgaMsqmsMJ/15FmsLikL2d7jjjaVsKCzxDZCHchqRS/4517fdulWib3vMUe3plZPm29+8uyTiddR5QFGaHi+VRagFlnDRWBOB140x7sfsbsaY4cAlwKMi0ivoYsY8Y4wZbowZnpOTc/AShyAlIbSyKK2sxhhDeXXtYL9pdwnfrt8d8XppyQkAbNtrLWOVHKIZbd22CghebstIqVUem6Ioi6e/WOfb3r6vDLCWuNbs1Mp7itJYeKks8oGurv1cYGuYvhMJWIIyxmy139cBnwHDGl7E6ISrlvf6gnz+PXujb2bhsHLb/pD9HdJTbGVhD3qH6jLUqY987ttOSYxjVO92fseP6lhr7N60u4RNhZEVhsOuImtG9vLcTZz6yBfMXVcY5QxFURoCL5XFPKCPiPQQkSQshRDk1SQiRwLZwGxXW7aIJNvb7YCRwHIPZQ2LRLBA3zNtWdASyortkZ920+2ZxVZ7ZlF2CC9DOdx1Zn+yUv0LSA3v3gaA9rY77cLNsQXpFZVXcaCskrcWWp5UHy/f0YCSKooSDs+UhTGmCrgJ+BBYAbxmjFkmIveJiNu7aRIw1fgnDOoHzBeRxcAsYIoxpkmURTRe/GaD3/7GwuKIuY8y7JnFXHu5KnBmcihSHsL43ybNWobKzW4FwM790Y3cAEVlVVz/nwUs2Ggpl39+tb6BpFQUJRIJXl7cGDMdmB7QdnfA/r0hzvsGGOSlbF4w5sgcZq0qYMveUnKzU4OO7y6u4MvVu4LajTERZzAtjcBcUKGM+E4QY1pyAonxQmFxRVAfsO5NfJxQXWMp4KLyKuat11QhitLYaAR3DPz9En9zScfMlCBbxtw7x3LtiT0B2Ly7NOR1/vnlupDtJU1kt9i2r5Tuk99n/obIRvm6EphlNpSycCvHNmlJ7AmjLMqraqiuMVx5QncADpRX0a9zpl8fJ8BRURTvUGURA2cN7sxfLx4SsU+HzBTaplvr8ntKQg98T362FoAL83L92g+UNY1HlBMHcdlzc6O6r9aFQKP9WYM7BfUZ1i2Lrm1acftpfclOTWJ3mHtWbqeGd1KFvPldPusC3JO/XhM8W6up0TToStNRVV3jc8Y4VFBlESPnD8vlTxdaK2OpybXutD87uSdv3XgCUOsOWhRl8D+pr79n0P6ypknX7SwFlVXWcOZjXzbYdUttG8VTl+axYcqZQWk+ADJTEvnyN6cwrFs2bdKSfDm2wl0rOzWJ+Dhh4aa9Qcr1wRkrWWF7of1i6kK6T36fnndOZ9Hmhi/heuPLC+g++X2u/dc8VUhKWJ7+Yh3D7//El/fs2/W7ufZf86iqbrk2SlUWdcAZXM8c1MkXlHfx8K4M65YN1Ho6/eaNJSHPT0uK55pRPThnSGe/9lfmbvJtH7ATDIZbsqoPX63exc9emh80uJW6DM/7Y5jdFJVXxeR95Fw3XEBjIFmpiewNM7Mos6/VKimONmm1HlUi8M7PR/r2x//tSzYVlvD2olrv7H8FOB80BNOXWqnoP1mxk/eXbmvw6yvhqaqu4a63lsbsZt2UrLK9Ih+buZql+fu48oVv+WTFTjbvCb1E3RJQZVEHxg/qxO/O7Me1o3r62hwFAZDh2n71200EUlFdQ1JCXJAx271s5RjA73+/4aKWL3tuLh8u2+H7nAdnrOCmV77jnneW1ek6t/13ET/993zy90T+sTpP/u68WpFo3SqRrXvLKA9RM8SneBLifZl+AYyBIV2zmDSim69tXoDtxcnf5RWlldU89OFKrc3RSCzavJeX527il/9b1NSiRMV5sFm4aS9n//0rkhKsoXbb3lIWbtrD/+ZvjnR6s0SVRR1IT07g2hN70jo1EWMHo6e5FERcnNC5dQoAd7y51O/cmhpDZbUhOcH/lrdLT2apy0DrHohDDZ6x8tePf6D75PcZOaU2g4oTCPj05+t4b8k2XxJEqHXpjcR3trvqnuLIy2ZPzFoDQI92aRH7OWSmJFJaWc3lz30bdMyZWaQkxjOgS2bQ8UyX3Ot2+dsyVmw74GkJ1798tIonZq3lgie/0VKxjUhldfO/1xUBy02J8dbvfn9ZFec/+Q2/fj306kNzRpVFPXny0qM5pns2qQFLLf+8ojZJntvQ6/zzOE8Y024ayZe/GUNmSgLrCopZmm8pjD9OX+k756XZVvryA2WVPPThyphjMorLq/jbTCvh4Za9tU/XZz3+Fd+EMAZfeUL38IlYXDi2lV1F5dTUGDbsCp1a3FmX7ZCZEpO83dpabsZzQ6RKKXUpi7sm9PObSYC/sg6M1dhXWsl7S6ylooYazFu3SuTkvlZqmR2uzwvnAac0PM1fVdQ6ZjgU2Lngnvxsja8t0MW8uaPKop6c1r8D/7v+hKAlpZ6uBHn3v18bR+i4jybZTxiDc7Po2iaVu87sBwQ/FVvnr+DTlTs4+/GveGLWWqYtDpctxWJvSQXdJ7/PgHs+DNvn3SXb/J7GAdqmJXGn4K8IAAAgAElEQVSgvIrSimq+27QnrBHOeaIrKCrnyc/WMPrhz4JyQIH1Yw5M7xGJswbX2nDKAgL4nB9dSmIcw7u34cEL/MNv4ly3/4Pvg0vb3vzqQs549At63DG9Qda6yyqr/VKVOGwo1JocXuOzsYVQ/DU1xheL0xwItyqwJL92FSGWbMvNCVUWDUxKYjw3n9IbsPIXbSws5s63lnLhU98AwelDhnbNAmDqt5tD/rNf/eJ8NtiDXGUUTwqnmFAkEuOF1CRLWVw9sgfv3zKKtumWW+pr8zdzwZPfBEWlg3+keWFRBbPtnEyPfvJDUN/i8io/W040WrdK9A3AgfaQUp+Bu3YG9+eLBvP+LaMA/xiOA/aT2qQRXfnThYOItzXJStvY+NkPB1eRr6KqhvKqmpBLduG8uZSGw4lH2lUUfK/PfPwret05Pai9qYglm/SO/WWNIEnDocrCA07qW5sB995py3hl7ibfE3ig0dfJmTR7XSHD7/8YwGf3CKQqypNTuKSHbv49eyO7Syq45Nhu3H12fwZ0bu2LD3nDLi50//srfN4cDu412HcWbfF5g723ZJvPn7ymxrBg426Kyqr8lodi4YHzrXoXgcs5ZS4Dt8OPh3dlQOfWfsfd/HLckVx8TDeOaOsfRX/3O8tYvaPumWqNMbyzaIsv4216cgIzbj3Rr4+WhvUeZ1l3y95SDrjczYvKq3yu002NMYZPV+5g1fYDDD8im0FdWoftq8pC4Rg7SR74p+IGGDegg99+vGsdZY/t7dO1Tapf/QeH/CiBc5E8qH5zxpG+7YqqGhJcn9vOnlm4p8gzvt/GzgO1/8yVrielldsP8JXL9uGkZX9t/mYufGo2W/eVxWQwd9PVTo+yOWBmUWwPEIG2IYdLjz2C3OxWZKXW3i/n+4Tyxlq2te6Dyo0vf8etUxdx2l+tTLrtMpKDFFFTzyzy95Rw69SFfoPoocT3W/b52d+ueXE+36zZxYPTV7AkvzaeJtrs22t+9tICrn5xPpt2l9A+M5l3bx7lO/bUpXm+ImAAW/eqslCAf109Aqg1bDlkhVACgWSkJLD4nnHcNKa3X7szO6msrglZByPU1PeuCf1Yeu84bhztf62EuNo/fY49uLp59JPVjH7oM99+pB+hk9fpOVdSv7Tk2NxmfTJkJJOcEMeGXf7KotCetWSnJYU6je7t0vjqt6f4yrX++vRapRhqKSySh5kxhncXb+X7LfvYvLvEZxSfYdtCnPvbITOF1KQEX8bcjOSEIINmY/Pwh6t4Z9HWkC7bhwJnPf4VD324yrdfVF7FJf+cy9NfrGOOqyJjLCV6veQjVxyS87ta98cJfHzbSYwf1Inzh9XWf5vTwtLrq7LwiB5tLUP3DvvpfOIxXfnrxUNIiA++5YFLGpn2bKRdeu0A2SWrFTNX7uThD1fR564Z9L/b34jtHsz/ffUInvnJ0Tzzk6P5yfFH+GY3bsOs2zDcLsN/IHae0ksqqn3XdZahbhnbJ0j+r1fvoqyymtUuY/fekrr9aEWEwbmt+WJ1ATv2l7F6xwHWFRT5ytgmhrhvbhzPK/eM6ZGLh9K/k7+77W/f8HdpdjN96XZufnUhZz3+FSf+eRbP2EWXAnM8Dsm17Ezd7b9xTmYyB8qrmtR91glGrOt9bwkE3tcBnTP9Akof+7TWw2hfEysLN06Kmrg4oY+dxcC9PLs+jDdhc0WVhUc4y0g79pUhAg9eMIjzh+WG7NuvUya3ju1DtzbW0oYT+ewYnk/r34E+HdIB+Pus2h/GN2t3+dxZ3T+ek/rmMG5AR8YN6EiKaynmpWuO9W27y786Bm+Av00c6uds8oC9tOV4QvVol+rzdBrX31pS+2DZdlbv8PeKcmwKdeG4nm1Zs7OIY/84k9P++gWvL8iP+VwnfsVt1+mS1SooCWQkAiv2PThjJTU1BmNq625ArfuzsxTVPiOZFdv20+euGU2eAsS9lFgXqmtMk8sejsCYhbTkhLAD7d4GVBZTZqyk++T36/0QkBllFcG9zNsSUGXhERkpCYhYa+4pCfFRU5Dfdlpf7jm7P1Bb48FZyimvquG6E3sGnXPJs3P55WuLAfjiB6sGuWMoDkVORjLPXj4cgI1h3EjPHdrFT/E4RjhnhpEYH0exvQQ2Z10h4wd2BGDmSmv6/eSleTx5aR6XHOsfDxELblsP1Noc3Ou84UiwjftVAQFbORnBS2zOUtS/vtnAfe8up6yymh92HPD73rV9re/dKyc96FgP2026oz2rqaox5DdBOgf3IP/Vml31GtzOfOxLRv7p0+gdm4AXvt4AwPnDurDy/86gT/vgv4XDvgacWf3jcyvx54E6xEN0cjmnuJd63bx+/fGcPaQzZZU1IR00miue1rM4nLGiuVuxZW+pn9tnJE45qj3PXTHc503VupW1PNQ2LYn2mcGDHliV4qqqa7jplYVA+JrhDif0stb2h9guu6Fw3GSTEuJ8A6izJJMYH+eb6u8vq+LiY7oy4/vtPPrJapIT4hg/sGO9a3MEDux7SiqJE7g0BsVzxQndmb9hD5OO7erX7tgtRvVux/nDuvDL/y1my55SerRL455pVrqTpVv2Mm/DHrpktQq67mLbeHpa/w5ceHSuz3MMLON675x05m+sTfexu6TCF2TYWATaqsoqa2L+n3NY6fJ++2HHAQ6UVXH0EdkNIt/BYIxhygwrUPWojhmkJMZzx4R+vGznUztrcCdf4CXA3tKGdzS4482lPHFJXtR+VdU1viwJAF2yg/+fwKoSuW5XMe8u3srSLfuCHpKaKzqz8BBnmSIlIbbbLCKM7dfBtz6f1y2LBy8YxL3nDIjoiur2qkiK8llpyQnM+tXooOC2D39xEh/8wrKdTLlgEB0ykzl9QEfmrCvk929/71sSSoqPY7DtDvjYpGEc0bY2CDE9OeGgijgFTtuf/9oymMdyzfYZKfz3Z8fTPsPf7VhE+O73p/H3S4bRzlZGu4sr/Hz1522wBnvH2+a9m0f56mc4FflSEuO56OhcxhzZ3nde61aJjBvQkc9XFfjadhc3fqCVo9CPtNfFl9ahvkf+nhI/F85r/zWfcX/9gguf+oYPlwUHOTY2v/pfbVoMxy6VnpzAuzeNonPrFM4b2sWvf0PZbOZt2E3XNtZg//mqgphmaw9/ZMUcndinHf+6eoQv0j8Ux/WwHtoC0+03Z1RZeIjjPlpdzzVPEWHSiG60bpXoW+oIhTOoQngXUzc92qX52TIAjuyYwVEdLWPwxBHdmHvnqeR1y6KssoaX5mz09UtPSeD+8wfxt4lDOWdIZ3q0S/PZC8JVu4uVUO62DbGM3iYtiazUJJ8dacX2A0E1N9wM7NKa207rC9QmI0xJDP9TGX1U7aCwO0reLC9w0kY4s88fPz07prxia3YWMepPs7jZnpUCfLKi1pvn8x8KQp3WqDixPwCnD+jo2x6U25pv7hjL4FzrwSU9OYH4OGmQGhL5e0r40T9m+2J+isqr/Gx84Viw0XIh75WTHlFRQO3fyp1xurmjysJDnNxBO2KsLx0JEeFNu25GoEeSO+J6YIQgoLoSalmmY2YK6ckJnOt6ooslWjUW0pOClcV5QzuH6Fk/HLfl37/9PSWVodehX7zKyu2VmWINPk7OrlBxLw6/Gnck795k+dM3xczCiR1xOxWc/fhXUc9zZhTfNnClRK8ItbTWLj2ZS4/txqs/PY4uWa0aJEdXqGC5B10528Lh2LUmjz8qal/nYW1x/j6fkmnuqLLwkIY2XuV1y2beXadyq0tZuLPYPnjBoJiT98VC5xDKIlRbQxEXF7zc9JcfD22w6zvLgv06ZYYtZetkyhWx6n4vtpWFO+gvkMT4OAZ2ySQ1KZ6te8tYW1DEu2HyeC3J30v3ye+zvB7BgeFwaoFcdHStt90PO6I/CYdb3Pv816M5skNGUIyQl1RW19B98vth40TCOTnExQkPnD+IQbmt6dYmlY0NUPHRPTvsZ7tez1y5M+JS1MbCYtbvKqZXTvCsPRqBsUXNFVUWHjLOnjbfGiI2ob7kZCT7RX1/PfkU33bbMIFr9cXxynKWYEb2buv32Q4D7dTha/84oUE/Hwj5efVFRDhnSGeK7aSJbrq2acXlxx/hc18OpGPryEpSROjTPp3VOw8w4W9fcvOrC32Dyw87DvDOIitvl2P7+WpNwy3xODaLtmlJPHpx7MrVXSv9zgm1T8OZKYl0bJ3iS2/SGJSUW9/hwen+WQgch4zLjjsi6jVys1uxJQZvtMrqmoipNtzH3GlEwhX+2l9WyckPfcbc9btZW1D32ImGUHCNgXpDecgvxvbh7MGdfAE5DcnDPxrCmp1FfgrC7anTEGSlJjHj1hNJS0rgpIdm+Rmz3fznmmPZvLu0QQb2zJQE9pdVcVzPNvWK1YhGtzapTFu8lUv/Odev/dWfHkdudmhFcWFebsgluUCO7JjBpyt3+pbl9pVWkpWaxLi/fgFYHlWOXachajJUVtdw+XPf+tyGWyXF+9msDpRVkpGSiDGGP7y7nFOOau+Xt6zYlQXgupN68eZ3W1i5/QDpKQmkpyTw+Q8FFNUxKWR9cex6gTEVNcYwIkZvoazUJHYVlVNZXRMxiPMP7y7jP3M28f0fTg/53QJTzqQkxlFWWeOLwzlQVklCXJxvWey3rtoUJ/aJPdvyt3eNZcQDM1tMrW6dWXiIO3Kzobno6Fwmjz/Kz1PIcbVtSPp1yqRb21Teu3kUd03oF7JPVmoSg3IbZmB/88YTeO1nxzP1uuP5/Vn9G+Sabk7o3TZke3Zq+HsX6DkWjr4dMvy8rAKXcd5euNWXJt2duqK+fLpyJ7PXFfLl6l2IWEuSQ7vVukQ7T8j7y6p48ZsNXP78t+xxOSE4teIdG9hbN47ko9tOIjE+jvm2HePthbWZjPP3lPDN2uB6KHWhsKic7pPf93la3fnWUv4zZ6PvCb4sIG1KaWVNzOV5nWs88nFwJmSAv3+6mknPzOGD760ZQrhZSP6eUr/sCd/faymV+Rv2sL+skkH3fkS/uz/wZYl2Bwi6lwKj0T4jhd7t0/3+Js0ZT5WFiJwhIqtEZI2ITA5x/K8issh+/SAie13HrhCR1fbrCi/lbOmc1r8DKYlx9MqJrTJdfRjYpXWdM8nWh97tMxjRwzu/8z7t/ZX3qvvP4KPbTgr53V659lh+PqZXVHdkh44B2YIDlcXDH63yW+IIVzckVtxZhjNTEhER2mekMPW64wCrLvnTn6/1W3JzZjZV1TX83i6re8PJvQBrZtLXfrh5fJIVV+CeLY55+DMueXYuJz80iyueD65qGAtOSphn7bidV+Zu4ndvf+8303MHGZZXVsfsev7bM6yltHCD78Mf/cDsdYW+J/k/f7AypF0xf08p/Tpl0r9TJndN6EdCfBx9O6TzwbLtDL73I1+/l+daXoLuGJXeEQIGQ5GdmuhXVrk545myEJF44AlgPNAfmCQifo+KxpjbjDFDjTFDgceBN+1z2wD3AMcCI4B7RKTpI4SaKc9ePpyV/zf+oGIcDhfaBNh1khNqB8hATujdjl+fHt2zxSHQY6rAHpQcJ4TdxRV+BW9CpaYoq6yO2VYQ74oQHuaaUXS27SuV1YYHZ6z0W256afYGAL88XqHcgp06K+7qg87S2cbCknq71ToKLpbElGDZY2INMOzfOZMBnTPD2iMyAh4IZq7cyVG//yCo35Y9pXTJasX0W0/kpydZmRPahUi2WVhU4Vfn5c8XDg7KRRYNa+nsMFcWWIP8GmPMOmNMBTAVODdC/0nAq/b26cDHxpjdxpg9wMfAGR7KqhwmuJ+Uf2nHUjQUWQHLgIVFFRhjgtbhnWAvx6hbUVXDzgNlvkj84x6cGVOeJveMwZ1aIjcgctj5HKgtXesu6RnqISMpIY7s1EQKimoH3rpUPwzFnz5YyYVPzQagotr43JIDKThQzv+9t5yvVu9iY2FJneo+RPKIimVmXFZZza6i8iCvv1Dn7iut5KoXrRnW8COy+fExXev8wPbt+t2s2Vnky67cnPFSWXQBNrv28+22IETkCKAH4CSniflcRakvNzeglxpYT7YZyVZOsPg4obC4nCo7EaEbJ8CyuMLKVNv3dzMY8cBMLnl2ri8obnlAMZ+1BUVB/vjOMgj4p4SPixO/xIdn/7025sLJmPuGXVXxjRtOCPt94uOE/8zZxOLNe3ln0Ra/GiZgJdqrS/2Ipz5b69uurK5h6rzQbrITHvuS575az2XPWUtTc9bFHodwRNs0Nu8uCVri276vjO1hlM5VL1i2nOVb9/tmGo5CdwilAl78ZgNfr7HSjP/i1Po9eDjJODc3QU6xuuKlsgh1f8M9Lk0EXjfGOI9AMZ0rIteJyHwRmV9Q0PTRpsrhTXycsPQPp7P+wTNpk5ZEYVGFbzD9icv1s6vtnvv2wi1+AY3u4Lg56wp5bOZquk9+n5oaw9i/fM6FT832ueBCbRzPpBFdg5wPOoXw3kpKiPOtjzvxDJEc2JzlkTe+y+fWqYuCjv/j87Vh3UkDCZwpGWN8+Z2ikRQlPb2b3u3Tqaw2nPmYf1Dibf8Nlt9h1qoC/jt/M1e8UGuHGdnLfxaVHCZ2wvF+GlUHLyg3E0dYec/2tgC7hZfKIh9wZ3XLBUJHKlnK4lXXfkznGmOeMcYMN8YMz8mJHF6vKA7zf3cq37jiU7ygpLyKqfM2M9suzNPT5Xww9ijrafLpL9b5apkHsrGwxOfVU+Iywt46dZEvLf28DXvo3T6dBy8YTPuAYMzeIbLk9u2QzkfLd9B98vu+tu5h3KHd7IyQgWD9ruKY8iYVBxTrqks8wotXHxNzX8fAvCqgfO4OVzrwgV0y+fausX7VI42ptdNAcFLL5DBG9lXbD0RMyhkNJ0vtmp3NP0eUl8piHtBHRHqISBKWQpgW2ElEjgSygdmu5g+BcSKSbRu2x9ltinLQtEtP9jQSHWoD5a7513zAf6rsBJoBLAuT9G9DYe1guiGgdkNhUYUv82+4dfhQgaCdAwILe7RLC1uBEOCT208iJTGOuetrFVqrxHiuHdXDt//Qh6t4bf7mUKf74ch7w+heIb32Tu6bQ5wQsv5IpFQrgYTzRnJco0/r34H3bj6R9hkpHOvyunvqszV8vHwHbdOSWHT3aUG2hxtG9/LLwnvLKVblyZ0HykmtY8S2m85ZrWifkRxUD6Y54pmyMMZUATdhDfIrgNeMMctE5D4ROcfVdRIw1bgeT4wxu4H/w1I484D77DZFaRHcMT44JuXXpx/JBcO6+KUO2RbG8+nL1bX2gbPsPE9OosXdxeVsto24N5wcXOcEoFvbVL78zRi/tsBBN9ryTu/2GUw8ppuvNrxDYO6j+Rv2EI159hLb8T3bcs2oWpmHdM1izh1jee6K4ax78EzOGhycC6xTlOh5N+nJCSGX1py683e7YneOPqINS+4dR3qyFQgKlidWVoiYmw6ZKbxxwwn84lRLCQ93BQqm17HefCCdslqxbX8Z5VXV3P7fRazZeSD6SU2Ap47zxpjpwPSAtrsD9u8Nc+7zwPOeCacoHrJ1n7/BMiE+jp+P6RHU77NVwba2dumh3SmP6pjBvA172F1cSVW1tTQULuocLNvI3yYO5ZjubchslciNL3/ndzw5QiZd9zXc3HRKbxLi49gw5Uz6/m4GFVU1YQ2RblbvKCJO4PhebfnEZec4ult2UHyKwwPnD+TSY6On+QjkqpE9mBqQY2rL3lIuPqZr0PfJTEkMWiKLxC9O7cv1J/fy86o72DQ7nTJTWLeriPkb9vDmwi28uXALlxzbjT+eH1swaGOhEdyK4gEX5vlH8vbrFDqWw6mh4U5pHS5zsJOfak9xhS/1RLRCS+cO7ULnrFakJyf4loKc5ZfiGCrAub2qvpl8Cj8f09u3P+eOsXRrk+qb5QSyq6icf32zAWMMawuK6NEujcT4OL+lrzsmhI9jqW8iQ0sBVPs8oowx7CutDBulX9cKAimJ8STGx/lq2lcfZB79jq1T+GFHEX/5qDaq/5W5m2JKM9+YqLJQFA9wB2edNbgTRx/hH5V+/jB/T/C/XzLMl+bcvYT1zs9H+rY720/gBUXlbNpdQlZqIpkpsa/nXzzc8hlxCjvFYmQ+tV+H2s8PsPO0SUtibL/2LNy012d0d/Pb15dwz7RlvDRnI1v3lvme6p1yuUd2yAiZw8mJxI5UcyQSma2sBRMnUeL6XcVU1xjChUC0D1F6Nxac1B71rVfj4BjTv9u01689VCGnfSWVXPrPOT7HCbDSutS3TnhdUGWhKB7gTrfeM4Rn0l8vHsqXvxlDckIcf/nREDJSEhmU25oNU87kyI4ZvH/LKKbdNJIhXbN80dmbdpfQLj2ZmSt2sHL7gbAZcsNxybHdWPfHCXVy82yVFE96coKfp5CbUb3bUVFdw5qdRRwoq+TdxVt9brJOcNzd7yxj6ZZ9vif7vh0yGJzbmj9eEDrtuFOONLByYqxk2Ap0f6mlLJxMv7vDpAF5/spjOK1/h5DHIjF+UCeSEuJ8yre+dM4KvQwXSt4Hpi/n6zWFTHp2DgDPf7WeY/84k6ft9CleollnFcUjrhnVg+e+Wk98mEfarm1SWXHfGSHreLgz7jrLPIVFFewqKvflNhpXjwEuLk7ISElkwqCOnDMktjjXxfeMC1v7wjGa/2LqIt/SWFF5FZNGdOOojhl+LqGZtiG4VVI80+xZVCjOHtwJYwwTBnWKSb5AnM9xZjuOjLeHidgf2KU1z14+3M+lOBa6ZLXih/vH10tGN+cO6cIzX6xnxbb9dMxM4ZGLh3DJs3ND5ox6bX6+3/597y0HYOaKHVxv5/jyClUWiuIRPx/TmziBiSO6hu0TSlEE8vbPR/KHd5fz8EVDGHJfbSI7p1BTfXjy0qNj7hsp9bzjurvJZbfIt1N8B8ZnVMa4ti8ifpUY64pvZmEri8LiCpIS4oLyggXyzs9HRkxt7hVxcUJetyxWbNvP9v1lPjkfn7mGEwKDAxPifIGc5z7xta+9qNx7+4YuQymKR7RJS+KuM/sfdPXC3OxUnr18OK1TE+nuMmh7/SQZC6HqQZRX1lBaUc3SgBiSc4c0XIncSDg2i/2llSzYuJsV2/aTk54cNW/TkK5Z9O9ct0SADYXbHbmNvVwXKmBz9JG1jhCLN9faOC7M8z4bks4sFKUF8cpPj+OEKZ9y2XHdIgbUNRahFGFFdQ3Lt+2ntLKapPg4KqpruGZUD47tGbqWSEPjuBNf/59aV+EhDVRvxSsyUhK5/PgjOLFPDu0zU2jdKtEvRUr+nhKSE+KpClE067dnHMW1J4aOt2lIdGahKC2IzvY6+f3nNQ8f/KSEON8y1eTxR5GUEMe/Z2/0eSI5GXfr4rV1sISK+A6VYry5cd+5A32G9gvyulDj8nAa9adZjP3LZ1RU1zCka5ZfhH5g0kOvUGWhKC2MWIsxNRb/vGI4YEVnO/Ud1hVYhu1BdszI6QPrboxvSFqnNp6yagj2FFdQXFHNhl3FvniL/WVVVFTVkJIQx20uY31jlL0FXYZSFOUgGXNke5b94XS/PFV/eNfy0nn4R0PYtLuEozo2ri2gXXqyX23rrhEi3ZsjRx+RzduLtrK2oMgvPUxFdU2QcmgsZdG8HlEURWmROIrif9cf79fePiO5XjEMB8uMW0/k8UnD+Pi2kzhzcCeuOTE41Upz5sQ+liF7b0mlLyklWFHtjnJwIsgPNjdVrKiyUBSlwTiyo39ak6wmWv7JyUjm7CGd6dMhgycuyWtUm0lD4Dgv7Cmp8Itkz99T6rO/3H5aX04f0IFeIYI+vUCVhaIoDUZmSiL3n1cbma114etHZkoC8XHC/e+v4JS/fO53zHGfHTegI0//ZHijxYaoslAUpUFx8l7dMLrp40BaKiJCVph0J2P7NY2zgBq4FUVpUNKSE1h897hGW0s/VOnWNpXCgPxQj/x4SBNJozMLRVE8oHVqYsQ0IUp0AhNFvnvTKC4ISH3fmKiyUBRFaYYELkOlxFCsyktUWSiKojRDrhnV06+meMpB1PpuCFRZKIqiNEO6tU3lk9tP9mUXboqMuG7UAqUoitKM+ffVI3jju3w6ZDZtfitVFoqiKM2Yrm1S+cWpoQs3NSa6DKUoiqJERZWFoiiKEhVPlYWInCEiq0RkjYhMDtPnxyKyXESWicgrrvZqEVlkv6Z5KaeiKIoSGc9sFiISDzwBnAbkA/NEZJoxZrmrTx/gDmCkMWaPiLR3XaLUGDPUK/kURVGU2PFyZjECWGOMWWeMqQCmAucG9Pkp8IQxZg+AMWanh/IoiqIo9cRLZdEF2Ozaz7fb3PQF+orI1yIyR0TOcB1LEZH5dvt5HsqpKIqiRMFL19lQiWECq40nAH2A0UAu8KWIDDTG7AW6GWO2ikhP4FMRWWqMWev3ASLXAdcBdOvWraHlVxRFUWy8nFnkA11d+7nA1hB93jHGVBpj1gOrsJQHxpit9vs64DNgWOAHGGOeMcYMN8YMz8nJafhvoCiKogAgxgQ+7DfQhUUSgB+AscAWYB5wiTFmmavPGcAkY8wVItIOWAgMBWqAEmNMud0+GzjXbRwP8XkFwMaDELkdsOsgzvea5i4fNH8Zm7t8oDI2BM1dPmheMh5hjIn6tO3ZMpQxpkpEbgI+BOKB540xy0TkPmC+MWaafWyciCwHqoFfG2MKReQE4GkRqcGa/UyJpCjszzuoqYWIzDfGDD+Ya3hJc5cPmr+MzV0+UBkbguYuH7QMGQPxNN2HMWY6MD2g7W7XtgFut1/uPt8Ag7yUTVEURYkdjeBWFEVRoqLKopZnmlqAKDR3+aD5y9jc5QOVsSFo7vJBy5DRD88M3IqiKMqhg84sFEVRlKgc9soilmSHjSRHVxGZJSIr7KSKt9rtbUTkYxFZbb9n2+0iIo/Zci8RkbxGkjNeRBaKyHv2fg8RmWvL918RSbLbk+39Nfbx7o0kX7JhgpkAAAiWSURBVJaIvC4iK+17eXxzuocicpv99/1eRF4VkZSmvoci8ryI7BSR711tdb5nInKF3X+1iFzRCDI+ZP+dl4jIWyKS5Tp2hy3jKhE53dXuye89lHyuY78SEWOHATTZPTxojDGH7QvLpXct0BNIAhYD/ZtIlk5Anr2dgRWj0h/4MzDZbp8M/MnengDMwIqUPw6Y20hy3g68Arxn778GTLS3/wHcYG/fCPzD3p4I/LeR5PsXcK29nQRkNZd7iJXuZj3QynXvrmzqewicBOQB37va6nTPgDbAOvs9297O9ljGcUCCvf0nl4z97d9yMtDD/o3He/l7DyWf3d4VK0RgI9CuKe/hQX/HphagSb88HA986Nq/A7ijqeWyZXkHK2PvKqCT3dYJWGVvP40V0Oj09/XzUKZcYCZwCvCe/c++y/WD9d1P+wdyvL2dYPcTj+XLtAdjCWhvFveQ2nxpbex78h5wenO4h0D3gIG4TvcMmAQ87Wr36+eFjAHHzgdetrf9fsfOffT69x5KPuB1YAiwgVpl0WT38GBeh/syVCzJDhsde7lhGDAX6GCM2QZgvztp3JtC9keB32BF2AO0BfYaY6pCyOCTzz6+z+7vJT2BAuAFe6nsnyKSRjO5h8aYLcDDwCZgG9Y9WUDzuocOdb1nTf1buhrraZ0IsjSqjCJyDrDFGLM44FCzkK+uHO7KIpZkh42KiKQDbwC/MMbsj9Q1RJtnsovIWcBOY8yCGGVoinubgLUU8JQxZhhQjLWEEo7GvofZWGn6ewCdgTRgfAQZmt3/J+FlajJZReQuoAp42WkKI0ujySgiqcBdwN2hDoeRozn+vX0c7soilmSHjYaIJGIpipeNMW/azTtEpJN9vBPg1PxobNlHAueIyAas2iSnYM00ssTKAxYog08++3hrYLeH8jmfmW+MmWvvv46lPJrLPTwVWG+MKTDGVAJvAifQvO6hQ13vWZP8lmwj8FnApcZeu2kmMvbCeihYbP9mcoHvRKRjM5GvzhzuymIe0Mf2RknCMiI2SQlXERHgOWCFMeYR16FpgOMVcQWWLcNpv9z2rDgO2OcsG3iBMeYOY0yuMaY71n361BhzKTALuCiMfI7cF9n9PX1KMsZsBzaLyJF201hgOc3kHmItPx0nIqn239uRr9ncQxd1vWdOnrdsewY1zm7zDLESkf4WOMcYUxIg+0Tbm6wHVibrb2nE37sxZqkxpr0xprv9m8nHcmDZTjO6h3WiqY0mTf3C8kz4ActL4q4mlGMU1pRzCbDIfk3AWqOeCay239vY/QWrbO1aYCkwvBFlHU2tN1RPrB/iGuB/QLLdnmLvr7GP92wk2YYC8+37+DaWV0mzuYfAH4CVwPfAS1geO016D4FXsWwolViD2jX1uWdYdoM19uuqRpBxDdYav/N7+Yer/122jKuA8a52T37voeQLOL6BWgN3k9zDg31pBLeiKIoSlcN9GUpRFEWJAVUWiqIoSlRUWSiKoihRUWWhKIqiREWVhaIoihIVVRZKi0JEqkVkkYgsFpHvxKrXHql/lojcGMN1PxORFlUT2WtE5EURuSh6T+VwQJWF0tIoNcYMNcYMwUoE92CU/llY2VubJa7IbUVp1qiyUFoymcAesHJqichMe7axVETOtftMAXrZs5GH7L6/sfssFpEpruv9SES+FZEfROREu2+8XTdhnl174Gd2eycR+cK+7vdOfzciskFE/mRf81sR6W23vygij4jILOBPYtWOeNu+/hwRGez6Ti/Ysi4RkQvt9nEiMtv+rv+z84khIlNEZLnd92G77Ue2fItF5Iso30lE5O/2Nd6nNnmgoqBPNUpLo5WILMKKbu6ElaMKoAw43xizX6wiM3NEZBpWIsGBxpihACIyHjgPONYYUyIibVzXTjDGjBCRCcA9WLmcrsFKx3CMiCQDX4vIR8AFWOmuHxCReCA1jLz77WtejpVL6yy7vS9wqjGmWkQeBxYaY84TkVOAf2NFov/e/uxBtuzZ9nf7nX1usYj8FrhdRP6Olab7KGOMkdpCQHcDpxtjtrjawn2nYcCRwCCgA1Yqkudj+qsohzyqLJSWRqlr4D8e+LeIDMRKofBHETkJK4V6F6wBL5BTgReMnUvIGONOzOckb1yAVZsArPw8g11r962xcg3NA54XK/nj28aYRWHkfdX1/ldX+/+MMdX29ijgQlueT0WkrYi0tmWd6JxgjNkjVvbf/lgDPFhFfGYD+7EU5j/tWcF79mlfAy+KyGuu7xfuO50EvGrLtVVEPg3znZTDEFUWSovFGDPbftLOwcr5kwMcbYypFCvTZ0qI04TwaZ/L7fdqan8bAtxsjAlK6GYrpjOBl0TkIWPMv0OJGWa7OECmUOeFklWAj40xk0LIMwIrOeFE4CbgFGPM9SJyrC3nIhEZGu472TMqzf+jhERtFkqLRUSOwiqVWYj1dLzTVhRjgCPsbgewytQ6fARcLVa9AQKWoULxIXCDPYNARPqKSJqIHGF/3rNY2YLD1e++2PU+O0yfL4BL7euPBnYZq5bJR1iDvvN9s4E5wEiX/SPVlikdaG2MmQ78AmsZCxHpZYyZa4y5G6vSnlPmM+g72XJMtG0anYAxUe6NchihMwulpeHYLMB6Qr7CXvd/GXhXROZjZSBdCWCMKRSRr0Xke2CGMebX9tP1fBGpAKYDd0b4vH9iLUl9J9a6TwGWzWM08GsRqQSKgMvDnJ8sInOxHsyCZgM292JV91sClFCbGvx+4Alb9mrgD8aYN0XkSuBV294Alg3jAPCOiKTY9+U2+9hDItLHbpuJVXd6SZjv9BaWDWgpVmbWzyPcF+UwQ7POKopH2Ethw40xu5paFkU5WHQZSlEURYmKziwURVGUqOjMQlEURYmKKgtFURQlKqosFEVRlKioslAURVGiospCURRFiYoqC0VRFCUq/w9LlTBbKYOjrwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_clas.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_clas.save('third_acc861_f889')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results\n",
    "|Dataset|epoch|train_loss|valid_loss|accuracy|f_beta  |seed|test split|\n",
    "|-------|-----|----------|----------|--------|--------|----|----------|\n",
    "|ASTD-B |6\t  |0.438674  |0.448438  |0.864662|0.857988|42  |0.1       |\n",
    "|ASTD   |6\t  |0.659341  |0.857126  |0.698801|0.677395|42  |0.2       |\n",
    "|AJGT   |5    |0.429798\t |0.452757\t|0.805556|0.745665|14  |0.2       |\n",
    "|LEV    |16   |1.185471  |1.251261  |0.523750|0.511322|14  |0.2       |\n",
    "|HARD   |7    |0.121737  |0.128373  |0.956695|0.956682|43  |0.2       |"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}