disaster-tweet-ulmfit-fastai.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import path\n",
    "import seaborn as sns\n",
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.callbacks import *\n",
    "from fastai.metrics import *\n",
    "\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('../input/nlp-getting-started/train.csv'),\n",
       " PosixPath('../input/nlp-getting-started/test.csv'),\n",
       " PosixPath('../input/nlp-getting-started/sample_submission.csv')]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path= Path('../input/nlp-getting-started')\n",
    "path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id keyword location                                               text  \\\n",
       "0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n",
       "1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n",
       "2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n",
       "3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n",
       "4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n",
       "\n",
       "   target  \n",
       "0       1  \n",
       "1       1  \n",
       "2       1  \n",
       "3       1  \n",
       "4       1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train=pd.read_csv(path/'train.csv')\n",
    "test=pd.read_csv(path/'test.csv')\n",
    "submission=pd.read_csv(path/'sample_submission.csv')\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7613 entries, 0 to 7612\n",
      "Data columns (total 5 columns):\n",
      "id          7613 non-null int64\n",
      "keyword     7552 non-null object\n",
      "location    5080 non-null object\n",
      "text        7613 non-null object\n",
      "target      7613 non-null int64\n",
      "dtypes: int64(2), object(3)\n",
      "memory usage: 297.5+ KB\n"
     ]
    }
   ],
   "source": [
    "# train = train.drop(columns='id')\n",
    "# test = test.drop(columns='id')\n",
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Distribution of target variable:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Target variable distribution')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFjBJREFUeJzt3Xu4XXV95/H3B6KiwyUoASEBQ2t0xFZRM3ittWgRr2C9TKjWTMVBO15qO49TdXxEUTo6dQbvWkaj6KiAWgQdOojgdRQwVEQEkYgKkVs0XL2gwe/8sX/BzeGck/1Lzz77xLxfz3Oes9Zv/dZa37WT7E9+a629dqoKSZJGtcOkC5AkbVsMDklSF4NDktTF4JAkdTE4JEldDA5JUheDQxpBkrOT/PsR+16T5DEzLDs0ybo5qunFST7fpu+W5JYk+8zRtt+Q5F1t+t8m2TQX223bu1+SG+Zqe5p/BodG0t6UNv/8JskvhuafO8+17JSkkiybr31W1cFVddJ87a9XVd1aVTtX1VWz9Rs1uKrq6Kp66VzUNjVIq+p7VbV4LratyVg06QK0baiqnTdPJ/kh8MKq+vzWbCvJoqqas//BjlOSHQCq6jeTrmW+bEt/PpoMRxyaE0keneTcJDcmuSrJcUkWtWWbRwh/leT7wEWt/SlJLktyQ5K3JTknyfOGtvmiJJcm2Zjk/yRZ2hZ9uf2+tI14Dp9Sy79p7fcdalvaRkm7J1mS5J+TbGjbPjXJ3kN9z0lyTJJzgZ8D+wzX1k7dfLGtuyHJCUl2mfKSPCrJd1uf45PcbYbXbd+2/58kuTzJi2d5jfdMcnqSm5J8HbjP0LI7jMKSHNb2f3OSK5O8PMm9gFOA3xsaLd4ryZuTfCzJSUluBla1tvdP2f+Lk1zd/nxfNtR+YpLXDs3fPqpJ8glgT+BzbX8vn3rqK8l+7bg2JvlektVDy96c5KNJPt6O5cIkB870Gml+GByaK78GXgrcE/gj4GnAC6f0eSrwMOAhSe4NnAT8DbAEuKotAyDJKuAVbTt7Ad8E/ndb/Nj2+/7t9Mynh3dSVT8DTgOOGGpeBZxRVdcz+Hv/PmA/YP+2/LgptT4PeD6wC3DNNMd7DHBv4A+B+wP/dcryI4CD27KHAK+cuoEkOwKnA18D9gEOBV6T5I+n2R/A8cBGBq/HXwEvmKEfwBrg+VW1C3Ag8JWq+inwDODy9rrt3NoAngmcAOwGfGqa7e0IPBL4PeApwBsyw3WcYVX1bOA64JC2v3dM0+0TwKXA3sCfA8clefTQ8me041kMnAW8bUv71XgZHJoTVXVeVX2jqm6rqu8D7wemvgEeW1U3VNUvgKcD36iqz1bVr4G3AtcP9X0R8KZ2PvzXwBuAxyTZa8SSPsYdg+PPWxtVdW1VnVpVv6iqG4H/Nk2t76+qS6vq11NP21TVd6vq7Kr6VVVdw+CNbOr6b6+qq6pqQ9v+EdzZY4CdquotbVvfAz7IIOTuIMlODF6z17a6LwA+OsvxbwIemGSXqvppVX1zlr4AX6qq06vqN+3PZzpHt31vDvHpjqlLkhXAg4HXtOs0axkE2F8MdTu7qs6sqtuAjzAIQk2QwaE5keSAdvrn2iQ3Aa8D9pjS7cqh6X2G59s1hB8PLb8P8L52GusGYAODN8NRL4ifAeyV5MFJ7gesAD7Tat0lyZokV7RaP7eFWqce6z5JPpHkx239929h/R+1453qPsDyzcfYjvNvGYxkpro3kGm2O5PDGYwirsjgjrCVs/SdWu8ofWY6pl77ABumhNWPgKVD88Mjvp8DO6OJMjg0V/4X8C/A71fVrgxO5WRKn+FHMV/NUAhkcBF6+M3iSuA/VNXioZ+7V9X5U7YzrTZK+SSD/xU/Fzhl6M3pVW3f/67VesgWap3qH4CfAX/Q1n/hNOvvOzS9H4NTcVNdCXx3yjHuUlXPmKbvNa2mqdudVlV9vaqeyuC01ueAj2/huEZ5TPZMx/Qz4B5Dy6YG32zbvgpYkuTuU7b94xn6awEwODRXdgFurKpbkjwQ+I9b6H8a8PAkT87gIvrfArsPLX8f8Nok9wdoF7WfCYNbT4EbGZxvn83HGJz2OaJND9f6c+CGJHsAr51m3dnsAtwC3JRkv1b7VC9Psnfb/qsYXM+Z6qsASV7RLm4vSvKgJA+d2rGqfslgxPSGJHdP8iAGgXgnGdwcsCrJrgyuPd0M3NYWXwvsmWRr/td+dNv3gxmcStp8TBcAT02yOIMbGF42Zb1rmfnPah1wIfCmDD6L8lBgNbOfhtOEGRyaK38DvDDJLcC7mf6N8nZVdTWDN/R3AD9hMAL4NnBrW/5x4F3AP7XTQRcAfzq0idcBn2ineJ4+w26+zOCi7m7A8K3Db2VwaumnDN68Tx/9MG/f92MYhNcpTH8x+UTgC8Bl7bj++9QObVT0ZOBRDE7PbADey8ynYl7EYARxLfCPDK6HzOQFbZs3MrjIv/lOpW8xCO0ftdfunrNsY9htwLnAD4D/CxxTVZvvblvDIACuAD7Lb0c3mx0LHNv2d4fPhtTgC4GeAxzAYFR1EvDKqvrKiHVpAuIXOWkhaKOOa4CnVdXXJ12PpJk54tDEJHlSkt3aHUNHMzh9dP6Ey5K0BQaHJumxDE59XAc8HnhGVf1qsiVJ2hJPVUmSujjikCR1+Z18yOEee+xRy5cvn3QZkrRNOf/8839SVUu21O93MjiWL1/O2rVrJ12GJG1Tksz2NILbeapKktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1OV38pPjc+Fhr/zwpEvQAnT+Pzx/0iVIE+eIQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUpexB0eSHZN8M8ln2/z+Sc5NclmSk5LctbXfrc2va8uXD23j1a390iRPHHfNkqSZzceI46+BS4bm3wIcV1UrgOuBI1v7kcD1VXVf4LjWjyQHAKuABwKHAu9JsuM81C1JmsZYgyPJMuApwPvbfICDgU+2LicAh7fpw9o8bfnjW//DgBOr6taq+gGwDjhonHVLkmY27hHH24D/Avymzd8LuKGqNrX59cDSNr0UuBKgLb+x9b+9fZp1bpfkqCRrk6zdsGHDXB+HJKkZW3AkeSpwXVWdP9w8TdfawrLZ1vltQ9XxVbWyqlYuWbKku15J0mjG+Q2AjwaenuTJwE7ArgxGIIuTLGqjimXAVa3/emBfYH2SRcBuwMah9s2G15EkzbOxjTiq6tVVtayqljO4uH12VT0X+ALwrNZtNXBqmz6tzdOWn11V1dpXtbuu9gdWAOeNq25J0uwm8Z3jfwecmORNwDeBD7T2DwAfSbKOwUhjFUBVfSfJycDFwCbgJVV12/yXLUmCeQqOqvoi8MU2fTnT3BVVVb8Enj3D+scCx46vQknSqPzkuCSpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqcuiSRcgqc8Vx/zhpEvQArTf6749b/tyxCFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqMrbgSLJTkvOSfCvJd5K8obXvn+TcJJclOSnJXVv73dr8urZ8+dC2Xt3aL03yxHHVLEnasnGOOG4FDq6qBwMHAocmeQTwFuC4qloBXA8c2fofCVxfVfcFjmv9SHIAsAp4IHAo8J4kO46xbknSLMYWHDVwS5u9S/sp4GDgk639BODwNn1Ym6ctf3yStPYTq+rWqvoBsA44aFx1S5JmN9ZrHEl2THIBcB1wJvB94Iaq2tS6rAeWtumlwJUAbfmNwL2G26dZZ3hfRyVZm2Tthg0bxnE4kiTGHBxVdVtVHQgsYzBKeMB03drvzLBspvap+zq+qlZW1colS5ZsbcmSpC2Yl7uqquoG4IvAI4DFSTY/zn0ZcFWbXg/sC9CW7wZsHG6fZh1J0jwb511VS5IsbtN3B54AXAJ8AXhW67YaOLVNn9bmacvPrqpq7avaXVf7AyuA88ZVtyRpduP8Iqe9gRPaHVA7ACdX1WeTXAycmORNwDeBD7T+HwA+kmQdg5HGKoCq+k6Sk4GLgU3AS6rqtjHWLUmaxdiCo6ouBB4yTfvlTHNXVFX9Enj2DNs6Fjh2rmuUJPXzk+OSpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuIwVHkrNGaZMk/e6b9atjk+wE3APYI8nuQNqiXYF9xlybJGkB2tJ3jr8IeAWDkDif3wbHTcC7x1iXJGmBmjU4qurtwNuTvKyq3jlPNUmSFrAtjTgAqKp3JnkUsHx4nar68JjqkiQtUCMFR5KPAL8PXADc1poLMDgkaTszUnAAK4EDqqrGWYwkaeEb9XMcFwH3HmchkqRtw6gjjj2Ai5OcB9y6ubGqnj6WqiRJC9aowfH6cRYhSdp2jHpX1ZfGXYgkadsw6l1VNzO4iwrgrsBdgJ9V1a7jKkyStDCNOuLYZXg+yeHAQWOpSJK0oG3V03Gr6tPAwXNciyRpGzDqqao/G5rdgcHnOvxMhyRth0a9q+ppQ9ObgB8Ch815NZKkBW/Uaxx/Oe5CJEnbhlG/yGlZklOSXJfk2iSfSrJs3MVJkhaeUS+OfxA4jcH3ciwFPtPaJEnbmVGDY0lVfbCqNrWfDwFLxliXJGmBGjU4fpLkeUl2bD/PA346zsIkSQvTqMHxAuA5wDXA1cCzgFkvmCfZN8kXklyS5DtJ/rq13zPJmUkua793b+1J8o4k65JcmOShQ9ta3fpflmT11hyoJGlujBocbwRWV9WSqtqTQZC8fgvrbAL+c1U9AHgE8JIkBwCvAs6qqhXAWW0e4EnAivZzFPBeGAQNcDTwcAafVj96c9hIkubfqMHxoKq6fvNMVW0EHjLbClV1dVX9S5u+GbiEwYX1w4ATWrcTgMPb9GHAh2vgHGBxkr2BJwJnVtXGVsOZwKEj1i1JmmOjBscOw//Lb6OAUT88SJLlDILmXGCvqroaBuEC7Nm6LQWuHFptfWubqX3qPo5KsjbJ2g0bNoxamiSp06hv/v8D+FqSTzJ41MhzgGNHWTHJzsCngFdU1U1JZuw6TVvN0n7HhqrjgeMBVq5c6eNQJGlMRhpxVNWHgWcC1wIbgD+rqo9sab0kd2EQGh+tqn9qzde2U1C039e19vXAvkOrLwOumqVdkjQBIz8dt6ourqp3VdU7q+riLfXPYGjxAeCSqvqfQ4tOAzbfGbUaOHWo/fnt7qpHADe2U1lnAIck2b2dLjuktUmSJmDk6xRb4dHAXwDfTnJBa3sN8Gbg5CRHAlcAz27LTgeeDKwDfk673beqNiZ5I/CN1u+YdnFekjQBYwuOqvoq01+fAHj8NP0LeMkM21oDrJm76iRJW2urvshJkrT9MjgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSF4NDktTF4JAkdTE4JEldDA5JUheDQ5LUxeCQJHUxOCRJXQwOSVIXg0OS1MXgkCR1MTgkSV0MDklSl7EFR5I1Sa5LctFQ2z2TnJnksvZ799aeJO9Isi7JhUkeOrTO6tb/siSrx1WvJGk04xxxfAg4dErbq4CzqmoFcFabB3gSsKL9HAW8FwZBAxwNPBw4CDh6c9hIkiZjbMFRVV8GNk5pPgw4oU2fABw+1P7hGjgHWJxkb+CJwJlVtbGqrgfO5M5hJEmaR/N9jWOvqroaoP3es7UvBa4c6re+tc3UfidJjkqyNsnaDRs2zHnhkqSBhXJxPNO01Sztd26sOr6qVlbVyiVLlsxpcZKk35rv4Li2nYKi/b6uta8H9h3qtwy4apZ2SdKEzHdwnAZsvjNqNXDqUPvz291VjwBubKeyzgAOSbJ7uyh+SGuTJE3IonFtOMnHgccBeyRZz+DuqDcDJyc5ErgCeHbrfjrwZGAd8HPgLwGqamOSNwLfaP2OqaqpF9wlSfNobMFRVUfMsOjx0/Qt4CUzbGcNsGYOS5Mk/SsslIvjkqRthMEhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSuhgckqQuBockqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFJ6mJwSJK6GBySpC4GhySpi8EhSepicEiSumwzwZHk0CSXJlmX5FWTrkeStlfbRHAk2RF4N/Ak4ADgiCQHTLYqSdo+bRPBARwErKuqy6vqV8CJwGETrkmStkuLJl3AiJYCVw7NrwcePtwhyVHAUW32liSXzlNt24M9gJ9MuoiFIG9dPekSdEf+3dzs6MzFVu4zSqdtJTime0XqDjNVxwPHz08525cka6tq5aTrkKby7+ZkbCunqtYD+w7NLwOumlAtkrRd21aC4xvAiiT7J7krsAo4bcI1SdJ2aZs4VVVVm5K8FDgD2BFYU1XfmXBZ2xNPAWqh8u/mBKSqttxLkqRmWzlVJUlaIAwOSVIXg0Oz8lEvWoiSrElyXZKLJl3L9sjg0Ix81IsWsA8Bh066iO2VwaHZ+KgXLUhV9WVg46Tr2F4ZHJrNdI96WTqhWiQtEAaHZrPFR71I2v4YHJqNj3qRdCcGh2bjo14k3YnBoRlV1SZg86NeLgFO9lEvWgiSfBz4OnD/JOuTHDnpmrYnPnJEktTFEYckqYvBIUnqYnBIkroYHJKkLgaHJKmLwSFthSSLk/ynedjP45I8atz7kXoYHNLWWQyMHBwZ2Jp/b48DDA4tKH6OQ9oKSTY/KfhS4AvAg4DdgbsAr62qU5MsB/65LX8kcDjwBODvGDy65TLg1qp6aZIlwPuA/douXgH8GDgHuA3YALysqr4yH8cnzcbgkLZCC4XPVtUfJFkE3KOqbkqyB4M3+xXAfYDLgUdV1TlJ9gG+BjwUuBk4G/hWC46PAe+pqq8m2Q84o6oekOT1wC1V9db5PkZpJosmXYD0OyDA3yd5LPAbBo+e36st+1FVndOmDwK+VFUbAZJ8ArhfW/YE4IDk9gcS75pkl/koXuplcEj/es8FlgAPq6pfJ/khsFNb9rOhftM9pn6zHYBHVtUvhhuHgkRaMLw4Lm2dm4HNI4LdgOtaaPwJg1NU0zkP+OMku7fTW88cWvY5Bg+UBCDJgdPsR1oQDA5pK1TVT4H/l+Qi4EBgZZK1DEYf351hnR8Dfw+cC3weuBi4sS1+edvGhUkuBl7c2j8DPCPJBUn+aGwHJHXw4rg0j5LsXFW3tBHHKcCaqjpl0nVJPRxxSPPr9UkuAC4CfgB8esL1SN0ccUiSujjikCR1MTgkSV0MDklSF4NDktTF4JAkdfn/07tKEBnKpIYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.target).set_title('Target variable distribution')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>text</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>target</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>keyword</th>\n",
       "      <td>0.801261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>location</th>\n",
       "      <td>33.272035</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          percentage\n",
       "id          0.000000\n",
       "text        0.000000\n",
       "target      0.000000\n",
       "keyword     0.801261\n",
       "location   33.272035"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#let's look at the Nan value\n",
    "(100.0 * train.isna().sum() / train.shape[0]).to_frame(name='percentage').sort_values(by='percentage')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "33% percent missing location, 0.8% keyword"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#let's see the correlation between keyword and disasters\n",
    "def is_keyword_in(data):\n",
    "    if data.keyword in data.text.split():\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "train['keyword_appears'] = train[['keyword', 'text']].dropna().apply(is_keyword_in, axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of keyword appearence in disasters\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>58.563023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>41.436977</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     percentage\n",
       "0.0   58.563023\n",
       "1.0   41.436977"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Percentage of keyword appearence in disasters')\n",
    "100.0 * train[train.target == 1].keyword_appears.value_counts(normalize=True).to_frame(name='percentage')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of keyword appearence in non disasters\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.0</th>\n",
       "      <td>56.1647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>43.8353</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     percentage\n",
       "0.0     56.1647\n",
       "1.0     43.8353"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Percentage of keyword appearence in non disasters')\n",
    "100.0 * train[train.target == 0].keyword_appears.value_counts(normalize=True).to_frame(name='percentage')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Almost the same numbers of keywords in both situation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Which locations have the most frequent disasters?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>USA</th>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York</th>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>United States</th>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>London</th>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Canada</th>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cuttack, Orissa</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Morganville, Texas.</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Yooooooo</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hartford, Connecticut</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Greenville, S.C.</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3341 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       count\n",
       "USA                      104\n",
       "New York                  71\n",
       "United States             50\n",
       "London                    45\n",
       "Canada                    29\n",
       "...                      ...\n",
       "Cuttack, Orissa            1\n",
       "Morganville, Texas.        1\n",
       "Yooooooo                   1\n",
       "Hartford, Connecticut      1\n",
       "Greenville, S.C.           1\n",
       "\n",
       "[3341 rows x 1 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.location.dropna().value_counts().to_frame(name='count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "fatalities               45\n",
       "armageddon               42\n",
       "deluge                   42\n",
       "sinking                  41\n",
       "body%20bags              41\n",
       "                         ..\n",
       "forest%20fire            19\n",
       "epicentre                12\n",
       "threat                   11\n",
       "inundation               10\n",
       "radiation%20emergency     9\n",
       "Name: keyword, Length: 221, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Distinct keywords in train dataset\n",
    "dist_keyword=train['keyword'].value_counts()\n",
    "dist_keyword"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Let's start the Fastai model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs=128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                    0\n",
       "keyword              61\n",
       "location           2533\n",
       "text                  0\n",
       "target                0\n",
       "keyword_appears      61\n",
       "dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#count the null values in each column\n",
    "train.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             0\n",
       "keyword       26\n",
       "location    1105\n",
       "text           0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we won't delete the location and keyword columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "train2 =copy.copy(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "## replacing na values in location,keyword with No\n",
    "train2['location'].fillna('No',inplace = True)\n",
    "train2['keyword'].fillna('No',inplace = True) \n",
    "# train2.drop(['keyword_appears'], axis=1 ,inplace = True)\n",
    "test['keyword'].fillna('No',inplace = True) \n",
    "test['location'].fillna('No',inplace = True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Our Deeds are the Reason of this #earthquake M...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Forest fire near La Ronge Sask. Canada</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>All residents asked to 'shelter in place' are ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>13,000 people receive #wildfires evacuation or...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7608</th>\n",
       "      <td>10869</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Two giant cranes holding a bridge collapse int...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7609</th>\n",
       "      <td>10870</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>@aria_ahrary @TheTawniest The out of control w...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7610</th>\n",
       "      <td>10871</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>M1.94 [01:04 UTC]?5km S of Volcano Hawaii.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7611</th>\n",
       "      <td>10872</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Police investigating after an e-bike collided ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7612</th>\n",
       "      <td>10873</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>The Latest: More Homes Razed by Northern Calif...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7613 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id keyword location  \\\n",
       "0         1      No       No   \n",
       "1         4      No       No   \n",
       "2         5      No       No   \n",
       "3         6      No       No   \n",
       "4         7      No       No   \n",
       "...     ...     ...      ...   \n",
       "7608  10869      No       No   \n",
       "7609  10870      No       No   \n",
       "7610  10871      No       No   \n",
       "7611  10872      No       No   \n",
       "7612  10873      No       No   \n",
       "\n",
       "                                                   text  target  \n",
       "0     Our Deeds are the Reason of this #earthquake M...       1  \n",
       "1                Forest fire near La Ronge Sask. Canada       1  \n",
       "2     All residents asked to 'shelter in place' are ...       1  \n",
       "3     13,000 people receive #wildfires evacuation or...       1  \n",
       "4     Just got sent this photo from Ruby #Alaska as ...       1  \n",
       "...                                                 ...     ...  \n",
       "7608  Two giant cranes holding a bridge collapse int...       1  \n",
       "7609  @aria_ahrary @TheTawniest The out of control w...       1  \n",
       "7610       M1.94 [01:04 UTC]?5km S of Volcano Hawaii.         1  \n",
       "7611  Police investigating after an e-bike collided ...       1  \n",
       "7612  The Latest: More Homes Razed by Northern Calif...       1  \n",
       "\n",
       "[7613 rows x 5 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def remove_emoji(string):\n",
    "    emoji_pattern = re.compile(\"[\"\n",
    "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           u\"\\U00002702-\\U000027B0\"\n",
    "                           u\"\\U000024C2-\\U0001F251\"\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "    return emoji_pattern.sub(r'', string)\n",
    "\n",
    "\n",
    "def remove_urls(text):\n",
    "    \n",
    "    url_pattern = re.sub(r'''(?i)\\b((?:https?://|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:'\".,<>?«»“”‘’]))''', \" \", text)\n",
    "    return url_pattern\n",
    "\n",
    "\n",
    "def remove_html(text):\n",
    "    cleanr = re.compile('<.*?>')\n",
    "    cleantext = re.sub(cleanr, '', text)\n",
    "    return cleantext\n",
    "    \n",
    "\n",
    "\n",
    "train2['text'] = train2['text'].apply(lambda x: remove_urls(x))\n",
    "train2['text'] = train2['text'].apply(lambda x: remove_html(x))\n",
    "\n",
    "test['text'] = test['text'].apply(lambda x: remove_urls(x))\n",
    "test['text'] = test['text'].apply(lambda x: remove_html(x))\n",
    "        #df[col] = df[col].apply(lambda text: remove_punctuation(text))\n",
    "#         df[col] = df[col].apply(lambda x: clean_text(x))\n",
    "     #   df[col] = df[col].apply(lambda x: replace_typical_misspell(x))\n",
    "      #  df[col] = df[col].apply(lambda x: unslang(x))\n",
    "train2['text'] = train2['text'].apply(lambda x: remove_emoji(x))\n",
    "test['text']= test['text'].apply(lambda x: remove_emoji(x))\n",
    "train2.drop(['keyword_appears'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply tokenization and numericalization\n",
    "# it basically creates a separate unit (i.e. a “token”) for each separate part of a word\n",
    "#o get the data ready for a language model\n",
    "# Language model data\n",
    "# data_lm = TextLMDataBunch.from_df('.', train_df=train2 , valid_df=test ,text_cols = 'text', bs=48 ).split_by_rand_pct(0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## create databunch with both train and test text and label for language model\n",
    "bs = 48\n",
    "data_lm = (TextList.from_df(pd.concat([train[['text']], test[['text']]]), '.')\n",
    "           .split_by_rand_pct(0.1, seed=42)\n",
    "           .label_for_lm()\n",
    "           .databunch(bs=bs, num_workers=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5456, 9789)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_lm.vocab.itos),len(data_lm.train_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>idx</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>from xxmaj xxunk # xxmaj alaska as smoke from # wildfires xxunk into a school xxbos # rockyfire xxmaj update = &gt; xxmaj california xxmaj hwy . 20 closed in both xxunk due to xxmaj lake xxmaj county fire - # xxunk # wildfires xxbos # flood # disaster xxmaj heavy rain causes flash flooding of streets in xxmaj xxunk , xxmaj colorado xxmaj springs areas xxbos i 'm on</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>xxunk game xxbos xxunk xxunk xxmaj people ' annihilated by last nights weather ' ... xxmaj really xxmaj xxunk thought you would have forecast that ... xxbos xxmaj xxunk other xxunk be annihilated xxunk up - to - the - minute feat : xxunk xxbos xxunk xxunk i seen xxmaj xxunk xxmaj xxunk in 2005 when xxmaj rob came back ; xxmaj xxunk as support . xxmaj fucking annihilated the</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>chicago - xxmaj area xxmaj gay xxmaj bar xxmaj admits to xxmaj arson xxmaj scheme : xxmaj frank xxmaj elliott pleaded guilty to hiring an arsonist to to ... http : / / t.co / xxunk xxbos xxmaj trusting xxmaj iran to stop terrorism is like xxunk an arsonist to join the fire brigade - xxmaj telegraph http : / / t.co / xxunk xxbos xxmaj big xxmaj top xxmaj</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>east xxmaj xxunk xxunk building http : / / t.co / xxunk xxbos niggas love hating . xxbos xxunk xxmaj do n't be like that babes &lt;3 you 'll have a ball of fun with me at xxup xxunk ! :) xxbos i liked a @youtube video from xxunk http : / / t.co / xxunk xxmaj minecraft xxmaj xxunk - o xxup blaze xxup que xxup usa xxup hack</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>to be a happy day but it 's a bloody friday xxunk xxrep 4 xxunk xxbos xxunk xxunk i should bloody hope so * she said xxunk her arms sitting back in her chair * xxbos ' i came to kill xxmaj xxunk ... for xxup fun ' : xxmaj video of xxunk and xxunk xxmaj pakistani killer shows him xxunk . http : / / t.co / xxunk xxbos</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "## check tokenisation looks ok on training set\n",
    "data_lm.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List of unique possible tokens is called the vocabulary. Listing below first 20 unique tokens in order of frequency:\n",
    "# data_lm.vocab.itos[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we replace the token with word ids\n",
    "# data_lm.train_ds[0][0].data[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save and load\n",
    "# data_lm.save('tmp_lm')\n",
    "# data_lm = TextLMDataBunch.load(path, 'tmp_lm', bs=48)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://s3.amazonaws.com/fast-ai-modelzoo/wt103-fwd\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#dwnld wiki103t to train model weight\n",
    "learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=1.).to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='1' class='' max='2', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      50.00% [1/2 00:05<00:05]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.485817</td>\n",
       "      <td>#na#</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='25' class='' max='73', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      34.25% [25/73 00:01<00:03 8.4672]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Min numerical gradient: 6.92E-02\n",
      "Min loss divided by 10: 4.37E-02\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4XWW59/HvnbmZ2rRJOrdpOjG30FBoy1QmrQqIgoKCDB4riuAB9Rw9vq/66sHh4HBURKgoogyKIDggWByglFJKCp0YWjqk85CkbeY59/vH3oUYMjU7a68Mv8917St7r/Xste+nO+2vzxqeZe6OiIhIbyWEXYCIiAxsChIREYmJgkRERGKiIBERkZgoSEREJCYKEhERiYmCREREYqIgERGRmChIREQkJklhF3C0cnNzvaCgIOwyREQGlNWrV5e5e14Q2x5wQVJQUEBxcXHYZYiIDChmtj2obWvXloiIxERBIiIiMVGQiIhITBQkIiISEwWJiIjEREEiIiIxUZCIiEhMBtx1JAPZxn1V/O31/WSkJDIiPYXh6cmMTE9hQs4wRmakYGZhlygictQUJFH1TS2s311BwagM8rJSj/r9Ow/W8ujLu6iqb+Z9J41l9sQRbwVDWXUD3396E79ZtYNW7/j9GSmJTByZzsSR6eRnpTIqM5W8zBTyslI5bcoocjJSYumeiEhghnSQbC2t5p8bS3l2Uykvbi2nobkVgOPGZnPmjFzOnJZHanICpVUNlFU3UFrVQEpiAqOHpzEmO43R2Wm8uqeC3xXv4oWt5ZhBcmICP1++jcLcDN5/8niSExO485+bqW1q4WPzCvjMudNIMONwbSOH65oor25k16FatpfXsvNgLdvLa1i9/RCHahvxaOikJCZw3rH5XDZnAmfPyCMpUXskRaT/MPdO/ovcTxUVFXmsU6S4Oz/+x2a+//QmAKbmZXD2jHzmThnJltJqlm0qZfX2QzS3Gz4kGB2OKCaNTOfyORP4wJwJZKUl8eT6vfz+5d28uO0gAOcek89/vedYpuVn9rjG5pZWDtU2sfNQLU+s28vjr+ymvKaR3MxU3nfSWC6aNY5TJo3Q7jAR6REzW+3uRYFse6gFSWNzK1/8/Tp+//JuPnDyeG65YAYTR6a/o111QzPFJQdJMCMvK5W8rFRy0lNoamnlQGUD+yrr2VdZz+isVE4tGElCwjv/Qd91qJbDtU2cMH54r+s9oqmllWc2lvLo6l38Y+MBGptbGT9iGO89aSwfKpp4VCElIkOPgqSNWILkcG0jN9y/mpVbD3LL+TO4+bxpA/J/9FX1TTz92n7+vG4vyzaV0urOJbPH89nzplOQmxF2eSLSDylI2uhtkGwvr+G6X77EroN1fOeyE7n05AkBVBd/ZdUN/GzZVu57oYSmFucDJ4/nxoXTug2U3YfrWLG5jBVbytlzuI7LiyZy8axxpCTp+IvIYKQgaaO3QfLPjQf4/MNrufOjp3Ba4agAKgvXgap6fvrMFh54cQeNza2cXjiSDxVNZNEJYxmWkkh5dQMvbC1nxZZyVmwuo6S8FoDczBSy05LZWlbD2OFpfPyMKVwxdxKZqUk0NrdS19hCU2sruZlHfyabiPQfCpI2Ytm1VdPQTEbq4D5RbX9lPY+s3sXDxTvZXl5LVmoS40YMY+P+KgCyUpM4rXAk86fmMn/aKGaOzgLgmU2l3PXMFl7cdpCUxAQcp6nl7d+NY8dm88FTxnPJ7PG9Oj1aRMKlIGmjL87aGgpaW51VJQd5uHgnZdWNnDZlJPOnjuLE8cO7PH34lR2HeHLDPhITjIyURIalJNHc0spf1u9l7a4KEhOMc2bk8d6TxrJwZr6ubxEZIBQkbShIwvPm/ioefXk3j7+ym32V9SQmGEWTczj/2NGMzxmGO7S60+pOZmoSo7PT3rq4MrGDs9pEJH4UJG0oSMLX2uqs313B317fz9Ov7eeNfVVdtk8wmDwqg1Mm5VBUkEPR5Bym5mV2eMp0R3YerOW5N8s4VNtIZX0TlXXNVDc0k2CQlJBAcqKRnJjA2BFpFOZmMCU3k8mj0klLTuyL7ooMCgqSNhQk/c/eijoq65oxi4QGGFX1TRyoauBAVQP7K+p5Y18VL+84xMGaRgCSE43stGSyhyWTnZZEbmYqM8dkcezYbI4dm/3WhZ1/XLuHl3ccfuuzUhITyB6WTGZqIg40tzhNLa00NLdSUdf0VjszOHdmPp9/10yOHZsd3z8QkX5IQdKGgmTgcne2ldVQvP0QW0trqKpvorK+mar6JvZV1LP5QPU7ZhM4ZkwWF80ax6ITxjBuxLAuRxlV9U2UlNWytaya1/ZW8tCLO6hqaObiWeO49YIZTB4V32tsKuubSDQjPSWx2+uVDtU0svS1fZgZF88ap9GU9LkBGyRmVgJUAS1Ac/tOmNlw4H5gEpF5v77r7vd2tU0FyeDV0NzC5gPVvLankoM1jZx7TD7To2eV9UZFbRN3LdvCvc9vo7nFmZafSWNzZPTS0NxCU0vkeM6RYzsZqUkcNzab48dlc/y44YwZnsauQ0fmQKvlUG0jE0emMy0/k6l5mUwamU5NQzPlNY0cqmmkrLqBLaU1bNpfxZsHqimtagAgMcHISksiOy2ZCTnDmDE6i5ljspgxOpNtZbX8ed0elr9Z9laI5mamct2CAq46fTLDhyX3yZ+tyEAPkiJ3L+tk/X8Bw939P80sD9gIjHH3xs62qSCRo3Wgsp67nt3KzkO1pCYlkJacSGpSAkkJRkKCkWBGgsHBmiZe3VPR4cgoPzpFzo6DtdQ1tXT6WRkpiUwbncX0aNgkWGRkUlXfTEVdE9vLa9m0v4raxre3MX7EMN43aywXnTSOyvom7np2K8s2lZKZmsQHThnP/Km5nFqQwyhdyyMxCDJIwr6owoEsi4z7M4GDQHO4Jclgk5+dxlcuOq7H7RuaW9i0r5rS6nom5KQzMSedYSmRXU2trc7eynq2HKhm16E6MtOSGJWRQk56CrnRaf+7243V2ursPlzHpv1VjMpMZdaE4f/ynvlTc9mwu4K7l23lty/t5FcvbAdgWn4mZ0zL5boFBXHfTSfSlaBHJNuAQ0QC4253X9JufRbwR+AYIAv4sLs/0dU2NSKRoaShuYUNuyt4cdtBXtp2kOe3lNPc0spFs8bx6XOmMXNM73f9ydAykHdtjXP3PWaWDzwN3OTuy9qsvwxYANwKTI22meXule22sxhYDDBp0qQ527dvD6xmkf7sQGU99yzfxv0rt1Pb2MIFx43mE2cWcmpBzoCcgFTiZ8AGyb98kNnXgGp3/26bZU8A33b356Kv/wF80d1XdbYdjUhEImd53buihPtWlFBR18Tx47K5fsEU3jdrLKlJOuNL3inIIAlsqlczy4juusLMMoALgQ3tmu0Azou2GQ3MBLYGVZPIYJGTkcKtF8xg5ZfO47ZLT6ChuZXP/W4tC779D772x1dZvf0QA+3Ufhm4AhuRmFkh8Fj0ZRLwoLvfZmY3ALj7XWY2DvglMBYwIqOT+7varkYkIu/k7izfXMb9K7fzz42lb9347H0njeXyoglMy9exlKFuUOza6isKEpGuHbnx2Z/W7uG56PUppxeO5KrTJ3PhcWN0z5khSkHShoJEpOfKqht4uHgnD764g12H6sjLSuX8Y0dzeuFI5hWOIj87LewSJU4UJG0oSESOXkurs2xTKb95aQcrNpdT1RC5XKswL4PFZxZyxdxJIVcoQRvMFySKSBwkJhgLj8ln4TH5tLQ6r+6p4IUt5Tz16j6++Pv1bNxfxf9573Ga7l96RUEiMsQkJhgnTRjBSRNG8G9nFnLbE6/zi+e3UVJWw4+uPJmsNM3vJUdHR91EhrDEBOMrFx3HbZeewLI3y7jspy+w82Bt2GXJAKMgERE+etpk7rtuLnsr6vjQ3S+wvbwm7JJkAFGQiAgAZ0zP5befnEddUwtXLlmpkYn0mIJERN5y7NhsHvi306hpbOEKhYn0kIJERP7F8eOG88C/nUZVfRNX/mwluw/XhV2S9HMKEhF5hxPGD+fXHz+Niromrlyykn0V9WGXJP2YgkREOjRr4gh+df1cyqsb+Mg9K9+6dbBIewoSEenUyZNyuPe6uew9XM9V97zIwZpO74ItQ5iCRES6NHfKSH5+TREl5TVc/fMXqahrCrsk6WcUJCLSrfnTcrn76jm8ub+aa+9dRX1TS9glST+iIBGRHjlnZj4/unI2a3Ye5guPrNONs+QtChIR6bF3nzCWL7xrJn9au4c7/rE57HKkn9CkjSJyVD519lQ276/me09vYlp+JotOHBt2SRIyjUhE5KiYGd/8wImcPGkEtz68lg27K8IuSUKmIBGRo5aWnMjdV88hJz2ZT/yqmPJqXWMylClIRKRX8rPSWPKxIsprGrnl4bW0turg+1ClIBGRXjth/HC+etFxLNtUyp3P6OD7UKUgEZGYfGTuJC6ZPY7vP72JFVvKwi5HQqAgEZGYmBnfvPREpuRm8NnfrOFAlSZ4HGoCDRIzKzGz9Wa2xsyKO1j/hei6NWa2wcxazGxkkDWJSN/LSE3izo/Ooaq+ic8+tIYWHS8ZUuIxIlno7rPdvaj9Cne/PbpuNvAl4Fl3PxiHmkSkj80ck8XXLz6BF7aW88CL28MuR+KoP+3auhJ4KOwiRKT3Li+awJnTc7n9qY3sr9QurqEi6CBxYKmZrTazxZ01MrN04N3AowHXIyIBMjO+cckJNLS08o0/vxZ2ORInQQfJAnc/BVgE3GhmZ3XS7iLg+c52a5nZYjMrNrPi0tLSoGoVkT5QkJvBZxZO48/r9vLsJv19HQoCDRJ33xP9eQB4DJjbSdMr6GK3lrsvcfcidy/Ky8vr+0JFpE998uxCCvMy+L+Pb9CU80NAYEFiZhlmlnXkOXAhsKGDdsOBs4E/BFWLiMRXalIi//3+E9hxsFazBA8BQY5IRgPLzWwtsAp4wt2fMrMbzOyGNu0uBZa6e02AtYhInM2fmssHThnP3cu2sHFfVdjlSIBsoN2cpqioyIuL33FJioj0Q+XVDbzrf5eRm5nK4zcuIC05MeyShiwzW93RZRh9oT+d/isig8yozFRuv3wWb+yr4jtPvRF2ORIQBYmIBGrhzHyunV/Avc+X8MzGA2GXIwFQkIhI4L646Bhmjs7i879bR5nuXTLoKEhEJHBpyYn88MrZVNY38Z+PrGOgHZuVrilIRCQujhmTzZcWHcPf3zjAb1/aGXY50ocUJCISN9fOL2DulJF868k3dHveQURBIiJxY2b89/tPoKahWWdxDSIKEhGJqxmjs/j4mVN4uHgXxSW6a8RgoCARkbi7+dzpjBuexpcf20BTS2vY5UiMFCQiEncZqUl89eLj2bi/ivtWlIRdjsRIQSIiobjwuNGce0w+P3h6E3sr6sIuR2KgIBGRUJgZX7voeJpbne88qQPvA5mCRERCM2lUOtefMYXH1+xhw+6KsMuRXlKQiEioPnXOVHLSk/nWk6/rivcBSkEiIqHKTkvmpnOn8/zmcpa9WRZ2OdILChIRCd1Vp09m0sh0vv3kG7S0alQy0ChIRCR0KUkJfOFdM3l9byWPv7I77HLkKClIRKRfeO+JYzlpwnC+t3Qj9U0tYZcjR0FBIiL9QkKC8cVFx7Cnop5f6iLFAUVBIiL9xvypuSycmcdPn9lCRV1T2OVIDylIRKRf+fy7ZlJR18SSZVvCLkV6SEEiIv3K8eOGc9GscfxieQkHqurDLkd6QEEiIv3OrRfMoLGllZ/8Y3PYpUgPBBokZlZiZuvNbI2ZFXfS5pzo+lfN7Nkg6xGRgWFKbgYfKprIg6t2sPNgbdjlSDfiMSJZ6O6z3b2o/QozGwHcCVzs7scDl8ehHhEZAD573nQSzPjB3zaFXYp0I+xdWx8Bfu/uOwDc/UDI9YhIPzFmeBrXzC/gsVd2s2l/VdjlSBeCDhIHlprZajNb3MH6GUCOmT0TbfOxjjZiZovNrNjMiktLSwMtWET6j0+dPZXMlCS+t3Rj2KVIF4IOkgXufgqwCLjRzM5qtz4JmAO8F3gX8H/NbEb7jbj7EncvcveivLy8gEsWkf4iJyOF68+Ywl9f3c/reyvDLkc6EWiQuPue6M8DwGPA3HZNdgFPuXuNu5cBy4BZQdYkIgPL9QumkJmaxI//8WbYpUgnAgsSM8sws6wjz4ELgQ3tmv0BONPMkswsHTgNeD2omkRk4Bmensy18wv4y/p9bNynYyX9UZAjktHAcjNbC6wCnnD3p8zsBjO7AcDdXweeAtZF29zj7u3DRkSGuI+fMYX0lETu+KeuK+mPkoLasLtvpYPdVO5+V7vXtwO3B1WHiAx8ORkpfGxeAXcv28Jnz5vOtPzMsEuSNsI+/VdEpEc+ceYU0pISuUPHSvodBYmIDAijMlO5et5k/rh2D1tLq8MuR9pQkIjIgPGJMwtJTkzQsZJ+RkEiIgNGXlYqV50+mcdf2c22spqwy5GoHgWJmU01s9To83PM7OboPFkiInH1ybMLSUlK4Md/17GS/qKnI5JHgRYzmwb8HJgCPBhYVSIincjPSuNj8wp4fM1utuhYSb/Q0yBpdfdm4FLgf939FmBscGWJiHRu8VmFpCYl8iONSvqFngZJk5ldCVwD/Dm6LDmYkkREupabmco18wv449o9vKmZgUPX0yC5DpgH3Obu28xsCnB/cGWJiHRt8VmFpCcn8kONSkLXoyBx99fc/WZ3f8jMcoAsd/92wLWJiHRqZEYK1y4o4In1ezUHV8h6etbWM2aWbWYjgbXAvWb2/WBLExHp2ifOLCQjJYkf/l13UQxTT3dtDXf3SuADwL3uPgc4P7iyRES6NyI9hWvnF/Dkhn1sL9d1JWHpaZAkmdlY4EO8fbBdRCR0V8+bTKIZ96/cHnYpQ1ZPg+TrwF+BLe7+kpkVAjrCJSKhG52dxrtPGMNvX9pJXWNL2OUMST092P47dz/J3T8Vfb3V3T8YbGkiIj1zzfwCKuub+cOa3WGXMiT19GD7BDN7zMwOmNl+M3vUzCYEXZyISE8UTc7h2LHZ3PfCdtw97HKGnJ7u2roX+CMwDhgP/Cm6TEQkdGbGNfMm8/reSl4qORR2OUNOT4Mkz93vdffm6OOXQF6AdYmIHJVLZo8nOy2J+14oCbuUIaenQVJmZleZWWL0cRVQHmRhIiJHY1hKIh8+dSJ/3bCPfRX1YZczpPQ0SK4ncurvPmAvcBmRaVNERPqNq08voMWdB1ftCLuUIaWnZ23tcPeL3T3P3fPd/f1ELk4UEek3Jo1K59yZ+Tz44g4amnUqcLzEcofEW/usChGRPnLN/ALKqhv489q9YZcyZMQSJNZtA7MSM1tvZmvMrLiD9eeYWUV0/Roz+0oM9YiIcOb0XKbnZ/Lz5dt0KnCcxBIkPf2GFrr7bHcv6mT9c9H1s9396zHUIyKCmXH9GVN4bW8lK7ceDLucIaHLIDGzKjOr7OBRReSaEhGRfufSk8czMiOFny/fFnYpQ0KXQeLuWe6e3cEjy92TerB9B5aa2WozW9xJm3lmttbMnjSz44+6ByIi7aQlJ/LR0ybx9zf2U1KmWYGDFsuurZ5Y4O6nAIuAG83srHbrXwYmu/ss4MfA4x1txMwWm1mxmRWXlpYGW7GIDApXnz6ZpATj3uc1KglaoEHi7nuiPw8AjwFz262vdPfq6PO/AMlmltvBdpa4e5G7F+Xl6YJ6EelefnYaF80ax+9W76Kirinscga1wILEzDLMLOvIc+BCYEO7NmPMzKLP50br0RXzItInPn7GFGobW/iNLlAMVJAjktHAcjNbC6wCnnD3p8zsBjO7IdrmMmBDtM2PgCtc5+uJSB85ftxwTi8cyX0rSmhqaQ27nEGrJwfMe8XdtwKzOlh+V5vndwB3BFWDiMj1C6aw+Ner+dtr+1l04tiwyxmUgj7YLiISqvOOHc34EcO4/0XdijcoChIRGdQSE4wr507k+c3lbCmtDrucQUlBIiKD3odOnUhSgvHASh10D4KCREQGvfysNN51whgeWb2TukbNCtzXFCQiMiRcffpkRuzdye6PXgfZ2ZCQEPn56U/Dli1hlzeg2UA727aoqMiLi98xkbCISJf8L3+h4f0fJLGlmeTW5rdXJCdHHo88AosWhVdgwMxsdReT58ZEIxIRGfy2bMEuv5y0pvp/DRGApiaorYXLLtPIpJcUJCIy+H3ve5HA6EpTE/zgB/GpZ5BRkIjI4Hf//T0Lkl//Oj71DDIKEhEZ/Kp7eP1IT9vJv1CQiMjgl5nZt+3kXyhIRGTwu+qqyJlZXUlOhquvjk89g4yCREQGv899rmdBcsst8alnkFGQiMjgN3Vq5DqR9PR3BIonJ0eWP/JIpJ0cNQWJiAwNixbBunWweDFkZ+NmVKWks+niKyLLB/HFiEEL7H4kIiL9ztSpcMcdkYc7H/rRchIMntBIJCYakYjIkGQWmV7+1T2VrN9VEXY5A5qCRESGrEtmjyctOYGHXtL08rFQkIjIkDV8WDLvPXEcf3hlNzUNzd2/QTqkIBGRIe0jp02kprGFP67dE3YpA5aCRESGtFMm5XDMmCzuW1HCQLutRn+hIBGRIc3MuHZ+AW/sq+KlkkNhlzMgKUhEZMi7ZPZ4hg9L5r4VJWGXMiAFGiRmVmJm681sjZl1eltDMzvVzFrM7LIg6xER6ciwlEQ+fOpEnnp1H/sq6sMuZ8CJx4hkobvP7uwWj2aWCHwH+GscahER6dBVp02m1Z0HXtwedikDTn/YtXUT8ChwIOxCRGTomjQqnfOOyeehVTtoaG4Ju5wBJeggcWCpma02s8XtV5rZeOBS4K6uNmJmi82s2MyKS0tLAypVRIa6j80roKy6kb+s3xt2KQNK0EGywN1PARYBN5rZWe3W/y/wn+7eZfy7+xJ3L3L3ory8vKBqFZEh7oxpuRTmZXDfCu3eOhqBBom774n+PAA8Bsxt16QI+I2ZlQCXAXea2fuDrElEpDMJCcbHTp/Mmp2HWbvzcNjlDBiBBYmZZZhZ1pHnwIXAhrZt3H2Kuxe4ewHwCPBpd388qJpERLrzwTkTyExN4mfPbQ27lAEjyBHJaGC5ma0FVgFPuPtTZnaDmd0Q4OeKiPRaVloyV50+mb+s38u2spqwyxkQAgsSd9/q7rOij+Pd/bbo8rvc/R0H1939Wnd/JKh6RER66uNnTCE5MYG7n90SdikDQn84/VdEpF/Jy0rlw6dO5NGXd7G3oi7scvo9BYmISAcWn1WIO/xs2bawS+n3FCQiIh2YkJPOJbPH89CqHZRXN4RdTr+mIBER6cSnzimkvrmFX2oyxy4pSEREOjEtP4t3Hz+GX64ooaq+Kexy+i0FiYhIFz59zjSq6pv59Upd7d4ZBYmISBdOnDCcs2fkcc9z26ht1H3dO6IgERHpxs3nTedgTSMPrNwRdin9koJERKQbcybncOb0XO5etoW6Rk0x356CRESkB24+bzpl1Y08uEqjkvYUJCIiPXBqwUjmFY7irme3UN+kUUlbChIRkR66+bzplFY18NuXdoZdSr+iIBER6aF5U0cxd8pIfvrMFt2Otw0FiYjIUfjsedPZV1nPw8W7wi6l31CQiIgchflTRzFncg53/nOzjpVEKUhERI6CmXHrBTPYW1GvYyVRChIRkaM0P3qs5CcalQAKEhGRo2ZmfO6CGRyoauB+zcGlIBER6Y3TCkdxxrRc7np2y5Cfg0tBIiLSS7dcMIOy6kbuWzG0RyUKEhGRXpozOYdzZuZx97ItQ/p+JQoSEZEY3HL+DA7XNvHL50vCLiU0gQaJmZWY2XozW2NmxR2sv8TM1h1Zb2ZnBFmPiEhfmzVxBOcfO5olz22lonZojkriMSJZ6O6z3b2og3V/B2a5+2zgeuCeONQjItKnPnfhDKrqm1ny3JawSwlFqLu23L3a3T36MgPwrtqLiPRHx47N5qJZ4/jF8hJKqxrCLifugg4SB5aa2WozW9xRAzO71MzeAJ4gMioRERlwbjl/Oo0trdz5zOawS4m7oINkgbufAiwCbjSzs9o3cPfH3P0Y4P3ANzraiJktjh5DKS4tLQ22YhGRXijMy+SyUybwwMod7D5cF3Y5cRVokLj7nujPA8BjwNwu2i4DpppZbgfrlrh7kbsX5eXlBVaviEgsbj5/OgA//vubIVcSX4EFiZllmFnWkefAhcCGdm2mmZlFn58CpADlQdUkIhKk8SOG8ZHTJvG71bvYVlYTdjlxE+SIZDSw3MzWAquAJ9z9KTO7wcxuiLb5ILDBzNYAPwE+3Obgu4jIgHPjwmmkJCbw/ac3hV1K3CQFtWF33wrM6mD5XW2efwf4TlA1iIjEW15WKtefUcBP/rmFj542idMLR4VdUuB0ZbuISB/7zMLpTBqZzpd+v35ITDOvIBER6WPDUhL55qUnsq2shh//Y/AfeFeQiIgE4IzpuVw2ZwJ3P7uV1/ZUhl1OoBQkIiIB+fJ7jmVEejJf/P06mltawy4nMAoSEZGA5GSk8NWLjmfdrgp+uaIk7HICoyAREQnQ+04ay3nH5PPdpRvZV1EfdjmBUJCIiATIzPjaxcfT0uqD9sC7gkREJGATR6ZzxamT+O1LO9lePviueFeQiIjEwU3nTiMp0fjBILziXUEiIhIH+dlpXDt/Cn9Yu4c39g2u04EVJCIicXLD2YVkpibxvaWDa1SiIBERiZMR6Sl88qxCnn5tPy/vOBR2OX1GQSIiEkfXLZjCqIwUbn9qI4NlsnMFiYhIHGWkJvGZc6fxwtZyrvr5i6zfVRF2STFTkIiIxNk18wr4yvuO47U9lVx0x3JueugVdpTXhl1Wr9lAG1oVFRV5cXFx2GWIiMSssr6JJc9u5Z7lW2lpda44dRI3nTuN/Oy0Pv8sM1vt7kV9vmEUJCIiodtfWc8P//4mD7+0k6RE45p5Bdxw9lRyMlIAqG9qoaKuiZTEhLeWHS0FSRsKEhEZrLaX1/DDv73JY2t2Myw5kczUJCrqmmhojswc/OlzpvIf7z6mV9sOMkgCu9WuiIgcncmjMvj+h2fzqXOmcu+KElpanBHpyQxPT2b4sGROGj8i7BI7pCAREelnpo/O4puXnhh2GT1cct7AAAAJG0lEQVSms7ZERCQmChIREYmJgkRERGIS6DESMysBqoAWoLn9GQNm9lHgP6Mvq4FPufvaIGsSEZG+FY+D7QvdvayTdduAs939kJktApYAp8WhJhER6SOhnrXl7ivavFwJTAirFhER6Z2gj5E4sNTMVpvZ4m7afhx4sqMVZrbYzIrNrLi0tLTPixQRkd4LekSywN33mFk+8LSZveHuy9o3MrOFRILkjI424u5LiOz2oqioaGBdii8iMsjFbYoUM/saUO3u3223/CTgMWCRu3d72zAzKwW2t1s8HGg/F3P7ZV29PvK87bJcoLNjO93pqJ6jaXO0/enueSx96a7W7toMpu+mJ31pvyzI70a/Z10vH6i/Z52ti/W7yXD3vG4r7w13D+QBZABZbZ6vAN7drs0kYDMwP8bPWtLdsq5eH3nebllxX9ZzNG2Otj/dPY+lL7H2ZzB9Nz3pSzy/G/2eDc7fs/743XT3CHLX1mjgMTODyC60B939KTO7AcDd7wK+AowC7oy2e8cpwj30px4s6+r1nzpp01s92U5XbY62Pz15HotY+jOYvpue9KX9siC/G/2edb18oP6edbYuzO+mSwNu9t94MbPiXoZavzOY+gKDqz/qS/81mPoTdF90ZXvnloRdQB8aTH2BwdUf9aX/Gkz9CbQvGpGIiEhMNCIREZGYDPogMbNfmNkBM9vQi/fOMbP1ZrbZzH5k0TMCoutuMrONZvaqmf1P31bdZU193h8z+5qZ7TazNdHHe/q+8g7rCeS7ia7/vJm5meX2XcXd1hTEd/MNM1sX/V6Wmtm4vq+8w3qC6MvtZvZGtD+PmVnc7tIUUH8uj/79bzWzwI+lxNKHTrZ3jZm9GX1c02Z5l3+3OhTkKWH94QGcBZwCbOjFe1cB8wAjctX9oujyhcDfgNTo6/wB3p+vAZ8fDN9NdN1E4K9ErjfKHcj9AbLbtLkZuGsA9+VCICn6/DvAdwb4d3MsMBN4Bijqr32I1lfQbtlIYGv0Z070eU5X/e3qMehHJB65kv5g22VmNtXMnopO3fKcmb3jJshmNpbIX+IXPPKn+yvg/dHVnwK+7e4N0c84EGwv3hZQf0IRYF9+APwHkSl64iaI/rh7ZZumGcSpTwH1Zam7N0ebxnVuvYD687q7b4xH/dHP61UfOvEu4Gl3P+juh4CngXf39t+JQR8knVgC3OTuc4DPA3d20GY8sKvN613RZQAzgDPN7EUze9bMTg202u7F2h+Az0R3OfzCzHKCK7VbMfXFzC4Gdnv/uR1BzN+Nmd1mZjuBjxK59iosffF7dsT1dDK3Xhz1ZX/C0pM+dGQ8sLPN6yP96lV/h9w9280sE5gP/K7Nrr/Ujpp2sOzI/waTiAwHTwdOBR42s8JogsdVH/Xnp8A3oq+/AXyPyF/0uIq1L2aWDnyZyC6U0PXRd4O7fxn4spl9CfgM8NU+LrVbfdWX6La+DDQDD/RljUejL/sTlq76YGbXAZ+NLpsG/MXMGoFt7n4pnferV/0dckFCZBR22N1nt11oZonA6ujLPxL5x7Xt0HsCsCf6fBfw+2hwrDKzViJz2YQxNXHM/XH3/W3e9zPgz0EW3IVY+zIVmAKsjf7FmgC8bGZz3X1fwLV3pC9+19p6EHiCEIKEPupL9KDu+4DzwviPVxt9/d2EocM+ALj7vcC9AGb2DHCtu5e0abILOKfN6wlEjqXsojf9DfoAUX94AAW0OUBFZN6vy6PPDZjVyfteIjLqOHLQ6T3R5TcAX48+n0FkiGgDuD9j27S5BfjNQO1LuzYlxPFge0DfzfQ2bW4CHhnAfXk38BqQF8/vJOjfNeJ0sL23faDzg+3biOxZyYk+H9mT/nZYVxhfaJx/eR4C9gJNRNL240T+1/oUsDb6i/2VTt5bBGwAtgB38PYFnCnA/dF1LwPnDvD+/BpYD6wj8r+wsQO1L+3alBDfs7aC+G4ejS5fR2TepPEDuC+bifyna030EZcz0ALsz6XRbTUA+4G/9sc+0EGQRJdfH/1ONgPXddffrh66sl1ERGIyVM/aEhGRPqIgERGRmChIREQkJgoSERGJiYJERERioiCRQcHMquP8efeY2XF9tK0Wi8zuu8HM/tTdrLhmNsLMPt0Xny3SF3T6rwwKZlbt7pl9uL0kf3uCwUC1rd3M7gM2ufttXbQvAP7s7ifEoz6R7mhEIoOWmeWZ2aNm9lL0sSC6fK6ZrTCzV6I/Z0aXX2tmvzOzPwFLzewcM3vGzB6xyH00Hjhyb4bo8qLo8+roxIprzWylmY2OLp8aff2SmX29h6OmF3h7AspMM/u7mb1skftDXBJt821ganQUc3u07Rein7POzP5fH/4xinRLQSKD2Q+BH7j7qcAHgXuiy98AznL3k4nMpvvNNu+ZB1zj7udGX58M/DtwHFAILOjgczKAle4+C1gGfKLN5/8w+vndzlcUnefpPCKzCwDUA5e6+ylE7oHzvWiQfRHY4u6z3f0LZnYhMB2YC8wG5pjZWd19nkhfGYqTNsrQcT5wXJuZUbPNLAsYDtxnZtOJzGya3OY9T7t723s+rHL3XQBmtobIXEfL231OI29PdLkauCD6fB5v38vhQeC7ndQ5rM22VxO5NwRE5jr6ZjQUWomMVEZ38P4Lo49Xoq8ziQTLsk4+T6RPKUhkMEsA5rl7XduFZvZj4J/ufmn0eMMzbVbXtNtGQ5vnLXT8d6bJ3z7Y2FmbrtS5+2wzG04kkG4EfkTk/iN5wBx3bzKzEiCtg/cb8C13v/soP1ekT2jXlgxmS4ncvwMAMzsy3fZwYHf0+bUBfv5KIrvUAK7orrG7VxC5ne7nzSyZSJ0HoiGyEJgcbVoFZLV561+B66P3p8DMxptZfh/1QaRbChIZLNLNbFebx61E/lEuih6Afo3I9P8A/wN8y8yeBxIDrOnfgVvNbBUwFqjo7g3u/gqRmVyvIHLjpyIzKyYyOnkj2qYceD56uvDt7r6UyK6zF8xsPfAI/xo0IoHS6b8iAYnesbHO3d3MrgCudPdLunufyECjYyQiwZkD3BE90+owIdy+WCQeNCIREZGY6BiJiIjEREEiIiIxUZCIiEhMFCQiIhITBYmIiMREQSIiIjH5/x1BpPv/5W5UAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "#train last layer\n",
    "learn.recorder.plot(skip_end=15 ,suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.06918309709189363"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# access the corresponding learning rate \n",
    "min_grad_lr = learn.recorder.min_grad_lr\n",
    "min_grad_lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-2\n",
    "lr *= bs/48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.851868</td>\n",
       "      <td>3.762139</td>\n",
       "      <td>0.355283</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.978094</td>\n",
       "      <td>3.399087</td>\n",
       "      <td>0.399814</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3.565922</td>\n",
       "      <td>3.301796</td>\n",
       "      <td>0.412091</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3.365664</td>\n",
       "      <td>3.284777</td>\n",
       "      <td>0.414881</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# just fine-tuning the last layers\n",
    "learn.fit_one_cycle(4, lr ,moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>3.189915</td>\n",
       "      <td>3.204182</td>\n",
       "      <td>0.423847</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.086931</td>\n",
       "      <td>3.145515</td>\n",
       "      <td>0.432143</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2.957583</td>\n",
       "      <td>3.068869</td>\n",
       "      <td>0.446019</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.812179</td>\n",
       "      <td>3.028883</td>\n",
       "      <td>0.454836</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.668868</td>\n",
       "      <td>3.026144</td>\n",
       "      <td>0.455171</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2.526497</td>\n",
       "      <td>3.014939</td>\n",
       "      <td>0.461123</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.393943</td>\n",
       "      <td>3.017669</td>\n",
       "      <td>0.462798</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.295583</td>\n",
       "      <td>3.028005</td>\n",
       "      <td>0.465737</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.229485</td>\n",
       "      <td>3.034883</td>\n",
       "      <td>0.465476</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.187034</td>\n",
       "      <td>3.038361</td>\n",
       "      <td>0.464993</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(10, lr/8, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "## save model and encoder\n",
    "learn.save('fine_tuned')\n",
    "learn.save_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas = (TextList.from_df(train[['text', 'target']], '.', vocab=data_lm.vocab)\n",
    "#           .split_by_folder(valid='test')\n",
    "             .split_by_rand_pct(0.1)\n",
    "             .label_from_df('target')\n",
    "             .add_test(TextList.from_df(test[['text']], '.', vocab=data_lm.vocab))\n",
    "#              .filter_missing_y()\n",
    "             .databunch(bs=bs, num_workers=1))\n",
    "data_clas.save('tmp_clas')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>xxbos _ \\n  xxrep 5 ? xxup retweet \\n  xxrep 7 ? \\n  xxrep 5 ? xxup follow xxup all xxup who xxup rt \\n  xxrep 7 ? \\n  xxrep 5 ? xxup xxunk \\n  xxrep 7 ? \\n  xxrep 5 ? xxup xxunk xxup with \\n  xxrep 7 ? \\n  xxrep 5 ? xxup follow ? xxunk # xxup xxunk</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos . : . : . : . : . : . : . : . : . : . : . : . : . : . : . : . : . : . : . : . : . : xxup rt xxunk : # xxunk \\n \\n  xxmaj indian xxmaj army xxunk _ http : / / t.co / xxunk g</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj truth ... \\n  https : / / t.co / xxunk \\n  # xxmaj news \\n  # xxup bbc \\n  # xxup cnn \\n  # xxmaj islam \\n  # xxmaj truth \\n  # god \\n  # xxup isis \\n  # terrorism \\n  # xxmaj quran \\n  # xxmaj lies http : / / t.co / xxunk</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj truth ... \\n  https : / / t.co / xxunk \\n  # xxmaj news \\n  # xxup bbc \\n  # xxup cnn \\n  # xxmaj islam \\n  # xxmaj truth \\n  # god \\n  # xxup isis \\n  # terrorism \\n  # xxmaj quran \\n  # xxmaj lies http : / / t.co / xxunk</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>xxbos xxmaj no # news of # hostages in # xxmaj libya \\n \\n  http : / / t.co / xxunk \\n \\n  # xxmaj india # terrorism # xxmaj africa # xxup ap # xxup ts # xxup nri # xxmaj news # xxup trs # xxup tdp # xxup bjp http : / / t.co / xxunk</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data_clas.show_batch()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Text classifier learner **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5).to_fp16()\n",
    "learn.load_encoder('fine_tuned_enc')\n",
    "learn.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='95' class='' max='142', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      66.90% [95/142 00:02<00:01 1.8268]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Min numerical gradient: 6.31E-07\n",
      "Min loss divided by 10: 1.20E-02\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8nGW99/HPL5N9bdImbZp030v3hrKplEUEVCoICh7c8MjjQfC44FFfPo948EHxiPt6EEEBhcOi57DKvigW2pTSQkv3LemWNEubZJJMZuY6f8xkmqZpE9rMzD3J9/16zYu57/u65/7N0MxvruW+LnPOISIiApCW7ABERMQ7lBRERCRGSUFERGKUFEREJEZJQUREYpQUREQkRklBRERilBRERCRGSUFERGLSkx3AOzVq1Cg3ceLEZIchIpJSVq1adcA5V9pfuZRLChMnTqS6ujrZYYiIpBQz2zmQcmo+EhGRGCUFERGJUVIQEZEYJQUREYlRUhARkRglBRERiVFSEBGRGCUFEZEU8JNnN/G3zfVxv46SgoiIxznn+PnzW3h1W0Pcr6WkICLicYFQmFDYkZsZ/0kolBRERDyuPRACIDfTF/drxS0pmNmdZlZnZm8d4/hMM1tuZp1mdmO84hARSXX+oZAUgN8DFx7neCPwBeC2OMYgIpLy/IEgADmp3HzknHuZyBf/sY7XOedWAl3xikFEZCiI1RQyUrumMGjM7Fozqzaz6vr6+A/JEhHxklhSyFJSAMA5d7tzrso5V1Va2u8aESIiQ8rhjuYUbj4SEZHB0RbtU0j1jmYRERkE3c1HOQnoU4hbXcTM7gOWAqPMrBa4CcgAcM79xszGANVAIRA2sy8Cs51zh+IVk4hIKkrkfQpxSwrOuav6Ob4PqIzX9UVEhorumkJelvoURESGPX8giBlkpcf/K1tJQUTE4/yBELkZPsws7tdSUhAR8Th/IJSQu5lBSUFExPPaA0HyEnDjGigpiIh4XlsglJDhqKCkICLiee2BUEKGo4KSgoiI5/kDwYRMcQFKCiIinudXTUFERLq1dykpiIhIVFunhqSKiEhUeyComoKIiIBzDn9XiDwlBRER6QyGcS4x6zODkoKIiKe1dSZugR1QUhAR8bTYAjtKCiIi0t4VXUtBzUciIuJP4KproKQgIuJp/mifgpqPRERENQURETnM39WdFNSnICIy7LUHNCRVRESi2jrVfCQiIlHdQ1LV0SwiIvgDQdLTjExfYr6ulRRERDzMHwiRk+nDzBJyPSUFEREP83cmboEdUFIQEfE0f1coYcNRQUlBRMTTErnADigpiIh4mj+g5iMREYlqCyRufWZQUhAR8bT2QJDcDNUUREQENR+JiEgP7YEQuVlDICmY2Z1mVmdmbx3juJnZz8xsi5mtNbNF8YpFRCRVtQWCQ2ZI6u+BC49z/CJgWvRxLfDrOMYiIpJywmFHR1eYnKHQp+CcexloPE6RZcDdLuJVYISZlccrHhGRVNPeldgZUiG5fQoVQE2P7droPhERoceqa1lDo/moP33N7uT6LGh2rZlVm1l1fX19nMMSEfEGf/cCO0Oh+WgAaoFxPbYrgT19FXTO3e6cq3LOVZWWliYkOBGRZEv0+syQ3KTwCPCJ6Cik04GDzrm9SYxHRMRTupNCohbYAYhbQ5WZ3QcsBUaZWS1wE5AB4Jz7DfAEcDGwBfADn45XLCIiqag9mhTyEtinELcrOeeu6ue4Az4fr+uLiKS6tmifwpAYkioiIienfZj1KYiIyHEc7mgeHkNSRUTkOGJDUofC3EciInJyYjUF9SmIiIg/ECLTl0a6L3Ff1UoKIiIe1R4IJvQeBVBSEBHxLH8gRJ6SgoiIQCQpqKYgIiJAZPRRIoejgpKCiIhnqaYgIiIx7V3qUxARkai2TjUfiYhIVLuaj0REpJu/K5TQyfBASUFExLP8gZCaj0REBIKhMIFgWDUFERGJNB1BYtdSACUFERFPak/C+sygpCAi4knd02bnqU9BRETaOqPrM6umICIi7epTEBGRbofXZ1ZSEBEZ9tq712dWn4KIiLR1qqYgIiJR3fcpqKNZRETUfCQiIod1dzTnZKimICIy7PkDIbIz0vClWUKvq6QgIuJByVifGZQUREQ8yR8IJbzpCJQUREQ8qT2Q+AV2QElBRMST2pQURESk20F/gBG5mQm/rpKCiIgHNbQFKMkbYknBzC40s41mtsXMvt7H8Qlm9pyZrTWzF82sMp7xiIikisahlhTMzAf8ErgImA1cZWazexW7DbjbOTcPuBn4XrziERFJFR1dIfyBkHeTgplNMbOs6POlZvYFMxvRz2lLgC3OuW3OuQBwP7CsV5nZwHPR5y/0cVxEZNhpaAsAMNKrSQF4GAiZ2VTgd8Ak4E/9nFMB1PTYro3u62kN8OHo80uBAjMbOcCYRESGpMbWSFLwbE0BCDvngkS+uH/inPsSUN7POX3dm+16bd8InG1mq4Gzgd1A8KgXMrvWzKrNrLq+vn6AIYuIpKaGtk4ARuZ7Nyl0mdlVwCeBx6L7Mvo5pxYY12O7EtjTs4Bzbo9z7jLn3ELgm9F9B3u/kHPududclXOuqrS0dIAhi4ikpsa27ppCVsKvPdCk8GngDOAW59x2M5sE3NvPOSuBaWY2ycwygSuBR3oWMLNRZtYdwzeAOwceuojI0BRLCl69T8E5t9459wXn3H1mVgwUOOdu7eecIHA98BTwNvCAc26dmd1sZpdEiy0FNprZJmA0cMuJvhERkaGisS1AeppRmJP4CfEGdEUzexG4JFr+DaDezF5yzn35eOc5554Anui171s9nj8EPPQOYxYRGdIa2wIU52Vilthps2HgzUdFzrlDwGXAXc65xcD58QtLRGT4amgLJGU4Kgw8KaSbWTnwEQ53NIuISBwk625mGHhSuJlI38BW59xKM5sMbI5fWCIiw1cyk8KA+hSccw8CD/bY3sbhm85ERGQQNbR2erv5yMwqzewvZlZnZvvN7GFNXiciMvi6QmEOdQSTco8CDLz56C4i9xiMJTJVxaPRfSIiMoiaYjeu9Xd/cHwMNCmUOufucs4Fo4/fA7q1WERkkDX6k3c3Mww8KRwws6vNzBd9XA00xDMwEZHhKJmT4cHAk8I1RIaj7gP2ApcTmfpCREQGUWza7CRMhgcDn+Zil3PuEudcqXOuzDn3ISI3somIyCA6PBmeh5PCMRx3igsREXnnGtoCmEFxEibDg5NLComflENEZIhrbOtkRE4GvrTkfMWeTFLovWCOiIicpO7J8JLluHc0m1kLfX/5G5ATl4hERIaxhtbkTYYH/SQF51xBogIREZFITWFyaV7Srn8yzUciIjLImvyBpN24BkoKIiKeEQ47mvxdSW0+UlIQEfGIg+1dhMIuafcogJKCiIhnJPtuZlBSEBHxjGTfzQxKCiIintHY1gkk725mUFIQEfEMNR+JiEhMsqfNBiUFERHPaPQHyM9KJyvdl7QYlBRERDyisS2Q1FoCKCmIiHiGkoKIiMQkezI8UFIQEfGMZE+bDUoKIiKe4JyjsU01BRERAVo7gwRCYfUpiIiIN6a4ACUFERFP8MLdzKCkICLiCU2xmkLyFtgBJQUREU+I1RSGcvORmV1oZhvNbIuZfb2P4+PN7AUzW21ma83s4njGIyLiVQdaIzOkDtk+BTPzAb8ELgJmA1eZ2exexf4v8IBzbiFwJfCreMUjIuJla2qaqSzOIS8rPalxxLOmsATY4pzb5pwLAPcDy3qVcUBh9HkRsCeO8YiIeFIo7Fi+tYGzpoxKdihxTQoVQE2P7drovp6+DVxtZrXAE8ANfb2QmV1rZtVmVl1fXx+PWEVEkmbdnoMc6ghy5tSRyQ4lrknB+tjnem1fBfzeOVcJXAzcY2ZHxeScu905V+WcqyotLY1DqCIiyfOPrQ0AnDFlaCeFWmBcj+1Kjm4e+gzwAIBzbjmQDSS//iQikkCvbDnA9NH5lBVkJzuUuCaFlcA0M5tkZplEOpIf6VVmF3AegJnNIpIUBrd9aOtWuO46KCyEtLTIf6+7LrJfRCTJAsEwK3c0cqYH+hMgjknBORcErgeeAt4mMsponZndbGaXRIt9Bfisma0B7gM+5Zzr3cR04p58EubNgzvugJYWcC7y3zvuiOx/8slBu5SIyIlYvauJjq4wZ3qg6QggrmOfnHNPEOlA7rnvWz2erwfOisvFt26Fyy8Hv//oY11dkcfll8PatTBlSlxCEBHpzytbG0gzOG2yN5LC0L2j+Yc/jHzxH09XF/z4x4mJR0SkD8u3HmBuRRFFORnJDgUYyknh3nsHlhTuuScx8YiI9NLWGWT1rmbOnOqN/gQYykmhtXVwy4mIDLIVOxoJhp0nblrrNnSTQn7+4JYTERlky7c2kOlLY/GE4mSHEjN0k8LVV0NGP210GRnw8Y8nJh7pU7M/QFtnMNlhiCTFK1sOsGjCCHIyfckOJWboJoWvfKXfpOAyMuBLX0pQQIOvrTPItvpW6lo66OgKMZijeQdDOOyoafTz1u6DhMNHxtYeCHHbUxtZcstznP7d5/jeE2+z72BHkiIVSbymtgDr9x7yzP0J3ZI7HV88TZkCDz0UGXbaPQQ1KuRLp9N8vHDTzwm15rDv5a3sP9RJZXEOH1pQQXGSp67tT1cozL2v7uSnz22m2X/4fWX4jLKCbMaX5DKuJIdJo/L52JLxFOUmZlSDPxDk+Q11PP92HRv2tbDtQCsdXWEARhdm8f65Y/nA/HLqDnXwncfeZndzO8sWjCUUdvz2b9u485XtXDK/gmvfM5kZYwoSErNIsqzbcwjnoMpDTUcA5rVfl/2pqqpy1dXVAz9h69bIsNN77ol0KufnE776am4YfTaP+/NixbIz0ujoCpPpS+N9c8bw0apxzBtXREFWOmZ9TeOUGKGwoysUJuxcbCbFW5/cwLYDbZw1dSSXLqykPRCkpTPIofYg+w62s6vRT01TO/UtnYwvyeXXVy/ilLFFA7qePxDkjZpmNu5rYW5FEQvHF+NLO/b7r23ys2J7I0+v28+Lm+ro6AozMi+TuZVFTCnNZ2pZPlnpafz1rX28uKmeQDCSJGaOKeDmZXNYMqkEgJpGP7/7+3b+a2UN7V0hls4o5dr3TOaMySOT+vmLxMvDq2r5yoNreOHGpUwaldf/CSfJzFY556r6LTfkk8IxtAdCrN7VxKiCLMYUZVOQlc6GfS3818oa/rJ6NwfbI7/AszPSKCvIprI4h49UjeP988rJ8B1udQuFHa9tbyA9LY2qCcWk9fgCdc6xamcTT6/fT7M/QGtnkNbOEAZMKc1nxph8po0uoLI4h6KcDLLSI+2K2w+08dzb+3n27f2s3NFEqFfTy5TSPL75/lmcM6PsuF+Yq3Y2cd0fV9Hs7+KWS+dy+eLKI2Lbd6iDrXVtbK1vZUtdK2/UNLN+76Ejrlecm8HSGWUsmVRCMOzwdwbxB0LsaGhj5fZG9kSbfMoKsrhozhgumlvOqRNL+kwkhzq6eGbdfgCWLRhLuu/o1sumtgD3vLqTP/xjBw1tAeZXFvG1C2d6asieyGD49Ytb+f5fN7Du39+XkDUUlBROQkdXiJc21bOrwc/+Qx3UtXTy1p6DbKtvo2JEDp8+ayKLJhTzxNq9PLJmD3UtkRWTxhRm84F55bxvzhjW1DRz/8oattS1kulLoyQvk7wsH/nZGQRDYbbWH25a6ZaVnkZupo+maJPQjNEFnD2jlKKcDHxpRnqaUVqQxcVzj0xMx3OgtZMb/rSa5dsaOGdGKWEX+XVf29ROZ/Dw9Quy0jmlopCqCSUsnlDM9DEFrN7VxPNv1/HCxrpYTN3GFGZTNbGYUyeWcOrEEmaOKTgiIZ6sjq4QD79ey69e2Mru5nbOm1nGNy6exdQyjRaToeHmR9fzXyt3se7mCxNyPSWFQRYOO17cVMdvXtrGiu2NQKQN/5wZZXxoYQXBsOORN/bw0qY6ukKRz3TR+BFceep43j+v/KhfAqGwo7bJz6b9rew72M6hjiAH27to6ehi+ugCzp81mnEluYMSezAU5kfPbOKhVbWUFWYxrjiXyuIcxpfkMqUsn6ml+ZQWZB2z1hEKO/Y0t5Od4SM300dOhm9QE8DxdHSFuOuVHfzqhS34u0JceMoYJo3Ko6I4h4oROWSlp9HeFaKjK0RXyHHa5BJPzDQp0p/r//Q6b+0+yItfPSch11NSiKM3aprZVt/KuTPLGJF7ZKf0QX8Xf9tSz7SyAnWWDqKG1k5+9txmnn27jn2HOo5qUuuW6UvjA/PLueasScypGFg/ikgyfOQ/l4ODBz53RkKup6QgQ1YwFGZ/Sye7m9oJhsJkZURqL8FwmIdX1fLgqlr8gRCnTizmqiXjuWhOuafGgYsAnHPbi8weW8gvP7YoIdcbaFIYukNSZchK96VRMSLSfNTbvMoRfPmCGTxYXcM9r+7kyw+s4Vv/s44Pzi/nvJmjyc3ykZXuIys9jall+WRnKFlIctS3dFJWkJXsMI6ipCBDTlFOBv/87sl85l2TWLmjiQeqa/jv1Xu4b0XNEeUmjMzlzk+dypRSdV5LYvkDQVo7g57s/1JSkCHLzFgyqYQlk0r49iWnsGl/C51dYQKhMI1tnfz/x97m0l++wm8+vthzd5XK0FZ3KDJisVQ1BZHkyM9KZ9H4I+8crZpQwmf+sJJP/G4Ft1w6h4+eOj5J0clw0z2M3YvNR0N37iORfowryeWhfzmTM6eO4msPv8ln765mTU1zssOSYaC+OykUKimIeEphdgZ3frKKr7x3Oiu2N7Lsl6/w8d+9xmvbGpIdmgxhdS3dMwF4r09BSUGGvXRfGjecN41Xvn4uX79oJm/vbeGjt7/KZ++uZldDH2t8i5ykupZO0tOMER5ZgrMnJQWRqPysdD539hT+/rVz+NqFM/nHlgOc/+OXuO2pjfgDWvNBBk/doU5KC7ISNjPAO6GOZpFesjN8/MvSKVy2qILvP7mBX7ywhd/9fTvjSnKojE4RsnhCMe87ZcwR9zk451i5o4l/bD3ARXPKdUe7HFN9qzfvUQAlBZFjGl2YzY8+uoB/On0Cj6/dG5tIcOX2Ru5evpPC7HQuXVjBsoUVvFl7kD++tpNN+yNrfv/k2c1cMHs01587lXmVI5L8TsRr6g51UFk8OHObDTYlBZF+LJ5QfMQauuGw49XtDdy/oob7VtTwh+U7AZhXWcR/fHge754+ivtX1HDXK9t5ev1+zptZxncvm8voQu91Kkpy1Ld0sshji+t0U1IQeYfS0owzp4zizCmjaGoL8NyGOqaPzj+iRvCl907nn989iXte3cnPn9vChT95mduumM95s0YnMXLxgq5QmIa2AKX5aj4SGXKK8zKPWLyop4LsDK5bOpULZo/hC/et5jN/qOZTZ07kGxfPjC2o1G13czt3/n07T63bx5yxRZw3q4xzZ5Yx0qNfHHLiGloDgDfvUQAlBZG4m1qWz18+fya3PrmBu17ZwZ9fr2X+uBEsHF/MzDEFPLN+P4+u2YMD3j1tFG/UNPPXdfswg3dPK+UXH1tIYbb3hi7KifHyPQqgpCCSEFnpPm764CmcP2s0j63dyxs1zfzi+c2EHeRm+vjkmRO55l2TqBiRg3OOdXsO8fS6ffzqxa1cd+/r3PmpU8lM1wjyoaB73iONPhIRzpo6irOi6023dQbZsK+FqaX5FOUergmYGXMqiphTUcS4kly++tBavvHnN7ntinnHXZNbUkP3vEdenAwPlBREkiYvK/2IUU19uaJqHLub2/nJs5sZV5LDF8+fnqDoJF665z0a5dH+IiUFEY/71/OmUdsUSQxjCrO5colmc01ldS0dlORlerY5UElBxOPMjO9dNpe6lk6+/uc3qW/p5Ppzp6opKUXVeXTFtW7eTFUicoQMXxq//cRiLl1YwQ+f2cRXHlxDZzCU7LDkBNS1dHq2PwFUUxBJGVnpPn70kflMHJnHj5/dRG1TO7+4aiFlulM6pdQf6mBK6chkh3FMca0pmNmFZrbRzLaY2df7OP5jM3sj+thkZlrhROQ4zIx/PX8aP71yAW/saubMW5/n/9xTzQsb6giFXbLDk34456KT4Xk3kcetpmBmPuCXwHuBWmClmT3inFvfXcY596Ue5W8AFsYrHpGhZNmCCuZXjuC+Fbt4aFUtT63bT3lRNpcsGMuy+RXMKi9Qn4MHNfu76Ao5T/cpxLP5aAmwxTm3DcDM7geWAeuPUf4q4KY4xiMypEwclcc3Lp7FVy6YwXNv7+eB6hp+97ft/OdL25hals8H5pWzdEYZcyuK8Hlw3v7hqM7Dy3B2i2dSqABqemzXAqf1VdDMJgCTgOePcfxa4FqA8eM1HE+kp8z0NC6aW85Fc8tpbAvwxJt7eWTNHn7y7GZ+8uxmRuRmcNbUUVw8p5yL5ozx5MIuw0X3FBdenQwP4psU+vqXd6xGzyuBh5xzfQ6ncM7dDtwOUFVVpYZTkWMoycvk6tMncPXpE2ho7eTvWw7wt80HeHlTPY+v3cvMMQV89X0zOHdmmZqXkqA+VlMYhn0KRGoG43psVwJ7jlH2SuDzcYxFZNgZmZ/FsgUVLFtQQTjseHTtHn70zCY+84dqFo4fwXVLp3LOjFLSfRqZniix5qNh2qewEphmZpOA3US++D/Wu5CZzQCKgeVxjEVkWEtLM5YtqODiueU8WF3Lz5/fzGfvrqa0IIvLFlVwxeJxTB6Vp6alOKs71Elepo+8LO/eDRC3yJxzQTO7HngK8AF3OufWmdnNQLVz7pFo0auA+51zahYSibMMXxofO208V1RV8sKGOh6oruWOaOe0L80oyctkZF4mY4qymTO2iHmVRcwfN0Krxg2SupYOTzcdQZxvXnPOPQE80Wvft3ptfzueMYjI0TJ8aVxwyhguOGUMdYc6eGr9fvYdbKehNcCB1gC1TX7+tvlA7N6HyaV5fOL0CVxeNY58D//K9br6lk5PdzKD7mgWGfbKCrP5+OkTjtrfHgixfu8h1tQ088iaPXz70fXc9vQmrqiq5J9OG8/UsoIkRJva6ls6mTW2MNlhHJeSgoj0KSfTx+IJxSyeUMw175rE6l1N/P4fO7hn+U7uemUHM8cUcMmCsXxw3ljGleQmO9yka+sMsv1AG22dQWaOKTxijYzWziDLtzaw92AHZ88oTWKU/bNUa8qvqqpy1dXVyQ5DZNiqa+ng8bWReyFW74rMTPOe6aV8+qyJnD2tdNh0Vje2BXh0zR6eXr+PLXWt7I+uqNatYkQOp4wtpKUjSPXORrpCjtxMHz+9ciHvnT064fGa2SrnXFW/5ZQURORE1TT6+cvq3dz76k7qWjqZXJrH1adN4IwpI5k+umDI3UkdCIZ5fkMdf369lhc21tEVcswYXcCciiIml+YxeVQeOZk+3t7bwro9B1m/5xBZGT7eM30UZ08vpWpCSdLWUVBSEJGECQTDPPHmXu56ZTtrag8CkJfpY25lEUtnlPHJMyaSk+lLcpQnbuO+Fh6sruEvq3fT0BZgVH4Wly4cy2WLKplV7u0+gm5KCiKScM45djX6Wb2rmdW7mli1q4m3dh9iTGE2N75vBpctrEip5qVN+1u46X/WsXxbA+lpxvmzRnNFVSVnT0+9m/6UFETEE1Zsb+SWx9ezpvYgs8sLj2hP96UZ75o2ioXjRnhq2o32QIifPb+Z3768jbysdD5/zhQ+vKiSkR4fTno8Sgoi4hnd02zc9vRGahrbjzo+rSyfj1SN49JFFUld0D4Udjz+5l6+/+QGdje3c/niSr5x0cyUTgbdlBRExHOcc0fUCFo7gzy2Zg8PVNfw+q5mzGDmmEJOnVhM1cQSzpg8MiFLV3aFwvzPG3v41Qtb2HagjRmjC/jOh+awZFJJ3K+dKEoKIpJSNu9v4fE391K9o4nXdzXhD4TI9KVx1ZJxfP6cqSc8PUQwFObfHlrLGzXNlI/IZkxhDqMLswgEwzT6AzT7u9i4r4Xdze3MKi/khnOncuEpQ2+KcSUFEUlZwVCY9XsP8afXdvHgqlrS04yPnz6B686ZSkle5jt6rVuf3MBvXtrK0hmlHGrvYt/BDva3dJKVnkZxbibFeRmMLsjmqiXjOW/W0J1SXElBRIaEnQ1t/Oy5LfxldS0leVn89MoFnDV11IDO/etb+/jcvav42Gnj+e6lc2P7ezdjDQcDTQqpNaZKRIadCSPz+OFH5vP4F97NiNwMrv7da/zo6Y0EQ+HjnretvpUbH1zD/Moibvrg7COODbeE8E4oKYhISphVXsgj15/F5Ysq+dnzW/jYHa/x2No9bNrfQiB4OEE452j2B/jcvavI8Bm/unoxWempe+NcomlCPBFJGbmZ6fzgivmcMWUk/++/3+L6P60GID3NKB+RTXsgxMH2LrpCjjSDu685jYoROUmOOrUoKYhIyrlsUSUXzy1na30rm/e3sml/ZPRQbmY6RTkZFOVksGDcCM6YMjLZoaYcJQURSUnZGT5OGVvEKWOLkh3KkKI+BRERiVFSEBGRGCUFERGJUVIQEZEYJQUREYlRUhARkRglBRERiVFSEBGRmJSbJdXMDgKb+zhUBBw8zr7ex7u3+yozCjhwAuH1FcNAjg+X2Pt7rtgHflyxeyP242339bznvkTHPs051/+dfs65lHoAtw90f899vY93b/dVBqgezNgU+8CeK3bFnmqxH2/7GPH23Jf0z72vRyo2Hz36DvY/epzjjw6gzDvV3/nDPfaBPD8Riv3ofYr9+AYr9uNt9/X8ZOMeyGu8k8/9KCnXfJQIZlbtBrAYhRcp9uRQ7Mmh2AdfKtYUEuH2ZAdwEhR7cij25FDsg0w1BRERiVFNQUREYoZ8UjCzO82szszeOoFzF5vZm2a2xcx+Zj0WdjWzG8xso5mtM7P/GNyoY9cY9NjN7NtmttvM3og+Lh78yOP3uUeP32hmzswGtnr7O79+PD7375jZ2uhn/rSZjU2RuH9gZhuisf/FzEYMdtzR68Qj9iuif59hMxv0tvuTifkYr/dJM9scfXyyx/7j/j0MuhMZEpVKD+A9wCLgrRM4dwVwBmDAk8BF0f3nAM8CWdHtshSK/dvAjan4uUePjQOeAnaY3SxQAAAGW0lEQVQCo1IldqCwR5kvAL9JkbgvANKjz78PfD+FPvNZwAzgRaDKKzFH45nYa18JsC363+Lo8+Ljvb94PYZ8TcE59zLQ2HOfmU0xs7+a2Soz+5uZzex9npmVE/lDXu4i/2fuBj4UPfwvwK3Ouc7oNepSKPaEiGPsPwb+DYhbZ1g8YnfOHepRNC8e8ccp7qedc8Fo0VeBysGOO46xv+2c2xiPeE8m5mN4H/CMc67ROdcEPANcmIy/5SGfFI7hduAG59xi4EbgV32UqQBqe2zXRvcBTAfebWavmdlLZnZqXKM90snGDnB9tDngTjMrjl+oRzmp2M3sEmC3c25NvAPtw0l/7mZ2i5nVAP8EfCuOsfY0GP9eul1D5Jdqogxm7IkykJj7UgHU9Njufh8Jf3/Dbo1mM8sHzgQe7NE0l9VX0T72df+6SydSxTsdOBV4wMwmRzN53AxS7L8GvhPd/g7wQyJ/7HF1srGbWS7wTSLNGQk1SJ87zrlvAt80s28A1wM3DXKoRwYzSHFHX+ubQBD442DGeCyDGXuiHC9mM/s08K/RfVOBJ8wsAGx3zl3Ksd9Hwt/fsEsKRGpHzc65BT13mpkPWBXdfITIl2fPqnIlsCf6vBb4czQJrDCzMJF5TOrjGTiDELtzbn+P834LPBbPgHs42dinAJOANdE/uErgdTNb4pzb5/HYe/sT8DhxTgoMUtzRTs8PAOfF+4dPD4P9mSdCnzEDOOfuAu4CMLMXgU8553b0KFILLO2xXUmk76GWRL+/eHZYeOUBTKRHZxDwD+CK6HMD5h/jvJVEagPdHTwXR/d/Drg5+nw6kWqfpUjs5T3KfAm4P1U+915ldhCnjuY4fe7TepS5AXgoReK+EFgPlMbrs473vxfi1NF8ojFz7I7m7URaIIqjz0sG8v4G/T3F+390sh/AfcBeoItI1v0MkV+cfwXWRP/Bf+sY51YBbwFbgV9w+Ga/TODe6LHXgXNTKPZ7gDeBtUR+aZWnSuy9yuwgfqOP4vG5Pxzdv5bIHDQVKRL3FiI/et6IPgZ91FQcY780+lqdwH7gKS/ETB9JIbr/mujnvQX49Dv5exjMh+5oFhGRmOE6+khERPqgpCAiIjFKCiIiEqOkICIiMUoKIiISo6QgQ4KZtSb4eneY2exBeq2QRWZPfcvMHu1vJlIzG2Fm1w3GtUV605BUGRLMrNU5lz+Ir5fuDk8EF1c9YzezPwCbnHO3HKf8ROAx59ycRMQnw4tqCjJkmVmpmT1sZiujj7Oi+5eY2T/MbHX0vzOi+z9lZg+a2aPA02a21MxeNLOHLLKmwB+757KP7q+KPm+NTna3xsxeNbPR0f1TotsrzezmAdZmlnN4AsB8M3vOzF63yHz6y6JlbgWmRGsXP4iW/Wr0OmvN7N8H8WOUYUZJQYaynwI/ds6dCnwYuCO6fwPwHufcQiKzlX63xzlnAJ90zp0b3V4IfBGYDUwGzurjOnnAq865+cDLwGd7XP+n0ev3O19NdF6f84jcaQ7QAVzqnFtEZA2PH0aT0teBrc65Bc65r5rZBcA0YAmwAFhsZu/p73oifRmOE+LJ8HE+MLvHjJWFZlYAFAF/MLNpRGaczOhxzjPOuZ5z5K9wztUCmNkbROa6+Xuv6wQ4PLHgKuC90edncHju+z8Btx0jzpwer72KyFz6EJnr5rvRL/gwkRrE6D7OvyD6WB3dzieSJF4+xvVEjklJQYayNOAM51x7z51m9nPgBefcpdH2+Rd7HG7r9RqdPZ6H6Ptvpssd7pw7VpnjaXfOLTCzIiLJ5fPAz4isu1AKLHbOdZnZDiC7j/MN+J5z7j/f4XVFjqLmIxnKniaybgEAZtY9pXERsDv6/FNxvP6rRJqtAK7sr7Bz7iCRpTpvNLMMInHWRRPCOcCEaNEWoKDHqU8B10Tn88fMKsysbJDegwwzSgoyVOSaWW2Px5eJfMFWRTtf1xOZ8hzgP4DvmdkrgC+OMX0R+LKZrQDKgYP9neCcW01khs0riSxoU2Vm1URqDRuiZRqAV6JDWH/gnHuaSPPUcjN7E3iII5OGyIBpSKpInERXi2t3zjkzuxK4yjm3rL/zRJJJfQoi8bMY+EV0xFAzCVj2VORkqaYgIiIx6lMQEZEYJQUREYlRUhARkRglBRERiVFSEBGRGCUFERGJ+V/21DhqIgZOxwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.lr_find()\n",
    "learn.recorder.plot(suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=2e-2\n",
    "lr *= bs/48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.523350</td>\n",
       "      <td>0.483231</td>\n",
       "      <td>0.760841</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1, lr, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('first')\n",
    "learn.load('first')\n",
    "learn.freeze_to(-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.517741</td>\n",
       "      <td>0.466405</td>\n",
       "      <td>0.787122</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.462745</td>\n",
       "      <td>0.440266</td>\n",
       "      <td>0.785808</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.411852</td>\n",
       "      <td>0.449109</td>\n",
       "      <td>0.793693</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(3, slice(lr/(2.6**4),lr), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.411695</td>\n",
       "      <td>0.435463</td>\n",
       "      <td>0.814717</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.save('second')\n",
    "learn.load('second');\n",
    "learn.freeze_to(-3)\n",
    "learn.fit_one_cycle(1, slice(lr/2/(2.6**4),lr/2), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.328826</td>\n",
       "      <td>0.498895</td>\n",
       "      <td>0.780552</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.271888</td>\n",
       "      <td>0.493785</td>\n",
       "      <td>0.806833</td>\n",
       "      <td>00:07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.save('third')\n",
    "learn.load('third');\n",
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(2, slice(lr/10/(2.6**4),lr/10), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pd.DataFrame()\n",
    "for i,row in test.iterrows():\n",
    "    result.loc[i,\"target\"] = learn.predict(row[\"text\"])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "## get test set predictions and ids\n",
    "# preds, _ = learn.get_preds(ds_type=DatasetType.Test,  ordered=True)\n",
    "# preds = preds.argmax(dim=-1)\n",
    "\n",
    "result[\"id\"] = test.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  target  id\n",
       "0      1   0\n",
       "1      1   2\n",
       "2      1   3\n",
       "3      1   9\n",
       "4      1  11"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[[\"id\",\"target\"]].to_csv(\"submission.csv\",index=False)\n",
    "result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}