asrs_new-factors-clean.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multilable text classification with fastai v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ACN</th>\n",
       "      <th>Date</th>\n",
       "      <th>Local Time Of Day</th>\n",
       "      <th>Locale Reference</th>\n",
       "      <th>State Reference</th>\n",
       "      <th>Relative Position.Angle.Radial</th>\n",
       "      <th>Relative Position.Distance.Nautical Miles</th>\n",
       "      <th>Altitude.AGL.Single Value</th>\n",
       "      <th>Altitude.MSL.Single Value</th>\n",
       "      <th>Flight Conditions</th>\n",
       "      <th>...</th>\n",
       "      <th>When Detected</th>\n",
       "      <th>Result</th>\n",
       "      <th>Contributing Factors / Situations</th>\n",
       "      <th>Primary Problem</th>\n",
       "      <th>Narrative</th>\n",
       "      <th>Callback</th>\n",
       "      <th>Narrative.1</th>\n",
       "      <th>Callback.1</th>\n",
       "      <th>Synopsis</th>\n",
       "      <th>Unnamed: 96</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>925545</td>\n",
       "      <td>201101</td>\n",
       "      <td>0001-0600</td>\n",
       "      <td>ZNY.ARTCC</td>\n",
       "      <td>NY</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19000.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Air Traffic Control Issued New Clearance</td>\n",
       "      <td>Aircraft; Equipment / Tooling; Human Factors</td>\n",
       "      <td>Human Factors</td>\n",
       "      <td>[I was] working air carrier 1 at FL190. Sector...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Air carrier 1 [was] being worked by Sector 56 ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ZNY controllers and flight crew personnel desc...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>925560</td>\n",
       "      <td>201101</td>\n",
       "      <td>1201-1800</td>\n",
       "      <td>ZZZ.Tower</td>\n",
       "      <td>US</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Taxi</td>\n",
       "      <td>Air Traffic Control Issued New Clearance; Air ...</td>\n",
       "      <td>Human Factors</td>\n",
       "      <td>Human Factors</td>\n",
       "      <td>Aircraft X; a CRJ2; landed Runway XXL and was ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Disabled aircraft causes airport authority to ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>925563</td>\n",
       "      <td>201101</td>\n",
       "      <td>1201-1800</td>\n",
       "      <td>ZOB.ARTCC</td>\n",
       "      <td>OH</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>In-flight</td>\n",
       "      <td>Air Traffic Control Issued New Clearance; Air ...</td>\n",
       "      <td>Human Factors; Procedure; Staffing</td>\n",
       "      <td>Human Factors</td>\n",
       "      <td>I was working combined sectors with no D-side....</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ZOB Controller reported that when an aircraft ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>925810</td>\n",
       "      <td>201101</td>\n",
       "      <td>0001-0600</td>\n",
       "      <td>ZZZ.Airport</td>\n",
       "      <td>US</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Aircraft In Service At Gate</td>\n",
       "      <td>Flight Crew Returned To Gate; General Maintena...</td>\n",
       "      <td>Procedure; Human Factors; Aircraft; Staffing; ...</td>\n",
       "      <td>Aircraft</td>\n",
       "      <td>[We] arrived to a cold airplane with ambient t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A CRJ flight crew suffered a hot start on a fi...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>925811</td>\n",
       "      <td>201101</td>\n",
       "      <td>0601-1200</td>\n",
       "      <td>ZZZ.Airport</td>\n",
       "      <td>US</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32500.0</td>\n",
       "      <td>IMC</td>\n",
       "      <td>...</td>\n",
       "      <td>In-flight</td>\n",
       "      <td>Air Traffic Control Provided Assistance; Aircr...</td>\n",
       "      <td>Aircraft; Weather</td>\n",
       "      <td>Aircraft</td>\n",
       "      <td>During the climbout we entered the ragged over...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>B717 Captain experienced simultaneous overspee...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 97 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ACN    Date Local Time Of Day Locale Reference State Reference  \\\n",
       "0  925545  201101         0001-0600        ZNY.ARTCC              NY   \n",
       "1  925560  201101         1201-1800        ZZZ.Tower              US   \n",
       "2  925563  201101         1201-1800        ZOB.ARTCC              OH   \n",
       "3  925810  201101         0001-0600      ZZZ.Airport              US   \n",
       "4  925811  201101         0601-1200      ZZZ.Airport              US   \n",
       "\n",
       "   Relative Position.Angle.Radial  Relative Position.Distance.Nautical Miles  \\\n",
       "0                             NaN                                        NaN   \n",
       "1                             NaN                                        NaN   \n",
       "2                             NaN                                        NaN   \n",
       "3                             NaN                                        NaN   \n",
       "4                             NaN                                        NaN   \n",
       "\n",
       "   Altitude.AGL.Single Value  Altitude.MSL.Single Value Flight Conditions  \\\n",
       "0                        NaN                    19000.0               NaN   \n",
       "1                        0.0                        NaN               NaN   \n",
       "2                        NaN                    15000.0               NaN   \n",
       "3                        0.0                        NaN               NaN   \n",
       "4                        NaN                    32500.0               IMC   \n",
       "\n",
       "      ...                     When Detected  \\\n",
       "0     ...                               NaN   \n",
       "1     ...                              Taxi   \n",
       "2     ...                         In-flight   \n",
       "3     ...       Aircraft In Service At Gate   \n",
       "4     ...                         In-flight   \n",
       "\n",
       "                                              Result  \\\n",
       "0           Air Traffic Control Issued New Clearance   \n",
       "1  Air Traffic Control Issued New Clearance; Air ...   \n",
       "2  Air Traffic Control Issued New Clearance; Air ...   \n",
       "3  Flight Crew Returned To Gate; General Maintena...   \n",
       "4  Air Traffic Control Provided Assistance; Aircr...   \n",
       "\n",
       "                   Contributing Factors / Situations Primary Problem  \\\n",
       "0       Aircraft; Equipment / Tooling; Human Factors   Human Factors   \n",
       "1                                      Human Factors   Human Factors   \n",
       "2                 Human Factors; Procedure; Staffing   Human Factors   \n",
       "3  Procedure; Human Factors; Aircraft; Staffing; ...        Aircraft   \n",
       "4                                  Aircraft; Weather        Aircraft   \n",
       "\n",
       "                                           Narrative Callback  \\\n",
       "0  [I was] working air carrier 1 at FL190. Sector...      NaN   \n",
       "1  Aircraft X; a CRJ2; landed Runway XXL and was ...      NaN   \n",
       "2  I was working combined sectors with no D-side....      NaN   \n",
       "3  [We] arrived to a cold airplane with ambient t...      NaN   \n",
       "4  During the climbout we entered the ragged over...      NaN   \n",
       "\n",
       "                                         Narrative.1 Callback.1  \\\n",
       "0  Air carrier 1 [was] being worked by Sector 56 ...        NaN   \n",
       "1                                                NaN        NaN   \n",
       "2                                                NaN        NaN   \n",
       "3                                                NaN        NaN   \n",
       "4                                                NaN        NaN   \n",
       "\n",
       "                                            Synopsis  Unnamed: 96  \n",
       "0  ZNY controllers and flight crew personnel desc...          NaN  \n",
       "1  Disabled aircraft causes airport authority to ...          NaN  \n",
       "2  ZOB Controller reported that when an aircraft ...          NaN  \n",
       "3  A CRJ flight crew suffered a hot start on a fi...          NaN  \n",
       "4  B717 Captain experienced simultaneous overspee...          NaN  \n",
       "\n",
       "[5 rows x 97 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn = pd.read_csv(path/'train_Robinson.csv', skiprows=1,low_memory=False, dtype={'Contributing Factors / Situations':str})\n",
    "df_val = pd.read_csv(path/'valid.csv', skiprows=1,low_memory=False, dtype={'Contributing Factors / Situations':str})\n",
    "df_trn.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((4500, 97), (2948, 97))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trn.shape, df_val.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_trn.dropna(axis=0, subset=['Contributing Factors / Situations'], inplace=True)\n",
    "df_val.dropna(axis=0, subset=['Contributing Factors / Situations'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4485, 2935)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_trn), len(df_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Replace label delimiter '; ' by single char '|' to allow compatibility with fastai 1.0.28\n",
    "\n",
    "df_trn['Contributing Factors / Situations'] = df_trn['Contributing Factors / Situations'].str.replace('; ', '|').astype(str)\n",
    "\n",
    "df_val['Contributing Factors / Situations'] = df_val['Contributing Factors / Situations'].str.replace('; ', '|').astype(str)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Language model\n",
    "\n",
    "Fine tune the language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([df_trn, df_val], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = (TextList.from_df(df, path, cols='Narrative')\n",
    "                .random_split_by_pct(0.1)\n",
    "                .label_for_lm()\n",
    "                .databunch(bs=bs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm.save('tmp_lm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm = language_model_learner(data_lm, pretrained_model=URLs.WT103, drop_mult=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xl8VOXZ//HPlT1kY0mCbGFXQBGEACpCxa3W3brX+oii1KUu7c/HLj61WqvWLrYurZaqKFrrjkvdsNYNQSHsIChb2CEJgSyQPffvj5nEGEMSIGfOTPJ9v155ZebMmTnXZPvmPuc+1zHnHCIiIgBRfhcgIiLhQ6EgIiL1FAoiIlJPoSAiIvUUCiIiUk+hICIi9RQKIiJST6EgIiL1FAoiIlIvxu8C9ld6errr16+f32WIiESUBQsWFDjnMlpaL+JCoV+/fuTk5PhdhohIRDGzDa1ZT7uPRESknkJBRETqKRRERKSeQkFEROopFEREpJ5CQURE6ikURESknkJBRCQC/OU/X/HJ6nzPt6NQEBEJc845Hnx/NZ+vK/R8WwoFEZEwt6eyhloHqYneN6FQKIiIhLmS8ioAUhJiPd+WQkFEJMwVl1UDkKpQEBGRr0cK2n0kItLhFQdDITVRIwURkQ6vpDyw+0gjBRERobgsOFLQMQURESnWSEFEROoUl1cRFxNFQmy059tSKIiIhLmS8mpSQzBKAIWCiEjYKy6rCsnxBFAoiIiEvZLy6pAcTwCFgohI2CsurwrJOQqgUBARCXsaKYiISD0dUxARkXoaKYiICABVNbWUVdVopCAiIqHtewQKBRGRsFbf9yhEs488jR4zywVKgBqg2jmX3ejx44HXgPXBRa84537jZU0iIpHk65FCOwiFoEnOuYJmHv/EOXdGCOoQEYk49ddS0O4jEREJ5fWZwftQcMAsM1tgZlP3sc4xZrbEzN42s8M9rkdEJKLUX585MTQjBa+3Mt45t9XMMoH3zGyVc+7jBo8vBPo650rN7DTgVWBw4xcJBspUgKysLI9LFhEJH8XtaaTgnNsa/JwHzATGNnq82DlXGrz9FhBrZulNvM4051y2cy47IyPDy5JFRMJKcXk1ZpASH+HHFMwsycxS6m4DpwDLG61ziJlZ8PbYYD07vapJRCTSlJRXkRwXQ1SUhWR7XkZPd2Bm8G9+DPCsc+4dM7sGwDn3KHA+cK2ZVQNlwMXOOedhTSIiEaW4rDpk5yiAh6HgnFsHjGhi+aMNbj8MPOxVDSIika64vCpkZzODpqSKiIS1kvLQdUgFhYKISFgL7D7SSEFERICSiqqQTUcFhYKISFgrLqsOWYsLUCiIiIQt5xwl5RopiIgIsKeyhloXuhYXoFAQEQlboW6GBwoFEZGwVd8MT6EgIiJfjxS0+0hEpMOrv8BOCNtcKBRERMLU15fi1EhBRKTDKy6ruxSnRgoiIh1esUYKIiJSp7i8iriYKBJio0O2TYWCiEiYKikPbYsLUCiIiISt4rLQts0GhYKISNgqKa8O6fEEUCiIiISt4vKqkJ6jAAoFEZGwpZGCiIjU0zEFERGpp5GCiIgAUFVTS1lVjUYKIiLiT98jUCiIiISl+r5Hmn0kIiJfjxQUCiIiHV79tRS0+0hERPy4PjMoFEREwlL99ZkTNVIQEenwijVSEBGROsXl1ZhBSrxGCiIiHV5xWRXJcTFERVlIt6tQEBEJQyXl1SE/RwEUCiIiYam4vCrkZzODQkFEJCyVlIe+QyooFEREwlJxWeg7pIJCQUQkLBXuqaRLUlzIt6tQEBEJM845du6pICMlPuTb9jQUzCzXzJaZ2WIzy2lmvTFmVmNm53tZj4hIJCgqq6KqxpGeHPpQCMUOq0nOuYJ9PWhm0cB9wLshqEVEJOwVlFYAkJ7cMXcf3QC8DOT5XYiISDjIKwmEQoYPIwWvQ8EBs8xsgZlNbfygmfUCzgUe9bgOEZGIUVBaCUC6D8cUvN59NN45t9XMMoH3zGyVc+7jBo//BfiZc67GbN+ncgcDZSpAVlaWpwWLiPitoKRu91E7Gyk457YGP+cBM4GxjVbJBp4zs1zgfOBvZnZOE68zzTmX7ZzLzsjI8LJkERHfFZRWEBNldPahzYVnIwUzSwKinHMlwdunAL9puI5zrn+D9Z8E/u2ce9WrmkREIkFBaQXdkuNC3gwPvN191B2YGdwtFAM865x7x8yuAXDO6TiCiEgTCkorfdl1BB6GgnNuHTCiieVNhoFzbrJXtYiIRJKC0grfQiEcpqSKiEgD+SUKBRERIdjiorSS9JTQn7gGCgURkbBSXFZNZU2tLyeugUJBRCSs5AdbXPjRDA8UCiIiYeXrvkcKBRGRDk+hICIi9b5ucaEDzSIiHV5+aQXRUUaXTgoFEZEOr6Ckkm5J/rS4AIWCiEhY8fNsZlAoiIiElYLSCl+uo1AnFJfjbFc2Fe7lufkbeX3JVqLN6J6awCFpCfTuksgPxvWlV+dEv0sUkQhWUFrJwMxk37avUGhCba3j3RXbWZNXSnxsFPEx0URFGe+v3MFHX+VjwMRDM0iOj2FHcTkLN+7izaXbeHz2eq47fhBTJw4gITba77chIhHGOUd+aYVvZzODQuFb5q0v5O43v2DJ5qJvPZaZEs8NkwZx0disb40Ituwu4543V3L/e1/x4oJN3H7G4Zw8rHuoyhaRdqCkoprK6lpfjykoFII27NzDPW+t5N0VOzgkNYE/XjCCM0f0oKrGUVFVQ0V1LZkp8cREN30YplfnRP566SguXVPAHW+s4OoZOZwyrDt3nn04PdK0S0lEWpZf4m+LC2hlKJjZQGCzc67CzI4HjgRmOOd2e1lcKNTWOp6am8t976wi2oxbTjmUKccNIDEusPsnPgaS41ufnccOSufNGycw/dP13P/eV5x8/8fceuphXDquL845tu4uZ0PhHrp0iuOIXmkevSsRiUR+Xpu5Tmv/2r0MZJvZIOBx4HXgWeA0rwoLhY0793LLS0uYt76Q4w/L4N7vD2+T/+pjo6OYOnEg3zuiB7+cuYzbX1vBA/9ZTVFZFdW1rn694w/L4P+dfBjDeyscRCRwkBnwrW02tD4Uap1z1WZ2LvAX59xDZrbIy8K89u6K7fzk+cVEm/H784/kgtG9CV46tM306dqJGVeO5fUlW/nwy3x6dk6gb9cksrp1YvGm3Tz60VrOfHg2pwzrzmnDexAbHUVMtBEXE8W4/l3pFKe9eyIdid99j6D1oVBlZpcAlwNnBpfFelOS9+asKeCGZxcxrGcqf7t0FD09nEZqZpw9shdnj+z1jeVHD+jGpeOyeGJ2Lo99so5ZX+z4xuNH9k7j2auP3q9dVyIS2QpKK4gyfGtxAa0PhSuAa4C7nXPrzaw/8Ix3ZXln2eYirp6RQ//0JJ66YixpnfzLtpSEWG46aTBXTejP9uJyqmscVTW1rNpews9eXso1Ty/gicljiIuJjHMMl28pYvfeKkb17axRjsgBKCitoGtSPNE+tbiAVoaCc+4L4EYAM+sCpDjnfudlYV5Yl1/K5Onz6NwpjhlT/A2EhpLiYxiY8fXJKnUHoG95cQk/fWExD158lG99UFpSVVPLW8u28eScXBZtDMw7iIkyRvTpzDEDunHOUb0Y5OOJOCKRJL+k0rfuqHVaO/voQ+Cs4PqLgXwz+8g591MPa2tTO4rLuezxeQA8PWUs3VMTfK6oeeeP7s3O0grufXsV3ZLiuOOsw9v8mEd1TS2vLt5K58RYvnNYBrH7mG4LUFPrWLJ5Nws37GLX3kp2761i994qcjYUsqO4gv7pSdxx5jD6ZyTz2bqdzFm7k0c+WsvfP17LjyYO5McnDNIJfSItyC+t8HU6KrR+91Gac67YzK4Cpjvnfm1mS70srK0t3rSb0opqnpkyjgEZkfGf64++M5CC0gr+8cl6zIxfnTGszYaV24rKuPFfi5ifuwsI9G4/a0QvzjmqJ2mJsZRWVLO3sobtReV89FU+H6zKY+eewMyI6Cijc2IsnTvFMrxXGr87ry/fGZxRP5r5zqEZQGAofM9bK3n4gzW8sXQrvz3nCCYMzmiT+ltSU+t8HYKLHIiCkgoGpif5WkNrQyHGzHoAFwK3eViPZ757+CEcPaAbaYnhscuotX7xvaE4B4/NXs/Gwr08eMlRB33w+T9f7OCWl5ZQWV3Lny4YQVpiLC8v3Mwzn23giU/Xf2v91IQYjj8skxOHZjJ+UDrdkuJaNWpJT47n/gtHcv6o3vzfq8u57PF5HN4zleMGpzNxcAajsrqwJq+UOWsL+HTtTnIL9nD+6N5cMb4fKQmt/z4t31LESws2szqvhPySCvJLKti1t4penRM5KqszR2V1YUTvNBJio6l1jloXGCUVlQVGO7vLqoiJMk4cmknvLp3262sp0lacc743wwMw51zLK5ldAPwK+NQ5d62ZDQD+4Jw7z+sCG8vOznY5OTmh3qzvnv5sA3e8voJDu6fw+OXZ9EhLYHtxOat3lLJrbyVHD+jW7C6xmlrHvPWFvLxwMy8t2MzhPVN56JKjvjFq2rWnkg+/ysM56BQXQ3J8DGmJsQztkbLPM7lbq7yqhmc+28B7X+xg4cZdVNV88+ducGYymanxfLpmJ2mJsUydOIDLj+23zwAsKqvi9cVbeG7+JlZsLSY+JophPVPJTIknIyWeLp3iWFewh0UbdrG1qLzVdY7s05nTh/fgzBE9OSQtvHcxSvtSUl7F8Dtm8cvThjB14sA2f30zW+Ccy25xvdaEQjjpqKEA8NFX+Vz/z4WYgXNQWlH9jceH90rjxKGZjOzTmeoaR3l1DWWVNSzdXMTby7dTUFpBQmwUl47ry62nHkZ8jD/7+PdUVPP5+p0s3LCbQZnJHDuwG5nBQFu2uYi//Ocr3l+VR0pCDN89/BBOP7IH4wemExNlzFm7kxcXbOKd5dupqK7l8J6pXDymD2eN7LXPUeD2onK+2FZEdY0jyozoqMBHWnAXWOfEOHbtreSt5dt4a9k2lm8pJi46isnj+3H98YPCZkKCtG/rC/Yw6Y8fcv+FI/j+qN5t/vptGgpm1ht4CBgPOGA2cJNzbvPBFrq/OnIoAHy5vYSH/ruarklxDM5MZlBmCikJMXz0VT7vr9zBok27afwtTYiN4oQhmZw+vCeThmRExHTRxZt2M2NOLu99sYOSimrSEmNJiotma1E5aYmxnD2yJxdm9/GkVUhuwR7++sEaXlq4mbTEWG44YTCXHd03YqYGS2San1vIBY/OZcaVY5l4aNsfe2vrUHiPQFuLp4OLfghc6pw7+aCqPAAdPRRasrO0gnUFe4iPiSIxNpqE2GjSk+PrezlFmorqGmavLuDNpdsoLq/i7JG9OHlY95DMZPpiazH3vr2ST1YX0D01nkvH9eWSsVm+zw6R9untZdu49p8LefumCQztkdrmr9/WobDYOTeypWWhoFCQUPtkdT7TPl7HJ6sLiIuO4vQje3DjiYPp7/MsEWlfZszN5fbXVjD/tpM8+cejtaHQ2v0IBWb2Q+BfwfuXADsPtDiRSDJhcAYTBmewNr+UGXNyeWnBZv6zcgePXDqa4wan+12etBMFJYEWF12T/D15rbU7Sa8kMB11O7ANOJ9A6wuRDmNgRjJ3nn0E79w8kZ5piVw+fR7PfLbB77KkncgvraRrUpzv59e0KhSccxudc2c55zKcc5nOuXOA73tcm0hY6tO1Ey9dewwTB6fzf68u5843VlBdU+t3WRLh8kvKfe2OWudgplNETIsLkbaWkhDLY5eP4crx/Zn+aS6nPfgJ76/cQaRN8ZbwsamwjD5d/T958mBCQT0EpEOLjjJuP3MYj/5wFFU1jilP5XDRtM9YtHGX36VJhHHOsbFwL1kRHgr6l0gEOPWIHsz6yUTuOvtw1uWXcu7f5vDoR2v9LksiyM49lZRV1dCni//Xc2929pGZldD0H38D/K9eJEzERkdx2TH9OHdUb37xyjJ+9/Yqdu2t5OenDmnz7rbS/mws3AtAVjf/RwrNhoJzLiVUhYi0B8nxMfzlopGkJsTw94/WUbS3irvPHe77jBIJb5vqQiEMdh952u/AzHKBEqAGqG584oSZnQ3cBdQC1cDNzrnZXtYk4rXoKOO35xxBl05xPPzBGgr3VPKz7w35xoWURBqqC4Vw6NIbiiY4k5xzBft47H3gdeecM7MjgReAISGoScRTZsYt3z2Mzp1iufftVcz6YgdHZXXm/NG9OWN4TzXZk2/YWLiXzJT4sLgQla+d0ZxzpQ3uJqGD19LOXDVhAGeN6Mmri7fwYs5mbpu5nNtmLqdvt04MOSSFIYekcszAbozr31XHHjqwjYV7w2I6KngfCg6YZWYO+LtzblrjFczsXOBeIBM4vakXMbOpwFSArKws76oV8UBmagJTJw7k6gkDWLaliA+/zOfL7SWs3F7MrC928MD7qxnZpzM3nDCIE4ZkKhw6oE2FZYzt39XvMgDvQ2G8c26rmWUC75nZKufcxw1XcM7NBGaa2UQCxxdOavwiwTCZBoGGeB7XLOIJM+PI3p05snfn+mV7K6uZuWgLj3y4lilP5TC0Ryo/O/Uwjj8s08dKJZSqamrZVhQeJ67BwZ2n0CLn3Nbg5zxgJjC2mXU/BgaamTqMSYfRKS6GS8f15YNbjuePF4ygvKqGydPnc/tryymvqvG7PAmBrbvLqHWExTkK4GEomFmSmaXU3QZOAZY3WmeQBcfKZjYKiEPdV6UDio2O4vzRvXn7pglMOa4/M+Zu4IyHZrNia5HfpYnHNobRdFTwdqTQHZhtZkuAecCbzrl3zOwaM7smuM55wHIzWwz8FbjIqXmMdGAJsdH86oxhPD1lLMVlVZzz10/5x8frqK3Vr0V7VRcK4bL7yLNjCs65dcCIJpY/2uD2fcB9XtUgEqkmDM7gnZsn8vOXl3L3Wyv5eHU+f7pgRP21rKX92FRYRlx0FN3D5Huri86KhKmuSXH8/bLR3HPucObnFnLqA4FOrNK+bCrcS+8uiWFz1rtCQSSMmRk/GJfFv284ju6pCUx5KodbX1rC7r2VfpcmbWRj4V56h8muI1AoiESEQZkpzLzuWH70nQG8vHALJ93/Ea8t3qLrN7QDm3btJatreMw8AoWCSMRIiI3mF98byhs/Po5eXTpx03OLuXz6fPKKy/0uTQ5QcXkVu/dW0ScMeh7VUSiIRJhhPVN55dpjuePMYcxfX8iZD89m8abdfpclByCcuqPWUSiIRKDoKGPy+P68ct2xxMVEceGjc3khZ5PfZcl+2hRm01FBoSAS0Yb2SOX1649jTP8u3PrSUu54fQXVNbV+lyWtFG7nKIBCQSTidUmK46krxjLluP48OSeXqU8vYG9ltd9lSStsKiwjLTGWtMTwaaWuUBBpB2Kio/jVGcO465wj+PDLPC6Z9hkFpRV+lyUtCLTMDp+ZR6BQEGlXLju6L4/+cDRf7ijhvEfmsL5gj98lSTM2Fe4Nq4PMoFAQaXdOOfwQnr36aIrLqjjvkTks2rjL75KkCbW1js27wqdldh2Fgkg7NCqrC69cN56k+Gh+8I/P1R4jDO0oKaeypjaszlEAhYJIu9U/PYlXrh3PoMxkrp6Rw3PzNvpdkjSwqbAMCK9zFEChINKuZaTE89zUo5kwOIOfv7KMP836khq14Q4L4TgdFRQKIu1eUnwMj12ezYXZvXnov2uYPH0ehXvUUM9vGwv3Yga9Omv2kYiEWGx0FPeddyT3fn84n68v5PQHP2HBBh2A9tPa/FKyunYiLia8/gyHVzUi4hkz45KxWbxy7bHERBsX/X0uj89er06rPlmzo5TBmcl+l/EtCgWRDuaIXmn8+4YJTBqSyV3//oJrnllAUVmV32V1KNU1tawrKGVQZorfpXyLQkGkA0pLjGXaZaP5v9OH8v7KPM546BOWbS7yu6wOY0PhXqpqnEYKIhI+zIyrJgzg+R8dQ02N47xH5vDKws1+l9UhrN5RCsDg7goFEQkzo/t24c0bJ5Ddrwv/78Ul/EvnM3huTV4JAAMzFAoiEoa6JMXxxOQxHH9oBr94ZRlPzcn1u6R2bXVeKb06J5IUH+N3Kd+iUBARIHC5z0cvG83Jw7rz69dX8I+P1/ldUru1Jq80LHcdgUJBRBqIj4nmb5eO4vThPbj7rZUaMXigptaxJq+UQWG46wgg/MYuIuKr2OgoHrh4JJU1tdz5xgp6d0nkxKHd/S6r3diyq4yK6lqNFEQkcsQEg+GIXmn8+NlFLN+i6aptZXXwIHM4nqMACgUR2YdOcYGeSV2T4rjyyfls3V3md0ntwuq8wHTUQWF4jgIoFESkGZkpCTwxeQxllTVc+eR8nfncBlbvKKV7anxYXZe5IYWCiDTrsENSeOSHo1mbX8rk6fMoraj2u6SItiavhMFhuusIFAoi0grHDU7noUtGsXRzEVc+OZ+yyhq/S4pIzjlW55WG7a4jUCiISCudesQh/PmikeTkFnL1jBzKqxQM+2trUTl7K2vCduYRKBREZD+cNaInvz9/BLPXFHDtMwuoqFYw7I/VOwIzj7T7SETajfNH9+aec4fzwZf5TJ2xQCOG/bAmOPMoHLuj1lEoiMh++8G4LH73/eF8vDqfq57K0TGGVlqTV0p6chxdkuL8LmWfFAoickAuHpvFH88fwZy1BUyePo89mpXUonA/yAwKBRE5COeN7h04+LxhF5c/oWBojnOO1TvCezoqeBwKZpZrZsvMbLGZ5TTx+KVmtjT4McfMRnhZj4i0vbNH9uKBi0eycOMuzUpqRn5JBcXl1WE98whCM1KY5Jwb6ZzLbuKx9cB3nHNHAncB00JQj4i0sTOO7MkfLxjB3HU7ufaZBVRW1/pdUtipb28Rpt1R6/i6+8g5N8c5tyt49zOgt5/1iMiB+/6o3vz2nCP44Mt8bnpuEdU1CoaGVmwNNBU89JAOvPsIcMAsM1tgZlNbWHcK8LbH9YiIhy4d15f/O30oby/fzi0vLqGm1vldUtiYt34X/dOTSE+O97uUZnl9PYXxzrmtZpYJvGdmq5xzHzdeycwmEQiF45p6kWCgTAXIysrysl4ROUhXTRhARXUtf3j3S2od3H/hCGKiO/acltpaR86GQk4ZFv7XpfD0O+Wc2xr8nAfMBMY2XsfMjgQeA852zu3cx+tMc85lO+eyMzIyvCxZRNrA9ZMG8bNTh/D6kq3c9Nxiqjr4rqS1+aXs3lvFmH5d/S6lRZ6NFMwsCYhyzpUEb58C/KbROlnAK8BlzrmvvKpFRELv2uMHEhNl3P3WSmpqHQ9echRxMR1zxDAvtxAgIkLBy+9Qd2C2mS0B5gFvOufeMbNrzOya4Dq3A92Av+1r2qqIRK6rJw7g9jOG8c6K7Ux9OqfDnscwf30hGSnx9O3Wye9SWuTZSME5tw741nkHzrlHG9y+CrjKqxpExH9XHtefxLhobpu5jIumzeWJyWPITEnwu6yQmp+7i7H9umJmfpfSoo45lhORkLpkbBaPXZ7N2rw9nPvXOawJXqe4I9iyu4wtu8sY06+L36W0ikJBRELihCHdef5HR1NRXct5j8xlyabdfpcUEjnB4wnZEXA8ARQKIhJCR/buzMzrjiU5Pobrn11IcXn7v+bzvPWFpMTHMLRHqt+ltIpCQURCqk/XTjx4yVFsKyrn9leX+12O5+bnFjKqbxeio8L/eAIoFETEB6P7duHGEwbz6uKtvLpoi9/leGbXnkq+2lHK2P6RsesIFAoi4pPrJw0ku28XfvXqcjYV7vW7HE8s2BBo7ZbdNzIOMoNCQUR8EhMdxZ8vGgnAzc8vbpcN9ObnFhIXHcWIPp39LqXVFAoi4ps+XTvx23OPYMGGXfzilWXUtrMGevNyCzmydxoJsdF+l9JqCgUR8dXZI3tx44mDeXHBZm57dTnOtY9gKKusYdnmIsZE0PEE8L5LqohIi35y0mCqa2r524driY027jzr8Ig4+7c5c9cVUF3rGBsh5yfUUSiIiO/MjP/97mFU1dTyj0/WExMVxa/OGBrRwfCveZtIT45j/KB0v0vZLwoFEQkLZsYvTxtKda3jiU/Xk5IQw09OPtTvsg7I9qJy/rsqj6snDIi4zrAKBREJG2bG7WcMY09FNQ+8v5puyXH8zzH9/C5rv72Qs4maWsfFY/r4Xcp+UyiISFgxM+45dziFe6r49esr6NIpjjNH9PS7rFarqXU8N28jxw1Kp196kt/l7LfIGteISIcQEx3Fwz84iuy+XfjpC4v5ZHW+3yW12kdf5bG1qJwfjIvMSwcrFEQkLCXERvPY5WMYmJHMj55ewLz1hX6X1CrPfh44wHzS0PC/HnNTFAoiErbSEmOZMWUsPdISmDx9XtgHw7aiMv67agcXZPeJuAPMdSKzahHpMDJTEvjX1KPrg+HzdTv9Lmmfnp+/iVpHRB5grqNQEJGwVxcMPTsncsWT88MyGKpranl+/iYmDE6nb7fIO8BcR6EgIhEhMyWBZ68eR8/OiUyePj/sDj6/kLOZbUXlETmFtiGFgohEjMyUBJ6bejT90pOY8mQO7yzf7ndJAOytrObP//mK0X27cNLQTL/LOSgKBRGJKOnJ8Tx39dEc0SuV659dyMxFm/0uicc/WU9+SQW/PG1IRLfmAIWCiESgtE6xPD1lHOP6d+Unzy/h168t57XFW1i9oyTk12UoKK3g0Y/W8t3DuzO6b2Q1v2uKzmgWkYiUFB/DE5PHcOtLS3l23kaemrsBgLiYKC4dl8Vtpw0lJtr7/3sfen815dW13HrqEM+3FQoKBRGJWAmx0Tx4yVFUVo9gbX4pK7cVM3tNAdM/zWVNXikP/2AUaYmxnm0/t2AP//x8IxeN6cPAjGTPthNK2n0kIhEvLiaKoT1S+f6o3tx/4UjuO284c9fu5LxH5rBxpzfXf3bOce/bK4mLieLmkwZ7sg0/KBREpN25aEwWT08ZR0FpBWf/dTYffJnXpq9fXVPLrS8t5d0VO7h+0iAyUxLa9PX9pFAQkXbpmIHdmHndeNKT47li+nx++vxidu2pPOjXLa+q4ZpnFvLigs3cdOJgrjt+YBtUGz4UCiLSbvVPT+LfNx7HjScM4vUlWznp/o94Y8nWA74OdFFZFf8EXFpfAAAKDUlEQVTz+DzeX7WDO886nJ+cfGjET0FtzCLtItnZ2dkuJyfH7zJEJMKs3FbMz15eytLNRUw8NIM7zzqc/q283sGW3WW8MH8Tz8/fxM49FfzpwpGcFUHXeAAwswXOuewW11MoiEhHUV1Ty4y5G7j/va+orK7lmu8M4LpJg0iIjaaqppZdeyrZuaeSgtIKdpYGPs9Zu5MPv8zDARMGZ/DjSYMY2z/yzkdQKIiI7ENecTl3v7WS1xZvrZ+yWlRW1eS63VPjuTC7Dxdm96FP106hLLNNtTYUdJ6CiHQ4makJPHDxUVw8JosXF2wiOT6GrklxdEuOp1tSHOnJ8XRLjiM9KZ7UxJh2d9ygOQoFEemwjhnYjWMGdvO7jLCi2UciIlJPoSAiIvUUCiIiUs/TUDCzXDNbZmaLzexbU4bMbIiZzTWzCjO7xctaRESkZaE40DzJOVewj8cKgRuBc0JQh4iItMDX3UfOuTzn3Hyg6QnCIiISUl6HggNmmdkCM5vq8bZEROQgeb37aLxzbquZZQLvmdkq59zH+/siwUCZCpCVldXWNYqISJCnoeCc2xr8nGdmM4GxwH6HgnNuGjANwMyKzGx1E6ulAUXNLGv8eN39ptZJB/Z1HKQlTdXRmsdbqr/x/aZuq/7wqB8O/D20VH9z6zRXb+P77bH+hrfDof7m6mx4P1R/g/q26tnOOU8+gCQgpcHtOcCp+1j3DuCWVr7utNYub7is8eN195taB8g5iPfdZH0HW39z76fxe1H9/tZ/MO+hpfr35z10tPrb4meoLetvrs5mvu6e/w609OHlSKE7MDPYMyQGeNY5946ZXQPgnHvUzA4BcoBUoNbMbgaGOeeKm3ndN/Zj+RvNPP5GK9Y5EC29xoHW3/h+U7dVf/uvv7l1mqu38f32WH9rt9+ctqy/8bJw+RvUrIjrkhoKZpbjWtFNMFypfv9F+ntQ/f7ys36d0dy0aX4XcJBUv/8i/T2ofn/5Vr9GCiIiUk8jBRERqdfuQ8HMnjCzPDNbfgDPHR3s3bTGzB60BlfaMLMbzOxLM1thZr9v26q/UUOb129md5jZlmBPqsVmdlrbV15fgydf/+Djt5iZM7P0tqv4WzV48fW/y8yWBr/2s8zMs4v9elT/H8xsVfA9zDSzzm1f+Tfq8OI9XBD83a01szbfd38wNe/j9S43s9XBj8sbLG/2d+SAHMzUpUj4ACYCo4DlB/DcecAxgAFvA98LLp8E/AeID97PjLD676CVU4DDsf7gY32Ad4ENQHok1Q+kNljnRuDRCKv/FCAmePs+4L5I+xkChgKHAR8C2eFSc7Cefo2WdQXWBT93Cd7u0tz7O5iPdj9ScIEzqAsbLjOzgWb2TrD9xidmNqTx88ysB4Ff3rku8NWfwdeN+64FfuecqwhuIy/C6g8ZD+v/M3ArgVYqnvGifvfNKddJePgePKp/lnOuOrjqZ0Bvr+r38D2sdM59GW4178N3gfecc4XOuV3Ae8CpXv2Ot/tQ2IdpwA3OudHALcDfmlinF7C5wf3NwWUAhwITzOxzM/vIzMZ4Wu23HWz9AD8ODv+fMLMu3pXapIOq38zOArY455Z4Xeg+HPTX38zuNrNNwKXA7R7W2pS2+PmpcyWB/1BDrS3fQ6i0puam9AI2Nbhf9z48eX8d7hrNZpYMHAu82GD3W3xTqzaxrO4/uhgCw7ijgTHAC2Y2IJjWnmqj+h8B7grevwv4E4Ffbs8dbP1m1gm4jcAujJBro68/zrnbgNvM7BfAj4Fft3GpTWqr+oOvdRtQDfyzLWtsSVu+h1BprmYzuwK4KbhsEPCWmVUC651z57Lv9+HJ++twoUBgdLTbOTey4UIziwYWBO++TuAPZ8NhcW9ga/D2ZuCVYAjMM7NaAr1K8r0sPOig63fO7WjwvH8A//ay4EYOtv6BQH9gSfCXqzew0MzGOue2e1w7tM3PT0PPAm8SolCgjeoPHuw8AzgxFP8MNdLW34NQaLJmAOfcdGA6gJl9CEx2zuU2WGUzcHyD+70JHHvYjBfvr60PsITjB9CPBgd8CPRhuiB424AR+3jefAKjgbqDOKcFl18D/CZ4+1ACQzuLoPp7NFjnJ8BzkfT1b7ROLh4eaPbo6z+4wTo3AC9FWP2nAl8AGV7WHYqfITw60HygNbPvA83rCeyd6BK83bU17++A6g7VN9WvD+BfwDYCF/LZDEwh8J/mO8CS4A/37ft4bjawHFgLPMzXJ/vFAc8EH1sInBBh9T8NLAOWEviPqkck1d9onVy8nX3kxdf/5eDypQT61PSKsPrXEPhHaHHww7PZUx6+h3ODr1UB7ADeDYeaaSIUgsuvDH7d1wBX7M/vyP5+6IxmERGp11FnH4mISBMUCiIiUk+hICIi9RQKIiJST6EgIiL1FArSLphZaYi395iZDWuj16qxQMfU5Wb2RktdR82ss5ld1xbbFmlMU1KlXTCzUudcchu+Xoz7uumbpxrWbmZPAV855+5uZv1+wL+dc0eEoj7pWDRSkHbLzDLM7GUzmx/8GB9cPtbM5pjZouDnw4LLJ5vZi2b2BjDLzI43sw/N7CULXD/gn3X96oPLs4O3S4MN7paY2Wdm1j24fGDw/nwz+00rRzNz+brxX7KZvW9mCy3QM//s4Dq/AwYGRxd/CK77v8HtLDWzO9vwyygdjEJB2rMHgD8758YA5wGPBZevAiY6544i0KH0ngbPOQa43Dl3QvD+UcDNwDBgADC+ie0kAZ8550YAHwNXN9j+A8Htt9iTJti750QCZ5kDlAPnOudGEbiGx5+CofRzYK1zbqRz7n/N7BRgMDAWGAmMNrOJLW1PpCkdsSGedBwnAcMadKVMNbMUIA14yswGE+gqGdvgOe855xr2wZ/nnNsMYGaLCfSzmd1oO5V83VRwAXBy8PYxfN3f/lngj/uoM7HBay8g0C8fAv1s7gn+ga8lMILo3sTzTwl+LAreTyYQEh/vY3si+6RQkPYsCjjGOVfWcKGZPQR84Jw7N7h//sMGD+9p9BoVDW7X0PTvTJX7+uDcvtZpTplzbqSZpREIl+uBBwlcayEDGO2cqzKzXCChiecbcK9z7u/7uV2Rb9HuI2nPZhG4VgEAZlbXtjgN2BK8PdnD7X9GYLcVwMUtreycKyJwec5bzCyWQJ15wUCYBPQNrloCpDR46rvAlcGe/ZhZLzPLbKP3IB2MQkHai05mtrnBx08J/IHNDh58/YJAy3OA3wP3mtmnQLSHNd0M/NTM5gE9gKKWnuCcW0Sgi+bFBC5ek21mOQRGDauC6+wEPg1OYf2Dc24Wgd1Tc81sGfAS3wwNkVbTlFQRjwSvElfmnHNmdjFwiXPu7JaeJ+InHVMQ8c5o4OHgjKHdhOiSpyIHQyMFERGpp2MKIiJST6EgIiL1FAoiIlJPoSAiIvUUCiIiUk+hICIi9f4/t7rTk7V5zCoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.lr_find()\n",
    "learn_lm.recorder.plot(skip_end=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:47\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      4.108119    3.942737    0.276310  (00:47)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn_lm.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.save('fit_head')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.load('fit_head')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 09:58\n",
      "epoch  train_loss  valid_loss  accuracy\n",
      "1      3.855438    3.800400    0.291591  (00:59)\n",
      "2      3.759814    3.692157    0.303298  (00:59)\n",
      "3      3.602196    3.617920    0.310050  (00:59)\n",
      "4      3.506228    3.559112    0.316812  (00:59)\n",
      "5      3.384718    3.523955    0.319978  (01:00)\n",
      "6      3.304082    3.505268    0.322260  (01:00)\n",
      "7      3.205504    3.498160    0.323293  (00:59)\n",
      "8      3.128214    3.500733    0.323048  (01:00)\n",
      "9      3.084575    3.503936    0.323252  (00:59)\n",
      "10     3.047713    3.508037    0.322524  (01:00)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn_lm.unfreeze()\n",
    "learn_lm.fit_one_cycle(10, 1e-3, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3Xd4VFX6wPHvm0klCUkIAQIBEpqUECBEBEHpSFFUxBXWritr3xXX/WFZbKuiuOraRV3svSKggAKiVKmhibTQQgktlABp5/fHvZm0SQFmMpnk/TzPPLnl3DvvwXHeOffec44YY1BKKaUA/LwdgFJKqepDk4JSSiknTQpKKaWcNCkopZRy0qSglFLKSZOCUkopJ00KSimlnDQpKKWUctKkoJRSysnf2wGcrvr165v4+Hhvh6GUUj5l2bJl+40xMRWV83hSEBEHsBTYZYy5uMS+scBfgFwgA7jJGLOtvPPFx8ezdOlST4WrlFI1koiU+91aoCouH/0NWF/GvhVAijEmCfgCeKYK4lFKKVUGjyYFEYkDhgFvudpvjJljjMmyVxcBcZ6MRymlVPk83VJ4AfgnkF+JsjcD37vaISJjRGSpiCzNyMhwZ3xKKaWK8Ng9BRG5GNhnjFkmIn0qKHsNkAL0drXfGDMJmASQkpKiY30rVUPk5OSwc+dOTp486e1Qaozg4GDi4uIICAg4o+M9eaO5JzBcRIYCwUBdEfnAGHNN0UIiMgB4EOhtjDnlwXiUUtXMzp07CQ8PJz4+HhHxdjg+zxjDgQMH2LlzJwkJCWd0Do9dPjLG3G+MiTPGxAOjgNkuEkIX4A1guDFmn6diUUpVTydPniQ6OloTgpuICNHR0WfV8qryzmsi8piIDLdXJwJhwOcislJEplR1PEop79KE4F5n++9ZJZ3XjDFzgbn28vgi2wdUxfsD/JZ2kF/+yODOfq0J9NeO3Eop5Uqt+XZcvu0QL87eRG5+ZR6EUkrVdAcOHKBz58507tyZRo0a0aRJE+d6dnZ2pc5x4403smHDBg9HWrV8bpiLM+VnN6ny9dklpRQQHR3NypUrAXjkkUcICwvjH//4R7EyxhiMMfj5uf79PHnyZI/HWdVqTUuh4DJbvtGsoJQq26ZNm0hMTOTWW28lOTmZ3bt3M2bMGFJSUujQoQOPPfaYs2yvXr1YuXIlubm5REZGMm7cODp16kSPHj3Yt883n52pNS2FgpsvmhOUqp4e/W4t69KPuPWc7RvX5eFLOpz2cevWrWPy5Mm8/vrrAEyYMIF69eqRm5tL3759GTlyJO3bty92TGZmJr1792bChAmMHTuW//3vf4wbN84t9ahKtaelYP81mhWUUhVo2bIl5557rnP9448/Jjk5meTkZNavX8+6detKHRMSEsKQIUMA6Nq1K2lpaVUVrlvVmpaCn50VNCcoVT2dyS96TwkNDXUub9y4kf/+978sWbKEyMhIrrnmGpf9AAIDA53LDoeD3NzcKonV3WpNS8HPr+BGs2YFpVTlHTlyhPDwcOrWrcvu3buZMWOGt0PyqFrTUii4fKRPHymlTkdycjLt27cnMTGRFi1a0LNnT2+H5FHia9fYU1JSzJlMsvPBom089M0aljzYnwbhwR6ITCl1utavX0+7du28HUaN4+rfVUSWGWNSKjq29lw+0qePlFKqQrUmKWg/BaWUqlitSQr69JFSSlWs1iQFEX36SCmlKlJ7koL9V3OCUkqVrdYkBb3RrJRSFas1SUFvNCulSurTp0+pzmgvvPACt99+e5nHhIWFAZCens7IkSPLPG9Fj86/8MILZGVlOdeHDh3K4cOHKxu6x9SapOBsKXg5DqVU9TF69Gg++eSTYts++eQTRo8eXeGxjRs35osvvjjj9y6ZFKZPn05kZOQZn89dak1S0JaCUqqkkSNHMnXqVE6dOgVAWloa6enpdO7cmf79+5OcnEzHjh359ttvSx2blpZGYmIiACdOnGDUqFEkJSVx1VVXceLECWe52267zTns9sMPPwzAiy++SHp6On379qVv374AxMfHs3//fgCee+45EhMTSUxM5IUXXnC+X7t27bjlllvo0KEDgwYNKvY+7lJ7hrlw3lPQpKBUtfT9ONiz2r3nbNQRhkwoc3d0dDTdunXjhx9+4NJLL+WTTz7hqquuIiQkhK+//pq6deuyf/9+unfvzvDhw8uc//i1116jTp06pKamkpqaSnJysnPfE088Qb169cjLy6N///6kpqZy991389xzzzFnzhzq169f7FzLli1j8uTJLF68GGMM5513Hr179yYqKoqNGzfy8ccf8+abb/KnP/2JL7/8kmuuucY9/1a2WtNS0H4KSilXil5CKrh0ZIzhgQceICkpiQEDBrBr1y727t1b5jnmzZvn/HJOSkoiKSnJue+zzz4jOTmZLl26sHbtWpfDbhf166+/cvnllxMaGkpYWBgjRozgl19+ASAhIYHOnTsDnhueu9a0FHQ6TqWquXJ+0XvSZZddxtixY1m+fDknTpwgOTmZd955h4yMDJYtW0ZAQADx8fEuh8suylUrYuvWrTz77LP89ttvREVFccMNN1R4nvKuZgQFBTmXHQ6HRy4f1ZqWQuEoqZoVlFKFwsLC6NOnDzfddJPzBnNmZiYNGjQgICCAOXPmsG3btnLPceGFF/Lhhx8CsGbNGlJTUwFr2O3Q0FAiIiLYu3cv33//vfOY8PBwjh496vJc33zzDVlZWRw/fpyvv/6aCy64wF3VrVCtaSnodJxKqbKMHj2aESNGOC8jXX311VxyySWkpKTQuXNn2rZtW+7xt912GzfeeCNJSUl07tyZbt26AdCpUye6dOlChw4dSg27PWbMGIYMGUJsbCxz5sxxbk9OTuaGG25wnuMvf/kLXbp0qbKZ3GrN0Nkz1u7hr+8vY+pdvUhsEuGByJRSp0uHzvYMHTq7EvzKeGpAKaVUIY8nBRFxiMgKEZnqYl+QiHwqIptEZLGIxHsqDj/tp6CUUhWqipbC34D1Zey7GThkjGkFPA887akgCjuveeodlFJnwtcuYVd3Z/vv6dGkICJxwDDgrTKKXAq8ay9/AfSXsnqHnH0sgH4AlapOgoODOXDggP5/6SbGGA4cOEBw8JlPOezpp49eAP4JhJexvwmwA8AYkysimUA0sN/dgWg/BaWqn7i4OHbu3ElGRoa3Q6kxgoODiYuLO+PjPZYURORiYJ8xZpmI9CmrmIttpb62RWQMMAagWbNmZxZPwcn1F4lS1UZAQAAJCQneDkMV4cnLRz2B4SKSBnwC9BORD0qU2Qk0BRARfyACOFjyRMaYScaYFGNMSkxMzBkFU7Sl8OO6veTZTYZr3lrM/V+5ebwVpZTyUR5rKRhj7gfuB7BbCv8wxpQcuWkKcD2wEBgJzDYe+ilf8PTRD2v28L/5W3lwaDtO5ebx6ybrStWNPeNp07Csq1xKKVU7VHk/BRF5TESG26tvA9EisgkYC4zz1Pv6O6yq/m/+VgBW7jzMszP/cO6/4rUFnnprpZTyGVUyzIUxZi4w114eX2T7SeDKqoghyL94/lu9M7PY+tGTuRhjyhwaVymlaoNa06M5OMBRbH37QWvGo/nj+nFlV+tO/TkP/UDa/uNVHptSSlUXtSYplGwpFGgSGcIFbayb19l5+fzj81VVGZZSSlUrtSYplGwpAIxIbgLAxR1jnduWbjukj60qpWqtWjN0tsOv8F7B1qeGYkzh0Bd+fsIHN5/HNW8vBmDFjsMkN4vyRphKKeVVtaalEBESAMDA9g0REfz8pNhN5V6t6/PTvb0BePGnjV6JUSmlvK3WtBQC/f1ImzCs3DJNo+oAMHdDBiey8wgJLH3JSSmlarJa01KojMAiN6NnrtsDwL4jJ3nrly2l7jP8sjGDf32zxtkz+pEpa/l86Y6qC1YppTyg1rQUKmvV+EF0emwmj323DoefcOdHKwDo3DSSlPh6rNmVyW0fLmPHQWvC7PcXFZ+79cqUplUes1JKuYu2FEqIqBNAi/qhHDie7UwIAFdNWsSSrQe5+KVfnQlBKaVqGk0KLpzKzS+1LS/f8Kc3FlZ47PFTuZ4ISSmlqoQmBReeGtGxwjLDkmIZf3F72sXW5bs7e/HKn5MBSDugPaKVUr5L7ym4cGGbGNImDCMv3/Dl8p2c3zKakAAHG/Ye5c9vLmbCiI6M6mbN63BTL2ss+IJ+ECt3HKZD4wivxa6UUmdDk0I5HH7Cn4rcOD4/LKjMx1rj61uPsz749Rr6tW1AbERIlcSolFLupJeP3KROYGF+7fHUbC9GopRSZ06Tghs9e2Un5/J/Zm7wYiRKKXVmNCm40ciucYy/uD0AL83eRPy4ac7ObUop5Qs0KbhZQkxosfUf1uzRxKCU8hmaFNzs3Ph69G/bgL/1bw3AHR8tZ9iLv5CviUEp5QM0KbhZWJA/b99wLvcMbEN0aCAAv+85yg3v/OblyJRSqmKaFDzoUFa2c3neHxmczMnzYjRKKVUxTQoe9O5N3bi8SxPn+orth70YjVJKVUyTggdd0DqG56/qzEd/OQ+A9MM6kJ5SqnrTpFAFkptbU3vu0qSglKrmNClUgeAAB/VCA9mScczboSilVLk0KVSRg8ez+WZlOm/O2+LtUJRSqkweSwoiEiwiS0RklYisFZFHXZRpJiJzRGSFiKSKyFBPxeNto+1RVZ+Yvt7LkSilVNk82VI4BfQzxnQCOgODRaR7iTIPAZ8ZY7oAo4BXPRiPVz15eaJz+ejJHC9GopRSZfNYUjCWgovoAfarZLdeA9S1lyOAdE/F41X5+YgIr19jTcTT8ZGZnPfkj2S7mOFNKaW8yaP3FETEISIrgX3ALGPM4hJFHgGuEZGdwHTgLk/G4xW5p2BSb/j1Bbo3D3du3nvkFDdMXuLFwJRSqjSPJgVjTJ4xpjMQB3QTkcQSRUYD7xhj4oChwPsiUiomERkjIktFZGlGRoYnQ3a/U8cgoin8+DCR7/XnPCm8p7Bg8wHix01zvt5bmOa1MJVSCqro6SNjzGFgLjC4xK6bgc/sMguBYKC+i+MnGWNSjDEpMTExHo7WzUKjYfRHMPoTyMni06DH+U/Aq/SPK110/Ldr2VzOY6sz1+5h7KcrPRisUqq28+TTRzEiEmkvhwADgN9LFNsO9LfLtMNKCj7WFKikc4bA7YvJ7XkvIwIW89bRW5nUdgV+FL+vcN3bZV9SGvP+Mr5asYu0/cc9Ha1SqpbyZEshFpgjIqnAb1j3FKaKyGMiMtwucy9wi4isAj4GbjDG1NwxpgPr4D9wPHL7QqRxFwalTWRTwrPM/XM4K8cPBKxezze6uNdQdOjtH9buITs3n0nzNpcaZG9a6m6em7lBB99TSp0R8bXv4JSUFLN06VJvh3H2jIE1X8KMB+DYPki5iZf9RvPsvH0AfHdnLzrGRXDoeDZdHp9V7NBOTSPJyc1n3e4j3DuwDXfZczcAxI+b5ly+IjmO//ypE0opJSLLjDEpFZXTHs3eIgIdR8Kdv8F5t8Kyyfw19SpG+M0DDCNfX8D01bu58+PlpQ5dteMw63YfAeA/s/5gza5MAA4cO1Ws3JfLd7J4ywGPV0UpVXNoUvC24AgYMgHG/IxfdAueC3ydTwMfp1nedm7/cDnzN1X8pX7xS7+Sk5fPLxv3AzC8U2MSm1jdP66atMij4SulahZNCtVFbBKOm2fC8JfoGJDO9MD7Gef/EXU46Szy5nUpzmk+X7+mKxd1aOjc9+WynfzdfjJp4pVJfHtHL+e+I9qDWilVSXpPoRpK276dNe/ew8V5P3I0sAH5gydQt/PliF/xHJ6bl0/miRy6/vvH4sdPGAbAjLV7+Ov7y/j2jp50ahpZZfErpaofvafgw+KbNePif30JN80kPKoBEVNuQj76ExwsPsKqv8OP6LCgYts2P1k4pmDLmFAAtuzXIbuVUpXj7+0AVDmanQdjfoYlk2DOE/BqD7jgXjj/bggIdhZb8+hFrN99hMTGETj8pPDweqH4CWzJ0H4NSqnK0ZZCdefwhx63W08pnTPUSg6vnQ+bfnIWCQvy59z4eoQEOoodGujvR9N6ddiind2UUpWkScFX1G0MV06Ga7+21j8YAZ9dD0fKH1i2Rf1QbSkopSpNk4KvadkPbl8IfR+CP36Al8+FBS9DnusnjFrEhLF1/7FiPaKVUqosmhR8kX8Q9L4Pbl8EzXvCzAfhjd6wvXSfhFYNwjiZk0965gkvBKqU8jWaFHxZvQT486cw6iM4dQT+dxF8cwcc3+8s0qpBGAC9np7jrSiVUj5Ek4KvE4G2w+COxdDrHkj9BF7qCksnQ34+7WLrOosePJ7txUCVUr5Ak0JNERgKAx6BW+dDo44w9e/w9kDC9qfyzo3nArAu/YhXQ1RKVX+aFGqaBm3h+u/g8klweBu82Zfz5l7NJX4LuPHt+XrDWSlVLk0KNZEIdLoK7lwKAx8n5OQ+Xgp8mflBd/PS+JtcPsaanZuPrw15opRyP00KNVlIJPS8G+5awdyur7AmP567HF9jnk+Ez66Drb+AMeTm5dPmoe8Z8t9fvB2xUsrLdJiL2sDPjz6XXMOdR9vzyOqVXO34kRs3zSVw3bcQ05YDba8llEb8vsfbgSqlvE1bCrXIs1d2YrtpyFO5V9PxyH95te5YjH8wDX95kEVBd/Ko/2ROpa/1dphKKS/SpFCLBAc4+PdliQCcIpBn9qWQsPU+Ljv1GDPzuzLKMYegSedz5PXB5K39BvJyvRyxUqqqaVKoZUZ2jSuxRVhpWnFvzu30OPUyT+eM4sjujTg+vx5e6Ag/P0Nu5m7y7KeWMk/kMDU1XW9KK1VDaVKoZYIDHHw6pjvPjExybosICeDhS9pzkLq8ljecC0+9wBN1x1uPt855gvznOjDniYth20LemLuJOz9awbJthwDIys5lf4m5oZVSvktnXqvFtu4/zrgvU3n16mSiw4KIHzetVJkE2c01jh+50vEzdSWLLX7xvHmqP0mD/8LoC9pz18cr+G5VOpufHFpsLgelVPVS2ZnXKpUURKQlsNMYc0pE+gBJwHvGmMNnHelp0qTgOSdz8nh3QRpNokK486MVxfaFcJJLHQu4zjGL9n7byJJQ6nS7jr7zWrHVxHLPgDb8bUBrL0WulKqIu6fj/BLIE5FWwNtAAvDRWcSnqqHgAAd/7d2SYR1jCQsqfFr55/v6EBsTzSd5/Ria/SRXnHqYWbmdyF40iTlB9/JewFOsmf0R+bl6Y1opX1fZlsJyY0yyiNwHnDTGvCQiK4wxXTwfYnHaUqg62bn5nMjJIyIkgJy8fFo/+D1B/n5cdW5T3lu4jfpkMsoxm2sDZtOQA+w09cnqeC1thtwJofW9Hb5Sqgh3txRyRGQ0cD0w1d4WcKbBKd8Q6O9HRIj1nznA4UfahGFs+PcQBrZvCMB+ItiWeAcZN//GX7PvIS2/IW3WPI95rh18NcaaMrSMyX+UUtVTZXs03wjcCjxhjNkqIgnAB+UdICLBwDwgyH6fL4wxD7so9yfgEcAAq4wxf658+MobLmgdw6x7LqR1w3AAcvLymZF/LjPyz6Vl7i4+S15H9IYvIfVTCImCthdDh8sgoTc49LeEUtXZaT99JCJRQFNjTGoF5QQINcYcE5EA4Ffgb8aYRUXKtAY+A/oZYw6JSANjzL7yzquXj6qnQ8ez2ZRxjCtfXwhA41BhwZXAum9gw/fWJEDBkcUThH+gd4NWqhap7OWjSrUURGQuMNwuvxLIEJGfjTFjyzrGWNnmmL0aYL9KZqBbgFeMMYfsY8pNCKr6igoNJDEgwrmeftzwVVYnRowYCrmnYPNsWPsNrJ8CKz+A4Ag4Z5iVIFr01QShVDVR2ctHEcaYIyLyF2CyMeZhESm3pQAgIg5gGdAK68t/cYkibexy8wEH8Igx5gcX5xkDjAFo1qxZJUNWVS0k0FFsfda6vYxIjrPmlD5niPXKPQVb5loJYsM0WPURBEVY+zpcBi37WeWVUl5R2aTgLyKxwJ+AByt7cmNMHtBZRCKBr0Uk0RizpsT7twb6AHHAL3aZwyXOMwmYBNblo8q+v6p6aROGAdDi/mk0rBtcuoB/ELS5yHrlZsPWn60E8ftUayrRoLpWgmhvJ4gAF+dQSnlMZZPCY8AMYL4x5jcRaQFsrOybGGMO25egBgNFk8JOYJExJgfYKiIbsJLEb5U9t6qeWjcIZ9fhE+UX8g+E1gOtV+7zsHUerPsafp9m3aQODIdzBlsJolV/CAipmuCVqsUqlRSMMZ8DnxdZ3wJcUd4xIhID5NgJIQQYADxdotg3wGjgHRGpj3U5aUvlw1fVVePIYNIrSgpF+QdC6wHW6+IX7ATxDayfCqs/h8AwaDMY2l9qJRFNEEp5RGVvNMcBLwE9sW4WFzxJtLOcw2KBd+37Cn7AZ8aYqSLyGLDUGDMFq/UxSETWAXnAfcaYA2deHVVdNI4MYeWOMxwFxRFgtQxa9Ydhz0Har3aC+A7WfAEBodblpw6XQauBEFjHvcErVYtV9vLRZKxhLa6016+xtw0s6wD7kdVSPZ6NMeOLLBtgrP1SNYgBDmXlsP/YKeqHncWNY0cAtOxrvYb+B7b9aj/F9B2s/QoC6kDrQVaCaD0IAkPdVgelaqPKJoUYY8zkIuvviMjfPRGQqhnaNAgDYPGWgwxLinXPSR3+0KKP9Rr2H9g2v/Ax13XfgH+IdWmpzUXQuAvUP8c6RilVaZX9P2a/iFwDfGyvjwb0Mo8q05UpTXnku3V8tXyn+5JCUX4OSLjQeg2dCNsWWIlh3RQrSQD4B0PDRGjcGWI7QWxnaNBOe1UrVY7KJoWbgJeB57GuDCzAGvpCKZdCg/w5p2E4mzOOVVz4DMxcu4f2jesSF1XHThAXWK8hE+HAJti9CnavhPSVsOpT+O0t60BHIDTsYCWIgmTRoL32jVDKVtmnj7Zj9Wh2si8fveCJoFTNMLxzYybO2EBmVg4Rddz363z273sZ8/4yAFIfGYQA4cH2+f38IKaN9Uqyb4Hl58OhrZC+ojBZrP0KltlXRP0CoGH7wtZE487QoIP2kVC10hnPvCYi240xVd69WMc+8h2/bMzg2reXMGFER0Z1O/uPyi3vLWXWur0u9xV0mqs0Y+BQWmFroiBZnLCmGcXPH2LaQWM7UcR2hkaJ+iis8lluHfuorPc4i2NVLZDY2BoLadxXq2ndMJyuzaNO6/ivlu+keXSo87iyEgLA9gNZNIs+jUdTRaBegvXqcLm1zRg4vL34pacN38MKe0BgcUDMOcUvPTXqqE88qRrlbJKCDjehyhUVWjjI3RWvLeCr288nuVlhYjDG8Oh36+jRMpqLOjQqduyCzfsZ+9kqAF4a3YX5m/YDEOAQcvIMt/Zuyes/b3aWn/37Xm7omXB2AYtAVHPr1X54QZBwZFfx1sSmH60xmwDED+q3sRNEEkQ2hfDGUDcWwhrqTW3lc8q9fCQiR3H95S9AiDGmyp/308tHvqVgxjaAzk0jGX9Je0a8uoDYiGB2Z550ltv85FAcfoWNz/hx01yeb/rdF9C+cV0AcvPyyTOGcx6yxlDc9MQQ/B2VnTfqLBgDR/eUvvR0dHeJggKhMVaCKEgUzr+xULex9Tc4wkpISnmQWy4fGWPC3ReSqo0CHH5MGNGRcV+tZuWOw4x4dQFAsYQAsC79CB3jrMtNd3+8AoCU5lEs3XaoWLl2sYUfSX+HX7EPcCs7+Xg8OYhYX+x1Y63B+wocP2C1Ko7utl5HdsPRdOtv5g7YsRhOHCx9voA6RZJEo+IJo+BveCNtdagqoT17lMeN6taMH9fv48f1pe8J/PLPvlzwzBwWbtlPx7gIvli2kymr0gHollCPQ1nZbM44DsCq8YMQF7+oJ994LjdOLhxD8ZkZG3hgaDsP1aYcodHWKzap7DI5J4skjfTiyePoHtixxNqWl13iwApaHSFRVnIJCLHucQSEWJ35/Kqg5VTb5OdDfo713yiv4G/J5dwytpdYrvA8OUXWc+Dcm60Omh6kSUFVifMS6vHj+r2EBjpY/chF+BW5VBQTHsST039n6/7jfLxkh3P79efHc8sFLXjr1y30a9uwzMda+57ToNj6pHlbvJMUKiMguPAGd1mMgayDha2Mkn/La3WU5B9sJ4uChFFkOaDkcln7y9nnHwT5eZCfC8b+m59XuK3gZfKLrJfYf1rH5ZYok2OtF3zBFlvOtb6cncv2/vwcF8tF3rPouqvzmnz3fR6KEevf0xFoPf3mCLRfAYV/sz3T76coTQqqSlx/fjw9WkaT2CSi1L6Mo6cAiiWE1EcGUdfue3DfRW0rPH+bhmFs3HeM3m1imLshg9y8/Kq5v+AJIoWtjkYdyy6XcxKO7bESxclMyDkOOSfsVxZkZ1l/nduOF+47dRSO7i2yPwuyj1tf0L5I/Kz+Jo4AqzOjc9m/8FV03RFglfEPhqBwF2UCrCFSnMtFzuvnb43qW/JLu6BcqS/zksuuvvADrfNXA5oUVJUI9PdzmRAAPrrlPP78ZuGkfKseLkwIlTXlzl7k5Ru+W5XO3A0Z7Dly0urtXJMFBENUvPVyl7ycihNKzonC/bmn7C9LR+EXq5RYdy477H1Ft5/tcQVf4j76A6Aa0qSgvK5bfD3n8ml3QrMFB1i/sgoSwfaDWTU/KXiCIwAcEdYTUapW0qSgvM7f4cefz2tG83pn/yWeEGN1JNuccZzzW9Y/6/MpVdtoUlDVwpOXl3Pt/DQ0jrDGK/rXN2u4tntzt5xTqdpEL8SpGqXoI6uZWTlejEQp36RJQdU444ZYTyt1emwmZzrgo1K1lSYFVePcVGQMpIPHS3YCU0qVR5OCqnEC/f3olmA90bT/mCYFpU6HJgVVI907sA1Q2DFOKVU5mhRUjdQ40poMZ9rqkiOXKqXKo0lB1UhxUVZS2H9MWwpKnQ5NCqpGEhGGdmzE6p2Z3g5FKZ+iSUHVWG0b1WXPkZM8/cPvbj2vPuaqajKPJQURCRaRJSKySkTWisij5ZQdKSJGRCqcFUipymoQHgTAa3M3V1DScvB4NvM37S/3S3/KqnQS7p/OviMnyyyjlC/zZEvhFNDPGNMJ6AwMFpHuJQuJSDhwN7C45D6lzsZcYpduAAAbQklEQVSfUpoSaA+ffexULmD9ys/Nyyc/v/QXf/Ljs7j6rcW8uyCtzHP+98c/AFi0tRJzGSjlgzyWFIylYEaIAPvl6ifY48AzgP70Um7l5yfc2a8VAIkPz2DVjsMk3D+dVg9+T4sHprM07SBHT1pDYfy+54jzuF83HSj7nPYwGnd/vIL4cdOIHzeNR6as9WAtlKpaHh0QT0QcwDKgFfCKMWZxif1dgKbGmKki8g9PxqJqp0s6Nea5Wdav+0tfmV9s38jXFwKQUD+UrfuPO7f/uH4vs9btZWD7hs5tWdm5BPs7XD7N9M6CNBrUDeL2Pq08UQWlqpRHbzQbY/KMMZ2BOKCbiCQW7BMRP+B54N6KziMiY0RkqYgszcjI8FzAqsZJqB/K1Lt6FdvWoXHdYutFE8J9F50DwC3vLQXglTmbiB83jfbjZzDw+Z85lJVDYpPixwM888MGZv9eeg5qpXxNlTx9ZIw5DMwFBhfZHA4kAnNFJA3oDkxxdbPZGDPJGJNijEmJiYmpgohVTZLYJIJhSbHO9aevSOJv/VuXKnfvwDbc1rulc/1Edh4TZ2xwrm/OsJLH/64/l81PDmXTE0MY0K6wNXHTO0s5meOj01kqZfPk00cxIhJpL4cAAwDns4HGmExjTH1jTLwxJh5YBAw3xiz1VEyq9nrlz8lseXIoyx4aQGKTCO4Z2Iavbz+/WJlm0XXw8xOev6oTAGvSXfdxaFA3GIef4O/wY+LIpGL7FusNaOXjPHlPIRZ4176v4Ad8Zt87eAxYaoyZ4sH3VqoUPz8hOizIud6lWRRpE4aRmZXDuwvTGN6pMQAt6ocBcKV9z6HPOTFMujaFR79by61FWhIAUaGBpE0YxqZ9Rxnw3Dx9VFX5PI8lBWNMKtDFxfbxZZTv46lYlCpPRJ0A7i5yOSmxSfH5iccObEOgvx9PlDM7XFN7KtHdmZoUlG/THs1KleDwE5Y82N+53qFxxZPYB/k7qB8WRPrhE54MTSmP0zmalXKhQXgwvz04gMwTOTj8pOIDgMaRwaRrS0H5OE0KSpUhJjyImPCgigvaGkeEsCnjWMUFlarG9PKRUm4SGxlM+uETOmCe8mmaFJRykyaRIWRl53HkRK63Q1HqjGlSUMpNYiOsiX126c1m5cM0KSjlJo0jgwHYeSjLy5EodeY0KSjlJi1irE5vRcdSUsrXaFJQyk0iQgKIrBPAtoPaUlC+S5OCUm50OCuHjxZvZ9O+o249b25ePnkuJgYC+HzpDuZs2OfW91O1lyYFpdyoSaR1s3nQ8/Pcds5dh0/Q6sHvafnAdPYdLd057r4vUrlx8m9uez9Vu2lSUMqNJl5pjZqab2Ba6u4Ky9/x4XLnDG6ZWTml9mdl5zLxB+fgwvx76vpi+w+4mPRHqbOhSUEpN0puFuVcvuOj5c65oV35ZWMG01YXJo5Oj83kuVl/kJOX79x26wfL+WZlunN9yqp0Dh7PBmBteiZd//2jO8NXSpOCUu4UHOBg+b8GOte7Pj6r2P5X51ozubX91/dc+/aSUse/+NNG58Q+x0/lMu8Pa6bBusGFI9Ik2+cc9uKvxY49Xk4CUqqyNCko5Wb1QgPZ+tRQAE7l5rNoywHy8w3HTuXyzA/WF/7JHKs1EBLg4MexFxY7ftK8LWzad4ykR2cC1n2KaXdfwJvXFU5K2OqB6aXeV0doVe6gSUEpDxAR3ri2KwCjJi2ixQPTSXx4Rqly9110Dq0ahPPE5Ym8f3M35/YBz/3sfNqoS7NImtarw8D2DfnwL+cBkGvvGzuwDV/e1gOAHdppTrmBjpKqlIe0a1TX5fY1j17Eml2ZpDSPwt9h/S67+rzmAPz++GDa/uuHYuUL9gH0bFWfi5NimWrfxO7eIpoEe6a4LRnH6dfW7dVQtYy2FJTykCZRIaW2nZdQj7Agf7q3iHYmhKKCAxyMHdjGub71qaH0aBldrMxtfQqnBO2WUI+oOgHUCXSQfljnclBnT1sKSnmIw09ImzCM/HyDsdcr4+7+rQkN8qdf2waIlD6mfWxdvrztfFo3tFoIIkKD8CCXfRiUOl2aFJTyML9KJoOibu6VUOY+EaFr86hi2xqEB7PvqPZZUGdPLx8pVQM0qBtEhiYF5QaaFJSqARpHhuisb8otNCkoVQPERYVwKjef7TpCqzpLmhSUqgGaRtUBoPfEuezQxKDOgiYFpWqApLgI5/LqXZlejET5Oo8lBREJFpElIrJKRNaKyKMuyowVkXUikioiP4lIc1fnUkqVLzosiM9vtXo2r9h+yMvRKF/myZbCKaCfMaYT0BkYLCLdS5RZAaQYY5KAL4BnPBiPUjXaufH1AHjzl60VXkLKys4965vS6YdP8PysP8gvY/If5Zs8lhSM5Zi9GmC/TIkyc4wxBZ/eRUCcp+JRqjYIt0dTnbIqvcwyOw5m0X78DBLun84rczaReaL0PA4PfbOaro/PKvMLf8baPZw/YTb//WkjN7yjE/zUJB69pyAiDhFZCewDZhljFpdT/Gbge0/Go1RNt/iB/gBMnLGB+79KdVlm1rq9zuWJMzbQ6dGZnMjOK1bmg0XbOXA8mzHvLyt1/J7Mk/y1yPaC4b1VzeDRpGCMyTPGdMZqAXQTkURX5UTkGiAFmFjG/jEislRElmZk6AdQqbLUCfQnsYk1EF9ZM79lZZeed2G5fR8i4+gp/vnFKuf2zRnHnMt7Mk9yx4fL6f7UT8WObd0g7KzjVtVHlQxzYYw5LCJzgcHAmqL7RGQA8CDQ2xjjskumMWYSMAkgJSVFL2AqVY6pd11A32fnsnX/cbYfyKJZdJ1i+5+d+Ydz+a5+rXhp9iaufqt0I75Ls0j2ZBaOp1QyGQC0iAllx6Es8vPNGQ3noaofTz59FCMikfZyCDAA+L1EmS7AG8BwY8w+T8WiVG1TMKXnzHV7ANh39CS7Dp/g2rcLv/xTHxlUbETWoh6/LJEhiY3YnXmSyfO3Ej9uWrH9s+/tTdqEYdzYM4GTOfk67lIN4smWQizwrog4sJLPZ8aYqSLyGLDUGDMF63JRGPC5PRrkdmPMcA/GpFSt8PmtPejx1Gz+PW09z87c4JzprcBd/VpRNzgAgLn/6EOfZ+cW2z+4QyM27DkKwKPfrSu2r22jcFrEWJeMEqJDAVizK5NGEcGeqIqqYh5LCsaYVKCLi+3jiywP8NT7K1WbxUYUzuVQMiEA/K1/a+dyfP1Q0iYM48jJHEw+pB04Tkx4EP4lLge9fk0yA9s3KjYEeEq8NVrrP75Yxcrxg9xdDeUFOnS2UjXUPwef45wTGqBOoIPpd19AfP1Ql+ULWg6d6kQCEBUayIND2zF/8376tIlhcGJsqWOCAxwAHM7KISs7lzqB+pXi68TXRlVMSUkxS5cu9XYYSvmMvHzDoaxs6ocFeeT8Hy3ezgNfr+b9m7txQesYj7yHOnsisswYk1JROR37SKkazuEnHksIAJd1aYy/n7Bg8wGPvYeqOpoUlFJnpU6gPx3jIliWpmMu1QSaFJRSZ61lTBhL0g56OwzlBpoUlFJnLf3wCQDWpR9x+7lz8/K57/NVTJq3mf/7IpU8HYDPo/RRAaXUWbt30DkseG0B36Wm075x3WL7Xp69kbQDWTx7ZadyzzH205V8tWKXc/3F0V0Y3qkxG/Ye5fNlO53bI+sE8Ma8Lbx1XQoD2jd0b0WUthSUUmcvuZn1GOtrczcX226M4dmZf/DFsp38sfco2w+4HtL7vYVpxRICwN0fr2Dcl6n8sfdose1vzNsCwO0fLXdT9KooTQpKqbMmIow6tylAseG2i84ZPej5eVw4cU6pY3Py8hn/7VqX5/3ktx3c86k1QN8nY4pPx5Kdm88u+7KVch9NCkopt2gRY3WKa/HAdO7/KhVjDDdOLj3XwrFT1iitvSfO4W+frODxqYXDaHx5Ww/SJgxj4sikUsd1bxFNg/Dij9a+8fPmUuXU2dGkoJRyi4KZ3wA+XrKDNbuOsGX/cQBuOD/euW/zvmO8+NNGth3I4tuV6cxca83vsGr8ILo2t84xsmscr/w52XnMF/ZUo3f2awXAyvEDAasntXIvvdGslHKLLs2iiq1f8vKvAFyRHMfDl7Tn0s6NufzVBazffYTnZhUO373nyEkGtGtIRJ0A5zYRYVhSLP3aDsZgnMNnXNu9OSOS4wgL8qdXq/psq2DaUXX6tKWglHKb3x8fzJQ7exbb9vDw9ogIHZtEADDuq9WljmtaL6TUNoCQQEex8ZREhLAgf3sZVu04zModh90VfqX42tBAp0uTglLKbYIDHCTFRRJiD5QHhQPt+TuKf90se2iAc9a2gqG4T0e+/eV82SvzzzTc0/b8rD9IuH86mTX4spUmBaWU271+bVcA1j12UbHtK8cPpEeLaF6/JpnosCBm3nMhn4zpztXdmp32e7w8uvCew9r0TMD6FV/yEVZ3+u9PGwFYtr3m9t7WUVKVUj4rdedhhr9stRTiokLYech6RPWZK5K4PLkJAY7T/907c+0emkeHck6j8GLbl28/xIhXFwAw5sIWPDC03VlGX7UqO0qq3mhWSvmsol/cBQkB4J9fprImPZO+bRsQ6PDD4Sd0bxFd7FhjDPM27mfzvmPc1CuBA8dOcSgrmzHvL6NxRDAL7u+PMYbpq/fQs1U0y7dZA/5FhASwemdm1VTQCzQpKKV8VpC/g2l392LYi7+W2vfewm28t3Cbc/3HsRfSqkFhEkl+fBaH7HsD3RLqMWrSImcfivTMkzz49Wo+XLy92DmjQwPp1bo+y7fX3BFh9Z6CUsqndWgcwcrxAxmR3ASAWy5IcFluw55jAJzIziN+3DRnQgB48aeNzoRQoGRCAGgXW5e4qBB2HDzByZw8d1WhWtGkoJTyeZF1AhlzYQsARiTH0bNV4aWic+15pBds3s+IV+fTbvwPzn2f/bUHLeqHMnPd3jLPfc+ANs7l5GaRtGlotTaK3tDee+QkL/60kazsXNbvPsK/vllT4aOrs9btJX7cNDbs8dyN8TOhl4+UUjVC20Z1SZswDIB3buzGhj1HSWwSgTGGhPunl/rl/9Et59EtoZ5znukCo85tyie/7QCsfhfBAQ5uOD+eb1ft4squTdmcYbU4fly313mTu0DRTnk9W9VncGIjZq7dw5j3l9G6QRivXp1Mazup3PKe9cDM+4vS+GCRFdvShwa4nCVv8vytPPrdOl75czLDkkrPle1O2lJQStU4AQ4/Eu3OciLi7A9RVEFnutHnWY/DzrrnQtImDGPCFUmsffQitj411JkwIuoEcF2PeEICHSTUt8Z4enH2pnJjuPWDZcz7I4Mx7y8DYOO+Ywx8fh5TVqUXa0UUJASAq95Y6PJcn9pJKjossOLKnyVNCkqpGm/q3b2cyy1jQrmgdX3C7U5113ZvTtqEYc5f8AChQf6IiMtzhQb5E+Aova9glNiirvvfklLb7v54BX//dKXLc2/OOM6BY6eKbes9cQ6/25eYSj5B5Ql6+UgpVeMF+Tv46vbzmZ66mweHtSvzC7+yXriqC3d8tNw5EZAxBhHhn4Pb8uum/TSOCGbk64W/+lc/Mojs3Hy6/vtHAL5dmV7qnD1aRLNwywG+XrGLv1zQgrx8w4rth9hmz0Hx+KUdzirmytKkoJSqFZKbRZFcYtC+MzUsKZa2sb1paQ/PUZBk6oUGMrxTYwAS6oeydf9x6ocFOVsl88f1o+eE2QAMaNeAN69LYf3uozSJDMHhEBIfnsG/p63HT4THigwpPvmGc+nbtoFbYq+Ix3o0i0gwMA8Iwko+XxhjHi5RJgh4D+gKHACuMsaklXde7dGslPIF+fmG7Lz8Ujey52zYx/rdR7i9T6tSx8SPm+byXFueHIqf39m1birbo9mT9xROAf2MMZ2AzsBgEeleoszNwCFjTCvgeeBpD8ajlFJVxs9PSiUEgL7nNHCZEADevalbsfX/G9yWFf8aeNYJ4XR47PKRsZogx+zVAPtVsllyKfCIvfwF8LKIiPG1AZmUUsoNereJYcmD/dlx8AQz1u7h5l4JBPpX7fNAHr2nICIOYBnQCnjFGLO4RJEmwA4AY0yuiGQC0cB+T8allFLVVYPwYBqEB9O1uXvuf5wuj6YgY0yeMaYzEAd0E5HEEkVctYlKtRJEZIyILBWRpRkZGZ4IVSmlFFXUT8EYcxiYCwwusWsn0BRARPyBCKDUQOXGmEnGmBRjTEpMTIyHo1VKqdrLY0lBRGJEJNJeDgEGAL+XKDYFuN5eHgnM1vsJSinlPZ68pxALvGvfV/ADPjPGTBWRx4ClxpgpwNvA+yKyCauFMMqD8SillKqAJ58+SgW6uNg+vsjySeBKT8WglFLq9OjYR0oppZw0KSillHLSpKCUUsrJY2MfeYqIZADbKizoWn1qTse4mlQXqFn10bpUT7W9Ls2NMRU+0+9zSeFsiMjSygwI5QtqUl2gZtVH61I9aV0qRy8fKaWUctKkoJRSyqm2JYVJ3g7AjWpSXaBm1UfrUj1pXSqhVt1TUEopVb7a1lJQSilVjlqTFERksIhsEJFNIjLO2/G4IiL/E5F9IrKmyLZ6IjJLRDbaf6Ps7SIiL9r1SRWR5CLHXG+X3ygi17t6ryqoS1MRmSMi60VkrYj8zVfrIyLBIrJERFbZdXnU3p4gIovtuD4VkUB7e5C9vsneH1/kXPfb2zeIyEVVXZcicThEZIWITLXXfbIuIpImIqtFZKWILLW3+dxnzI4hUkS+EJHf7f9venilLsaYGv8CHMBmoAUQCKwC2ns7LhdxXggkA2uKbHsGGGcvjwOetpeHAt9jzUnRHVhsb68HbLH/RtnLUV6oSyyQbC+HA38A7X2xPnZMYfZyALDYjvEzYJS9/XXgNnv5duB1e3kU8Km93N7+7AUBCfZn0uGlz9pY4CNgqr3uk3UB0oD6Jbb53GfMjuNd4C/2ciAQ6Y26VPmH0RsvoAcwo8j6/cD93o6rjFjjKZ4UNgCx9nIssMFefgMYXbIcMBp4o8j2YuW8WK9vgYG+Xh+gDrAcOA+r85B/yc8YMAPoYS/72+Wk5OeuaLkqrkMc8BPQD5hqx+ardUmjdFLwuc8YUBfYin2f15t1qS2Xj5zTftp22tt8QUNjzG4A+28De3tZdap2dbUvOXTB+oXtk/WxL7esBPYBs7B+GR82xuS6iKvYNLNAwTSz1aIuwAvAP4F8ez0a362LAWaKyDIRGWNv88XPWAsgA5hsX9Z7S0RC8UJdaktSqNS0nz6mrDpVq7qKSBjwJfB3Y8yR8oq62FZt6mNKTC0LtHNVzP5bbesiIhcD+4wxy4pudlG02tfF1tMYkwwMAe4QkQvLKVud6+KPden4NWNMF+A41uWisnisLrUlKTin/bTFAeleiuV07RWRWAD77z57e1l1qjZ1FZEArITwoTHmK3uzz9YHik0t2x2IFGsa2ZJxlTXNbHWoS09guIikAZ9gXUJ6Ad+sC8aYdPvvPuBrrITti5+xncBOY8xie/0LrCRR5XWpLUnhN6C1/YRFINYNsylejqmyik5Zej3WtfmC7dfZTyF0BzLt5uUMYJCIRNlPKgyyt1UpERGsmfXWG2OeK7LL5+ojrqeWXQ/MwZpGFkrXxdU0s1OAUfYTPQlAa2BJ1dTCYoy53xgTZ4yJx/r/YLYx5mp8sC4iEioi4QXLWJ+NNfjgZ8wYswfYISLn2Jv6A+vwRl2q+saQt15Yd+v/wLoW/KC34ykjxo+B3UAOVsa/Gev67U/ARvtvPbusAK/Y9VkNpBQ5z03AJvt1o5fq0gur2ZoKrLRfQ32xPkASsMKuyxpgvL29BdYX4SbgcyDI3h5sr2+y97cocq4H7TpuAIZ4+fPWh8Knj3yuLnbMq+zX2oL/r33xM2bH0BlYan/OvsF6eqjK66I9mpVSSjnVlstHSimlKkGTglJKKSdNCkoppZw0KSillHLSpKCUUspJk4KqdkQkzx71cpWILBeR8ysoHykit1fivHNFpEbM0esuIvKOiIysuKSqLTQpqOrohDGmszGmE9bAa09VUD4SazTPaqlIT2Glqj1NCqq6qwscAmscJRH5yW49rBaRS+0yE4CWdutiol32n3aZVSIyocj5rhRrboQ/ROQCu6xDRCaKyG/22PR/tbfHisg8+7xrCsoXJdZ4/k/b51wiIq3s7e+IyHMiMgd4Wqxx8b+xz79IRJKK1GmyHWuqiFxhbx8kIgvtun5ujyGFiEwQkXV22WftbVfa8a0SkXkV1ElE5GX7HNMoHGBNKYs3e1TqS1+uXkAeVg/o37FG5exqb/cH6trL9bF6bAqlhxsfAiwA6tjrBb1A5wL/sZeHAj/ay2OAh+zlIKxepQnAvRT2knUA4S5iTStS5joKewi/gzUstcNefwl42F7uB6y0l58GXihyvii7bvOAUHvb/wHjscbI30DhNLqR9t/VQJMS28qq0wisUV4dQGPgMDDS2//N9VV9XtqsVdXRCWONSIqI9ADeE5FErATwpFgjYeZjDQnc0MXxA4DJxpgsAGPMwSL7CgbmW4aVTMAaHyapyLX1CKyxfH4D/ifWwH7fGGNWlhHvx0X+Pl9k++fGmDx7uRdwhR3PbBGJFpEIO9ZRBQcYYw6JNZJpe2C+NYQUgcBC4AhwEnjL/pU/1T5sPvCOiHxWpH5l1elC4GM7rnQRmV1GnVQtpUlBVWvGmIUiUh+Iwfp1H4PVcsgRa6TPYBeHCWUPF3zK/ptH4edfgLuMMaUGDrMT0DDgfRGZaIx5z1WYZSwfLxGTq+NcxSrALGPMaBfxdMMaLG0UcCfQzxhzq4icZ8e5UkQ6l1UnERnq4v2UctJ7CqpaE5G2WJc6DmD92t1nJ4S+QHO72FGsKT8LzARuEpE69jnqVfA2M4Db7BYBItLGHoGzuf1+b2KN+JpcxvFXFfm7sIwy84Cr7fP3AfYba36JmVhf7gX1jQIWAT2L3J+oY8cUBkQYY6YDf8caQA0RaWmMWWyMGY81M1rTsupkxzHKvucQC/St4N9G1TLaUlDVUYhYs5yB9Yv3emNMnoh8CHwn1gTtBfccMMYcEJH5IrIG+N4Yc5/9a3mpiGQD04EHynm/t7AuJS0X63pNBnAZ1iii94lIDnAM656BK0EishjrR1apX/e2R7Bm1UoFsigcDvnfwCt27HnAo8aYr0TkBuBjEQmyyz2Elfy+FZFg+9/lHnvfRBFpbW/7CWvU0NQy6vQ11j2N1VijBv9czr+LqoV0lFSlzoJ9CSvFGLPf27Eo5Q56+UgppZSTthSUUko5aUtBKaWUkyYFpZRSTpoUlFJKOWlSUEop5aRJQSmllJMmBaWUUk7/D9AkRPiA3on/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.recorder.plot_losses()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.save('fine_tuned')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.save_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fine tuned models *fit_head*, *fine_tuned* and *fine_tuned_enc* are saved in default `path/models` folder\n",
    "\n",
    "Numeralized text and the integer to string dictionary are saved in user defined `path/tmp_lm` folder "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = TextLMDataBunch.load(path, 'tmp_lm', bs=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def precision(log_preds, targs, thresh=0.5, epsilon=1e-8):\n",
    "    pred_pos = (log_preds > thresh).float()\n",
    "    tpos = torch.mul((targs == pred_pos).float(), targs.float())\n",
    "    return (tpos.sum()/(pred_pos.sum() + epsilon))#.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recall(log_preds, targs, thresh=0.5, epsilon=1e-8):\n",
    "    pred_pos = (log_preds > thresh).float()\n",
    "    tpos = torch.mul((targs == pred_pos).float(), targs.float())\n",
    "    return (tpos.sum()/(targs.sum() + epsilon))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas = TextClasDataBunch.from_df(path, train_df=df_trn, valid_df=df_val, \n",
    "                                  vocab=data_lm.vocab, \n",
    "                                  text_cols='Narrative', \n",
    "                                  label_cols='Contributing Factors / Situations',\n",
    "                                  label_delim='|',\n",
    "                                  bs=bs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas.save('tmp_clas')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = text_classifier_learner(data_clas, drop_mult=0.5)\n",
    "learn.metrics = [accuracy_thresh, precision, recall]\n",
    "learn.load_encoder('fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 00:36\n",
      "epoch  train_loss  valid_loss  accuracy_thresh  precision  recall  \n",
      "1      0.239528    0.219730    0.907879         0.771040   0.441893  (00:36)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze()\n",
    "learn.fit_one_cycle(1, 3e-2, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('first_factors')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('first_factors')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time: 01:25\n",
      "epoch  train_loss  valid_loss  accuracy_thresh  precision  recall  \n",
      "1      0.226213    0.217303    0.909732         0.756436   0.501886  (00:42)\n",
      "2      0.216116    0.214490    0.909625         0.739172   0.519424  (00:42)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.freeze_to(-2)\n",
    "learn.fit_one_cycle(2, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7), wd=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.save('second_factors')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.load('second_factors')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7), wd=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/krasin/anaconda2/envs/fastai.1/lib/python3.6/site-packages/torch/nn/functional.py:1126: UserWarning: nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\n",
      "  warnings.warn(\"nn.functional.sigmoid is deprecated. Use torch.sigmoid instead.\")\n"
     ]
    }
   ],
   "source": [
    "# Predict on the validation dataset\n",
    "y_pred, y_true =  learn.get_preds()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Classifier models *first_factors* and *second_factors* are saved in default `path/models` folder\n",
    "\n",
    "Numeralized text, labels and the integer to string dictionary are saved in user defined `path/tmp_clas` folder "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score, recall_score, accuracy_score, f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6599488864549106"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_true, y_pred>0.35, average='micro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = y_true.numpy()\n",
    "scores = y_pred.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2935, 16), (2935, 16))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores.shape, y_true.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              precision    recall  f1-score   support\n",
      "\n",
      "                         Equipment / Tooling       0.00      0.00      0.00        56\n",
      "                                    Aircraft       0.79      0.92      0.85      1635\n",
      "    ATC Equipment / Nav Facility / Buildings       0.00      0.00      0.00        60\n",
      "           Environment - Non Weather Related       0.29      0.18      0.23       238\n",
      "                                     Weather       0.61      0.60      0.60       270\n",
      "                               Human Factors       0.73      0.88      0.80      1672\n",
      "                                    Staffing       0.50      0.02      0.04        52\n",
      "                               Logbook Entry       0.29      0.08      0.13        59\n",
      "Incorrect / Not Installed / Unavailable Part       0.23      0.15      0.18        71\n",
      "                          Airspace Structure       0.50      0.05      0.10        92\n",
      "                                   Procedure       0.37      0.88      0.52       754\n",
      "                                     Manuals       0.50      0.38      0.43       163\n",
      "                                     Airport       0.53      0.44      0.48       212\n",
      "                                         MEL       0.40      0.32      0.36        84\n",
      "                        Chart Or Publication       0.49      0.58      0.53       259\n",
      "                              Company Policy       0.64      0.52      0.57       624\n",
      "\n",
      "                                 avg / total       0.61      0.72      0.64      6301\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics = classification_report(y_true, scores>0.35, target_names=data_clas.valid_ds.classes)\n",
    "print(metrics)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}