train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEBUG = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "sys.path = [\n",
    "    './efficientnet/EfficientNet-PyTorch/EfficientNet-PyTorch-master',\n",
    "] + sys.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./efficientnet/EfficientNet-PyTorch/EfficientNet-PyTorch-master',\n",
       " '/home/under/ddw02141/PANDA',\n",
       " '/home/under/ddw02141/miniconda3/envs/sangwon/lib/python37.zip',\n",
       " '/home/under/ddw02141/miniconda3/envs/sangwon/lib/python3.7',\n",
       " '/home/under/ddw02141/miniconda3/envs/sangwon/lib/python3.7/lib-dynload',\n",
       " '',\n",
       " '/home/under/ddw02141/.local/lib/python3.7/site-packages',\n",
       " '/home/under/ddw02141/miniconda3/envs/sangwon/lib/python3.7/site-packages',\n",
       " '/home/under/ddw02141/.local/lib/python3.7/site-packages/IPython/extensions',\n",
       " '/home/under/ddw02141/.ipython']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sys.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import skimage.io\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import cv2\n",
    "import PIL.Image\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torch.utils.data.sampler import SubsetRandomSampler, RandomSampler, SequentialSampler\n",
    "from warmup_scheduler import GradualWarmupScheduler\n",
    "from efficientnet_pytorch import model as enet\n",
    "import albumentations\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import os\n",
    "import warnings \n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = './prostate-cancer-grade-assessment/'\n",
    "df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))\n",
    "image_folder = os.path.join(data_dir, 'train_images/')\n",
    "\n",
    "kernel_type = 'efficientnet_b1_36x256x256_final'\n",
    "\n",
    "enet_type = 'efficientnet-b1'\n",
    "fold = 0\n",
    "tile_size = 256\n",
    "image_size = 256\n",
    "n_tiles = 36\n",
    "batch_size =8\n",
    "num_workers = 16\n",
    "out_dim = 5\n",
    "init_lr = 3e-4\n",
    "warmup_factor = 10\n",
    "\n",
    "warmup_epo = 1\n",
    "n_epochs = 1 if DEBUG else 50\n",
    "df_train = df_train.sample(100).reset_index(drop=True) if DEBUG else df_train\n",
    "\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"2,3,4,5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>data_provider</th>\n",
       "      <th>isup_grade</th>\n",
       "      <th>gleason_score</th>\n",
       "      <th>fold</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0005f7aaab2800f6170c399693a96917</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000920ad0b612851f8e01bcc880d9b3d</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0018ae58b01bdadc8e347995b69f99aa</td>\n",
       "      <td>radboud</td>\n",
       "      <td>4</td>\n",
       "      <td>4+4</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>001c62abd11fa4b57bf7a6c603a11bb9</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>4</td>\n",
       "      <td>4+4</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>001d865e65ef5d2579c190a0e0350d8f</td>\n",
       "      <td>karolinska</td>\n",
       "      <td>0</td>\n",
       "      <td>0+0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           image_id data_provider  isup_grade gleason_score  \\\n",
       "0  0005f7aaab2800f6170c399693a96917    karolinska           0           0+0   \n",
       "1  000920ad0b612851f8e01bcc880d9b3d    karolinska           0           0+0   \n",
       "2  0018ae58b01bdadc8e347995b69f99aa       radboud           4           4+4   \n",
       "3  001c62abd11fa4b57bf7a6c603a11bb9    karolinska           4           4+4   \n",
       "4  001d865e65ef5d2579c190a0e0350d8f    karolinska           0           0+0   \n",
       "\n",
       "   fold  \n",
       "0     4  \n",
       "1     0  \n",
       "2     3  \n",
       "3     4  \n",
       "4     4  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skf = StratifiedKFold(5, shuffle=True, random_state=42)\n",
    "df_train['fold'] = -1\n",
    "for i, (train_idx, valid_idx) in enumerate(skf.split(df_train, df_train['isup_grade'])):\n",
    "    df_train.loc[valid_idx, 'fold'] = i\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pretrained_model = {\n",
    "    'efficientnet-b1': './efficientnet/efficientnet-b1-dbc7070a.pth'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class enetv2(nn.Module):\n",
    "    def __init__(self, backbone, out_dim):\n",
    "        super(enetv2, self).__init__()\n",
    "        self.enet = enet.EfficientNet.from_name(backbone)\n",
    "        self.enet.load_state_dict(torch.load(pretrained_model[backbone]))\n",
    "\n",
    "        self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)\n",
    "        self.enet._fc = nn.Identity()\n",
    "\n",
    "    def extract(self, x):\n",
    "        return self.enet(x)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.extract(x)\n",
    "        x = self.myfc(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tiles(img, mode=0):\n",
    "        result = []\n",
    "        h, w, c = img.shape\n",
    "        pad_h = (tile_size - h % tile_size) % tile_size + ((tile_size * mode) // 2)\n",
    "        pad_w = (tile_size - w % tile_size) % tile_size + ((tile_size * mode) // 2)\n",
    "\n",
    "        img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)\n",
    "        img3 = img2.reshape(\n",
    "            img2.shape[0] // tile_size,\n",
    "            tile_size,\n",
    "            img2.shape[1] // tile_size,\n",
    "            tile_size,\n",
    "            3\n",
    "        )\n",
    "\n",
    "        img3 = img3.transpose(0,2,1,3,4).reshape(-1, tile_size, tile_size,3)\n",
    "        n_tiles_with_info = (img3.reshape(img3.shape[0],-1).sum(1) < tile_size ** 2 * 3 * 255).sum()\n",
    "        if len(img3) < n_tiles:\n",
    "            img3 = np.pad(img3,[[0,n_tiles-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)\n",
    "        idxs = np.argsort(img3.reshape(img3.shape[0],-1).sum(-1))[:n_tiles]\n",
    "        img3 = img3[idxs]\n",
    "        for i in range(len(img3)):\n",
    "            result.append({'img':img3[i], 'idx':i})\n",
    "        return result, n_tiles_with_info >= n_tiles\n",
    "\n",
    "\n",
    "class PANDADataset(Dataset):\n",
    "    def __init__(self,\n",
    "                 df,\n",
    "                 image_size,\n",
    "                 n_tiles=n_tiles,\n",
    "                 tile_mode=0,\n",
    "                 rand=False,\n",
    "                 transform=None,\n",
    "                ):\n",
    "\n",
    "        self.df = df.reset_index(drop=True)\n",
    "        self.image_size = image_size\n",
    "        self.n_tiles = n_tiles\n",
    "        self.tile_mode = tile_mode\n",
    "        self.rand = rand\n",
    "        self.transform = transform\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.df.shape[0]\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        row = self.df.iloc[index]\n",
    "        img_id = row.image_id\n",
    "        \n",
    "        tiff_file = os.path.join(image_folder, f'{img_id}.tiff')\n",
    "        image = skimage.io.MultiImage(tiff_file)[-2]\n",
    "        tiles, OK = get_tiles(image, self.tile_mode)\n",
    "\n",
    "        if self.rand:\n",
    "            idxes = np.random.choice(list(range(self.n_tiles)), self.n_tiles, replace=False)\n",
    "        else:\n",
    "            idxes = list(range(self.n_tiles))\n",
    "\n",
    "        n_row_tiles = int(np.sqrt(self.n_tiles))\n",
    "        images = np.zeros((image_size * n_row_tiles, image_size * n_row_tiles, 3))\n",
    "        for h in range(n_row_tiles):\n",
    "            for w in range(n_row_tiles):\n",
    "                i = h * n_row_tiles + w\n",
    "    \n",
    "                if len(tiles) > idxes[i]:\n",
    "                    this_img = tiles[idxes[i]]['img']\n",
    "                else:\n",
    "                    this_img = np.ones((self.image_size, self.image_size, 3)).astype(np.uint8) * 255\n",
    "                this_img = 255 - this_img\n",
    "                if self.transform is not None:\n",
    "                    this_img = self.transform(image=this_img)['image']\n",
    "                h1 = h * image_size\n",
    "                w1 = w * image_size\n",
    "                images[h1:h1+image_size, w1:w1+image_size] = this_img\n",
    "\n",
    "        if self.transform is not None:\n",
    "            images = self.transform(image=images)['image']\n",
    "        images = images.astype(np.float32)\n",
    "        images /= 255\n",
    "        images = images.transpose(2, 0, 1)\n",
    "\n",
    "        label = np.zeros(5).astype(np.float32)\n",
    "        label[:row.isup_grade] = 1.\n",
    "        return torch.tensor(images), torch.tensor(label)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Augmentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms_train = albumentations.Compose([\n",
    "    albumentations.Transpose(p=0.5),\n",
    "    albumentations.VerticalFlip(p=0.5),\n",
    "    albumentations.HorizontalFlip(p=0.5),\n",
    "])\n",
    "transforms_val = albumentations.Compose([])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dataset_show = PANDADataset(df_train, image_size, n_tiles, 0, transform=transforms_train)\n",
    "from pylab import rcParams\n",
    "rcParams['figure.figsize'] = 20,10\n",
    "for i in range(2):\n",
    "    f, axarr = plt.subplots(1,5)\n",
    "    for p in range(5):\n",
    "        idx = np.random.randint(0, len(dataset_show))\n",
    "        img, label = dataset_show[idx]\n",
    "        axarr[p].imshow(1. - img.transpose(0, 1).transpose(1,2).squeeze())\n",
    "        axarr[p].set_title(str(sum(label)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.BCEWithLogitsLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train & Val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_epoch(loader, optimizer):\n",
    "\n",
    "    model.train()\n",
    "    train_loss = []\n",
    "    bar = tqdm(loader)\n",
    "    for (data, target) in bar:\n",
    "        \n",
    "        data, target = data.to(device), target.to(device)\n",
    "        loss_func = criterion\n",
    "        optimizer.zero_grad()\n",
    "        logits = model(data)\n",
    "        loss = loss_func(logits, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        loss_np = loss.detach().cpu().numpy()\n",
    "        train_loss.append(loss_np)\n",
    "        smooth_loss = sum(train_loss[-100:]) / min(len(train_loss), 100)\n",
    "        bar.set_description('loss: %.5f, smth: %.5f' % (loss_np, smooth_loss))\n",
    "    return train_loss\n",
    "\n",
    "\n",
    "def val_epoch(loader, get_output=False):\n",
    "\n",
    "    model.eval()\n",
    "    val_loss = []\n",
    "    LOGITS = []\n",
    "    PREDS = []\n",
    "    TARGETS = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for (data, target) in tqdm(loader):\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            logits = model(data)\n",
    "\n",
    "            loss = criterion(logits, target)\n",
    "\n",
    "            pred = logits.sigmoid().sum(1).detach().round()\n",
    "            LOGITS.append(logits)\n",
    "            PREDS.append(pred)\n",
    "            TARGETS.append(target.sum(1))\n",
    "\n",
    "            val_loss.append(loss.detach().cpu().numpy())\n",
    "        val_loss = np.mean(val_loss)\n",
    "\n",
    "    LOGITS = torch.cat(LOGITS).cpu().numpy()\n",
    "    PREDS = torch.cat(PREDS).cpu().numpy()\n",
    "    TARGETS = torch.cat(TARGETS).cpu().numpy()\n",
    "    acc = (PREDS == TARGETS).mean() * 100.\n",
    "    \n",
    "    qwk = cohen_kappa_score(PREDS, TARGETS, weights='quadratic')\n",
    "    qwk_k = cohen_kappa_score(PREDS[df_valid['data_provider'] == 'karolinska'], df_valid[df_valid['data_provider'] == 'karolinska'].isup_grade.values, weights='quadratic')\n",
    "    qwk_r = cohen_kappa_score(PREDS[df_valid['data_provider'] == 'radboud'], df_valid[df_valid['data_provider'] == 'radboud'].isup_grade.values, weights='quadratic')\n",
    "    print('qwk', qwk, 'qwk_k', qwk_k, 'qwk_r', qwk_r)\n",
    "\n",
    "    if get_output:\n",
    "        return LOGITS\n",
    "    else:\n",
    "        return val_loss, acc, qwk\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Dataloader & Model & Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8492 2124\n"
     ]
    }
   ],
   "source": [
    "train_idx = np.where((df_train['fold'] != fold))[0]\n",
    "valid_idx = np.where((df_train['fold'] == fold))[0]\n",
    "\n",
    "df_this  = df_train.loc[train_idx]\n",
    "df_valid = df_train.loc[valid_idx]\n",
    "\n",
    "dataset_train = PANDADataset(df_this , image_size, n_tiles, transform=transforms_train,  rand=True)\n",
    "dataset_valid = PANDADataset(df_valid, image_size, n_tiles, transform=transforms_val,  rand=True)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=batch_size, sampler=RandomSampler(dataset_train), num_workers=num_workers)\n",
    "valid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=batch_size, sampler=SequentialSampler(dataset_valid), num_workers=num_workers)\n",
    "\n",
    "model = enetv2(enet_type, out_dim=out_dim)\n",
    "model = nn.DataParallel(model, device_ids = [0,1,2,3])\n",
    "model = model.to(device)\n",
    "\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=init_lr/warmup_factor)\n",
    "scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, n_epochs-warmup_epo)\n",
    "scheduler = GradualWarmupScheduler(optimizer, multiplier=warmup_factor, total_epoch=warmup_epo, after_scheduler=scheduler_cosine)\n",
    "\n",
    "print(len(dataset_train), len(dataset_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'checkpoints' not in os.listdir():\n",
    "    !mkdir checkpoints\n",
    "\n",
    "if 'logs' not in os.listdir():\n",
    "    !mkdir logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fri Jul 31 01:12:31 2020 Epoch: 1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6aee2d825fc9475e8456bd6e090529e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2525240442542688c8a7c5916dac06b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.6852177459427153 qwk_k 0.6375299046293832 qwk_r 0.6319657437028228\n",
      "Fri Jul 31 01:22:48 2020 Epoch 1, lr: 0.0000300, train loss: 0.48728, val loss: 0.35194, acc: 32.67420, qwk: 0.68522\n",
      "score2 (0.000000 --> 0.685218).  Saving model ...\n",
      "Fri Jul 31 01:22:48 2020 Epoch: 2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c6feee78ce241848448cd0595a48abf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36d3f63cc789486a8361565e99865e2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.7454934308877695 qwk_k 0.7113386956404304 qwk_r 0.6863973646286833\n",
      "Fri Jul 31 01:32:59 2020 Epoch 2, lr: 0.0003000, train loss: 0.36829, val loss: 0.31629, acc: 36.62900, qwk: 0.74549\n",
      "score2 (0.685218 --> 0.745493).  Saving model ...\n",
      "Fri Jul 31 01:33:00 2020 Epoch: 3\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68a1ece304754a45885d0eaeadcb53b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c16394a917bc49a69991f46f9cb6ba35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8383836477378939 qwk_k 0.8367210841928616 qwk_r 0.7976045761153205\n",
      "Fri Jul 31 01:43:09 2020 Epoch 3, lr: 0.0000300, train loss: 0.27242, val loss: 0.24789, acc: 56.82674, qwk: 0.83838\n",
      "score2 (0.745493 --> 0.838384).  Saving model ...\n",
      "Fri Jul 31 01:43:09 2020 Epoch: 4\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5f3b54e50e54ae2b47381f133f10dd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c37fa26de4a54a56aa8cbe76cb0fdb10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8017132308410737 qwk_k 0.7986607132275255 qwk_r 0.7575776722247339\n",
      "Fri Jul 31 01:53:26 2020 Epoch 4, lr: 0.0002988, train loss: 0.28789, val loss: 0.28162, acc: 61.95857, qwk: 0.80171\n",
      "Fri Jul 31 01:53:26 2020 Epoch: 5\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f40f90a85ba9495882f0d29ca2c9ca4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d583fad9c96c4b3dbf199b7bf5a25d39",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8367194497719429 qwk_k 0.8484122727699887 qwk_r 0.7849960328629897\n",
      "Fri Jul 31 02:03:43 2020 Epoch 5, lr: 0.0002972, train loss: 0.26203, val loss: 0.25778, acc: 57.25047, qwk: 0.83672\n",
      "Fri Jul 31 02:03:43 2020 Epoch: 6\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "395c7e34ca0e4b9b99fa558196bd40bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7fc5cd376b194d4abfc21c99dff37d7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8056864283788165 qwk_k 0.792594428775657 qwk_r 0.7522288278035356\n",
      "Fri Jul 31 02:13:58 2020 Epoch 6, lr: 0.0002951, train loss: 0.24874, val loss: 0.28150, acc: 48.44633, qwk: 0.80569\n",
      "Fri Jul 31 02:13:58 2020 Epoch: 7\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40d6d71d4b3c4812b7884b5ba15ab866",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5bf807735a4940e6ab3103a5523b5b0b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8483630127833808 qwk_k 0.8698452326924156 qwk_r 0.7910578559693895\n",
      "Fri Jul 31 02:24:14 2020 Epoch 7, lr: 0.0002924, train loss: 0.23590, val loss: 0.24244, acc: 59.51036, qwk: 0.84836\n",
      "score2 (0.838384 --> 0.848363).  Saving model ...\n",
      "Fri Jul 31 02:24:14 2020 Epoch: 8\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f150ee2df38b44c68dacd498b760abee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98c77e5fb74645f999f6fc6097eaf8e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8448285847751659 qwk_k 0.8404331581142184 qwk_r 0.8010380207065689\n",
      "Fri Jul 31 02:34:28 2020 Epoch 8, lr: 0.0002890, train loss: 0.22646, val loss: 0.24412, acc: 61.39360, qwk: 0.84483\n",
      "Fri Jul 31 02:34:28 2020 Epoch: 9\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94a4a30457934587b3fcabc8112ba57c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9790202c9e9643239d0c28de342de6d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8563584868146222 qwk_k 0.8676280014804439 qwk_r 0.8087361066340405\n",
      "Fri Jul 31 02:44:41 2020 Epoch 9, lr: 0.0002851, train loss: 0.21952, val loss: 0.23171, acc: 62.00565, qwk: 0.85636\n",
      "score2 (0.848363 --> 0.856358).  Saving model ...\n",
      "Fri Jul 31 02:44:41 2020 Epoch: 10\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7bd6f34617a44a59fc985a7052cfc7c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a22f34a6e2a416cb8f1470b10240e47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8473610666479467 qwk_k 0.8661214419208741 qwk_r 0.7960761783399599\n",
      "Fri Jul 31 02:54:56 2020 Epoch 10, lr: 0.0002807, train loss: 0.21155, val loss: 0.25910, acc: 61.58192, qwk: 0.84736\n",
      "Fri Jul 31 02:54:56 2020 Epoch: 11\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0cda609b6b7e4bb5aa8c6595ceaf1ee1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86126302fa5049ba91e54e10e2007bd3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8571437203257938 qwk_k 0.8824983013121552 qwk_r 0.804686095688199\n",
      "Fri Jul 31 03:05:09 2020 Epoch 11, lr: 0.0002757, train loss: 0.20425, val loss: 0.24255, acc: 62.09981, qwk: 0.85714\n",
      "score2 (0.856358 --> 0.857144).  Saving model ...\n",
      "Fri Jul 31 03:05:09 2020 Epoch: 12\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d405936aea94bdf97fc7f695e724667",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81c38656bf57431f9fca0e2530ab6d04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8628680153797403 qwk_k 0.8874337881315255 qwk_r 0.8065448243913088\n",
      "Fri Jul 31 03:15:23 2020 Epoch 12, lr: 0.0002702, train loss: 0.19756, val loss: 0.22886, acc: 61.39360, qwk: 0.86287\n",
      "score2 (0.857144 --> 0.862868).  Saving model ...\n",
      "Fri Jul 31 03:15:23 2020 Epoch: 13\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79ff29c494c949df845471f32b49df2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bcea1cdb6dda483a8393602727889c0a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8660315288802635 qwk_k 0.8949300896459076 qwk_r 0.8112613862418382\n",
      "Fri Jul 31 03:25:38 2020 Epoch 13, lr: 0.0002642, train loss: 0.19369, val loss: 0.24613, acc: 64.87759, qwk: 0.86603\n",
      "score2 (0.862868 --> 0.866032).  Saving model ...\n",
      "Fri Jul 31 03:25:38 2020 Epoch: 14\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c8d81fcebd642d7aabc45829c5f5516",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07e2dd8c587e41eb97c9c320fa0a1f9e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8496104441546302 qwk_k 0.872832049044096 qwk_r 0.7974219786078718\n",
      "Fri Jul 31 03:35:51 2020 Epoch 14, lr: 0.0002578, train loss: 0.18799, val loss: 0.24163, acc: 66.05461, qwk: 0.84961\n",
      "Fri Jul 31 03:35:51 2020 Epoch: 15\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ab4ad62109f4e50aad0ffaa5469c53b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "670aa7a0732848769325a2019ba311b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8760494931961256 qwk_k 0.8862365886863101 qwk_r 0.8375849609766433\n",
      "Fri Jul 31 06:19:17 2020 Epoch 30, lr: 0.0001166, train loss: 0.09263, val loss: 0.26277, acc: 68.26742, qwk: 0.87605\n",
      "Fri Jul 31 06:19:17 2020 Epoch: 31\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ad6505cfcb249f0b4ffcd0323a83457",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18ef056109084f1ea647dab2f05812a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8739362330611272 qwk_k 0.8904644182283193 qwk_r 0.8304523452455994\n",
      "Fri Jul 31 06:29:35 2020 Epoch 31, lr: 0.0001073, train loss: 0.09059, val loss: 0.27439, acc: 68.17326, qwk: 0.87394\n",
      "Fri Jul 31 06:29:35 2020 Epoch: 32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7e60edcb9604a259b4401954595fedc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb944fd4ab3644268f3de1a4c70e1e6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8777090818767095 qwk_k 0.8939520960660595 qwk_r 0.8315541715544507\n",
      "Fri Jul 31 06:39:51 2020 Epoch 32, lr: 0.0000982, train loss: 0.08415, val loss: 0.25896, acc: 67.23164, qwk: 0.87771\n",
      "score2 (0.876166 --> 0.877709).  Saving model ...\n",
      "Fri Jul 31 06:39:51 2020 Epoch: 33\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "626b0d680e224c5cba1cf225f0546434",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34ad7fdda8fe490287be6bac9f761ac3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8652786022024671 qwk_k 0.8821859162048821 qwk_r 0.8213188385972838\n",
      "Fri Jul 31 06:50:07 2020 Epoch 33, lr: 0.0000893, train loss: 0.07885, val loss: 0.29701, acc: 68.40866, qwk: 0.86528\n",
      "Fri Jul 31 06:50:07 2020 Epoch: 34\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e316d54186047469bfa8c60c8287929",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c4632db9957426180e3b361d4bdef8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8660483437779767 qwk_k 0.8853418183601133 qwk_r 0.8186856253139227\n",
      "Fri Jul 31 07:00:20 2020 Epoch 34, lr: 0.0000806, train loss: 0.07550, val loss: 0.29093, acc: 68.22034, qwk: 0.86605\n",
      "Fri Jul 31 07:00:20 2020 Epoch: 35\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57f910896d8e4476923f6108d19e6794",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46e7e1ebf4864e2589a033dbf8aaa2e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8656030112029305 qwk_k 0.884694649131911 qwk_r 0.8200976123279963\n",
      "Fri Jul 31 07:10:37 2020 Epoch 35, lr: 0.0000722, train loss: 0.07180, val loss: 0.29977, acc: 68.40866, qwk: 0.86560\n",
      "Fri Jul 31 07:10:37 2020 Epoch: 36\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aca76c618cd64b0e97b357475cab70d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a5227870da64c2f86c6ef5bb873c34c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8689620916470051 qwk_k 0.883694387287806 qwk_r 0.8261364049963053\n",
      "Fri Jul 31 07:20:50 2020 Epoch 36, lr: 0.0000642, train loss: 0.06816, val loss: 0.29129, acc: 68.50282, qwk: 0.86896\n",
      "Fri Jul 31 07:20:50 2020 Epoch: 37\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12a569d4e14b4c37a0aec2e360173bee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "488f4e4d167245e5b0479c2a2f667567",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.87000863979565 qwk_k 0.8867937561975644 qwk_r 0.8250575699595354\n",
      "Fri Jul 31 07:31:06 2020 Epoch 37, lr: 0.0000565, train loss: 0.06662, val loss: 0.30470, acc: 67.89077, qwk: 0.87001\n",
      "Fri Jul 31 07:31:06 2020 Epoch: 38\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91d52128f13c46e8a8289b72b662f0f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63eb3a8f508e4e2dab91ff7ccd32d5c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8656908828088783 qwk_k 0.8871914363090031 qwk_r 0.8167934017382561\n",
      "Fri Jul 31 07:41:21 2020 Epoch 38, lr: 0.0000492, train loss: 0.06030, val loss: 0.31497, acc: 67.32580, qwk: 0.86569\n",
      "Fri Jul 31 07:41:21 2020 Epoch: 39\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ceda09cb71c644388e3757967c69bb3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "233c9598124a47a8b9d7bf154e0e546c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8769466905453945 qwk_k 0.8911028014026597 qwk_r 0.8359396681307717\n",
      "Fri Jul 31 07:51:37 2020 Epoch 39, lr: 0.0000422, train loss: 0.06160, val loss: 0.29281, acc: 68.07910, qwk: 0.87695\n",
      "Fri Jul 31 07:51:37 2020 Epoch: 40\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09715de2f0044acfbf3a18823e3cfae1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d3e69a04bc6408f99131dc54206e301",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8724693829694422 qwk_k 0.8850723744853098 qwk_r 0.8319246161306602\n",
      "Fri Jul 31 08:01:52 2020 Epoch 40, lr: 0.0000358, train loss: 0.05638, val loss: 0.31547, acc: 69.11488, qwk: 0.87247\n",
      "Fri Jul 31 08:01:52 2020 Epoch: 41\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59aeaa793d6f41fe8d517fbf77cf209a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2c45b386cff4e9e8479fc76cc091996",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8721494507689789 qwk_k 0.8891110348862389 qwk_r 0.8293120693496367\n",
      "Fri Jul 31 08:12:11 2020 Epoch 41, lr: 0.0000298, train loss: 0.05708, val loss: 0.31981, acc: 69.58569, qwk: 0.87215\n",
      "Fri Jul 31 08:12:11 2020 Epoch: 42\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc96b2ac5f3e40b4a50302dff8540f77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f6baa0144cc4f3faf9aa34b79473440",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8726528300385328 qwk_k 0.8913311060653573 qwk_r 0.8285338639615649\n",
      "Fri Jul 31 08:22:26 2020 Epoch 42, lr: 0.0000243, train loss: 0.05326, val loss: 0.31598, acc: 68.64407, qwk: 0.87265\n",
      "Fri Jul 31 08:22:26 2020 Epoch: 43\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b31f3404c994294ab4977f29cd804d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7814bd6ffae4660a8f6100669325a7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a182868d93843b7832d5d1be84a979d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6a249dc1440408fb4a71654df71fc68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8757704353362852 qwk_k 0.8909242406207053 qwk_r 0.8349276150631681\n",
      "Fri Jul 31 09:13:44 2020 Epoch 47, lr: 0.0000049, train loss: 0.04592, val loss: 0.33006, acc: 69.49153, qwk: 0.87577\n",
      "Fri Jul 31 09:13:44 2020 Epoch: 48\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bb55424e4774e1687bd9fd137b575c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0234fe98ac14d25a4b3cb696fb65448",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=266.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "qwk 0.8742718433026364 qwk_k 0.8901641623560541 qwk_r 0.8318805983722093\n",
      "Fri Jul 31 09:34:14 2020 Epoch 49, lr: 0.0000012, train loss: 0.04644, val loss: 0.32033, acc: 69.77401, qwk: 0.87427\n",
      "Fri Jul 31 09:34:14 2020 Epoch: 50\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36b89581d86c437c8fc86b9359475561",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1062.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "qwk_max = 0.\n",
    "best_file = f'checkpoints/{kernel_type}_best_fold{fold}.pth'\n",
    "for epoch in range(1, n_epochs+1):\n",
    "    print(time.ctime(), 'Epoch:', epoch)\n",
    "    scheduler.step(epoch-1)\n",
    "\n",
    "    train_loss = train_epoch(train_loader, optimizer)\n",
    "    val_loss, acc, qwk = val_epoch(valid_loader)\n",
    "\n",
    "    content = time.ctime() + ' ' + f'Epoch {epoch}, lr: {optimizer.param_groups[0][\"lr\"]:.7f}, train loss: {np.mean(train_loss):.5f}, val loss: {np.mean(val_loss):.5f}, acc: {(acc):.5f}, qwk: {(qwk):.5f}'\n",
    "    print(content)\n",
    "    with open(f'logs/log_{kernel_type}.txt', 'a') as appender:\n",
    "        appender.write(content + '\\n')\n",
    "\n",
    "    if qwk > qwk_max:\n",
    "        print('score2 ({:.6f} --> {:.6f}).  Saving model ...'.format(qwk_max, qwk))\n",
    "        torch.save(model.state_dict(), best_file)\n",
    "        qwk_max = qwk\n",
    "\n",
    "torch.save(model.state_dict(), os.path.join(f'{kernel_type}_final_fold{fold}.pth'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean Train log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fri Jul 31 01:22:48 2020 Epoch 1, lr: 0.0000300, train loss: 0.48728, val loss: 0.35194, acc: 32.67420, qwk: 0.68522\n",
    "\n",
    "Fri Jul 31 01:32:59 2020 Epoch 2, lr: 0.0003000, train loss: 0.36829, val loss: 0.31629, acc: 36.62900, qwk: 0.74549\n",
    "\n",
    "Fri Jul 31 01:43:09 2020 Epoch 3, lr: 0.0000300, train loss: 0.27242, val loss: 0.24789, acc: 56.82674, qwk: 0.83838\n",
    "\n",
    "Fri Jul 31 01:53:26 2020 Epoch 4, lr: 0.0002988, train loss: 0.28789, val loss: 0.28162, acc: 61.95857, qwk: 0.80171\n",
    "\n",
    "Fri Jul 31 02:03:43 2020 Epoch 5, lr: 0.0002972, train loss: 0.26203, val loss: 0.25778, acc: 57.25047, qwk: 0.83672\n",
    "\n",
    "Fri Jul 31 02:13:58 2020 Epoch 6, lr: 0.0002951, train loss: 0.24874, val loss: 0.28150, acc: 48.44633, qwk: 0.80569\n",
    "\n",
    "Fri Jul 31 02:24:14 2020 Epoch 7, lr: 0.0002924, train loss: 0.23590, val loss: 0.24244, acc: 59.51036, qwk: 0.84836\n",
    "\n",
    "Fri Jul 31 02:34:28 2020 Epoch 8, lr: 0.0002890, train loss: 0.22646, val loss: 0.24412, acc: 61.39360, qwk: 0.84483\n",
    "\n",
    "Fri Jul 31 02:44:41 2020 Epoch 9, lr: 0.0002851, train loss: 0.21952, val loss: 0.23171, acc: 62.00565, qwk: 0.85636\n",
    "\n",
    "Fri Jul 31 02:54:56 2020 Epoch 10, lr: 0.0002807, train loss: 0.21155, val loss: 0.25910, acc: 61.58192, qwk: 0.84736\n",
    "\n",
    "Fri Jul 31 03:05:09 2020 Epoch 11, lr: 0.0002757, train loss: 0.20425, val loss: 0.24255, acc: 62.09981, qwk: 0.85714\n",
    "\n",
    "Fri Jul 31 03:15:23 2020 Epoch 12, lr: 0.0002702, train loss: 0.19756, val loss: 0.22886, acc: 61.39360, qwk: 0.86287\n",
    "\n",
    "Fri Jul 31 03:25:38 2020 Epoch 13, lr: 0.0002642, train loss: 0.19369, val loss: 0.24613, acc: 64.87759, qwk: 0.86603\n",
    "\n",
    "Fri Jul 31 03:35:51 2020 Epoch 14, lr: 0.0002578, train loss: 0.18799, val loss: 0.24163, acc: 66.05461, qwk: 0.84961\n",
    "\n",
    "Fri Jul 31 03:46:04 2020 Epoch 15, lr: 0.0002508, train loss: 0.17917, val loss: 0.26223, acc: 64.31262, qwk: 0.84652\n",
    "\n",
    "Fri Jul 31 03:56:17 2020 Epoch 16, lr: 0.0002435, train loss: 0.17006, val loss: 0.23943, acc: 63.84181, qwk: 0.84214\n",
    "\n",
    "Fri Jul 31 04:06:30 2020 Epoch 17, lr: 0.0002358, train loss: 0.16587, val loss: 0.23266, acc: 64.97175, qwk: 0.86491\n",
    "\n",
    "Fri Jul 31 04:16:43 2020 Epoch 18, lr: 0.0002278, train loss: 0.16135, val loss: 0.21974, acc: 66.61959, qwk: 0.87617\n",
    "\n",
    "Fri Jul 31 04:26:57 2020 Epoch 19, lr: 0.0002194, train loss: 0.15672, val loss: 0.23681, acc: 64.35970, qwk: 0.85780\n",
    "\n",
    "Fri Jul 31 04:37:10 2020 Epoch 20, lr: 0.0002107, train loss: 0.14988, val loss: 0.25504, acc: 62.28814, qwk: 0.85248\n",
    "\n",
    "Fri Jul 31 04:47:25 2020 Epoch 21, lr: 0.0002018, train loss: 0.14172, val loss: 0.24680, acc: 64.83051, qwk: 0.86191\n",
    "\n",
    "Fri Jul 31 04:57:37 2020 Epoch 22, lr: 0.0001927, train loss: 0.13453, val loss: 0.31979, acc: 65.16008, qwk: 0.82527\n",
    "\n",
    "Fri Jul 31 05:07:50 2020 Epoch 23, lr: 0.0001834, train loss: 0.13046, val loss: 0.24314, acc: 65.67797, qwk: 0.86635\n",
    "\n",
    "Fri Jul 31 05:18:03 2020 Epoch 24, lr: 0.0001739, train loss: 0.12409, val loss: 0.24002, acc: 67.18456, qwk: 0.87073\n",
    "\n",
    "Fri Jul 31 05:28:18 2020 Epoch 25, lr: 0.0001644, train loss: 0.12092, val loss: 0.27853, acc: 66.00753, qwk: 0.86430\n",
    "\n",
    "Fri Jul 31 05:38:32 2020 Epoch 26, lr: 0.0001548, train loss: 0.11514, val loss: 0.24585, acc: 67.84369, qwk: 0.87188\n",
    "\n",
    "Fri Jul 31 05:48:40 2020 Epoch 27, lr: 0.0001452, train loss: 0.10879, val loss: 0.27445, acc: 64.87759, qwk: 0.86390\n",
    "\n",
    "Fri Jul 31 05:58:52 2020 Epoch 28, lr: 0.0001356, train loss: 0.10443, val loss: 0.26355, acc: 67.56121, qwk: 0.87602\n",
    "\n",
    "Fri Jul 31 06:09:04 2020 Epoch 29, lr: 0.0001261, train loss: 0.09894, val loss: 0.26092, acc: 67.27872, qwk: 0.87175\n",
    "\n",
    "Fri Jul 31 06:19:17 2020 Epoch 30, lr: 0.0001166, train loss: 0.09263, val loss: 0.26277, acc: 68.26742, qwk: 0.87605\n",
    "\n",
    "\n",
    "Fri Jul 31 06:29:35 2020 Epoch 31, lr: 0.0001073, train loss: 0.09059, val loss: 0.27439, acc: 68.17326, qwk: 0.87394\n",
    "\n",
    "Fri Jul 31 06:39:51 2020 Epoch 32, lr: 0.0000982, train loss: 0.08415, val loss: 0.25896, acc: 67.23164, qwk: 0.87771\n",
    "\n",
    "Fri Jul 31 06:50:07 2020 Epoch 33, lr: 0.0000893, train loss: 0.07885, val loss: 0.29701, acc: 68.40866, qwk: 0.86528\n",
    "\n",
    "Fri Jul 31 07:00:20 2020 Epoch 34, lr: 0.0000806, train loss: 0.07550, val loss: 0.29093, acc: 68.22034, qwk: 0.86605\n",
    "\n",
    "Fri Jul 31 07:10:37 2020 Epoch 35, lr: 0.0000722, train loss: 0.07180, val loss: 0.29977, acc: 68.40866, qwk: 0.86560\n",
    "\n",
    "Fri Jul 31 07:20:50 2020 Epoch 36, lr: 0.0000642, train loss: 0.06816, val loss: 0.29129, acc: 68.50282, qwk: 0.86896\n",
    "\n",
    "Fri Jul 31 07:31:06 2020 Epoch 37, lr: 0.0000565, train loss: 0.06662, val loss: 0.30470, acc: 67.89077, qwk: 0.87001\n",
    "\n",
    "Fri Jul 31 07:41:21 2020 Epoch 38, lr: 0.0000492, train loss: 0.06030, val loss: 0.31497, acc: 67.32580, qwk: 0.86569\n",
    "\n",
    "Fri Jul 31 07:51:37 2020 Epoch 39, lr: 0.0000422, train loss: 0.06160, val loss: 0.29281, acc: 68.07910, qwk: 0.87695\n",
    "\n",
    "Fri Jul 31 08:01:52 2020 Epoch 40, lr: 0.0000358, train loss: 0.05638, val loss: 0.31547, acc: 69.11488, qwk: 0.87247\n",
    "\n",
    "Fri Jul 31 08:12:11 2020 Epoch 41, lr: 0.0000298, train loss: 0.05708, val loss: 0.31981, acc: 69.58569, qwk: 0.87215\n",
    "\n",
    "Fri Jul 31 08:22:26 2020 Epoch 42, lr: 0.0000243, train loss: 0.05326, val loss: 0.31598, acc: 68.64407, qwk: 0.87265\n",
    "\n",
    "Fri Jul 31 08:32:43 2020 Epoch 43, lr: 0.0000193, train loss: 0.04858, val loss: 0.32906, acc: 69.06780, qwk: 0.87208\n",
    "\n",
    "Fri Jul 31 08:42:59 2020 Epoch 44, lr: 0.0000149, train loss: 0.04714, val loss: 0.32263, acc: 69.35028, qwk: 0.87109\n",
    "\n",
    "Fri Jul 31 08:53:14 2020 Epoch 45, lr: 0.0000110, train loss: 0.04844, val loss: 0.31060, acc: 69.16196, qwk: 0.87609\n",
    "\n",
    "Fri Jul 31 09:03:30 2020 Epoch 46, lr: 0.0000076, train loss: 0.04556, val loss: 0.32499, acc: 69.91525, qwk: 0.87472\n",
    "\n",
    "Fri Jul 31 09:13:44 2020 Epoch 47, lr: 0.0000049, train loss: 0.04592, val loss: 0.33006, acc: 69.49153, qwk: 0.87577\n",
    "\n",
    "Fri Jul 31 09:23:59 2020 Epoch 48, lr: 0.0000028, train loss: 0.04433, val loss: 0.32192, acc: 69.63277, qwk: 0.87289\n",
    "\n",
    "Fri Jul 31 09:34:14 2020 Epoch 49, lr: 0.0000012, train loss: 0.04644, val loss: 0.32033, acc: 69.77401, qwk: 0.87427\n",
    "\n",
    "Fri Jul 31 09:44:26 2020 Epoch 50, lr: 0.0000003, train loss: 0.04275, val loss: 0.33012, acc: 69.25612, qwk: 0.87218"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sangwon",
   "language": "python",
   "name": "sangwon"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "117c736dca8349beb197997b569c520b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "1aba7eeb9d7040459953ee8cc1c57ddb": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "282293ec10b14213911a555ab28f5273": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "32ead8e2299b47f69d264072a7830317": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "41156f41dccc48bb919888b37a171178": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_282293ec10b14213911a555ab28f5273",
       "placeholder": "​",
       "style": "IPY_MODEL_1aba7eeb9d7040459953ee8cc1c57ddb",
       "value": " 10/10 [00:10&lt;00:00,  1.04s/it]"
      }
     },
     "465d0704ece943f99b00159e192738ad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d3fb9db3ef8141e49d5cc78ba676ad17",
       "placeholder": "​",
       "style": "IPY_MODEL_117c736dca8349beb197997b569c520b",
       "value": " 40/40 [00:42&lt;00:00,  1.06s/it]"
      }
     },
     "65ec03d6ea05405ba89d074994812ab4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_cb5c37ea332e4db7a3c6e7ac0e8c2f47",
       "max": 10,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_32ead8e2299b47f69d264072a7830317",
       "value": 10
      }
     },
     "7566820607054425a234e25c418c0e49": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8e83d561f435460e8ff19ea062636702": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b28f9a83767a4fb1a086f4a2d98b08ca": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "b8403be6607b4fc8a7b1fa25579bc970": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "loss: 0.68076, smth: 0.66723: 100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8e83d561f435460e8ff19ea062636702",
       "max": 40,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_b28f9a83767a4fb1a086f4a2d98b08ca",
       "value": 40
      }
     },
     "cb5c37ea332e4db7a3c6e7ac0e8c2f47": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "cfb548ddc9ee4774958243653d06cf26": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_b8403be6607b4fc8a7b1fa25579bc970",
        "IPY_MODEL_465d0704ece943f99b00159e192738ad"
       ],
       "layout": "IPY_MODEL_e83736ac502445259bd0fc4162e484cb"
      }
     },
     "d3fb9db3ef8141e49d5cc78ba676ad17": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e83736ac502445259bd0fc4162e484cb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f1e9d76e042c4f269eeea3275804f577": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_65ec03d6ea05405ba89d074994812ab4",
        "IPY_MODEL_41156f41dccc48bb919888b37a171178"
       ],
       "layout": "IPY_MODEL_7566820607054425a234e25c418c0e49"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}