Model_train_eval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.keras.backend as K\n",
    "from tensorflow.keras.callbacks import *\n",
    "\n",
    "from Model import HourglassNet\n",
    "from Data_utils import DataGenerator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = HourglassNet(num_classes=7, \n",
    "                     num_stacks=1, \n",
    "                     num_channels=128, \n",
    "                     inres=(128,128), \n",
    "                     outres=(32,32)).get_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mse_4x_penalty(y_true, y_pred):\n",
    "    mse = tf.keras.losses.mean_squared_error(y_true, y_pred)*4\n",
    "    a = K.less(K.max(K.argmax(y_true[:,:,:,2], axis=2)), K.max(K.argmax(y_pred[:,:,:,0], axis=2)))\n",
    "    b = K.less(K.max(K.argmax(y_pred[:,:,:,1], axis=2)), K.max(K.argmax(y_true[:,:,:,2], axis=2)))\n",
    "    cond = tf.cond(\n",
    "        K.any(K.stack([a, b], axis=0), axis=0),\n",
    "        lambda: mse*10,\n",
    "        lambda: mse\n",
    "    )\n",
    "    return cond\n",
    "\n",
    "def mae_4x(y_true, y_pred):\n",
    "    return tf.keras.metrics.mean_absolute_error(y_true, y_pred)*4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam(lr=5e-4)\n",
    "model.compile(optimizer=optimizer, loss=mse_4x_penalty, metrics=[mae_4x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 128, 128, 1)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "front_conv_1x1_x1 (Conv2D)      (None, 64, 64, 64)   3200        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_53 (BatchNo (None, 64, 64, 64)   256         front_conv_1x1_x1[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x1_conv_1x1_x1 ( (None, 64, 64, 32)   2080        batch_normalization_53[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_54 (BatchNo (None, 64, 64, 32)   128         front_residual_x1_conv_1x1_x1[0][\n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x1_conv_3x3_x2 ( (None, 64, 64, 32)   9248        batch_normalization_54[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_55 (BatchNo (None, 64, 64, 32)   128         front_residual_x1_conv_3x3_x2[0][\n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x1_conv_1x1_x3 ( (None, 64, 64, 64)   2112        batch_normalization_55[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_56 (BatchNo (None, 64, 64, 64)   256         front_residual_x1_conv_1x1_x3[0][\n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x1_residual (Add (None, 64, 64, 64)   0           batch_normalization_53[0][0]     \n",
      "                                                                 batch_normalization_56[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)  (None, 32, 32, 64)   0           front_residual_x1_residual[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x2_conv_1x1_x1 ( (None, 32, 32, 32)   2080        max_pooling2d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_57 (BatchNo (None, 32, 32, 32)   128         front_residual_x2_conv_1x1_x1[0][\n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x2_conv_3x3_x2 ( (None, 32, 32, 32)   9248        batch_normalization_57[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_58 (BatchNo (None, 32, 32, 32)   128         front_residual_x2_conv_3x3_x2[0][\n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x2_conv_1x1_x3 ( (None, 32, 32, 64)   2112        batch_normalization_58[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_59 (BatchNo (None, 32, 32, 64)   256         front_residual_x2_conv_1x1_x3[0][\n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x2_residual (Add (None, 32, 32, 64)   0           max_pooling2d_4[0][0]            \n",
      "                                                                 batch_normalization_59[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x3_conv_1x1_x1 ( (None, 32, 32, 64)   4160        front_residual_x2_residual[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_60 (BatchNo (None, 32, 32, 64)   256         front_residual_x3_conv_1x1_x1[0][\n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x3_conv_3x3_x2 ( (None, 32, 32, 64)   36928       batch_normalization_60[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_61 (BatchNo (None, 32, 32, 64)   256         front_residual_x3_conv_3x3_x2[0][\n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x3_conv_1x1_x3 ( (None, 32, 32, 128)  8320        batch_normalization_61[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x3skip (Conv2D)  (None, 32, 32, 128)  8320        front_residual_x2_residual[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_62 (BatchNo (None, 32, 32, 128)  512         front_residual_x3_conv_1x1_x3[0][\n",
      "__________________________________________________________________________________________________\n",
      "front_residual_x3_residual (Add (None, 32, 32, 128)  0           front_residual_x3skip[0][0]      \n",
      "                                                                 batch_normalization_62[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l1_conv_1x1_x1 (Conv2D)     (None, 32, 32, 64)   8256        front_residual_x3_residual[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_63 (BatchNo (None, 32, 32, 64)   256         hg0_l1_conv_1x1_x1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l1_conv_3x3_x2 (Conv2D)     (None, 32, 32, 64)   36928       batch_normalization_63[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_64 (BatchNo (None, 32, 32, 64)   256         hg0_l1_conv_3x3_x2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l1_conv_1x1_x3 (Conv2D)     (None, 32, 32, 128)  8320        batch_normalization_64[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_65 (BatchNo (None, 32, 32, 128)  512         hg0_l1_conv_1x1_x3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l1_residual (Add)           (None, 32, 32, 128)  0           front_residual_x3_residual[0][0] \n",
      "                                                                 batch_normalization_65[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2D)  (None, 16, 16, 128)  0           hg0_l1_residual[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l2_conv_1x1_x1 (Conv2D)     (None, 16, 16, 64)   8256        max_pooling2d_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_66 (BatchNo (None, 16, 16, 64)   256         hg0_l2_conv_1x1_x1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l2_conv_3x3_x2 (Conv2D)     (None, 16, 16, 64)   36928       batch_normalization_66[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_67 (BatchNo (None, 16, 16, 64)   256         hg0_l2_conv_3x3_x2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l2_conv_1x1_x3 (Conv2D)     (None, 16, 16, 128)  8320        batch_normalization_67[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_68 (BatchNo (None, 16, 16, 128)  512         hg0_l2_conv_1x1_x3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l2_residual (Add)           (None, 16, 16, 128)  0           max_pooling2d_5[0][0]            \n",
      "                                                                 batch_normalization_68[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2D)  (None, 8, 8, 128)    0           hg0_l2_residual[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l4_conv_1x1_x1 (Conv2D)     (None, 8, 8, 64)     8256        max_pooling2d_6[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_69 (BatchNo (None, 8, 8, 64)     256         hg0_l4_conv_1x1_x1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l4_conv_3x3_x2 (Conv2D)     (None, 8, 8, 64)     36928       batch_normalization_69[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_70 (BatchNo (None, 8, 8, 64)     256         hg0_l4_conv_3x3_x2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l4_conv_1x1_x3 (Conv2D)     (None, 8, 8, 128)    8320        batch_normalization_70[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_71 (BatchNo (None, 8, 8, 128)    512         hg0_l4_conv_1x1_x3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l4_residual (Add)           (None, 8, 8, 128)    0           max_pooling2d_6[0][0]            \n",
      "                                                                 batch_normalization_71[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_7 (MaxPooling2D)  (None, 4, 4, 128)    0           hg0_l4_residual[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l8_conv_1x1_x1 (Conv2D)     (None, 4, 4, 64)     8256        max_pooling2d_7[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_72 (BatchNo (None, 4, 4, 64)     256         hg0_l8_conv_1x1_x1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l8_conv_3x3_x2 (Conv2D)     (None, 4, 4, 64)     36928       batch_normalization_72[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_73 (BatchNo (None, 4, 4, 64)     256         hg0_l8_conv_3x3_x2[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l8_conv_1x1_x3 (Conv2D)     (None, 4, 4, 128)    8320        batch_normalization_73[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_74 (BatchNo (None, 4, 4, 128)    512         hg0_l8_conv_1x1_x3[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "hg0_l8_residual (Add)           (None, 4, 4, 128)    0           max_pooling2d_7[0][0]            \n",
      "                                                                 batch_normalization_74[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x1_conv_1x1_x1 (Conv2D)   (None, 4, 4, 64)     8256        hg0_l8_residual[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_78 (BatchNo (None, 4, 4, 64)     256         0_lf8_x1_conv_1x1_x1[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x1_conv_3x3_x2 (Conv2D)   (None, 4, 4, 64)     36928       batch_normalization_78[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_79 (BatchNo (None, 4, 4, 64)     256         0_lf8_x1_conv_3x3_x2[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x1_conv_1x1_x3 (Conv2D)   (None, 4, 4, 128)    8320        batch_normalization_79[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_80 (BatchNo (None, 4, 4, 128)    512         0_lf8_x1_conv_1x1_x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x1_residual (Add)         (None, 4, 4, 128)    0           hg0_l8_residual[0][0]            \n",
      "                                                                 batch_normalization_80[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x2_conv_1x1_x1 (Conv2D)   (None, 4, 4, 64)     8256        0_lf8_x1_residual[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_81 (BatchNo (None, 4, 4, 64)     256         0_lf8_x2_conv_1x1_x1[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x2_conv_3x3_x2 (Conv2D)   (None, 4, 4, 64)     36928       batch_normalization_81[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_82 (BatchNo (None, 4, 4, 64)     256         0_lf8_x2_conv_3x3_x2[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x2_conv_1x1_x3 (Conv2D)   (None, 4, 4, 128)    8320        batch_normalization_82[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_83 (BatchNo (None, 4, 4, 128)    512         0_lf8_x2_conv_1x1_x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x2_residual (Add)         (None, 4, 4, 128)    0           0_lf8_x1_residual[0][0]          \n",
      "                                                                 batch_normalization_83[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x3_conv_1x1_x1 (Conv2D)   (None, 4, 4, 64)     8256        0_lf8_x2_residual[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_conv_1x1_x1 (Conv2D)      (None, 4, 4, 64)     8256        hg0_l8_residual[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_84 (BatchNo (None, 4, 4, 64)     256         0_lf8_x3_conv_1x1_x1[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_75 (BatchNo (None, 4, 4, 64)     256         0_lf8_conv_1x1_x1[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf4_connect_conv_1x1_x1 (Co (None, 8, 8, 64)     8256        hg0_l4_residual[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x3_conv_3x3_x2 (Conv2D)   (None, 4, 4, 64)     36928       batch_normalization_84[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_conv_3x3_x2 (Conv2D)      (None, 4, 4, 64)     36928       batch_normalization_75[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_87 (BatchNo (None, 8, 8, 64)     256         hg0_rf4_connect_conv_1x1_x1[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_85 (BatchNo (None, 4, 4, 64)     256         0_lf8_x3_conv_3x3_x2[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_76 (BatchNo (None, 4, 4, 64)     256         0_lf8_conv_3x3_x2[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf4_connect_conv_3x3_x2 (Co (None, 8, 8, 64)     36928       batch_normalization_87[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x3_conv_1x1_x3 (Conv2D)   (None, 4, 4, 128)    8320        batch_normalization_85[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_conv_1x1_x3 (Conv2D)      (None, 4, 4, 128)    8320        batch_normalization_76[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_88 (BatchNo (None, 8, 8, 64)     256         hg0_rf4_connect_conv_3x3_x2[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_86 (BatchNo (None, 4, 4, 128)    512         0_lf8_x3_conv_1x1_x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_77 (BatchNo (None, 4, 4, 128)    512         0_lf8_conv_1x1_x3[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf4_connect_conv_1x1_x3 (Co (None, 8, 8, 128)    8320        batch_normalization_88[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_x3_residual (Add)         (None, 4, 4, 128)    0           0_lf8_x2_residual[0][0]          \n",
      "                                                                 batch_normalization_86[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "0_lf8_residual (Add)            (None, 4, 4, 128)    0           hg0_l8_residual[0][0]            \n",
      "                                                                 batch_normalization_77[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_89 (BatchNo (None, 8, 8, 128)    512         hg0_rf4_connect_conv_1x1_x3[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 4, 4, 128)    0           0_lf8_x3_residual[0][0]          \n",
      "                                                                 0_lf8_residual[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf4_connect_residual (Add)  (None, 8, 8, 128)    0           hg0_l4_residual[0][0]            \n",
      "                                                                 batch_normalization_89[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "up_sampling2d_3 (UpSampling2D)  (None, 8, 8, 128)    0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 8, 8, 128)    0           hg0_rf4_connect_residual[0][0]   \n",
      "                                                                 up_sampling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf4_connect_conv_conv_1x1_x (None, 8, 8, 64)     8256        add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf2_connect_conv_1x1_x1 (Co (None, 16, 16, 64)   8256        hg0_l2_residual[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_90 (BatchNo (None, 8, 8, 64)     256         hg0_rf4_connect_conv_conv_1x1_x1[\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_93 (BatchNo (None, 16, 16, 64)   256         hg0_rf2_connect_conv_1x1_x1[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf4_connect_conv_conv_3x3_x (None, 8, 8, 64)     36928       batch_normalization_90[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf2_connect_conv_3x3_x2 (Co (None, 16, 16, 64)   36928       batch_normalization_93[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_91 (BatchNo (None, 8, 8, 64)     256         hg0_rf4_connect_conv_conv_3x3_x2[\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_94 (BatchNo (None, 16, 16, 64)   256         hg0_rf2_connect_conv_3x3_x2[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf4_connect_conv_conv_1x1_x (None, 8, 8, 128)    8320        batch_normalization_91[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf2_connect_conv_1x1_x3 (Co (None, 16, 16, 128)  8320        batch_normalization_94[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_92 (BatchNo (None, 8, 8, 128)    512         hg0_rf4_connect_conv_conv_1x1_x3[\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_95 (BatchNo (None, 16, 16, 128)  512         hg0_rf2_connect_conv_1x1_x3[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf4_connect_conv_residual ( (None, 8, 8, 128)    0           add_6[0][0]                      \n",
      "                                                                 batch_normalization_92[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf2_connect_residual (Add)  (None, 16, 16, 128)  0           hg0_l2_residual[0][0]            \n",
      "                                                                 batch_normalization_95[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "up_sampling2d_4 (UpSampling2D)  (None, 16, 16, 128)  0           hg0_rf4_connect_conv_residual[0][\n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 16, 16, 128)  0           hg0_rf2_connect_residual[0][0]   \n",
      "                                                                 up_sampling2d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf2_connect_conv_conv_1x1_x (None, 16, 16, 64)   8256        add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf1_connect_conv_1x1_x1 (Co (None, 32, 32, 64)   8256        hg0_l1_residual[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_96 (BatchNo (None, 16, 16, 64)   256         hg0_rf2_connect_conv_conv_1x1_x1[\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_99 (BatchNo (None, 32, 32, 64)   256         hg0_rf1_connect_conv_1x1_x1[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf2_connect_conv_conv_3x3_x (None, 16, 16, 64)   36928       batch_normalization_96[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf1_connect_conv_3x3_x2 (Co (None, 32, 32, 64)   36928       batch_normalization_99[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_97 (BatchNo (None, 16, 16, 64)   256         hg0_rf2_connect_conv_conv_3x3_x2[\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_100 (BatchN (None, 32, 32, 64)   256         hg0_rf1_connect_conv_3x3_x2[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf2_connect_conv_conv_1x1_x (None, 16, 16, 128)  8320        batch_normalization_97[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf1_connect_conv_1x1_x3 (Co (None, 32, 32, 128)  8320        batch_normalization_100[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_98 (BatchNo (None, 16, 16, 128)  512         hg0_rf2_connect_conv_conv_1x1_x3[\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_101 (BatchN (None, 32, 32, 128)  512         hg0_rf1_connect_conv_1x1_x3[0][0]\n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf2_connect_conv_residual ( (None, 16, 16, 128)  0           add_7[0][0]                      \n",
      "                                                                 batch_normalization_98[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf1_connect_residual (Add)  (None, 32, 32, 128)  0           hg0_l1_residual[0][0]            \n",
      "                                                                 batch_normalization_101[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "up_sampling2d_5 (UpSampling2D)  (None, 32, 32, 128)  0           hg0_rf2_connect_conv_residual[0][\n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 32, 32, 128)  0           hg0_rf1_connect_residual[0][0]   \n",
      "                                                                 up_sampling2d_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf1_connect_conv_conv_1x1_x (None, 32, 32, 64)   8256        add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_102 (BatchN (None, 32, 32, 64)   256         hg0_rf1_connect_conv_conv_1x1_x1[\n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf1_connect_conv_conv_3x3_x (None, 32, 32, 64)   36928       batch_normalization_102[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_103 (BatchN (None, 32, 32, 64)   256         hg0_rf1_connect_conv_conv_3x3_x2[\n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf1_connect_conv_conv_1x1_x (None, 32, 32, 128)  8320        batch_normalization_103[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_104 (BatchN (None, 32, 32, 128)  512         hg0_rf1_connect_conv_conv_1x1_x3[\n",
      "__________________________________________________________________________________________________\n",
      "hg0_rf1_connect_conv_residual ( (None, 32, 32, 128)  0           add_8[0][0]                      \n",
      "                                                                 batch_normalization_104[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "0_conv_1x1_x1 (Conv2D)          (None, 32, 32, 128)  16512       hg0_rf1_connect_conv_residual[0][\n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_105 (BatchN (None, 32, 32, 128)  512         0_conv_1x1_x1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "0_conv_1x1_parts (Conv2D)       (None, 32, 32, 7)    903         batch_normalization_105[0][0]    \n",
      "==================================================================================================\n",
      "Total params: 871,431\n",
      "Trainable params: 862,855\n",
      "Non-trainable params: 8,576\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_gen = DataGenerator('dataset/', 'train_landmarks.csv', \n",
    "                          inres=(128,128), outres=(32,32), nparts=7, is_train=True)\n",
    "valid_gen = DataGenerator('dataset/', 'valid_landmarks.csv', \n",
    "                          inres=(128,128), outres=(32,32), nparts=7)\n",
    "test_gen = DataGenerator('dataset/', 'test_landmarks.csv', \n",
    "                         inres=(128,128), outres=(32,32), nparts=7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_call = [\n",
    "    ModelCheckpoint('train/model.h5', save_best_only=True),\n",
    "    EarlyStopping(patience=40),\n",
    "    TensorBoard(log_dir='train/'),\n",
    "    ReduceLROnPlateau(patience=20),\n",
    "    CSVLogger('train/history.csv')\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "83/83 [==============================] - 28s 338ms/step - loss: 361.0574 - mae_4x: 1.3191 - val_loss: 1449.3931 - val_mae_4x: 3.2746\n",
      "Epoch 2/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 28.6250 - mae_4x: 0.4375 - val_loss: 3326.4771 - val_mae_4x: 4.8439\n",
      "Epoch 3/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 16.6540 - mae_4x: 0.3244 - val_loss: 1020.3224 - val_mae_4x: 2.0693\n",
      "Epoch 4/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 12.1424 - mae_4x: 0.2706 - val_loss: 172.8026 - val_mae_4x: 0.8207\n",
      "Epoch 5/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 9.2370 - mae_4x: 0.2272 - val_loss: 28.4455 - val_mae_4x: 0.4315\n",
      "Epoch 6/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 7.7203 - mae_4x: 0.2012 - val_loss: 84.4099 - val_mae_4x: 0.5632\n",
      "Epoch 7/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 6.7589 - mae_4x: 0.1828 - val_loss: 94.4071 - val_mae_4x: 0.6901\n",
      "Epoch 8/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 6.0724 - mae_4x: 0.1682 - val_loss: 111.2618 - val_mae_4x: 0.7007\n",
      "Epoch 9/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 5.5496 - mae_4x: 0.1562 - val_loss: 19.3157 - val_mae_4x: 0.3249\n",
      "Epoch 10/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 5.1686 - mae_4x: 0.1469 - val_loss: 6.9433 - val_mae_4x: 0.1913\n",
      "Epoch 11/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 4.8578 - mae_4x: 0.1389 - val_loss: 5.1857 - val_mae_4x: 0.1530\n",
      "Epoch 12/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 4.6138 - mae_4x: 0.1322 - val_loss: 4.7276 - val_mae_4x: 0.1384\n",
      "Epoch 13/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 4.4145 - mae_4x: 0.1265 - val_loss: 4.5547 - val_mae_4x: 0.1336\n",
      "Epoch 14/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 4.2407 - mae_4x: 0.1213 - val_loss: 4.2869 - val_mae_4x: 0.1241\n",
      "Epoch 15/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 4.0912 - mae_4x: 0.1167 - val_loss: 4.1279 - val_mae_4x: 0.1185\n",
      "Epoch 16/1000\n",
      "83/83 [==============================] - 10s 116ms/step - loss: 3.9646 - mae_4x: 0.1127 - val_loss: 3.9947 - val_mae_4x: 0.1142\n",
      "Epoch 17/1000\n",
      "83/83 [==============================] - 10s 116ms/step - loss: 3.8536 - mae_4x: 0.1091 - val_loss: 3.8851 - val_mae_4x: 0.1100\n",
      "Epoch 18/1000\n",
      "83/83 [==============================] - 10s 116ms/step - loss: 3.7570 - mae_4x: 0.1059 - val_loss: 3.7724 - val_mae_4x: 0.1062\n",
      "Epoch 19/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 3.6719 - mae_4x: 0.1030 - val_loss: 3.6985 - val_mae_4x: 0.1036\n",
      "Epoch 20/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 3.5969 - mae_4x: 0.1004 - val_loss: 3.6262 - val_mae_4x: 0.1012\n",
      "Epoch 21/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 3.5287 - mae_4x: 0.0980 - val_loss: 3.5822 - val_mae_4x: 0.0997\n",
      "Epoch 22/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 3.4660 - mae_4x: 0.0959 - val_loss: 3.5010 - val_mae_4x: 0.0963\n",
      "Epoch 23/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 3.4140 - mae_4x: 0.0941 - val_loss: 3.4491 - val_mae_4x: 0.0947\n",
      "Epoch 24/1000\n",
      "83/83 [==============================] - 10s 114ms/step - loss: 3.3573 - mae_4x: 0.0922 - val_loss: 3.3702 - val_mae_4x: 0.0917\n",
      "Epoch 25/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 3.3069 - mae_4x: 0.0905 - val_loss: 3.3266 - val_mae_4x: 0.0897\n",
      "Epoch 26/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 3.2601 - mae_4x: 0.0890 - val_loss: 3.2890 - val_mae_4x: 0.0883\n",
      "Epoch 27/1000\n",
      "83/83 [==============================] - 10s 116ms/step - loss: 3.2150 - mae_4x: 0.0875 - val_loss: 3.2416 - val_mae_4x: 0.0868\n",
      "Epoch 28/1000\n",
      "83/83 [==============================] - 10s 116ms/step - loss: 3.1724 - mae_4x: 0.0861 - val_loss: 3.1925 - val_mae_4x: 0.0855\n",
      "Epoch 29/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 3.1295 - mae_4x: 0.0848 - val_loss: 3.1865 - val_mae_4x: 0.0844\n",
      "Epoch 30/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 3.0892 - mae_4x: 0.0836 - val_loss: 3.1297 - val_mae_4x: 0.0822\n",
      "Epoch 31/1000\n",
      "83/83 [==============================] - 10s 118ms/step - loss: 3.0469 - mae_4x: 0.0824 - val_loss: 3.0842 - val_mae_4x: 0.0809\n",
      "Epoch 32/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 3.0054 - mae_4x: 0.0812 - val_loss: 3.0573 - val_mae_4x: 0.0803\n",
      "Epoch 33/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.9661 - mae_4x: 0.0800 - val_loss: 3.0235 - val_mae_4x: 0.0788\n",
      "Epoch 34/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 2.9269 - mae_4x: 0.0789 - val_loss: 2.9977 - val_mae_4x: 0.0777\n",
      "Epoch 35/1000\n",
      "83/83 [==============================] - 10s 116ms/step - loss: 2.8894 - mae_4x: 0.0779 - val_loss: 2.9415 - val_mae_4x: 0.0767\n",
      "Epoch 36/1000\n",
      "83/83 [==============================] - 10s 116ms/step - loss: 2.8525 - mae_4x: 0.0769 - val_loss: 2.9040 - val_mae_4x: 0.0761\n",
      "Epoch 37/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 2.8238 - mae_4x: 0.0762 - val_loss: 2.9189 - val_mae_4x: 0.0767\n",
      "Epoch 38/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.7839 - mae_4x: 0.0753 - val_loss: 2.8788 - val_mae_4x: 0.0758\n",
      "Epoch 39/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 2.7454 - mae_4x: 0.0743 - val_loss: 2.8294 - val_mae_4x: 0.0741\n",
      "Epoch 40/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.7070 - mae_4x: 0.0734 - val_loss: 2.8270 - val_mae_4x: 0.0738\n",
      "Epoch 41/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.6681 - mae_4x: 0.0725 - val_loss: 2.7673 - val_mae_4x: 0.0726\n",
      "Epoch 42/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 2.6262 - mae_4x: 0.0716 - val_loss: 2.7325 - val_mae_4x: 0.0716\n",
      "Epoch 43/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 2.5846 - mae_4x: 0.0707 - val_loss: 2.7097 - val_mae_4x: 0.0708\n",
      "Epoch 44/1000\n",
      "83/83 [==============================] - 10s 116ms/step - loss: 2.5424 - mae_4x: 0.0697 - val_loss: 2.6530 - val_mae_4x: 0.0697\n",
      "Epoch 45/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 2.4998 - mae_4x: 0.0688 - val_loss: 2.6388 - val_mae_4x: 0.0690\n",
      "Epoch 46/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.4572 - mae_4x: 0.0678 - val_loss: 2.6200 - val_mae_4x: 0.0677\n",
      "Epoch 47/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 2.4175 - mae_4x: 0.0669 - val_loss: 2.5956 - val_mae_4x: 0.0681\n",
      "Epoch 48/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.3778 - mae_4x: 0.0660 - val_loss: 2.5210 - val_mae_4x: 0.0663\n",
      "Epoch 49/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 2.3343 - mae_4x: 0.0649 - val_loss: 2.5408 - val_mae_4x: 0.0660\n",
      "Epoch 50/1000\n",
      "83/83 [==============================] - 9s 113ms/step - loss: 2.2923 - mae_4x: 0.0639 - val_loss: 2.4854 - val_mae_4x: 0.0652\n",
      "Epoch 51/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.2519 - mae_4x: 0.0630 - val_loss: 2.4628 - val_mae_4x: 0.0646\n",
      "Epoch 52/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 2.2148 - mae_4x: 0.0622 - val_loss: 2.5159 - val_mae_4x: 0.0692\n",
      "Epoch 53/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 2.2142 - mae_4x: 0.0629 - val_loss: 2.4805 - val_mae_4x: 0.0672\n",
      "Epoch 54/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 2.2470 - mae_4x: 0.0654 - val_loss: 4.6980 - val_mae_4x: 0.1427\n",
      "Epoch 55/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 4.2231 - mae_4x: 0.1207 - val_loss: 38.8673 - val_mae_4x: 0.4615\n",
      "Epoch 56/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 3.3867 - mae_4x: 0.0930 - val_loss: 8.3803 - val_mae_4x: 0.1987\n",
      "Epoch 57/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 2.9079 - mae_4x: 0.0769 - val_loss: 4.1072 - val_mae_4x: 0.1235\n",
      "Epoch 58/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 2.7133 - mae_4x: 0.0712 - val_loss: 3.0758 - val_mae_4x: 0.0854\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 59/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 2.7542 - mae_4x: 0.0755 - val_loss: 7.5047 - val_mae_4x: 0.1764\n",
      "Epoch 60/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 2.6139 - mae_4x: 0.0695 - val_loss: 3.1010 - val_mae_4x: 0.0922\n",
      "Epoch 61/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 2.4660 - mae_4x: 0.0645 - val_loss: 2.7464 - val_mae_4x: 0.0725\n",
      "Epoch 62/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 2.3773 - mae_4x: 0.0618 - val_loss: 2.6498 - val_mae_4x: 0.0672\n",
      "Epoch 63/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 2.3046 - mae_4x: 0.0600 - val_loss: 2.5866 - val_mae_4x: 0.0654\n",
      "Epoch 64/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 2.2430 - mae_4x: 0.0585 - val_loss: 2.5344 - val_mae_4x: 0.0643\n",
      "Epoch 65/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 2.1907 - mae_4x: 0.0571 - val_loss: 2.5187 - val_mae_4x: 0.0632\n",
      "Epoch 66/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.1420 - mae_4x: 0.0559 - val_loss: 2.4619 - val_mae_4x: 0.0616\n",
      "Epoch 67/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.1003 - mae_4x: 0.0548 - val_loss: 2.4252 - val_mae_4x: 0.0604\n",
      "Epoch 68/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.0613 - mae_4x: 0.0537 - val_loss: 2.3830 - val_mae_4x: 0.0591\n",
      "Epoch 69/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 2.0197 - mae_4x: 0.0527 - val_loss: 2.3744 - val_mae_4x: 0.0579\n",
      "Epoch 70/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 1.9851 - mae_4x: 0.0518 - val_loss: 2.3512 - val_mae_4x: 0.0565\n",
      "Epoch 71/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 1.9558 - mae_4x: 0.0511 - val_loss: 2.3086 - val_mae_4x: 0.0555\n",
      "Epoch 72/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 1.9221 - mae_4x: 0.0503 - val_loss: 2.2764 - val_mae_4x: 0.0545\n",
      "Epoch 73/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.8936 - mae_4x: 0.0496 - val_loss: 2.2887 - val_mae_4x: 0.0536\n",
      "Epoch 74/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 1.8699 - mae_4x: 0.0489 - val_loss: 2.2727 - val_mae_4x: 0.0531\n",
      "Epoch 75/1000\n",
      "83/83 [==============================] - 9s 114ms/step - loss: 1.8462 - mae_4x: 0.0484 - val_loss: 2.2370 - val_mae_4x: 0.0518\n",
      "Epoch 76/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.8186 - mae_4x: 0.0478 - val_loss: 2.2441 - val_mae_4x: 0.0519\n",
      "Epoch 77/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.7932 - mae_4x: 0.0472 - val_loss: 2.2554 - val_mae_4x: 0.0518\n",
      "Epoch 78/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 1.7720 - mae_4x: 0.0467 - val_loss: 2.2356 - val_mae_4x: 0.0510\n",
      "Epoch 79/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 1.7512 - mae_4x: 0.0462 - val_loss: 2.2245 - val_mae_4x: 0.0504\n",
      "Epoch 80/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.7281 - mae_4x: 0.0457 - val_loss: 2.2332 - val_mae_4x: 0.0493\n",
      "Epoch 81/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.7103 - mae_4x: 0.0453 - val_loss: 2.2544 - val_mae_4x: 0.0494\n",
      "Epoch 82/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.6901 - mae_4x: 0.0449 - val_loss: 2.2516 - val_mae_4x: 0.0494\n",
      "Epoch 83/1000\n",
      "83/83 [==============================] - 10s 115ms/step - loss: 1.6701 - mae_4x: 0.0446 - val_loss: 2.2192 - val_mae_4x: 0.0492\n",
      "Epoch 84/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.6511 - mae_4x: 0.0443 - val_loss: 2.2270 - val_mae_4x: 0.0487\n",
      "Epoch 85/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.6297 - mae_4x: 0.0440 - val_loss: 2.2637 - val_mae_4x: 0.0489\n",
      "Epoch 86/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.6172 - mae_4x: 0.0437 - val_loss: 2.2765 - val_mae_4x: 0.0487\n",
      "Epoch 87/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.5905 - mae_4x: 0.0434 - val_loss: 2.2543 - val_mae_4x: 0.0486\n",
      "Epoch 88/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.5742 - mae_4x: 0.0431 - val_loss: 2.3097 - val_mae_4x: 0.0492\n",
      "Epoch 89/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.5538 - mae_4x: 0.0428 - val_loss: 2.2917 - val_mae_4x: 0.0485\n",
      "Epoch 90/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.5385 - mae_4x: 0.0426 - val_loss: 2.3284 - val_mae_4x: 0.0485\n",
      "Epoch 91/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.5249 - mae_4x: 0.0423 - val_loss: 2.3164 - val_mae_4x: 0.0492\n",
      "Epoch 92/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.5017 - mae_4x: 0.0421 - val_loss: 2.3260 - val_mae_4x: 0.0494\n",
      "Epoch 93/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.4867 - mae_4x: 0.0419 - val_loss: 2.4082 - val_mae_4x: 0.0510\n",
      "Epoch 94/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.4768 - mae_4x: 0.0417 - val_loss: 2.5059 - val_mae_4x: 0.0521\n",
      "Epoch 95/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.4588 - mae_4x: 0.0415 - val_loss: 2.4968 - val_mae_4x: 0.0523\n",
      "Epoch 96/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.4398 - mae_4x: 0.0414 - val_loss: 2.5688 - val_mae_4x: 0.0511\n",
      "Epoch 97/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.4204 - mae_4x: 0.0412 - val_loss: 2.5577 - val_mae_4x: 0.0531\n",
      "Epoch 98/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.4230 - mae_4x: 0.0412 - val_loss: 2.6534 - val_mae_4x: 0.0529\n",
      "Epoch 99/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.4025 - mae_4x: 0.0412 - val_loss: 2.6580 - val_mae_4x: 0.0530\n",
      "Epoch 100/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.3900 - mae_4x: 0.0413 - val_loss: 2.6290 - val_mae_4x: 0.0554\n",
      "Epoch 101/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.3867 - mae_4x: 0.0414 - val_loss: 2.8400 - val_mae_4x: 0.0601\n",
      "Epoch 102/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.3922 - mae_4x: 0.0414 - val_loss: 2.7625 - val_mae_4x: 0.0619\n",
      "Epoch 103/1000\n",
      "83/83 [==============================] - 10s 117ms/step - loss: 1.3841 - mae_4x: 0.0412 - val_loss: 2.5638 - val_mae_4x: 0.0534\n",
      "Epoch 104/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.3229 - mae_4x: 0.0358 - val_loss: 2.3828 - val_mae_4x: 0.0419\n",
      "Epoch 105/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.2579 - mae_4x: 0.0346 - val_loss: 2.4715 - val_mae_4x: 0.0423\n",
      "Epoch 106/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.2373 - mae_4x: 0.0344 - val_loss: 2.4935 - val_mae_4x: 0.0428\n",
      "Epoch 107/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.2182 - mae_4x: 0.0343 - val_loss: 2.5208 - val_mae_4x: 0.0428\n",
      "Epoch 108/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.2111 - mae_4x: 0.0343 - val_loss: 2.5570 - val_mae_4x: 0.0433\n",
      "Epoch 109/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.1939 - mae_4x: 0.0342 - val_loss: 2.6019 - val_mae_4x: 0.0435\n",
      "Epoch 110/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.1853 - mae_4x: 0.0342 - val_loss: 2.6541 - val_mae_4x: 0.0443\n",
      "Epoch 111/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.1701 - mae_4x: 0.0341 - val_loss: 2.6983 - val_mae_4x: 0.0442\n",
      "Epoch 112/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.1555 - mae_4x: 0.0340 - val_loss: 2.6790 - val_mae_4x: 0.0442\n",
      "Epoch 113/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.1450 - mae_4x: 0.0340 - val_loss: 2.6662 - val_mae_4x: 0.0441\n",
      "Epoch 114/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.1345 - mae_4x: 0.0339 - val_loss: 2.7439 - val_mae_4x: 0.0446\n",
      "Epoch 115/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.1250 - mae_4x: 0.0339 - val_loss: 2.7718 - val_mae_4x: 0.0447\n",
      "Epoch 116/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.1214 - mae_4x: 0.0339 - val_loss: 2.7587 - val_mae_4x: 0.0446\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 117/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.1060 - mae_4x: 0.0338 - val_loss: 2.7800 - val_mae_4x: 0.0444\n",
      "Epoch 118/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.0980 - mae_4x: 0.0338 - val_loss: 2.8466 - val_mae_4x: 0.0446\n",
      "Epoch 119/1000\n",
      "83/83 [==============================] - 9s 109ms/step - loss: 1.0894 - mae_4x: 0.0337 - val_loss: 2.8180 - val_mae_4x: 0.0447\n",
      "Epoch 120/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.0852 - mae_4x: 0.0337 - val_loss: 2.8248 - val_mae_4x: 0.0451\n",
      "Epoch 121/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.0786 - mae_4x: 0.0337 - val_loss: 2.8044 - val_mae_4x: 0.0446\n",
      "Epoch 122/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.0770 - mae_4x: 0.0337 - val_loss: 2.8815 - val_mae_4x: 0.0449\n",
      "Epoch 123/1000\n",
      "83/83 [==============================] - 9s 108ms/step - loss: 1.0701 - mae_4x: 0.0336 - val_loss: 2.8231 - val_mae_4x: 0.0457\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7f92dc93e940>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 32\n",
    "n_stack = 1\n",
    "epochs = 1000\n",
    "sigma = 1.0\n",
    "\n",
    "model.fit_generator(train_gen.generator(batch_size, n_stack, sigma=sigma), len(train_gen)//batch_size,\n",
    "                    validation_data=valid_gen.generator(batch_size, n_stack, sigma=sigma), \n",
    "                    validation_steps=len(valid_gen)//batch_size,\n",
    "                    epochs=epochs, \n",
    "                    callbacks=my_call)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0859802779959864, 0.04566506155242678]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate_generator(test_gen.generator(1,n_stack,sigma=sigma), steps=len(test_gen))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Train.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}