top_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from keras import optimizers\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dropout, Flatten, Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import keras.backend as K\n",
    "from keras.models import load_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VGG16 Bottleneck"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train=np.load('train_data_reduced/x_train_150000_bottleneck.npy')\n",
    "x_cv=np.load('train_data_reduced/x_cv_25000_bottleneck.npy')\n",
    "\n",
    "y_train=np.load('train_data_reduced/y_train_150000.npy')\n",
    "y_cv=np.load('train_data_reduced/y_cv_25000.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Custom_lr(keras.callbacks.Callback):\n",
    "\n",
    "    def on_train_begin(self, logs={}):\n",
    "\t    K.set_value(self.model.optimizer.lr, 0.001)\n",
    " \n",
    "    def on_epoch_begin(self, epoch, logs={}):\n",
    "        lr_present=K.get_value(self.model.optimizer.lr)\n",
    "        #print(epoch)\n",
    "        if (epoch%10==0) and epoch:\n",
    "        \t\n",
    "            K.set_value(self.model.optimizer.lr, lr_present/((epoch)**0.5))\n",
    "            print(K.get_value(self.model.optimizer.lr))\n",
    "            print(lr_present/((epoch)**0.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(64, activation='relu'))\n",
    "top_model.add(Dropout(0.5))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_6 (Flatten)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 64)                131136    \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 131,201\n",
      "Trainable params: 131,201\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [\n",
    "\t    Custom_lr()\n",
    "\t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.6290 - acc: 0.6680 - val_loss: 0.5196 - val_acc: 0.7350\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 16s 110us/step - loss: 0.5444 - acc: 0.7069 - val_loss: 0.4953 - val_acc: 0.7519\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.5162 - acc: 0.7274 - val_loss: 0.4614 - val_acc: 0.7717\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 16s 108us/step - loss: 0.4982 - acc: 0.7413 - val_loss: 0.4572 - val_acc: 0.7760\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 16s 108us/step - loss: 0.4816 - acc: 0.7488 - val_loss: 0.4456 - val_acc: 0.7815\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 16s 108us/step - loss: 0.4698 - acc: 0.7566 - val_loss: 0.4258 - val_acc: 0.7948\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 16s 108us/step - loss: 0.4563 - acc: 0.7646 - val_loss: 0.4210 - val_acc: 0.7988\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 16s 110us/step - loss: 0.4476 - acc: 0.7695 - val_loss: 0.4185 - val_acc: 0.8060\n",
      "Epoch 9/35\n",
      "150000/150000 [==============================] - 16s 108us/step - loss: 0.4367 - acc: 0.7753 - val_loss: 0.4081 - val_acc: 0.8108\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.4301 - acc: 0.7785 - val_loss: 0.4072 - val_acc: 0.8106\n",
      "Epoch 11/35\n",
      "0.0003162278\n",
      "0.00031622778103685084\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3979 - acc: 0.7952 - val_loss: 0.3865 - val_acc: 0.8226\n",
      "Epoch 12/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3892 - acc: 0.7991 - val_loss: 0.3846 - val_acc: 0.8262\n",
      "Epoch 13/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3823 - acc: 0.8020 - val_loss: 0.3827 - val_acc: 0.8268\n",
      "Epoch 14/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3772 - acc: 0.8051 - val_loss: 0.3817 - val_acc: 0.8309\n",
      "Epoch 15/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3724 - acc: 0.8075 - val_loss: 0.3824 - val_acc: 0.8320\n",
      "Epoch 16/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3674 - acc: 0.8095 - val_loss: 0.3814 - val_acc: 0.8325\n",
      "Epoch 17/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3636 - acc: 0.8110 - val_loss: 0.3799 - val_acc: 0.8344\n",
      "Epoch 18/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3606 - acc: 0.8124 - val_loss: 0.3775 - val_acc: 0.8360\n",
      "Epoch 19/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3592 - acc: 0.8127 - val_loss: 0.3794 - val_acc: 0.8362\n",
      "Epoch 20/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3524 - acc: 0.8171 - val_loss: 0.3775 - val_acc: 0.8367\n",
      "Epoch 21/35\n",
      "7.071068e-05\n",
      "7.071068264135355e-05\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3432 - acc: 0.8222 - val_loss: 0.3766 - val_acc: 0.8386\n",
      "Epoch 22/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3392 - acc: 0.8229 - val_loss: 0.3768 - val_acc: 0.8402\n",
      "Epoch 23/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3361 - acc: 0.8250 - val_loss: 0.3788 - val_acc: 0.8396\n",
      "Epoch 24/35\n",
      "150000/150000 [==============================] - 16s 108us/step - loss: 0.3343 - acc: 0.8239 - val_loss: 0.3783 - val_acc: 0.8422\n",
      "Epoch 25/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3342 - acc: 0.8249 - val_loss: 0.3790 - val_acc: 0.8420\n",
      "Epoch 26/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3313 - acc: 0.8254 - val_loss: 0.3788 - val_acc: 0.8424\n",
      "Epoch 27/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3299 - acc: 0.8270 - val_loss: 0.3797 - val_acc: 0.8429\n",
      "Epoch 28/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3300 - acc: 0.8270 - val_loss: 0.3807 - val_acc: 0.8426\n",
      "Epoch 29/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3292 - acc: 0.8284 - val_loss: 0.3805 - val_acc: 0.8426\n",
      "Epoch 30/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3265 - acc: 0.8275 - val_loss: 0.3830 - val_acc: 0.8422\n",
      "Epoch 31/35\n",
      "1.2909945e-05\n",
      "1.2909945018820682e-05\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3241 - acc: 0.8315 - val_loss: 0.3820 - val_acc: 0.8425\n",
      "Epoch 32/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3242 - acc: 0.8308 - val_loss: 0.3824 - val_acc: 0.8434\n",
      "Epoch 33/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3215 - acc: 0.8318 - val_loss: 0.3833 - val_acc: 0.8428\n",
      "Epoch 34/35\n",
      "150000/150000 [==============================] - 16s 108us/step - loss: 0.3227 - acc: 0.8302 - val_loss: 0.3837 - val_acc: 0.8430\n",
      "Epoch 35/35\n",
      "150000/150000 [==============================] - 16s 109us/step - loss: 0.3230 - acc: 0.8318 - val_loss: 0.3833 - val_acc: 0.8426\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2bbbb623630>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv), callbacks=callbacks,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save('top_model_64_adam_custom_lr.h5') # best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import LeakyReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_8 (Flatten)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 32)                65568     \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 66,657\n",
      "Trainable params: 66,657\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(32, activation='relu'))\n",
    "top_model.add(Dropout(0.3))\n",
    "top_model.add(Dense(32))\n",
    "top_model.add(LeakyReLU())\n",
    "top_model.add(Dropout(0.3))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 20s 130us/step - loss: 0.5470 - acc: 0.7017 - val_loss: 0.5080 - val_acc: 0.7396\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.5177 - acc: 0.7231 - val_loss: 0.4836 - val_acc: 0.7608\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.4939 - acc: 0.7398 - val_loss: 0.4641 - val_acc: 0.7672\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.4783 - acc: 0.7492 - val_loss: 0.4527 - val_acc: 0.7714\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.4614 - acc: 0.7572 - val_loss: 0.4437 - val_acc: 0.7851\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.4525 - acc: 0.7637 - val_loss: 0.4389 - val_acc: 0.7855\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.4408 - acc: 0.7702 - val_loss: 0.4372 - val_acc: 0.7921\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.4328 - acc: 0.7739 - val_loss: 0.4289 - val_acc: 0.7945\n",
      "Epoch 9/35\n",
      "150000/150000 [==============================] - 20s 133us/step - loss: 0.4247 - acc: 0.7791 - val_loss: 0.4301 - val_acc: 0.7902\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 19s 129us/step - loss: 0.4194 - acc: 0.7824 - val_loss: 0.4246 - val_acc: 0.7981\n",
      "Epoch 11/35\n",
      "150000/150000 [==============================] - 20s 130us/step - loss: 0.4123 - acc: 0.7861 - val_loss: 0.4275 - val_acc: 0.8013\n",
      "Epoch 12/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.4076 - acc: 0.7899 - val_loss: 0.4171 - val_acc: 0.8031\n",
      "Epoch 13/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.4024 - acc: 0.7935 - val_loss: 0.4172 - val_acc: 0.8034\n",
      "Epoch 14/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.3976 - acc: 0.7958 - val_loss: 0.4206 - val_acc: 0.8065\n",
      "Epoch 15/35\n",
      "150000/150000 [==============================] - 20s 131us/step - loss: 0.3926 - acc: 0.7995 - val_loss: 0.4177 - val_acc: 0.8071\n",
      "Epoch 16/35\n",
      "150000/150000 [==============================] - 19s 129us/step - loss: 0.3863 - acc: 0.8025 - val_loss: 0.4103 - val_acc: 0.8112\n",
      "Epoch 17/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.3837 - acc: 0.8046 - val_loss: 0.4167 - val_acc: 0.8067\n",
      "Epoch 18/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.3784 - acc: 0.8070 - val_loss: 0.4116 - val_acc: 0.8106\n",
      "Epoch 19/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.3749 - acc: 0.8096 - val_loss: 0.4131 - val_acc: 0.8151\n",
      "Epoch 20/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.3727 - acc: 0.8127 - val_loss: 0.4127 - val_acc: 0.8183\n",
      "Epoch 21/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.3684 - acc: 0.8147 - val_loss: 0.4114 - val_acc: 0.8184\n",
      "Epoch 22/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.3672 - acc: 0.8149 - val_loss: 0.4319 - val_acc: 0.8191\n",
      "Epoch 23/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.3616 - acc: 0.8175 - val_loss: 0.4106 - val_acc: 0.8159\n",
      "Epoch 24/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.3599 - acc: 0.8188 - val_loss: 0.4195 - val_acc: 0.8165\n",
      "Epoch 25/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.3579 - acc: 0.8201 - val_loss: 0.4300 - val_acc: 0.8208\n",
      "Epoch 26/35\n",
      "150000/150000 [==============================] - 19s 126us/step - loss: 0.3550 - acc: 0.8204 - val_loss: 0.4131 - val_acc: 0.8147\n",
      "Epoch 27/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.3522 - acc: 0.8226 - val_loss: 0.4062 - val_acc: 0.8135\n",
      "Epoch 28/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.3500 - acc: 0.8224 - val_loss: 0.4155 - val_acc: 0.8220\n",
      "Epoch 29/35\n",
      "150000/150000 [==============================] - 19s 126us/step - loss: 0.3491 - acc: 0.8252 - val_loss: 0.4134 - val_acc: 0.8220\n",
      "Epoch 30/35\n",
      "150000/150000 [==============================] - 19s 129us/step - loss: 0.3455 - acc: 0.8268 - val_loss: 0.4088 - val_acc: 0.8250\n",
      "Epoch 31/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.3449 - acc: 0.8281 - val_loss: 0.4222 - val_acc: 0.8232\n",
      "Epoch 32/35\n",
      "150000/150000 [==============================] - 19s 128us/step - loss: 0.3438 - acc: 0.8281 - val_loss: 0.4212 - val_acc: 0.8215\n",
      "Epoch 33/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.3418 - acc: 0.8282 - val_loss: 0.4273 - val_acc: 0.8270\n",
      "Epoch 34/35\n",
      "150000/150000 [==============================] - 19s 127us/step - loss: 0.3389 - acc: 0.8296 - val_loss: 0.4264 - val_acc: 0.8238\n",
      "Epoch 35/35\n",
      "150000/150000 [==============================] - 19s 126us/step - loss: 0.3382 - acc: 0.8324 - val_loss: 0.4049 - val_acc: 0.8232\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2bbc8a12748>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save('top_model_2_layer_leaky_relu.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_12 (Flatten)         (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 32)                65568     \n",
      "_________________________________________________________________\n",
      "dropout_12 (Dropout)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 65,601\n",
      "Trainable params: 65,601\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(32, activation='relu'))\n",
    "top_model.add(Dropout(0.35))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 18s 119us/step - loss: 0.5983 - acc: 0.6776 - val_loss: 0.5123 - val_acc: 0.7374\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.5286 - acc: 0.7216 - val_loss: 0.4885 - val_acc: 0.7550\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.5031 - acc: 0.7390 - val_loss: 0.4658 - val_acc: 0.7708\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.4836 - acc: 0.7493 - val_loss: 0.4478 - val_acc: 0.7815\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.4672 - acc: 0.7582 - val_loss: 0.4413 - val_acc: 0.7886\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.4545 - acc: 0.7659 - val_loss: 0.4355 - val_acc: 0.7945\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.4434 - acc: 0.7725 - val_loss: 0.4267 - val_acc: 0.7999\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.4339 - acc: 0.7774 - val_loss: 0.4212 - val_acc: 0.8038\n",
      "Epoch 9/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.4269 - acc: 0.7801 - val_loss: 0.4175 - val_acc: 0.8020\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.4196 - acc: 0.7861 - val_loss: 0.4126 - val_acc: 0.8097\n",
      "Epoch 11/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.4106 - acc: 0.7897 - val_loss: 0.4171 - val_acc: 0.8105\n",
      "Epoch 12/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.4071 - acc: 0.7927 - val_loss: 0.4136 - val_acc: 0.8129\n",
      "Epoch 13/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.4012 - acc: 0.7944 - val_loss: 0.4125 - val_acc: 0.8140\n",
      "Epoch 14/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3968 - acc: 0.7987 - val_loss: 0.4181 - val_acc: 0.8098\n",
      "Epoch 15/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3922 - acc: 0.7998 - val_loss: 0.4093 - val_acc: 0.8148\n",
      "Epoch 16/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3866 - acc: 0.8027 - val_loss: 0.4098 - val_acc: 0.8178\n",
      "Epoch 17/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3837 - acc: 0.8045 - val_loss: 0.4075 - val_acc: 0.8165\n",
      "Epoch 18/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3778 - acc: 0.8078 - val_loss: 0.4085 - val_acc: 0.8194\n",
      "Epoch 19/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3763 - acc: 0.8082 - val_loss: 0.4134 - val_acc: 0.8218\n",
      "Epoch 20/35\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3737 - acc: 0.8097 - val_loss: 0.4175 - val_acc: 0.8178\n",
      "Epoch 21/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3687 - acc: 0.8127 - val_loss: 0.4127 - val_acc: 0.8214\n",
      "Epoch 22/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3658 - acc: 0.8147 - val_loss: 0.4124 - val_acc: 0.8207\n",
      "Epoch 23/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3652 - acc: 0.8144 - val_loss: 0.4035 - val_acc: 0.8224\n",
      "Epoch 24/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3605 - acc: 0.8170 - val_loss: 0.4119 - val_acc: 0.8186\n",
      "Epoch 25/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3591 - acc: 0.8174 - val_loss: 0.4175 - val_acc: 0.8226\n",
      "Epoch 26/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3556 - acc: 0.8184 - val_loss: 0.4178 - val_acc: 0.8246\n",
      "Epoch 27/35\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3554 - acc: 0.8193 - val_loss: 0.4129 - val_acc: 0.8230\n",
      "Epoch 28/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3521 - acc: 0.8187 - val_loss: 0.4181 - val_acc: 0.8240\n",
      "Epoch 29/35\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3512 - acc: 0.8219 - val_loss: 0.4115 - val_acc: 0.8276\n",
      "Epoch 30/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3497 - acc: 0.8221 - val_loss: 0.4193 - val_acc: 0.8265\n",
      "Epoch 31/35\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3483 - acc: 0.8225 - val_loss: 0.4266 - val_acc: 0.8259\n",
      "Epoch 32/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3455 - acc: 0.8235 - val_loss: 0.4261 - val_acc: 0.8244\n",
      "Epoch 33/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3446 - acc: 0.8259 - val_loss: 0.4210 - val_acc: 0.8277\n",
      "Epoch 34/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3413 - acc: 0.8268 - val_loss: 0.4271 - val_acc: 0.8244\n",
      "Epoch 35/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3415 - acc: 0.8265 - val_loss: 0.4232 - val_acc: 0.8227\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2bbcc67b978>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv),)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save('top_model_32_adam.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train best top model architecture on larger data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train=np.load('train_data/x_train_bottleneck.npy')\n",
    "x_cv=np.load('train_data/x_cv_bottleneck.npy')\n",
    "\n",
    "y_train=np.load('train_data/y_train.npy')\n",
    "y_cv=np.load('train_data/y_cv.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(64, activation='relu'))\n",
    "top_model.add(Dropout(0.5))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_2 (Flatten)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 64)                131136    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 131,201\n",
      "Trainable params: 131,201\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [\n",
    "\t    Custom_lr()\n",
    "\t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 245509 samples, validate on 105219 samples\n",
      "Epoch 1/35\n",
      "245509/245509 [==============================] - 29s 120us/step - loss: 0.6017 - acc: 0.6863 - val_loss: 0.5006 - val_acc: 0.7479\n",
      "Epoch 2/35\n",
      "245509/245509 [==============================] - 29s 120us/step - loss: 0.5225 - acc: 0.7241 - val_loss: 0.4587 - val_acc: 0.7744\n",
      "Epoch 3/35\n",
      "245509/245509 [==============================] - 29s 117us/step - loss: 0.4957 - acc: 0.7401 - val_loss: 0.4426 - val_acc: 0.7843\n",
      "Epoch 4/35\n",
      "245509/245509 [==============================] - 29s 117us/step - loss: 0.4786 - acc: 0.7508 - val_loss: 0.4298 - val_acc: 0.7941\n",
      "Epoch 5/35\n",
      "245509/245509 [==============================] - 29s 118us/step - loss: 0.4646 - acc: 0.7583 - val_loss: 0.4165 - val_acc: 0.8000\n",
      "Epoch 6/35\n",
      "245509/245509 [==============================] - 29s 118us/step - loss: 0.4545 - acc: 0.7632 - val_loss: 0.4088 - val_acc: 0.8051\n",
      "Epoch 7/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.4441 - acc: 0.7689 - val_loss: 0.3916 - val_acc: 0.8151\n",
      "Epoch 8/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.4356 - acc: 0.7748 - val_loss: 0.3874 - val_acc: 0.8211\n",
      "Epoch 9/35\n",
      "245509/245509 [==============================] - 29s 118us/step - loss: 0.4275 - acc: 0.7786 - val_loss: 0.3798 - val_acc: 0.8236\n",
      "Epoch 10/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.4220 - acc: 0.7819 - val_loss: 0.3851 - val_acc: 0.8225\n",
      "Epoch 11/35\n",
      "0.0003162278\n",
      "0.00031622778103685084\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3929 - acc: 0.7946 - val_loss: 0.3598 - val_acc: 0.8357\n",
      "Epoch 12/35\n",
      "245509/245509 [==============================] - 28s 116us/step - loss: 0.3840 - acc: 0.8000 - val_loss: 0.3560 - val_acc: 0.8385\n",
      "Epoch 13/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3772 - acc: 0.8027 - val_loss: 0.3553 - val_acc: 0.8396\n",
      "Epoch 14/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3720 - acc: 0.8060 - val_loss: 0.3496 - val_acc: 0.8417\n",
      "Epoch 15/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3685 - acc: 0.8063 - val_loss: 0.3491 - val_acc: 0.8416\n",
      "Epoch 16/35\n",
      "245509/245509 [==============================] - 29s 118us/step - loss: 0.3654 - acc: 0.8092 - val_loss: 0.3475 - val_acc: 0.8442\n",
      "Epoch 17/35\n",
      "245509/245509 [==============================] - 29s 117us/step - loss: 0.3630 - acc: 0.8108 - val_loss: 0.3471 - val_acc: 0.8445\n",
      "Epoch 18/35\n",
      "245509/245509 [==============================] - 29s 117us/step - loss: 0.3602 - acc: 0.8117 - val_loss: 0.3464 - val_acc: 0.8455\n",
      "Epoch 19/35\n",
      "245509/245509 [==============================] - 29s 118us/step - loss: 0.3581 - acc: 0.8142 - val_loss: 0.3459 - val_acc: 0.8462\n",
      "Epoch 20/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3546 - acc: 0.8155 - val_loss: 0.3437 - val_acc: 0.8475\n",
      "Epoch 21/35\n",
      "7.071068e-05\n",
      "7.071068264135355e-05\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3432 - acc: 0.8207 - val_loss: 0.3417 - val_acc: 0.8498\n",
      "Epoch 22/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3415 - acc: 0.8211 - val_loss: 0.3405 - val_acc: 0.8501\n",
      "Epoch 23/35\n",
      "245509/245509 [==============================] - 29s 117us/step - loss: 0.3404 - acc: 0.8227 - val_loss: 0.3405 - val_acc: 0.8508\n",
      "Epoch 24/35\n",
      "245509/245509 [==============================] - 29s 117us/step - loss: 0.3378 - acc: 0.8241 - val_loss: 0.3404 - val_acc: 0.8507\n",
      "Epoch 25/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3371 - acc: 0.8228 - val_loss: 0.3398 - val_acc: 0.8513\n",
      "Epoch 26/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3358 - acc: 0.8243 - val_loss: 0.3399 - val_acc: 0.8517\n",
      "Epoch 27/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3345 - acc: 0.8247 - val_loss: 0.3406 - val_acc: 0.8519\n",
      "Epoch 28/35\n",
      "245509/245509 [==============================] - 29s 117us/step - loss: 0.3344 - acc: 0.8256 - val_loss: 0.3400 - val_acc: 0.8510\n",
      "Epoch 29/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3331 - acc: 0.8261 - val_loss: 0.3394 - val_acc: 0.8527\n",
      "Epoch 30/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3328 - acc: 0.8265 - val_loss: 0.3397 - val_acc: 0.8523\n",
      "Epoch 31/35\n",
      "1.2909945e-05\n",
      "1.2909945018820682e-05\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3300 - acc: 0.8275 - val_loss: 0.3393 - val_acc: 0.8522\n",
      "Epoch 32/35\n",
      "245509/245509 [==============================] - 28s 116us/step - loss: 0.3289 - acc: 0.8281 - val_loss: 0.3395 - val_acc: 0.8527\n",
      "Epoch 33/35\n",
      "245509/245509 [==============================] - 28s 116us/step - loss: 0.3284 - acc: 0.8275 - val_loss: 0.3397 - val_acc: 0.8524\n",
      "Epoch 34/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3289 - acc: 0.8286 - val_loss: 0.3400 - val_acc: 0.8528\n",
      "Epoch 35/35\n",
      "245509/245509 [==============================] - 29s 116us/step - loss: 0.3282 - acc: 0.8284 - val_loss: 0.3398 - val_acc: 0.8525\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x273b745bc50>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv), callbacks=callbacks,)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The best model with small data could not bring test loss lower than 0.37"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save('top_model_full_data_custom_lr.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_model('top_model_full_data_custom_lr.h5').save_weights('top_model_full_data_custom_lr_weights.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Resnet50 bottleneck</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Custom_lr_2(keras.callbacks.Callback):\n",
    "\n",
    "    def on_train_begin(self, logs={}):\n",
    "        K.set_value(self.model.optimizer.lr, 0.001)\n",
    "        self.losses=[]\n",
    " \n",
    "    def on_epoch_begin(self, epoch, logs={}):\n",
    "        lr_present=K.get_value(self.model.optimizer.lr)\n",
    "        #print(epoch)\n",
    "        if epoch==0 or epoch==1:\n",
    "            return\n",
    "        \n",
    "        if abs(self.losses[epoch-1]-self.losses[epoch-2])<1e-2:\n",
    "            K.set_value(self.model.optimizer.lr, lr_present/((epoch)**0.5))\n",
    "            print(K.get_value(self.model.optimizer.lr))\n",
    "            print(lr_present/((epoch)**0.5))\n",
    "    \n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        self.losses.append(logs.get('loss'))\n",
    "        return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train=np.load('train_data_reduced/x_train_resnet_150000_bottleneck.npy')\n",
    "x_cv=np.load('train_data_reduced/x_cv_resnet_25000_bottleneck.npy')\n",
    "\n",
    "y_train=np.load('train_data_reduced/y_train_150000.npy')\n",
    "y_cv=np.load('train_data_reduced/y_cv_25000.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150000, 2, 2, 2048)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(64, activation='relu'))\n",
    "top_model.add(Dropout(0.5))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 8192)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 64)                524352    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 524,417\n",
      "Trainable params: 524,417\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [\n",
    "\t    Custom_lr()\n",
    "\t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 28s 190us/step - loss: 0.5212 - acc: 0.7295 - val_loss: 0.4122 - val_acc: 0.8050\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 27s 180us/step - loss: 0.4316 - acc: 0.7864 - val_loss: 0.3560 - val_acc: 0.8350\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 27s 183us/step - loss: 0.3878 - acc: 0.8108 - val_loss: 0.3241 - val_acc: 0.8555\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 27s 178us/step - loss: 0.3555 - acc: 0.8287 - val_loss: 0.3004 - val_acc: 0.8693\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 27s 179us/step - loss: 0.3325 - acc: 0.8430 - val_loss: 0.2921 - val_acc: 0.8711\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 26s 176us/step - loss: 0.3118 - acc: 0.8539 - val_loss: 0.2805 - val_acc: 0.8798\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 26s 175us/step - loss: 0.2977 - acc: 0.8607 - val_loss: 0.2696 - val_acc: 0.8836\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 25s 168us/step - loss: 0.2809 - acc: 0.8705 - val_loss: 0.2630 - val_acc: 0.8897\n",
      "Epoch 9/35\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.2698 - acc: 0.8755 - val_loss: 0.2508 - val_acc: 0.8951\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 26s 172us/step - loss: 0.2596 - acc: 0.8809 - val_loss: 0.2462 - val_acc: 0.8962\n",
      "Epoch 11/35\n",
      "0.0003162278\n",
      "0.00031622778103685084\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.2187 - acc: 0.9005 - val_loss: 0.2322 - val_acc: 0.9060\n",
      "Epoch 12/35\n",
      "150000/150000 [==============================] - 26s 176us/step - loss: 0.2048 - acc: 0.9075 - val_loss: 0.2347 - val_acc: 0.9090\n",
      "Epoch 13/35\n",
      "150000/150000 [==============================] - 26s 172us/step - loss: 0.1975 - acc: 0.9110 - val_loss: 0.2330 - val_acc: 0.9119\n",
      "Epoch 14/35\n",
      "150000/150000 [==============================] - 25s 169us/step - loss: 0.1890 - acc: 0.9144 - val_loss: 0.2368 - val_acc: 0.9112\n",
      "Epoch 15/35\n",
      "150000/150000 [==============================] - 27s 179us/step - loss: 0.1839 - acc: 0.9171 - val_loss: 0.2368 - val_acc: 0.9112\n",
      "Epoch 16/35\n",
      "150000/150000 [==============================] - 26s 174us/step - loss: 0.1801 - acc: 0.9193 - val_loss: 0.2330 - val_acc: 0.9128\n",
      "Epoch 17/35\n",
      "150000/150000 [==============================] - 26s 172us/step - loss: 0.1762 - acc: 0.9209 - val_loss: 0.2453 - val_acc: 0.9110\n",
      "Epoch 18/35\n",
      "150000/150000 [==============================] - 26s 172us/step - loss: 0.1706 - acc: 0.9228 - val_loss: 0.2456 - val_acc: 0.9133\n",
      "Epoch 19/35\n",
      "150000/150000 [==============================] - 27s 177us/step - loss: 0.1677 - acc: 0.9241 - val_loss: 0.2465 - val_acc: 0.9144\n",
      "Epoch 20/35\n",
      "150000/150000 [==============================] - 26s 170us/step - loss: 0.1649 - acc: 0.9256 - val_loss: 0.2528 - val_acc: 0.9132\n",
      "Epoch 21/35\n",
      "7.071068e-05\n",
      "7.071068264135355e-05\n",
      "150000/150000 [==============================] - 25s 169us/step - loss: 0.1542 - acc: 0.9308 - val_loss: 0.2511 - val_acc: 0.9161\n",
      "Epoch 22/35\n",
      "150000/150000 [==============================] - 26s 174us/step - loss: 0.1481 - acc: 0.9343 - val_loss: 0.2575 - val_acc: 0.9161\n",
      "Epoch 23/35\n",
      "150000/150000 [==============================] - 26s 175us/step - loss: 0.1458 - acc: 0.9348 - val_loss: 0.2581 - val_acc: 0.9159\n",
      "Epoch 24/35\n",
      "150000/150000 [==============================] - 26s 176us/step - loss: 0.1442 - acc: 0.9362 - val_loss: 0.2603 - val_acc: 0.9170\n",
      "Epoch 25/35\n",
      "150000/150000 [==============================] - 26s 177us/step - loss: 0.1432 - acc: 0.9358 - val_loss: 0.2650 - val_acc: 0.9172\n",
      "Epoch 26/35\n",
      "150000/150000 [==============================] - 26s 172us/step - loss: 0.1407 - acc: 0.9381 - val_loss: 0.2673 - val_acc: 0.9165\n",
      "Epoch 27/35\n",
      "150000/150000 [==============================] - 27s 178us/step - loss: 0.1405 - acc: 0.9373 - val_loss: 0.2724 - val_acc: 0.9159\n",
      "Epoch 28/35\n",
      "150000/150000 [==============================] - 27s 180us/step - loss: 0.1377 - acc: 0.9390 - val_loss: 0.2734 - val_acc: 0.9167\n",
      "Epoch 29/35\n",
      "150000/150000 [==============================] - 27s 178us/step - loss: 0.1363 - acc: 0.9397 - val_loss: 0.2721 - val_acc: 0.9163\n",
      "Epoch 30/35\n",
      "150000/150000 [==============================] - 27s 183us/step - loss: 0.1364 - acc: 0.9392 - val_loss: 0.2778 - val_acc: 0.9172\n",
      "Epoch 31/35\n",
      "1.2909945e-05\n",
      "1.2909945018820682e-05\n",
      "150000/150000 [==============================] - 28s 184us/step - loss: 0.1318 - acc: 0.9418 - val_loss: 0.2779 - val_acc: 0.9170\n",
      "Epoch 32/35\n",
      "150000/150000 [==============================] - 27s 182us/step - loss: 0.1313 - acc: 0.9417 - val_loss: 0.2803 - val_acc: 0.9166\n",
      "Epoch 33/35\n",
      "150000/150000 [==============================] - 26s 173us/step - loss: 0.1320 - acc: 0.9413 - val_loss: 0.2813 - val_acc: 0.9162\n",
      "Epoch 34/35\n",
      "150000/150000 [==============================] - 27s 180us/step - loss: 0.1311 - acc: 0.9421 - val_loss: 0.2820 - val_acc: 0.9167\n",
      "Epoch 35/35\n",
      "150000/150000 [==============================] - 28s 188us/step - loss: 0.1297 - acc: 0.9424 - val_loss: 0.2843 - val_acc: 0.9166\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2c0c53942e8>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv), \n",
    "            callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save_weights('top_model_resnet_64_adam_custom_lr.h5') # best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model_custom_lr2=Sequential()\n",
    "top_model_custom_lr2.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model_custom_lr2.add(Dense(64, activation='relu'))\n",
    "top_model_custom_lr2.add(Dropout(0.5))\n",
    "top_model_custom_lr2.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model_custom_lr2.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_5 (Flatten)          (None, 8192)              0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 64)                524352    \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 524,417\n",
      "Trainable params: 524,417\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model_custom_lr2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [\n",
    "\t    Custom_lr_2()\n",
    "\t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 26s 174us/step - loss: 0.5270 - acc: 0.7237 - val_loss: 0.4177 - val_acc: 0.8006\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 26s 172us/step - loss: 0.4367 - acc: 0.7824 - val_loss: 0.3712 - val_acc: 0.8286\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 26s 175us/step - loss: 0.3943 - acc: 0.8061 - val_loss: 0.3291 - val_acc: 0.8514\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 26s 173us/step - loss: 0.3610 - acc: 0.8226 - val_loss: 0.3049 - val_acc: 0.8661\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 25s 167us/step - loss: 0.3361 - acc: 0.8366 - val_loss: 0.2967 - val_acc: 0.8696\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 28s 187us/step - loss: 0.3160 - acc: 0.8478 - val_loss: 0.2788 - val_acc: 0.8792\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 28s 186us/step - loss: 0.3001 - acc: 0.8568 - val_loss: 0.2749 - val_acc: 0.8802\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 27s 183us/step - loss: 0.2873 - acc: 0.8620 - val_loss: 0.2590 - val_acc: 0.8866\n",
      "Epoch 9/35\n",
      "150000/150000 [==============================] - 27s 181us/step - loss: 0.2757 - acc: 0.8688 - val_loss: 0.2577 - val_acc: 0.8924\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 26s 174us/step - loss: 0.2664 - acc: 0.8734 - val_loss: 0.2485 - val_acc: 0.8934\n",
      "Epoch 11/35\n",
      "0.0003162278\n",
      "0.00031622778103685084\n",
      "150000/150000 [==============================] - 25s 164us/step - loss: 0.2287 - acc: 0.8909 - val_loss: 0.2326 - val_acc: 0.9025\n",
      "Epoch 12/35\n",
      "150000/150000 [==============================] - 25s 166us/step - loss: 0.2139 - acc: 0.8971 - val_loss: 0.2392 - val_acc: 0.9051\n",
      "Epoch 13/35\n",
      "150000/150000 [==============================] - 27s 178us/step - loss: 0.2056 - acc: 0.9016 - val_loss: 0.2372 - val_acc: 0.9060\n",
      "Epoch 14/35\n",
      "8.7705805e-05\n",
      "8.770580754042039e-05\n",
      "150000/150000 [==============================] - 28s 187us/step - loss: 0.1882 - acc: 0.9101 - val_loss: 0.2378 - val_acc: 0.9098\n",
      "Epoch 15/35\n",
      "150000/150000 [==============================] - 28s 189us/step - loss: 0.1844 - acc: 0.9114 - val_loss: 0.2394 - val_acc: 0.9102\n",
      "Epoch 16/35\n",
      "2.2645541e-05\n",
      "2.264554137652806e-05\n",
      "150000/150000 [==============================] - 28s 187us/step - loss: 0.1786 - acc: 0.9125 - val_loss: 0.2409 - val_acc: 0.9104\n",
      "Epoch 17/35\n",
      "5.6613853e-06\n",
      "5.661385330313351e-06\n",
      "150000/150000 [==============================] - 28s 189us/step - loss: 0.1786 - acc: 0.9135 - val_loss: 0.2413 - val_acc: 0.9101\n",
      "Epoch 18/35\n",
      "1.3730877e-06\n",
      "1.373087629659075e-06\n",
      "150000/150000 [==============================] - 28s 187us/step - loss: 0.1773 - acc: 0.9143 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 19/35\n",
      "3.2363988e-07\n",
      "3.2363987094422374e-07\n",
      "150000/150000 [==============================] - 28s 185us/step - loss: 0.1774 - acc: 0.9151 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 20/35\n",
      "7.424808e-08\n",
      "7.424808083619956e-08\n",
      "150000/150000 [==============================] - 27s 181us/step - loss: 0.1772 - acc: 0.9152 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 21/35\n",
      "1.6602375e-08\n",
      "1.6602374940126426e-08\n",
      "150000/150000 [==============================] - 27s 182us/step - loss: 0.1771 - acc: 0.9147 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 22/35\n",
      "3.622935e-09\n",
      "3.6229351551142435e-09\n",
      "150000/150000 [==============================] - 26s 177us/step - loss: 0.1778 - acc: 0.9136 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 23/35\n",
      "7.7241236e-10\n",
      "7.724123507549834e-10\n",
      "150000/150000 [==============================] - 27s 178us/step - loss: 0.1792 - acc: 0.9139 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 24/35\n",
      "1.6105911e-10\n",
      "1.6105910988392012e-10\n",
      "150000/150000 [==============================] - 26s 174us/step - loss: 0.1798 - acc: 0.9135 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 25/35\n",
      "3.2876053e-11\n",
      "3.287605326115812e-11\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.1773 - acc: 0.9150 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 26/35\n",
      "6.5752104e-12\n",
      "6.575210514947471e-12\n",
      "150000/150000 [==============================] - 27s 181us/step - loss: 0.1800 - acc: 0.9136 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 27/35\n",
      "1.2895049e-12\n",
      "1.289504856901084e-12\n",
      "150000/150000 [==============================] - 27s 179us/step - loss: 0.1794 - acc: 0.9134 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 28/35\n",
      "2.481653e-13\n",
      "2.4816532461377917e-13\n",
      "150000/150000 [==============================] - 26s 173us/step - loss: 0.1791 - acc: 0.9145 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 29/35\n",
      "4.6898835e-14\n",
      "4.6898835700645123e-14\n",
      "150000/150000 [==============================] - 26s 173us/step - loss: 0.1780 - acc: 0.9144 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 30/35\n",
      "8.7088955e-15\n",
      "8.708895084826156e-15\n",
      "150000/150000 [==============================] - 26s 173us/step - loss: 0.1784 - acc: 0.9139 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 31/35\n",
      "1.5900195e-15\n",
      "1.5900194972382688e-15\n",
      "150000/150000 [==============================] - 25s 168us/step - loss: 0.1787 - acc: 0.9140 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 32/35\n",
      "2.8557594e-16\n",
      "2.855759403641947e-16\n",
      "150000/150000 [==============================] - 26s 175us/step - loss: 0.1779 - acc: 0.9143 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 33/35\n",
      "5.048317e-17\n",
      "5.0483171027882134e-17\n",
      "150000/150000 [==============================] - 28s 185us/step - loss: 0.1789 - acc: 0.9133 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 34/35\n",
      "8.787992e-18\n",
      "8.787991946475677e-18\n",
      "150000/150000 [==============================] - 28s 187us/step - loss: 0.1771 - acc: 0.9141 - val_loss: 0.2415 - val_acc: 0.9104\n",
      "Epoch 35/35\n",
      "1.5071281e-18\n",
      "1.5071281330498602e-18\n",
      "150000/150000 [==============================] - 28s 183us/step - loss: 0.1778 - acc: 0.9145 - val_loss: 0.2415 - val_acc: 0.9104\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2c24ae0b208>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model_custom_lr2.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv), \n",
    "            callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model_custom_lr2.save_weights('top_model_resnet_64_adam_custom_lr2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import LeakyReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_6 (Flatten)          (None, 8192)              0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 32)                262176    \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 263,265\n",
      "Trainable params: 263,265\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(32, activation='relu'))\n",
    "top_model.add(Dropout(0.2))\n",
    "top_model.add(Dense(32))\n",
    "top_model.add(LeakyReLU())\n",
    "top_model.add(Dropout(0.2))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 29s 190us/step - loss: 0.5093 - acc: 0.7337 - val_loss: 0.4084 - val_acc: 0.8019\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 30s 200us/step - loss: 0.4010 - acc: 0.8042 - val_loss: 0.3521 - val_acc: 0.8371\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 29s 192us/step - loss: 0.3467 - acc: 0.8344 - val_loss: 0.3225 - val_acc: 0.8561\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 30s 198us/step - loss: 0.3071 - acc: 0.8562 - val_loss: 0.3038 - val_acc: 0.8641\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 30s 202us/step - loss: 0.2768 - acc: 0.8723 - val_loss: 0.2846 - val_acc: 0.8753\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 29s 190us/step - loss: 0.2549 - acc: 0.8828 - val_loss: 0.2781 - val_acc: 0.8839\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 28s 189us/step - loss: 0.2367 - acc: 0.8923 - val_loss: 0.2692 - val_acc: 0.8852\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 30s 200us/step - loss: 0.2212 - acc: 0.8996 - val_loss: 0.2653 - val_acc: 0.8897\n",
      "Epoch 9/35\n",
      "150000/150000 [==============================] - 30s 199us/step - loss: 0.2088 - acc: 0.9060 - val_loss: 0.2717 - val_acc: 0.8878\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 29s 194us/step - loss: 0.1985 - acc: 0.9110 - val_loss: 0.2750 - val_acc: 0.8921\n",
      "Epoch 11/35\n",
      "150000/150000 [==============================] - 30s 198us/step - loss: 0.1909 - acc: 0.9150 - val_loss: 0.2639 - val_acc: 0.8932\n",
      "Epoch 12/35\n",
      "0.00030151135\n",
      "0.00030151135889878403\n",
      "150000/150000 [==============================] - 28s 189us/step - loss: 0.1542 - acc: 0.9310 - val_loss: 0.2680 - val_acc: 0.9048\n",
      "Epoch 13/35\n",
      "150000/150000 [==============================] - 29s 193us/step - loss: 0.1417 - acc: 0.9367 - val_loss: 0.2879 - val_acc: 0.9032\n",
      "Epoch 14/35\n",
      "150000/150000 [==============================] - 29s 192us/step - loss: 0.1360 - acc: 0.9396 - val_loss: 0.2878 - val_acc: 0.9070\n",
      "Epoch 15/35\n",
      "8.05823e-05\n",
      "8.058229706621705e-05\n",
      "150000/150000 [==============================] - 28s 185us/step - loss: 0.1242 - acc: 0.9452 - val_loss: 0.3019 - val_acc: 0.9082\n",
      "Epoch 16/35\n",
      "150000/150000 [==============================] - 29s 191us/step - loss: 0.1217 - acc: 0.9462 - val_loss: 0.3006 - val_acc: 0.9084\n",
      "Epoch 17/35\n",
      "2.0145575e-05\n",
      "2.0145575035712682e-05\n",
      "150000/150000 [==============================] - 28s 189us/step - loss: 0.1167 - acc: 0.9490 - val_loss: 0.3111 - val_acc: 0.9086\n",
      "Epoch 18/35\n",
      "4.8860197e-06\n",
      "4.886019633002921e-06\n",
      "150000/150000 [==============================] - 29s 194us/step - loss: 0.1157 - acc: 0.9493 - val_loss: 0.3128 - val_acc: 0.9088\n",
      "Epoch 19/35\n",
      "1.1516458e-06\n",
      "1.151645893266536e-06\n",
      "150000/150000 [==============================] - 29s 195us/step - loss: 0.1162 - acc: 0.9489 - val_loss: 0.3129 - val_acc: 0.9089\n",
      "Epoch 20/35\n",
      "2.6420568e-07\n",
      "2.6420567717844374e-07\n",
      "150000/150000 [==============================] - 28s 187us/step - loss: 0.1157 - acc: 0.9490 - val_loss: 0.3129 - val_acc: 0.9090\n",
      "Epoch 21/35\n",
      "5.9078186e-08\n",
      "5.907818635020035e-08\n",
      "150000/150000 [==============================] - 30s 197us/step - loss: 0.1169 - acc: 0.9482 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 22/35\n",
      "1.2891917e-08\n",
      "1.2891917085827184e-08\n",
      "150000/150000 [==============================] - 28s 189us/step - loss: 0.1164 - acc: 0.9492 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 23/35\n",
      "2.7485658e-09\n",
      "2.748565911321258e-09\n",
      "150000/150000 [==============================] - 28s 185us/step - loss: 0.1154 - acc: 0.9494 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 24/35\n",
      "5.731156e-10\n",
      "5.731155913270351e-10\n",
      "150000/150000 [==============================] - 28s 188us/step - loss: 0.1160 - acc: 0.9498 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 25/35\n",
      "1.1698674e-10\n",
      "1.169867341174648e-10\n",
      "150000/150000 [==============================] - 30s 200us/step - loss: 0.1163 - acc: 0.9491 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 26/35\n",
      "2.3397348e-11\n",
      "2.3397347947984316e-11\n",
      "150000/150000 [==============================] - 30s 201us/step - loss: 0.1158 - acc: 0.9486 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 27/35\n",
      "4.5885977e-12\n",
      "4.5885975200834576e-12\n",
      "150000/150000 [==============================] - 30s 200us/step - loss: 0.1166 - acc: 0.9486 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 28/35\n",
      "8.8307606e-13\n",
      "8.830760450552225e-13\n",
      "150000/150000 [==============================] - 29s 193us/step - loss: 0.1153 - acc: 0.9491 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 29/35\n",
      "1.6688568e-13\n",
      "1.668856880393761e-13\n",
      "150000/150000 [==============================] - 29s 192us/step - loss: 0.1151 - acc: 0.9490 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 30/35\n",
      "3.0989893e-14\n",
      "3.0989893503994936e-14\n",
      "150000/150000 [==============================] - 28s 188us/step - loss: 0.1159 - acc: 0.9493 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 31/35\n",
      "5.6579544e-15\n",
      "5.657954483168427e-15\n",
      "150000/150000 [==============================] - 27s 182us/step - loss: 0.1147 - acc: 0.9496 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 32/35\n",
      "1.0161986e-15\n",
      "1.0161986004922124e-15\n",
      "150000/150000 [==============================] - 27s 183us/step - loss: 0.1157 - acc: 0.9489 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 33/35\n",
      "1.7964022e-16\n",
      "1.7964022246513694e-16\n",
      "150000/150000 [==============================] - 28s 186us/step - loss: 0.1139 - acc: 0.9495 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 34/35\n",
      "3.1271348e-17\n",
      "3.127134780830015e-17\n",
      "150000/150000 [==============================] - 28s 185us/step - loss: 0.1148 - acc: 0.9493 - val_loss: 0.3128 - val_acc: 0.9090\n",
      "Epoch 35/35\n",
      "5.362992e-18\n",
      "5.3629919358883e-18\n",
      "150000/150000 [==============================] - 28s 185us/step - loss: 0.1157 - acc: 0.9489 - val_loss: 0.3128 - val_acc: 0.9090\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2c249d6b208>"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv),\n",
    "             callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save_weights('top_model_resnet_32_32_adam_custom_lr2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_7 (Flatten)          (None, 8192)              0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 32)                262176    \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 262,209\n",
      "Trainable params: 262,209\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(32, activation='relu'))\n",
    "top_model.add(Dropout(0.35))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.5166 - acc: 0.7265 - val_loss: 0.4223 - val_acc: 0.7938\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.4340 - acc: 0.7795 - val_loss: 0.3789 - val_acc: 0.8221\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 24s 161us/step - loss: 0.3955 - acc: 0.8008 - val_loss: 0.3504 - val_acc: 0.8390\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 25s 166us/step - loss: 0.3651 - acc: 0.8154 - val_loss: 0.3313 - val_acc: 0.8493\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 25s 168us/step - loss: 0.3420 - acc: 0.8273 - val_loss: 0.3192 - val_acc: 0.8557\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 25s 163us/step - loss: 0.3249 - acc: 0.8361 - val_loss: 0.3053 - val_acc: 0.8640\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 24s 161us/step - loss: 0.3093 - acc: 0.8439 - val_loss: 0.2963 - val_acc: 0.8671\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 24s 159us/step - loss: 0.2975 - acc: 0.8487 - val_loss: 0.3031 - val_acc: 0.8663\n",
      "Epoch 9/35\n",
      "150000/150000 [==============================] - 24s 163us/step - loss: 0.2849 - acc: 0.8557 - val_loss: 0.2966 - val_acc: 0.8729\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 24s 161us/step - loss: 0.2779 - acc: 0.8589 - val_loss: 0.2878 - val_acc: 0.8771\n",
      "Epoch 11/35\n",
      "0.0003162278\n",
      "0.00031622778103685084\n",
      "150000/150000 [==============================] - 24s 163us/step - loss: 0.2423 - acc: 0.8735 - val_loss: 0.2786 - val_acc: 0.8854\n",
      "Epoch 12/35\n",
      "150000/150000 [==============================] - 25s 166us/step - loss: 0.2288 - acc: 0.8793 - val_loss: 0.2871 - val_acc: 0.8871\n",
      "Epoch 13/35\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.2234 - acc: 0.8810 - val_loss: 0.2833 - val_acc: 0.8880\n",
      "Epoch 14/35\n",
      "8.7705805e-05\n",
      "8.770580754042039e-05\n",
      "150000/150000 [==============================] - 27s 178us/step - loss: 0.2089 - acc: 0.8873 - val_loss: 0.2881 - val_acc: 0.8904\n",
      "Epoch 15/35\n",
      "150000/150000 [==============================] - 27s 179us/step - loss: 0.2040 - acc: 0.8888 - val_loss: 0.2950 - val_acc: 0.8894\n",
      "Epoch 16/35\n",
      "2.2645541e-05\n",
      "2.264554137652806e-05\n",
      "150000/150000 [==============================] - 27s 177us/step - loss: 0.2004 - acc: 0.8906 - val_loss: 0.2972 - val_acc: 0.8900\n",
      "Epoch 17/35\n",
      "5.6613853e-06\n",
      "5.661385330313351e-06\n",
      "150000/150000 [==============================] - 27s 181us/step - loss: 0.1971 - acc: 0.8930 - val_loss: 0.2983 - val_acc: 0.8900\n",
      "Epoch 18/35\n",
      "1.3730877e-06\n",
      "1.373087629659075e-06\n",
      "150000/150000 [==============================] - 27s 177us/step - loss: 0.1976 - acc: 0.8923 - val_loss: 0.2984 - val_acc: 0.8901\n",
      "Epoch 19/35\n",
      "3.2363988e-07\n",
      "3.2363987094422374e-07\n",
      "150000/150000 [==============================] - 27s 177us/step - loss: 0.1982 - acc: 0.8918 - val_loss: 0.2985 - val_acc: 0.8902\n",
      "Epoch 20/35\n",
      "7.424808e-08\n",
      "7.424808083619956e-08\n",
      "150000/150000 [==============================] - 27s 177us/step - loss: 0.1981 - acc: 0.8917 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 21/35\n",
      "1.6602375e-08\n",
      "1.6602374940126426e-08\n",
      "150000/150000 [==============================] - 26s 175us/step - loss: 0.1983 - acc: 0.8927 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 22/35\n",
      "3.622935e-09\n",
      "3.6229351551142435e-09\n",
      "150000/150000 [==============================] - 26s 176us/step - loss: 0.1989 - acc: 0.8919 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 23/35\n",
      "7.7241236e-10\n",
      "7.724123507549834e-10\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.1973 - acc: 0.8924 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 24/35\n",
      "1.6105911e-10\n",
      "1.6105910988392012e-10\n",
      "150000/150000 [==============================] - 26s 176us/step - loss: 0.1973 - acc: 0.8926 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 25/35\n",
      "3.2876053e-11\n",
      "3.287605326115812e-11\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.1972 - acc: 0.8929 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 26/35\n",
      "6.5752104e-12\n",
      "6.575210514947471e-12\n",
      "150000/150000 [==============================] - 26s 172us/step - loss: 0.1976 - acc: 0.8930 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 27/35\n",
      "1.2895049e-12\n",
      "1.289504856901084e-12\n",
      "150000/150000 [==============================] - 26s 173us/step - loss: 0.1976 - acc: 0.8923 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 28/35\n",
      "2.481653e-13\n",
      "2.4816532461377917e-13\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.1982 - acc: 0.8922 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 29/35\n",
      "4.6898835e-14\n",
      "4.6898835700645123e-14\n",
      "150000/150000 [==============================] - 26s 176us/step - loss: 0.1995 - acc: 0.8920 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 30/35\n",
      "8.7088955e-15\n",
      "8.708895084826156e-15\n",
      "150000/150000 [==============================] - 26s 174us/step - loss: 0.1991 - acc: 0.8919 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 31/35\n",
      "1.5900195e-15\n",
      "1.5900194972382688e-15\n",
      "150000/150000 [==============================] - 26s 171us/step - loss: 0.1978 - acc: 0.8925 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 32/35\n",
      "2.8557594e-16\n",
      "2.855759403641947e-16\n",
      "150000/150000 [==============================] - 26s 175us/step - loss: 0.1969 - acc: 0.8932 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 33/35\n",
      "5.048317e-17\n",
      "5.0483171027882134e-17\n",
      "150000/150000 [==============================] - 27s 179us/step - loss: 0.1980 - acc: 0.8927 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 34/35\n",
      "8.787992e-18\n",
      "8.787991946475677e-18\n",
      "150000/150000 [==============================] - 27s 183us/step - loss: 0.1980 - acc: 0.8919 - val_loss: 0.2984 - val_acc: 0.8902\n",
      "Epoch 35/35\n",
      "1.5071281e-18\n",
      "1.5071281330498602e-18\n",
      "150000/150000 [==============================] - 26s 173us/step - loss: 0.1975 - acc: 0.8925 - val_loss: 0.2984 - val_acc: 0.8902\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2c249d77b70>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv),\n",
    "             callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save_weights('top_model_resnet_32_adam_custom_lr2.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>VGG19 bottleneck</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train=np.load('train_data_reduced/x_train_vgg19_150000_bottleneck.npy')\n",
    "x_cv=np.load('train_data_reduced/x_cv_vgg19_25000_bottleneck.npy')\n",
    "\n",
    "y_train=np.load('train_data_reduced/y_train_150000.npy')\n",
    "y_cv=np.load('train_data_reduced/y_cv_25000.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150000, 2, 2, 512)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(64, activation='relu'))\n",
    "top_model.add(Dropout(0.5))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_8 (Flatten)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 64)                131136    \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 131,201\n",
      "Trainable params: 131,201\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [\n",
    "\t    Custom_lr()\n",
    "\t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 18s 118us/step - loss: 0.6378 - acc: 0.6668 - val_loss: 0.5237 - val_acc: 0.7270\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.5461 - acc: 0.7090 - val_loss: 0.4982 - val_acc: 0.7515\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.5208 - acc: 0.7236 - val_loss: 0.4711 - val_acc: 0.7677\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.5019 - acc: 0.7347 - val_loss: 0.4611 - val_acc: 0.7744\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 18s 118us/step - loss: 0.4858 - acc: 0.7448 - val_loss: 0.4491 - val_acc: 0.7840\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4720 - acc: 0.7525 - val_loss: 0.4339 - val_acc: 0.7923\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.4628 - acc: 0.7569 - val_loss: 0.4317 - val_acc: 0.7959\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.4520 - acc: 0.7616 - val_loss: 0.4229 - val_acc: 0.8013\n",
      "Epoch 9/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4440 - acc: 0.7668 - val_loss: 0.4259 - val_acc: 0.8022\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.4391 - acc: 0.7688 - val_loss: 0.4177 - val_acc: 0.8062\n",
      "Epoch 11/35\n",
      "0.0003162278\n",
      "0.00031622778103685084\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.4086 - acc: 0.7832 - val_loss: 0.4010 - val_acc: 0.8168\n",
      "Epoch 12/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3964 - acc: 0.7888 - val_loss: 0.3980 - val_acc: 0.8197\n",
      "Epoch 13/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3897 - acc: 0.7923 - val_loss: 0.3995 - val_acc: 0.8209\n",
      "Epoch 14/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3856 - acc: 0.7942 - val_loss: 0.3965 - val_acc: 0.8220\n",
      "Epoch 15/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3823 - acc: 0.7967 - val_loss: 0.3946 - val_acc: 0.8250\n",
      "Epoch 16/35\n",
      "150000/150000 [==============================] - 18s 117us/step - loss: 0.3779 - acc: 0.7987 - val_loss: 0.3974 - val_acc: 0.8235\n",
      "Epoch 17/35\n",
      "150000/150000 [==============================] - 18s 118us/step - loss: 0.3745 - acc: 0.7995 - val_loss: 0.3982 - val_acc: 0.8246\n",
      "Epoch 18/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3710 - acc: 0.8025 - val_loss: 0.3944 - val_acc: 0.8265\n",
      "Epoch 19/35\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3672 - acc: 0.8034 - val_loss: 0.3977 - val_acc: 0.8282\n",
      "Epoch 20/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3643 - acc: 0.8050 - val_loss: 0.3960 - val_acc: 0.8274\n",
      "Epoch 21/35\n",
      "7.071068e-05\n",
      "7.071068264135355e-05\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3554 - acc: 0.8081 - val_loss: 0.3978 - val_acc: 0.8298\n",
      "Epoch 22/35\n",
      "150000/150000 [==============================] - 17s 110us/step - loss: 0.3515 - acc: 0.8101 - val_loss: 0.3969 - val_acc: 0.8303\n",
      "Epoch 23/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3483 - acc: 0.8118 - val_loss: 0.3991 - val_acc: 0.8300\n",
      "Epoch 24/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3458 - acc: 0.8140 - val_loss: 0.4012 - val_acc: 0.8300\n",
      "Epoch 25/35\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3462 - acc: 0.8128 - val_loss: 0.4015 - val_acc: 0.8307\n",
      "Epoch 26/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3442 - acc: 0.8151 - val_loss: 0.4044 - val_acc: 0.8313\n",
      "Epoch 27/35\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3444 - acc: 0.8134 - val_loss: 0.4032 - val_acc: 0.8318\n",
      "Epoch 28/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3436 - acc: 0.8154 - val_loss: 0.4036 - val_acc: 0.8320\n",
      "Epoch 29/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3401 - acc: 0.8172 - val_loss: 0.4047 - val_acc: 0.8314\n",
      "Epoch 30/35\n",
      "150000/150000 [==============================] - 17s 111us/step - loss: 0.3404 - acc: 0.8163 - val_loss: 0.4071 - val_acc: 0.8325\n",
      "Epoch 31/35\n",
      "1.2909945e-05\n",
      "1.2909945018820682e-05\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3383 - acc: 0.8179 - val_loss: 0.4068 - val_acc: 0.8325\n",
      "Epoch 32/35\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3373 - acc: 0.8182 - val_loss: 0.4067 - val_acc: 0.8325\n",
      "Epoch 33/35\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3362 - acc: 0.8182 - val_loss: 0.4073 - val_acc: 0.8318\n",
      "Epoch 34/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3381 - acc: 0.8168 - val_loss: 0.4074 - val_acc: 0.8322\n",
      "Epoch 35/35\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3372 - acc: 0.8172 - val_loss: 0.4078 - val_acc: 0.8324\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2c24cb8d5c0>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv), \n",
    "            callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save_weights('top_model_vgg19_64_adam_custom_lr.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model_custom_lr2=Sequential()\n",
    "top_model_custom_lr2.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model_custom_lr2.add(Dense(64, activation='relu'))\n",
    "top_model_custom_lr2.add(Dropout(0.5))\n",
    "top_model_custom_lr2.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model_custom_lr2.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_9 (Flatten)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 64)                131136    \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 131,201\n",
      "Trainable params: 131,201\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model_custom_lr2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [\n",
    "\t    Custom_lr_2()\n",
    "\t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 18s 120us/step - loss: 0.6348 - acc: 0.6672 - val_loss: 0.5458 - val_acc: 0.7196\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 18s 117us/step - loss: 0.5492 - acc: 0.7085 - val_loss: 0.4900 - val_acc: 0.7549\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.5236 - acc: 0.7269 - val_loss: 0.4797 - val_acc: 0.7651\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.5069 - acc: 0.7375 - val_loss: 0.4620 - val_acc: 0.7782\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4913 - acc: 0.7461 - val_loss: 0.4457 - val_acc: 0.7814\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 18s 120us/step - loss: 0.4783 - acc: 0.7542 - val_loss: 0.4480 - val_acc: 0.7831\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4676 - acc: 0.7601 - val_loss: 0.4377 - val_acc: 0.7924\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 18s 117us/step - loss: 0.4580 - acc: 0.7652 - val_loss: 0.4262 - val_acc: 0.7983\n",
      "Epoch 9/35\n",
      "0.00035355342\n",
      "0.0003535534073861587\n",
      "150000/150000 [==============================] - 18s 119us/step - loss: 0.4286 - acc: 0.7812 - val_loss: 0.4092 - val_acc: 0.8109\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4196 - acc: 0.7861 - val_loss: 0.4033 - val_acc: 0.8145\n",
      "Epoch 11/35\n",
      "0.000111803405\n",
      "0.00011180340807931047\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4019 - acc: 0.7949 - val_loss: 0.3995 - val_acc: 0.8160\n",
      "Epoch 12/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3974 - acc: 0.7970 - val_loss: 0.3978 - val_acc: 0.8193\n",
      "Epoch 13/35\n",
      "3.2274864e-05\n",
      "3.227486303314903e-05\n",
      "150000/150000 [==============================] - 18s 117us/step - loss: 0.3921 - acc: 0.8010 - val_loss: 0.3964 - val_acc: 0.8197\n",
      "Epoch 14/35\n",
      "8.951437e-06\n",
      "8.951436633180429e-06\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3890 - acc: 0.8027 - val_loss: 0.3966 - val_acc: 0.8201\n",
      "Epoch 15/35\n",
      "2.3923722e-06\n",
      "2.3923721532825226e-06\n",
      "150000/150000 [==============================] - 17s 117us/step - loss: 0.3890 - acc: 0.8020 - val_loss: 0.3967 - val_acc: 0.8198\n",
      "Epoch 16/35\n",
      "6.177078e-07\n",
      "6.17707833721772e-07\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3891 - acc: 0.8030 - val_loss: 0.3967 - val_acc: 0.8196\n",
      "Epoch 17/35\n",
      "1.5442696e-07\n",
      "1.544269565556533e-07\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3899 - acc: 0.8019 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 18/35\n",
      "3.745404e-08\n",
      "3.745403843068401e-08\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3896 - acc: 0.8016 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 19/35\n",
      "8.828001e-09\n",
      "8.828001858195394e-09\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3898 - acc: 0.8026 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 20/35\n",
      "2.0252824e-09\n",
      "2.0252824291915945e-09\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3897 - acc: 0.8019 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 21/35\n",
      "4.528669e-10\n",
      "4.528669089270345e-10\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3894 - acc: 0.8022 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 22/35\n",
      "9.8823665e-11\n",
      "9.882366179722631e-11\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3904 - acc: 0.8029 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 23/35\n",
      "2.1069277e-11\n",
      "2.1069276070554952e-11\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3897 - acc: 0.8017 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 24/35\n",
      "4.393248e-12\n",
      "4.39324785063103e-12\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3893 - acc: 0.8022 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 25/35\n",
      "8.9676796e-13\n",
      "8.967679761012581e-13\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3902 - acc: 0.8019 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 26/35\n",
      "1.7935359e-13\n",
      "1.793535914403066e-13\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3872 - acc: 0.8028 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 27/35\n",
      "3.5174134e-14\n",
      "3.517413264485197e-14\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3902 - acc: 0.8016 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 28/35\n",
      "6.7692654e-15\n",
      "6.769265266250403e-15\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3886 - acc: 0.8019 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 29/35\n",
      "1.2792709e-15\n",
      "1.2792709132402144e-15\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3892 - acc: 0.8024 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 30/35\n",
      "2.3755465e-16\n",
      "2.3755464158274595e-16\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3900 - acc: 0.8012 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 31/35\n",
      "4.3371346e-17\n",
      "4.3371347123994214e-17\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3894 - acc: 0.8025 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 32/35\n",
      "7.789724e-18\n",
      "7.78972369602735e-18\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3887 - acc: 0.8026 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 33/35\n",
      "1.3770416e-18\n",
      "1.3770415970805965e-18\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3891 - acc: 0.8019 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 34/35\n",
      "2.3971216e-19\n",
      "2.3971217101263455e-19\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3896 - acc: 0.8022 - val_loss: 0.3967 - val_acc: 0.8197\n",
      "Epoch 35/35\n",
      "4.1110295e-20\n",
      "4.111029617511719e-20\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3881 - acc: 0.8027 - val_loss: 0.3967 - val_acc: 0.8197\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2c24adbbcf8>"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model_custom_lr2.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv), \n",
    "            callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model_custom_lr2.save_weights('top_model_vgg19_64_adam_custom_lr2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import LeakyReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_10 (Flatten)         (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 32)                65568     \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 32)                1056      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)    (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dropout_12 (Dropout)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 66,657\n",
      "Trainable params: 66,657\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(32, activation='relu'))\n",
    "top_model.add(Dropout(0.2))\n",
    "top_model.add(Dense(32))\n",
    "top_model.add(LeakyReLU())\n",
    "top_model.add(Dropout(0.2))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 24s 163us/step - loss: 0.6415 - acc: 0.6297 - val_loss: 0.5525 - val_acc: 0.7036\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 23s 153us/step - loss: 0.5465 - acc: 0.7040 - val_loss: 0.5009 - val_acc: 0.7351\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 23s 155us/step - loss: 0.4986 - acc: 0.7346 - val_loss: 0.4732 - val_acc: 0.7579\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 22s 150us/step - loss: 0.4693 - acc: 0.7532 - val_loss: 0.4507 - val_acc: 0.7744\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 22s 149us/step - loss: 0.4475 - acc: 0.7676 - val_loss: 0.4402 - val_acc: 0.7803\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 25s 164us/step - loss: 0.4313 - acc: 0.7747 - val_loss: 0.4403 - val_acc: 0.7813\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 22s 149us/step - loss: 0.4204 - acc: 0.7808 - val_loss: 0.4270 - val_acc: 0.7907\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 23s 151us/step - loss: 0.4076 - acc: 0.7865 - val_loss: 0.4337 - val_acc: 0.7912\n",
      "Epoch 9/35\n",
      "150000/150000 [==============================] - 23s 150us/step - loss: 0.3966 - acc: 0.7926 - val_loss: 0.4220 - val_acc: 0.7981\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 22s 147us/step - loss: 0.3875 - acc: 0.7981 - val_loss: 0.4160 - val_acc: 0.8022\n",
      "Epoch 11/35\n",
      "0.0003162278\n",
      "0.00031622778103685084\n",
      "150000/150000 [==============================] - 23s 152us/step - loss: 0.3594 - acc: 0.8110 - val_loss: 0.4170 - val_acc: 0.8065\n",
      "Epoch 12/35\n",
      "150000/150000 [==============================] - 22s 148us/step - loss: 0.3503 - acc: 0.8142 - val_loss: 0.4190 - val_acc: 0.8103\n",
      "Epoch 13/35\n",
      "9.12871e-05\n",
      "9.128709875630674e-05\n",
      "150000/150000 [==============================] - 22s 148us/step - loss: 0.3375 - acc: 0.8203 - val_loss: 0.4251 - val_acc: 0.8094\n",
      "Epoch 14/35\n",
      "150000/150000 [==============================] - 23s 150us/step - loss: 0.3340 - acc: 0.8217 - val_loss: 0.4303 - val_acc: 0.8094\n",
      "Epoch 15/35\n",
      "2.4397503e-05\n",
      "2.4397502764723083e-05\n",
      "150000/150000 [==============================] - 22s 149us/step - loss: 0.3297 - acc: 0.8242 - val_loss: 0.4343 - val_acc: 0.8109\n",
      "Epoch 16/35\n",
      "6.299408e-06\n",
      "6.299408131501269e-06\n",
      "150000/150000 [==============================] - 22s 149us/step - loss: 0.3286 - acc: 0.8243 - val_loss: 0.4347 - val_acc: 0.8109\n",
      "Epoch 17/35\n",
      "1.574852e-06\n",
      "1.5748520354463835e-06\n",
      "150000/150000 [==============================] - 22s 144us/step - loss: 0.3273 - acc: 0.8254 - val_loss: 0.4354 - val_acc: 0.8110\n",
      "Epoch 18/35\n",
      "3.8195773e-07\n",
      "3.819577227567298e-07\n",
      "150000/150000 [==============================] - 22s 147us/step - loss: 0.3281 - acc: 0.8252 - val_loss: 0.4354 - val_acc: 0.8110\n",
      "Epoch 19/35\n",
      "9.00283e-08\n",
      "9.00283010431194e-08\n",
      "150000/150000 [==============================] - 22s 148us/step - loss: 0.3282 - acc: 0.8249 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 20/35\n",
      "2.0653909e-08\n",
      "2.065390880230928e-08\n",
      "150000/150000 [==============================] - 22s 149us/step - loss: 0.3280 - acc: 0.8253 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 21/35\n",
      "4.6183546e-09\n",
      "4.618354409008315e-09\n",
      "150000/150000 [==============================] - 23s 151us/step - loss: 0.3278 - acc: 0.8255 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 22/35\n",
      "1.0078076e-09\n",
      "1.0078075944784399e-09\n",
      "150000/150000 [==============================] - 22s 149us/step - loss: 0.3271 - acc: 0.8261 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 23/35\n",
      "2.148653e-10\n",
      "2.1486530549621976e-10\n",
      "150000/150000 [==============================] - 22s 145us/step - loss: 0.3287 - acc: 0.8244 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 24/35\n",
      "4.4802512e-11\n",
      "4.480251340324632e-11\n",
      "150000/150000 [==============================] - 22s 144us/step - loss: 0.3287 - acc: 0.8242 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 25/35\n",
      "9.145275e-12\n",
      "9.145274546764337e-12\n",
      "150000/150000 [==============================] - 22s 147us/step - loss: 0.3278 - acc: 0.8253 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 26/35\n",
      "1.829055e-12\n",
      "1.829054963042864e-12\n",
      "150000/150000 [==============================] - 22s 149us/step - loss: 0.3284 - acc: 0.8243 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 27/35\n",
      "3.5870718e-13\n",
      "3.58707181802099e-13\n",
      "150000/150000 [==============================] - 22s 147us/step - loss: 0.3280 - acc: 0.8252 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 28/35\n",
      "6.9033226e-14\n",
      "6.903322848885075e-14\n",
      "150000/150000 [==============================] - 23s 152us/step - loss: 0.3289 - acc: 0.8242 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 29/35\n",
      "1.3046053e-14\n",
      "1.3046053415925462e-14\n",
      "150000/150000 [==============================] - 22s 144us/step - loss: 0.3263 - acc: 0.8258 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 30/35\n",
      "2.4225913e-15\n",
      "2.4225912464808003e-15\n",
      "150000/150000 [==============================] - 21s 139us/step - loss: 0.3290 - acc: 0.8238 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 31/35\n",
      "4.4230263e-16\n",
      "4.423026358783204e-16\n",
      "150000/150000 [==============================] - 21s 141us/step - loss: 0.3276 - acc: 0.8256 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 32/35\n",
      "7.94399e-17\n",
      "7.943989782121984e-17\n",
      "150000/150000 [==============================] - 21s 141us/step - loss: 0.3276 - acc: 0.8250 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 33/35\n",
      "1.4043123e-17\n",
      "1.4043122672423347e-17\n",
      "150000/150000 [==============================] - 22s 145us/step - loss: 0.3276 - acc: 0.8255 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 34/35\n",
      "2.444594e-18\n",
      "2.4445939366251747e-18\n",
      "150000/150000 [==============================] - 22s 147us/step - loss: 0.3282 - acc: 0.8246 - val_loss: 0.4354 - val_acc: 0.8111\n",
      "Epoch 35/35\n",
      "4.192444e-19\n",
      "4.192444044349093e-19\n",
      "150000/150000 [==============================] - 22s 144us/step - loss: 0.3291 - acc: 0.8239 - val_loss: 0.4354 - val_acc: 0.8111\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2c2a47452e8>"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv),\n",
    "             callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save_weights('top_model_vgg19_32_32_adam_custom_lr2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_11 (Flatten)         (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 32)                65568     \n",
      "_________________________________________________________________\n",
      "dropout_13 (Dropout)         (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 1)                 33        \n",
      "=================================================================\n",
      "Total params: 65,601\n",
      "Trainable params: 65,601\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(32, activation='relu'))\n",
    "top_model.add(Dropout(0.35))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 150000 samples, validate on 25000 samples\n",
      "Epoch 1/35\n",
      "150000/150000 [==============================] - 18s 118us/step - loss: 0.6137 - acc: 0.6722 - val_loss: 0.5269 - val_acc: 0.7289\n",
      "Epoch 2/35\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.5443 - acc: 0.7092 - val_loss: 0.5034 - val_acc: 0.7446\n",
      "Epoch 3/35\n",
      "150000/150000 [==============================] - 18s 119us/step - loss: 0.5206 - acc: 0.7224 - val_loss: 0.4861 - val_acc: 0.7570\n",
      "Epoch 4/35\n",
      "150000/150000 [==============================] - 18s 118us/step - loss: 0.5019 - acc: 0.7324 - val_loss: 0.4683 - val_acc: 0.7692\n",
      "Epoch 5/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4877 - acc: 0.7397 - val_loss: 0.4686 - val_acc: 0.7694\n",
      "Epoch 6/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4761 - acc: 0.7456 - val_loss: 0.4563 - val_acc: 0.7742\n",
      "Epoch 7/35\n",
      "150000/150000 [==============================] - 18s 117us/step - loss: 0.4633 - acc: 0.7523 - val_loss: 0.4456 - val_acc: 0.7830\n",
      "Epoch 8/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4535 - acc: 0.7582 - val_loss: 0.4389 - val_acc: 0.7901\n",
      "Epoch 9/35\n",
      "0.00035355342\n",
      "0.0003535534073861587\n",
      "150000/150000 [==============================] - 17s 117us/step - loss: 0.4267 - acc: 0.7712 - val_loss: 0.4271 - val_acc: 0.7961\n",
      "Epoch 10/35\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.4162 - acc: 0.7758 - val_loss: 0.4270 - val_acc: 0.7981\n",
      "Epoch 11/35\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.4078 - acc: 0.7800 - val_loss: 0.4242 - val_acc: 0.7992\n",
      "Epoch 12/35\n",
      "0.00010660037\n",
      "0.00010660036695378094\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3966 - acc: 0.7848 - val_loss: 0.4258 - val_acc: 0.8032\n",
      "Epoch 13/35\n",
      "150000/150000 [==============================] - 18s 118us/step - loss: 0.3921 - acc: 0.7879 - val_loss: 0.4249 - val_acc: 0.8022\n",
      "Epoch 14/35\n",
      "2.9565623e-05\n",
      "2.9565622488023183e-05\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3876 - acc: 0.7889 - val_loss: 0.4258 - val_acc: 0.8041\n",
      "Epoch 15/35\n",
      "7.901745e-06\n",
      "7.90174505503605e-06\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3849 - acc: 0.7917 - val_loss: 0.4258 - val_acc: 0.8042\n",
      "Epoch 16/35\n",
      "2.040222e-06\n",
      "2.040221901784762e-06\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3860 - acc: 0.7907 - val_loss: 0.4258 - val_acc: 0.8042\n",
      "Epoch 17/35\n",
      "5.100555e-07\n",
      "5.100554858472606e-07\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3844 - acc: 0.7921 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 18/35\n",
      "1.2370663e-07\n",
      "1.2370662606317582e-07\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3858 - acc: 0.7901 - val_loss: 0.4259 - val_acc: 0.8044\n",
      "Epoch 19/35\n",
      "2.9157931e-08\n",
      "2.9157931648509776e-08\n",
      "150000/150000 [==============================] - 18s 117us/step - loss: 0.3856 - acc: 0.7913 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 20/35\n",
      "6.6892882e-09\n",
      "6.689288179952593e-09\n",
      "150000/150000 [==============================] - 17s 117us/step - loss: 0.3858 - acc: 0.7904 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 21/35\n",
      "1.4957703e-09\n",
      "1.4957703213298468e-09\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3844 - acc: 0.7917 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 22/35\n",
      "3.2640385e-10\n",
      "3.2640383595964826e-10\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3866 - acc: 0.7915 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 23/35\n",
      "6.9589536e-11\n",
      "6.958953429885186e-11\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3868 - acc: 0.7892 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 24/35\n",
      "1.45104215e-11\n",
      "1.4510421224069895e-11\n",
      "150000/150000 [==============================] - 17s 116us/step - loss: 0.3842 - acc: 0.7921 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 25/35\n",
      "2.9619274e-12\n",
      "2.9619273877211838e-12\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3835 - acc: 0.7924 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 26/35\n",
      "5.9238546e-13\n",
      "5.92385472272805e-13\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3845 - acc: 0.7915 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 27/35\n",
      "1.1617634e-13\n",
      "1.16176347207855e-13\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3850 - acc: 0.7919 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 28/35\n",
      "2.2358148e-14\n",
      "2.2358147460189495e-14\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3849 - acc: 0.7918 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 29/35\n",
      "4.225293e-15\n",
      "4.225292783757977e-15\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3858 - acc: 0.7912 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 30/35\n",
      "7.846172e-16\n",
      "7.846172108214538e-16\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3879 - acc: 0.7890 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 31/35\n",
      "1.4325085e-16\n",
      "1.4325084775171024e-16\n",
      "150000/150000 [==============================] - 17s 112us/step - loss: 0.3845 - acc: 0.7924 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 32/35\n",
      "2.5728611e-17\n",
      "2.5728611697486635e-17\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3856 - acc: 0.7910 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 33/35\n",
      "4.5482187e-18\n",
      "4.5482188065756e-18\n",
      "150000/150000 [==============================] - 17s 114us/step - loss: 0.3844 - acc: 0.7922 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 34/35\n",
      "7.9174326e-19\n",
      "7.917432472354881e-19\n",
      "150000/150000 [==============================] - 17s 115us/step - loss: 0.3850 - acc: 0.7914 - val_loss: 0.4259 - val_acc: 0.8043\n",
      "Epoch 35/35\n",
      "1.3578285e-19\n",
      "1.3578284966353205e-19\n",
      "150000/150000 [==============================] - 17s 113us/step - loss: 0.3865 - acc: 0.7896 - val_loss: 0.4259 - val_acc: 0.8043\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2c2a61cb048>"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv),\n",
    "             callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save_weights('top_model_vgg19_32_adam_custom_lr2.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Best model on full data (Resnet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train=np.load('train_data/x_train_resnet_bottleneck.npy')\n",
    "x_cv=np.load('train_data/x_cv_resnet_bottleneck.npy')\n",
    "\n",
    "y_train=np.load('train_data/y_train.npy')\n",
    "y_cv=np.load('train_data/y_cv.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(245509, 2, 2, 2048)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model=Sequential()\n",
    "top_model.add(Flatten(input_shape=x_train.shape[1:]))\n",
    "top_model.add(Dense(64, activation='relu'))\n",
    "# top_model.add(Dropout(0.5))\n",
    "top_model.add(Dense(1, activation='sigmoid'))\n",
    "top_model.load_weights('top_model_resnet_64_adam_custom_lr_full_data.h5')\n",
    "\n",
    "top_model.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_3 (Flatten)          (None, 8192)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 64)                524352    \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 524,417\n",
      "Trainable params: 524,417\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "top_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [\n",
    "\t    Custom_lr()\n",
    "\t]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 245509 samples, validate on 105219 samples\n",
      "Epoch 1/35\n",
      "245509/245509 [==============================] - 52s 214us/step - loss: 0.4926 - acc: 0.7486 - val_loss: 0.3728 - val_acc: 0.8264\n",
      "Epoch 2/35\n",
      "245509/245509 [==============================] - 50s 205us/step - loss: 0.4016 - acc: 0.8042 - val_loss: 0.3160 - val_acc: 0.8588\n",
      "Epoch 3/35\n",
      "245509/245509 [==============================] - 51s 207us/step - loss: 0.3572 - acc: 0.8283 - val_loss: 0.2934 - val_acc: 0.8726\n",
      "Epoch 4/35\n",
      "245509/245509 [==============================] - 48s 196us/step - loss: 0.3280 - acc: 0.8448 - val_loss: 0.2663 - val_acc: 0.8839\n",
      "Epoch 5/35\n",
      "245509/245509 [==============================] - 49s 198us/step - loss: 0.3046 - acc: 0.8557 - val_loss: 0.2549 - val_acc: 0.8920\n",
      "Epoch 6/35\n",
      "245509/245509 [==============================] - 48s 195us/step - loss: 0.2887 - acc: 0.8648 - val_loss: 0.2414 - val_acc: 0.8974\n",
      "Epoch 7/35\n",
      "245509/245509 [==============================] - 48s 194us/step - loss: 0.2738 - acc: 0.8709 - val_loss: 0.2354 - val_acc: 0.9023\n",
      "Epoch 8/35\n",
      "245509/245509 [==============================] - 47s 193us/step - loss: 0.2634 - acc: 0.8761 - val_loss: 0.2234 - val_acc: 0.9059\n",
      "Epoch 9/35\n",
      "245509/245509 [==============================] - 47s 193us/step - loss: 0.2522 - acc: 0.8819 - val_loss: 0.2180 - val_acc: 0.9106\n",
      "Epoch 10/35\n",
      "245509/245509 [==============================] - 48s 196us/step - loss: 0.2405 - acc: 0.8879 - val_loss: 0.2155 - val_acc: 0.9140\n",
      "Epoch 11/35\n",
      "0.0003162278\n",
      "0.00031622778103685084\n",
      "245509/245509 [==============================] - 47s 192us/step - loss: 0.2071 - acc: 0.9022 - val_loss: 0.2008 - val_acc: 0.9209\n",
      "Epoch 12/35\n",
      "245509/245509 [==============================] - 47s 193us/step - loss: 0.1949 - acc: 0.9073 - val_loss: 0.2009 - val_acc: 0.9230\n",
      "Epoch 13/35\n",
      "245509/245509 [==============================] - 48s 196us/step - loss: 0.1875 - acc: 0.9109 - val_loss: 0.1971 - val_acc: 0.9243\n",
      "Epoch 14/35\n",
      "245509/245509 [==============================] - 48s 196us/step - loss: 0.1832 - acc: 0.9118 - val_loss: 0.2015 - val_acc: 0.9254\n",
      "Epoch 15/35\n",
      "245509/245509 [==============================] - 48s 197us/step - loss: 0.1774 - acc: 0.9143 - val_loss: 0.2028 - val_acc: 0.9237\n",
      "Epoch 16/35\n",
      "245509/245509 [==============================] - 48s 197us/step - loss: 0.1743 - acc: 0.9166 - val_loss: 0.2029 - val_acc: 0.9259\n",
      "Epoch 17/35\n",
      "245509/245509 [==============================] - 48s 195us/step - loss: 0.1726 - acc: 0.9172 - val_loss: 0.2012 - val_acc: 0.9264\n",
      "Epoch 18/35\n",
      "245509/245509 [==============================] - 47s 192us/step - loss: 0.1692 - acc: 0.9175 - val_loss: 0.2042 - val_acc: 0.9262\n",
      "Epoch 19/35\n",
      "245509/245509 [==============================] - 47s 190us/step - loss: 0.1662 - acc: 0.9199 - val_loss: 0.1981 - val_acc: 0.9280\n",
      "Epoch 20/35\n",
      "245509/245509 [==============================] - 47s 192us/step - loss: 0.1637 - acc: 0.9209 - val_loss: 0.2027 - val_acc: 0.9277\n",
      "Epoch 21/35\n",
      "7.071068e-05\n",
      "7.071068264135355e-05\n",
      "245509/245509 [==============================] - 47s 193us/step - loss: 0.1534 - acc: 0.9254 - val_loss: 0.2036 - val_acc: 0.9294\n",
      "Epoch 22/35\n",
      "245509/245509 [==============================] - 47s 193us/step - loss: 0.1478 - acc: 0.9277 - val_loss: 0.2073 - val_acc: 0.9298\n",
      "Epoch 23/35\n",
      "245509/245509 [==============================] - 47s 192us/step - loss: 0.1479 - acc: 0.9269 - val_loss: 0.2080 - val_acc: 0.9303\n",
      "Epoch 24/35\n",
      "245509/245509 [==============================] - 47s 191us/step - loss: 0.1456 - acc: 0.9283 - val_loss: 0.2117 - val_acc: 0.9301\n",
      "Epoch 25/35\n",
      "245509/245509 [==============================] - 48s 195us/step - loss: 0.1437 - acc: 0.9290 - val_loss: 0.2141 - val_acc: 0.9303\n",
      "Epoch 26/35\n",
      "245509/245509 [==============================] - 48s 196us/step - loss: 0.1429 - acc: 0.9292 - val_loss: 0.2147 - val_acc: 0.9300\n",
      "Epoch 27/35\n",
      "245509/245509 [==============================] - 47s 190us/step - loss: 0.1422 - acc: 0.9297 - val_loss: 0.2170 - val_acc: 0.9308\n",
      "Epoch 28/35\n",
      "245509/245509 [==============================] - 46s 187us/step - loss: 0.1414 - acc: 0.9304 - val_loss: 0.2176 - val_acc: 0.9307\n",
      "Epoch 29/35\n",
      "245509/245509 [==============================] - 46s 189us/step - loss: 0.1406 - acc: 0.9302 - val_loss: 0.2196 - val_acc: 0.9302\n",
      "Epoch 30/35\n",
      "245509/245509 [==============================] - 46s 189us/step - loss: 0.1386 - acc: 0.9312 - val_loss: 0.2204 - val_acc: 0.9299\n",
      "Epoch 31/35\n",
      "1.2909945e-05\n",
      "1.2909945018820682e-05\n",
      "245509/245509 [==============================] - 47s 190us/step - loss: 0.1371 - acc: 0.9325 - val_loss: 0.2211 - val_acc: 0.9308\n",
      "Epoch 32/35\n",
      "245509/245509 [==============================] - 46s 189us/step - loss: 0.1354 - acc: 0.9329 - val_loss: 0.2212 - val_acc: 0.9309\n",
      "Epoch 33/35\n",
      "245509/245509 [==============================] - 46s 189us/step - loss: 0.1353 - acc: 0.9333 - val_loss: 0.2220 - val_acc: 0.9312\n",
      "Epoch 34/35\n",
      "245509/245509 [==============================] - 48s 194us/step - loss: 0.1351 - acc: 0.9331 - val_loss: 0.2230 - val_acc: 0.9308\n",
      "Epoch 35/35\n",
      "245509/245509 [==============================] - 47s 192us/step - loss: 0.1351 - acc: 0.9324 - val_loss: 0.2228 - val_acc: 0.9308\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2c2a4746c88>"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.fit(x_train, y_train,\n",
    "          epochs=35,\n",
    "          batch_size=32,\n",
    "          validation_data=(x_cv, y_cv), \n",
    "            callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_model.save_weights('top_model_resnet_64_adam_custom_lr_full_data.h5') # best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<keras.layers.core.Flatten at 0x2c249d77d30>,\n",
       " <keras.layers.core.Dense at 0x2c2a47333c8>,\n",
       " <keras.layers.core.Dropout at 0x2c2a4733400>,\n",
       " <keras.layers.core.Dense at 0x2c2a646e550>]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "105219/105219 [==============================] - 9s 89us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.22277329155604367, 0.9308109752040982]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_model.evaluate(x_cv, y_cv)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}