final model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Architechture\n",
    "\n",
    "Since the input consists of two parts, an **image vector** and a **partial caption**, we cannot use the Sequential API provided by the Keras library. \n",
    "\n",
    "For this reason, we use the Functional API "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Model,Input\n",
    "from keras.applications.inception_v3 import InceptionV3,preprocess_input\n",
    "from keras.layers import Embedding,Dense,BatchNormalization,Dropout,LSTM,add\n",
    "from keras.utils import plot_model\n",
    "from keras.utils import plot_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standard variables\n",
    "MAX_LENGTH = 39\n",
    "VOCAB_SIZE = 1665\n",
    "NPIX = 299\n",
    "TARGET_SIZE = (NPIX,NPIX,3)\n",
    "EMBEDDING_SIZE = 300\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\rdbz3b\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\rdbz3b\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    }
   ],
   "source": [
    "# partial caption sequence model\n",
    "\n",
    "inputs2 = Input(shape=(MAX_LENGTH,))\n",
    "se1 = Embedding(VOCAB_SIZE, EMBEDDING_SIZE, mask_zero=True)(inputs2)\n",
    "se2 = Dropout(0.5)(se1)\n",
    "se3 = LSTM(300)(se2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# image feature extractor model\n",
    "inputs1 = Input(shape=(2048,))\n",
    "fe1 = Dropout(0.5)(inputs1)\n",
    "fe2 = Dense(300, activation='relu')(fe1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "decoder1 = add([fe2, se3])\n",
    "decoder2 = Dense(300, activation='relu')(decoder1)\n",
    "outputs = Dense(VOCAB_SIZE, activation='softmax')(decoder2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge the two input models\n",
    "# image_feature + partial caption ===> output\n",
    "model = Model(inputs=[inputs1, inputs2], outputs=outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 39)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 2048)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 39, 300)      499500      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 2048)         0           input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 39, 300)      0           embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 300)          614700      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 300)          721200      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 300)          0           dense_1[0][0]                    \n",
      "                                                                 lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 300)          90300       add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1665)         501165      dense_2[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 2,426,865\n",
      "Trainable params: 2,426,865\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setting wight of embedded matrix that we saved earlier for words\n",
    "from pickle import load,dump\n",
    "with open(\"embedding_matrix.pkl\",\"rb\") as f:\n",
    "    embedding_matrix = load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<keras.layers.embeddings.Embedding object at 0x000001EE86BC9748>\n"
     ]
    }
   ],
   "source": [
    "print(model.layers[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.layers[2].set_weights([embedding_matrix])\n",
    "model.layers[2].trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# keras library import  for Saving and loading model and weights\n",
    "\n",
    "from keras.models import model_from_json\n",
    "from keras.models import load_model\n",
    "\n",
    "model_json = model.to_json()\n",
    "\n",
    "\n",
    "with open(\"model.json\", \"w\") as json_file:\n",
    "    json_file.write(model_json)\n",
    "\n",
    "# serialize weights to HDF5\n",
    "model.save_weights(\"model.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}