Preprocessed.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import cv2, numpy as np\n",
    "import time\n",
    "import theano\n",
    "import os\n",
    "from collections import OrderedDict, defaultdict\n",
    "import six.moves.cPickle as pkl\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, LSTM\n",
    "from keras.layers import Embedding\n",
    "from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D \n",
    "from keras.optimizers import SGD\n",
    "from keras.layers import GRU, TimeDistributed, RepeatVector, Merge, TimeDistributedDense\n",
    "import h5py\n",
    "import json\n",
    "from collections import Counter\n",
    "import matplotlib.pyplot as plt\n",
    "import skimage.transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MEAN_VALUES = np.array([104, 117, 123]).reshape((3,1,1))\n",
    "SEQUENCE_LENGTH = 32\n",
    "MAX_SENTENCE_LENGTH = SEQUENCE_LENGTH - 3 # 1 for image, 1 for start token, 1 for end token\n",
    "BATCH_SIZE = 20\n",
    "CNN_FEATURE_SIZE = 1000\n",
    "EMBEDDING_SIZE = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def word_processing(dataset):\n",
    "    allwords = Counter()\n",
    "    for item in dataset:\n",
    "        for sentence in item['sentences']:\n",
    "            allwords.update(sentence['tokens'])\n",
    "            \n",
    "    vocab = [k for k, v in allwords.items()]\n",
    "    vocab.insert(0, '<NULL>')\n",
    "    vocab.append('<START>')\n",
    "    vocab.append('<END>')\n",
    "    vocab.append('<UNK>')\n",
    "\n",
    "    word_to_index = {w: i for i, w in enumerate(vocab)}\n",
    "    index_to_word = {i: w for i, w in enumerate(vocab)}\n",
    "    return vocab, word_to_index, index_to_word\n",
    "\n",
    "def import_flickr8kdataset():\n",
    "    dataset = json.load(open('captions/dataset_flickr8k.json'))['images']\n",
    "    #reduced length to a 300 for testing\n",
    "    val_set = list(filter(lambda x: x['split'] == 'val', dataset))\n",
    "    train_set = list(filter(lambda x: x['split'] == 'train', dataset))\n",
    "    test_set = list(filter(lambda x: x['split'] == 'test', dataset))\n",
    "    return train_set[:800]+val_set[:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def floatX(arr):\n",
    "    return np.asarray(arr, dtype=theano.config.floatX)\n",
    "\n",
    "#Prep Image uses an skimage transform\n",
    "def prep_image(im):\n",
    "    if len(im.shape) == 2:\n",
    "        im = im[:, :, np.newaxis]\n",
    "        im = np.repeat(im, 3, axis=2)\n",
    "    # Resize so smallest dim = 224, preserving aspect ratio\n",
    "    h, w, _ = im.shape\n",
    "    if h < w:\n",
    "        im = skimage.transform.resize(im, (224, w*224/h), preserve_range=True)\n",
    "    else:\n",
    "        im = skimage.transform.resize(im, (h*224/w, 224), preserve_range=True)\n",
    "\n",
    "    # Central crop to 224x224\n",
    "    h, w, _ = im.shape\n",
    "    im = im[h//2-112:h//2+112, w//2-112:w//2+112]\n",
    "    \n",
    "    rawim = np.copy(im).astype('uint8')\n",
    "    \n",
    "    # Shuffle axes to c01\n",
    "    im = np.swapaxes(np.swapaxes(im, 1, 2), 0, 1)\n",
    "    \n",
    "    # Convert to BGR\n",
    "    im = im[::-1, :, :]\n",
    "\n",
    "    im = im - MEAN_VALUES\n",
    "    return rawim, floatX(im[np.newaxis])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def VGG_16(weights_path=None):\n",
    "    model = Sequential()\n",
    "    model.add(ZeroPadding2D((1,1),input_shape=(3,224,224)))\n",
    "    model.add(Convolution2D(64, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(64, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2,2), strides=(2,2)))\n",
    "\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(128, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(128, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2,2), strides=(2,2)))\n",
    "\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(256, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(256, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(256, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2,2), strides=(2,2)))\n",
    "\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2,2), strides=(2,2)))\n",
    "\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1,1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2,2), strides=(2,2)))\n",
    "\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(4096, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(4096, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(1000, activation='softmax'))\n",
    "\n",
    "    if weights_path:\n",
    "        model.load_weights(weights_path)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = import_flickr8kdataset()\n",
    "# Currently testing it out\n",
    "dataset = [i for i in dataset[:100]]\n",
    "vocab,word_to_index, index_to_word = word_processing(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def chunks(l, n):\n",
    "    for i in xrange(0, len(l), n):\n",
    "        yield l[i:i + n]\n",
    "\n",
    "def process_cnn_features(dataset, model, coco=False, d_set=\"Flicker8k_Dataset\"):\n",
    "    ind_process = 1\n",
    "    total = len(dataset)\n",
    "    for chunk in chunks(dataset, 25):\n",
    "        cnn_input = floatX(np.zeros((len(chunk), 3, 224, 224)))\n",
    "        for i, image in enumerate(chunk):\n",
    "            print \"ind_process %s total %s\" %(str(ind_process),str(total))\n",
    "            ind_process+=1\n",
    "            if coco:\n",
    "                fn = './coco/{}/{}'.format(image['filepath'], image['filename'])\n",
    "            else:\n",
    "                fn = d_set+'/{}'.format(image['filename'])\n",
    "            try:\n",
    "                im = plt.imread(fn)\n",
    "                _, cnn_input[i] = prep_image(im)\n",
    "            except IOError:\n",
    "                continue\n",
    "        features = model.predict(cnn_input)\n",
    "        print \"Processing Features For Chunk\"\n",
    "        for i, image in enumerate(chunk):\n",
    "            image['cnn features'] = features[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_caption_features(dataset, coco=False, d_set=\"Flicker8k_Dataset\"):\n",
    "    ind_process = 1\n",
    "    total = len(dataset)\n",
    "    for image in dataset:\n",
    "        partial_caption_ar = np.zeros(SEQUENCE_LENGTH, dtype=np.int)\n",
    "        #print SEQUENCE_LENGTH, len(image['sentences'][0]['tokens'])\n",
    "        words = ['<START>'] + image['sentences'][0]['tokens'] + ['<END>']\n",
    "        #print image['sentences'][0]['tokens']\n",
    "        assert len(words)<=SEQUENCE_LENGTH\n",
    "        for i in range(len(words)):\n",
    "            \n",
    "            if words[i] in word_to_index:\n",
    "                partial_caption_ar[i] = word_to_index[words[i]]\n",
    "            else:\n",
    "                partial_caption_ar[i] = word_to_index[\"<UNK>\"]\n",
    "        image['captions'] = partial_caption_ar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/danielsampetethiyagu/anaconda/lib/python2.7/site-packages/keras/backend/theano_backend.py:1500: UserWarning: DEPRECATION: the 'ds' parameter is not going to exist anymore as it is going to be replaced by the parameter 'ws'.\n",
      "  mode='max')\n",
      "/Users/danielsampetethiyagu/anaconda/lib/python2.7/site-packages/keras/backend/theano_backend.py:1500: UserWarning: DEPRECATION: the 'st' parameter is not going to exist anymore as it is going to be replaced by the parameter 'stride'.\n",
      "  mode='max')\n",
      "/Users/danielsampetethiyagu/anaconda/lib/python2.7/site-packages/keras/backend/theano_backend.py:1500: UserWarning: DEPRECATION: the 'padding' parameter is not going to exist anymore as it is going to be replaced by the parameter 'pad'.\n",
      "  mode='max')\n"
     ]
    }
   ],
   "source": [
    "model = VGG_16('weights/vgg16_weights.h5')\n",
    "sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "model.compile(optimizer=sgd, loss='categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "dataset = json.load(open('captions/dataset_flickr8k.json'))['images']\n",
    "val_set = list(filter(lambda x: x['split'] == 'val', dataset))\n",
    "train_set = list(filter(lambda x: x['split'] == 'train', dataset))\n",
    "test_set = list(filter(lambda x: x['split'] == 'test', dataset))\n",
    "dataset = train_set[:200]+val_set[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ind_process 1 total 250\n",
      "ind_process 2 total 250\n",
      "ind_process 3 total 250\n",
      "ind_process 4 total 250\n",
      "ind_process 5 total 250\n",
      "ind_process 6 total 250\n",
      "ind_process 7 total 250\n",
      "ind_process 8 total 250\n",
      "ind_process 9 total 250\n",
      "ind_process 10 total 250\n",
      "ind_process 11 total 250\n",
      "ind_process 12 total 250\n",
      "ind_process 13 total 250\n",
      "ind_process 14 total 250\n",
      "ind_process 15 total 250\n",
      "ind_process 16 total 250\n",
      "ind_process 17 total 250\n",
      "ind_process 18 total 250\n",
      "ind_process 19 total 250\n",
      "ind_process 20 total 250\n",
      "ind_process 21 total 250\n",
      "ind_process 22 total 250\n",
      "ind_process 23 total 250\n",
      "ind_process 24 total 250\n",
      "ind_process 25 total 250\n",
      "Processing Features For Chunk\n",
      "ind_process 26 total 250\n",
      "ind_process 27 total 250\n",
      "ind_process 28 total 250\n",
      "ind_process 29 total 250\n",
      "ind_process 30 total 250\n",
      "ind_process 31 total 250\n",
      "ind_process 32 total 250\n",
      "ind_process 33 total 250\n",
      "ind_process 34 total 250\n",
      "ind_process 35 total 250\n",
      "ind_process 36 total 250\n",
      "ind_process 37 total 250\n",
      "ind_process 38 total 250\n",
      "ind_process 39 total 250\n",
      "ind_process 40 total 250\n",
      "ind_process 41 total 250\n",
      "ind_process 42 total 250\n",
      "ind_process 43 total 250\n",
      "ind_process 44 total 250\n",
      "ind_process 45 total 250\n",
      "ind_process 46 total 250\n",
      "ind_process 47 total 250\n",
      "ind_process 48 total 250\n",
      "ind_process 49 total 250\n",
      "ind_process 50 total 250\n",
      "Processing Features For Chunk\n",
      "ind_process 51 total 250\n",
      "ind_process 52 total 250\n",
      "ind_process 53 total 250\n",
      "ind_process 54 total 250\n",
      "ind_process 55 total 250\n",
      "ind_process 56 total 250\n",
      "ind_process 57 total 250\n",
      "ind_process 58 total 250\n",
      "ind_process 59 total 250\n",
      "ind_process 60 total 250\n",
      "ind_process 61 total 250\n",
      "ind_process 62 total 250\n",
      "ind_process 63 total 250\n",
      "ind_process 64 total 250\n",
      "ind_process 65 total 250\n",
      "ind_process 66 total 250\n",
      "ind_process 67 total 250\n",
      "ind_process 68 total 250\n",
      "ind_process 69 total 250\n",
      "ind_process 70 total 250\n",
      "ind_process 71 total 250\n",
      "ind_process 72 total 250\n",
      "ind_process 73 total 250\n",
      "ind_process 74 total 250\n",
      "ind_process 75 total 250\n",
      "Processing Features For Chunk\n",
      "ind_process 76 total 250\n",
      "ind_process 77 total 250\n",
      "ind_process 78 total 250\n",
      "ind_process 79 total 250\n",
      "ind_process 80 total 250\n",
      "ind_process 81 total 250\n",
      "ind_process 82 total 250\n",
      "ind_process 83 total 250\n",
      "ind_process 84 total 250\n",
      "ind_process 85 total 250\n",
      "ind_process 86 total 250\n",
      "ind_process 87 total 250\n",
      "ind_process 88 total 250\n",
      "ind_process 89 total 250\n",
      "ind_process 90 total 250\n",
      "ind_process 91 total 250\n",
      "ind_process 92 total 250\n",
      "ind_process 93 total 250\n",
      "ind_process 94 total 250\n",
      "ind_process 95 total 250\n",
      "ind_process 96 total 250\n",
      "ind_process 97 total 250\n",
      "ind_process 98 total 250\n",
      "ind_process 99 total 250\n",
      "ind_process 100 total 250\n",
      "Processing Features For Chunk\n",
      "ind_process 101 total 250\n",
      "ind_process 102 total 250\n",
      "ind_process 103 total 250\n",
      "ind_process 104 total 250\n",
      "ind_process 105 total 250\n",
      "ind_process 106 total 250\n",
      "ind_process 107 total 250\n",
      "ind_process 108 total 250\n",
      "ind_process 109 total 250\n",
      "ind_process 110 total 250\n",
      "ind_process 111 total 250\n",
      "ind_process 112 total 250\n",
      "ind_process 113 total 250\n",
      "ind_process 114 total 250\n",
      "ind_process 115 total 250\n",
      "ind_process 116 total 250\n",
      "ind_process 117 total 250\n",
      "ind_process 118 total 250\n",
      "ind_process 119 total 250\n",
      "ind_process 120 total 250\n",
      "ind_process 121 total 250\n",
      "ind_process 122 total 250\n",
      "ind_process 123 total 250\n",
      "ind_process 124 total 250\n",
      "ind_process 125 total 250\n",
      "Processing Features For Chunk\n",
      "ind_process 126 total 250\n",
      "ind_process 127 total 250\n",
      "ind_process 128 total 250\n",
      "ind_process 129 total 250\n",
      "ind_process 130 total 250\n",
      "ind_process 131 total 250\n",
      "ind_process 132 total 250\n",
      "ind_process 133 total 250\n",
      "ind_process 134 total 250\n",
      "ind_process 135 total 250\n",
      "ind_process 136 total 250\n",
      "ind_process 137 total 250\n",
      "ind_process 138 total 250\n",
      "ind_process 139 total 250\n",
      "ind_process 140 total 250\n",
      "ind_process 141 total 250\n",
      "ind_process 142 total 250\n",
      "ind_process 143 total 250\n",
      "ind_process 144 total 250\n",
      "ind_process 145 total 250\n",
      "ind_process 146 total 250\n",
      "ind_process 147 total 250\n",
      "ind_process 148 total 250\n",
      "ind_process 149 total 250\n",
      "ind_process 150 total 250\n",
      "Processing Features For Chunk\n",
      "ind_process 151 total 250\n",
      "ind_process 152 total 250\n",
      "ind_process 153 total 250\n",
      "ind_process 154 total 250\n",
      "ind_process 155 total 250\n",
      "ind_process 156 total 250\n",
      "ind_process 157 total 250\n",
      "ind_process 158 total 250\n",
      "ind_process 159 total 250\n",
      "ind_process 160 total 250\n",
      "ind_process 161 total 250\n",
      "ind_process 162 total 250\n",
      "ind_process 163 total 250\n",
      "ind_process 164 total 250\n",
      "ind_process 165 total 250\n",
      "ind_process 166 total 250\n",
      "ind_process 167 total 250\n",
      "ind_process 168 total 250\n",
      "ind_process 169 total 250\n",
      "ind_process 170 total 250\n",
      "ind_process 171 total 250\n",
      "ind_process 172 total 250\n",
      "ind_process 173 total 250\n",
      "ind_process 174 total 250\n",
      "ind_process 175 total 250\n",
      "Processing Features For Chunk\n",
      "ind_process 176 total 250\n",
      "ind_process 177 total 250\n",
      "ind_process 178 total 250\n",
      "ind_process 179 total 250\n",
      "ind_process 180 total 250\n",
      "ind_process 181 total 250\n",
      "ind_process 182 total 250\n",
      "ind_process 183 total 250\n",
      "ind_process 184 total 250\n",
      "ind_process 185 total 250\n",
      "ind_process 186 total 250\n",
      "ind_process 187 total 250\n",
      "ind_process 188 total 250\n",
      "ind_process 189 total 250\n",
      "ind_process 190 total 250\n",
      "ind_process 191 total 250\n",
      "ind_process 192 total 250\n",
      "ind_process 193 total 250\n",
      "ind_process 194 total 250\n",
      "ind_process 195 total 250\n",
      "ind_process 196 total 250\n",
      "ind_process 197 total 250\n",
      "ind_process 198 total 250\n",
      "ind_process 199 total 250\n",
      "ind_process 200 total 250\n",
      "Processing Features For Chunk\n",
      "ind_process 201 total 250\n",
      "ind_process 202 total 250\n",
      "ind_process 203 total 250\n",
      "ind_process 204 total 250\n",
      "ind_process 205 total 250\n",
      "ind_process 206 total 250\n",
      "ind_process 207 total 250\n",
      "ind_process 208 total 250\n",
      "ind_process 209 total 250\n",
      "ind_process 210 total 250\n",
      "ind_process 211 total 250\n",
      "ind_process 212 total 250\n",
      "ind_process 213 total 250\n",
      "ind_process 214 total 250\n",
      "ind_process 215 total 250\n",
      "ind_process 216 total 250\n",
      "ind_process 217 total 250\n",
      "ind_process 218 total 250\n",
      "ind_process 219 total 250\n",
      "ind_process 220 total 250\n",
      "ind_process 221 total 250\n",
      "ind_process 222 total 250\n",
      "ind_process 223 total 250\n",
      "ind_process 224 total 250\n",
      "ind_process 225 total 250\n",
      "Processing Features For Chunk\n",
      "ind_process 226 total 250\n",
      "ind_process 227 total 250\n",
      "ind_process 228 total 250\n",
      "ind_process 229 total 250\n",
      "ind_process 230 total 250\n",
      "ind_process 231 total 250\n",
      "ind_process 232 total 250\n",
      "ind_process 233 total 250\n",
      "ind_process 234 total 250\n",
      "ind_process 235 total 250\n",
      "ind_process 236 total 250\n",
      "ind_process 237 total 250\n",
      "ind_process 238 total 250\n",
      "ind_process 239 total 250\n",
      "ind_process 240 total 250\n",
      "ind_process 241 total 250\n",
      "ind_process 242 total 250\n",
      "ind_process 243 total 250\n",
      "ind_process 244 total 250\n",
      "ind_process 245 total 250\n",
      "ind_process 246 total 250\n",
      "ind_process 247 total 250\n",
      "ind_process 248 total 250\n",
      "ind_process 249 total 250\n",
      "ind_process 250 total 250\n",
      "Processing Features For Chunk\n"
     ]
    }
   ],
   "source": [
    "process_cnn_features(dataset, model, False, \"Flicker8k_Dataset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "process_caption_features(dataset, coco=False, d_set=\"Flicker8k_Dataset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "img_cnn_input = []#np.zeros((len(dataset),CNN_FEATURE_SIZE))\n",
    "sentences_input = []#np.zeros((len(dataset),SEQUENCE_LENGTH - 1))\n",
    "nextwords_input = []#np.zeros((len(dataset),SEQUENCE_LENGTH - 1))\n",
    "\n",
    "for ind, x in enumerate(dataset):\n",
    "    img_cnn_input.append(x['cnn features'])\n",
    "    sentences_input.append(x['captions'][0:len(x['captions'])-1])\n",
    "    nextwords_input.append(x['captions'][1:])\n",
    "img_cnn_input = np.array(img_cnn_input)#np.zeros((len(dataset),CNN_FEATURE_SIZE))\n",
    "sentences_input = np.array(sentences_input)#np.zeros((len(dataset),SEQUENCE_LENGTH - 1))\n",
    "nextwords_input = np.array(nextwords_input)#np.zeros((len(dataset),SEQUENCE_LENGTH - 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "VOCAB_COUNT = len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AssertionError",
     "evalue": "Keyword argument not understood: input_length",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-67-cdc2ac165053>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;31m# let's repeat the image vector to turn it into a sequence.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mimage_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDense\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEMBEDDING_SIZE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0minput_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mCNN_FEATURE_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;31m# the output of both models will be tensors of shape (samples, max_caption_len, 128).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/danielsampetethiyagu/anaconda/lib/python2.7/site-packages/keras/layers/core.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, output_dim, init, activation, weights, W_regularizer, b_regularizer, activity_regularizer, W_constraint, b_constraint, bias, input_dim, **kwargs)\u001b[0m\n\u001b[1;32m    684\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    685\u001b[0m             \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'input_shape'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 686\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDense\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/danielsampetethiyagu/anaconda/lib/python2.7/site-packages/keras/engine/topology.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m    305\u001b[0m                           'create_input_layer'}\n\u001b[1;32m    306\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mkwarg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 307\u001b[0;31m             \u001b[0;32massert\u001b[0m \u001b[0mkwarg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mallowed_kwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Keyword argument not understood: '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mkwarg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    309\u001b[0m         \u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'name'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAssertionError\u001b[0m: Keyword argument not understood: input_length"
     ]
    }
   ],
   "source": [
    "\n",
    "image_model = Sequential()\n",
    "language_model = Sequential()\n",
    "language_model.add(Embedding(VOCAB_COUNT, EMBEDDING_SIZE, input_length=SEQUENCE_LENGTH - 1))\n",
    "language_model.add(GRU(output_dim=128, return_sequences=True))\n",
    "language_model.add(TimeDistributed(Dense(128)))\n",
    "\n",
    "# let's repeat the image vector to turn it into a sequence.\n",
    "image_model.add(Dense(EMBEDDING_SIZE,input_shape=(CNN_FEATURE_SIZE))\n",
    "\n",
    "# the output of both models will be tensors of shape (samples, max_caption_len, 128).\n",
    "# let's concatenate these 2 vector sequences.\n",
    "imcap_model = Sequential()\n",
    "imcap_model.add(Merge([image_model, language_model], mode='concat', concat_axis=-1))\n",
    "# let's encode this vector sequence into a single vector\n",
    "imcap_model.add(GRU(EMBEDDING_SIZE, return_sequences=False))\n",
    "# which will be used to compute a probability\n",
    "# distribution over what the next word in the caption should be!\n",
    "imcap_model.add(Dense(VOCAB_COUNT))\n",
    "imcap_model.add(Activation('softmax'))\n",
    "\n",
    "imcap_model.compile(loss='categorical_crossentropy', optimizer='rmsprop')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}