CNN+LSTM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "clevr(1)-Copy1.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "TI2f3JF3JqA6",
        "colab_type": "code",
        "colab": {},
        "outputId": "81225a9d-2881-4217-bc94-c78859da4bd4"
      },
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "from __future__ import print_function\n",
        "import json\n",
        "import os.path\n",
        "import random as ra\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import keras\n",
        "from keras.optimizers import Adam\n",
        "from keras import backend as K\n",
        "from keras.layers import Input, Dense, Dropout, BatchNormalization, Reshape, Lambda, Embedding, LSTM, Conv2D, MaxPooling2D, TimeDistributed, RepeatVector, Concatenate, Multiply\n",
        "from keras.models import Model\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing import sequence\n",
        "from keras.callbacks import ModelCheckpoint, TensorBoard\n",
        "from scipy import ndimage, misc\n",
        "from imageio import imread\n",
        "from keras.utils.vis_utils import plot_model\n",
        "from numpy import array\n",
        "from numpy import argmax\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.preprocessing import OneHotEncoder\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "1ipVN44LJqBL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n",
        "json1_file = open('Training/Quest_Answers.json')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UWWgCDSsJqBQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "json1_str = json1_file.read()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "x0VhiSS8JqBU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "json1_data = json.loads(json1_str)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cTfwUMjaJqBZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "json1_data = json1_data['quest_answers']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kItXK8iJJqBd",
        "colab_type": "code",
        "colab": {},
        "outputId": "17a90997-764e-460c-8531-64d142318bfb"
      },
      "cell_type": "code",
      "source": [
        "len(json1_data)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "135020"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "Ck0XGVOsJqBg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "q=[a['Question'] for a in json1_data ]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "V4IBqsuYJqBj",
        "colab_type": "code",
        "colab": {},
        "outputId": "671e356f-e9f0-43ad-8ab1-cc77b8319573"
      },
      "cell_type": "code",
      "source": [
        "len(q)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "135020"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "metadata": {
        "id": "043nP-RcJqBn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "lens = []\n",
        "for i in range (len(q)):\n",
        "    lens.extend(q[i].split())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "scrolled": false,
        "id": "Ee4xkzdGJqBq",
        "colab_type": "code",
        "colab": {},
        "outputId": "c1e790bf-a146-4d72-982b-016f561ecdcf"
      },
      "cell_type": "code",
      "source": [
        "lens[0:5]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['There', 'is', 'a', 'metal', 'thing']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "metadata": {
        "id": "aZrugK6JJqBt",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from collections import Counter "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "EzuuWFBPJqBw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "Counter = Counter(lens) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QbgeBNMbJqBz",
        "colab_type": "code",
        "colab": {},
        "outputId": "35a217c0-e6cc-4139-e836-5474fc3ab218"
      },
      "cell_type": "code",
      "source": [
        "most_occur = Counter.most_common(115) \n",
        "  \n",
        "print(most_occur) "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('the', 348314), ('is', 144688), ('of', 142252), ('that', 79365), ('same', 68596), ('as', 61568), ('thing', 43831), ('big', 42187), ('object', 41881), ('are', 41818), ('rubber', 39696), ('large', 39479), ('tiny', 39446), ('matte', 39221), ('small', 38922), ('What', 35438), ('a', 33043), ('There', 30215), ('number', 29080), ('left', 28754), ('behind', 28630), ('right', 28513), ('in', 28472), ('front', 28472), ('there', 27403), ('what', 26588), ('metallic', 26506), ('metal', 26384), ('shiny', 26197), ('things', 23554), ('brown', 22984), ('shape', 22920), ('red', 22734), ('color', 22685), ('purple', 22638), ('cyan', 22468), ('cylinder', 22459), ('green', 22409), ('gray', 22358), ('yellow', 22267), ('blue', 22010), ('size', 21623), ('objects', 21256), ('Is', 20800), ('material', 20767), ('to', 19161), ('on', 19119), ('side', 19119), ('have', 16962), ('thing?', 16799), ('object?', 16727), ('How', 16214), ('many', 16018), ('and', 15616), ('Are', 15404), ('cylinder?', 14372), ('other', 13868), ('it?', 13488), ('or', 12096), ('ball', 11479), ('sphere', 11388), ('block', 11273), ('cube', 11250), ('any', 11081), ('made', 10488), ('its', 10258), ('cylinders', 9242), ('than', 9133), ('The', 7727), ('cube?', 7359), ('sphere?', 7284), ('block?', 7225), ('ball?', 7169), ('it', 6469), ('thing;', 6436), ('object;', 6427), ('color?', 6101), ('either', 6006), ('shape?', 5865), ('cylinder;', 5725), ('material?', 5013), ('has', 4928), ('size?', 4796), ('Do', 4766), ('cubes', 4642), ('balls', 4588), ('spheres', 4559), ('Does', 4456), ('blocks', 4455), ('both', 3745), ('things?', 3535), ('objects?', 3491), ('cylinders?', 3376), ('of?', 3368), ('sphere;', 2988), ('ball;', 2969), ('block;', 2883), ('cube;', 2787), ('does', 2326), ('fewer', 2303), ('greater', 2289), ('less', 2288), ('more', 2253), ('another', 2210), ('how', 2168), ('anything', 2087), ('else', 2087), ('cubes?', 1748), ('spheres?', 1741), ('balls?', 1706), ('blocks?', 1647), ('there?', 1512), ('an', 1028), ('equal', 1028), ('visible?', 478)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "xGSIDLO2JqB2",
        "colab_type": "code",
        "colab": {},
        "outputId": "e9d819bd-4956-44a9-dc18-6423a0d0e1dc"
      },
      "cell_type": "code",
      "source": [
        "len(lens)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2473483"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "metadata": {
        "id": "LxXAxRjIJqB5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "setlens = set(lens)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OGzx7093JqB-",
        "colab_type": "code",
        "colab": {},
        "outputId": "3785d924-b7ec-4d50-bd15-a251d96232f1"
      },
      "cell_type": "code",
      "source": [
        "len(setlens)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "115"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "metadata": {
        "id": "VqaQL4OGJqCD",
        "colab_type": "code",
        "colab": {},
        "outputId": "bde293a5-1353-415f-ff99-e6bd602ca94d"
      },
      "cell_type": "code",
      "source": [
        "isinstance(1,bool)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "metadata": {
        "id": "oplBl6ZQJqCJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "for i in range(len(p)):\n",
        "    if (p[i] is False):\n",
        "        p[i] = 'False'\n",
        "    if (p[i] is True):\n",
        "        p[i] = 'True'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cuIoGoMWJqCM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "p = set(p)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "X7-bEObbJqCP",
        "colab_type": "code",
        "colab": {},
        "outputId": "4fceb374-60cb-48ac-e4c1-dfe6d37d09c2"
      },
      "cell_type": "code",
      "source": [
        "len(p)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "26"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 51
        }
      ]
    },
    {
      "metadata": {
        "scrolled": true,
        "id": "4-r0VmS-JqCS",
        "colab_type": "code",
        "colab": {},
        "outputId": "a9f77943-63a4-417c-f67f-eb3996c471a2"
      },
      "cell_type": "code",
      "source": [
        "0 is False"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "metadata": {
        "id": "JkcX9uF1JqCY",
        "colab_type": "code",
        "colab": {},
        "outputId": "e2440570-4a8c-41ed-b9ac-92ee71dbd91e"
      },
      "cell_type": "code",
      "source": [
        "False is 0"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "metadata": {
        "scrolled": true,
        "id": "WJiqzEnXJqCa",
        "colab_type": "code",
        "colab": {},
        "outputId": "2b5cf003-deaa-47d6-f0cc-306a2bf37d42"
      },
      "cell_type": "code",
      "source": [
        "array(q)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['0', 'False', 'large', ..., '0', '0', '1'], dtype='<U11')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "metadata": {
        "id": "6bYZoa2rJqCd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 229
        },
        "outputId": "cf37cbc0-38e2-4e51-8009-0b80eeaf7719"
      },
      "cell_type": "code",
      "source": [
        "\n",
        "#One-hot encoding labels\n",
        "label_encoder = LabelEncoder()\n",
        "integer_encoded = label_encoder.fit_transform(q)\n",
        "# print(integer_encoded)\n",
        "# binary encode\n",
        "onehot_encoder = OneHotEncoder(sparse=False)\n",
        "integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)\n",
        "onehot_encoded = onehot_encoder.fit_transform(integer_encoded)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-18cae3f5e7ae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlabel_encoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLabelEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0minteger_encoded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlabel_encoder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# print(integer_encoded)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# binary encode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0monehot_encoder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOneHotEncoder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msparse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'LabelEncoder' is not defined"
          ]
        }
      ]
    },
    {
      "metadata": {
        "scrolled": false,
        "id": "Xv_amFyBJqCg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 162
        },
        "outputId": "bcd8305c-8042-4f84-f000-6912ae564d8b"
      },
      "cell_type": "code",
      "source": [
        "onehot_encoded[1]"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-e836a5eaeb96>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0monehot_encoded\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'onehot_encoded' is not defined"
          ]
        }
      ]
    },
    {
      "metadata": {
        "id": "QdvzHMB-JqCk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        " label_encoder.inverse_transform([argmax(2)])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IuYVwG5mJqCp",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "o9xTxPc5JqCr",
        "colab_type": "code",
        "colab": {},
        "outputId": "66cbe575-fe30-4367-c690-b4730488d403"
      },
      "cell_type": "code",
      "source": [
        "tf.VERSION"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'1.12.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "metadata": {
        "id": "9-jfOaAmJqCu",
        "colab_type": "code",
        "colab": {},
        "outputId": "1b8e44c4-99e6-408e-8b5a-3b10b10f8865"
      },
      "cell_type": "code",
      "source": [
        "False is False"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "metadata": {
        "id": "uuOmjINhJqCx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "path = 'Training'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AYowryX-JqCz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "lab = {}\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Nwl_EW6CJqC1",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def load_data(n, vocab_size, sequence_length, tokenizer=None):\n",
        "    # Dataset paths\n",
        "    # \tpath = '/'\n",
        "    questions_path = path + '/Quest_Answers.json'\n",
        "#     subset_questions_path = path + '/subsetQs' + '.json'\n",
        "    images_path = path + '/images/' \n",
        "\n",
        "    x_text = []     # List of questions\n",
        "    x_image = []    # List of images\n",
        "    y = []          # List of answers\n",
        "    num_labels = 0  # Current number of labels, used to create index mapping\n",
        "    labels = {}     # Dictionary mapping of ints to labels\n",
        "    images = {}     # Dictionary of images, to minimize number of imread ops\n",
        "\n",
        "    # Attempt to load saved JSON subset of the questions\n",
        "    print('Loading data...')\n",
        "\n",
        " \n",
        "    with open(questions_path) as f:\n",
        "        data = json.load(f)\n",
        "\n",
        "    data = data['quest_answers'][0:n]\n",
        "    print('JSON subset saved to file...')\n",
        "\n",
        "    # Store image data and labels in dictionaries\n",
        "    print('Storing image data...')\n",
        "\n",
        "    for q in data[0:n]:\n",
        "        # Create an index for each answer\n",
        "        if not q['Answer'] in labels :\n",
        "            labels[q['Answer']] = num_labels\n",
        "            num_labels += 1\n",
        "        \n",
        "\n",
        "        # Create an index for each image\n",
        "        if not q['Image'] in images:\n",
        "            images[q['Image']] = imread(images_path + q['Image'] + '.png', pilmode='RGB')\n",
        "\n",
        "        x_text.append(q['Question'])\n",
        "        x_image.append(images[q['Image']])\n",
        "        y.append(labels[q['Answer']])\n",
        "\n",
        "    # Convert question corpus into sequential encoding for LSTM\n",
        "    print('Processing text data...')\n",
        "\n",
        "    if not tokenizer:\n",
        "        tokenizer = Tokenizer(num_words=vocab_size)\n",
        "\n",
        "    tokenizer.fit_on_texts(x_text)\n",
        "    sequences = tokenizer.texts_to_sequences(x_text)\n",
        "    x_text = sequence.pad_sequences(sequences, maxlen=sequence_length)\n",
        "\n",
        "    # Convert x_image to np array\n",
        "    x_image = np.array(x_image)\n",
        "\n",
        "    # Convert labels to categorical labels\n",
        "    y = keras.utils.to_categorical(y, num_labels)\n",
        "    print(len(data))\n",
        "    print('Text: ', x_text.shape)\n",
        "    print('Image: ', x_image.shape)\n",
        "    print('Labels: ', y.shape)\n",
        "    print(num_labels)\n",
        "    return ([x_text, x_image], y), num_labels, tokenizer\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FSBJO30eJqC4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def process_image(x):\n",
        "    target_height, target_width = 128, 128\n",
        "#     rotation_range = .05  # In radians\n",
        "#     degs = ra.uniform(-rotation_range, rotation_range)\n",
        "    x = tf.image.resize_images(x, (target_height, target_width), method=tf.image.ResizeMethod.AREA)\n",
        "#     x = tf.contrib.image.rotate(x, degs)\n",
        "    return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "NZkprDnxJqC5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def get_relation_vectors(x):\n",
        "    objects = []\n",
        "    relations = []\n",
        "    shape = K.int_shape(x)\n",
        "    k = 30     # Hyperparameter which controls how many objects are considered\n",
        "    keys = []\n",
        "\n",
        "    # Get k unique random objects\n",
        "    while k > 0:\n",
        "        i = ra.randint(0, shape[1]-1)\n",
        "        j = ra.randint(0, shape[2]-1)\n",
        "        if not (i, j) in keys:\n",
        "            keys.append((i, j))\n",
        "            objects.append(x[:, i, j, :])\n",
        "            k -= 1\n",
        "\n",
        "    # Concatenate each pair of objects to form a relation vector\n",
        "    for i in range(len(objects)):\n",
        "        for j in range(i, len(objects)):\n",
        "            relations.append(K.concatenate([objects[i], objects[j]], axis=1))\n",
        "\n",
        "    # Restack objects into Keras tensor [batch, relation_ID, relation_vectors]\n",
        "    return K.permute_dimensions(K.stack([r for r in relations], axis=0), [1, 0, 2])\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "G-jnhaayJqC8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "samples = 20000"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DRLPQbexJqC9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#setting hyperparameters\n",
        "learning_rate = .00025\n",
        "vocab_size = 512\n",
        "sequence_length = 64\n",
        "img_rows, img_cols = 120, 160\n",
        "image_input_shape = (img_rows, img_cols, 3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "CdVYL2V8JqDB",
        "colab_type": "code",
        "colab": {},
        "outputId": "359dd0d4-2d46-43ea-b9b9-fae0fb0efd12"
      },
      "cell_type": "code",
      "source": [
        "(x_train, y_train), num_labels, tokenizer = load_data(samples, vocab_size, sequence_length)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading data...\n",
            "JSON subset saved to file...\n",
            "Storing image data...\n",
            "Processing text data...\n",
            "20000\n",
            "Text:  (20000, 64)\n",
            "Image:  (20000, 120, 160, 3)\n",
            "Labels:  (20000, 24)\n",
            "24\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "5pfDcbSOJqDJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#LSTM Head\n",
        "text_inputs = Input(shape=(sequence_length,), name='text_input')\n",
        "text_x = Embedding(vocab_size, 128)(text_inputs)\n",
        "text_x = LSTM(128,return_sequences=True,activation='tanh')(text_x)\n",
        "text_x = Dropout(0.5)(text_x)\n",
        "text_x = LSTM(128)(text_x)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "qYxQ519RJqDN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "LSTM above\n",
        "CNN Below"
      ]
    },
    {
      "metadata": {
        "id": "vyASSytyJqDO",
        "colab_type": "code",
        "colab": {},
        "outputId": "526bb515-f12c-4355-988d-317b58a63471"
      },
      "cell_type": "code",
      "source": [
        "#CNN head\n",
        "\n",
        "image_inputs = Input(shape=image_input_shape, name='image_input')\n",
        "image_x = Lambda(process_image)(image_inputs)\n",
        "print(image_x)\n",
        "image_x = Conv2D(8, kernel_size=(3, 3), strides=1, activation='relu')(image_x)\n",
        "image_x = BatchNormalization()(image_x)\n",
        "image_x = Conv2D(8, kernel_size=(3, 3), strides=1, activation='relu')(image_x)\n",
        "image_x = BatchNormalization()(image_x)\n",
        "image_x = MaxPooling2D(pool_size=2)(image_x)\n",
        "\n",
        "image_x = Conv2D(16, kernel_size=(3, 3), strides=1, activation='relu')(image_x)\n",
        "image_x = BatchNormalization()(image_x)\n",
        "image_x = Conv2D(16, kernel_size=(3, 3), strides=1, activation='relu')(image_x)\n",
        "image_x = BatchNormalization()(image_x)\n",
        "image_x = MaxPooling2D(pool_size=2)(image_x)\n",
        "\n",
        "print(image_x)\n",
        "shape = K.int_shape(image_x)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tensor(\"lambda_1/resize_images/ResizeArea:0\", shape=(?, 128, 128, 3), dtype=float32)\n",
            "Tensor(\"max_pooling2d_2/MaxPool:0\", shape=(?, 29, 29, 16), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "scrolled": false,
        "id": "MIcX18lWJqDV",
        "colab_type": "code",
        "colab": {},
        "outputId": "c824afea-d293-4e93-b450-7ae7ce06d1ca"
      },
      "cell_type": "code",
      "source": [
        "print(shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(None, 29, 29, 16)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "6Nh9n785JqDc",
        "colab_type": "code",
        "colab": {},
        "outputId": "5a0cd242-e8b7-4c86-cf88-b8d8788998b0"
      },
      "cell_type": "code",
      "source": [
        "K.int_shape(image_x)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(None, 29, 29, 16)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "metadata": {
        "id": "OcFGxnBtJqDf",
        "colab_type": "code",
        "colab": {},
        "outputId": "f1e135c0-8a76-4b6c-a1ef-68fd8af76ab5"
      },
      "cell_type": "code",
      "source": [
        "K.int_shape(text_x)[1]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "128"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "metadata": {
        "id": "eUTkas44JqDg",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "RN below\n"
      ]
    },
    {
      "metadata": {
        "id": "LmSedoqoJqDk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#Concatenating CNN and RNN output vectors and adding a few dense layers\n",
        "\n",
        "RN_inputs = Input(shape=(1, (2 * shape[3]) + K.int_shape(text_x)[1]))\n",
        "RN_x = Dense(256, activation='relu')(RN_inputs)\n",
        "RN_x = Dense(256, activation='relu')(RN_x)\n",
        "RN_x = Dense(256, activation='relu')(RN_x)\n",
        "RN_x = Dropout(.5)(RN_x)\n",
        "RN_outputs = Dense(256, activation='relu')(RN_x)\n",
        "RN = Model(inputs=RN_inputs, outputs=RN_outputs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_eM97lVMJqDm",
        "colab_type": "code",
        "colab": {},
        "outputId": "4e2f17f9-e243-45cf-8a25-9e6b68bd03f6"
      },
      "cell_type": "code",
      "source": [
        "# image_x.shape\n",
        "get_relation_vectors(image_x)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'transpose:0' shape=(?, 465, 32) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "metadata": {
        "id": "nUp2vQNzJqDo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#\n",
        "# Implements g_theta\n",
        "#\n",
        "relations = Lambda(get_relation_vectors)(image_x)           # Get tensor [batch, relation_ID, relation_vectors]\n",
        "question = RepeatVector(K.int_shape(relations)[1])(text_x)  # Shape question vector to same size as relations\n",
        "relations = Concatenate(axis=2)([relations, question])      # Merge tensors [batch, relation_ID, relation_vectors, question_vector]\n",
        "g = TimeDistributed(RN)(relations)                          # TimeDistributed applies RN to relation vectors.\n",
        "g = Lambda(lambda x: K.sum(x, axis=1))(g) # Sum over relation_ID"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "q7aCMYInJqDr",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#\n",
        "# Define f_phi\n",
        "#\n",
        "f = Dense(256, activation='relu')(g)\n",
        "f = Dropout(.5)(f)\n",
        "f = Dense(256, activation='relu')(f)\n",
        "f = Dropout(.5)(f)\n",
        "outputs = Dense(num_labels, activation='softmax')(f)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "scrolled": true,
        "id": "rf_AMaI6JqDv",
        "colab_type": "code",
        "colab": {},
        "outputId": "382308d9-1278-4fb9-a9b0-b7f7220ff903"
      },
      "cell_type": "code",
      "source": [
        "\n",
        "model = Model(inputs=[text_inputs, image_inputs], outputs=outputs) \n",
        "print(model.summary())\n",
        "model.compile(optimizer=Adam(lr=learning_rate), loss='categorical_crossentropy', metrics=['accuracy'])\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "image_input (InputLayer)        (None, 120, 160, 3)  0                                            \n",
            "__________________________________________________________________________________________________\n",
            "lambda_1 (Lambda)               (None, 128, 128, 3)  0           image_input[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_1 (Conv2D)               (None, 126, 126, 8)  224         lambda_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_1 (BatchNor (None, 126, 126, 8)  32          conv2d_1[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_2 (Conv2D)               (None, 124, 124, 8)  584         batch_normalization_1[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_2 (BatchNor (None, 124, 124, 8)  32          conv2d_2[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_1 (MaxPooling2D)  (None, 62, 62, 8)    0           batch_normalization_2[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_3 (Conv2D)               (None, 60, 60, 16)   1168        max_pooling2d_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "text_input (InputLayer)         (None, 64)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_3 (BatchNor (None, 60, 60, 16)   64          conv2d_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, 64, 128)      65536       text_input[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2d_4 (Conv2D)               (None, 58, 58, 16)   2320        batch_normalization_3[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                   (None, 64, 128)      131584      embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "batch_normalization_4 (BatchNor (None, 58, 58, 16)   64          conv2d_4[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 64, 128)      0           lstm_1[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "max_pooling2d_2 (MaxPooling2D)  (None, 29, 29, 16)   0           batch_normalization_4[0][0]      \n",
            "__________________________________________________________________________________________________\n",
            "lstm_2 (LSTM)                   (None, 128)          131584      dropout_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "lambda_2 (Lambda)               (None, 465, 32)      0           max_pooling2d_2[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "repeat_vector_1 (RepeatVector)  (None, 465, 128)     0           lstm_2[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 465, 160)     0           lambda_2[0][0]                   \n",
            "                                                                 repeat_vector_1[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "time_distributed_1 (TimeDistrib (None, 465, 256)     238592      concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_3 (Lambda)               (None, 256)          0           time_distributed_1[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "dense_5 (Dense)                 (None, 256)          65792       lambda_3[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dropout_3 (Dropout)             (None, 256)          0           dense_5[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_6 (Dense)                 (None, 256)          65792       dropout_3[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dropout_4 (Dropout)             (None, 256)          0           dense_6[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_7 (Dense)                 (None, 24)           6168        dropout_4[0][0]                  \n",
            "==================================================================================================\n",
            "Total params: 709,536\n",
            "Trainable params: 709,440\n",
            "Non-trainable params: 96\n",
            "__________________________________________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "tRSomYUuJqDz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "plot_model(model)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "w28iUqDwJqD2",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "![image.png](attachment:image.png)\n"
      ]
    },
    {
      "metadata": {
        "id": "f4FFQgenJqD4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "epochs = 1\n",
        "batch_size = 256\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "scrolled": false,
        "id": "METOc1UeJqD7",
        "colab_type": "code",
        "colab": {},
        "outputId": "e01db1f1-8f80-4166-912b-8b8bb714861b"
      },
      "cell_type": "code",
      "source": [
        "model.fit(x_train, y_train, validation_split=0.01,\n",
        "\t      batch_size=batch_size, \n",
        "\t      epochs=epochs, \n",
        "\t      shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 99000 samples, validate on 1000 samples\n",
            "Epoch 1/1\n",
            "99000/99000 [==============================] - 199s 2ms/step - loss: 2.9160 - acc: 0.2984 - val_loss: 1.5390 - val_acc: 0.3450\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x1f5e5ef5550>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "metadata": {
        "id": "ZI2nF1ldJqEB",
        "colab_type": "code",
        "colab": {},
        "outputId": "99f7d00d-8022-48de-c2ac-a2f700412935"
      },
      "cell_type": "code",
      "source": [
        "model.fit(x_train, y_train, validation_split=0.01,\n",
        "\t      batch_size=batch_size, \n",
        "\t      epochs=epochs*3, \n",
        "\t      shuffle=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 99000 samples, validate on 1000 samples\n",
            "Epoch 1/3\n",
            "99000/99000 [==============================] - 195s 2ms/step - loss: 1.4336 - acc: 0.3623 - val_loss: 1.1575 - val_acc: 0.3930\n",
            "Epoch 2/3\n",
            "99000/99000 [==============================] - 197s 2ms/step - loss: 1.2152 - acc: 0.3899 - val_loss: 1.0494 - val_acc: 0.4440\n",
            "Epoch 3/3\n",
            "99000/99000 [==============================] - 196s 2ms/step - loss: 1.0832 - acc: 0.4183 - val_loss: 1.0073 - val_acc: 0.4180\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x1f5db351128>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "metadata": {
        "scrolled": false,
        "id": "eLhwlLYJJqEE",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "model.save('Models_nnfl/model_4180_1lakh.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Cpxa8MJuJqEI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "b3sU0TmoJqEJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}