NER-3rd.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n",
      "/home/user/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "# Keras==1.0.6\n",
    "import numpy as np\n",
    "from keras.models import Sequential\n",
    "from keras.layers.recurrent import LSTM\n",
    "from keras.layers.core import  Activation\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers.embeddings import Embedding\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, precision_recall_fscore_support\n",
    "import tensorflow_hub as hub\n",
    "from keras.layers import Dense,TimeDistributed\n",
    "import keras.layers as layers\n",
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input sequence length range:  144 1\n"
     ]
    }
   ],
   "source": [
    "raw = open('wikigold.conll.txt', 'r').readlines()\n",
    " \n",
    "all_x = []\n",
    "point = []\n",
    "for line in raw:\n",
    "    stripped_line = line.strip().split(' ')\n",
    "    point.append(stripped_line)\n",
    "    if line == '\\n':\n",
    "        all_x.append(point[:-1])\n",
    "        point = []\n",
    "all_x = all_x[:-1]\n",
    " \n",
    "lengths = [len(x) for x in all_x]\n",
    "print('Input sequence length range: ', max(lengths), min(lengths))\n",
    " \n",
    "short_x = [x for x in all_x if len(x) < 64]\n",
    " \n",
    "X = [[c[0] for c in x] for x in short_x]\n",
    "y = [[c[1] for c in y] for y in short_x]\n",
    " \n",
    "all_text = [c for x in X for c in x]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size: 8285 5\n"
     ]
    }
   ],
   "source": [
    "words = list(set(all_text))\n",
    "word2ind = {word: index for index, word in enumerate(words)}\n",
    "ind2word = {index: word for index, word in enumerate(words)}\n",
    "labels = list(set([c for x in y for c in x]))\n",
    "label2ind = {label: (index + 1) for index, label in enumerate(labels)}\n",
    "ind2label = {(index + 1): label for index, label in enumerate(labels)}\n",
    "print('Vocabulary size:', len(word2ind), len(label2ind))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum sequence length: 63\n",
      "{'I-PER': 1, 'O': 2, 'I-MISC': 3, 'I-LOC': 4, 'I-ORG': 5}\n"
     ]
    }
   ],
   "source": [
    "maxlen = max([len(x) for x in X])\n",
    "print('Maximum sequence length:', maxlen)\n",
    "print(label2ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encode(x, n):\n",
    "    result = np.zeros(n)\n",
    "    result[x] = 1\n",
    "    return result\n",
    " \n",
    "X_enc = [[word2ind[c] for c in x] for x in X]\n",
    "max_label = max(label2ind.values()) + 1\n",
    "y_enc = [[0] * (maxlen - len(ey)) + [label2ind[c] for c in ey] for ey in y]\n",
    "y_enc = [[encode(c, max_label) for c in ey] for ey in y_enc]\n",
    " \n",
    "X_enc = pad_sequences(X_enc, maxlen=maxlen)\n",
    "y_enc = pad_sequences(y_enc, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training and testing tensor shapes: (1440, 63) (352, 63) (1440, 63, 6) (352, 63, 6)\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_enc, y_enc, test_size=11*32, train_size=45*32, random_state=42)\n",
    "print('Training and testing tensor shapes:', X_train.shape, X_test.shape, y_train.shape, y_test.shape)\n",
    " \n",
    "max_features = len(word2ind)\n",
    "embedding_size = 300\n",
    "hidden_size = 32\n",
    "out_size = len(label2ind) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Downloading TF-Hub Module 'https://tfhub.dev/google/elmo/2'.\n",
      "INFO:tensorflow:Downloaded TF-Hub Module 'https://tfhub.dev/google/elmo/2'.\n"
     ]
    }
   ],
   "source": [
    "#elmo_model = hub.Module(\"https://tfhub.dev/google/elmo/1\", trainable=True)\n",
    "elmo_model = hub.Module(\"https://tfhub.dev/google/elmo/2\", trainable=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ElmoEmbedding(x):\n",
    "    return elmo_model(tf.squeeze(tf.cast(x, tf.string)), signature=\"default\", as_dict=True)[\"word_emb\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "sess = tf.Session()\n",
    "K.set_session(sess)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "sess.run(tf.tables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'embedding' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-4e9c1db5c2f1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0membedding\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'embedding' is not defined"
     ]
    }
   ],
   "source": [
    "embedding.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['OVERLOADABLE_OPERATORS',\n",
       " '__abs__',\n",
       " '__add__',\n",
       " '__and__',\n",
       " '__array_priority__',\n",
       " '__bool__',\n",
       " '__class__',\n",
       " '__copy__',\n",
       " '__delattr__',\n",
       " '__dict__',\n",
       " '__dir__',\n",
       " '__div__',\n",
       " '__doc__',\n",
       " '__eq__',\n",
       " '__floordiv__',\n",
       " '__format__',\n",
       " '__ge__',\n",
       " '__getattribute__',\n",
       " '__getitem__',\n",
       " '__gt__',\n",
       " '__hash__',\n",
       " '__init__',\n",
       " '__init_subclass__',\n",
       " '__invert__',\n",
       " '__iter__',\n",
       " '__le__',\n",
       " '__lt__',\n",
       " '__matmul__',\n",
       " '__mod__',\n",
       " '__module__',\n",
       " '__mul__',\n",
       " '__ne__',\n",
       " '__neg__',\n",
       " '__new__',\n",
       " '__nonzero__',\n",
       " '__or__',\n",
       " '__pow__',\n",
       " '__radd__',\n",
       " '__rand__',\n",
       " '__rdiv__',\n",
       " '__reduce__',\n",
       " '__reduce_ex__',\n",
       " '__repr__',\n",
       " '__rfloordiv__',\n",
       " '__rmatmul__',\n",
       " '__rmod__',\n",
       " '__rmul__',\n",
       " '__ror__',\n",
       " '__rpow__',\n",
       " '__rsub__',\n",
       " '__rtruediv__',\n",
       " '__rxor__',\n",
       " '__setattr__',\n",
       " '__sizeof__',\n",
       " '__str__',\n",
       " '__sub__',\n",
       " '__subclasshook__',\n",
       " '__truediv__',\n",
       " '__weakref__',\n",
       " '__xor__',\n",
       " '_as_node_def_input',\n",
       " '_as_tf_output',\n",
       " '_c_api_shape',\n",
       " '_consumers',\n",
       " '_dtype',\n",
       " '_get_input_ops_without_shapes',\n",
       " '_id',\n",
       " '_keras_history',\n",
       " '_keras_shape',\n",
       " '_op',\n",
       " '_override_operator',\n",
       " '_rank',\n",
       " '_shape',\n",
       " '_shape_as_list',\n",
       " '_shape_tuple',\n",
       " '_shape_val',\n",
       " '_tf_api_names',\n",
       " '_tf_api_names_v1',\n",
       " '_uses_learning_phase',\n",
       " '_value_index',\n",
       " 'consumers',\n",
       " 'device',\n",
       " 'dtype',\n",
       " 'eval',\n",
       " 'get_shape',\n",
       " 'graph',\n",
       " 'name',\n",
       " 'op',\n",
       " 'set_shape',\n",
       " 'shape',\n",
       " 'value_index']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir(embedding)\n",
    "\n",
    "#layer.get_output_at(0).get_shape().as_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ElmoEmbedding(x):\n",
    "    return elmo_model(tf.squeeze(tf.cast(x, tf.string)), signature=\"default\", as_dict=True)[\"word_emb\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'module_1_apply_default_1/bilm/Reshape_1:0' shape=(?, ?, 512) dtype=float32>"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_text = layers.Input(shape=(InputSize,300,), dtype=tf.string)\n",
    "\n",
    "ElmoEmbedding(input_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'main_input_10:0' shape=(?, 63) dtype=string>"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.cast(main_input, tf.string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'Squeeze_5:0' shape=<unknown> dtype=string>"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.squeeze(tf.cast(main_input, tf.string))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ElmoEmbedding(x):\n",
    "    return elmo_model(tf.squeeze(tf.cast(x, tf.string)), signature=\"default\", as_dict=True)[\"default\"]\n",
    "    #return elmo_model(tf.cast(x, tf.string), signature=\"default\", as_dict=True)[\"default\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'module_apply_default_4/pooling/truediv:0' shape=(?, 1024) dtype=float32>"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ElmoEmbedding(main_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for +: 'NoneType' and 'int'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-100-f7e2efaa60bd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;31m# A LSTM will transform the vector sequence into a single vector,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;31m# containing information about the entire sequence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m \u001b[0mlstm_out\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLSTM\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_sequences\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/layers/recurrent.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, initial_state, constants, **kwargs)\u001b[0m\n\u001b[1;32m    498\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    499\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0minitial_state\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mconstants\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 500\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mRNN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    501\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    502\u001b[0m         \u001b[0;31m# If any of `initial_state` or `constants` are specified and are Keras\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, **kwargs)\u001b[0m\n\u001b[1;32m    429\u001b[0m                                          \u001b[0;34m'You can build it manually via: '\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    430\u001b[0m                                          '`layer.build(batch_input_shape)`')\n\u001b[0;32m--> 431\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munpack_singleton\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shapes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    432\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    433\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/layers/recurrent.py\u001b[0m in \u001b[0;36mbuild\u001b[0;34m(self, input_shape)\u001b[0m\n\u001b[1;32m    459\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstep_input_shape\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mconstants_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    460\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 461\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcell\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep_input_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    462\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    463\u001b[0m         \u001b[0;31m# set or validate state_spec\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/layers/recurrent.py\u001b[0m in \u001b[0;36mbuild\u001b[0;34m(self, input_shape)\u001b[0m\n\u001b[1;32m   1797\u001b[0m                                       \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkernel_initializer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1798\u001b[0m                                       \u001b[0mregularizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkernel_regularizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1799\u001b[0;31m                                       constraint=self.kernel_constraint)\n\u001b[0m\u001b[1;32m   1800\u001b[0m         self.recurrent_kernel = self.add_weight(\n\u001b[1;32m   1801\u001b[0m             \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munits\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[1;32m     90\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/base_layer.py\u001b[0m in \u001b[0;36madd_weight\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, trainable, constraint)\u001b[0m\n\u001b[1;32m    247\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m             \u001b[0mdtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloatx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 249\u001b[0;31m         weight = K.variable(initializer(shape),\n\u001b[0m\u001b[1;32m    250\u001b[0m                             \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    251\u001b[0m                             \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/initializers.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, shape, dtype)\u001b[0m\n\u001b[1;32m    207\u001b[0m             \u001b[0mscale\u001b[0m \u001b[0;34m/=\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1.\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfan_out\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 209\u001b[0;31m             \u001b[0mscale\u001b[0m \u001b[0;34m/=\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1.\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfan_in\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mfan_out\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    210\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdistribution\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'normal'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    211\u001b[0m             \u001b[0;31m# 0.879... = scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +: 'NoneType' and 'int'"
     ]
    }
   ],
   "source": [
    "from keras.layers import Input, Embedding, LSTM, Dense, Lambda\n",
    "\n",
    "# Headline input: meant to receive sequences of 100 integers, between 1 and 10000.\n",
    "# Note that we can name any layer by passing it a \"name\" argument.\n",
    "main_input = Input(shape=(maxlen,), dtype=tf.string, name='main_input')\n",
    "\n",
    "#input_text = layers.Input(shape=(InputSize,300,), dtype=tf.string)\n",
    "\n",
    "#ElmoEmbedding(input_text)\n",
    "\n",
    "# This embedding layer will encode the input sequence\n",
    "# into a sequence of dense 512-dimensional vectors.\n",
    "#x = Embedding(max_features,embedding_size,input_length=maxlen)(main_input)\n",
    "x = Lambda(ElmoEmbedding, output_shape=(embedding_size,),name='Embedding_1')(main_input)\n",
    "\n",
    "\n",
    "# A LSTM will transform the vector sequence into a single vector,\n",
    "# containing information about the entire sequence\n",
    "lstm_out = LSTM(hidden_size, return_sequences=True)(x)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(None), Dimension(63)])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main_input.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(None), Dimension(63), Dimension(300)])"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hidden_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ElmoEmbedding(x):\n",
    "    #return elmo_model(tf.squeeze(tf.cast(x, tf.string)), signature=\"default\", as_dict=True)[\"default\"]\n",
    "    return elmo_model(tf.cast(x, tf.string), signature=\"default\", as_dict=True)[\"default\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(None), Dimension(63), Dimension(20), Dimension(22), Dimension(23)])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_text.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(None), Dimension(1024)])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Can't convert 'text': Shape TensorShape([Dimension(None), Dimension(63)]) is incompatible with TensorShape([Dimension(None)])",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-55-8b5d71e0a8fd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;31m#embedding = layers.Lambda(ElmoEmbedding, output_shape=(embedding_size,),name='Embedding_1')(input_text)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m \u001b[0membedding\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLambda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mElmoEmbedding\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membedding_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Embedding_1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;31m#None, 63, 300\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, **kwargs)\u001b[0m\n\u001b[1;32m    455\u001b[0m             \u001b[0;31m# Actually call the layer,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m             \u001b[0;31m# collecting output(s), mask(s), and shape(s).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 457\u001b[0;31m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    458\u001b[0m             \u001b[0moutput_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprevious_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    459\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/keras/layers/core.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, mask)\u001b[0m\n\u001b[1;32m    680\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mhas_arg\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'mask'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    681\u001b[0m             \u001b[0marguments\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'mask'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 682\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0marguments\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    683\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    684\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcompute_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-51-8331784a15c8>\u001b[0m in \u001b[0;36mElmoEmbedding\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mElmoEmbedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0;31m#return elmo_model(tf.squeeze(tf.cast(x, tf.string)), signature=\"default\", as_dict=True)[\"default\"]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0melmo_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msignature\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"default\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mas_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"default\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow_hub/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, _sentinel, signature, as_dict)\u001b[0m\n\u001b[1;32m    196\u001b[0m     dict_inputs = _prepare_dict_inputs(\n\u001b[1;32m    197\u001b[0m         inputs, self._spec.get_input_info_dict(signature=signature,\n\u001b[0;32m--> 198\u001b[0;31m                                                tags=self._tags))\n\u001b[0m\u001b[1;32m    199\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m     dict_outputs = self._impl.create_apply_graph(\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow_hub/module.py\u001b[0m in \u001b[0;36m_prepare_dict_inputs\u001b[0;34m(inputs, tensor_info_map)\u001b[0m\n\u001b[1;32m    358\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Signature expects multiple inputs. Use a dict.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    359\u001b[0m   \u001b[0;31m# Finally convert a dict of values into a dict of tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 360\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mtensor_info\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_compatible_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdict_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_info_map\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    361\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    362\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow_hub/tensor_info.py\u001b[0m in \u001b[0;36mmake_compatible_dict\u001b[0;34m(values, targets)\u001b[0m\n\u001b[1;32m    170\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    171\u001b[0m     result[key] = _convert_to_compatible_tensor(\n\u001b[0;32m--> 172\u001b[0;31m         value, targets[key], error_prefix=\"Can't convert %r\" % key)\n\u001b[0m\u001b[1;32m    173\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    174\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/tensorflow_hub/tensor_info.py\u001b[0m in \u001b[0;36m_convert_to_compatible_tensor\u001b[0;34m(value, target, error_prefix)\u001b[0m\n\u001b[1;32m    125\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m     raise TypeError(\"%s: Shape %r is incompatible with %r\" %\n\u001b[0;32m--> 127\u001b[0;31m                     (error_prefix, tensor.get_shape(), target.get_shape()))\n\u001b[0m\u001b[1;32m    128\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mtensor\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: Can't convert 'text': Shape TensorShape([Dimension(None), Dimension(63)]) is incompatible with TensorShape([Dimension(None)])"
     ]
    }
   ],
   "source": [
    "InputSize = maxlen\n",
    "batch_size = 32\n",
    "input_text = layers.Input(shape=(InputSize,), dtype=tf.string)\n",
    "#input_text = layers.Input(shape=(1,), dtype=tf.string)\n",
    "#wordInputs = Input(shape=(MAX_SENTS+1, MAX_WORDS), name=\"wordInputs\", dtype='float32')\n",
    "#wordInp = Flatten()(wordInputs)\n",
    "#wordEmbedding = Embedding(MAX_NB_WORDS, EMBEDDING_DIM, weights=[embedding_matrix], mask_zero=False, trainable=True, name='wordEmbedding')(wordInp) \n",
    "\n",
    "# l1 = Input(shape = (1000,), dtype='int32', name = 'l1')\n",
    "# e1 = Embedding(output_dim=60,input_dim=1000,input_length=1000) (l1)\n",
    "\n",
    "#input_text = layers.Input(shape=(InputSize,max_features), dtype=tf.string)\n",
    "\n",
    "#embedding = layers.Lambda(ElmoEmbedding, output_shape=(embedding_size,),name='Embedding_1')(input_text)\n",
    "embedding = layers.Lambda(ElmoEmbedding, output_shape=(embedding_size,),name='Embedding_1')(input_text)\n",
    "#None, 63, 300\n",
    "\n",
    "\n",
    "\n",
    "lstm_ = LSTM(hidden_size,batch_size=batch_size, return_sequences=True)(embedding)\n",
    "#model.add(Embedding(max_features,embedding_size,input_length=maxlen,weights=[embedding_weights]))\n",
    "timed_ = layers.TimeDistributed(layers.Dense(out_size))(lstm_)\n",
    "pred = layers.Activation('softmax')(timed_)\n",
    "model = Model(inputs=[input_text], outputs=pred)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxlen\n",
    "embedding_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([Dimension(None), Dimension(1024)])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding.get_shape()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
     ]
    }
   ],
   "source": [
    "InputSize = maxlen\n",
    "#model.add(Embedding(max_features, embedding_size, input_length=maxlen, mask_zero=True))\n",
    "input_text = layers.Input(shape=(InputSize,max_features), dtype=tf.string)\n",
    "embedding = layers.Lambda(ElmoEmbedding, output_shape=(embedding_size,))(input_text)\n",
    "#dense = layers.TimeDistributed(layers.Dense(out_size))(embedding)\n",
    "#pred = layers.Activation('softmax')(dense)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saver not created because there are no variables in the graph to restore\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "input_text = layers.Input(shape=(1,), dtype=tf.string)\n",
    "embedding = layers.Lambda(ElmoEmbedding, output_shape=(1024,))(input_text)\n",
    "\n",
    "\n",
    "#model.add(Embedding(max_features, embedding_size, input_length=maxlen, mask_zero=True))\n",
    "#model.add(Embedding(max_features,embedding_size,input_length=maxlen,weights=[embedding_weights]))\n",
    "\n",
    "#model.add(TimeDistributed(Dense(out_size)))\n",
    "dense = layers.TimeDistributed(Dense(out_size))\n",
    "\n",
    "#dense = layers.Dense(256, activation='relu')(embedding)\n",
    "#pred = layers.Dense(1, activation='sigmoid')(dense)\n",
    "##pred = layers.Activation('softmax')(dense)\n",
    "\n",
    "##model = Model(inputs=[input_text], outputs=pred)\n",
    "\n",
    "##model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "##model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(max_features, embedding_size, input_length=maxlen, mask_zero=True))\n",
    "model.add(Embedding(max_features,embedding_size,input_length=maxlen,weights=[embedding_weights]))\n",
    "model.add(LSTM(hidden_size, return_sequences=True))  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 63, 300)           2485500   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 63, 32)            42624     \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 63, 6)             198       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 63, 6)             0         \n",
      "=================================================================\n",
      "Total params: 2,528,322\n",
      "Trainable params: 2,528,322\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#model.add(TimeDistributedDense(out_size))\n",
    "model.add(TimeDistributed(Dense(out_size)))\n",
    "\n",
    "model.add(Activation('softmax'))\n",
    " \n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:2: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1440 samples, validate on 352 samples\n",
      "Epoch 1/10\n",
      "1440/1440 [==============================] - 4s 3ms/step - loss: 0.6218 - val_loss: 0.2713\n",
      "Epoch 2/10\n",
      "1440/1440 [==============================] - 3s 2ms/step - loss: 0.2324 - val_loss: 0.2049\n",
      "Epoch 3/10\n",
      "1440/1440 [==============================] - 4s 2ms/step - loss: 0.1810 - val_loss: 0.1683\n",
      "Epoch 4/10\n",
      "1440/1440 [==============================] - 3s 2ms/step - loss: 0.1438 - val_loss: 0.1407\n",
      "Epoch 5/10\n",
      "1440/1440 [==============================] - 3s 2ms/step - loss: 0.1134 - val_loss: 0.1195\n",
      "Epoch 6/10\n",
      "1440/1440 [==============================] - 3s 2ms/step - loss: 0.0869 - val_loss: 0.1022\n",
      "Epoch 7/10\n",
      "1440/1440 [==============================] - 3s 2ms/step - loss: 0.0650 - val_loss: 0.0901\n",
      "Epoch 8/10\n",
      "1440/1440 [==============================] - 3s 2ms/step - loss: 0.0476 - val_loss: 0.0816\n",
      "Epoch 9/10\n",
      "1440/1440 [==============================] - 3s 2ms/step - loss: 0.0348 - val_loss: 0.0773\n",
      "Epoch 10/10\n",
      "1440/1440 [==============================] - 4s 2ms/step - loss: 0.0262 - val_loss: 0.0753\n",
      "352/352 [==============================] - 0s 435us/step\n",
      "Raw test score: 0.07525161281228065\n"
     ]
    }
   ],
   "source": [
    "batch_size = 32\n",
    "model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=10, validation_data=(X_test, y_test))\n",
    "score = model.evaluate(X_test, y_test, batch_size=batch_size)\n",
    "print('Raw test score:', score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 0.9889061235500405\n",
      "Training confusion matrix:\n",
      "[[    0     0     0     0     0     0]\n",
      " [    1  1043     1    32     5    34]\n",
      " [    0     0  1205    16     1     4]\n",
      " [    0     7     1 24701    23    20]\n",
      " [    2     9     7    67   920    15]\n",
      " [    0    15     4    52    13  1458]]\n",
      "Testing accuracy: 0.9320243737305348\n",
      "Testing confusion matrix:\n",
      "[[   0    0    0    0    0    0]\n",
      " [   1  164    5   55   11   32]\n",
      " [   0    1  214   49    3   16]\n",
      " [   0    1    5 6162   17   19]\n",
      " [   0   10    2  119  115   27]\n",
      " [   0   10   15   91   13  228]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/user/anaconda3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([0.        , 0.88172043, 0.8879668 , 0.95151328, 0.72327044,\n",
       "        0.70807453]),\n",
       " array([0.        , 0.6119403 , 0.75618375, 0.99323017, 0.42124542,\n",
       "        0.63865546]),\n",
       " array([0.        , 0.72246696, 0.81679389, 0.97192429, 0.53240741,\n",
       "        0.67157585]),\n",
       " array([   0,  268,  283, 6204,  273,  357]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def score(yh, pr):\n",
    "    coords = [np.where(yhh > 0)[0][0] for yhh in yh]\n",
    "    yh = [yhh[co:] for yhh, co in zip(yh, coords)]\n",
    "    ypr = [prr[co:] for prr, co in zip(pr, coords)]\n",
    "    fyh = [c for row in yh for c in row]\n",
    "    fpr = [c for row in ypr for c in row]\n",
    "    return fyh, fpr\n",
    " \n",
    "pr = model.predict_classes(X_train)\n",
    "yh = y_train.argmax(2)\n",
    "fyh, fpr = score(yh, pr)\n",
    "print('Training accuracy:', accuracy_score(fyh, fpr))\n",
    "print('Training confusion matrix:')\n",
    "print(confusion_matrix(fyh, fpr))\n",
    "precision_recall_fscore_support(fyh, fpr)\n",
    " \n",
    "pr = model.predict_classes(X_test)\n",
    "yh = y_test.argmax(2)\n",
    "fyh, fpr = score(yh, pr)\n",
    "print('Testing accuracy:', accuracy_score(fyh, fpr))\n",
    "print('Testing confusion matrix:')\n",
    "print(confusion_matrix(fyh, fpr))\n",
    "precision_recall_fscore_support(fyh, fpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['John', 'was', 'a', 'member', 'of', 'US', 'Army']\n",
      "['I-PER', 'O', 'O', 'O', 'O', 'I-ORG', 'I-ORG']\n"
     ]
    }
   ],
   "source": [
    "from keras import backend as K\n",
    "string = \"John was a member of US Army\"\n",
    "wordlist = string.split(' ')\n",
    "ip = []\n",
    "for x in wordlist:\n",
    "    ip.append(word2ind[x])\n",
    "i=maxlen-len(ip)\n",
    "temp=[0]*i\n",
    "ip=temp+ip\n",
    "input_layer = model.layers[0].input\n",
    "output_layer = model.layers[3].output\n",
    "op = K.function([input_layer], [output_layer])\n",
    "out = op([[ip]])\n",
    "temp = []\n",
    "while i<maxlen:\n",
    "    for j in label2ind:\n",
    "        #print(out[0][i].tolist())\n",
    "        if label2ind[j]==out[0][0][i].tolist().index(max(out[0][0][i])):\n",
    "            temp.append(j)\n",
    "    i=i+1\n",
    "print(wordlist)\n",
    "print(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}