text-similarity/1.birnn-contrastive.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import collections\n",
    "from unidecode import unidecode\n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    count.extend(collections.Counter(words).most_common(n_words - 1))\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary\n",
    "\n",
    "def str_idx(corpus, dic, maxlen, UNK=3):\n",
    "    X = np.zeros((len(corpus),maxlen))\n",
    "    for i in range(len(corpus)):\n",
    "        for no, k in enumerate(corpus[i][:maxlen][::-1]):\n",
    "            val = dic[k] if k in dic else UNK\n",
    "            X[i,-1 - no]= val\n",
    "    return X\n",
    "\n",
    "def cleaning(string):\n",
    "    string = unidecode(string).replace('.', ' . ').replace(',', ' , ')\n",
    "    string = re.sub('[^A-Za-z\\- ]+', ' ', string)\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    return string.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('quora_duplicate_questions.tsv', delimiter='\\t').dropna()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "left, right, label = df['question1'].tolist(), df['question2'].tolist(), df['is_duplicate'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([255024, 149263]))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(label, return_counts = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 404287/404287 [00:07<00:00, 54874.65it/s]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(len(left))):\n",
    "    left[i] = cleaning(left[i])\n",
    "    right[i] = cleaning(right[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 87661\n",
      "Most common words [('the', 377593), ('what', 324635), ('is', 269934), ('i', 223893), ('how', 220876), ('a', 212757)]\n",
      "Sample data [5, 6, 4, 1285, 62, 1285, 2501, 10, 564, 11] ['what', 'is', 'the', 'step', 'by', 'step', 'guide', 'to', 'invest', 'in']\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(left + right).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 dict_size, learning_rate, dropout):\n",
    "        \n",
    "        def cells(size, reuse=False):\n",
    "            cell = tf.nn.rnn_cell.LSTMCell(size,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "            return tf.contrib.rnn.DropoutWrapper(cell,output_keep_prob=dropout)\n",
    "        \n",
    "        def birnn(inputs, scope):\n",
    "            with tf.variable_scope(scope, reuse = tf.AUTO_REUSE):\n",
    "                for n in range(num_layers):\n",
    "                    (out_fw, out_bw), (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                        cell_fw = cells(size_layer // 2),\n",
    "                        cell_bw = cells(size_layer // 2),\n",
    "                        inputs = inputs,\n",
    "                        dtype = tf.float32,\n",
    "                        scope = 'bidirectional_rnn_%d'%(n))\n",
    "                    inputs = tf.concat((out_fw, out_bw), 2)\n",
    "                return inputs[:,-1]\n",
    "        \n",
    "        self.X_left = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_right = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.float32, [None])\n",
    "        self.batch_size = tf.shape(self.X_left)[0]\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        embedded_left = tf.nn.embedding_lookup(encoder_embeddings, self.X_left)\n",
    "        embedded_right = tf.nn.embedding_lookup(encoder_embeddings, self.X_right)\n",
    "        \n",
    "        def contrastive_loss(y,d):\n",
    "            tmp= y * tf.square(d)\n",
    "            tmp2 = (1-y) * tf.square(tf.maximum((1 - d),0))\n",
    "            return tf.reduce_sum(tmp +tmp2)/tf.cast(self.batch_size,tf.float32)/2\n",
    "        \n",
    "        self.output_left = birnn(embedded_left, 'left')\n",
    "        self.output_right = birnn(embedded_right, 'right')\n",
    "        self.distance = tf.sqrt(tf.reduce_sum(tf.square(tf.subtract(self.output_left,self.output_right)),\n",
    "                                              1,keep_dims=True))\n",
    "        self.distance = tf.div(self.distance, tf.add(tf.sqrt(tf.reduce_sum(tf.square(self.output_left),\n",
    "                                                                           1,keep_dims=True)),\n",
    "                                                     tf.sqrt(tf.reduce_sum(tf.square(self.output_right),\n",
    "                                                                           1,keep_dims=True))))\n",
    "        self.distance = tf.reshape(self.distance, [-1])\n",
    "        self.cost = contrastive_loss(self.Y,self.distance)\n",
    "        \n",
    "        self.temp_sim = tf.subtract(tf.ones_like(self.distance),\n",
    "                                    tf.rint(self.distance))\n",
    "        correct_predictions = tf.equal(self.temp_sim, self.Y)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, \"float\"))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 256\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "learning_rate = 1e-3\n",
    "maxlen = 50\n",
    "batch_size = 128\n",
    "dropout = 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "vectors_left = str_idx(left, dictionary, maxlen)\n",
    "vectors_right = str_idx(right, dictionary, maxlen)\n",
    "train_X_left, test_X_left, train_X_right, test_X_right, train_Y, test_Y = train_test_split(vectors_left,\n",
    "                                                                                           vectors_right,\n",
    "                                                                                           label,\n",
    "                                                                                           test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From <ipython-input-9-679a746d3c87>:6: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-9-679a746d3c87>:17: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn.py:443: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/rnn_cell_impl.py:1259: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From <ipython-input-9-679a746d3c87>:37: calling reduce_sum_v1 (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n",
      "WARNING:tensorflow:From <ipython-input-9-679a746d3c87>:41: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,num_layers,embedded_size,len(dictionary),learning_rate,dropout)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:54<00:00,  3.32it/s, accuracy=0.762, cost=0.0892]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:30<00:00,  7.02it/s, accuracy=0.611, cost=0.114] \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.721205\n",
      "time taken: 865.0403523445129\n",
      "epoch: 0, training loss: 0.102127, training acc: 0.692444, valid loss: 0.095351, valid acc: 0.721205\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:50<00:00,  3.28it/s, accuracy=0.733, cost=0.0808]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:31<00:00,  6.93it/s, accuracy=0.644, cost=0.106] \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.721205, current acc: 0.743804\n",
      "time taken: 861.1270344257355\n",
      "epoch: 0, training loss: 0.092396, training acc: 0.733870, valid loss: 0.089960, valid acc: 0.743804\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:51<00:00,  3.32it/s, accuracy=0.802, cost=0.0735]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:30<00:00,  6.93it/s, accuracy=0.644, cost=0.105] \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.743804, current acc: 0.754440\n",
      "time taken: 861.8881492614746\n",
      "epoch: 0, training loss: 0.088065, training acc: 0.751199, valid loss: 0.087837, valid acc: 0.754440\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:50<00:00,  3.27it/s, accuracy=0.842, cost=0.0697]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:30<00:00,  7.03it/s, accuracy=0.667, cost=0.104] \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.754440, current acc: 0.757023\n",
      "time taken: 861.7144618034363\n",
      "epoch: 0, training loss: 0.085004, training acc: 0.764099, valid loss: 0.086727, valid acc: 0.757023\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:51<00:00,  3.22it/s, accuracy=0.812, cost=0.0724]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:31<00:00,  6.93it/s, accuracy=0.633, cost=0.11]  \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.757023, current acc: 0.760754\n",
      "time taken: 862.4683222770691\n",
      "epoch: 0, training loss: 0.082544, training acc: 0.773236, valid loss: 0.085892, valid acc: 0.760754\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:50<00:00,  3.32it/s, accuracy=0.782, cost=0.0759]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:30<00:00,  6.95it/s, accuracy=0.656, cost=0.108] \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 861.5845947265625\n",
      "epoch: 0, training loss: 0.080261, training acc: 0.781377, valid loss: 0.086369, valid acc: 0.757438\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:50<00:00,  3.31it/s, accuracy=0.832, cost=0.0661]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:31<00:00,  6.97it/s, accuracy=0.656, cost=0.102] \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.760754, current acc: 0.763077\n",
      "time taken: 861.8206684589386\n",
      "epoch: 0, training loss: 0.078398, training acc: 0.788314, valid loss: 0.084990, valid acc: 0.763077\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:50<00:00,  3.29it/s, accuracy=0.842, cost=0.0661]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:30<00:00,  6.96it/s, accuracy=0.656, cost=0.103] \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 861.3552474975586\n",
      "epoch: 0, training loss: 0.076674, training acc: 0.795231, valid loss: 0.085479, valid acc: 0.759256\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:50<00:00,  3.23it/s, accuracy=0.851, cost=0.0647]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:31<00:00,  6.93it/s, accuracy=0.656, cost=0.1]   \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.763077, current acc: 0.763510\n",
      "time taken: 861.3164525032043\n",
      "epoch: 0, training loss: 0.075192, training acc: 0.800024, valid loss: 0.084781, valid acc: 0.763510\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:49<00:00,  3.28it/s, accuracy=0.822, cost=0.0684]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:30<00:00,  6.92it/s, accuracy=0.667, cost=0.107] \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.763510, current acc: 0.765012\n",
      "time taken: 860.8371865749359\n",
      "epoch: 0, training loss: 0.073777, training acc: 0.805469, valid loss: 0.084846, valid acc: 0.765012\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:51<00:00,  3.27it/s, accuracy=0.842, cost=0.0651]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:31<00:00,  6.90it/s, accuracy=0.644, cost=0.104] \n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 862.1494925022125\n",
      "epoch: 0, training loss: 0.072904, training acc: 0.808442, valid loss: 0.084983, valid acc: 0.762664\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:50<00:00,  3.29it/s, accuracy=0.802, cost=0.0664]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:31<00:00,  6.96it/s, accuracy=0.678, cost=0.0966]\n",
      "train minibatch loop:   0%|          | 0/2527 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 861.4347906112671\n",
      "epoch: 0, training loss: 0.072015, training acc: 0.811395, valid loss: 0.084607, valid acc: 0.763842\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [12:48<00:00,  3.33it/s, accuracy=0.851, cost=0.0605]\n",
      "test minibatch loop: 100%|██████████| 632/632 [01:31<00:00,  6.98it/s, accuracy=0.667, cost=0.0982]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 859.5523777008057\n",
      "epoch: 0, training loss: 0.070824, training acc: 0.816009, valid loss: 0.085312, valid acc: 0.761277\n",
      "\n",
      "break epoch:0\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 3, 0, 0, 0\n",
    "\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n' % (EPOCH))\n",
    "        break\n",
    "\n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    pbar = tqdm(range(0, len(train_X_left), batch_size), desc='train minibatch loop')\n",
    "    for i in pbar:\n",
    "        batch_x_left = train_X_left[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        batch_x_right = train_X_right[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        batch_y = train_Y[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X_left : batch_x_left, \n",
    "                                        model.X_right: batch_x_right,\n",
    "                                        model.Y : batch_y})\n",
    "        assert not np.isnan(loss)\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(range(0, len(test_X_left), batch_size), desc='test minibatch loop')\n",
    "    for i in pbar:\n",
    "        batch_x_left = test_X_left[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        batch_x_right = test_X_right[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        batch_y = test_Y[i:min(i+batch_size,train_X_left.shape[0])]\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X_left : batch_x_left, \n",
    "                                        model.X_right: batch_x_right,\n",
    "                                        model.Y : batch_y})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "    \n",
    "    train_loss /= (len(train_X_left) / batch_size)\n",
    "    train_acc /= (len(train_X_left) / batch_size)\n",
    "    test_loss /= (len(test_X_left) / batch_size)\n",
    "    test_acc /= (len(test_X_left) / batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print(\n",
    "            'epoch: %d, pass acc: %f, current acc: %f'\n",
    "            % (EPOCH, CURRENT_ACC, test_acc)\n",
    "        )\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "    \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0.], dtype=float32), array([0.13218915], dtype=float32)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "left = str_idx(['a person is outdoors, on a horse.'], dictionary, maxlen)\n",
    "right = str_idx(['a person on a horse jumps over a broken down airplane.'], dictionary, maxlen)\n",
    "sess.run([model.temp_sim,1-model.distance], feed_dict = {model.X_left : left, \n",
    "                                        model.X_right: right})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}