abstractive-summarization/9.dilated-seq2seq-self-attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import collections\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from tensor2tensor.utils import beam_search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ctexts.json','r') as fopen:\n",
    "    ctexts = json.load(fopen)\n",
    "    \n",
    "with open('headlines.json','r') as fopen:\n",
    "    headlines = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "import random\n",
    "\n",
    "max_len = 500\n",
    "\n",
    "def textcleaning(string):\n",
    "    string = unidecode(string).replace('.', '. ').replace(',', ' , ')\n",
    "    string = re.sub('[^\\'\"A-Za-z\\- ]+', ' ', string)\n",
    "    string = re.sub(r'[ ]+', ' ', string.lower()).strip().split()\n",
    "    if len(string) > max_len:\n",
    "        string = random.sample(string, max_len)\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 872 ms, sys: 8 ms, total: 880 ms\n",
      "Wall time: 881 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "h, c = [], []\n",
    "for i in range(len(ctexts)):\n",
    "    try:\n",
    "        c.append(textcleaning(ctexts[i]))\n",
    "        h.append(textcleaning(headlines[i]))\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    count.extend(collections.Counter(words).most_common(n_words))\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 45162\n",
      "Most common words [('the', 82065), ('dot', 62590), ('comma', 61024), ('and', 34489), ('to', 33637), ('of', 31862)]\n",
      "Sample data [4, 10027, 7, 11772, 636, 12, 448, 7084, 11, 3514] ['the', 'daman', 'and', 'diu', 'administration', 'on', 'wednesday', 'withdrew', 'a', 'circular']\n",
      "filtered vocab size: 45166\n",
      "% of vocab used: 100.01%\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(c).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])\n",
    "print('filtered vocab size:',len(dictionary))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary)/vocabulary_size,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'daman and diu revokes mandatory rakshabandhan in offices order EOS'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(len(h)):\n",
    "    h[i] = h[i] + ' EOS'\n",
    "h[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic, UNK=3):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i.split():\n",
    "            ints.append(dic.get(k, UNK))\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(c, dictionary)\n",
    "Y = str_idx(h, dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embed_seq(x, vocab_sz, embed_dim, name, zero_pad=True): \n",
    "    embedding = tf.get_variable(name, [vocab_sz, embed_dim]) \n",
    "    if zero_pad:\n",
    "        embedding = tf.concat([tf.zeros([1, embed_dim]), embedding[1:, :]], 0) \n",
    "    x = tf.nn.embedding_lookup(embedding, x)\n",
    "    return x\n",
    "\n",
    "def position_encoding(inputs):\n",
    "    T = tf.shape(inputs)[1]\n",
    "    repr_dim = inputs.get_shape()[-1].value\n",
    "    pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1])\n",
    "    i = np.arange(0, repr_dim, 2, np.float32)\n",
    "    denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1])\n",
    "    enc = tf.expand_dims(tf.concat([tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0)\n",
    "    return tf.tile(enc, [tf.shape(inputs)[0], 1, 1])\n",
    "\n",
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    return gamma * normalized + beta\n",
    "\n",
    "\n",
    "def cnn_block(x, dilation_rate, pad_sz, hidden_dim, kernel_size):\n",
    "    x = layer_norm(x)\n",
    "    pad = tf.zeros([tf.shape(x)[0], pad_sz, hidden_dim])\n",
    "    x =  tf.layers.conv1d(inputs = tf.concat([pad, x, pad], 1),\n",
    "                          filters = hidden_dim,\n",
    "                          kernel_size = kernel_size,\n",
    "                          dilation_rate = dilation_rate)\n",
    "    x = x[:, :-pad_sz, :]\n",
    "    x = tf.nn.relu(x)\n",
    "    return x\n",
    "\n",
    "def Attention(Q, inputs, num_units, num_heads = 8, activation = None):\n",
    "    inputs = tf.layers.dropout(inputs, 0.1, training=True)\n",
    "    T_q = tf.shape(Q)[1]\n",
    "    T_k = tf.shape(inputs)[1]\n",
    "    K_V = tf.layers.dense(inputs, 2*num_units, activation)\n",
    "    K, V = tf.split(K_V, 2, -1)\n",
    "    Q_ = tf.concat(tf.split(Q, num_heads, axis=2), 0)\n",
    "    K_ = tf.concat(tf.split(K, num_heads, axis=2), 0)\n",
    "    V_ = tf.concat(tf.split(V, num_heads, axis=2), 0)\n",
    "    p_gen = tf.layers.dense(K * V, 1)\n",
    "    p_gen = tf.sigmoid(p_gen)\n",
    "    align = tf.matmul(Q_, K_, transpose_b=True)\n",
    "    align *= tf.rsqrt(tf.to_float(K_.get_shape()[-1].value))\n",
    "    paddings = tf.fill(tf.shape(align), float('-inf'))\n",
    "    lower_tri = tf.ones([T_q, T_k])\n",
    "    lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()\n",
    "    masks = tf.tile(tf.expand_dims(lower_tri,0), [tf.shape(align)[0],1,1])\n",
    "    align = tf.where(tf.equal(masks, 0), paddings, align)\n",
    "    align = tf.nn.softmax(align)\n",
    "    alignments = tf.transpose(align, [0, 2, 1]) \n",
    "    x = tf.matmul(align, V_)\n",
    "    x = tf.concat(tf.split(x, num_heads, axis=0), 2)\n",
    "    x += Q\n",
    "    x = layer_norm(x)\n",
    "    return x, alignments, p_gen\n",
    "\n",
    "class Summarization:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, \n",
    "                 dict_size, learning_rate, \n",
    "                 kernel_size = 2, n_attn_heads = 16):\n",
    "\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype = tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype = tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        self.batch_size = batch_size\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        self.embedding = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        \n",
    "        self.num_layers = num_layers\n",
    "        self.kernel_size = kernel_size\n",
    "        self.size_layer = size_layer\n",
    "        self.n_attn_heads = n_attn_heads\n",
    "        self.dict_size = dict_size\n",
    "        \n",
    "        self.training_logits = self.forward(self.X, decoder_input)\n",
    "\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "    def forward(self, x, y, reuse = False):\n",
    "        with tf.variable_scope('forward',reuse=reuse):\n",
    "            with tf.variable_scope('forward',reuse=reuse):\n",
    "                encoder_embedded = tf.nn.embedding_lookup(self.embedding, x)\n",
    "                decoder_embedded = tf.nn.embedding_lookup(self.embedding, y)\n",
    "                encoder_embedded += position_encoding(encoder_embedded)\n",
    "\n",
    "                for i in range(self.num_layers): \n",
    "                    dilation_rate = 2 ** i\n",
    "                    pad_sz = (self.kernel_size - 1) * dilation_rate \n",
    "                    with tf.variable_scope('block_%d'%i,reuse=reuse):\n",
    "                        encoder_embedded += cnn_block(encoder_embedded, dilation_rate, \n",
    "                                                      pad_sz, self.size_layer, self.kernel_size)\n",
    "\n",
    "                g = tf.identity(decoder_embedded)\n",
    "                for i in range(self.num_layers):\n",
    "                    dilation_rate = 2 ** i\n",
    "                    pad_sz = (self.kernel_size - 1) * dilation_rate\n",
    "                    with tf.variable_scope('decode_%d'%i,reuse=reuse):\n",
    "                        attn_res = h = cnn_block(decoder_embedded, dilation_rate, \n",
    "                                                 pad_sz, self.size_layer, self.kernel_size)\n",
    "                        with tf.variable_scope('attention_%d'%i,reuse=reuse):\n",
    "                            h, alignment, p_gen = Attention(attn_res, encoder_embedded, self.size_layer)\n",
    "                        \n",
    "                        vocab_dist = tf.nn.softmax(encoder_embedded) * p_gen\n",
    "                        alignments = alignment[0] * (1 - p_gen)\n",
    "                        encoder_embedded += vocab_dist\n",
    "                        decoder_embedded += h\n",
    "\n",
    "                return tf.layers.dense(decoder_embedded, self.dict_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 4\n",
    "embedded_size = 128\n",
    "learning_rate = 1e-3\n",
    "batch_size = 8\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def beam_search_decoding(length = 20, beam_width = 5):\n",
    "    initial_ids = tf.fill([model.batch_size], GO)\n",
    "    \n",
    "    def symbols_to_logits(ids):\n",
    "        x = tf.contrib.seq2seq.tile_batch(model.X, beam_width)\n",
    "        logits = model.forward(x, ids, reuse = True)\n",
    "        return logits[:, tf.shape(ids)[1]-1, :]\n",
    "\n",
    "    final_ids, final_probs = beam_search.beam_search(\n",
    "        symbols_to_logits,\n",
    "        initial_ids,\n",
    "        beam_width,\n",
    "        length,\n",
    "        len(dictionary),\n",
    "        0.0,\n",
    "        eos_id = EOS)\n",
    "    \n",
    "    return final_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Summarization(size_layer, num_layers, embedded_size, \n",
    "                      len(dictionary), learning_rate)\n",
    "model.generate = beam_search_decoding()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:45<00:00,  8.59it/s, accuracy=0.171, cost=7.02] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:03<00:00, 35.69it/s, accuracy=0.101, cost=7.02]\n",
      "train minibatch loop:   0%|          | 2/440 [00:00<00:25, 16.97it/s, accuracy=0.107, cost=6.77]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, avg loss: 7.576314, avg accuracy: 0.114468\n",
      "epoch: 0, avg loss test: 7.016410, avg accuracy test: 0.122362\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:28<00:00,  5.94it/s, accuracy=0.122, cost=7.11] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:02<00:00, 39.22it/s, accuracy=0.118, cost=7.11]\n",
      "train minibatch loop:   0%|          | 2/440 [00:00<00:26, 16.51it/s, accuracy=0.133, cost=5.85]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 6.328432, avg accuracy: 0.138315\n",
      "epoch: 1, avg loss test: 7.112424, avg accuracy test: 0.136168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:27<00:00, 10.62it/s, accuracy=0.19, cost=5.94]  \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:02<00:00, 37.58it/s, accuracy=0.165, cost=5.94]\n",
      "train minibatch loop:   0%|          | 2/440 [00:00<00:28, 15.53it/s, accuracy=0.279, cost=3.56]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, avg loss: 5.327238, avg accuracy: 0.166704\n",
      "epoch: 2, avg loss test: 5.940526, avg accuracy test: 0.146419\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:28<00:00,  8.47it/s, accuracy=0.196, cost=4.56]\n",
      "test minibatch loop: 100%|██████████| 110/110 [00:02<00:00, 39.07it/s, accuracy=0.189, cost=4.56]\n",
      "train minibatch loop:   0%|          | 2/440 [00:00<00:26, 16.70it/s, accuracy=0.516, cost=2.6] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, avg loss: 4.019179, avg accuracy: 0.263450\n",
      "epoch: 3, avg loss test: 4.555007, avg accuracy test: 0.142204\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:27<00:00, 17.40it/s, accuracy=0.289, cost=3.8] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:02<00:00, 37.34it/s, accuracy=0.125, cost=3.8] \n",
      "train minibatch loop:   0%|          | 2/440 [00:00<00:26, 16.73it/s, accuracy=0.639, cost=1.73]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, avg loss: 2.797662, avg accuracy: 0.434791\n",
      "epoch: 4, avg loss test: 3.802958, avg accuracy test: 0.134433\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:26<00:00, 17.24it/s, accuracy=0.31, cost=3.17] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:02<00:00, 39.56it/s, accuracy=0.0989, cost=3.17]\n",
      "train minibatch loop:   0%|          | 2/440 [00:00<00:26, 16.43it/s, accuracy=0.667, cost=1.26]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, avg loss: 1.973520, avg accuracy: 0.575891\n",
      "epoch: 5, avg loss test: 3.170563, avg accuracy test: 0.118116\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:27<00:00,  7.52it/s, accuracy=0.644, cost=1.56] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:03<00:00, 35.93it/s, accuracy=0.149, cost=1.56] \n",
      "train minibatch loop:   0%|          | 2/440 [00:00<00:26, 16.82it/s, accuracy=0.789, cost=0.815]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, avg loss: 1.424597, avg accuracy: 0.689482\n",
      "epoch: 6, avg loss test: 1.563884, avg accuracy test: 0.124567\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:26<00:00, 17.25it/s, accuracy=0.727, cost=1.29] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:02<00:00, 39.58it/s, accuracy=0.11, cost=1.29]  \n",
      "train minibatch loop:   0%|          | 2/440 [00:00<00:26, 16.38it/s, accuracy=0.849, cost=0.661]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, avg loss: 1.058539, avg accuracy: 0.770024\n",
      "epoch: 7, avg loss test: 1.294635, avg accuracy test: 0.119202\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:26<00:00, 17.33it/s, accuracy=0.87, cost=0.584] \n",
      "test minibatch loop: 100%|██████████| 110/110 [00:02<00:00, 39.60it/s, accuracy=0.158, cost=0.584] \n",
      "train minibatch loop:   0%|          | 2/440 [00:00<00:25, 16.86it/s, accuracy=0.913, cost=0.502]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, avg loss: 0.819929, avg accuracy: 0.825999\n",
      "epoch: 8, avg loss test: 0.584340, avg accuracy test: 0.116095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 440/440 [00:26<00:00, 17.35it/s, accuracy=0.891, cost=0.581]\n",
      "test minibatch loop: 100%|██████████| 110/110 [00:02<00:00, 39.51it/s, accuracy=0.0682, cost=0.581]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, avg loss: 0.668940, avg accuracy: 0.862275\n",
      "epoch: 9, avg loss test: 0.581209, avg accuracy test: 0.115498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "import time\n",
    "\n",
    "for EPOCH in range(10):\n",
    "    lasttime = time.time()\n",
    "    total_loss, total_accuracy, total_loss_test, total_accuracy_test = 0, 0, 0, 0\n",
    "    train_X, train_Y = shuffle(train_X, train_Y)\n",
    "    test_X, test_Y = shuffle(test_X, test_Y)\n",
    "    pbar = tqdm(range(0, len(train_X), batch_size), desc='train minibatch loop')\n",
    "    for k in pbar:\n",
    "        batch_x, _ = pad_sentence_batch(train_X[k: min(k+batch_size,len(train_X))], PAD)\n",
    "        batch_y, _ = pad_sentence_batch(train_Y[k: min(k+batch_size,len(train_X))], PAD)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(range(0, len(test_X), batch_size), desc='test minibatch loop')\n",
    "    for k in pbar:\n",
    "        batch_x, _ = pad_sentence_batch(test_X[k: min(k+batch_size,len(test_X))], PAD)\n",
    "        batch_y, _ = pad_sentence_batch(test_Y[k: min(k+batch_size,len(test_X))], PAD)\n",
    "        acc, losslop = sess.run([model.accuracy, model.cost], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss_test += loss\n",
    "        total_accuracy_test += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "        \n",
    "    total_loss /= (len(train_X) / batch_size)\n",
    "    total_accuracy /= (len(train_X) / batch_size)\n",
    "    total_loss_test /= (len(test_X) / batch_size)\n",
    "    total_accuracy_test /= (len(test_X) / batch_size)\n",
    "        \n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(EPOCH, total_loss, total_accuracy))\n",
    "    print('epoch: %d, avg loss test: %f, avg accuracy test: %f'%(EPOCH, total_loss_test, total_accuracy_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'GO navjot killed as swift UNK tries to overtake lamborghini EOS'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated = [rev_dictionary[i] for i in sess.run(model.generate, feed_dict = {model.X: [test_X[0]]})[0,0,:]]\n",
    "' '.join(generated)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cross the border for better fireworks gambhir to separatist EOS'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join([rev_dictionary[i] for i in test_Y[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}