abstractive-summarization/5.xueyouluo-pointer-generator-bahdanau.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import collections\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from pointer_generator_helper import (PointerGeneratorDecoder, \n",
    "                                      PointerGeneratorGreedyEmbeddingHelper, \n",
    "                                      PointerGeneratorBahdanauAttention,\n",
    "                                      PointerGeneratorAttentionWrapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('ctexts.json','r') as fopen:\n",
    "    ctexts = json.load(fopen)\n",
    "    \n",
    "with open('headlines.json','r') as fopen:\n",
    "    headlines = json.load(fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "import random\n",
    "\n",
    "max_len = 500\n",
    "\n",
    "def textcleaning(string):\n",
    "    string = unidecode(string).replace('.', '. ').replace(',', ' , ')\n",
    "    string = re.sub('[^\\'\"A-Za-z\\- ]+', ' ', string)\n",
    "    string = re.sub(r'[ ]+', ' ', string.lower()).strip().split()\n",
    "    if len(string) > max_len:\n",
    "        string = random.sample(string, max_len)\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 868 ms, sys: 12 ms, total: 880 ms\n",
      "Wall time: 879 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "h, c = [], []\n",
    "for i in range(len(ctexts)):\n",
    "    try:\n",
    "        c.append(textcleaning(ctexts[i]))\n",
    "        h.append(textcleaning(headlines[i]))\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    count.extend(collections.Counter(words).most_common(n_words))\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 45213\n",
      "Most common words [('the', 81961), ('dot', 62551), ('comma', 61164), ('and', 34531), ('to', 33620), ('of', 31758)]\n",
      "Sample data [4, 10002, 7, 11765, 633, 12, 466, 7448, 11, 3433] ['the', 'daman', 'and', 'diu', 'administration', 'on', 'wednesday', 'withdrew', 'a', 'circular']\n",
      "filtered vocab size: 45217\n",
      "% of vocab used: 100.01%\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(c).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])\n",
    "print('filtered vocab size:',len(dictionary))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary)/vocabulary_size,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'daman and diu revokes mandatory rakshabandhan in offices order EOS'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for i in range(len(h)):\n",
    "    h[i] = h[i] + ' EOS'\n",
    "h[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(c, h, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sent2idx(sent, vocab, UNK=UNK):\n",
    "    tokens = sent.split()\n",
    "    oovs = []\n",
    "    extend_tokens = []\n",
    "    tokenized = []\n",
    "    for token in tokens:\n",
    "        if token not in vocab:\n",
    "            tokenized.append(UNK)\n",
    "            if token not in oovs:\n",
    "                oovs.append(token)\n",
    "            extend_tokens.append(len(vocab) + oovs.index(token))\n",
    "        else:\n",
    "            extend_tokens.append(vocab[token])\n",
    "            tokenized.append(vocab[token])\n",
    "    return tokenized, extend_tokens, oovs\n",
    "\n",
    "def target2idx(sent, oovs, vocab,UNK=UNK):\n",
    "    tokens = sent.split()\n",
    "    tokenized = []\n",
    "    for token in tokens:\n",
    "        if token not in vocab:\n",
    "            if token not in oovs:\n",
    "                tokenized.append(UNK)\n",
    "            else:\n",
    "                tokenized.append(len(vocab) + oovs.index(token))\n",
    "        else:\n",
    "            tokenized.append(vocab[token])\n",
    "    return tokenized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Summarization:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, dict_size):\n",
    "        \n",
    "        def lstm_cell(reuse=False):\n",
    "            return tf.nn.rnn_cell.GRUCell(size_layer, reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        self.source_oov_words = tf.placeholder(tf.int32, shape=[])\n",
    "        self.source_extend_tokens = tf.placeholder(tf.int32, shape=[None, None])\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        condition = tf.less(decoder_input, dict_size)\n",
    "        self.decoder_input = decoder_input\n",
    "        self.decoder_input_length = self.Y_seq_len\n",
    "        self.predict_count = tf.reduce_sum(self.decoder_input_length)\n",
    "        \n",
    "        embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        \n",
    "        encoder_embedded = tf.nn.embedding_lookup(embeddings, self.X)\n",
    "        encoder_cells = tf.nn.rnn_cell.MultiRNNCell([lstm_cell() for _ in range(num_layers)])\n",
    "        self.encoder_out, self.encoder_state = tf.nn.dynamic_rnn(cell = encoder_cells, \n",
    "                                                                 inputs = encoder_embedded, \n",
    "                                                                 sequence_length = self.X_seq_len,\n",
    "                                                                 dtype = tf.float32)\n",
    "        self.decode_initial_state = self.encoder_state[-1]\n",
    "        print(self.decode_initial_state)\n",
    "        \n",
    "        atten_mech = PointerGeneratorBahdanauAttention(\n",
    "                size_layer, self.encoder_out, memory_sequence_length=self.X_seq_len,\n",
    "        coverage = True)\n",
    "        decoder_cells = [lstm_cell() for _ in range(num_layers)]\n",
    "        decoder_cells[0] = PointerGeneratorAttentionWrapper(\n",
    "                cell=decoder_cells[0],\n",
    "                attention_mechanism=atten_mech,\n",
    "                attention_layer_size=size_layer,\n",
    "                alignment_history = True,\n",
    "                coverage = True\n",
    "            )\n",
    "        initial_state = [self.decode_initial_state for i in range(num_layers)]\n",
    "        attention_cell_state = decoder_cells[0].zero_state(\n",
    "                dtype=tf.float32, batch_size=batch_size)\n",
    "        initial_state[0] = attention_cell_state.clone(\n",
    "                cell_state=initial_state[0])\n",
    "        self.initial_state = tuple(initial_state)\n",
    "        decoder_cells = tf.contrib.rnn.MultiRNNCell(decoder_cells)\n",
    "        \n",
    "        decoded = tf.nn.embedding_lookup(embeddings, self.decoder_input)\n",
    "        \n",
    "        training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "            decoded,\n",
    "            self.decoder_input_length\n",
    "        )\n",
    "        dense_layer = tf.layers.Dense(dict_size)\n",
    "        \n",
    "        training_decoder = PointerGeneratorDecoder(\n",
    "            source_extend_tokens = self.source_extend_tokens,\n",
    "            source_oov_words = self.source_oov_words,\n",
    "            coverage = True,\n",
    "            cell=decoder_cells,\n",
    "            helper=training_helper,\n",
    "            initial_state=self.initial_state,\n",
    "            output_layer=dense_layer\n",
    "        )\n",
    "        \n",
    "        maxlen = tf.reduce_max(self.decoder_input_length)\n",
    "        train_dec_outputs, train_dec_last_state, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            training_decoder,\n",
    "            output_time_major=False,\n",
    "            impute_finished=True,\n",
    "            maximum_iterations=maxlen,\n",
    "            swap_memory=True)\n",
    "        logits = train_dec_outputs.rnn_output\n",
    "        \n",
    "        masks = tf.sequence_mask(\n",
    "            self.decoder_input_length, maxlen, \n",
    "            dtype=tf.float32)\n",
    "        \n",
    "        targets = tf.slice(self.Y, [0, 0], [-1, maxlen])\n",
    "        i1, i2 = tf.meshgrid(tf.range(batch_size),\n",
    "                     tf.range(maxlen), indexing=\"ij\")\n",
    "        indices = tf.stack((i1,i2,targets),axis=2)\n",
    "        probs = tf.gather_nd(logits, indices)\n",
    "        probs = tf.where(tf.less_equal(probs,0),tf.ones_like(probs)*1e-10,probs)\n",
    "        crossent = -tf.log(probs)\n",
    "        self.cost = tf.reduce_sum(crossent * masks) / tf.to_float(batch_size)\n",
    "        alignment_history = train_dec_last_state[0].alignment_history.stack()\n",
    "        alignment_history = tf.transpose(alignment_history,[1,2,0])\n",
    "        coverage_loss = tf.minimum(alignment_history,tf.cumsum(alignment_history, axis=2, exclusive=True))\n",
    "        self.coverage_loss = tf.reduce_sum(coverage_loss / tf.to_float(batch_size))\n",
    "        self.cost = self.cost + self.coverage_loss\n",
    "        \n",
    "        self.optimizer = tf.train.AdamOptimizer().minimize(self.cost)\n",
    "        \n",
    "        helper = PointerGeneratorGreedyEmbeddingHelper(\n",
    "            embedding=embeddings,\n",
    "            start_tokens=tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "            end_token=EOS\n",
    "        )\n",
    "        \n",
    "        inference_decoder = PointerGeneratorDecoder(\n",
    "            source_extend_tokens = self.source_extend_tokens,\n",
    "            source_oov_words = self.source_oov_words,\n",
    "            coverage = True,\n",
    "            cell=decoder_cells,\n",
    "            helper=helper,\n",
    "            initial_state=self.initial_state,\n",
    "            output_layer=dense_layer\n",
    "        )\n",
    "        \n",
    "        dec_outputs, dec_last_state, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "            inference_decoder,\n",
    "            output_time_major=False,\n",
    "            maximum_iterations=tf.reduce_max(self.X_seq_len),\n",
    "            swap_memory=True)\n",
    "        \n",
    "        self.beam_predictions = dec_outputs.sample_id\n",
    "        print(self.beam_predictions)\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        y_t = tf.argmax(logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "batch_size = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"rnn/while/Exit_4:0\", shape=(?, 128), dtype=float32)\n",
      "<pointer_generator_helper.PointerGeneratorBahdanauAttention object at 0x7fd8c6b70b70>\n",
      "Tensor(\"decoder/while/PGDecoderStep/decoder/multi_rnn_cell/cell_0/cell_0/pointer_generator_attention_wrapper/Softmax:0\", shape=(?, ?), dtype=float32) Tensor(\"decoder/while/PGDecoderStep/decoder/multi_rnn_cell/cell_0/cell_0/pointer_generator_attention_wrapper/Softmax:0\", shape=(?, ?), dtype=float32)\n",
      "<pointer_generator_helper.PointerGeneratorBahdanauAttention object at 0x7fd8c6b70b70>\n",
      "Tensor(\"decoder/while/PGDecoderStep/decoder/multi_rnn_cell/cell_0/cell_0/pointer_generator_attention_wrapper/Softmax_1:0\", shape=(?, ?), dtype=float32) Tensor(\"decoder/while/PGDecoderStep/decoder/multi_rnn_cell/cell_0/cell_0/pointer_generator_attention_wrapper/Softmax_1:0\", shape=(?, ?), dtype=float32)\n",
      "Tensor(\"decoder_1/transpose_1:0\", shape=(?, ?), dtype=int32)\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Summarization(size_layer, num_layers, embedded_size, len(dictionary))\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batching(X, Y):\n",
    "    s_, es_, oovs_, target_ = [], [], [], []\n",
    "    for x, y in zip(X, Y):\n",
    "        s,es,oovs = sent2idx(x, dictionary)\n",
    "        target = target2idx(y, oovs,dictionary)\n",
    "        s_.append(s)\n",
    "        es_.append(es)\n",
    "        oovs_.append(oovs)\n",
    "        target_.append(target)\n",
    "    s_ = pad_sequences(s_,padding='post')\n",
    "    es_ = pad_sequences(es_,padding='post')\n",
    "    target_ = pad_sequences(target_,padding='post')\n",
    "    maxlen = max([len(o) for o in oovs_])\n",
    "    return s_, es_, target_, maxlen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [02:03<00:00,  1.81it/s, accuracy=0.156, cost=65.5] \n",
      "test minibatch loop: 100%|██████████| 55/55 [00:09<00:00,  5.89it/s, accuracy=0.113, cost=74.4]\n",
      "train minibatch loop:   0%|          | 0/220 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, avg loss: 76.652437, avg accuracy: 0.099290\n",
      "epoch: 0, avg loss test: 72.695572, avg accuracy test: 0.133109\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [02:03<00:00,  1.79it/s, accuracy=0.159, cost=65.2]\n",
      "test minibatch loop: 100%|██████████| 55/55 [00:09<00:00,  6.12it/s, accuracy=0.14, cost=72.1] \n",
      "train minibatch loop:   0%|          | 0/220 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 68.791295, avg accuracy: 0.140208\n",
      "epoch: 1, avg loss test: 70.518493, avg accuracy test: 0.143250\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [02:03<00:00,  1.78it/s, accuracy=0.148, cost=63.6]\n",
      "test minibatch loop: 100%|██████████| 55/55 [00:08<00:00,  6.45it/s, accuracy=0.167, cost=66.3]\n",
      "train minibatch loop:   0%|          | 0/220 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, avg loss: 64.951911, avg accuracy: 0.164101\n",
      "epoch: 2, avg loss test: 68.836975, avg accuracy test: 0.160287\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [02:03<00:00,  1.82it/s, accuracy=0.228, cost=55]  \n",
      "test minibatch loop: 100%|██████████| 55/55 [00:08<00:00,  6.11it/s, accuracy=0.146, cost=66.8]\n",
      "train minibatch loop:   0%|          | 0/220 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, avg loss: 59.688779, avg accuracy: 0.203397\n",
      "epoch: 3, avg loss test: 66.158990, avg accuracy test: 0.163988\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [02:02<00:00,  1.77it/s, accuracy=0.269, cost=57.6]\n",
      "test minibatch loop: 100%|██████████| 55/55 [00:08<00:00,  6.25it/s, accuracy=0.142, cost=65.9]\n",
      "train minibatch loop:   0%|          | 0/220 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, avg loss: 53.233103, avg accuracy: 0.245670\n",
      "epoch: 4, avg loss test: 65.842302, avg accuracy test: 0.167900\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [02:02<00:00,  1.85it/s, accuracy=0.317, cost=39.9]\n",
      "test minibatch loop: 100%|██████████| 55/55 [00:08<00:00,  6.22it/s, accuracy=0.187, cost=61.5]\n",
      "train minibatch loop:   0%|          | 0/220 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, avg loss: 47.825079, avg accuracy: 0.298827\n",
      "epoch: 5, avg loss test: 65.852169, avg accuracy test: 0.162368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [02:01<00:00,  1.82it/s, accuracy=0.407, cost=41.7]\n",
      "test minibatch loop: 100%|██████████| 55/55 [00:09<00:00,  6.25it/s, accuracy=0.147, cost=69.9]\n",
      "train minibatch loop:   0%|          | 0/220 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, avg loss: 43.178482, avg accuracy: 0.350300\n",
      "epoch: 6, avg loss test: 68.023724, avg accuracy test: 0.164362\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [02:00<00:00,  1.85it/s, accuracy=0.326, cost=50.5]\n",
      "test minibatch loop: 100%|██████████| 55/55 [00:08<00:00,  6.14it/s, accuracy=0.134, cost=73.9]\n",
      "train minibatch loop:   0%|          | 0/220 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, avg loss: 38.934335, avg accuracy: 0.399625\n",
      "epoch: 7, avg loss test: 69.417469, avg accuracy test: 0.161455\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [01:59<00:00,  1.93it/s, accuracy=0.471, cost=38.4]\n",
      "test minibatch loop: 100%|██████████| 55/55 [00:08<00:00,  5.59it/s, accuracy=0.179, cost=72.4]\n",
      "train minibatch loop:   0%|          | 0/220 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, avg loss: 35.275156, avg accuracy: 0.441227\n",
      "epoch: 8, avg loss test: 71.623262, avg accuracy test: 0.160715\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 220/220 [01:59<00:00,  1.88it/s, accuracy=0.425, cost=34.2]\n",
      "test minibatch loop: 100%|██████████| 55/55 [00:08<00:00,  6.22it/s, accuracy=0.134, cost=71.1] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, avg loss: 32.010688, avg accuracy: 0.481275\n",
      "epoch: 9, avg loss test: 73.853065, avg accuracy test: 0.155170\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "import time\n",
    "\n",
    "for EPOCH in range(10):\n",
    "    lasttime = time.time()\n",
    "    total_loss, total_accuracy, total_loss_test, total_accuracy_test = 0, 0, 0, 0\n",
    "    train_X, train_Y = shuffle(train_X, train_Y)\n",
    "    test_X, test_Y = shuffle(test_X, test_Y)\n",
    "    pbar = tqdm(range(0, len(train_X), batch_size), desc='train minibatch loop')\n",
    "    for k in pbar:\n",
    "        index = min(k+batch_size,len(train_X))\n",
    "        batch_x, batch_es, batch_y, maxlen = batching(train_X[k: index],\n",
    "                                                     train_Y[k: index])\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                 model.source_extend_tokens:batch_es,\n",
    "                                                model.Y:batch_y,\n",
    "                                                model.source_oov_words:maxlen})\n",
    "        total_loss += loss\n",
    "        total_accuracy += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "        \n",
    "    pbar = tqdm(range(0, len(test_X), batch_size), desc='test minibatch loop')\n",
    "    for k in pbar:\n",
    "        index = min(k+batch_size,len(test_X))\n",
    "        batch_x, batch_es, batch_y, maxlen = batching(test_X[k: index],\n",
    "                                                     test_Y[k: index])\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                 model.source_extend_tokens:batch_es,\n",
    "                                                model.Y:batch_y,\n",
    "                                                model.source_oov_words:maxlen})\n",
    "        total_loss_test += loss\n",
    "        total_accuracy_test += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "        \n",
    "    total_loss /= (len(train_X) / batch_size)\n",
    "    total_accuracy /= (len(train_X) / batch_size)\n",
    "    total_loss_test /= (len(test_X) / batch_size)\n",
    "    total_accuracy_test /= (len(test_X) / batch_size)\n",
    "        \n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(EPOCH, total_loss, total_accuracy))\n",
    "    print('epoch: %d, avg loss test: %f, avg accuracy test: %f'%(EPOCH, total_loss_test, total_accuracy_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x, batch_es, batch_y, maxlen = batching(test_X[:1], test_Y[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "out = [rev_dictionary[i] for i in sess.run(model.beam_predictions, feed_dict = {model.X: batch_x,\n",
    "                                             model.Y: batch_y,\n",
    "                                             model.source_extend_tokens:batch_es,\n",
    "                                             model.source_oov_words:maxlen})[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f7(seq):\n",
    "    seen = set()\n",
    "    seen_add = seen.add\n",
    "    return ' '.join([x for x in seq if not (x in seen or seen_add(x))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'unsaid forty rules bengaluru of expedition signboard dandwate lake months mudit EOS'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f7(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'man loses arm fighting a crocodile to save dogs EOS'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f7([rev_dictionary[i] for i in batch_y[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}