topic-generator/4.dilated-seq2seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://github.com/huseinzol05/Malaya-Dataset/raw/master/news/news.zip\n",
    "# !unzip news.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = os.listdir('news')\n",
    "news = ['news/' + i for i in labels if '.json' in i]\n",
    "labels = [i.replace('.json','') for i in labels]\n",
    "len(news)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import malaya\n",
    "import re\n",
    "\n",
    "tokenizer = malaya.preprocessing.SocialTokenizer().tokenize\n",
    "splitter = malaya.texts._text_functions.split_into_sentences\n",
    "accept_tokens = ',-.()\"\\''\n",
    "\n",
    "def is_number_regex(s):\n",
    "    if re.match(\"^\\d+?\\.\\d+?$\", s) is None:\n",
    "        return s.isdigit()\n",
    "    return True\n",
    "\n",
    "def detect_money(word):\n",
    "    if word[:2] == 'rm' and is_number_regex(word[2:]):\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "def preprocessing(string):\n",
    "    tokenized = tokenizer(string)\n",
    "    tokenized = [w.lower() for w in tokenized if len(w) > 1 or w in accept_tokens]\n",
    "    tokenized = ['<NUM>' if is_number_regex(w) else w for w in tokenized]\n",
    "    tokenized = ['<MONEY>' if detect_money(w) else w for w in tokenized]\n",
    "    return tokenized\n",
    "\n",
    "def clean_label(label):\n",
    "    string = re.sub('[^A-Za-z\\- ]+', ' ', label)\n",
    "    return re.sub(r'[ ]+', ' ', string.lower()).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [clean_label(label) for label in labels]\n",
    "rejected_text = ['javascript', 'malaysiakini']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "import json\n",
    "\n",
    "maxlen = 150\n",
    "min_len = 20\n",
    "\n",
    "x, y = [], []\n",
    "for no, n in enumerate(news):\n",
    "    with open(n) as fopen: \n",
    "        news_ = json.load(fopen)\n",
    "    for row in news_:\n",
    "        if len(row['text'].split()) > min_len:\n",
    "            split = splitter(row['text'])\n",
    "            split = [s for s in split if not any([r in s.lower() for r in rejected_text])]\n",
    "            split = ' '.join(split)\n",
    "            p = preprocessing(split)\n",
    "            p = p[:maxlen]\n",
    "            if len(p) > 10:\n",
    "                x.append(p)\n",
    "                y.append(labels[no])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13963, 13963)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x), len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "def build_dataset(words, n_words, atleast=2):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 61551\n",
      "Most common words [(',', 89449), ('.', 77833), ('-', 32111), ('yang', 31940), ('the', 29150), ('dan', 28032)]\n",
      "Sample data [58, 65, 772, 3394, 273, 204, 223, 3534, 2286, 11740] ['kuala', 'lumpur', 'former', 'transport', 'minister', 'tan', 'sri', 'chan', 'kong', 'choy']\n",
      "filtered vocab size: 37920\n",
      "% of vocab used: 61.61%\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "\n",
    "concat = list(itertools.chain(*x)) + ' '.join(labels).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])\n",
    "print('filtered vocab size:',len(dictionary))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary)/vocabulary_size,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(x)):\n",
    "    x.append('EOS')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensor2tensor.utils import beam_search\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embed_seq(x, vocab_sz, embed_dim, name, zero_pad=True): \n",
    "    embedding = tf.get_variable(name, [vocab_sz, embed_dim]) \n",
    "    if zero_pad:\n",
    "        embedding = tf.concat([tf.zeros([1, embed_dim]), embedding[1:, :]], 0) \n",
    "    x = tf.nn.embedding_lookup(embedding, x)\n",
    "    return x\n",
    "\n",
    "def position_encoding(inputs):\n",
    "    T = tf.shape(inputs)[1]\n",
    "    repr_dim = inputs.get_shape()[-1].value\n",
    "    pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1])\n",
    "    i = np.arange(0, repr_dim, 2, np.float32)\n",
    "    denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1])\n",
    "    enc = tf.expand_dims(tf.concat([tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0)\n",
    "    return tf.tile(enc, [tf.shape(inputs)[0], 1, 1])\n",
    "\n",
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    return gamma * normalized + beta\n",
    "\n",
    "\n",
    "def cnn_block(x, dilation_rate, pad_sz, hidden_dim, kernel_size):\n",
    "    x = layer_norm(x)\n",
    "    pad = tf.zeros([tf.shape(x)[0], pad_sz, hidden_dim])\n",
    "    x =  tf.layers.conv1d(inputs = tf.concat([pad, x, pad], 1),\n",
    "                          filters = hidden_dim,\n",
    "                          kernel_size = kernel_size,\n",
    "                          dilation_rate = dilation_rate)\n",
    "    x = x[:, :-pad_sz, :]\n",
    "    x = tf.nn.relu(x)\n",
    "    return x\n",
    "\n",
    "class Summarization:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, \n",
    "                 dict_size, learning_rate, \n",
    "                 kernel_size = 2, n_attn_heads = 16):\n",
    "\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "\n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype = tf.int32)\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype = tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        self.batch_size = batch_size\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        \n",
    "        self.embedding = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        \n",
    "        self.num_layers = num_layers\n",
    "        self.kernel_size = kernel_size\n",
    "        self.size_layer = size_layer\n",
    "        self.n_attn_heads = n_attn_heads\n",
    "        self.dict_size = dict_size\n",
    "        \n",
    "        self.training_logits = self.forward(self.X, decoder_input)\n",
    "\n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "        \n",
    "    def forward(self, x, y, reuse = False):\n",
    "        with tf.variable_scope('forward',reuse=reuse):\n",
    "            with tf.variable_scope('forward',reuse=reuse):\n",
    "                encoder_embedded = tf.nn.embedding_lookup(self.embedding, x)\n",
    "                decoder_embedded = tf.nn.embedding_lookup(self.embedding, y)\n",
    "                encoder_embedded += position_encoding(encoder_embedded)\n",
    "\n",
    "                for i in range(self.num_layers): \n",
    "                    dilation_rate = 2 ** i\n",
    "                    pad_sz = (self.kernel_size - 1) * dilation_rate \n",
    "                    with tf.variable_scope('block_%d'%i,reuse=reuse):\n",
    "                        encoder_embedded += cnn_block(encoder_embedded, dilation_rate, \n",
    "                                                      pad_sz, self.size_layer, self.kernel_size)\n",
    "\n",
    "                g = tf.identity(decoder_embedded)\n",
    "                for i in range(self.num_layers):\n",
    "                    dilation_rate = 2 ** i\n",
    "                    pad_sz = (self.kernel_size - 1) * dilation_rate\n",
    "                    with tf.variable_scope('decode_%d'%i,reuse=reuse):\n",
    "                        attn_res = h = cnn_block(decoder_embedded, dilation_rate, \n",
    "                                                 pad_sz, self.size_layer, self.kernel_size)\n",
    "                        C = []\n",
    "                        for j in range(self.n_attn_heads):\n",
    "                            h_ = tf.layers.dense(h, self.size_layer//self.n_attn_heads)\n",
    "                            g_ = tf.layers.dense(g, self.size_layer//self.n_attn_heads)\n",
    "                            zu_ = tf.layers.dense(encoder_embedded, self.size_layer//self.n_attn_heads)\n",
    "                            ze_ = tf.layers.dense(encoder_embedded, self.size_layer//self.n_attn_heads)\n",
    "\n",
    "                            d = tf.layers.dense(h_, self.size_layer//self.n_attn_heads) + g_\n",
    "                            dz = tf.matmul(d, tf.transpose(zu_, [0, 2, 1]))\n",
    "                            a = tf.nn.softmax(dz)\n",
    "                            c_ = tf.matmul(a, ze_)\n",
    "                            C.append(c_)\n",
    "\n",
    "                        c = tf.concat(C, 2)\n",
    "                        h = tf.layers.dense(attn_res + c, self.size_layer)\n",
    "                        decoder_embedded += h\n",
    "\n",
    "                return tf.layers.dense(decoder_embedded, self.dict_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 256\n",
    "num_layers = 4\n",
    "embedded_size = 256\n",
    "learning_rate = 1e-3\n",
    "batch_size = 32\n",
    "epoch = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def beam_search_decoding(length = 50, beam_width = 5):\n",
    "    initial_ids = tf.fill([model.batch_size], GO)\n",
    "    \n",
    "    def symbols_to_logits(ids):\n",
    "        x = tf.contrib.seq2seq.tile_batch(model.X, beam_width)\n",
    "        logits = model.forward(x, ids, reuse = True)\n",
    "        return logits[:, tf.shape(ids)[1]-1, :]\n",
    "\n",
    "    final_ids, final_probs, _ = beam_search.beam_search(\n",
    "        symbols_to_logits,\n",
    "        initial_ids,\n",
    "        beam_width,\n",
    "        length,\n",
    "        len(dictionary),\n",
    "        0.0,\n",
    "        eos_id = EOS)\n",
    "    \n",
    "    return final_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From <ipython-input-13-6bf9c95e37ae>:11: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From <ipython-input-13-6bf9c95e37ae>:32: conv1d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.conv1d instead.\n",
      "WARNING:tensorflow:From <ipython-input-13-6bf9c95e37ae>:98: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Summarization(size_layer, num_layers, embedded_size, \n",
    "                len(dictionary), learning_rate)\n",
    "model.generate = beam_search_decoding()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i:\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = [i.split() for i in y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(y, dictionary)\n",
    "Y = str_idx(x, dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:28<00:00,  4.42it/s, accuracy=0.104, cost=6.92] \n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:21,  5.37it/s, accuracy=0.0929, cost=6.6]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, avg loss: 6.923699, avg accuracy: 0.102635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.54it/s, accuracy=0.121, cost=5.87] \n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:21,  5.37it/s, accuracy=0.131, cost=6.01]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, avg loss: 5.832101, avg accuracy: 0.154718\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.53it/s, accuracy=0.155, cost=5.18]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.41it/s, accuracy=0.12, cost=6.07]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, avg loss: 5.372792, avg accuracy: 0.179991\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.51it/s, accuracy=0.205, cost=4.58]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:21,  5.32it/s, accuracy=0.136, cost=5.23]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, avg loss: 5.030723, avg accuracy: 0.200721\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.52it/s, accuracy=0.259, cost=4.12]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.41it/s, accuracy=0.179, cost=5.17]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 5, avg loss: 4.749640, avg accuracy: 0.221752\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.56it/s, accuracy=0.321, cost=3.78]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.44it/s, accuracy=0.179, cost=4.78]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 6, avg loss: 4.478085, avg accuracy: 0.244396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.56it/s, accuracy=0.353, cost=3.5] \n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.41it/s, accuracy=0.224, cost=4.85]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 7, avg loss: 4.286467, avg accuracy: 0.261283\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.53it/s, accuracy=0.361, cost=3.28]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:21,  5.34it/s, accuracy=0.235, cost=4.3]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 8, avg loss: 4.111535, avg accuracy: 0.278317\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.56it/s, accuracy=0.41, cost=3.06] \n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.41it/s, accuracy=0.232, cost=4.41]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 9, avg loss: 3.975817, avg accuracy: 0.292393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.53it/s, accuracy=0.4, cost=2.94]  \n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:21,  5.32it/s, accuracy=0.231, cost=4.34]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, avg loss: 3.848144, avg accuracy: 0.306767\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.54it/s, accuracy=0.427, cost=2.8] \n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.39it/s, accuracy=0.286, cost=4.21]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 11, avg loss: 3.737917, avg accuracy: 0.318649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.53it/s, accuracy=0.45, cost=2.65] \n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:21,  5.36it/s, accuracy=0.282, cost=3.81]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 12, avg loss: 3.633036, avg accuracy: 0.330552\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.54it/s, accuracy=0.489, cost=2.49]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:21,  5.37it/s, accuracy=0.308, cost=3.79]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 13, avg loss: 3.541286, avg accuracy: 0.340664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.53it/s, accuracy=0.506, cost=2.41]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:21,  5.35it/s, accuracy=0.33, cost=3.54]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 14, avg loss: 3.459093, avg accuracy: 0.350461\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:22<00:00,  5.57it/s, accuracy=0.508, cost=2.32]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.40it/s, accuracy=0.37, cost=3.18]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 15, avg loss: 3.371084, avg accuracy: 0.362246\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:21<00:00,  5.52it/s, accuracy=0.542, cost=2.17]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.42it/s, accuracy=0.375, cost=3.12]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 16, avg loss: 3.283447, avg accuracy: 0.373162\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:21<00:00,  5.51it/s, accuracy=0.559, cost=2.09]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:21,  5.32it/s, accuracy=0.356, cost=3.24]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 17, avg loss: 3.204799, avg accuracy: 0.383106\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:21<00:00,  5.59it/s, accuracy=0.566, cost=2.04]\n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.40it/s, accuracy=0.377, cost=3.15]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 18, avg loss: 3.174452, avg accuracy: 0.386813\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:21<00:00,  5.60it/s, accuracy=0.606, cost=1.91] \n",
      "minibatch loop:   0%|          | 1/437 [00:00<01:20,  5.44it/s, accuracy=0.374, cost=3.07]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 19, avg loss: 3.115246, avg accuracy: 0.394881\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 437/437 [01:21<00:00,  5.60it/s, accuracy=0.6, cost=1.89]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 20, avg loss: 3.058292, avg accuracy: 0.402294\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for i in range(epoch):\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    pbar = tqdm(\n",
    "        range(0, len(X), batch_size), desc = 'minibatch loop')\n",
    "    for k in pbar:\n",
    "        index = min(k+batch_size, len(X))\n",
    "        batch_x, seq_x = pad_sentence_batch(X[k: index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(Y[k: index], PAD)\n",
    "        accuracy,loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += accuracy\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    total_loss /= (len(X) / batch_size)\n",
    "    total_accuracy /= (len(X) / batch_size)\n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(i+1, total_loss, total_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_topic = 'isu najib mahathir'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[29, 134, 112]]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_topic_idx = str_idx([test_topic.split()], dictionary)\n",
    "batch_test, _ = pad_sentence_batch(test_topic_idx, PAD)\n",
    "batch_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 51)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated = sess.run(model.generate, feed_dict = {model.X: batch_test})[0]\n",
    "generated.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GO - ( ubah saiz teks ) shah alam - percutian perdana menteri , datuk seri najib tun razak dan presiden amerika syarikat ( as ) , prof . datuk kamarudin ahmad razif abdul rahman ( reuters ) - malaysia ' mikhail kolyada of an internal investigation by the end of\n",
      "GO - ( ubah saiz teks ) shah alam - percutian perdana menteri , datuk seri najib tun razak dan presiden amerika syarikat ( as ) , prof . datuk kamarudin ahmad razif abdul rahman ( reuters ) - malaysia ' mikhail kolyada of an internal investigation on the island ,\n",
      "GO - ( ubah saiz teks ) shah alam - percutian perdana menteri , datuk seri najib tun razak dan presiden amerika syarikat ( as ) , prof . datuk kamarudin ahmad razif abdul rahman ( reuters ) - malaysia ' mikhail kolyada of an internal investigation on the island of\n",
      "GO - ( ubah saiz teks ) shah alam - percutian perdana menteri , datuk seri najib tun razak dan presiden amerika syarikat ( as ) , prof . datuk kamarudin ahmad razif abdul rahman ( reuters ) - malaysia ' mikhail kolyada of an internal investigation on the island in\n",
      "GO - ( ubah saiz teks ) shah alam - percutian perdana menteri , datuk seri najib tun razak dan presiden amerika syarikat ( as ) , prof . datuk kamarudin ahmad razif abdul rahman ( reuters ) - malaysia ' mikhail kolyada to an internal investigation after the company '\n"
     ]
    }
   ],
   "source": [
    "for g in generated:\n",
    "    print(' '.join([rev_dictionary[i] for i in g]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}