Neural Machine Translation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "try:\n",
    "    import cPickle\n",
    "except:\n",
    "    import pickle\n",
    "from collections import Counter\n",
    "import matplotlib.pyplot as plt\n",
    "from nltk.translate.bleu_score import sentence_bleu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1920209, 1920209)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# save sentences line by line in a list\n",
    "\n",
    "def read_sentences(file_path):\n",
    "    sentences = []\n",
    "    with open(file_path, 'r', encoding = 'utf-8') as reader:\n",
    "        sentences = reader.readlines()\n",
    "    return sentences\n",
    "\n",
    "en_sentences = read_sentences('C:\\\\Users\\\\vatsa\\\\Downloads\\\\CS533\\\\Project\\\\data\\\\europarl-v7.de-en.en')\n",
    "de_sentences = read_sentences('C:\\\\Users\\\\vatsa\\\\Downloads\\\\CS533\\\\Project\\\\data\\\\europarl-v7.de-en.de')\n",
    "\n",
    "len(en_sentences), len(de_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_dataset(en_sentences, de_sentences):\n",
    "\n",
    "    en_vocab_dict = Counter(word.strip(',.\" ;:)(][?!') for sentence in en_sentences for word in sentence.split())\n",
    "    de_vocab_dict = Counter(word.strip(',.\" ;:)(][?!') for sentence in de_sentences for word in sentence.split())\n",
    "\n",
    "    en_vocab = list(map(lambda x: x[0], sorted(en_vocab_dict.items(), key = lambda x: -x[1])))\n",
    "    de_vocab = list(map(lambda x: x[0], sorted(de_vocab_dict.items(), key = lambda x: -x[1])))\n",
    "\n",
    "    en_vocab = en_vocab[:100000]\n",
    "    de_vocab = de_vocab[:100000]\n",
    "\n",
    "    start_idx = 2\n",
    "    en_word2idx = dict([(word, idx+start_idx) for idx, word in enumerate(en_vocab)])\n",
    "    en_word2idx['<ukn>'] = 0\n",
    "    en_word2idx['<pad>'] = 1\n",
    "    en_idx2word = dict([(idx, word) for word, idx in en_word2idx.items()])\n",
    "\n",
    "    start_idx = 4\n",
    "    de_word2idx = dict([(word, idx+start_idx) for idx, word in enumerate(de_vocab)])\n",
    "    de_word2idx['<ukn>'] = 0\n",
    "    de_word2idx['<go>']  = 1 # start tag used to detect beginning of sentence in german. Not needed for encoder\n",
    "    de_word2idx['<eos>'] = 2 # end tag used to detect end of sentence in german. Not needed for encoder\n",
    "    de_word2idx['<pad>'] = 3\n",
    "    de_idx2word = dict([(idx, word) for word, idx in de_word2idx.items()])\n",
    "\n",
    "    x = [[en_word2idx.get(word.strip(',.\" ;:)(][?!'), 0) for word in sentence.split()] for sentence in en_sentences]\n",
    "    y = [[de_word2idx.get(word.strip(',.\" ;:)(][?!'), 0) for word in sentence.split()] for sentence in de_sentences]\n",
    "    X = []\n",
    "    Y = []\n",
    "    for i in range(len(x)):\n",
    "        n1 = len(x[i])\n",
    "        n2 = len(y[i])\n",
    "        n = n1 if n1 < n2 else n2 \n",
    "        if abs(n1 - n2) <= 0.3 * n:\n",
    "            if n1 <= 40 and n2 <= 40:\n",
    "                X.append(x[i])\n",
    "                Y.append(y[i])\n",
    "    return X, Y, en_word2idx, en_idx2word, en_vocab, de_word2idx, de_idx2word, de_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def save_dataset(file_path, obj):\n",
    "    with open(file_path, 'wb') as f:\n",
    "        pickle.dump(obj, f, -1)\n",
    "\n",
    "save_dataset('./data.pkl', create_dataset(en_sentences, de_sentences))\n",
    "\n",
    "def read_dataset(file_path):\n",
    "    with open(file_path, 'rb') as f:\n",
    "        return pickle.load(f)\n",
    "    \n",
    "X, Y, en_word2idx, en_idx2word, en_vocab, de_word2idx, de_idx2word, de_vocab = read_dataset('data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sentence in English - encoded: [3163, 1210, 203, 10, 14, 13229, 314, 3608]\n",
      "Sentence in German - encoded: [30, 621, 49, 28, 8, 46, 8938, 8, 3046]\n",
      "Decoded:\n",
      "------------------------\n",
      "Please rise then for this minute' s silence \n",
      "\n",
      "Ich bitte Sie sich zu einer Schweigeminute zu erheben "
     ]
    }
   ],
   "source": [
    "# inspect data\n",
    "print ('Sentence in English - encoded:', X[2])\n",
    "print ('Sentence in German - encoded:', Y[2])\n",
    "print ('Decoded:\\n------------------------')\n",
    "\n",
    "for i in range(len(X[2])):\n",
    "    print (en_idx2word[X[2][i]], end=\" \")\n",
    "\n",
    "print ('\\n')\n",
    "\n",
    "for i in range(len(Y[2])):\n",
    "    print (de_idx2word[Y[2][i]], end=\" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def data_padding(x, y, length = 40):\n",
    "    for i in range(len(x)):\n",
    "        x[i] = x[i] + (length - len(x[i])) * [en_word2idx['<pad>']]\n",
    "        y[i] = [de_word2idx['<go>']] + y[i] + [de_word2idx['<eos>']] + (length-len(y[i])) * [de_word2idx['<pad>']]\n",
    "\n",
    "data_padding(X, Y)\n",
    "\n",
    "X_train,  X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.1)\n",
    "\n",
    "del X\n",
    "del Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_seq_len = 40\n",
    "output_seq_len = 42\n",
    "en_vocab_size = len(en_vocab) + 2 # + <pad>, <ukn>\n",
    "de_vocab_size = len(de_vocab) + 4 # + <pad>, <ukn>, <eos>, <go>\n",
    "\n",
    "# placeholders\n",
    "encoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'encoder{}'.format(i)) for i in range(input_seq_len)]\n",
    "decoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'decoder{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "targets = [decoder_inputs[i+1] for i in range(output_seq_len-1)]\n",
    "# add one more target\n",
    "targets.append(tf.placeholder(dtype = tf.int32, shape = [None], name = 'last_target'))\n",
    "target_weights = [tf.placeholder(dtype = tf.float32, shape = [None], name = 'target_w{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "# output projection\n",
    "size = 512\n",
    "w_t = tf.get_variable('proj_w', [de_vocab_size, size], tf.float32)\n",
    "b = tf.get_variable('proj_b', [de_vocab_size], tf.float32)\n",
    "w = tf.transpose(w_t)\n",
    "output_projection = (w, b)\n",
    "\n",
    "outputs, states = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(\n",
    "                                            encoder_inputs,\n",
    "                                            decoder_inputs,\n",
    "                                            tf.contrib.rnn.BasicLSTMCell(size),\n",
    "                                            num_encoder_symbols = en_vocab_size,\n",
    "                                            num_decoder_symbols = de_vocab_size,\n",
    "                                            embedding_size = 100,\n",
    "                                            feed_previous = False,\n",
    "                                            output_projection = output_projection,\n",
    "                                            dtype = tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\ops\\nn_impl.py:1310: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# define our loss function\n",
    "\n",
    "# sampled softmax loss - returns: A batch_size 1-D tensor of per-example sampled softmax losses\n",
    "def sampled_loss(labels, logits):\n",
    "    return tf.nn.sampled_softmax_loss(\n",
    "                        weights = w_t,\n",
    "                        biases = b,\n",
    "                        labels = tf.reshape(labels, [-1, 1]),\n",
    "                        inputs = logits,\n",
    "                        num_sampled = 512,\n",
    "                        num_classes = de_vocab_size)\n",
    "\n",
    "# Weighted cross-entropy loss for a sequence of logits\n",
    "loss = tf.contrib.legacy_seq2seq.sequence_loss(outputs, targets, target_weights, softmax_loss_function = sampled_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let's define some helper functions\n",
    "\n",
    "# simple softmax function\n",
    "def softmax(x):\n",
    "    n = np.max(x)\n",
    "    e_x = np.exp(x - n)\n",
    "    return e_x / e_x.sum()\n",
    "\n",
    "# feed data into placeholders\n",
    "def feed_dict(x, y, batch_size = 64):\n",
    "    feed = {}\n",
    "    \n",
    "    idxes = np.random.choice(len(x), size = batch_size, replace = False)\n",
    "    \n",
    "    for i in range(input_seq_len):\n",
    "        feed[encoder_inputs[i].name] = np.array([x[j][i] for j in idxes], dtype = np.int32)\n",
    "        \n",
    "    for i in range(output_seq_len):\n",
    "        feed[decoder_inputs[i].name] = np.array([y[j][i] for j in idxes], dtype = np.int32)\n",
    "        \n",
    "    feed[targets[len(targets)-1].name] = np.full(shape = [batch_size], fill_value = de_word2idx['<pad>'], dtype = np.int32)\n",
    "    \n",
    "    for i in range(output_seq_len-1):\n",
    "        batch_weights = np.ones(batch_size, dtype = np.float32)\n",
    "        target = feed[decoder_inputs[i+1].name]\n",
    "        for j in range(batch_size):\n",
    "            if target[j] == de_word2idx['<pad>']:\n",
    "                batch_weights[j] = 0.0\n",
    "        feed[target_weights[i].name] = batch_weights\n",
    "        \n",
    "    feed[target_weights[output_seq_len-1].name] = np.zeros(batch_size, dtype = np.float32)\n",
    "    \n",
    "    return feed\n",
    "\n",
    "# decode output sequence\n",
    "def decode_output(output_seq):\n",
    "    words = []\n",
    "    for i in range(output_seq_len):\n",
    "        smax = softmax(output_seq[i])\n",
    "        idx = np.argmax(smax)\n",
    "        words.append(de_idx2word[idx])\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ops and hyperparameters\n",
    "learning_rate = 5e-3\n",
    "batch_size = 64\n",
    "steps = 2500\n",
    "\n",
    "# ops for projecting outputs\n",
    "outputs_proj = [tf.matmul(outputs[i], output_projection[0]) + output_projection[1] for i in range(output_seq_len)]\n",
    "\n",
    "# training op\n",
    "optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(loss)\n",
    "\n",
    "# init op\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "# forward step\n",
    "def forward_step(sess, feed):\n",
    "    output_sequences = sess.run(outputs_proj, feed_dict = feed)\n",
    "    return output_sequences\n",
    "\n",
    "# training step\n",
    "def backward_step(sess, feed):\n",
    "    sess.run(optimizer, feed_dict = feed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------TRAINING------------------\n",
      "step: 0, loss: 9.911262512207031\n",
      "step: 4, loss: 9.896167755126953\n",
      "step: 9, loss: 9.885457038879395\n",
      "step: 14, loss: 9.908164024353027\n",
      "step: 19, loss: 9.938472747802734\n",
      "Checkpoint is saved\n",
      "step: 24, loss: 9.915271759033203\n",
      "step: 29, loss: 9.941328048706055\n",
      "step: 34, loss: 9.894632339477539\n",
      "step: 39, loss: 9.841753005981445\n",
      "Checkpoint is saved\n",
      "step: 44, loss: 9.804807662963867\n",
      "step: 49, loss: 9.748702049255371\n",
      "step: 54, loss: 9.686172485351562\n",
      "step: 59, loss: 9.282543182373047\n",
      "Checkpoint is saved\n",
      "step: 64, loss: 8.424149513244629\n",
      "step: 69, loss: 7.651979446411133\n",
      "step: 74, loss: 7.711560249328613\n",
      "step: 79, loss: 7.148109436035156\n",
      "Checkpoint is saved\n",
      "step: 84, loss: 6.980451583862305\n",
      "step: 89, loss: 6.807934761047363\n",
      "step: 94, loss: 7.455887794494629\n",
      "step: 99, loss: 6.36277961730957\n",
      "Checkpoint is saved\n",
      "step: 104, loss: 6.61954402923584\n",
      "step: 109, loss: 6.206734657287598\n",
      "step: 114, loss: 6.342323303222656\n",
      "step: 119, loss: 6.127676486968994\n",
      "Checkpoint is saved\n",
      "step: 124, loss: 6.457366466522217\n",
      "step: 129, loss: 6.648914337158203\n",
      "step: 134, loss: 6.124096870422363\n",
      "step: 139, loss: 5.613900184631348\n",
      "Checkpoint is saved\n",
      "step: 144, loss: 5.779672145843506\n",
      "step: 149, loss: 5.600014686584473\n",
      "step: 154, loss: 6.036521911621094\n",
      "step: 159, loss: 5.5854058265686035\n",
      "Checkpoint is saved\n",
      "step: 164, loss: 5.519305229187012\n",
      "step: 169, loss: 5.525867938995361\n",
      "step: 174, loss: 5.616455554962158\n",
      "step: 179, loss: 5.325733184814453\n",
      "Checkpoint is saved\n",
      "step: 184, loss: 5.4259233474731445\n",
      "step: 189, loss: 5.378445625305176\n",
      "step: 194, loss: 5.457329750061035\n",
      "step: 199, loss: 5.416937351226807\n",
      "Checkpoint is saved\n",
      "step: 204, loss: 5.623446941375732\n",
      "step: 209, loss: 5.1536407470703125\n",
      "step: 214, loss: 5.523046493530273\n",
      "step: 219, loss: 5.258944988250732\n",
      "Checkpoint is saved\n",
      "step: 224, loss: 5.017424583435059\n",
      "step: 229, loss: 5.061684608459473\n",
      "step: 234, loss: 5.168099880218506\n",
      "step: 239, loss: 5.077337265014648\n",
      "Checkpoint is saved\n",
      "step: 244, loss: 5.135865211486816\n",
      "step: 249, loss: 4.973653793334961\n",
      "step: 254, loss: 5.462830066680908\n",
      "step: 259, loss: 4.99891471862793\n",
      "Checkpoint is saved\n",
      "step: 264, loss: 4.866244316101074\n",
      "step: 269, loss: 4.8519086837768555\n",
      "step: 274, loss: 4.700699329376221\n",
      "step: 279, loss: 4.859347343444824\n",
      "Checkpoint is saved\n",
      "step: 284, loss: 4.878484725952148\n",
      "step: 289, loss: 4.752120494842529\n",
      "step: 294, loss: 4.71595573425293\n",
      "step: 299, loss: 4.773390769958496\n",
      "Checkpoint is saved\n",
      "step: 304, loss: 4.911589622497559\n",
      "step: 309, loss: 4.507783889770508\n",
      "step: 314, loss: 4.673243522644043\n",
      "step: 319, loss: 4.3129472732543945\n",
      "Checkpoint is saved\n",
      "step: 324, loss: 4.458073616027832\n",
      "step: 329, loss: 4.576162338256836\n",
      "step: 334, loss: 4.5395121574401855\n",
      "step: 339, loss: 4.385821342468262\n",
      "Checkpoint is saved\n",
      "step: 344, loss: 4.6280694007873535\n",
      "step: 349, loss: 4.40574836730957\n",
      "step: 354, loss: 4.17890739440918\n",
      "step: 359, loss: 4.4329023361206055\n",
      "Checkpoint is saved\n",
      "step: 364, loss: 4.176817893981934\n",
      "step: 369, loss: 4.524710655212402\n",
      "step: 374, loss: 4.781718730926514\n",
      "step: 379, loss: 4.01705265045166\n",
      "Checkpoint is saved\n",
      "step: 384, loss: 3.8193721771240234\n",
      "step: 389, loss: 4.343754768371582\n",
      "step: 394, loss: 4.379310131072998\n",
      "step: 399, loss: 4.4066314697265625\n",
      "Checkpoint is saved\n",
      "step: 404, loss: 4.1380743980407715\n",
      "step: 409, loss: 4.115804672241211\n",
      "step: 414, loss: 4.379770278930664\n",
      "step: 419, loss: 4.067921161651611\n",
      "Checkpoint is saved\n",
      "step: 424, loss: 4.020147800445557\n",
      "step: 429, loss: 4.164481163024902\n",
      "step: 434, loss: 4.239485740661621\n",
      "step: 439, loss: 4.154537200927734\n",
      "Checkpoint is saved\n",
      "step: 444, loss: 4.199252128601074\n",
      "step: 449, loss: 3.9659790992736816\n",
      "step: 454, loss: 4.073634624481201\n",
      "step: 459, loss: 3.897879123687744\n",
      "Checkpoint is saved\n",
      "step: 464, loss: 3.868072509765625\n",
      "step: 469, loss: 3.972306728363037\n",
      "step: 474, loss: 3.991210460662842\n",
      "step: 479, loss: 3.8299684524536133\n",
      "Checkpoint is saved\n",
      "step: 484, loss: 4.024656295776367\n",
      "step: 489, loss: 3.752838611602783\n",
      "step: 494, loss: 4.046872615814209\n",
      "step: 499, loss: 3.8215692043304443\n",
      "Checkpoint is saved\n",
      "step: 504, loss: 3.951993703842163\n",
      "step: 509, loss: 3.8156485557556152\n",
      "step: 514, loss: 3.6413819789886475\n",
      "step: 519, loss: 3.619907855987549\n",
      "Checkpoint is saved\n",
      "step: 524, loss: 3.5771520137786865\n",
      "step: 529, loss: 3.7075414657592773\n",
      "step: 534, loss: 3.6666319370269775\n",
      "step: 539, loss: 3.692042827606201\n",
      "Checkpoint is saved\n",
      "step: 544, loss: 3.473306179046631\n",
      "step: 549, loss: 3.675520420074463\n",
      "step: 554, loss: 3.610142469406128\n",
      "step: 559, loss: 3.4983348846435547\n",
      "Checkpoint is saved\n",
      "step: 564, loss: 3.7733154296875\n",
      "step: 569, loss: 3.365407705307007\n",
      "step: 574, loss: 3.5618491172790527\n",
      "step: 579, loss: 3.3524527549743652\n",
      "Checkpoint is saved\n",
      "step: 584, loss: 3.5139315128326416\n",
      "step: 589, loss: 3.4671621322631836\n",
      "step: 594, loss: 3.446045398712158\n",
      "step: 599, loss: 3.10022234916687\n",
      "Checkpoint is saved\n",
      "step: 604, loss: 3.255415439605713\n",
      "step: 609, loss: 3.349513530731201\n",
      "step: 614, loss: 3.412287473678589\n",
      "step: 619, loss: 3.222783088684082\n",
      "Checkpoint is saved\n",
      "step: 624, loss: 3.307830333709717\n",
      "step: 629, loss: 2.9573941230773926\n",
      "step: 634, loss: 3.3370537757873535\n",
      "step: 639, loss: 3.26871919631958\n",
      "Checkpoint is saved\n",
      "step: 644, loss: 3.461709976196289\n",
      "step: 649, loss: 3.2116808891296387\n",
      "step: 654, loss: 3.0281245708465576\n",
      "step: 659, loss: 3.16115140914917\n",
      "Checkpoint is saved\n",
      "step: 664, loss: 3.0575451850891113\n",
      "step: 669, loss: 3.2734079360961914\n",
      "step: 674, loss: 3.421271562576294\n",
      "step: 679, loss: 3.2951886653900146\n",
      "Checkpoint is saved\n",
      "step: 684, loss: 2.9400811195373535\n",
      "step: 689, loss: 3.113271474838257\n",
      "step: 694, loss: 3.1332314014434814\n",
      "step: 699, loss: 2.9801127910614014\n",
      "Checkpoint is saved\n",
      "step: 704, loss: 3.2560276985168457\n",
      "step: 709, loss: 3.270071506500244\n",
      "step: 714, loss: 2.90644907951355\n",
      "step: 719, loss: 3.0193991661071777\n",
      "Checkpoint is saved\n",
      "step: 724, loss: 2.7565243244171143\n",
      "step: 729, loss: 3.0065088272094727\n",
      "step: 734, loss: 2.86618709564209\n",
      "step: 739, loss: 2.7725446224212646\n",
      "Checkpoint is saved\n",
      "step: 744, loss: 2.734163284301758\n",
      "step: 749, loss: 3.068617105484009\n",
      "step: 754, loss: 2.815662384033203\n",
      "step: 759, loss: 2.9933135509490967\n",
      "Checkpoint is saved\n",
      "step: 764, loss: 3.2221083641052246\n",
      "step: 769, loss: 2.6814887523651123\n",
      "step: 774, loss: 2.9855642318725586\n",
      "step: 779, loss: 2.883629322052002\n",
      "Checkpoint is saved\n",
      "step: 784, loss: 2.7815802097320557\n",
      "step: 789, loss: 2.857929229736328\n",
      "step: 794, loss: 2.601090431213379\n",
      "step: 799, loss: 2.7837557792663574\n",
      "Checkpoint is saved\n",
      "step: 804, loss: 2.628904342651367\n",
      "step: 809, loss: 2.715324878692627\n",
      "step: 814, loss: 2.6001243591308594\n",
      "step: 819, loss: 3.0309393405914307\n",
      "Checkpoint is saved\n",
      "step: 824, loss: 2.5337095260620117\n",
      "step: 829, loss: 2.6125411987304688\n",
      "step: 834, loss: 2.600849151611328\n",
      "step: 839, loss: 2.632874011993408\n",
      "Checkpoint is saved\n",
      "step: 844, loss: 2.821291446685791\n",
      "step: 849, loss: 2.648266315460205\n",
      "step: 854, loss: 2.570312023162842\n",
      "step: 859, loss: 2.6897072792053223\n",
      "Checkpoint is saved\n",
      "step: 864, loss: 2.556267738342285\n",
      "step: 869, loss: 2.422346830368042\n",
      "step: 874, loss: 2.5420613288879395\n",
      "step: 879, loss: 2.672328472137451\n",
      "Checkpoint is saved\n",
      "step: 884, loss: 2.6529159545898438\n",
      "step: 889, loss: 2.6170787811279297\n",
      "step: 894, loss: 2.3163013458251953\n",
      "step: 899, loss: 2.703950881958008\n",
      "Checkpoint is saved\n",
      "step: 904, loss: 2.5606491565704346\n",
      "step: 909, loss: 2.6901769638061523\n",
      "step: 914, loss: 2.716808795928955\n",
      "step: 919, loss: 2.70426344871521\n",
      "Checkpoint is saved\n",
      "step: 924, loss: 2.635282039642334\n",
      "step: 929, loss: 2.2347514629364014\n",
      "step: 934, loss: 2.4787402153015137\n",
      "step: 939, loss: 2.3557567596435547\n",
      "Checkpoint is saved\n",
      "step: 944, loss: 2.4443912506103516\n",
      "step: 949, loss: 2.4792027473449707\n",
      "step: 954, loss: 2.3993303775787354\n",
      "step: 959, loss: 2.491461753845215\n",
      "Checkpoint is saved\n",
      "step: 964, loss: 2.7570793628692627\n",
      "step: 969, loss: 2.4496521949768066\n",
      "step: 974, loss: 2.151270627975464\n",
      "step: 979, loss: 2.5346426963806152\n",
      "Checkpoint is saved\n",
      "step: 984, loss: 2.3607523441314697\n",
      "step: 989, loss: 2.329322576522827\n",
      "step: 994, loss: 2.4123353958129883\n",
      "step: 999, loss: 2.2988762855529785\n",
      "Checkpoint is saved\n",
      "step: 1004, loss: 2.1498284339904785\n",
      "step: 1009, loss: 2.4245476722717285\n",
      "step: 1014, loss: 2.5499773025512695\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step: 1019, loss: 2.2640016078948975\n",
      "Checkpoint is saved\n",
      "step: 1024, loss: 2.0340726375579834\n",
      "step: 1029, loss: 2.4034717082977295\n",
      "step: 1034, loss: 2.3374979496002197\n",
      "step: 1039, loss: 2.4103236198425293\n",
      "Checkpoint is saved\n",
      "step: 1044, loss: 2.675180435180664\n",
      "step: 1049, loss: 2.0938820838928223\n",
      "step: 1054, loss: 2.3074140548706055\n",
      "step: 1059, loss: 2.4116368293762207\n",
      "Checkpoint is saved\n",
      "step: 1064, loss: 2.0223608016967773\n",
      "step: 1069, loss: 2.4092493057250977\n",
      "step: 1074, loss: 2.2447595596313477\n",
      "step: 1079, loss: 2.0402650833129883\n",
      "Checkpoint is saved\n",
      "step: 1084, loss: 2.2536187171936035\n",
      "step: 1089, loss: 2.371462345123291\n",
      "step: 1094, loss: 2.2895302772521973\n",
      "step: 1099, loss: 2.0741422176361084\n",
      "Checkpoint is saved\n",
      "step: 1104, loss: 2.389817714691162\n",
      "step: 1109, loss: 2.213097333908081\n",
      "step: 1114, loss: 2.1745100021362305\n",
      "step: 1119, loss: 2.106513023376465\n",
      "Checkpoint is saved\n",
      "step: 1124, loss: 2.265883207321167\n",
      "step: 1129, loss: 2.2433042526245117\n",
      "step: 1134, loss: 2.2311768531799316\n",
      "step: 1139, loss: 1.88706374168396\n",
      "Checkpoint is saved\n",
      "step: 1144, loss: 2.358902931213379\n",
      "step: 1149, loss: 2.2245545387268066\n",
      "step: 1154, loss: 2.2256107330322266\n",
      "step: 1159, loss: 2.141623020172119\n",
      "Checkpoint is saved\n",
      "step: 1164, loss: 2.0318455696105957\n",
      "step: 1169, loss: 2.22365665435791\n",
      "step: 1174, loss: 2.1921629905700684\n",
      "step: 1179, loss: 2.1713242530822754\n",
      "Checkpoint is saved\n",
      "step: 1184, loss: 1.9745285511016846\n",
      "step: 1189, loss: 2.027390956878662\n",
      "step: 1194, loss: 2.0185413360595703\n",
      "step: 1199, loss: 2.364781379699707\n",
      "Checkpoint is saved\n",
      "step: 1204, loss: 2.063305377960205\n",
      "step: 1209, loss: 1.8266737461090088\n",
      "step: 1214, loss: 2.260223865509033\n",
      "step: 1219, loss: 1.8868188858032227\n",
      "Checkpoint is saved\n",
      "step: 1224, loss: 2.175737142562866\n",
      "step: 1229, loss: 1.929107427597046\n",
      "step: 1234, loss: 2.1234312057495117\n",
      "step: 1239, loss: 2.072995662689209\n",
      "Checkpoint is saved\n",
      "step: 1244, loss: 2.3467516899108887\n",
      "step: 1249, loss: 2.0113930702209473\n",
      "step: 1254, loss: 2.085843563079834\n",
      "step: 1259, loss: 2.1887316703796387\n",
      "Checkpoint is saved\n",
      "step: 1264, loss: 1.9684009552001953\n",
      "step: 1269, loss: 1.833166241645813\n",
      "step: 1274, loss: 1.8755195140838623\n",
      "step: 1279, loss: 1.9361822605133057\n",
      "Checkpoint is saved\n",
      "step: 1284, loss: 2.135859489440918\n",
      "step: 1289, loss: 1.9900202751159668\n",
      "step: 1294, loss: 1.9829763174057007\n",
      "step: 1299, loss: 1.73282790184021\n",
      "Checkpoint is saved\n",
      "step: 1304, loss: 1.901231288909912\n",
      "step: 1309, loss: 1.9601075649261475\n",
      "step: 1314, loss: 1.9290974140167236\n",
      "step: 1319, loss: 1.8694512844085693\n",
      "Checkpoint is saved\n",
      "step: 1324, loss: 1.8367030620574951\n",
      "step: 1329, loss: 1.7577307224273682\n",
      "step: 1334, loss: 1.7911760807037354\n",
      "step: 1339, loss: 1.8370999097824097\n",
      "Checkpoint is saved\n",
      "step: 1344, loss: 2.0024774074554443\n",
      "step: 1349, loss: 2.011418342590332\n",
      "step: 1354, loss: 1.9551135301589966\n",
      "step: 1359, loss: 1.991275429725647\n",
      "Checkpoint is saved\n",
      "step: 1364, loss: 1.9056775569915771\n",
      "step: 1369, loss: 1.9169085025787354\n",
      "step: 1374, loss: 1.9399006366729736\n",
      "step: 1379, loss: 2.0089876651763916\n",
      "Checkpoint is saved\n",
      "step: 1384, loss: 1.8560354709625244\n",
      "step: 1389, loss: 1.9815888404846191\n",
      "step: 1394, loss: 1.9360734224319458\n",
      "step: 1399, loss: 1.8383227586746216\n",
      "Checkpoint is saved\n",
      "step: 1404, loss: 1.9160313606262207\n",
      "step: 1409, loss: 1.6831388473510742\n",
      "step: 1414, loss: 1.8826559782028198\n",
      "step: 1419, loss: 1.827967882156372\n",
      "Checkpoint is saved\n",
      "step: 1424, loss: 1.918924331665039\n",
      "step: 1429, loss: 1.7485828399658203\n",
      "step: 1434, loss: 2.1905698776245117\n",
      "step: 1439, loss: 1.8376826047897339\n",
      "Checkpoint is saved\n",
      "step: 1444, loss: 1.9538098573684692\n",
      "step: 1449, loss: 1.839423656463623\n",
      "step: 1454, loss: 1.7813465595245361\n",
      "step: 1459, loss: 1.7487797737121582\n",
      "Checkpoint is saved\n",
      "step: 1464, loss: 2.0027236938476562\n",
      "step: 1469, loss: 1.6653612852096558\n",
      "step: 1474, loss: 1.9138476848602295\n",
      "step: 1479, loss: 1.8505548238754272\n",
      "Checkpoint is saved\n",
      "step: 1484, loss: 1.8623261451721191\n",
      "step: 1489, loss: 1.8875036239624023\n",
      "step: 1494, loss: 1.720641851425171\n",
      "step: 1499, loss: 1.7753859758377075\n",
      "Checkpoint is saved\n",
      "step: 1504, loss: 1.7133655548095703\n",
      "step: 1509, loss: 1.7471356391906738\n",
      "step: 1514, loss: 1.6719727516174316\n",
      "step: 1519, loss: 1.752631664276123\n",
      "Checkpoint is saved\n",
      "step: 1524, loss: 1.8123527765274048\n",
      "step: 1529, loss: 1.961004376411438\n",
      "step: 1534, loss: 1.9540008306503296\n",
      "step: 1539, loss: 1.734950065612793\n",
      "Checkpoint is saved\n",
      "step: 1544, loss: 1.7859042882919312\n",
      "step: 1549, loss: 1.649460792541504\n",
      "step: 1554, loss: 1.5826165676116943\n",
      "step: 1559, loss: 1.6211904287338257\n",
      "Checkpoint is saved\n",
      "step: 1564, loss: 1.8435331583023071\n",
      "step: 1569, loss: 1.5774672031402588\n",
      "step: 1574, loss: 1.7427905797958374\n",
      "step: 1579, loss: 1.7830026149749756\n",
      "Checkpoint is saved\n",
      "step: 1584, loss: 1.8615635633468628\n",
      "step: 1589, loss: 1.6455016136169434\n",
      "step: 1594, loss: 1.769737958908081\n",
      "step: 1599, loss: 1.7444751262664795\n",
      "Checkpoint is saved\n",
      "step: 1604, loss: 1.69480562210083\n",
      "step: 1609, loss: 1.7970013618469238\n",
      "step: 1614, loss: 1.7408257722854614\n",
      "step: 1619, loss: 1.9310498237609863\n",
      "Checkpoint is saved\n",
      "step: 1624, loss: 1.5740092992782593\n",
      "step: 1629, loss: 1.7167940139770508\n",
      "step: 1634, loss: 1.5184438228607178\n",
      "step: 1639, loss: 1.5292682647705078\n",
      "Checkpoint is saved\n",
      "step: 1644, loss: 1.9823392629623413\n",
      "step: 1649, loss: 1.6652491092681885\n",
      "step: 1654, loss: 1.6679695844650269\n",
      "step: 1659, loss: 1.6779518127441406\n",
      "Checkpoint is saved\n",
      "step: 1664, loss: 1.8020045757293701\n",
      "step: 1669, loss: 1.557186484336853\n",
      "step: 1674, loss: 1.8979500532150269\n",
      "step: 1679, loss: 1.6372718811035156\n",
      "Checkpoint is saved\n",
      "step: 1684, loss: 1.518216848373413\n",
      "step: 1689, loss: 1.7978465557098389\n",
      "step: 1694, loss: 1.664821982383728\n",
      "step: 1699, loss: 1.678322434425354\n",
      "Checkpoint is saved\n",
      "step: 1704, loss: 1.5341737270355225\n",
      "step: 1709, loss: 1.5614235401153564\n",
      "step: 1714, loss: 1.7057292461395264\n",
      "step: 1719, loss: 1.5161709785461426\n",
      "Checkpoint is saved\n",
      "step: 1724, loss: 1.5476734638214111\n",
      "step: 1729, loss: 1.476776123046875\n",
      "step: 1734, loss: 1.598168134689331\n",
      "step: 1739, loss: 1.7599225044250488\n",
      "Checkpoint is saved\n",
      "step: 1744, loss: 1.7617034912109375\n",
      "step: 1749, loss: 1.490013599395752\n",
      "step: 1754, loss: 1.591275691986084\n",
      "step: 1759, loss: 1.7138464450836182\n",
      "Checkpoint is saved\n",
      "step: 1764, loss: 1.7848074436187744\n",
      "step: 1769, loss: 1.5737318992614746\n",
      "step: 1774, loss: 1.5647327899932861\n",
      "step: 1779, loss: 1.4236171245574951\n",
      "Checkpoint is saved\n",
      "step: 1784, loss: 1.5847424268722534\n",
      "step: 1789, loss: 1.6151597499847412\n",
      "step: 1794, loss: 1.6065254211425781\n",
      "step: 1799, loss: 1.5703712701797485\n",
      "Checkpoint is saved\n",
      "step: 1804, loss: 1.7083529233932495\n",
      "step: 1809, loss: 1.7886621952056885\n",
      "step: 1814, loss: 1.5512677431106567\n",
      "step: 1819, loss: 1.554257869720459\n",
      "Checkpoint is saved\n",
      "step: 1824, loss: 1.7084290981292725\n",
      "step: 1829, loss: 1.6603882312774658\n",
      "step: 1834, loss: 1.618959665298462\n",
      "step: 1839, loss: 1.6049234867095947\n",
      "Checkpoint is saved\n",
      "step: 1844, loss: 1.7624691724777222\n",
      "step: 1849, loss: 1.7200899124145508\n",
      "step: 1854, loss: 1.401193618774414\n",
      "step: 1859, loss: 1.4660065174102783\n",
      "Checkpoint is saved\n",
      "step: 1864, loss: 1.386596918106079\n",
      "step: 1869, loss: 1.6846644878387451\n",
      "step: 1874, loss: 1.5327049493789673\n",
      "step: 1879, loss: 1.5892202854156494\n",
      "Checkpoint is saved\n",
      "step: 1884, loss: 1.4747297763824463\n",
      "step: 1889, loss: 1.5583839416503906\n",
      "step: 1894, loss: 1.591275691986084\n",
      "step: 1899, loss: 1.5475075244903564\n",
      "Checkpoint is saved\n",
      "step: 1904, loss: 1.6297578811645508\n",
      "step: 1909, loss: 1.735494613647461\n",
      "step: 1914, loss: 1.5065248012542725\n",
      "step: 1919, loss: 1.5114312171936035\n",
      "Checkpoint is saved\n",
      "step: 1924, loss: 1.5633902549743652\n",
      "step: 1929, loss: 1.4908299446105957\n",
      "step: 1934, loss: 1.3686063289642334\n",
      "step: 1939, loss: 1.6115288734436035\n",
      "Checkpoint is saved\n",
      "step: 1944, loss: 1.4381049871444702\n",
      "step: 1949, loss: 1.6452873945236206\n",
      "step: 1954, loss: 1.543336033821106\n",
      "step: 1959, loss: 1.8123801946640015\n",
      "Checkpoint is saved\n",
      "step: 1964, loss: 1.523854374885559\n",
      "step: 1969, loss: 1.5444920063018799\n",
      "step: 1974, loss: 1.4040145874023438\n",
      "step: 1979, loss: 1.530921220779419\n",
      "Checkpoint is saved\n",
      "step: 1984, loss: 1.6570780277252197\n",
      "step: 1989, loss: 1.7430126667022705\n",
      "step: 1994, loss: 1.4140541553497314\n",
      "step: 1999, loss: 1.43466055393219\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checkpoint is saved\n",
      "step: 2004, loss: 1.5810883045196533\n",
      "step: 2009, loss: 1.4909790754318237\n",
      "step: 2014, loss: 1.7791677713394165\n",
      "step: 2019, loss: 1.5257623195648193\n",
      "Checkpoint is saved\n",
      "step: 2024, loss: 1.4997544288635254\n",
      "step: 2029, loss: 1.7327592372894287\n",
      "step: 2034, loss: 1.4393284320831299\n",
      "step: 2039, loss: 1.5061869621276855\n",
      "Checkpoint is saved\n",
      "step: 2044, loss: 1.3929513692855835\n",
      "step: 2049, loss: 1.6205239295959473\n",
      "step: 2054, loss: 1.3990164995193481\n",
      "step: 2059, loss: 1.4685313701629639\n",
      "Checkpoint is saved\n",
      "step: 2064, loss: 1.4276872873306274\n",
      "step: 2069, loss: 1.576460599899292\n",
      "step: 2074, loss: 1.5305659770965576\n",
      "step: 2079, loss: 1.683668613433838\n",
      "Checkpoint is saved\n",
      "step: 2084, loss: 1.309368371963501\n",
      "step: 2089, loss: 1.4812687635421753\n",
      "step: 2094, loss: 1.4720590114593506\n",
      "step: 2099, loss: 1.5217511653900146\n",
      "Checkpoint is saved\n",
      "step: 2104, loss: 1.5398826599121094\n",
      "step: 2109, loss: 1.5198982954025269\n",
      "step: 2114, loss: 1.254844307899475\n",
      "step: 2119, loss: 1.5853968858718872\n",
      "Checkpoint is saved\n",
      "step: 2124, loss: 1.4711450338363647\n",
      "step: 2129, loss: 1.5638930797576904\n",
      "step: 2134, loss: 1.6007301807403564\n",
      "step: 2139, loss: 1.4584840536117554\n",
      "Checkpoint is saved\n",
      "step: 2144, loss: 1.4003533124923706\n",
      "step: 2149, loss: 1.377600908279419\n",
      "step: 2154, loss: 1.4057354927062988\n",
      "step: 2159, loss: 1.5474281311035156\n",
      "Checkpoint is saved\n",
      "step: 2164, loss: 1.5524230003356934\n",
      "step: 2169, loss: 1.4346848726272583\n",
      "step: 2174, loss: 1.4684350490570068\n",
      "step: 2179, loss: 1.362292766571045\n",
      "Checkpoint is saved\n",
      "step: 2184, loss: 1.4814006090164185\n",
      "step: 2189, loss: 1.5547280311584473\n",
      "step: 2194, loss: 1.447525978088379\n",
      "step: 2199, loss: 1.609824776649475\n",
      "Checkpoint is saved\n",
      "step: 2204, loss: 1.535713791847229\n",
      "step: 2209, loss: 1.4821081161499023\n",
      "step: 2214, loss: 1.4426368474960327\n",
      "step: 2219, loss: 1.4933724403381348\n",
      "Checkpoint is saved\n",
      "step: 2224, loss: 1.4226984977722168\n",
      "step: 2229, loss: 1.3445461988449097\n",
      "step: 2234, loss: 1.4020898342132568\n",
      "step: 2239, loss: 1.2847493886947632\n",
      "Checkpoint is saved\n",
      "step: 2244, loss: 1.4097813367843628\n",
      "step: 2249, loss: 1.568271517753601\n",
      "step: 2254, loss: 1.5133404731750488\n",
      "step: 2259, loss: 1.4987834692001343\n",
      "Checkpoint is saved\n",
      "step: 2264, loss: 1.507989525794983\n",
      "step: 2269, loss: 1.4852511882781982\n",
      "step: 2274, loss: 1.4777650833129883\n",
      "step: 2279, loss: 1.3511040210723877\n",
      "Checkpoint is saved\n",
      "step: 2284, loss: 1.4127696752548218\n",
      "step: 2289, loss: 1.2072038650512695\n",
      "step: 2294, loss: 1.4080049991607666\n",
      "step: 2299, loss: 1.5161023139953613\n",
      "Checkpoint is saved\n",
      "step: 2304, loss: 1.4769426584243774\n",
      "step: 2309, loss: 1.2836517095565796\n",
      "step: 2314, loss: 1.4343427419662476\n",
      "step: 2319, loss: 1.4478111267089844\n",
      "Checkpoint is saved\n",
      "step: 2324, loss: 1.4184023141860962\n",
      "step: 2329, loss: 1.3601326942443848\n",
      "step: 2334, loss: 1.499678134918213\n",
      "step: 2339, loss: 1.5114514827728271\n",
      "Checkpoint is saved\n",
      "step: 2344, loss: 1.3327395915985107\n",
      "step: 2349, loss: 1.1837177276611328\n",
      "step: 2354, loss: 1.4897533655166626\n",
      "step: 2359, loss: 1.3496159315109253\n",
      "Checkpoint is saved\n",
      "step: 2364, loss: 1.3826675415039062\n",
      "step: 2369, loss: 1.3615355491638184\n",
      "step: 2374, loss: 1.4522740840911865\n",
      "step: 2379, loss: 1.1497610807418823\n",
      "Checkpoint is saved\n",
      "step: 2384, loss: 1.426743745803833\n",
      "step: 2389, loss: 1.3151304721832275\n",
      "step: 2394, loss: 1.4107041358947754\n",
      "step: 2399, loss: 1.4688737392425537\n",
      "Checkpoint is saved\n",
      "step: 2404, loss: 1.548513650894165\n",
      "step: 2409, loss: 1.3311679363250732\n",
      "step: 2414, loss: 1.191833257675171\n",
      "step: 2419, loss: 1.372937560081482\n",
      "Checkpoint is saved\n",
      "step: 2424, loss: 1.3497962951660156\n",
      "step: 2429, loss: 1.441138505935669\n",
      "step: 2434, loss: 1.2872695922851562\n",
      "step: 2439, loss: 1.4058560132980347\n",
      "Checkpoint is saved\n",
      "step: 2444, loss: 1.3748681545257568\n",
      "step: 2449, loss: 1.4682421684265137\n",
      "step: 2454, loss: 1.5515596866607666\n",
      "step: 2459, loss: 1.3953197002410889\n",
      "Checkpoint is saved\n",
      "step: 2464, loss: 1.5248165130615234\n",
      "step: 2469, loss: 1.331953525543213\n",
      "step: 2474, loss: 1.23206627368927\n",
      "step: 2479, loss: 1.3819597959518433\n",
      "Checkpoint is saved\n",
      "step: 2484, loss: 1.3727867603302002\n",
      "step: 2489, loss: 1.3561768531799316\n",
      "step: 2494, loss: 1.3136134147644043\n",
      "step: 2499, loss: 1.427410364151001\n",
      "Checkpoint is saved\n",
      "Training time for 2500 steps: 18958.83646583557s\n"
     ]
    }
   ],
   "source": [
    "# let's train the model\n",
    "\n",
    "# we will use this list to plot losses through steps\n",
    "losses = []\n",
    "\n",
    "# save a checkpoint so we can restore the model later \n",
    "saver = tf.train.Saver()\n",
    "\n",
    "print ('------------------TRAINING------------------')\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    t = time.time()\n",
    "    for step in range(steps):\n",
    "        feed = feed_dict(X_train, Y_train)\n",
    "            \n",
    "        backward_step(sess, feed)\n",
    "        \n",
    "        if step % 5 == 4 or step == 0:\n",
    "            loss_value = sess.run(loss, feed_dict = feed)\n",
    "            print ('step: {}, loss: {}'.format(step, loss_value))\n",
    "            losses.append(loss_value)\n",
    "        \n",
    "        if step % 20 == 19:\n",
    "            saver.save(sess, 'checkpoints/', global_step=step)\n",
    "            print ('Checkpoint is saved')\n",
    "            \n",
    "    print ('Training time for {} steps: {}s'.format(steps, time.time() - t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd8lFXWwPHfmUnvQBIIEAi9Sg1N\nFKTYe9e1rbqL765rd12xr+tadn112XUbu5bV146sHRUQRBBBQHrvCTUhvbf7/vE8M5lJAoQymSTP\n+X4++WSeMpl7Q5gzt50rxhiUUko5lyvYBVBKKRVcGgiUUsrhNBAopZTDaSBQSimH00CglFIOp4FA\nKaUcLmCBQEReEZGDIrLW51xbEZktIlvs720C9fpKKaUaJ5AtgteAc+qcexCYa4zpBcy1j5VSSgWR\nBHJBmYikAZ8aYwbax5uAM4wx+0QkBZhvjOkTsAIopZQ6qpAmfr32xph9AHYwSD7cjSIyBZgCEB0d\nPbxv374n/OLGgMgJ/xillGoRli9fnm2MSTrafU0dCBrNGDMdmA6Qnp5uli1bdsw/Y9aafRigqKyK\nBz5YDcDUc/ty2/geJ7OoSinVLInIrsbc19SB4ICIpPh0DR0M5Iu9uWQ3C7dm+53745ebuGhIR1Li\nIwP50kop1WI09fTRj4Gb7Mc3AR8F8sX+c8tI/nbdMO47szcbf3cO3z4wAZdLeHbWxkC+rFJKtSiB\nnD76NrAY6CMimSJyK/AscKaIbAHOtI8Dxu0SzjslhTsm9SIi1E1q2yhuG9edj1buZdnOnEC+tFJK\ntRgBnTV0shzvGEFDSiqqmPS/35AcG86Ht49FdPRYKdVKichyY0z60e5z3MriqLAQ7pzUi1WZ+Sze\nfijYxVFKqaBzXCAAuHRoJ1wCi7dpIFBKKUcGgohQN3GRoeSVVAa7KEopFXSODAQACZGh5JVqIFBK\nKccGgvioMPI1ECillHMDQUJkKPklFcEuhlJKBZ1jA0G8dg0ppRTg4ECQEKWDxUopBU4OBJGhFJRV\nUlPT/BfUKaVUIDk2EMRHhWEMFJRpq0Ap5WyODQQx4W4Aiiuqg1wSpZQKLscGgvAQKxCUVWogUEo5\nm2MDQUSoVXUNBEopp3NsIAgPtVoE5VU1QS6JUkoFl2MDQYR2DSmlFODkQGB3DZVXaotAKeVsjg0E\nOlislFIWxwYC72BxlQYCpZSzOTgQ2IPF2jWklHI4xwcC7RpSSjmdgwOBp2tIWwRKKWdzbCDQwWKl\nlLI4NhC4XUKoWyjTMQKllMM5NhCAtaisXGcNKaUcztGBIDzUrS0CpZTjOToQRIS6KNcxAqWUwzk6\nEISHuHRBmVLK8RwdCCK0a0gppTQQ6GCxUsrpHB4IXNoiUEo5nrMDQYhbF5QppRzP0YEgPNSlgUAp\n5XiODgRWi0C7hpRSzuboQBAe6tY9i5VSjufoQKALypRSKkiBQETuEZF1IrJWRN4WkYhglCMi1K0L\nypRSjtfkgUBEOgF3AunGmIGAG7imqcsB1sriympDdY0JxssrpVSzEKyuoRAgUkRCgChgbzAK4d2u\nUlsFSikHa/JAYIzZAzwP7Ab2AfnGmK/q3iciU0RkmYgsy8rKCkhZIkLsXcp05pBSysGC0TXUBrgY\n6AZ0BKJF5Pq69xljphtj0o0x6UlJSQEpi+5brJRSwekamgzsMMZkGWMqgZnAqUEohwYCpZQiOIFg\nNzBaRKJERIBJwIYglINw7RpSSqmgjBEsAWYAK4A1dhmmN3U5QAeLlVIKgjRryBjzuDGmrzFmoDHm\nBmNMeTDKER5qVf/VRTuD8fJKKdUsOHplcXiI1SL4eFVQZq8qpVSz4OhAkBwb7n1sjC4qU0o5k6MD\nQWrbKH55Rg8ASnXmkFLKoRwdCABSEiIBKCqvCnJJlFIqOBwfCKLDrHGCknJtESilnEkDQXgIAFsP\nFpH24Gd8uloHjpVSzuL4QBBjB4KlO3MAmLliTzCLo5RSTc7xgcDTIsgrqQAgxCXBLI5SSjU5DQT2\nGEF+aSUAIW4NBEopZ9FAYLcIvIHA5fhfiVLKYRz/rlcbCKzpo9o1pJRyGg0EdtdQbrE1RuDWQKCU\nchjHB4IQt4uY8BCyisq9x0op5ST6rgd0T4r2bmCvXUNKKafRQAD0TI7xPjZo8jmllLNoIAB6Jcd6\nH1dWaSBQSjmLBgKgl0+LoKJat61USjmLBgKgd/vaFoEGAqWU02ggADq1iSTC3rayokoDgVLKWTQQ\nYK0d8AwYayBQSjmNBgLb7y4eSKhbqNSuIaWUw2ggsA3t0oYRaW21RaCUchwNBD5C3S6/FsE3m7PY\nerAwiCVSSqnA00DgIyzExarMfN5bloExhpteWcrkFxYEu1hKKRVQGgh8uMVKL/HAjNXkllQGuTRK\nKdU0NBD48OxJALAjuziIJVFKqaajgcBHjp2KGmoDgWdPY6WUaq00EPg4VFzufZyRUwJAQlRosIqj\nlFJNQgOBj0N2iyA8xEW2vT9BuU4nVUq1choIfFw4qCNgvfm/uWQ3AEVlVcEsklJKBZwGAh//e9Vg\npozr7neutLKaKl1trJRqxTQQ+Ah1u4iPrD8mUFxeHYTSKKVU09BAUEdDKSYKy3VNgVKq9dJAUMfP\nx3Xn9F6JfueKynWcQCnVegUlEIhIgojMEJGNIrJBRMYEoxwNiQkP4feXnAJAv5Q4AIo1ECilWrFg\ntQimAV8YY/oCg4ENQSpHg7q0i+LLu8fx24sGAFBYVkVBWSWlFTpWoJRqfZo8EIhIHDAOeBnAGFNh\njMlr6nIcTZ8OsbSxF5MVlVcx6ImvOGeaJqBTSrU+wWgRdAeygFdF5EcR+beIRNe9SUSmiMgyEVmW\nlZXV9KUEYiKs9BKetQS7DpUEpRxKKRVIwQgEIcAw4O/GmKFAMfBg3ZuMMdONMenGmPSkpKSmLiNQ\nm2dIB4uVUq1ZMAJBJpBpjFliH8/ACgzNTnSYFQiyiyqOcqdSSrVcTR4IjDH7gQwR6WOfmgSsb+py\nNIbLJUSHudmbVxrsoiilVMAEK8fyHcCbIhIGbAduDlI5jio6PISPV+0NdjGUUipgghIIjDErgfRg\nvPaxyi3x7xYqq6wmItQdpNIopdTJpyuLj6Ky2vgdF5RpugmlVOuigeAYFZRWUVNTGxwKyio1OCil\nWjQNBMfob/O20v2hz71v/oOe+Ir0p+YEuVRKKXX8GhUIROQuEYkTy8siskJEzgp04ZqDOfdaqSYG\nd44HYOaPewDYk1s7k6ihjKVKKdVSNLZFcIsxpgA4C0jCmuXzbMBK1Yz0TI7lplPT+PD2sfS3k9AB\nHCwsp6yyNvdQ2oOfYYxp6EcopVSz1thAIPb384BXjTGrfM45gojwhysGcd2oLgBkFZbz20/8lz8U\n+qxAnr3+AK8v3tmEJVRKqePT2Omjy0XkK6AbMFVEYgHH9YcM7BTPw+f3480lu7n//VX1ru/LKyOu\ng5Wo7uevLwPgxjFpTVlEpZQ6Zo1tEdyKlQ9ohDGmBAijGS8CC6SosMPHzn35ugJZKdXyNDYQGKA/\ncKd9HA1EBKRELcjTl57C05ee4j3en19W7x4dN1BKNXeN7Rr6G1ZX0ETgSaAQ+AAYEaByNWu/u2Qg\nUaFuLh/emU980k/sbSAQlFRUEx0erEweSil1dI19hxpljBkmIj8CGGNy7TxBjnTD6K7ex7ERtb/C\n/Q10DRWUVWogUEo1a43tGqoUETdWFxEikoQDB4sb4nbVTp7al1/Gmsx8v3UFhWW6l4FSqnlr7EfV\nPwP/BZJF5PfAFcAjAStVC+I7ePz99kNc+NJCv+sFpZp+QinVvDUqEBhj3hSR5Vh7BwhwiTGmWW04\nHyzDuiQw7ZohLN52iHd+yKh3XVsESqnmrrEpJnoAO4wxfwXWAmeKSEJAS9ZCiAgXD+lEt8R62y4D\nmq1UKdX8NXaM4AOgWkR6Av/GWlj2VsBK1QIlxoQDMGVcd8Z0b+c9X6AtAqVUM9fYMYIaY0yViFwG\nTDPG/MUzg0hZzh+UQliIi3MHduDVRTtZvP0QALsPFQe5ZEopdWTHMmvoWuBG4FP7XGhgitQyRYS6\nuXBwR0LcLm49rRuv3jyCIakJrMrM97vv2y1Z/GnO5iCVUiml6mtsILgZGAP83hizQ0S6Af8XuGK1\nbC6XMKFPMkNSE1i6I4c3vt9F+lOzmbkikxteXsqf5mwJdhGVUsqrUYHAGLPeGHOnMeZtEWkDxBpj\nHJGG+kRclZ6KS+DRD9eSXVTB52v2ea/5prBWSqlgauysofn2xjRtgVXAqyLyQmCL1vL17xjHJUM7\neY/nbDjofXy49QXlVdWc9tzXzFl/IODlU0opaHzXULy9Mc1lWPsRDAcmB65YrUdqmygAIkPdfufz\nDhMIMnJKyMwt5elZukxDKdU0GhsIQkQkBbiK2sFi1Qj97F3NHrmgH+N6J9GlrRUY/jRnc4OZSQ8V\nVQCQEKlj8UqpptHYQPAk8CWwzRjzg4h0B3TEsxHOHtCeD35xKj8Z2YXXbxnJSz8ZCsDna/bzY0Ye\nAEt35DB/k9VttL/AymCaEOXYnH5KqSbW2BQT7wPv+xxvBy4PVKFaExFheNc23uM2Pm/wuw+VkFtc\nwa3/sXYz2/ns+d49DbRFoJRqKo0KBCLSGfgLMBYrA+lC4C5jTGYAy9YqxUfVvsHf/e5Kv2vGGPbZ\ngcA3q6lSSgVSY7uGXgU+BjoCnYBP7HPqGMUeYW+C7KIKDthdQ2VVmuVbKdU0GhsIkowxrxpjquyv\n14CkAJar1RIRpl0zpMFrC7dmsfVgEQClFbrOQCnVNBobCLJF5HoRcdtf1wOHAlmw1uziIbVrCyJC\na/8J7nl3FVvsQFBeVU1xeRX3vbeKH3bmkFtc0eTlVEo5Q2MDwS1YU0f3A/uwNqa5OVCFcpKp5/Zr\n8HxReRW3v7WCD1Zk8sTH6xj6u9l8tHJPE5dOKeUEjU0xsdsYc5ExJskYk2yMuQRrcZk6QTedmsaK\nR8+sd/7H3XnM35QFwLq9BQAs2JzdpGVTSjnDieyqfi/wp5NVEKd55rJTqLEXlLWNPvyagfAQF+X2\nwHGISyivqrbPuw/7HKWUOhYnEgh0fuMJuHZkl0bd1zM5xtsiEIGLX1pEdY3hlZ+OINVepayUUiei\nsWMEDamfH0EdtzZRDS8g65kc4328OjOfjfsL2XKwiNP/MI+aGv0nUEqduCO2CESkkIbf8AWIDEiJ\nHOrtKaN5f1kmRWVVuFzw9tIMAHok1QaC9fsK/J6TXVROclxEk5ZTKdX6HLFFYIyJNcbENfAVa4w5\nkW4l7GmoP4qIJrED+naI49EL+vPcFYNoFx3uPX9VeiqdEmpjbrvoMH538QAAnvtiE9XaKlBKnaAT\n6Ro6UXcBmmu5AZ61BeEhLjrER7DowYn8akJP+5qb9LS2AHywIpNvNlvJ6rZnFfHcFxsZ/fRc3fRG\nKXVMTuhT/fGycxedD/wea/aR8hEVZv2z+H7W/+nYNF6at9VqIbSpbSFsPVjEoq2HeHnhDu+5zNwS\neibHNlVxlVItXFACAda00weAw75bicgUYApAly6Nm2HTWnhmA1X45BtKjAln/ZNnExHixuWTkO7p\nzzfWe/7kFxbw7QMTdFaRUqpRmrxrSEQuAA4aY5Yf6T5jzHRjTLoxJj0pyVlpjdLaNfwGHhUW4g0C\n8+4/w7vJTUN8WwhKKXUkwRgjGAtcJCI7gXeAiSLyf0EoR7PVmE/y3RKjuXxY58NeLyhreCtMpZSq\nq8m7howxU4GpACJyBnC/Meb6pi5HcxYR2rhVw3dO6smAjnHklFTwwIzVftfW7skPRNGUUq1QsMYI\n1FG8d9sYEg6zyMxDRJjcvz0AiTFh3PKatdPZead04Kt1B6isriHU7d/oG/n7OVw9IpX7zuoTmIIr\npVqcYE4fxRgz3xhzQTDL0FyN7NaW3u0bP/NnbM9E2seF8/frhjG5X3uqagzbsor4xzfbWLc3n7LK\naqqqazhYWM5fvt4awJIrpVoabRG0EuEhbpY8NBmA1Zl5ALyxeBdvLtnNs7PAJXDZEcYUlFLOFdQW\ngQqMHkkxhLiEN5fs9p6rMTBjuW4xrZSqTwNBKxQdHsLTl57iPT69V2K9e7YeLGzKIimlmjHtGmql\nrhqRyoGCMpJiw/ly3f561ye/sIAPbx/LkNSEIJROKdWcaIugFbtjUi+uGdmF6sPkpbvkr4tYsTu3\nwWv78ksxRhPaKeUEGggc4OaxaQB8dPtYhnbxbwH89JWlHCwoo7SimspqK6XFqow8xjzzNTNXWHsk\nF5RVsnG/fwpspVTroYHAASb0SWbns+czODWBYV3a+F0rKKti7d58+j32Bb96awUAq+3FaEt2HAJg\nyuvLOOdP32pWU6VaKQ0EDuO7t0FSrLXvgWc20ZfrDpBXUsErdp6isBDrz+P77TmArlZWqrXSwWKH\nuenUNPqmxPL9tkOce0oK5077ls/X1A4mD3lytvdxXkml3zjBit253r0QlFKthwYCh3G7hFN7JHJq\nj0SMMbjEWmPQkH35ZWTmlnqPtx4sAuC3n6zj3R8y+OKucXQ5TKZUpVTLoV1DDiYi3iBwpp2zyKNf\nShz78krZnVPiPZdbYmU0fXXRTkoqqtnSwFqEZz7fwPDfza53XinVfGkgUADcMbEnw+wZRZGhbs7s\n3569+WXc9MpSALq0jSK3uILSitoBY09g8PXPBds5VFzRNIVWSp0UGggUAL3bx3Lrad0BqDGGHknR\nAFTZTYb+KXEs25VLv8e+8D4n9whv+J6pqEqp5k8DgcON6d4OsPZAaBsdBlh7JfdIivG7zzPDyFdO\nSW0g2JNXyh++qN02s7i8KgClVUoFgg4WO9wbt46k2p4Z5A0ExtAtMdp7z6k92tGmgb0R/j5/G+N7\nJ9EzOYYHZqxi0dZD3mtF5VUkRIX53T915moGdU7g2pHO2oNaqeZOWwQOF+J2ER5i7YjmCQQ1xkpc\nd++ZvfngF6fy1s9He7uI6rpm+vec+szXlFX6dwVd/NIi8kv9xxDeXprB1JlrAlALpdSJ0ECgvDyf\n+i8YlALAnZN6MbyrtRLZEwZ+MqoLt4zt5ve8iuoawurshHaouIJvNmd5jzVvkVLNl3YNKa8Qt4vv\np07ytgx8/eKMHoSHuLh9Qk9C3S4+XrWX7KJy7/W80voziBZszuK8gR0IcbsortD0FEo1V9oiUH46\nxEd4U0v4iosI5e7Jvb17IH9+52l+1zfsq5+UbsbyTHo+PAtjTL1uon35pUecdaSUajoaCNRxSY6L\nICU+olH3/ue7neTXWXMw5pmvOeP5+QEomVLqWGkgUMfNM6V0RJo1jjAkNYHZ94yrd98Tn6zn/eUZ\n3mPPGoO6rYS6GhpX2Jdfyvg/zmNndvFxl1sp5U8DgTpuf79+OI+c34/nrxwMwJXpnUnzmXb6wS/G\neB+/umin97Fv2op5Gw+SU1zhfdOvqKrhNzNW8+ysjQx8/Ete+nqL32v+98c97DpUwv99v+uIZTPG\n6KI2pRpJB4vVceuUEMnPTrdWI29+6ly/sYVxvZNIiY9s8Hnr9taOJ9z82g+MTGvLpH7J/HPBdv5n\nfHfeXVbbepg2dwvXjOxCYozV+qixp7G6XXLEsj0zayPTF2xn29PnHfVepZxOWwTqpPANAksfmsT0\nG4Y3uBoZ4M63f/Q7/mFXDh+t3EtOcQVPf167OvmOiT2prDbMWX/Ae67ueoXDmb5gOwDFFbrCWamj\n0UCgTrrkuAgiQt2Eul10T4zmsqGduGZEqt89D57b1/tYsFJcp9VJaX3J0E4AfknsDhSUAZDTwIyj\nRVuzKa/yn6ZaUq7TVpU6Gg0EKqC+vv8MXrh6CM9ePoiXfjIUgHvP7M3/jO/BWz8bxfCubagx1qK0\nc09J8T5vaJcEuidGExnqJs8np9F+OxC8vzyTtAc/865lWLojh+v+vYS/fr3V7/WLNOeRUkelgUA1\nmXMGdODP1w7l9gk9ATi1ZyJ/vGKQ9/rZAzp4H7/5s1GICAlRofzr2x38a8F2bnntB77dku33M9dk\nWttnrs7MA+BgYbnf9RLtGlLqqHSwWDWZELeLiwZ39DuX1i6a2IgQ4iND6ZcS6z0fGWrlP4qPDGVf\nfhm//3xDgz9zZUYeCVGh3gHosBAXmbm1s5I8u6q5XcKAjvEntT5KtRYaCFRQuVzC9aO7EhcR6k1+\nB9buab7ffV04uCOfrNoLWLOKps3dQt8OVhB5Z2kGry+unVp673urvI+XPDSJpTty6JYYzS/fXMHf\nrhvGwE5WcDDGkJlbSmrb+ltvZhWWk1dSQa/2sfWuKdUaaNeQCrrfnNOXX5zRo8Frvl07z1x2Cu9O\nGc1weyc1Xxv3W9tmVhxh7cD32w9xx9s/csFfFrI7p4QXZm/2Xvt41V5O/8M8vtuWXe95Z734DWe+\nuMDvXE5xBWkPfsbCLfXvV6ql0UCgmpWY8BCiw2pbBiV2srrHLujPNSNSGdW9HYk+01J7JsfU+xkA\nN43pWu/cm0t2+x3nlVRQWFZJcXkVO7Ot7qR5Gw/We55nS87qGuNd+LYyIxeAfy7Y1ui6KdVcaSBQ\nzcoPD09m2SNneo9L7Fk/552S4u0mSrIXl8WGhzDn3vHEhFs9nJ4AEuZ2ce9Zfer97KU7cvyO80sr\nueudlQx4/EsWbrVSZr/zQwYlFVWszszzLl7z6PXw55z354VA7XqGULf+F1Itn/4Vq2YlMsxNpE+L\n4HeXDCQpNtxvcZrncWWN9WbcLsZKmz3E7jKqNsavVXE4+/LL+NpuAfyw0/qEX1hWxc/+s4yLXlrE\nPe+tpNonGNQYK8vq/vwy77TVyuoaXlu0w+++k+G9ZRlk+KTiUCqQNBCoZu2yYZ354eHJfmkiEuus\nWPa0EPq0jwOsLpyQw3xSH9Wtrfexp9vJd1vO1LaRfLfN2nLzo5V7efeHDOoa/cxcdh2y3qS/3ZLN\nE5+sZ8GWLGpqDN9szjrhTXjKq6p5YMZqrvjHdyf0c5RqrCYPBCKSKiLzRGSDiKwTkbuaugyqZYsN\nD+Gnp6bx5s9GA/DHKwdzzoAOXDg45SjPhH4pcd7H3ZOsAOAbHG4cneb3Oit25zb4c15euMPvOLe4\nghkrMrnplaXMXLGn0XVpSEGp1R12oKD8KHcqdXIEo0VQBdxnjOkHjAZuF5H+QSiHaqFEhCcuGuDd\nRrNbYjT/uGE4PeoMHHdtF+VNkV373NrHr9w0gsuHdWZyv/bec6f2bOd93D055rCBoK7M3FL25JYC\nDW/ScywKyo6cnlupk63JA4ExZp8xZoX9uBDYAHRq6nKo1icuwtpz2bOm4JtfT+Bv1w33u2dsj0Q+\nu/M05tw7jrTEaP73qsGkJNRusNOvg9Vi6JQQSde2UWzPaty+Bxk5Jd5UGJvtRWx1zVyRycP/XcO6\nvfnU1BjmbTzIJ6v21utKKqizT0NpRTUzV2Tqvs8qYIK6oExE0oChwJIGrk0BpgB06dKlSculWq75\n959BG589l5Niw3nmslOY0CeZsBBXg/sxt4uuHXNwuYSv7hlHQlQof/hiEwC928ew+UD9N/c3bh3J\nxyv3si2riIzcEu+CuAWbs3h76W6Ky6u4bFhn2kaHUVZZ7V3c9uaS3Tx58QAe+2gdAI98uJa59433\nptr23bBn68FCXlm0k7eW7KZzmyhG+nRjeWTmlvDO0gwuGtKR3sew6O2N73fx6IdrWffbs4kO17Wl\nTha0wWIRiQE+AO42xtRrSxtjphtj0o0x6UlJSU1fQNUipSVGEx8Z6nfu2pFd6BAf0WAQAGgT7X9/\n7/axJMdGcNekXvz67D58cdc4Yuu8UYa5XZzeK4k/XjmYnskxbNhXyKb9hYzpbnUtTZ25hqc+28CN\nryzBGFOvZeEJAmC98V//7yV8smovReVV3P/+au+1yS8sYHuWFYRemreVQ0X1xw3eWZrBS/O28uys\njfWu1fXhj3uY+Px8amoM//7WStW9L99K5FdQVqlJ+hwqKIFAREKxgsCbxpiZwSiDUh6eT/J1N7BJ\nbRvF7RN64nIJMRG1gSA2IoRP7zzNezyocwL5pZXsLyjjutFd/GYhrd1TwDebs9h8wFr5/MXdp/Pr\ns/sgAv19Bq437i/kjrd/ZODjX3qnpnrk2QvaFmzOYvhTc5j4/HxvOm6A9faYhG+OJY/NB6wA5XH3\nuyvZnl1MXmklYfbMqvxSq0tr0BNfMebpuUf9fanWJxizhgR4GdhgjHmhqV9fqYa8evMI5t47/rDX\nPV0nN4zuyg8PT/brghncuTblxbkDU7wtj9vGdyetXRT3v7+K77ZlE+ISuifG8IvxPdjw5Dm8dssI\nAO+CuMPZ6PNGDrA9u5jlu2oHsdfttTKwZuSUYowhI6eEtAc/Y9HWbM56cQFn/8k/PQZYKTI8i+Gy\nCisoq7Sm0hY2kxbByow8b5lU4AWjRTAWuAGYKCIr7a/zglAOpbwm9En222+5rmnXDOHM/u159IL+\nRIT6L1br3zGOm8em8eXd43C7hFJ7fcLAjvH88crBZBdV8N6yTM4e2IGwEBculxAR6iY5NoJlj0xm\nyUOT6N7Aa//67D71dnmbds0QAB767xoOFJSxYV8BBwrKSWsXRWllNb/5YDWL7XUQD86s7WJauCWb\nxz5a6z3OLanw7iq3L7+Uvo9+4b3W99FZDXYzHSwsY01mPsYY/jZ/Kx+t9J8mu2l/ITe8vIRin2Cy\n9WAhVce4d3RmbgmX/HURv/1kXb1rBwvK+PPcLfVWfasTE4xZQwuNMWKMGWSMGWJ/fd7U5VDqWAzo\nGM+/bkz325LTw+0SHr9wAH3s2Uqe7TG7J0WT3rUNsXa30mMX1J8lnRgTTnR4CE9dMhDAb0X07RN6\nct+Zvf3uP9/evCevpJJRT8/l3GnfEh3m5pdnWHs8vLcskzy7qycjp9T7vOtfXuKXlXXJ9kPerqFv\nNmf5vUZZZQ3/+KZ+DqWzX1zAhS8tZPb6A/zhi03c9c5Kv+s3v7qUb7dks2ZPPhVVNRwsLGPyCwv4\n+evL6v0s63WqeeLjdWTV2UPCc7xil7XHxNIdOfzsPz9QVV3DAx+s5oXZm3nnhwzeWbq73s9Ux0en\nCih1kj1/5WD+PHcLvZJjEbFgFiz7AAAQhklEQVRmIYW4XIfdwxmgY0IkYO3ZMOfesWTYaxKuHpFK\neKiLe95d5b1+Zv/2zPbZx/nsgR2Y3L+9NeoGzNlQmzhvYKc41u6pv67h+a9qM6/O35RV7zrA+r0F\nXPuv77l6RCp3TOzpTb731/n1g0RNjWGvPej8zKyN7Mkt9W5HOm9TFmWV1cxYnsnEvsneur63LIPX\nvttJeIiLqef18/4sz+ZCnjUfV/1zMWDtTueZWvvQf9d4fz++qcr35JWyaGs2V6X7b42qjkxTTCh1\nko1Ia8sbt47yth5S4iOPGATACgSdEiJ55rJT6Jkcy4Q+yYC1eO7SoZ397v3bdcNY+9uzeeg86432\n0qGdaBsdxuKpEwH/5Hrje1sz7jrERfCTUV1Y/sjkRtfj8r9/R35pJdMXbPebPrsqI88uG1RU1VBd\nY9ieXex3PbuonPvfr90LYsuBIh75cC3Xv1w7U9yz8M7lM0ifVVjObW8st86L+I0T5BRX1BvQzy+t\nJKe4wrvG4qp/LOaBGat19tMx0haBUs1AWIiLRQ9OPOz1xVMn4llPFup2Eep2MWVcDy4Z0onkOGtB\nXIe4CDq3iSQzt7ZL6MrhqezJLeWuyb39ZjN5nNYzkZ+P687UD1Z7P9GDNYDt+2a6qc6ANYAx1oyl\nqTPX1FtNHR7ioryqdmzAc317VjFr9+SzZk8+by+18jh5ZkC9tmgHT3yy3vuc6hrjHQgHa7e5nYf8\nZ0at2J3LLa8t4/EL+3PF8M7sybPqnltc4R2E35tXysHCchZuySIxJpxrRja8Lun2t1bQIymGe+t0\nx1l1NQ1uktRaaCBQqgVIiY9s8LwnCIDVerhncm/un7GK6DDrjbxruyj+dM3Qw/7cwanxjO+dRKc2\nkX6B4OHz+zF15hrv8bOzNhAXEUJBmRUcrhjemRnLM7nkr4v8fl5q20gyckr55Rk9iQ53szevjFcW\n7WDD/tpAccFfFvo9Z+aKPazOzPduK+qx6UAh7yytTfrnu9ucx5LtVutn1pr9fLF2v/d8Xkkloe4y\nlu3K4alPN7DfZ7rtNSO7MGf9AT5etZf7zupNh/gIfjNjNZ+t3gfA7RN6+O2Wd9sby8grqeTd28Yc\n7tfYaG8v3U2/lDiGpPpvrvTWkt0kxoQxsW/yYRMmBpIGAqVakcuHd+b8QSkYA4VllQ1+il3w6wn8\n8q3lrN1TQFt7VfWLVw/h9cW7mL7AWmTm6VLyKCirYmS3tt5up7sm9aK4vIpZa/cT5nbx7W8mkJFT\nwsqMPL5ad4DzB3WgZ3IsGTklViBoIP/S4NQEEqPDmLvxYL0g4PH+8kzaRYdxqLiiwev/tMtrMCzd\nUTuldunOHP48d4vfKm2Pkooqnp61ge1ZxXRLjKZncgwfrtzrvf7G4l10T4pmYt/2GGP4cp01HnPD\ny0u4YFAKlw7tTFiIi2lztrA1q4hpVw/B5RKyCst57KO1jO+dxKDOCXRpF+U3NfhgQRlTZ64hLiKE\n1U+c7T1fWlHNk5+uo6yyBrdL2PZ000+i1ECgVCvjmd4aeZg9Gbq0i+K5ywexMiOPy4dZ4w+d20Tx\n0Hn9uHpEKlsOFNExIZJVj58FwF3v/Mj8TVn0So4hMtTNN5uzSG0bxVUjUpm1dj9VNTW0j4ugfVwE\n6Wlt+dnp3b2v5Umb8f322gAybe4W7pjYk/vO6sPcDQcor6ph0bZsfFMprX7iLCb97zdkFZZzas9E\n7x7VAKO7t2V7VrF3UBlq95MY1a0tS3bk8LtPrS4ml1j7SPhaviuXnfaYxrS5W+r87lw89dkGAD6/\n83QSompXnS/cms23W7KZvf4A/7oxnRfnWAPu6V3bkBAVyvp9Bcxau59tWUVsPlDEyLS2PHfFIDbt\nL+ScgR34yA42njGRvXmlRIeHsHjbIe9GR9U1hrLKaiJC3VRU1WAwfq2TQNFAoJQDDegYz4CO8fXO\n90iKoUeSlcXVk6rjtZtHMmf9AUZ1b0tEqNu7Cc8pnaznd24TddjXqRuM7pzUi85tIrloSEcAJvVr\nz6R+7Xlh9mb+bL8pD05NIC4ilEuHdmL6gu10blPbLfbST4Z6p9B2m1p/1vl1o7uyxG61PHnxAJbt\nzOVjO4h8ePtYrvrnYp74eB01BtpEhXpnQnm8M2WMt7vrxTmb6WVntH3vtjEM79qGn766lDkbDnLP\nu7VTZx//2H+9wxa7dbN0Zw4Tnp8PWGNAldW1u9rV1BhOffZroP6Cwl2HSujTIZZvNmfxq7dW8N9f\njqV/xzgCSQOBUuqoJvevTdXtWU+XGBPOU5cMZGzPxCM+t3tStDfXktslXNnA1E7P3tPDuiTw+q2j\nALh4SEemL9jOxUM6Eh8ZSmFZJecOrN2y9LsHJ3rfTAEeOb8fZw+oLecNo7t6B86njOvOkNQErh/V\nlVcW7cDtEmb84lRu+PcS9uaX8egF/bllbBoiwuonzuI3M1Yza+1+7zTdvimxuF3CFcM78+2WbL+u\npLoaShJbYQ+cD+ocz+rMfE57rrbcReVVtI8L9+4/ceU/viMhKozyqmoMtftmBJIGAqXUcbt+dNej\n3jPnnvHsySulsOzwUzrP6t+eCwd35O7JvbyfkAd0jGfns+cD0LdD/U/EHRMi+fHRM6kxhujwkHor\nvkXEmyzQZQePqef1JSU+guFpbeiRFMOrN4/krnd+5MJBtQEmLiKUy4Z1ZpbP4LMnxfmINP/sr9Fh\nboorqpl2zRCmzdnC5P7tveMsDfnJyC6szlzjNzAPVkukusYw+YVvKCir8g7KJ8aE16tXIOg6AqVU\nQLlcQmrbqCN2b0SEuvnLtUO93VKN1SY6jHZ13ixjw0M4y27BeGbg+HbL/Hxcd4Z1sTYs6tMhli/u\nHuc3+wpgcr9k/nH9MMDam8KjY0Ikr98y0nvcId563shubfn6/jP41cSe3msz/mcMb/5slPf4ulFd\nOKVzbXecZ0Okh8/r5x20jgi1yjv13L707RDLXZN7HdPv43hpi0Ap1aqs+e3Z3gVmnnGMQZ3rj4cc\niYhwzsAUHr+wP6f38u/6Guwz9XP6jenMWX+ADnYgiYsIpWN8BHvzyxjQMZ7dObXrHn5/6SnegATw\n/JWDmLE8kxvG1Laq0tpFs3F/IcO6tuG28T2OqcwnQgOBUqrV8XTznNYrka/vG9/gYrrGuHlst3rn\nPIPofTvEWoPr4/1bMb3ax5JfWklkmLveivJQnzUCCVFhfjOsAF76yTD+Nn/rMQeuE6WBQCnVqnU/\nxu6mxvj2gQnER4U2eO3qEan0s/eaSLCDhu/MoDduHdng+gawBs1fuGrISS7t0UlL2Ac1PT3dLFvW\ncAZDpZRqzj5fs4/+KXFHTHMeKCKy3BiTfrT7tEWglFIBdJ697qE501lDSinlcBoIlFLK4TQQKKWU\nw2kgUEoph9NAoJRSDqeBQCmlHE4DgVJKOZwGAqWUcjgNBEop5XAaCJRSyuE0ECillMNpIFBKKYfT\nQKCUUg6ngUAppRxOA4FSSjmcBgKllHI4DQRKKeVwGgiUUsrhNBAopZTDaSBQSimHC0ogEJFzRGST\niGwVkQeDUQallFKWJg8EIuIG/gqcC/QHrhWR/k1dDqWUUpZgtAhGAluNMduNMRXAO8DFQSiHUkop\nICQIr9kJyPA5zgRG1b1JRKYAU+zDIhHZdJyvlwhkH+dzWyqtszNonVu/E61v18bcFIxAIA2cM/VO\nGDMdmH7CLyayzBiTfqI/pyXROjuD1rn1a6r6BqNrKBNI9TnuDOwNQjmUUkoRnEDwA9BLRLqJSBhw\nDfBxEMqhlFKKIHQNGWOqRORXwJeAG3jFGLMugC95wt1LLZDW2Rm0zq1fk9RXjKnXPa+UUspBdGWx\nUko5nAYCpZRyuFYdCFprKgsReUVEDorIWp9zbUVktohssb+3sc+LiPzZ/h2sFpFhwSv58RGRVBGZ\nJyIbRGSdiNxln2/NdY4QkaUissqu82/t891EZIld53ftCReISLh9vNW+nhbM8p8IEXGLyI8i8ql9\n3KrrLCI7RWSNiKwUkWX2uSb92261gaCVp7J4DTinzrkHgbnGmF7AXPsYrPr3sr+mAH9vojKeTFXA\nfcaYfsBo4Hb737I117kcmGiMGQwMAc4RkdHAc8CLdp1zgVvt+28Fco0xPYEX7ftaqruADT7HTqjz\nBGPMEJ81A037t22MaZVfwBjgS5/jqcDUYJfrJNYvDVjrc7wJSLEfpwCb7Mf/BK5t6L6W+gV8BJzp\nlDoDUcAKrBX42UCIfd77N441C2+M/TjEvk+CXfbjqGtnrDe+icCnWAtQW3uddwKJdc416d92q20R\n0HAqi05BKktTaG+M2Qdgf0+2z7eq34Pd/B8KLKGV19nuIlkJHARmA9uAPGNMlX2Lb728dbav5wPt\nmrbEJ8WfgAeAGvu4Ha2/zgb4SkSW26l1oIn/toORYqKpNCqVhQO0mt+DiMQAHwB3G2MKRBqqmnVr\nA+daXJ2NMdXAEBFJAP4L9GvoNvt7i6+ziFwAHDTGLBeRMzynG7i11dTZNtYYs1dEkoHZIrLxCPcG\npM6tuUXgtFQWB0QkBcD+ftA+3yp+DyISihUE3jTGzLRPt+o6exhj8oD5WOMjCSLi+QDnWy9vne3r\n8UBO05b0hI0FLhKRnVhZiSditRBac50xxuy1vx/ECvgjaeK/7dYcCJyWyuJj4Cb78U1Y/eie8zfa\nsw1GA/meJmdLIdZH/5eBDcaYF3wuteY6J9ktAUQkEpiMNYA6D7jCvq1unT2/iyuAr43didxSGGOm\nGmM6G2PSsP6/fm2MuY5WXGcRiRaRWM9j4CxgLU39tx3sgZIAD8KcB2zG6lt9ONjlOYn1ehvYB1Ri\nfUK4FatvdC6wxf7e1r5XsGZPbQPWAOnBLv9x1Pc0rObvamCl/XVeK6/zIOBHu85rgcfs892BpcBW\n4H0g3D4fYR9vta93D3YdTrD+ZwCftvY623VbZX+t87xPNfXftqaYUEoph2vNXUNKKaUaQQOBUko5\nnAYCpZRyOA0ESinlcBoIlFLK4TQQKOVDRB62s32utrNBjhKRu0UkKthlUypQdPqoUjYRGQO8AJxh\njCkXkUQgDPgOa752dlALqFSAaItAqVopQLYxphzAfuO/AugIzBOReQAicpaILBaRFSLyvp0DyZNX\n/jl7H4GlItLTPn+liKy19xZYEJyqKXV42iJQyma/oS/ESvs8B3jXGPONnfsm3RiTbbcSZgLnGmOK\nReQ3WCtdn7Tv+5cx5vciciNwlTHmAhFZA5xjjNkjIgnGyh2kVLOhLQKlbMaYImA41oYfWcC7IvLT\nOreNxtroaJGdIvomoKvP9bd9vo+xHy8CXhORnwPuwJReqePXmtNQK3XMjJX6eT4w3/4kf1OdWwSY\nbYy59nA/ou5jY8z/iMgo4HxgpYgMMcYcOrklV+r4aYtAKZuI9BGRXj6nhgC7gEIg1j73PTDWp/8/\nSkR6+zznap/vi+17ehhjlhhjHsPaRcs3jbBSQactAqVqxQB/sdM/V2FltZwCXAvMEpF9xpgJdnfR\n2yISbj/vEawstwDhIrIE60OWp9XwRzvACFYmyVVNUhulGkkHi5U6SXwHlYNdFqWOhXYNKaWUw2mL\nQCmlHE5bBEop5XAaCJRSyuE0ECillMNpIFBKKYfTQKCUUg73/y5M5funH1TcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2b5592b8588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot losses\n",
    "\n",
    "plt.plot(losses)\n",
    "plt.xlabel('Steps')\n",
    "plt.ylabel('Losses')\n",
    "plt.ylim((0, 10))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from checkpoints\\-2499\n",
      "1.\n",
      "--------------------------------\n",
      "English sentence:  ﻿Why are no-smoking areas not enforced?\n",
      "\n",
      "Expected translation:  ﻿Warum wird in den Nichtraucherzonen das Rauchverbot nicht durchgesetzt?\n",
      "\n",
      "Generated output: \n",
      "Die <ukn> sind nicht <ukn> \n",
      "Bleu score:  0.7801157731069053\n",
      "\n",
      "--------------------------------\n",
      "2.\n",
      "--------------------------------\n",
      "English sentence:  That was the decision.\n",
      "\n",
      "Expected translation:  Das war der Beschluß.\n",
      "\n",
      "Generated output: \n",
      "Das war die <ukn> \n",
      "Bleu score:  0.8408964152537145\n",
      "\n",
      "--------------------------------\n",
      "3.\n",
      "--------------------------------\n",
      "English sentence:  All of the others were of a different opinion.\n",
      "\n",
      "Expected translation:  Alle anderen waren anderer Meinung.\n",
      "\n",
      "Generated output: \n",
      "Alle anderen der anderen waren ein anderes \n",
      "Bleu score:  0.6944370541455951\n",
      "\n",
      "--------------------------------\n",
      "4.\n",
      "--------------------------------\n",
      "English sentence:  This gives them a competitive edge for the interim period.\n",
      "\n",
      "Expected translation:  Dadurch erwachsen ihnen vorübergehend Wettbewerbsvorteile.\n",
      "\n",
      "Generated output: \n",
      "Diese <ukn> ihnen für die <ukn> für die Logik \n",
      "Bleu score:  0.7291155827927387\n",
      "\n",
      "--------------------------------\n",
      "5.\n",
      "--------------------------------\n",
      "English sentence:  But I would like to say that safety is a priority objective for the Commission.\n",
      "\n",
      "Expected translation:  Ich will zum Ausdruck bringen, daß die Sicherheit ein vorrangiges Ziel der Kommission ist.\n",
      "\n",
      "Generated output: \n",
      "Aber ich möchte sagen dass Sicherheit ein Priorität für die <ukn> ist \n",
      "Bleu score:  0.7217950347929304\n",
      "\n",
      "--------------------------------\n",
      "6.\n",
      "--------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\nltk\\translate\\bleu_score.py:490: UserWarning: \n",
      "Corpus/Sentence contains 0 counts of 2-gram overlaps.\n",
      "BLEU scores might be undesirable; use SmoothingFunction().\n",
      "  warnings.warn(_msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English sentence:  In addition, the need for greater transparency has been pointed out.\n",
      "\n",
      "Expected translation:  Darüber hinaus wird auf die Notwendigkeit einer verstärkten Transparenz hingewiesen.\n",
      "\n",
      "Generated output: \n",
      "Im Dezember <ukn> ist mehr Transparenz <ukn> <ukn> \n",
      "Bleu score:  0.748406329216425\n",
      "\n",
      "--------------------------------\n",
      "7.\n",
      "--------------------------------\n",
      "English sentence:  Transport of dangerous goods by road\n",
      "\n",
      "Expected translation:  Sicherheitsberater für den Gefahrguttransport\n",
      "\n",
      "Generated output: \n",
      "Die gefährlichen von Straße \n",
      "Bleu score:  0.8254605646966103\n",
      "\n",
      "--------------------------------\n",
      "8.\n",
      "--------------------------------\n",
      "English sentence:  Some speakers have already mentioned unemployment and the fall in population.\n",
      "\n",
      "Expected translation:  Einige Kollegen haben bereits über die Arbeitslosigkeit und den Bevölkerungsrückgang gesprochen.\n",
      "\n",
      "Generated output: \n",
      "Einige Redner haben Arbeitslosigkeit und die <ukn> erwähnt \n",
      "Bleu score:  0.7431990115298822\n",
      "\n",
      "--------------------------------\n",
      "9.\n",
      "--------------------------------\n",
      "English sentence:  On this occasion we have had six minutes of questions from the floor of the House and we have had 29 minutes of response by the Commissioner, and his statement.\n",
      "\n",
      "Expected translation:  In diesem Fall standen uns sechs Minuten für Fragen des Hauses und 29 Minuten für die Antworten des Kommissionsmitglieds sowie für seine Erklärung zur Verfügung.\n",
      "\n",
      "Generated output: \n",
      "Am Ende des Parlaments haben wir sechs Minuten von den Fragen des Parlaments und der Antwort des Parlaments über die Antwort des Parlaments und seiner <ukn> \n",
      "Bleu score:  0.6186671349661511\n",
      "\n",
      "--------------------------------\n",
      "10.\n",
      "--------------------------------\n",
      "English sentence:  This plan is designed to increase its energy potential by 2%.\n",
      "\n",
      "Expected translation:  Lassen Sie mich jedoch folgendes ergänzen: Mit dieser Anlage will die Türkei ihr Energiepotential um lediglich 2 % steigern.\n",
      "\n",
      "Generated output: \n",
      "Diese Plan ist die <ukn> von den Potenzial von <ukn> \n",
      "Bleu score:  0.7037475090852178\n",
      "\n",
      "--------------------------------\n",
      "11.\n",
      "--------------------------------\n",
      "English sentence:  At the same time, it also wishes to acquire Cadou type reactors from Canada, which we now know have already been used for nuclear weapons in Pakistan and India.\n",
      "\n",
      "Expected translation:  Allerdings strebt sie angeblich an, Candu-Reaktoren zu erwerben. Pakistan und Indien verfügen über solche Reaktoren und sind auf diesem Wege an Atomwaffen gelangt.\n",
      "\n",
      "Generated output: \n",
      "Gleichzeitig haben wir auch <ukn> <ukn> <ukn> <ukn> <ukn> die wir bereits bereits in Pakistan bereits mit der Kernenergie und <ukn> verwendet wurden und <ukn> \n",
      "Bleu score:  0.6028448325695556\n",
      "\n",
      "--------------------------------\n",
      "12.\n",
      "--------------------------------\n",
      "English sentence:  The EQUAL initiative is not restricted to certain groups of people.\n",
      "\n",
      "Expected translation:  Die Initiative EQUAL ist nicht auf bestimmte Personengruppen beschränkt.\n",
      "\n",
      "Generated output: \n",
      "Die derzeitige Initiative ist nicht in bestimmte Gruppen \n",
      "Bleu score:  0.749634235443537\n",
      "\n",
      "--------------------------------\n",
      "13.\n",
      "--------------------------------\n",
      "English sentence:  However, for people with various kinds of physical disability who are in need of special transport and personal assistance, freedom of movement is still highly restricted.\n",
      "\n",
      "Expected translation:  Für Personen mit verschiedenen körperlichen Behinderungen, die besondere Beförderungsmittel und persönliche Hilfe benötigen, ist diese Freizügigkeit jedoch weiterhin sehr eingeschränkt.\n",
      "\n",
      "Generated output: \n",
      "<ukn> Personen mit Personen mit Personen die in der Notwendigkeit von der besonderen Verkehr und persönlichen <ukn> ist noch immer noch immer noch hoch \n",
      "Bleu score:  0.616800579706554\n",
      "\n",
      "--------------------------------\n",
      "14.\n",
      "--------------------------------\n",
      "English sentence:  This also applies, for example, to the very real projects which the Committee on Industry has introduced.\n",
      "\n",
      "Expected translation:  Gleiches gilt beispielsweise für ganz konkrete Projekte, die der Industrieausschuß zur Sprache gebracht hat.\n",
      "\n",
      "Generated output: \n",
      "Diese ebenfalls <ukn> <ukn> für den sehr echte Projekte die im Ausschuß für Industrie <ukn> <ukn> hat \n",
      "Bleu score:  0.6814839425935316\n",
      "\n",
      "--------------------------------\n",
      "15.\n",
      "--------------------------------\n",
      "English sentence:  We also need to recognise that people who live in glasshouses should not throw stones.\n",
      "Expected translation:  Ebenso sollten wir uns aber auch der Tatsache bewußt sein, daß jemand, der im Glashaus sitzt, nicht mit Steinen werfen darf.\n",
      "Generated output: \n",
      "Wir müssen auch darauf hingewiesen dass Personen die in der europäischen Bevölkerung leben \n",
      "Bleu score:  0.6574335789673229\n",
      "\n",
      "--------------------------------\n",
      "\n",
      "Average BLEU score is 0.714269171924\n"
     ]
    }
   ],
   "source": [
    "# let's test the model\n",
    "gen_out = ''\n",
    "total_bleu_score = []\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    \n",
    "    # placeholders\n",
    "    encoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'encoder{}'.format(i)) for i in range(input_seq_len)]\n",
    "    decoder_inputs = [tf.placeholder(dtype = tf.int32, shape = [None], name = 'decoder{}'.format(i)) for i in range(output_seq_len)]\n",
    "\n",
    "    # output projection\n",
    "    size = 512\n",
    "    w_t = tf.get_variable('proj_w', [de_vocab_size, size], tf.float32)\n",
    "    b = tf.get_variable('proj_b', [de_vocab_size], tf.float32)\n",
    "    w = tf.transpose(w_t)\n",
    "    output_projection = (w, b)\n",
    "    \n",
    "    # change the model so that output at time t can be fed as input at time t+1\n",
    "    outputs, states = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(\n",
    "                                                encoder_inputs,\n",
    "                                                decoder_inputs,\n",
    "                                                tf.contrib.rnn.BasicLSTMCell(size),\n",
    "                                                num_encoder_symbols = en_vocab_size,\n",
    "                                                num_decoder_symbols = de_vocab_size,\n",
    "                                                embedding_size = 100,\n",
    "                                                feed_previous = True, # <-----this is changed----->\n",
    "                                                output_projection = output_projection,\n",
    "                                                dtype = tf.float32)\n",
    "    \n",
    "    # ops for projecting outputs\n",
    "    outputs_proj = [tf.matmul(outputs[i], output_projection[0]) + output_projection[1] for i in range(output_seq_len)]\n",
    "\n",
    "    # let's translate these sentences     \n",
    "    en_sentences = read_sentences('C:\\\\Users\\\\vatsa\\\\Downloads\\\\CS533\\\\Project\\\\test_data\\\\testen.txt')\n",
    "    de_sentences = read_sentences('C:\\\\Users\\\\vatsa\\\\Downloads\\\\CS533\\\\Project\\\\test_data\\\\testde.txt')\n",
    "    en_sentences_encoded = [[en_word2idx.get(word, 0) for word in en_sentence.split()] for en_sentence in en_sentences]\n",
    "    \n",
    "    # padding to fit encoder input\n",
    "    for i in range(len(en_sentences_encoded)):\n",
    "        en_sentences_encoded[i] += (40 - len(en_sentences_encoded[i])) * [en_word2idx['<pad>']]\n",
    "    \n",
    "    # restore all variables - use the last checkpoint saved\n",
    "    saver = tf.train.Saver()\n",
    "    path = tf.train.latest_checkpoint('checkpoints')\n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        # restore\n",
    "        saver.restore(sess, path)\n",
    "        \n",
    "        # feed data into placeholders\n",
    "        feed = {}\n",
    "        for i in range(input_seq_len):\n",
    "            feed[encoder_inputs[i].name] = np.array([en_sentences_encoded[j][i] for j in range(len(en_sentences_encoded))], dtype = np.int32)\n",
    "            \n",
    "        feed[decoder_inputs[0].name] = np.array([de_word2idx['<go>']] * len(en_sentences_encoded), dtype = np.int32)\n",
    "        \n",
    "        # translate\n",
    "        output_sequences = sess.run(outputs_proj, feed_dict = feed)\n",
    "        \n",
    "        # decode seq.\n",
    "        for i in range(len(en_sentences_encoded)):\n",
    "            print ('{}.\\n--------------------------------'.format(i+1))\n",
    "            ouput_seq = [output_sequences[j][i] for j in range(output_seq_len)]\n",
    "            #decode output sequence\n",
    "            words = decode_output(ouput_seq)\n",
    "            print ('English sentence: ', en_sentences[i])\n",
    "            print('Expected translation: ', de_sentences[i])\n",
    "            print('Generated output: ')\n",
    "            for k in range(len(words)):\n",
    "                if words[k] not in ['<eos>', '<pad>', '<go>']:\n",
    "                    print (words[k], end=\" \")\n",
    "                    gen_out += ' ' + words[k]\n",
    "            score = sentence_bleu(de_sentences[i], gen_out) \n",
    "            print('\\nBleu score: ', score)\n",
    "            total_bleu_score.append(score)\n",
    "            gen_out = ''\n",
    "            print ('\\n--------------------------------')\n",
    "\n",
    "print('\\nAverage BLEU score is', np.mean(total_bleu_score))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}