NMT English to Dutch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "04461dd6",
   "metadata": {},
   "source": [
    "# Setting up the environment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db8ba617",
   "metadata": {},
   "source": [
    "## Importing relevant libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e2a7aa57",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.556988Z",
     "start_time": "2022-08-23T18:35:14.135460Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init Plugin\n",
      "Init Graph Optimizer\n",
      "Init Kernel\n"
     ]
    }
   ],
   "source": [
    "#__future__ to bring features from newer versions of Python \n",
    "# from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow_datasets as tfds\n",
    "\n",
    "import string\n",
    "from string import digits\n",
    "import re\n",
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import collections\n",
    "import unicodedata\n",
    "\n",
    "import os\n",
    "\n",
    "from bert import BertModelLayer\n",
    "from bert.loader import StockBertConfig, load_stock_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7677532b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.563053Z",
     "start_time": "2022-08-23T18:35:16.557916Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /var/folders/jd/ll0by25n4ln09rznyqbztdyw0000gn/T/ipykernel_90191/3505786474.py:2: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.config.list_physical_devices('GPU')` instead.\n",
      "GPU Available: Metal device set to: Apple M1\n",
      " True\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-24 00:05:16.559618: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2022-08-24 00:05:16.559716: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    }
   ],
   "source": [
    "tf.config.list_physical_devices('GPU')\n",
    "print(\"GPU Available: \", tf.test.is_gpu_available())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "660fd8cb",
   "metadata": {},
   "source": [
    "## Cleaning the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "91660001",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.571882Z",
     "start_time": "2022-08-23T18:35:16.563654Z"
    }
   },
   "outputs": [],
   "source": [
    "#reading data from NLD-ENG translation dataset\n",
    "with open('nld-eng/nld.txt','r') as f:\n",
    "  data = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c1de78cd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.589901Z",
     "start_time": "2022-08-23T18:35:16.573091Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "68954"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#removing trailing and leading characters\n",
    "#creating list of word translations by splitting on new line\n",
    "uncleaned_data_list = data.strip().split('\\n')\n",
    "len(uncleaned_data_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c4588a6d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.592724Z",
     "start_time": "2022-08-23T18:35:16.590616Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hi.\tHai!\tCC-BY 2.0 (France) Attribution: tatoeba.org #538123 (CM) & #6117420 (Raizin)\n"
     ]
    }
   ],
   "source": [
    "print(uncleaned_data_list[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b7b62070",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.596678Z",
     "start_time": "2022-08-23T18:35:16.593525Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Go.', 'Lopen!', 'CC-BY 2.0 (France) Attribution: tatoeba.org #2877272 (CM) & #7764436 (LinguisticFusion)']\n"
     ]
    }
   ],
   "source": [
    "for word in uncleaned_data_list:\n",
    "    print(word.split('\\t'))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c47d50e9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.599249Z",
     "start_time": "2022-08-23T18:35:16.597427Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Go.', 'Lopen!']\n"
     ]
    }
   ],
   "source": [
    "for word in uncleaned_data_list:\n",
    "    print(word.split('\\t')[:-1])\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "97a0cbeb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.638075Z",
     "start_time": "2022-08-23T18:35:16.599878Z"
    }
   },
   "outputs": [],
   "source": [
    "#separating english and dutch words \n",
    "eng_word = []\n",
    "nld_word = []\n",
    "for word in uncleaned_data_list:\n",
    "    eng_word.append(word.split('\\t')[:-1][0])\n",
    "    nld_word.append(word.split('\\t')[:-1][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e2668d3b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.656135Z",
     "start_time": "2022-08-23T18:35:16.638812Z"
    }
   },
   "outputs": [],
   "source": [
    "#creating pandas df with english words and their dutch equivalents\n",
    "data = pd.DataFrame(columns=['English','Dutch'])\n",
    "data['English'] = eng_word\n",
    "data['Dutch'] = nld_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e2c54576",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.735737Z",
     "start_time": "2022-08-23T18:35:16.658022Z"
    }
   },
   "outputs": [],
   "source": [
    "data.to_csv('data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ee046ab4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.792051Z",
     "start_time": "2022-08-23T18:35:16.736804Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>English</th>\n",
       "      <th>Dutch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Go.</td>\n",
       "      <td>Lopen!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Go.</td>\n",
       "      <td>Vooruit.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Hi.</td>\n",
       "      <td>Hoi.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Hi.</td>\n",
       "      <td>Hé!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Hi.</td>\n",
       "      <td>Hai!</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  English     Dutch\n",
       "0     Go.    Lopen!\n",
       "1     Go.  Vooruit.\n",
       "2     Hi.      Hoi.\n",
       "3     Hi.       Hé!\n",
       "4     Hi.      Hai!"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('data.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6fc55a44",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.795217Z",
     "start_time": "2022-08-23T18:35:16.792639Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"If someone who doesn't know your background says that you sound like a native speaker, it means they probably noticed something about your speaking that made them realize you weren't a native speaker. In other words, you don't really sound like a native speaker.\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eng_word[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5eaeb871",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.801894Z",
     "start_time": "2022-08-23T18:35:16.795913Z"
    }
   },
   "outputs": [],
   "source": [
    "#80% of data used for training, and 20% for testing the models\n",
    "# Shuffle dataset \n",
    "shuffle_df = data.sample(frac=1)\n",
    "\n",
    "# Define a size for train set \n",
    "train_size = int(0.8 * len(data))\n",
    "\n",
    "# Split dataset \n",
    "train_ex = shuffle_df[:train_size]\n",
    "test_ex = shuffle_df[train_size:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "87ff19c2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.804487Z",
     "start_time": "2022-08-23T18:35:16.802589Z"
    }
   },
   "outputs": [],
   "source": [
    "eng_train = train_ex['English'].values\n",
    "nld_train = train_ex['Dutch'].values\n",
    "eng_test = test_ex['English'].values\n",
    "nld_test = test_ex['Dutch'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "68d2dc6c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.817934Z",
     "start_time": "2022-08-23T18:35:16.805046Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-08-24 00:05:16.805982: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:305] Could not identify NUMA node of platform GPU ID 0, defaulting to 0. Your kernel may not have been built with NUMA support.\n",
      "2022-08-24 00:05:16.806001: I tensorflow/core/common_runtime/pluggable_device/pluggable_device_factory.cc:271] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 0 MB memory) -> physical PluggableDevice (device: 0, name: METAL, pci bus id: <undefined>)\n"
     ]
    }
   ],
   "source": [
    "#we get the slices of the arrays as objects by using tf.data.Dataset.from_tensor_slices() \n",
    "train_ex = tf.data.Dataset.from_tensor_slices((eng_train,nld_train))\n",
    "test_ex = tf.data.Dataset.from_tensor_slices((eng_test,nld_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4213d4ca",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.823104Z",
     "start_time": "2022-08-23T18:35:16.818689Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Let's go out.\n",
      "Laten we uitgaan.\n"
     ]
    }
   ],
   "source": [
    "#tf.compat.as_text() converts any string-like python input types to unicode.\n",
    "for en, nld in train_ex.take(1):\n",
    "  print(tf.compat.as_text(en.numpy()))\n",
    "  print(tf.compat.as_text(nld.numpy()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc0530af",
   "metadata": {},
   "source": [
    "# Preprocessing and Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "260e9c60",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.825558Z",
     "start_time": "2022-08-23T18:35:16.823772Z"
    }
   },
   "outputs": [],
   "source": [
    "def to_unicode(text):\n",
    "    #strings are stored as unicode in Python 3.0 and above\n",
    "    if isinstance(text,str):\n",
    "        return text\n",
    "    #conversion of bytes to unicode\n",
    "    elif isinstance(text,bytes):\n",
    "        return text.decode('utf-8','ignore')\n",
    "    else:\n",
    "        raise ValueError(\"Unsupported string type: %s\" % (type(text)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "31b2402c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.827997Z",
     "start_time": "2022-08-23T18:35:16.826064Z"
    }
   },
   "outputs": [],
   "source": [
    "#load vocab file into a dictionary\n",
    "def load_dict(vocab_file):\n",
    "    vocab = collections.OrderedDict()\n",
    "    index = 0\n",
    "    with open(vocab_file, \"r\") as reader:\n",
    "        while True:\n",
    "            token = to_unicode(reader.readline())\n",
    "            if not token:\n",
    "                break\n",
    "            token = token.strip()\n",
    "            vocab[token] = index\n",
    "            index += 1\n",
    "    return vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d3369206",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.830152Z",
     "start_time": "2022-08-23T18:35:16.828644Z"
    }
   },
   "outputs": [],
   "source": [
    "# removes whitespace from text and returns tokens\n",
    "def remove_whitespace(text):\n",
    "    text = text.strip()\n",
    "    if not text:\n",
    "        return []\n",
    "    token = text.split()\n",
    "    return token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1d765b03",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.832169Z",
     "start_time": "2022-08-23T18:35:16.830645Z"
    }
   },
   "outputs": [],
   "source": [
    "def convert_vocab(vocab,items):\n",
    "    output = []\n",
    "    for item in items:\n",
    "        output.append(vocab[item])\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "30799d03",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.835529Z",
     "start_time": "2022-08-23T18:35:16.832729Z"
    }
   },
   "outputs": [],
   "source": [
    "# for end-to-end tokenization\n",
    "class FullTokenizer(object):\n",
    "    def __init__(self, vocab_file, do_lower_case=True):\n",
    "        self.vocab = load_dict(vocab_file)\n",
    "        # map IDs to tokens\n",
    "        self.invert_vocab = {v: k for k, v in self.vocab.items()}\n",
    "        # basic tokenizer to break text into tokens based on whitespace\n",
    "        self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)\n",
    "        # Wordpiece tokenizer to generate sub-tokens out of the tokens from BasicTokenizer\n",
    "        self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab)\n",
    "    \n",
    "    # uses Wordpiece to tokenize tokens generated by the BasicTokenizer\n",
    "    def tokenize(self, text):\n",
    "        split_tokens = []\n",
    "        for token in self.basic_tokenizer.tokenize(text):\n",
    "            for sub_token in self.wordpiece_tokenizer.tokenize(token):\n",
    "                split_tokens.append(sub_token)\n",
    "        return split_tokens\n",
    "    \n",
    "    # after being provided the tokens, outputs the IDs of the tokens\n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return convert_vocab(self.vocab, tokens)\n",
    "    \n",
    "    # after being provided the IDs, outputs the tokens mapped to the IDs\n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return convert_vocab(self.invert_vocab, ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3828743f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.842086Z",
     "start_time": "2022-08-23T18:35:16.836163Z"
    }
   },
   "outputs": [],
   "source": [
    "# tokenizes based on whitespace and punctuation, also lower-cases the tokens\n",
    "class BasicTokenizer(object):\n",
    "    \n",
    "    # whether or not to lower-case the tokens\n",
    "    def __init__(self, do_lower_case=True):\n",
    "        self.do_lower_case = do_lower_case\n",
    "\n",
    "    # tokenizes the text\n",
    "    def tokenize(self, text):\n",
    "        text = to_unicode(text)\n",
    "        text = self.text_clean(text)\n",
    "        text = self._tokenize_chinese_chars(text)\n",
    "\n",
    "        orig_tokens = remove_whitespace(text)\n",
    "        split_tokens = []\n",
    "        for token in orig_tokens:\n",
    "            if self.do_lower_case:\n",
    "                token = token.lower()\n",
    "                token = self._run_strip_accents(token)\n",
    "            split_tokens.extend(self._run_split_on_punc(token))\n",
    "\n",
    "        output_tokens = remove_whitespace(\" \".join(split_tokens))\n",
    "        return output_tokens\n",
    "\n",
    "    def _run_strip_accents(self, text):\n",
    "        \"\"\"Strips accents from a piece of text.\"\"\"\n",
    "        text = unicodedata.normalize(\"NFD\", text)\n",
    "        output = []\n",
    "        for char in text:\n",
    "            cat = unicodedata.category(char)\n",
    "            if cat == \"Mn\":\n",
    "                continue\n",
    "            output.append(char)\n",
    "        return \"\".join(output)\n",
    "\n",
    "    def _run_split_on_punc(self, text):\n",
    "        \"\"\"Splits punctuation on a piece of text.\"\"\"\n",
    "        chars = list(text)\n",
    "        i = 0\n",
    "        start_new_word = True\n",
    "        output = []\n",
    "        while i < len(chars):\n",
    "            char = chars[i]\n",
    "            if _is_punctuation(char):\n",
    "                output.append([char])\n",
    "                start_new_word = True\n",
    "            else:\n",
    "                if start_new_word:\n",
    "                    output.append([])\n",
    "                start_new_word = False\n",
    "                output[-1].append(char)\n",
    "            i += 1\n",
    "\n",
    "        return [\"\".join(x) for x in output]\n",
    "\n",
    "    # adds whitespace around CJK characters\n",
    "    def _tokenize_chinese_chars(self, text):\n",
    "        output = []\n",
    "        for char in text:\n",
    "            # returns unicode for the codepoint\n",
    "            cp = ord(char)\n",
    "            if self._is_chinese_char(cp):\n",
    "                output.append(\" \")\n",
    "                output.append(char)\n",
    "                output.append(\" \")\n",
    "            else:\n",
    "                output.append(char)\n",
    "        return \"\".join(output)\n",
    "    \n",
    "    # checks if codepoint is a CJK character\n",
    "    def _is_chinese_char(self, cp):\n",
    "        if ((cp >= 0x4E00 and cp <= 0x9FFF) or  #\n",
    "                (cp >= 0x3400 and cp <= 0x4DBF) or  #\n",
    "                (cp >= 0x20000 and cp <= 0x2A6DF) or  #\n",
    "                (cp >= 0x2A700 and cp <= 0x2B73F) or  #\n",
    "                (cp >= 0x2B740 and cp <= 0x2B81F) or  #\n",
    "                (cp >= 0x2B820 and cp <= 0x2CEAF) or\n",
    "                (cp >= 0xF900 and cp <= 0xFAFF) or  #\n",
    "                (cp >= 0x2F800 and cp <= 0x2FA1F)):  #\n",
    "            return True\n",
    "\n",
    "        return False\n",
    "    \n",
    "    # removes invalid characters and cleans up whitespaces\n",
    "    def text_clean(self, text):\n",
    "        output = []\n",
    "        for char in text:\n",
    "            # get unicode of character\n",
    "            cp = ord(char)\n",
    "            # check if character is NULL, replacement character, or control character\n",
    "            if cp == 0 or cp == 0xfffd or _is_control(char):\n",
    "                continue\n",
    "            # check if character is a whitespace character\n",
    "            if _is_whitespace(char):\n",
    "                output.append(\" \")\n",
    "            else:\n",
    "                output.append(char)\n",
    "        return \"\".join(output)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "581357b2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.846362Z",
     "start_time": "2022-08-23T18:35:16.842708Z"
    }
   },
   "outputs": [],
   "source": [
    "# tokenization based on Wordpiece\n",
    "class WordpieceTokenizer(object):\n",
    "\n",
    "    def __init__(self, vocab, unk_token=\"[UNK]\", max_input_chars_per_word=200):\n",
    "        self.vocab = vocab\n",
    "        # unk_token represents an unknown word that is not present in the vocabulary\n",
    "        self.unk_token = unk_token\n",
    "        self.max_input_chars_per_word = max_input_chars_per_word\n",
    "    \n",
    "    # tokenizes text into word pieces\n",
    "    def tokenize(self, text):\n",
    "        \"\"\"\n",
    "        A very similar algorithm to BPE.\n",
    "        BPE is a compression algorithm which replaces consecutive bytes of data with a byte that does not occur\n",
    "        in the data. It does so by mapping individual characters of text to their frequency, inculding an EOW token.\n",
    "        In each further iteration, the most frequent pairing or characters or group of characters from the table\n",
    "        is merged together until token limit or iteration limit is reached.\n",
    "        \n",
    "        In WordPiece tokenization, the only difference is how the pairing to be merged is selected. At each \n",
    "        iterative step, WordPiece chooses a symbol pair which will result in the largest increase in likelihood \n",
    "        upon merging. P(AB)/[P(A)*P(B)]\n",
    "        The time complexity is O(K²) where K is the number of current word units in the table.\n",
    "        While we use probability, the algorithm is still greedy. For a probabilistic approach, the unigram tokenizer\n",
    "        is used.\n",
    "        \n",
    "        For example:\n",
    "          input = \"unaffable\"\n",
    "          output = [\"un\", \"##aff\", \"##able\"]\n",
    "        Here '##' indicates that the token is a suffix, and should be used in that context. \n",
    "    \n",
    "        Args:\n",
    "          text: A single token or whitespace separated tokens. This should have\n",
    "            already been passed through `BasicTokenizer.\n",
    "    \n",
    "        Returns:\n",
    "          A list of wordpiece tokens.\n",
    "        \"\"\"\n",
    "\n",
    "        text = to_unicode(text)\n",
    "\n",
    "        output_tokens = []\n",
    "        for token in remove_whitespace(text):\n",
    "            # splits token into chars\n",
    "            chars = list(token)\n",
    "            if len(chars) > self.max_input_chars_per_word:\n",
    "                output_tokens.append(self.unk_token)\n",
    "                continue\n",
    "\n",
    "            is_bad = False\n",
    "            start = 0\n",
    "            sub_tokens = []\n",
    "            while start < len(chars):\n",
    "                end = len(chars)\n",
    "                cur_substr = None\n",
    "                # finds longest substring from start of remaining word which is in vocab\n",
    "                while start < end:\n",
    "                    substr = \"\".join(chars[start:end])\n",
    "                    if start > 0:\n",
    "                        substr = \"##\" + substr\n",
    "                    if substr in self.vocab:\n",
    "                        cur_substr = substr\n",
    "                        break\n",
    "                    end -= 1\n",
    "                # if no such substring, the token is not in vocab\n",
    "                if cur_substr is None:\n",
    "                    is_bad = True\n",
    "                    break\n",
    "                sub_tokens.append(cur_substr)\n",
    "                # find next substring\n",
    "                start = end\n",
    "\n",
    "            if is_bad:\n",
    "                output_tokens.append(self.unk_token)\n",
    "            else:\n",
    "                output_tokens.extend(sub_tokens)\n",
    "        return output_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b5e8286c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.849975Z",
     "start_time": "2022-08-23T18:35:16.847109Z"
    }
   },
   "outputs": [],
   "source": [
    "# checks if char is whitespace \n",
    "def _is_whitespace(char):\n",
    "    # \\t, \\n, and \\r are technically controll characters but we treat them\n",
    "    # as whitespace since they are generally considered as such.\n",
    "    if char == \" \" or char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n",
    "        return True\n",
    "    # returns category of char\n",
    "    cat = unicodedata.category(char)\n",
    "    # char of category space separator\n",
    "    if cat == \"Zs\":\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "\n",
    "def _is_control(char):\n",
    "    # These are technically control characters but we count them as whitespace\n",
    "    # characters.\n",
    "    if char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n",
    "        return False\n",
    "    # returns category of char\n",
    "    cat = unicodedata.category(char)\n",
    "    # char of category control, format, private use, or surrogate\n",
    "    if cat.startswith(\"C\"):\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "# checks if char is a punctuation\n",
    "def _is_punctuation(char):\n",
    "    cp = ord(char)\n",
    "    # We treat all non-letter/number ASCII as punctuation.\n",
    "    # Characters such as \"^\", \"$\", and \"`\" are not in the Unicode\n",
    "    # Punctuation class but we treat them as punctuation anyways, for\n",
    "    # consistency.\n",
    "    if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or\n",
    "            (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):\n",
    "        return True\n",
    "    # returns category of char\n",
    "    cat = unicodedata.category(char)\n",
    "    # char of category punctuation\n",
    "    if cat.startswith(\"P\"):\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2c7ec5b",
   "metadata": {},
   "source": [
    "## Create a custom subwords tokenizer from the training dataset for the decoder.\n",
    "The encoder uses BERT Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2e806f24",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.885570Z",
     "start_time": "2022-08-23T18:35:16.850629Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17285 ----> Mooie \n",
      "14012 ----> kale \n",
      "580 ----> man\n",
      "18097 ----> .\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "vocab_file = 'vocab_nld'\n",
    "# The vocabulary is \"trained\" on a corpus and all wordpieces are stored in a vocabulary file\n",
    "if os.path.isfile(vocab_file + '.subwords'):\n",
    "    # Invertible TextEncoder using word pieces with a byte-level fallback\n",
    "    tokenizer_nld = tfds.deprecated.text.SubwordTextEncoder.load_from_file(vocab_file)\n",
    "else: \n",
    "    # if vocab file not stored, build it\n",
    "    tokenizer_nld = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(\n",
    "      (nld_train), target_vocab_size=2 ** 15)\n",
    "    tokenizer_nld.save_to_file('vocab_nld')\n",
    "\n",
    "sample_string = 'Mooie kale man.'\n",
    "tokenized_string = tokenizer_nld.encode(sample_string)\n",
    "for ts in tokenized_string:\n",
    "  print ('{} ----> {}'.format(ts, tokenizer_nld.decode([ts])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "2e99602b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:16.899279Z",
     "start_time": "2022-08-23T18:35:16.886231Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[101, 14262, 10497, 11514, 3012, 102]\n",
      "['[CLS]', 'ser', '##end', '##ip', '##ity', '[SEP]']\n"
     ]
    }
   ],
   "source": [
    "# english tokenizer using the custom FullTokenizer\n",
    "tokenizer_en = FullTokenizer(\n",
    "    vocab_file= 'vocab_en.txt',\n",
    "    do_lower_case=True)\n",
    "\n",
    "test_tokens = tokenizer_en.tokenize(\"serendipity\")\n",
    "# [CLS] token is used to indicate the task we want BERT to perform is next-sentence prediction, and not mask\n",
    "# word prediction. We can think about the output of [CLS] as a probability. Used to oraganize tasks as [CLS]\n",
    "# and [MASK]. [SEP] is also used for next-sentence predication tasks. It helps the model distinguish one \n",
    "# sentence from the next.\n",
    "test_ids = tokenizer_en.convert_tokens_to_ids(['[CLS]'] + test_tokens + ['[SEP]'])\n",
    "print(test_ids)\n",
    "print(tokenizer_en.convert_ids_to_tokens(test_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "44dc772e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:17.753433Z",
     "start_time": "2022-08-23T18:35:17.746581Z"
    }
   },
   "outputs": [],
   "source": [
    "MAX_SEQ_LENGTH = 50\n",
    "\n",
    "def encode(en, nld, seq_length=MAX_SEQ_LENGTH):\n",
    "    # converts Python input text to unicode\n",
    "    tokens_en = tokenizer_en.tokenize(tf.compat.as_text(en.numpy()))\n",
    "    lang1 = tokenizer_en.convert_tokens_to_ids(['[CLS]'] + tokens_en + ['[SEP]'])\n",
    "    if len(lang1)<seq_length:\n",
    "        # makes token list equal to length of seq_length\n",
    "        lang1 = lang1 + list(np.zeros(seq_length - len(lang1), 'int32'))\n",
    "    # lang2 is a list with first argument as size of vocab of Dutch tokenizer, followed by list of Dutch text converted\n",
    "    # to integers, and again the last argument is the size of the vocab+1\n",
    "    lang2 = [tokenizer_nld.vocab_size] + tokenizer_nld.encode(tf.compat.as_text(nld.numpy())) + [tokenizer_nld.vocab_size + 1]\n",
    "    if len(lang2)<seq_length:\n",
    "        lang2 = lang2 + list(np.zeros(seq_length - len(lang2), 'int32'))\n",
    "    return lang1, lang2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "059a93e8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:18.247533Z",
     "start_time": "2022-08-23T18:35:18.243794Z"
    }
   },
   "outputs": [],
   "source": [
    "# used to convert shape and type of output from tf.string to tf.int32, makes shape(None,) required for future \n",
    "# functions\n",
    "def tf_encode(en, nld):\n",
    "    # tf.py_function makes it possible to express control flow using Python constructs (if, while, for, etc.), \n",
    "    # instead of TensorFlow control flow constructs (tf.cond, tf.while_loop) for the function used in the argument\n",
    "    # tf.py_function(func, inp, Tout, name=None)\n",
    "    result_en, result_nld = tf.py_function(encode, [en, nld], [tf.int32, tf.int32])\n",
    "    # sets particular shape to the objects\n",
    "    result_en.set_shape([None])\n",
    "    result_nld.set_shape([None])\n",
    "    return result_en, result_nld"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "284c7593",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:18.328544Z",
     "start_time": "2022-08-23T18:35:18.325340Z"
    }
   },
   "outputs": [],
   "source": [
    "# checks if both english and dutch seentences are less than max_length\n",
    "def filter_max_length(x, y, max_length=MAX_SEQ_LENGTH):\n",
    "    # element-wise AND of its arguments\n",
    "    return tf.logical_and(tf.size(x) <= max_length,\n",
    "                        tf.size(y) <= max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b39fc057",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:18.571323Z",
     "start_time": "2022-08-23T18:35:18.428941Z"
    }
   },
   "outputs": [],
   "source": [
    "BUFFER_SIZE = 40000\n",
    "BATCH_SIZE = 64\n",
    "\n",
    "# applies tf_encode, filter_max_length functions to entire dataset\n",
    "train_dataset = train_ex.map(tf_encode)\n",
    "train_dataset = train_dataset.filter(filter_max_length)\n",
    "\n",
    "# cache the dataset to memory to get a speedup while reading from it.\n",
    "train_dataset = train_dataset.cache()\n",
    "\n",
    "# maintains a fixed-size buffer and chooses the next element uniformly at random from that buffer.\n",
    "train_dataset = train_dataset.shuffle(BUFFER_SIZE).padded_batch(\n",
    "    BATCH_SIZE, padded_shapes=([-1], [-1]), drop_remainder=True)\n",
    "# creates a Dataset that prefetches elements from this dataset. This allows later elements to \n",
    "# be prepared while the current element is being processed. \n",
    "train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
    "test_dataset = test_ex.map(\n",
    "    lambda en, nld: tf.py_function(encode, [en, nld], [tf.int32, tf.int32]))\n",
    "test_dataset = test_dataset.filter(filter_max_length)\n",
    "test_dataset = test_dataset.padded_batch(BATCH_SIZE, padded_shapes=([-1], [-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40918588",
   "metadata": {},
   "source": [
    "# Attention"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e491c613",
   "metadata": {},
   "source": [
    "## Positional Encoding\n",
    "The positional encoding vector is added to the embedding vector. Embeddings represent a token in a \n",
    "d-dimensional space where tokens with similar meaning will be closer to each other. But the embeddings do \n",
    "not encode the relative position of words in a sentence. So after adding the positional encoding, words will\n",
    "be closer to each other based on the similarity of their meaning and their position in the sentence, in the \n",
    "d-dimensional space.\n",
    "The formula for calculating the positional encoding is as follows:\n",
    "\n",
    "$$\\Large{PE_{(pos, 2i)} = \\sin(pos / 10000^{2i / d_{model}})} $$\n",
    "$$\\Large{PE_{(pos, 2i+1)} = \\cos(pos / 10000^{2i / d_{model}})} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c107e85b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:20.363534Z",
     "start_time": "2022-08-23T18:35:20.359349Z"
    }
   },
   "outputs": [],
   "source": [
    "# angles to which sin and cos will be applied\n",
    "def get_angles(pos, i, d_model):\n",
    "    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
    "    return pos * angle_rates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c41db276",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:20.478926Z",
     "start_time": "2022-08-23T18:35:20.473833Z"
    }
   },
   "outputs": [],
   "source": [
    "# calculates the positional encoding vector\n",
    "def positional_encoding(position, d_model):\n",
    "    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
    "                          np.arange(d_model)[np.newaxis, :],\n",
    "                          d_model)\n",
    "\n",
    "    # apply sin to even indices in the array; 2i, start at 0, skip 2\n",
    "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
    "\n",
    "    # apply cos to odd indices in the array; 2i+1, start at 1, skip 2\n",
    "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
    "\n",
    "    pos_encoding = angle_rads[np.newaxis, ...]\n",
    "\n",
    "    return tf.cast(pos_encoding, dtype=tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbee4040",
   "metadata": {},
   "source": [
    "## Masking\n",
    "\n",
    "Mask all the pad tokens in the batch of sequence. It ensures that the model does not treat padding as the input. The mask indicates where pad value 0 is present: it outputs a 1 at those locations, and a 0 otherwise."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d5827c10",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:20.624218Z",
     "start_time": "2022-08-23T18:35:20.617644Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_padding_mask(seq):\n",
    "    #tf.cast casts a tensor to a new type\n",
    "    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
    "\n",
    "    # add extra dimensions so that we can add the padding\n",
    "    # to the attention logits.\n",
    "    return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c42c39e0",
   "metadata": {},
   "source": [
    "The look-ahead mask is used to mask the future tokens in a sequence. In other words, the mask indicates which entries should not be used.\n",
    "\n",
    "This means that to predict the third word, only the first and second word will be used. Similarly to predict the fourth word, only the first, second and the third word will be used and so on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "52d08efd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:20.786090Z",
     "start_time": "2022-08-23T18:35:20.782262Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_look_ahead_mask(size):\n",
    "    # Copy a tensor setting everything outside a central band in each innermost matrix to zero\n",
    "    # The indicator function in_band(m, n) = (num_lower < 0 || (m-n) <= num_lower)) \n",
    "    # && (num_upper < 0 || (n-m) <= num_upper)\n",
    "    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0) # 1 - (lower triangle of matrix)\n",
    "    # lower triangle is not masked\n",
    "    return mask  # (seq_len, seq_len)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e111dac0",
   "metadata": {},
   "source": [
    "## Scaled dot product attention\n",
    "$$\\Large{Attention(Q, K, V) = softmax_k\\left(\\frac{QK^T}{\\sqrt{d_k}}\\right) V} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "fb777668",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:20.972236Z",
     "start_time": "2022-08-23T18:35:20.964385Z"
    },
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# the attention values decide the amount of importance given to the keys for each query.\n",
    "# The output represents the multiplication of the attention weights and the V (value) \n",
    "# vector. This ensures that the tokens we want to focus on are kept as-is and the \n",
    "# irrelevant tokens are flushed out.\n",
    "def scaled_dot_product_attention(q, k, v, mask):\n",
    "    \"\"\"Calculate the attention weights.\n",
    "    q, k, v must have matching leading dimensions.\n",
    "    k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n",
    "    The mask has different shapes depending on its type(padding or look ahead)\n",
    "    but it must be broadcastable for addition.\n",
    "\n",
    "    Args:\n",
    "    q: query shape == (..., seq_len_q, depth)\n",
    "    k: key shape == (..., seq_len_k, depth)\n",
    "    v: value shape == (..., seq_len_v, depth_v)\n",
    "    mask: Float tensor with shape broadcastable\n",
    "          to (..., seq_len_q, seq_len_k). Defaults to None.\n",
    "\n",
    "    Returns:\n",
    "    output, attention_weights\n",
    "    \"\"\"\n",
    "\n",
    "    matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n",
    "\n",
    "    # scale matmul_qk\n",
    "    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
    "    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
    "\n",
    "    # add the mask to the scaled tensor.\n",
    "    if mask is not None:\n",
    "        scaled_attention_logits += (mask * -1e9)\n",
    "\n",
    "    # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
    "    # add up to 1.\n",
    "    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  \n",
    "    # (..., seq_len_q, seq_len_k)\n",
    "\n",
    "    output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n",
    "\n",
    "    return output, attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fe980d4",
   "metadata": {},
   "source": [
    "## Multi-Head-Attention"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c4937a7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T05:36:24.132249Z",
     "start_time": "2022-08-08T05:36:24.123159Z"
    }
   },
   "source": [
    "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/multi_head_attention.png\" width=\"500\" alt=\"multi-head attention\">\n",
    "\n",
    "\n",
    "Multi-head attention consists of four parts:\n",
    "*    Linear layers.\n",
    "*    Scaled dot-product attention.\n",
    "*    Final linear layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "c87cd9ad",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:21.214936Z",
     "start_time": "2022-08-23T18:35:21.197056Z"
    }
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self,*, d_model, num_heads):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.num_heads = num_heads\n",
    "        self.d_model = d_model\n",
    "\n",
    "        assert d_model % self.num_heads == 0\n",
    "\n",
    "        self.depth = d_model // self.num_heads\n",
    "\n",
    "        self.wq = tf.keras.layers.Dense(d_model)\n",
    "        self.wk = tf.keras.layers.Dense(d_model)\n",
    "        self.wv = tf.keras.layers.Dense(d_model)\n",
    "\n",
    "        self.dense = tf.keras.layers.Dense(d_model)\n",
    "\n",
    "    def split_heads(self, x, batch_size):\n",
    "        \"\"\"Split the last dimension into (num_heads, depth).\n",
    "        Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n",
    "        \"\"\"\n",
    "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
    "        return tf.transpose(x, perm=[0, 2, 1, 3])\n",
    "\n",
    "    def call(self, v, k, q, mask):\n",
    "        batch_size = tf.shape(q)[0]\n",
    "\n",
    "        q = self.wq(q)  # (batch_size, seq_len, d_model)\n",
    "        k = self.wk(k)  # (batch_size, seq_len, d_model)\n",
    "        v = self.wv(v)  # (batch_size, seq_len, d_model)\n",
    "\n",
    "        q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
    "        k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
    "        v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
    "\n",
    "        # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
    "        # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
    "        scaled_attention, attention_weights = scaled_dot_product_attention(\n",
    "            q, k, v, mask)\n",
    "\n",
    "        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n",
    "\n",
    "        concat_attention = tf.reshape(scaled_attention,\n",
    "                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n",
    "\n",
    "        output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n",
    "\n",
    "        return output, attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc66077d",
   "metadata": {},
   "source": [
    "## Point wise feed forward network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "4989b45a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:21.686469Z",
     "start_time": "2022-08-23T18:35:21.682946Z"
    }
   },
   "outputs": [],
   "source": [
    "def point_wise_feed_forward_network(d_model, dff):\n",
    "  return tf.keras.Sequential([\n",
    "      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n",
    "      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n",
    "  ])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e2b0a75",
   "metadata": {},
   "source": [
    "# Encoder\n",
    "Use bert as encoder. The output shape is (batch_size, input_seq_len, d_model)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "19f1a718",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:22.285140Z",
     "start_time": "2022-08-23T18:35:22.281416Z"
    }
   },
   "outputs": [],
   "source": [
    "# using the bert_config.json from a pre-trained google model\n",
    "def build_encoder(config_file):\n",
    "    with tf.io.gfile.GFile(config_file, \"r\") as reader:\n",
    "        stock_params = StockBertConfig.from_json_string(reader.read())\n",
    "        bert_params = stock_params.to_bert_model_layer_params()\n",
    "\n",
    "    return BertModelLayer.from_params(bert_params, name=\"bert\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e9533d9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T06:15:32.661032Z",
     "start_time": "2022-08-08T06:15:32.658544Z"
    }
   },
   "source": [
    "# Decoder\n",
    "Each decoder layer consists of sublayers:\n",
    "\n",
    "1. Masked multi-head attention (with look ahead mask and padding mask)\n",
    "\n",
    "2. Multi-head attention (with padding mask). V (value) and K (key) receive the encoder output as inputs. Q (query) receives the output from the masked multi-head attention sublayer\n",
    "\n",
    "3. Point wise feed forward networks\n",
    "\n",
    "Each of these sublayers has a residual connection around it followed by a layer normalization. The output of each sublayer is LayerNorm(x + Sublayer(x)). The normalization is done on the d_model (last) axis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "df8bc2e2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:22.470380Z",
     "start_time": "2022-08-23T18:35:22.452850Z"
    }
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "\n",
    "        self.mha1 = MultiHeadAttention(d_model=d_model, num_heads=num_heads)\n",
    "        self.mha2 = MultiHeadAttention(d_model=d_model, num_heads=num_heads)\n",
    "\n",
    "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "\n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout3 = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "\n",
    "    def call(self, x, enc_output, training,\n",
    "             look_ahead_mask, padding_mask):\n",
    "        # enc_output.shape == (batch_size, input_seq_len, d_model)\n",
    "        # call function of MHA (v,k,q,mask)\n",
    "        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n",
    "        attn1 = self.dropout1(attn1, training=training)\n",
    "        # output of sublayer 1\n",
    "        out1 = self.layernorm1(attn1 + x)\n",
    "\n",
    "        attn2, attn_weights_block2 = self.mha2(\n",
    "            enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n",
    "        attn2 = self.dropout2(attn2, training=training)\n",
    "        out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n",
    "\n",
    "        ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n",
    "        ffn_output = self.dropout3(ffn_output, training=training)\n",
    "        out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n",
    "\n",
    "        return out3, attn_weights_block1, attn_weights_block2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "f8cd9e02",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:22.564115Z",
     "start_time": "2022-08-23T18:35:22.553880Z"
    }
   },
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.layers.Layer):\n",
    "  def __init__(self,*, num_layers, d_model, num_heads, dff, target_vocab_size,\n",
    "               rate=0.1):\n",
    "    super(Decoder, self).__init__()\n",
    "\n",
    "    # number of features expected in input\n",
    "    self.d_model = d_model\n",
    "    self.num_layers = num_layers\n",
    "\n",
    "    self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
    "    self.pos_encoding = positional_encoding(MAX_TOKENS, d_model)\n",
    "\n",
    "    self.dec_layers = [\n",
    "        DecoderLayer(d_model=d_model, num_heads=num_heads, dff=dff, rate=rate)\n",
    "        for _ in range(num_layers)]\n",
    "    self.dropout = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "  def call(self, x, enc_output, training,\n",
    "           look_ahead_mask, padding_mask):\n",
    "\n",
    "    seq_len = tf.shape(x)[1]\n",
    "    attention_weights = {}\n",
    "\n",
    "    x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n",
    "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "    x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "    x = self.dropout(x, training=training)\n",
    "\n",
    "    for i in range(self.num_layers):\n",
    "      x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n",
    "                                             look_ahead_mask, padding_mask)\n",
    "\n",
    "      attention_weights[f'decoder_layer{i+1}_block1'] = block1\n",
    "      attention_weights[f'decoder_layer{i+1}_block2'] = block2\n",
    "\n",
    "    # x.shape == (batch_size, target_seq_len, d_model)\n",
    "    return x, attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a0ac2a4",
   "metadata": {},
   "source": [
    "# Transformer\n",
    "A transformer consists of the encoder, decoder, and a final linear layer. The output of the decoder is the input to the linear layer and its output is returned.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "5cfe453e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:23.757964Z",
     "start_time": "2022-08-23T18:35:23.754546Z"
    }
   },
   "outputs": [],
   "source": [
    "class Config(object):\n",
    "  def __init__(self, num_layers, d_model, dff, num_heads):\n",
    "    self.num_layers = num_layers\n",
    "    self.d_model = d_model\n",
    "    self.dff = dff\n",
    "    self.num_heads= num_heads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "2c519e3b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:24.262686Z",
     "start_time": "2022-08-23T18:35:24.246224Z"
    }
   },
   "outputs": [],
   "source": [
    "from bert.loader import map_to_stock_variable_name\n",
    "\n",
    "class Transformer(tf.keras.Model):\n",
    "    def __init__(self, config,\n",
    "               target_vocab_size, \n",
    "               bert_config_file,\n",
    "               bert_training=False, \n",
    "               rate=0.1,\n",
    "               name='transformer'):\n",
    "      super(Transformer, self).__init__(name=name)\n",
    "\n",
    "      self.encoder = build_encoder(config_file=bert_config_file)\n",
    "      self.encoder.trainable = bert_training\n",
    "\n",
    "      self.decoder = Decoder(num_layers=config.num_layers, d_model=config.d_model,\n",
    "                           num_heads=config.num_heads, dff=config.dff,\n",
    "                           target_vocab_size=target_vocab_size, rate=rate)\n",
    "\n",
    "      self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
    "\n",
    "\n",
    "    def load_stock_weights(self, bert: BertModelLayer, ckpt_file):\n",
    "      assert isinstance(bert, BertModelLayer), \"Expecting a BertModelLayer instance as first argument\"\n",
    "      assert tf.compat.v1.train.checkpoint_exists(ckpt_file), \"Checkpoint does not exist: {}\".format(ckpt_file)\n",
    "      # returns CheckpointReader for checkpoint found in ckpt_dir_or_file.  \n",
    "      ckpt_reader = tf.train.load_checkpoint(ckpt_file)\n",
    "\n",
    "      bert_prefix = 'transformer/bert'\n",
    "\n",
    "      weights = []\n",
    "      for weight in bert.weights:\n",
    "          stock_name = map_to_stock_variable_name(weight.name, bert_prefix)\n",
    "          if ckpt_reader.has_tensor(stock_name):\n",
    "              value = ckpt_reader.get_tensor(stock_name)\n",
    "              weights.append(value)\n",
    "          else:\n",
    "              raise ValueError(\"No value for:[{}], i.e.:[{}] in:[{}]\".format(weight.name, stock_name, ckpt_file))\n",
    "      bert.set_weights(weights)\n",
    "      print(\"Done loading {} BERT weights from: {} into {} (prefix:{})\".format(\n",
    "          len(weights), ckpt_file, bert, bert_prefix))\n",
    "\n",
    "    def restore_encoder(self, bert_ckpt_file):\n",
    "      # loading the original pre-trained weights into the BERT layer:\n",
    "      self.load_stock_weights(self.encoder, bert_ckpt_file)\n",
    "\n",
    "    def call(self, inp, tar, training, look_ahead_mask, dec_padding_mask):\n",
    "      enc_output = self.encoder(inp, training=self.encoder.trainable)  # (batch_size, inp_seq_len, d_model)\n",
    "\n",
    "      # dec_output.shape == (batch_size, tar_seq_len, d_model)\n",
    "      dec_output, attention_weights = self.decoder(\n",
    "          tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
    "\n",
    "      final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n",
    "\n",
    "      return final_output, attention_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c16a44e",
   "metadata": {},
   "source": [
    "# Fine-tuning model and hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8558d952",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:24.416264Z",
     "start_time": "2022-08-23T18:35:24.411376Z"
    }
   },
   "outputs": [],
   "source": [
    "target_vocab_size = tokenizer_nld.vocab_size + 2\n",
    "dropout_rate = 0.15\n",
    "config = Config(num_layers=6, d_model=512, dff=1024, num_heads=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "2cb18fe4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:26.561412Z",
     "start_time": "2022-08-23T18:35:24.496824Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 50)\n",
      "(64, 50, 18309)\n",
      "WARNING:tensorflow:From /var/folders/jd/ll0by25n4ln09rznyqbztdyw0000gn/T/ipykernel_90191/3340841461.py:24: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "Done loading 196 BERT weights from: uncased_L-12_H-768_A-12/bert_model.ckpt into <bert.model.BertModelLayer object at 0x1056ca040> (prefix:transformer/bert)\n"
     ]
    }
   ],
   "source": [
    "MODEL_DIR = \"uncased_L-12_H-768_A-12\"\n",
    "bert_config_file = os.path.join(MODEL_DIR, \"bert_config.json\")\n",
    "bert_ckpt_file = os.path.join(MODEL_DIR, 'bert_model.ckpt')\n",
    "MAX_TOKENS = 128\n",
    "transformer = Transformer(config=config,\n",
    "                          target_vocab_size=target_vocab_size,\n",
    "                          bert_config_file=bert_config_file)\n",
    "  \n",
    "inp = tf.random.uniform((BATCH_SIZE, MAX_SEQ_LENGTH))\n",
    "tar_inp = tf.random.uniform((BATCH_SIZE, MAX_SEQ_LENGTH))\n",
    "fn_out, _ = transformer(inp, tar_inp, \n",
    "                        True,\n",
    "                        look_ahead_mask=None,\n",
    "                        dec_padding_mask=None)\n",
    "print(tar_inp.shape) # (batch_size, tar_seq_len) \n",
    "print(fn_out.shape)  # (batch_size, tar_seq_len, target_vocab_size) \n",
    "\n",
    "# init bert pre-trained weights\n",
    "transformer.restore_encoder(bert_ckpt_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "1933fcde",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:26.569896Z",
     "start_time": "2022-08-23T18:35:26.562282Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"transformer\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "bert (BertModelLayer)        multiple                  108890112 \n",
      "_________________________________________________________________\n",
      "decoder (Decoder)            multiple                  29873664  \n",
      "_________________________________________________________________\n",
      "dense_60 (Dense)             multiple                  9392517   \n",
      "=================================================================\n",
      "Total params: 148,156,293\n",
      "Trainable params: 39,266,181\n",
      "Non-trainable params: 108,890,112\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "transformer.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5add015f",
   "metadata": {},
   "source": [
    "Using the Adam optimizer with a custom learning rate scheduler according to the formula in ['Attention is all you need'](https://arxiv.org/abs/1706.03762).\n",
    "\n",
    "$$\\Large{lrate = d_{model}^{-0.5} * \\min(step{\\_}num^{-0.5}, step{\\_}num \\cdot warmup{\\_}steps^{-1.5})}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "a77e033f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:35.620484Z",
     "start_time": "2022-08-23T18:35:35.613948Z"
    }
   },
   "outputs": [],
   "source": [
    "# to modulate how the learning rate of your optimizer changes over time\n",
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "    def __init__(self, d_model, warmup_steps=4000):\n",
    "        super(CustomSchedule, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.d_model = tf.cast(self.d_model, tf.float32)\n",
    "\n",
    "        self.warmup_steps = warmup_steps\n",
    "    \n",
    "    def __call__(self, step):\n",
    "        # reciprocal of square root\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warmup_steps ** -1.5)\n",
    "\n",
    "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f663df41",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:36.747473Z",
     "start_time": "2022-08-23T18:35:36.743380Z"
    }
   },
   "outputs": [],
   "source": [
    "learning_rate = CustomSchedule(config.d_model)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98,\n",
    "                                     epsilon=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "0427c58c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:37.344260Z",
     "start_time": "2022-08-23T18:35:37.133414Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Train Step')"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAEGCAYAAABGnrPVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAynklEQVR4nO3de3yU9Zn//9eVhBASyAmSEM6nAIIHRASt1vMJW4tudau1W9vaZWll22672+J+263723771e3Z1mp1axftwdq1rmg9VFHRWg9ERQQRSUYOgUAmHAJJOOf6/TF3YAw5TMhMZpJ5Px+Pedz33Pf9ueeaG5iLz31/7us2d0dERCReMpIdgIiI9C9KLCIiEldKLCIiEldKLCIiEldKLCIiEldZyQ4gmYYNG+bjxo1LdhgiIn3K66+/Xu/uJR2tT+vEMm7cOCorK5MdhohIn2JmGzpbr1NhIiISV0osIiISV0osIiISV0osIiISV0osIiISVwlNLGZ2mZmtNbMqM1vUznozs9uD9SvNbGZXbc2s2MyeNrN1wbQoWH69ma2IerWY2YxEfj8RETlWwhKLmWUCdwBzgWnAdWY2rc1mc4GK4DUfuDOGtouApe5eASwN3uPuv3H3Ge4+A/g7YL27r0jU9xMRkfYlsscyG6hy95C7HwAeAOa12WYecJ9HvAIUmll5F23nAYuD+cXAle189nXA7+L6bVLE6xt2sGLTrmSHISLSoUQmlpHApqj3NcGyWLbprG2Zu9cCBNPSdj77E3SQWMxsvplVmlllOByO8aukjo/f+TJX3vESeo6OiKSqRCYWa2dZ21/DjraJpW37H2o2B2h291XtrXf3u919lrvPKinpsCJBSjrccvQQrN22J4mRiIh0LJGJpQYYHfV+FLAlxm06a7stOF1GMK1rs89r6aenwbbs2ntk/om3tyYxEhGRjiUysSwHKsxsvJllE/nBX9JmmyXAp4PRYWcADcHprc7aLgFuCOZvAB5p3ZmZZQDXELkm0+9UhRsBMIMnVtUmORoRkfYlLLG4+yFgIfAUsAZ40N1Xm9kCM1sQbPY4EAKqgHuAL3bWNmhzK3Cxma0DLg7etzoHqHH3UKK+VzKFwk0ALDx/Eu9ta6SqrjHJEYmIHCuh1Y3d/XEiySN62V1R8w7cFGvbYPl24MIO2jwPnHH8Eae26nAjBYMG8Mk5Y/jps1U8uaqWhRdUJDssEZEP0J33fUgo3MiEkjzKCwYxc0whT6zSdRYRST1KLH1IKNzExJLBAFx+Ujmrt+wmFNbpMBFJLUosfcSefQep27OfCSV5AFxxygjM4H/f3JzkyEREPkiJpY9ovXDf2mMpy8/hrInDeHjFZt0sKSIpRYmlj6gOTnlNDHosAFedOpJNO/by+oadyQpLROQYSix9RCjcRGaGMab4aGK57MThDBqQyR91OkxEUogSSx8Rqm9kTHEu2VlH/8jyBmZxyfQy/rSylv2HDicxOhGRo5RY+ojquiYmDMs7ZvlVp46kYe9Bnl3TtrKNiEhyKLH0AYdbnPe3NzGxdPAx686eNIzh+Tk8sHxTOy1FRHqfEksfsHnnXg4camm3x5KVmcHfnj6aF9aF2bSjOQnRiYh8kBJLH1BdHxkRNqHk2B4LwLWnj8aA36vXIiIpQImlD6iuO3aocbQRhYM4f0opv6/cxMHDLb0ZmojIMZRY+oBQfRMFgwZQnJfd4TbXzR5DeM9+lq7Z1ouRiYgcS4mlDwiFG5lYkodZew/WjDhvSgnlBTn85tWNvRiZiMixlFj6gOpwU4fXV1plZWbwydljeHFdPev02GIRSSIllhS3e99Bwnv2H6kR1pnrzxjLwKwM7n3p/V6ITESkfUosKa61+OSEDi7cRyvOy+ZvZo7kj29sZnvj/kSHJiLSLiWWFBdqp/hkZz531nj2H2rRtRYRSRollhTXXvHJzlSUDeHcySXc9/IG1Q8TkaRIaGIxs8vMbK2ZVZnZonbWm5ndHqxfaWYzu2prZsVm9rSZrQumRVHrTjazl81stZm9bWY5ifx+vaE6fGzxya7cePZ46hv36yFgIpIUCUssZpYJ3AHMBaYB15nZtDabzQUqgtd84M4Y2i4Clrp7BbA0eI+ZZQG/Bha4+3TgPOBgor5fb4k8jji23kqrD1cM48SR+fz8+WoO6YZJEellieyxzAaq3D3k7geAB4B5bbaZB9znEa8AhWZW3kXbecDiYH4xcGUwfwmw0t3fAnD37e7ep88FtRaf7GqocVtmxsLzK9iwvZlHV25JUHQiIu1LZGIZCUQXr6oJlsWyTWdty9y9FiCYlgbLJwNuZk+Z2Rtm9vX2gjKz+WZWaWaV4XD4OL5W7+ms+GRXLplWxpSyIfzs2SpaWvToYhHpPYlMLO3dJt72F66jbWJp21YWcDZwfTC9yswuPGYn7ne7+yx3n1VSUtLFLpPryOOI2ymX35WMDGPhBZOoDjfxxKqt8Q5NRKRDiUwsNcDoqPejgLbnZTraprO224LTZQTT1idc1QDL3L3e3ZuBx4GZ9GGtieV4eiwAl59UzoSSPH767Dr1WkSk1yQysSwHKsxsvJllA9cCS9psswT4dDA67AygITi91VnbJcANwfwNwCPB/FPAyWaWG1zIPxd4J1FfrjeE6psozO28+GRnMjOML19Ywbtb9+hai4j0moQlFnc/BCwk8oO/BnjQ3Veb2QIzWxBs9jgQAqqAe4AvdtY2aHMrcLGZrQMuDt7j7juBHxJJSiuAN9z9T4n6fr2huq6RCcM6Lz7ZlStOHsG08nx+8Of3OHBII8REJPGyErlzd3+cSPKIXnZX1LwDN8XaNli+HTjm2kmw7tdEhhz3C6H6Js6b3LPrQBkZxtcvm8JnfrWc3722kRs+NC4+wYmIdEB33qeo1uKT3R1q3J5zJ5cwZ3wxP312HU37D8UhOhGRjimxpKjuFJ/sipnxjblTqW88wD0vhnq8PxGRziixpKijxSd73mMBmDmmiMtPGs5dy6rZsmtvXPYpItIeJZYUVR1uDIpP5sZtn/96+Qm4w3cfXxO3fYqItKXEkqJC4SbGdrP4ZFdGFeXyhfMm8tjKWl4JbY/bfkVEoimxpKjqcGNcrq+0teDciYwsHMQtS1arQKWIJIQSSwo63OKsr2+Oy4iwtnIGZPLNj5zAu1v3cP8rG+K+fxERJZYUVLOzmQOHW7pdLj9Wl504nA9XDOP7T61lsy7ki0icKbGkoKNDjePfY4HI8OPvXnUSLQ7ffPhtIvepiojEhxJLCqqO81Dj9owuzuVrl0zmubVhHl1Zm7DPEZH0o8SSgqrDPSs+GavPnjWeU0YV8O9LVrOz6UBCP0tE0ocSSwoKhRsT2ltplZlh3Prxk2nYe5BvPbJKp8REJC6UWFJQdbjpuJ/B0l0nlOfzTxdP5rGVtTyyQqX1RaTnlFhSzO59B6lvjE/xyVgtOHcis8YW8a3/XUXNzuZe+1wR6Z+UWFJM64iwRA01bk9mhvGjT8ygxZ2vPfgWh/W0SRHpASWWFFNdFzyOuBd7LBAZJXbLx6bz6vs7uGtZda9+toj0L0osKSZU30hWhjF2aPyKT8bq6tNGccUpI/jBn9fycrVqiYnI8VFiSTHVdU2MKc5lQGbv/9GYGf/vb05i3LA8/vF3b1K3e1+vxyAifZ8SS4oJ1Sem+GSsBg/M4s7rT6Nx/0H+8XdvqlCliHRbQhOLmV1mZmvNrMrMFrWz3szs9mD9SjOb2VVbMys2s6fNbF0wLQqWjzOzvWa2InjdlcjvlgitxSd74x6WzkwZPoT/e+VJvPr+Dr7357VJjUVE+p6EJRYzywTuAOYC04DrzGxam83mAhXBaz5wZwxtFwFL3b0CWBq8b1Xt7jOC14LEfLPEaS0+mcweS6uPnzaKT84Zwy+WhfjjGzXJDkdE+pBE9lhmA1XuHnL3A8ADwLw228wD7vOIV4BCMyvvou08YHEwvxi4MoHfoVcdHWqc3B5Lq1uumM4ZE4pZ9NDbvL5hZ7LDEZE+IpGJZSSwKep9TbAslm06a1vm7rUAwbQ0arvxZvammS0zsw+3F5SZzTezSjOrDIfD3f1OCdVafLK3hxp3JDsrgzuvP43ywhz+4f5KldgXkZgkMrFYO8va3nnX0TaxtG2rFhjj7qcCXwV+a2b5x+zE/W53n+Xus0pKSrrYZe+qDjdR1AvFJ7ujKC+bX94wi/0HW/j84kr27DuY7JBEJMUlMrHUAKOj3o8C2haj6mibztpuC06XEUzrANx9v7tvD+ZfB6qByXH5Jr0k8jji1OitRJtUOoSfXT+Tddv2sODXr7P/0OFkhyQiKSyRiWU5UGFm480sG7gWWNJmmyXAp4PRYWcADcHprc7aLgFuCOZvAB4BMLOS4KI/ZjaByICAUOK+XvyFerH4ZHedO7mE2z5+Mi9VbeerD75Fi8q+iEgHshK1Y3c/ZGYLgaeATOBed19tZguC9XcBjwOXA1VAM/DZztoGu74VeNDMbgQ2AtcEy88B/j8zOwQcBha4+45Efb94a9gbKT45sTT1eiytPn7aKLY37ee7j7/LsLxsbvnYdMzaO2spIuksYYkFwN0fJ5I8opfdFTXvwE2xtg2WbwcubGf5Q8BDPQw5aUKtF+5TtMfSav45Ewnv2c89L75Pcd5AvnxRRbJDEpEUk9DEIrE7MtQ4hXssrW6eewI7mg7yo2feIyvTuOn8SckOSURSiBJLiqgOR4pPjinu/eKT3ZWRYfzn1SdzuKWF7z21lswMY8G5E5MdloikCCWWFBEKJ6/45PHIzDC+f80pHHa49Yl3ycowPv/hCckOS0RSgBJLikjVocadycrM4Ed/ewotLc53/rSGg4edL5ynnotIulNiSQGHW5wN25u5YGpp1xunmKzMDH587QwyM4zbnnyXXXsPsOiyqRotJpLGujzvYmaTzWypma0K3p9sZt9MfGjpo7X4ZKrUCOuuAZkZ/PgTM/jUGZGilf/68Co93lgkjcVyQv8e4GbgIIC7ryRyw6LEydEaYak91LgzGRnGf8w7kZvOn8jvXtvIlx54U3foi6SpWE6F5br7a21ObRxKUDxpKdWqGh8vM+NfLp1KwaABfPfxdwnv2c/df3cahbmpU/tMRBIvlh5LvZlNJCgCaWZXEyn4KHFSHW6kKHcARSlUfLIn5p8zkZ9cO4MVG3dx1c//yvr6pmSHJCK9KJbEchPwC2CqmW0GvgL0uYdopbLqcFOfGxHWlXkzRvKbv5/DruYDXPXzl1i+vs9U1xGRHoolsbi7XwSUAFPd/ewY20mMQuEmJvbh6ysdOX1cMQ9/8SyKcrO5/p5XeXD5pq4biUifF0uCeAjA3ZvcfU+w7H8SF1J6aS0+2d96LK3GDcvjj1/8ELPHF/P1h1byrw+/rYv6Iv1chxfvzWwqMB0oMLO/iVqVD+QkOrB00Vp8sq9fuO9MYW42iz83m+89tZa7llXzzpbd3PmpmZQXDEp2aCKSAJ31WKYAHwUKgSuiXjOBv094ZGmiOhgR1peHGsciM8NYNHcqdwYPDLvip3/h5ertyQ5LRBKgwx6Luz8CPGJmZ7r7y70YU1oJ9aHik/Ew96RyKsoGM//+1/nkf73CP54/iS9dWEFWH6mRJiJdi+U+ljfN7CYip8WOnAJz988lLKo0Uh1uZMzQvlN8Mh4mlQ7h0YVn8+0lq7n92Speqt7OT66dwaii9EiuIv1dLL9m9wPDgUuBZUSeP7+n0xYSs8jjiPvv9ZWO5A3M4vvXnMJPrp3B2q17mPuTF3n8bd0eJdIfxJJYJrn7t4Amd18MfAQ4KbFhpYdDh1vYsL2ZiaX9+/pKZ+bNGMnjX/owE0oG88XfvME//X4Fu5oPJDssEemBWBLLwWC6y8xOBAqAcQmLKI3U7NwbKT6Zhj2WaGOG5vI/C87kyxdW8OhbW7j4Ry/wzDvbkh2WiBynWBLL3WZWBHwTWAK8A9yW0KjSRKg+GGqcxj2WVgMyM/iniyfzvzedxdC8bD5/XyVffXAFDc0Hu24sIimly8Ti7v/l7jvd/QV3n+DupcCTsezczC4zs7VmVmVmi9pZb2Z2e7B+pZnN7KqtmRWb2dNmti6YFrXZ5xgzazSzf44lxmSqrguGGqd5jyXaiSMLWLLwbL50wSQeWbGFi3+0jMdWbsFdZfhF+opOE4uZnWlmV5tZafD+ZDP7LfCXrnZsZpnAHcBcYBpwnZlNa7PZXKAieM0H7oyh7SJgqbtXAEuD99F+BDzRVXypIFTfv4pPxkt2VgZfvWQKj9x0FqX5A1n42zf59L2vqZilSB/RYWIxs+8B9wIfB/5kZt8GngZeJZIIujIbqHL3kLsfAB4A5rXZZh5wn0e8AhSaWXkXbecBi4P5xcCVUTFfCYSA1THEl3TV4aZ+fcd9T504soBHbjqbW66Yxpsbd3HJj1/gJ8+sU0kYkRTXWY/lI8Cp7n4dcAmRnsHZ7v4Td98Xw75HAtFVB2uCZbFs01nbMnevBQimrb2pPOAbwL93FpSZzTezSjOrDIfDMXyNxAmFG/v9Hfc9lZlhfOas8Sz92rlcMq2MHz3zHpf9+EWeeWebTo+JpKjOEsve1gTi7juBte6+rhv7bu+h521/CTraJpa2bf078CN3b+xsI3e/291nufuskpKSLnaZOA3NB6lvPKAeS4zK8nP42SdnsvhzszGDz99Xyad++SprancnOzQRaaOzO+8nmtmSqPfjot+7+8e62HcNMDrq/ShgS4zbZHfSdpuZlbt7bXDarC5YPge42sz+k0h9sxYz2+fuP+sizqSorm99HLESS3ecO7mEp75yDr9+ZQM/fmYdH7n9RT5x+mi+evEUSoYMTHZ4IkLniaXt9ZAfdHPfy4EKMxsPbAauBT7ZZpslwEIze4BIYmgIEka4k7ZLgBuAW4PpIwDu/uHWnZrZLUBjqiYVOPo4Yp0K674BmRl89qzxXHXqSH6ydB33v7yBR9+q5R/OmcBnzx7P4IGxVCoSkUTprAjlsp7s2N0PmdlC4CkgE7jX3Veb2YJg/V3A48DlQBXQDHy2s7bBrm8FHjSzG4GNwDU9iTNZqtOs+GQiFOZm8+0rpvOpM8Zy6xPv8oOn3+NXf13PF8+byKfOGEvOgMxkhyiSliydL4DOmjXLKysrk/LZ/3B/JevqGnn2a+cl5fP7ozc37uQHf36Pv1TVMzw/hy9dWME1s0alVYFPkd5gZq+7+6yO1utfXJKENNQ47k4dU8SvPz+H3/79HEYU5vCvD7/NRT9cxu+Xb+TAoZZkhyeSNpRYkuDQ4RbWb2/S9ZUE+dDEYTz0hQ9x72dmMSQni2889Dbnfe85/vul99l3UPfAiCRal1c5zexRjh3q2wBUAr+I8Z4WiVKzcy8HD7t6LAlkZlwwtYzzp5Sy7L0wdzxXxS2PvsPPnqvixrMn8KkzxjAkZ0CywxTpl2LpsYSARuCe4LUb2AZMDt5LN1Ufec69eiyJZmacN6WUPyz4EL+ffwYnlOdz25Pvctatz/L/nlhDbcPeZIco0u/EMi7zVHc/J+r9o2b2grufY2Z9onRKqjky1FjFJ3vVnAlDmTNhKG9t2sUvXqjmnhdC/PLF97n8pHJuPHs8p4wuTHaIIv1CLImlxMzGuPtGiFQPBoYF6/REpuNQHW6kOC9bxSeT5JTRhfz8+tPYtKOZxX9dz++Xb2LJW1uYNbaIG88ezyXTh5OZ0V7xBxGJRSyJ5WvAX8ysmkiplfHAF4PaXIs7bSntijyOWKfBkm10cS7f/Og0vnxRBX+orOFXf32fL/zmDUYU5HDt7DF84vTRlOXnJDtMkT4npvtYzGwgMJVIYnm3v1ywT9Z9LLO+8zQXTi3jtqtP7vXPlo4dbnGeWbONX7+ygRfX1ZOZYVx8QhnXnzGGsyYOI0O9GBGg6/tYYq19cRqRxxFnASebGe5+XxziSzutxSc11Dj1ZGYYl04fzqXTh7O+vonfLd/IHypreHL1VsYOzeW62WO4+rRRDBusmmQinYlluPH9wERgBdB6E4ADSizHobX4pIYap7Zxw/K4ee4JfPXiyTy5aiu/eXUjtz7xLt9/ai3nTSnh6tNGccHUMrKzdCuYSFux9FhmAdM8nWu/xFF1XWtVY/VY+oKBWZnMmzGSeTNGUlW3hz+8XsPDb2zmmTV1FOYOYN4pI7j6tNGcODIfM50qE4HYEssqYDhQm+BY0kKovomsDGO0ik/2OZNKh3Dz3BP4l0um8Jeqeh56YzO/W76JxS9vYHLZYK46dRQfPblcf7aS9mJJLMOAd8zsNWB/68IYnsci7QiFGxk7NFeFEfuwrMwMzptSynlTSmnYe5A/razloTdquO3Jd7ntyXeZMbqQK04ZwUdOKmd4gUaVSfqJJbHckugg0kl1uEkP9+pHCgYN4JNzxvDJOWPYtKOZx1bW8tjKLfzHY+/wnT+9w+njirnilBHMPXG4LvpL2lDZ/F4cbnzocAsn/NuT3Hj2BBbNndprnyu9rzrcyGNv1fLoyi1U1TWSYXDGhKFcOn04F08rY0ThoGSHKHLcjnu4sZn9xd3PNrM9fLAIpQHu7vlxjDMtbAqKT+rCff83sWQwX76ogi9dOIm12/bw6FtbeHLVVr69ZDXfXrKak0YWcOn0Mi6ZPpyK0sG68C/9SmdPkDw7mA7pvXD6t5CKT6YdM2Pq8HymDs/nXy6dSlVdI0+/s40/v7OV7//5Pb7/5/cYNzSXS6YP55JpZZw6pkjlZKTPi+kGSTPLBMqit2+tHSaxa61qrOKT6WtS6WAmlQ7mC+dNZNvufTyzZhtPrd7Gr156n7tfCFGYO4BzKko4b0oJ504uYaiuy0gfFMsNkv8IfJtIqfzWx/A5oHok3RQKN6n4pBxRlp/D9XPGcv2csezed5Bla8M8t7aOF94Ls+StLZjBySMLOHdKKedPKeHkUYXqzUifEEuP5cvAFHff3t2dm9llwE+ATOC/3P3WNustWH850Ax8xt3f6KytmRUDvydSYmY98LfuvtPMZgN3t+4auMXdH+5uzIkUeRyxToPJsfJzBnDFKSO44pQRtLQ4q7fs5rm1dTy/to6fPbuO25euoyh3AOdMLuGcihLOmjRMQ5klZcWSWDYReWJktwSnz+4ALgZqgOVmtsTd34nabC5QEbzmAHcCc7pouwhY6u63mtmi4P03iNzIOcvdD5lZOfCWmT3q7oe6G3uiVIcbueiEsmSHISkuI8M4aVQBJ40q4EsXVrCz6QAvrAuzbG2YZe+FeWTFFiBSveGsicM4a9JQzpwwjIJcPRFTUkMsiSUEPG9mf+KDN0j+sIt2s4Eqdw8BmNkDwDwgOrHMA+4LysW8YmaFQVIY10nbecB5QfvFwPPAN9y9OWq/ORz7OOWk2tV8gO1NB5hYqh6LdE9RXvaRsjItLc6arbv5a9V2Xqqu56E3arj/lQ2YwYkjCvjQpKGcNXEYp48rZlB2ZrJDlzQVS2LZGLyyg1esRhLp7bSqIdIr6WqbkV20LXP3WgB3rzWz0taNzGwOcC8wFvi79norZjYfmA8wZsyYbnydnqnWUyMlDjIyjOkjCpg+ooC/P2cCBw618FbNLl6qquevVdu59y/v84tlIQZkGiePKuT0ccXMGV/MzLFFFAxSj0Z6R6eJJTglVeHunzqOfbd3lbFtL6KjbWJpe+wG7q8C083sBGCxmT3R9tkx7n43wbWYWbNm9VqvpnWose5hkXjKzsrg9HHFnD6umK9cBM0HDvHa+zt4uXo7r63fwX+9GOKuZdWYwdTh+cweV8Ts8UM5fXwRpUN0jUYSo9PE4u6HzazEzLLdvbuPIa4BRke9HwVsiXGb7E7abjOz8qC3Ug7UtRP3GjNrAk4Eev9JXu0I1TcxIFPFJyWxcrOzjtQxg0iiWbFxF6+t38Hy9Tt4sLKGxS9vAGDc0NwjSWnm2EImDBush5lJXMRyKmw98JKZLQGaWhfGcI1lOVBhZuOBzcC1wCfbbLMEWBhcQ5kDNAQJI9xJ2yXADcCtwfQRgGDbTcHF+7HAlCD2lFBd18iYYhWflN6Vm53FhyYN40OThgFw8HALqzY3sHz9Dl57fyd/fmcbf3i9BoAhOVnMGF3IqaMLOXVMETNGF2povByXWBLLluCVAcR8F37wA78QeIrIkOF73X21mS0I1t8FPE5kqHEVkeHGn+2sbbDrW4EHzexGItd+rgmWnw0sMrODRO63+aK718cab6KF6pv0cC9JugGZGZw6pohTxxQx/xxoaXFC9Y28uXEXb27axZsbd/Gz56poCU4Sjx+WF0k2Ywo5dXQRU8uH6D9H0iUVoeyFIpQqPil9SdP+Q6ysaWDFpl28uXEnb2zcRX1jZEDowKwMTijP56SRBZw0soATRxZQUTZYySbN9PiZ92ZWAnwdmE5kGC8A7n5BXCJMAyo+KX1J3sAszpw4lDMnDgXA3dm8ay8rNu1ixcZdvL25gYff3Mz9r0Su1QzMymBqeT4njcw/kmwml6lnk85iORX2GyJ3un8UWEDkukY4kUH1N62PI9apMOmLzIxRRbmMKsrloyePACKn0NZvb+LtzQ28XdPA25sb+N83t/DrVyIlBLOzMjhh+BCmjyzghPJ8ppUPYcrwfAYPjKk8ofRxsfwpD3X3X5rZl919GbDMzJYlOrD+JFSvqsbSv2RkGBNKBjOhZDDzZowEPphsVm2OJJtH39rCb189Wq92THEuU4cP4YTyfE4oj0xHF+VqNFo/E0tiORhMa83sI0Qu5I9KXEj9TyjcxNC8bApzNcJG+q/2ko27s6VhH2u27ObdrbtZs3UPa2p388yabUcGCORlZzJl+BCmludzQnk+U4cPYXLpEJWo6cNiSSzfMbMC4GvAT4F84J8SGlU/Ux1u1PUVSUtmxsjCQYwsHMRF047Wydt74DDvbdsTSTa1kWTzWJveTcmQgVSUDmZy2RAmlQ4+Mq8h0Kmvy8Ti7o8Fsw3A+YkNp38KhZu4eJqKT4q0GpSdySmjCzlldOGRZe5ObcM+1m7dw7q6Pazb1sh7dY38oXITTQcOH9lu2OBsJgVJpqJ0MJNKhzC5bLCeXZNCYhkVNplI1eEydz/RzE4GPubu30l4dP1Aa/FJ9VhEOmdmjCgcxIjCQZw/9UgJwCMJ571te6iqa2TdtkbW1e3h4Tc2s2f/0XKARbkDIqfihuUxviSPCcPymFAymDHFueQMUEHO3hTLqbB7gH8BfgHg7ivN7LeAEksMVHxSpGeiE05rqRqIJJxtu/ezrm4P721rpKpuD6FwE8veCx+pJhBpDyMLBx1NOsPymFASmY4oGKSBAwkQS2LJdffXIs/kOiJlnnGS6o48575UiUUknsyM4QU5DC/I4cMVJR9Y17j/EO+HmwjVN/J+fROhcBPv1zfxh/U7PnBabWBWBuODZDNuWB5ji3MZMzSXsUPzGJ6foyd2HqdYEku9mU0kqC5sZlcDtQmNqh+pDgfFJ4sGJTsUkbQxeGDWkYelRXN3wnv2E6pvChJOJPGs3baHp9/ZxqGWo5VIsjMzGFU0KJJoinMZM/Ro4tHptc7FklhuIlJmfqqZbQbeB65PaFT9SCjcyNiheWTpLmSRpDMzSvNzKM3P4YwJQz+w7tDhFmob9rFxRzMbtjezYUcTG7dH5ivX76Rx/wdP1JTlD2Rscd6RRDN2aC4jCwcxqiiX0iED0/oUWyyjwkLARWaWB2S4+x4z+wrw4wTH1i9Uhxt1x71IH5CVmcHo4lxGF+dy1qQPrnN3djYfZMP2piOJZ+OOZjZub+bFdWG27d7/ge0HZEauC0USzSBGFuZGpkWR98Pzc/r1fzZjrq/g7k1Rb7+KEkuXDh5uYeOOZi6eNjzZoYhID5gZxXnZFOdlc+qYomPW7z1wmJqdzdTs2svmnXup2bmXzbv2UrOzmefXhqnb88HEk5lhDM/POZJoRgU9nZFFkWQ0vCCnT59qO97CPenbx+uGTTuaOXjYVcpFpJ8blJ1JRdkQKsraf7LIvoOHqW3YR83O5mMSzyvV29m6ex8tbQrND83LZnhBDuUFgygvyKG8MCcyDd4PL8hhYFZqJp/jTSzpW2u/G0KtQ411KkwkreUMyDwy+qw9Bw+3sLVhH5uCxLO1YR+1u/dRGySf5et30LD34DHthuZlU16Yw/D8QYwozAkSUfKTT4eJxcz20H4CMUBDnGKg4pMiEosBUdd3OtJ84BC1DfvY2rCPLbsiyWdLwz62NnSdfErzcxieP5DhBTmU5UdeU4YPYWY7p/XiocPE4u4xPy1S2lddp+KTIhIfudlZTCwZ3OlgoKb9h9i6ex+1u/ZR27CX2oZ91Dbso273Prbu3sfbmxuobzwAwMdOGdH7iUV6LlSvEWEi0nvyBnadfA4caiHcuL/D9fHQf8e7pYDqcJNqhIlISsnOyjhScTpREppYzOwyM1trZlVmtqid9WZmtwfrV5rZzK7amlmxmT1tZuuCaVGw/GIze93M3g6mSX108q7mA+xQ8UkRSUMJSyxmlgncAcwFpgHXmdm0NpvNBSqC13wiVZS7arsIWOruFcDS4D1APXCFu59E5PHJ9yfoq8WktfikToWJSLpJZI9lNlDl7iF3PwA8AMxrs8084D6PeAUoNLPyLtrOAxYH84uBKwHc/U133xIsXw3kmFnSHtBQHRSf1FBjEUk3iUwsI4FNUe9rgmWxbNNZ2zJ3rwUIpqUc6+PAm+5+zBUqM5tvZpVmVhkOh7vxdbonpOKTIpKmEplY2rs7v+19MR1tE0vb9j/UbDpwG/AP7a1397vdfZa7zyopKWlvk7ioVvFJEUlTifzVqwFGR70fBWyJcZvO2m4LTpcRTOtaNzKzUcDDwKfdvToO3+G4hcKNTOjgLlsRkf4skYllOVBhZuPNLBu4FljSZpslwKeD0WFnAA3B6a3O2i4hcnGeYPoIgJkVAn8Cbnb3lxL4vbp08HALG7Y36+FeIpKWEnaDpLsfMrOFwFNAJnCvu682swXB+ruAx4HLgSqgGfhsZ22DXd8KPGhmNwIbgWuC5QuBScC3zOxbwbJL3P1Ij6a3bNrRzKEWV49FRNJSQu+8d/fHiSSP6GV3Rc07kQeJxdQ2WL4duLCd5d8BvtPDkOOitfikeiwiko50ZTkBWocaTxymxCIi6UeJJQFC4SaGDc6mIHdAskMREel1SiwJUB1uZIJ6KyKSppRYEiBUr+KTIpK+lFjibGdTpPikaoSJSLpSYomz1qdGqsciIulKiSXOVNVYRNKdEkucVYcbGZBpjFLxSRFJU0oscRYKN6n4pIikNf36xVl1uJGJur4iImlMiSWODh5uYeP2Zj3cS0TSmhJLHLUWn9SFexFJZ0oscdQ6IkxDjUUknSmxxFFIxSdFRJRY4qk63KjikyKS9pRY4igUblLxSRFJe0oscRSqb2Jiqa6viEh6U2KJk9bik+qxiEi6U2KJk9bik+qxiEi6S2hiMbPLzGytmVWZ2aJ21puZ3R6sX2lmM7tqa2bFZva0ma0LpkXB8qFm9pyZNZrZzxL5vdpTXRcMNVaPRUTSXMISi5llAncAc4FpwHVmNq3NZnOBiuA1H7gzhraLgKXuXgEsDd4D7AO+Bfxzor5TZ6rrVXxSRAQS22OZDVS5e8jdDwAPAPPabDMPuM8jXgEKzay8i7bzgMXB/GLgSgB3b3L3vxBJML2uuq6JcSo+KSKS0MQyEtgU9b4mWBbLNp21LXP3WoBgWhrHmI9bqL5Rd9yLiJDYxGLtLPMYt4ml7XExs/lmVmlmleFwOB67PFJ8UjXCREQSm1hqgNFR70cBW2LcprO224LTZQTTuu4E5e53u/ssd59VUlLSnaYd2hgUn1RVYxGRxCaW5UCFmY03s2zgWmBJm22WAJ8ORoedATQEp7c6a7sEuCGYvwF4JIHfISahI48j1qkwEZGsRO3Y3Q+Z2ULgKSATuNfdV5vZgmD9XcDjwOVAFdAMfLaztsGubwUeNLMbgY3ANa2faWbrgXwg28yuBC5x93cS9R1bVQfFJ9VjERFJYGIBcPfHiSSP6GV3Rc07cFOsbYPl24ELO2gzrgfhHrdQa/HJQSo+KSKisbFxEAo3qbciIhJQYokDPedeROQoJZYe2tF0gJ3NBzXUWEQkoMTSQ6EjF+7VYxERASWWHmsdaqzikyIiEUosPVQdbiQ7M0PFJ0VEAkosPVQdbmLs0FwVnxQRCejXsIdC9Y26cC8iEkWJpQdai0/qwr2IyFFKLD3QWnxSPRYRkaOUWHqguk5DjUVE2lJi6YFQfTDUWD0WEZEjlFh6oLqukWGDB6r4pIhIFCWWHgjVN+k0mIhIG0osPRAKa6ixiEhbSizH6WjxSfVYRESiKbEcp9bik+qxiIh8kBLLcapWVWMRkXYpsRynULgpKD6Zm+xQRERSihLLcaoONzFuWC6ZGZbsUEREUkpCE4uZXWZma82syswWtbPezOz2YP1KM5vZVVszKzazp81sXTAtilp3c7D9WjO7NJHfLRRu1DNYRETakbDEYmaZwB3AXGAacJ2ZTWuz2VygInjNB+6Moe0iYKm7VwBLg/cE668FpgOXAT8P9hN3Bw+3sHFHMxNLdX1FRKStRPZYZgNV7h5y9wPAA8C8NtvMA+7ziFeAQjMr76LtPGBxML8YuDJq+QPuvt/d3weqgv3E3YbtkeKT6rGIiBwrkYllJLAp6n1NsCyWbTprW+butQDBtLQbn4eZzTezSjOrDIfD3fpC0S4/aTjTRuQfd3sRkf4qkYmlvavaHuM2sbQ9ns/D3e9291nuPqukpKSLXbZvUulgfn79aZxQrsQiItJWIhNLDTA66v0oYEuM23TWdltwuoxgWteNzxMRkQRLZGJZDlSY2XgzyyZyYX1Jm22WAJ8ORoedATQEp7c6a7sEuCGYvwF4JGr5tWY20MzGExkQ8FqivpyIiLQvK1E7dvdDZrYQeArIBO5199VmtiBYfxfwOHA5kQvtzcBnO2sb7PpW4EEzuxHYCFwTtFltZg8C7wCHgJvc/XCivp+IiLTP3Lu6dNF/zZo1yysrK5MdhohIn2Jmr7v7rI7W6857ERGJKyUWERGJKyUWERGJKyUWERGJq7S+eG9mYWBDD3YxDKiPUzjxpLi6R3F1j+Lqnv4Y11h37/AO87ROLD1lZpWdjYxIFsXVPYqrexRX96RjXDoVJiIicaXEIiIicaXE0jN3JzuADiiu7lFc3aO4uift4tI1FhERiSv1WEREJK6UWEREJK6UWI6DmV1mZmvNrMrMFvXSZ643s7fNbIWZVQbLis3saTNbF0yLora/OYhvrZldGrX8tGA/VWZ2u5m194C0zuK418zqzGxV1LK4xRE89uD3wfJXzWxcD+K6xcw2B8dshZldnoS4RpvZc2a2xsxWm9mXU+GYdRJXUo+ZmeWY2Wtm9lYQ17+nyPHqKK5U+DuWaWZvmtljqXCsAHB3vbrxIlLGvxqYAGQDbwHTeuFz1wPD2iz7T2BRML8IuC2YnxbENRAYH8SbGax7DTiTyBM3nwDmdjOOc4CZwKpExAF8EbgrmL8W+H0P4roF+Od2tu3NuMqBmcH8EOC94POTesw6iSupxyzYx+BgfgDwKnBGChyvjuJKhb9jXwV+CzyWMv8eu/OjopcTHPynot7fDNzcC5+7nmMTy1qgPJgvB9a2FxOR59qcGWzzbtTy64BfHEcs4/jgD3jc4mjdJpjPInJnsB1nXB39o+/VuNp89iPAxalyzNqJK2WOGZALvAHMSaXj1SaupB4vIk/KXQpcwNHEkvRjpVNh3TcS2BT1viZYlmgO/NnMXjez+cGyMo88cZNgWtpFjCOD+bbLeyqecRxp4+6HgAZgaA9iW2hmKy1yqqz1lEBS4gpOI5xK5H+7KXPM2sQFST5mwamdFUQeO/60u6fE8eogLkju8fox8HWgJWpZ0o+VEkv3tXdNojfGbJ/l7jOBucBNZnZOJ9t2FGNvx348ccQzxjuBicAMoBb4QbLiMrPBwEPAV9x9d2eb9mZs7cSV9GPm7ofdfQaR/43PNrMTO/sKSY4racfLzD4K1Ln7613F3lsxtVJi6b4aYHTU+1HAlkR/qLtvCaZ1wMPAbGCbmZUDBNO6LmKsCebbLu+peMZxpI2ZZQEFwI7jCcrdtwU/Bi3APUSOWa/HZWYDiPx4/8bd/xgsTvoxay+uVDlmQSy7gOeBy0iB49VeXEk+XmcBHzOz9cADwAVm9mtS4FgpsXTfcqDCzMabWTaRC1pLEvmBZpZnZkNa54FLgFXB594QbHYDkfPkBMuvDUZ0jAcqgNeCbvEeMzsjGPXx6ag2PRHPOKL3dTXwrAcneLur9R9X4Coix6xX4wr280tgjbv/MGpVUo9ZR3El+5iZWYmZFQbzg4CLgHdJ/vFqN65kHi93v9ndR7n7OCK/Q8+6+6eSfaxag9Ormy/gciKjaKqB/9MLnzeByGiOt4DVrZ9J5FznUmBdMC2OavN/gvjWEjXyC5hF5C9/NfAzun+R93dEuvwHifxv5sZ4xgHkAH8AqoiMVJnQg7juB94GVgb/QMqTENfZRE4drARWBK/Lk33MOokrqccMOBl4M/j8VcC/xfvvepzjSvrfsaDteRy9eJ/0f48q6SIiInGlU2EiIhJXSiwiIhJXSiwiIhJXSiwiIhJXSiwiIhJXSiwix8HMhtrRirZb7YMVbrO7aDvLzG7v5ud9Lqg+u9LMVpnZvGD5Z8xsRE++i0i8abixSA+Z2S1Ao7t/P2pZlkdqK8Vj/6OAZUSqETcEZVhK3P19M3ueSBHEynh8lkg8qMciEidm9t9m9kMzew64zcxmm9lfLfKsjL+a2ZRgu/Ps6LMzbgmKFz5vZiEz+1I7uy4F9gCNAO7eGCSVq4nc2PaboKc0yCLP1VhmkWKlT0WV9njezH4cxLHKzGa38zkicaHEIhJfk4GL3P1rREqRnOPupwL/Bny3gzZTgUuJ1Jn6dlDDK9pbwDbgfTP7lZldAeDu/wNUAtd7pDjiIeCnwNXufhpwL/B/o/aT5+4fIvKMjXt7/E1FOpCV7ABE+pk/uPvhYL4AWGxmFUTKp7RNGK3+5O77gf1mVgeUEVXG3N0Pm9llwOnAhcCPzOw0d7+lzX6mACcCT0dKPpFJpMxNq98F+3vBzPLNrNAjBRVF4kqJRSS+mqLm/wN4zt2vssgzT57voM3+qPnDtPPv0iMXQ18DXjOzp4FfEXnIVDQDVrv7mR18TtsLqrrAKgmhU2EiiVMAbA7mP3O8OzGzEWY2M2rRDGBDML+HyKOFIVJYsMTMzgzaDTCz6VHtPhEsPxtocPeG441JpDPqsYgkzn8SORX2VeDZHuxnAPD9YFjxPiAMLAjW/Tdwl5ntJfKY2auB282sgMi/7x8TqYgNsNPM/grkA5/rQTwindJwY5E0oGHJ0pt0KkxEROJKPRYREYkr9VhERCSulFhERCSulFhERCSulFhERCSulFhERCSu/n8NIG8jg1ga6QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "temp_learning_rate_schedule = CustomSchedule(config.d_model)\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.xlabel(\"Train Step\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8b484b0",
   "metadata": {},
   "source": [
    "# Loss and Metrics\n",
    "Since the target sequences are padded, it is important to apply a padding mask when calculating the loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "9ce47421",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:38.378082Z",
     "start_time": "2022-08-23T18:35:38.374635Z"
    }
   },
   "outputs": [],
   "source": [
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "    from_logits=True, reduction='none')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "dbdf296c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:38.632350Z",
     "start_time": "2022-08-23T18:35:38.628179Z"
    }
   },
   "outputs": [],
   "source": [
    "def loss_function(real, pred):\n",
    "    # performs logical NOT of real==0\n",
    "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "    loss_ = loss_object(real, pred)\n",
    "\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "    \n",
    "    # computes the sum of elements across dimensions of a tensor.\n",
    "    return tf.reduce_sum(loss_)/tf.reduce_sum(mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "309babf0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:40.856434Z",
     "start_time": "2022-08-23T18:35:40.848724Z"
    }
   },
   "outputs": [],
   "source": [
    "def accuracy_function(real, pred):\n",
    "  accuracies = tf.equal(real, tf.argmax(pred, axis=2))\n",
    "\n",
    "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "  accuracies = tf.math.logical_and(mask, accuracies)\n",
    "\n",
    "  accuracies = tf.cast(accuracies, dtype=tf.float32)\n",
    "  mask = tf.cast(mask, dtype=tf.float32)\n",
    "  return tf.reduce_sum(accuracies)/tf.reduce_sum(mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "b1a0e574",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:41.860507Z",
     "start_time": "2022-08-23T18:35:41.839335Z"
    }
   },
   "outputs": [],
   "source": [
    "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
    "train_accuracy = tf.keras.metrics.Mean(name='train_accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8300fd9e",
   "metadata": {},
   "source": [
    "# Training and checkpointing\n",
    "Create the checkpoint path and the checkpoint manager. This will be used to save checkpoints every n epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "55fe03d0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:44.055865Z",
     "start_time": "2022-08-23T18:35:43.226539Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Latest checkpoint restored!!\n"
     ]
    }
   ],
   "source": [
    "checkpoint_path = './checkpoints/train'\n",
    "\n",
    "ckpt = tf.train.Checkpoint(transformer=transformer,\n",
    "                           optimizer=optimizer)\n",
    "\n",
    "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
    "\n",
    "# if a checkpoint exists, restore the latest checkpoint.\n",
    "if ckpt_manager.latest_checkpoint:\n",
    "  ckpt.restore(ckpt_manager.latest_checkpoint)\n",
    "  print('Latest checkpoint restored!!')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb3221fc",
   "metadata": {},
   "source": [
    "The target is divided into tar_inp and tar_real. tar_inp is passed as an input to the decoder. tar_real is that same input shifted by 1: At each location in tar_input, tar_real contains the next token that should be predicted.\n",
    "\n",
    "Training uses teacher forcing. Teacher forcing is passing the true output to the next time step regardless of what the model predicts at the current time step.\n",
    "\n",
    "As the transformer predicts each word, self-attention allows it to look at the previous words in the input sequence to better predict the next word.\n",
    "\n",
    "To prevent the model from peaking at the expected output, the model uses a look-ahead mask."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "756a9d8a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:44.875072Z",
     "start_time": "2022-08-23T18:35:44.869191Z"
    }
   },
   "outputs": [],
   "source": [
    "def create_masks(inp, tar):\n",
    "    # Used in the 2nd attention block in the decoder.\n",
    "    # This padding mask is used to mask the encoder outputs.\n",
    "    dec_padding_mask = create_padding_mask(inp)\n",
    "\n",
    "    # Used in the 1st attention block in the decoder.\n",
    "    # It is used to pad and mask future tokens in the input received by \n",
    "    # the decoder.\n",
    "    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
    "    dec_target_padding_mask = create_padding_mask(tar)\n",
    "    combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
    "\n",
    "    return combined_mask, dec_padding_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "19c40951",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:35:46.113862Z",
     "start_time": "2022-08-23T18:35:46.107810Z"
    }
   },
   "outputs": [],
   "source": [
    "# The @tf.function trace-compiles train_step into a TF graph for faster\n",
    "# execution. The function specializes to the precise shape of the argument\n",
    "# tensors. To avoid re-tracing due to the variable sequence lengths or variable\n",
    "# batch sizes (the last batch is smaller), use input_signature to specify\n",
    "# more generic shapes.\n",
    "@tf.function\n",
    "def train_st(inp, tar):\n",
    "    tar_inp = tar[:, :-1]\n",
    "    tar_real = tar[:, 1:]\n",
    "\n",
    "    combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
    "\n",
    "    # tf.GradientTape() records operations for automatic differentiation\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions, _ = transformer(inp, tar_inp, \n",
    "                                     True,\n",
    "                                     combined_mask,\n",
    "                                     dec_padding_mask)\n",
    "        loss = loss_function(tar_real, predictions)\n",
    "\n",
    "    gradients = tape.gradient(loss, transformer.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
    "\n",
    "    train_loss(loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c470a31",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:33:28.623529Z",
     "start_time": "2022-08-23T18:31:18.360738Z"
    }
   },
   "outputs": [],
   "source": [
    "EPOCHS = 20\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "\n",
    "    train_loss.reset_states()\n",
    "    train_accuracy.reset_states()\n",
    "\n",
    "    # inp -> dutch, tar -> english\n",
    "    for (batch, (inp, tar)) in enumerate(train_dataset):\n",
    "        train_st(inp, tar)\n",
    "\n",
    "        if batch % 10 == 0:\n",
    "            print('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
    "                epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n",
    "\n",
    "    if (epoch + 1) % 1 == 0:\n",
    "        ckpt_save_path = ckpt_manager.save()\n",
    "        print('Saving checkpoint for epoch {} at {}'.format(epoch + 1,\n",
    "                                                            ckpt_save_path))\n",
    "\n",
    "    print('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1,\n",
    "                                                        train_loss.result(),\n",
    "                                                        train_accuracy.result()))\n",
    "\n",
    "    print('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72fb637a",
   "metadata": {},
   "source": [
    "# Evaluate\n",
    "\n",
    "The following steps are used for evaluation:\n",
    "\n",
    "Encode the input sentence using the english tokenizer (tokenizer_en). Moreover, add the start and end token so the input is equivalent to what the model is trained with. This is the encoder input.\n",
    "\n",
    "The decoder input is the start token == tokenizer_nld.vocab_size.\n",
    "\n",
    "Calculate the padding masks and the look ahead masks.\n",
    "\n",
    "The decoder then outputs the predictions by looking at the encoder output and its own output (self-attention).\n",
    "\n",
    "Select the last word and calculate the argmax of that.\n",
    "\n",
    "Concatentate the predicted word to the decoder input as pass it to the decoder.\n",
    "\n",
    "In this approach, the decoder predicts the next word based on the previous words it predicted."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "e658be32",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:36:02.165081Z",
     "start_time": "2022-08-23T18:36:02.160010Z"
    }
   },
   "outputs": [],
   "source": [
    "def encode_en(en):\n",
    "    seq_length = MAX_SEQ_LENGTH\n",
    "    tokens_en = tokenizer_en.tokenize(en)\n",
    "    lang1 = tokenizer_en.convert_tokens_to_ids(['[CLS]'] + tokens_en + ['[SEP]'])\n",
    "    if len(lang1)<seq_length:\n",
    "        # makes token list equal to length of seq_length\n",
    "        lang1 = lang1 + list(np.zeros(seq_length - len(lang1), 'int32'))\n",
    "    return lang1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "b1edf94c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:36:03.159951Z",
     "start_time": "2022-08-23T18:36:03.152091Z"
    }
   },
   "outputs": [],
   "source": [
    "def evaluate(transformer, inp_sentence):\n",
    "    # normalize input sentence\n",
    "    inp_sentence = encode_en(inp_sentence)\n",
    "#     encoder_input = inp_sentence\n",
    "    encoder_input = tf.expand_dims(inp_sentence, 0)\n",
    "#     multiply = tf.constant([64])\n",
    "#     encoder_input = tf.reshape(tf.tile(encoder_input, multiply), [ multiply[0], tf.shape(encoder_input)[0]])\n",
    "\n",
    "    # as the target is english, the first word to the transformer should be the\n",
    "    # english start token.\n",
    "    decoder_input = [tokenizer_nld.vocab_size]\n",
    "    output = tf.expand_dims(decoder_input, 0)\n",
    "\n",
    "    for i in range(MAX_SEQ_LENGTH):\n",
    "        combined_mask, dec_padding_mask = create_masks(\n",
    "            encoder_input, output)\n",
    "\n",
    "        # predictions.shape == (batch_size, seq_len, vocab_size)\n",
    "        predictions, attention_weights = transformer(encoder_input,\n",
    "                                                     output,\n",
    "                                                     False,\n",
    "                                                     combined_mask,\n",
    "                                                     dec_padding_mask)\n",
    "\n",
    "        # select the last word from the seq_len dimension\n",
    "        predictions = predictions[:, -1:, :]  # (batch_size, 1, vocab_size)\n",
    "\n",
    "        predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
    "\n",
    "        # return the result if the predicted_id is equal to the end token\n",
    "        if tf.equal(predicted_id, tokenizer_nld.vocab_size + 1):\n",
    "            return tf.squeeze(output, axis=0), attention_weights\n",
    "\n",
    "        # concatentate the predicted_id to the output which is given to the decoder\n",
    "        # as its input.\n",
    "        output = tf.concat([output, predicted_id], axis=-1)\n",
    "\n",
    "    return tf.squeeze(output, axis=0), attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "e3fd3e59",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:36:04.872983Z",
     "start_time": "2022-08-23T18:36:04.868946Z"
    }
   },
   "outputs": [],
   "source": [
    "def translate(transformer, sentence):\n",
    "    result, attention_weights = evaluate(transformer, sentence)\n",
    "\n",
    "    predicted_sentence = tokenizer_nld.decode([i for i in result\n",
    "                                              if i < tokenizer_nld.vocab_size])\n",
    "\n",
    "    print('Input: {}'.format(sentence))\n",
    "    print('Predicted translation: {}'.format(predicted_sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d9d211a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:34:47.013896Z",
     "start_time": "2022-08-23T18:34:46.951564Z"
    }
   },
   "outputs": [],
   "source": [
    "translate(transformer, 'time to chase money.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c54d06be",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T14:09:55.560374Z",
     "start_time": "2022-08-08T14:09:55.511946Z"
    }
   },
   "outputs": [],
   "source": [
    "translate(transformer, 'when it rains, it pours.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5064f154",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T14:09:56.127490Z",
     "start_time": "2022-08-08T14:09:56.089413Z"
    }
   },
   "outputs": [],
   "source": [
    "translate(transformer, 'you fumble the bag.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "29b67bc1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T14:10:00.832122Z",
     "start_time": "2022-08-08T14:10:00.785366Z"
    }
   },
   "outputs": [],
   "source": [
    "translate(transformer, 'we want happiness.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "669d73f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "translate(transformer, \"please don't kill my vibe.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95be89d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "translate(transformer, 'i am proud of you.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2438580",
   "metadata": {},
   "outputs": [],
   "source": [
    "translate(transformer, 'plentiful')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "206cfc97",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T13:52:27.124013Z",
     "start_time": "2022-08-08T13:52:27.124008Z"
    }
   },
   "source": [
    "# Save weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "5df753d4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:36:16.694516Z",
     "start_time": "2022-08-23T18:36:15.110937Z"
    }
   },
   "outputs": [],
   "source": [
    "transformer.save_weights('bert_nmt_ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "a6889451",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-23T18:36:24.704380Z",
     "start_time": "2022-08-23T18:36:22.399649Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x16c00a940>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_transformer = Transformer(config=config,\n",
    "                          target_vocab_size=target_vocab_size,\n",
    "                          bert_config_file=bert_config_file)\n",
    "  \n",
    "fn_out, _ = new_transformer(inp, tar_inp, \n",
    "                        True,\n",
    "                        look_ahead_mask=None,\n",
    "                        dec_padding_mask=None)\n",
    "new_transformer.load_weights('bert_nmt_ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e6a4500",
   "metadata": {},
   "outputs": [],
   "source": [
    "translate(new_transformer, 'go to a party every day.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1697bb62",
   "metadata": {},
   "outputs": [],
   "source": [
    "translate(new_transformer, 'i dont like you, i love you.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac234afc",
   "metadata": {},
   "outputs": [],
   "source": [
    "translate(new_transformer, 'i drink a lot of water.')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:miniforge3-tensorflow] *",
   "language": "python",
   "name": "conda-env-miniforge3-tensorflow-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}