many_to_many_fine_tune_demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "many_to_many_fine_tune_demo.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dq0kkUAc5O5n",
        "colab_type": "text"
      },
      "source": [
        "## Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dQO2Qp272_ho",
        "colab_type": "code",
        "outputId": "da932459-2eab-4c52-a2fb-66e893bebcc3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        }
      },
      "source": [
        "!pip uninstall -y tensorflow numpy && pip install tensorflow-gpu==1.14.0 tf_sentencepiece"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Uninstalling tensorflow-1.14.0:\n",
            "  Successfully uninstalled tensorflow-1.14.0\n",
            "Uninstalling numpy-1.16.5:\n",
            "  Successfully uninstalled numpy-1.16.5\n",
            "Collecting tensorflow-gpu==1.14.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/76/04/43153bfdfcf6c9a4c38ecdb971ca9a75b9a791bb69a764d652c359aca504/tensorflow_gpu-1.14.0-cp36-cp36m-manylinux1_x86_64.whl (377.0MB)\n",
            "\u001b[K     |████████████████████████████████| 377.0MB 63kB/s \n",
            "\u001b[?25hCollecting tf_sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/dc/2c/20800032089a9271757921f3adc1f2c7ec2d294ec9fa07b3115fab9d27c2/tf_sentencepiece-0.1.83-py2.py3-none-manylinux1_x86_64.whl (2.7MB)\n",
            "\u001b[K     |████████████████████████████████| 2.7MB 39.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.8.0)\n",
            "Collecting numpy<2.0,>=1.14.5 (from tensorflow-gpu==1.14.0)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/e6/c3fdc53aed9fa19d6ff3abf97dfad768ae3afce1b7431f7500000816bda5/numpy-1.17.2-cp36-cp36m-manylinux1_x86_64.whl (20.4MB)\n",
            "\u001b[K     |████████████████████████████████| 20.4MB 40.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.1.7)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.1.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.11.2)\n",
            "Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.14.0)\n",
            "Requirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.14.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.8.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.12.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.33.6)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.1.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.15.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (3.7.1)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.2.2)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.0.8)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (0.15.6)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (41.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (3.1.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==1.14.0) (2.8.0)\n",
            "\u001b[31mERROR: stable-baselines 2.2.1 requires tensorflow>=1.5.0, which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: magenta 0.3.19 requires tensorflow>=1.12.0, which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: fancyimpute 0.4.3 requires tensorflow, which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Installing collected packages: numpy, tensorflow-gpu, tf-sentencepiece\n",
            "Successfully installed numpy-1.17.2 tensorflow-gpu-1.14.0 tf-sentencepiece-0.1.83\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xRYbl_We431V",
        "colab_type": "code",
        "outputId": "054f2c89-1c51-4391-a2f6-f032b5c782c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "!curl -o many_to_many_fine_tune.zip https://dl.dropboxusercontent.com/s/3ixjojbpho2emd8/many_to_many_fine_tune.zip?dl=0"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  114M  100  114M    0     0  42.5M      0  0:00:02  0:00:02 --:--:-- 42.4M\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b1kWSRVb5F-U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip -q many_to_many_fine_tune.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UA7Nd3qo5KM-",
        "colab_type": "code",
        "outputId": "d5ec02bf-9bc1-495f-a9ff-aecad29ce558",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "!curl -o mlt.zip https://codeload.github.com/suyash/mlt/zip/master"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100 38721    0 38721    0     0   115k      0 --:--:-- --:--:-- --:--:--  115k\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vg2sGM4N5Ln9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip -q mlt.zip && mv mlt-master/mlt ./"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SuEV4CUW5ROy",
        "colab_type": "text"
      },
      "source": [
        "## Many to Many Fine-Tuned Translation\n",
        "\n",
        "This takes the pretrained model from Many-to-Many and trains on pt-fr and pt-en with pt used as the 4th factor that was in the model but was never used when training many-to-many"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z1oy_H3o5MvI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import tf_sentencepiece as tfs\n",
        "\n",
        "from mlt.evaluation import predict\n",
        "from mlt.layers import Attention, ConditionalNormalization, Gelu, MultiplyConstant, PaddingAndLookaheadMask, PaddingMask, PositionalEncoding"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXVjFAP55Y9P",
        "colab_type": "code",
        "outputId": "b7676e83-298b-46fc-98ea-f781df0e5543",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tf.__version__"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'1.14.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1UrtqoA85aLG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sess = tf.InteractiveSession()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JAfwOm9x5baX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "encoding_model_file = \"./final/sentencepiece/en_de_fr_pt_shared/encoding.model\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sbrg8m1U5fIW",
        "colab_type": "code",
        "outputId": "294cf3f3-e9b4-4e67-e751-ee34cfc0ad81",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        }
      },
      "source": [
        "with sess.as_default():\n",
        "    model = tf.keras.experimental.load_from_saved_model(\"final/model114\", custom_objects={\n",
        "        \"MultiplyConstant\": MultiplyConstant,\n",
        "        \"PositionalEncoding\": PositionalEncoding,\n",
        "        \"PaddingMask\": PaddingMask,\n",
        "        \"PaddingAndLookaheadMask\": PaddingAndLookaheadMask,\n",
        "        \"Attention\": Attention,\n",
        "        \"ConditionalNormalization\": ConditionalNormalization,\n",
        "        \"Gelu\": Gelu,\n",
        "    })"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling Ones.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-lDWzHH7D-9k",
        "colab_type": "text"
      },
      "source": [
        "### English to Portuguese\n",
        "\n",
        "__NOTE:__ This is Few-Shot."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8KlXV7cLD_oS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"This is a problem that we need to solve.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qcMZ6j_lEB3g",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        },
        "outputId": "2aeb67a1-80ce-4fb9-8cf3-b87d48d349a5"
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 0.0, 0.0, 1.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensor2tensor/utils/beam_search.py:727: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.cast` instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:2403: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tLXca0f1EEVM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sb8fgqBqEHio",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QJ1nH7hGEJZW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "a4fc2caf-7c38-4c44-8da5-87bb4019b077"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['é um problema que precisamos de resolver .',\n",
              "  'isto é um problema que precisamos de resolver .',\n",
              "  'este é um problema que precisamos de resolver .',\n",
              "  'é um problema que temos de resolver .',\n",
              "  'isto é um problema para resolver .'],\n",
              " array([0.48925328, 0.40772614, 0.33876425, 0.20045164, 0.15399095],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VV5c-9BoIhQv",
        "colab_type": "text"
      },
      "source": [
        "### Portuguese to English\n",
        "\n",
        "__NOTE__: This is few shot"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "04vkIBCfIh77",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Este é um problema que temos que resolver.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GlOjcVg0IjmY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 0.0, 0.0, 1.0]),\n",
        "    tarf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xnuEG9V8IlQg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h9KbtgLSInRL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dAzAb_36Io51",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "151489d3-1a87-42c4-e603-e5ee605402ea"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['it is a problem that we have to solve .',\n",
              "  'is a problem that we have to solve .',\n",
              "  'eastern is a problem that we have to solve .',\n",
              "  'it is a problem that we need to solve .',\n",
              "  'is a problem that we need to solve .'],\n",
              " array([0.13502961, 0.12296236, 0.1034094 , 0.10161183, 0.09352253],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ivyXAH3II2Av",
        "colab_type": "text"
      },
      "source": [
        "### Portuguese to French\n",
        "\n",
        "__NOTE__: This is few shot"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yyrK4o05I7GC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Este é um problema que temos que resolver.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KFSr8BqQI85D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 0.0, 0.0, 1.0]),\n",
        "    tarf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-zubYwKKI-tD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0rUaAgZ4JAwG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rLMW1yARJCWB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "b58ca727-5724-4321-8397-980192f23d09"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([\"c'est un problème que nous devons régler .\",\n",
              "  \"c'est un problème auquel nous devons nous régler .\",\n",
              "  \"eh bien , c'est un problème que nous devons régler .\",\n",
              "  \"c'est un problème auquel nous devons résoudre .\",\n",
              "  \"c'est un problème dont nous devons régler .\"],\n",
              " array([0.29969594, 0.24713854, 0.23395146, 0.19806242, 0.19502234],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Ng0TKQ2JHaA",
        "colab_type": "text"
      },
      "source": [
        "### French to Portuguese\n",
        "\n",
        "__NOTE__: This is few shot"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iHGyGYDkJGvk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"c'est un problème que nous devons résoudre.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bjtrlKZ5JNlk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 0.0, 0.0, 1.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EuDqfs2lJPNv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ExDI-ER_JRCF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mF1cM2zFJS6x",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "a7f70f92-0fb4-4561-ad9f-6eaa780c3a34"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['é um problema que precisamos de resolver .',\n",
              "  'é um problema que temos de resolver .',\n",
              "  'é um problema que precisamos disso .',\n",
              "  'é um problema que precisamos de solucionar .',\n",
              "  'é um problema que temos que resolver .'],\n",
              " array([0.4475441 , 0.3857079 , 0.23617241, 0.20825288, 0.19200097],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hk_fKJPTJeq1",
        "colab_type": "text"
      },
      "source": [
        "### Portuguese to German\n",
        "\n",
        "__NOTE:__ This is zero-shot, training was done for en-fr, fr-de, fr-pt and pt-en"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AGE9Qx--JeOq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Este é um problema que temos que resolver.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mUJAijf0J0tK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 0.0, 0.0, 1.0]),\n",
        "    tarf=tf.constant([0.0, 0.0, 1.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UN-hzFmLJ1fA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2nb0U12EJ37y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZfxBKCbqJ6TH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "2b5a25ef-2552-42cf-8426-12619981c6fe"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['egonisch ist ein Problem, das wir begriffen müssen .',\n",
              "  'eh , es ist ein Problem, das wir begriffen müssen .',\n",
              "  'eh , es ist ein Problem, das wir bereichern müssen .',\n",
              "  'eh , es ist ein Problem , das wir uns müssen .',\n",
              "  'egonisch ist ein Problem, das wir begriffen müssen.'],\n",
              " array([0.11276496, 0.09135193, 0.07864954, 0.05628131, 0.04623901],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FRMFDIPYKEFo",
        "colab_type": "text"
      },
      "source": [
        "### German to Portuguese\n",
        "\n",
        "__NOTE:__ This is zero-shot, training was done for en-fr, fr-de, fr-pt and pt-en"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ypcMRxwKDmP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Dies ist ein Problem, das wir lösen müssen.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1GJEsoySKGFD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 0.0, 1.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 0.0, 0.0, 1.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gYF9wOwkKHqZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iEg6AIBaKJP9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OmDS_IwMKKvW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "98e95ca5-fdeb-4762-fdc4-a87b42261d8f"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['isto é um problema que temos de resolver .',\n",
              "  'isto é um problema que temos que resolver .',\n",
              "  'isto é um problema que precisamos de resolver .',\n",
              "  'é um problema que temos de resolver .',\n",
              "  'é um problema que temos que resolver .'],\n",
              " array([0.50007445, 0.3520581 , 0.306182  , 0.24348892, 0.1991398 ],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FgKgIm625jRP",
        "colab_type": "text"
      },
      "source": [
        "### English to French"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XH1XFVga5gZ5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"This is a problem that we need to solve.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cN0jhGry5odk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8MsOxURV5qH6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "66oTPUsR5req",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o883Aol85xNy",
        "colab_type": "code",
        "outputId": "0827ce08-961a-40fe-b500-e108c577ae8e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([\"C'est un problème que nous devons résoudre.\",\n",
              "  \"C'est un problème qui nous faut résoudre.\",\n",
              "  \"C'est un problème qu'il faut résoudre.\",\n",
              "  \"C'est un problème auquel nous devons résoudre.\",\n",
              "  \"C'est un problème pour résoudre ce problème.\"],\n",
              " array([0.4547706 , 0.31329823, 0.25585213, 0.25111327, 0.21929555],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UpaALHvr56x0",
        "colab_type": "text"
      },
      "source": [
        "### English to German"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QgjkgisG5yV1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"This is a problem that we need to solve.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tXYizgI06C_O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 0.0, 1.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "en6IYi_M6D5-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nmlqX_lw6FI6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c1dl-hAE6GLG",
        "colab_type": "code",
        "outputId": "adffe3b0-381a-4b1d-c439-c64e85ecb327",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Das ist ein Problem, das wir brauchen, um zu lösen.',\n",
              "  'Dies ist ein Problem, das wir brauchen, um zu lösen.',\n",
              "  'Das ist ein Problem, das wir beheben müssen.',\n",
              "  'Es ist ein Problem, das wir beheben müssen.',\n",
              "  'Das ist ein Problem, das wir brauchen.'],\n",
              " array([0.6969994 , 0.5009153 , 0.38670796, 0.2249565 , 0.22388114],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gt3aez4k6Lzv",
        "colab_type": "text"
      },
      "source": [
        "### French to English"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDy9PKeP6HbX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"c'est un problème que nous devons résoudre.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULV7SQu76S7f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yBceuuwB6Uaz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Waq32kLs6Vyy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DXXxpcLk6XBL",
        "colab_type": "code",
        "outputId": "20249281-079c-438a-b0a2-cd1863d677a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['This is a problem that we need to resolve.',\n",
              "  'This is a problem we need to resolve.',\n",
              "  'This is a problem that we have to solve.',\n",
              "  'This is a problem that we need to solve.',\n",
              "  'This is a problem that we have to resolve.'],\n",
              " array([0.3886431 , 0.30511987, 0.30445454, 0.28317395, 0.2635785 ],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LgWiupN-6b-N",
        "colab_type": "text"
      },
      "source": [
        "### German to English"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FWkLJP9Y6YX_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Hier finden Sie alle Antworten, die Sie brauchen.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k33TP3wZ6e7v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F2eNZu0K6f9Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_YeRSjk36hL0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Armf6bX6icz",
        "colab_type": "code",
        "outputId": "de66f172-ff90-43aa-aeae-2cf544b20ad8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Here you will find all the answers you need.',\n",
              "  'Here you can find all the answers you need.',\n",
              "  \"Here you'll find all the answers you need.\",\n",
              "  'Here you find all the answers you need.',\n",
              "  'Here you will find all the answers, you need answers.'],\n",
              " array([0.52666706, 0.37181923, 0.25515792, 0.25378272, 0.20513035],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wOXUw8D2AC8e",
        "colab_type": "text"
      },
      "source": [
        "### French to German\n",
        "\n",
        "__NOTE:__ This is zero-shot, training was done for en-fr and en-de"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jtRMi1z8AAcv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"c'est un problème que nous devons résoudre.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EYJOTUKAAFYg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 0.0, 1.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4oI86nyZAIDQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kw1RYMp5AJ2J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8z8tUfyNARRk",
        "colab_type": "code",
        "outputId": "19f57ed8-3380-4f60-bac3-15c43b0e64f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Das ist ein Problem, das wir uns tun müssen.',\n",
              "  'Das ist ein Problem, das wir uns ärkt wissen müssen.',\n",
              "  'Das ist ein Problem, das wir uns ärkt.',\n",
              "  'Das ist ein Problem, das wir ärkt.',\n",
              "  'Das ist ein Problem, das wir uns ärkt haben.'],\n",
              " array([0.4368739 , 0.22346888, 0.18264575, 0.18028234, 0.16595228],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g7eildiIAUf9",
        "colab_type": "text"
      },
      "source": [
        "### German to French\n",
        "\n",
        "__NOTE:__ This is zero-shot, training was done for en-fr and en-de"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MFlaF9qUASCc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Hier finden Sie alle Antworten, die Sie brauchen.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DYbeerUpAXRZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 0.0, 1.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uOc2M156AZVt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Za-CAH6ZAb-Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Dlm8_NkAdy6",
        "colab_type": "code",
        "outputId": "bc8394ba-e9a9-4379-a880-9066f2b187ed",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Vous trouverez ici toutes les réponses que vous avez besoin.',\n",
              "  'Ici vous trouverez toutes les réponses que vous avez besoin.',\n",
              "  'Ici vous trouverez toutes les réponses que vous devez.',\n",
              "  'Vous trouverez ici toutes les réponses que vous devez.',\n",
              "  'Ici, vous trouverez toutes les réponses que vous avez besoin.'],\n",
              " array([0.48981324, 0.38794842, 0.3431491 , 0.3424902 , 0.2872332 ],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2WKbUxHh6jlB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sess.close()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}