many_to_many_demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "many_to_many_demo.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dq0kkUAc5O5n",
        "colab_type": "text"
      },
      "source": [
        "## Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dQO2Qp272_ho",
        "colab_type": "code",
        "outputId": "5da14408-4d8f-4c5a-f190-5f6266d5eda7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 697
        }
      },
      "source": [
        "!pip uninstall -y tensorflow numpy && pip install tensorflow-gpu==1.14.0 tf_sentencepiece"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Uninstalling tensorflow-1.14.0:\n",
            "  Successfully uninstalled tensorflow-1.14.0\n",
            "Uninstalling numpy-1.16.5:\n",
            "  Successfully uninstalled numpy-1.16.5\n",
            "Collecting tensorflow-gpu==1.14.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/76/04/43153bfdfcf6c9a4c38ecdb971ca9a75b9a791bb69a764d652c359aca504/tensorflow_gpu-1.14.0-cp36-cp36m-manylinux1_x86_64.whl (377.0MB)\n",
            "\u001b[K     |████████████████████████████████| 377.0MB 49kB/s \n",
            "\u001b[?25hCollecting tf_sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/dc/2c/20800032089a9271757921f3adc1f2c7ec2d294ec9fa07b3115fab9d27c2/tf_sentencepiece-0.1.83-py2.py3-none-manylinux1_x86_64.whl (2.7MB)\n",
            "\u001b[K     |████████████████████████████████| 2.7MB 27.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.1.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.12.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.11.2)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.8.0)\n",
            "Collecting numpy<2.0,>=1.14.5 (from tensorflow-gpu==1.14.0)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/e6/c3fdc53aed9fa19d6ff3abf97dfad768ae3afce1b7431f7500000816bda5/numpy-1.17.2-cp36-cp36m-manylinux1_x86_64.whl (20.4MB)\n",
            "\u001b[K     |████████████████████████████████| 20.4MB 30.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.33.6)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.15.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.0.8)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.8.0)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.2.2)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.1.0)\n",
            "Requirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.14.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (3.7.1)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (0.1.7)\n",
            "Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==1.14.0) (1.14.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==1.14.0) (2.8.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (41.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (3.1.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow-gpu==1.14.0) (0.15.6)\n",
            "\u001b[31mERROR: stable-baselines 2.2.1 requires tensorflow>=1.5.0, which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: magenta 0.3.19 requires tensorflow>=1.12.0, which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: fancyimpute 0.4.3 requires tensorflow, which is not installed.\u001b[0m\n",
            "\u001b[31mERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[31mERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Installing collected packages: numpy, tensorflow-gpu, tf-sentencepiece\n",
            "Successfully installed numpy-1.17.2 tensorflow-gpu-1.14.0 tf-sentencepiece-0.1.83\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xRYbl_We431V",
        "colab_type": "code",
        "outputId": "ef1558f3-1593-40ba-b9eb-6ff7494ee9ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "!curl -o many_to_many.zip https://dl.dropboxusercontent.com/s/kzxryp6snnx5dw6/many_to_many.zip?dl=0"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  114M  100  114M    0     0  39.0M      0  0:00:02  0:00:02 --:--:-- 39.0M\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b1kWSRVb5F-U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip -q many_to_many.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UA7Nd3qo5KM-",
        "colab_type": "code",
        "outputId": "ecd7b898-2e15-45c4-ae85-09013bc7f968",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "!curl -o mlt.zip https://codeload.github.com/suyash/mlt/zip/master"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100 33661    0 33661    0     0   151k      0 --:--:-- --:--:-- --:--:--  151k\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vg2sGM4N5Ln9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!unzip -q mlt.zip && mv mlt-master/mlt ./"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SuEV4CUW5ROy",
        "colab_type": "text"
      },
      "source": [
        "## Many to Many Translation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z1oy_H3o5MvI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import tf_sentencepiece as tfs\n",
        "\n",
        "from mlt.evaluation import predict\n",
        "from mlt.layers import Attention, ConditionalNormalization, Gelu, MultiplyConstant, PaddingAndLookaheadMask, PaddingMask, PositionalEncoding"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXVjFAP55Y9P",
        "colab_type": "code",
        "outputId": "c524a6fd-f410-4332-8434-cd6f085f2ddb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "tf.__version__"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'1.14.0'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1UrtqoA85aLG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sess = tf.InteractiveSession()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JAfwOm9x5baX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "encoding_model_file = \"./final/sentencepiece/en_de_fr_pt_shared/encoding.model\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sbrg8m1U5fIW",
        "colab_type": "code",
        "outputId": "64b0f953-156b-4094-b83e-7fda8fcfe1e9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        }
      },
      "source": [
        "with sess.as_default():\n",
        "    model = tf.keras.experimental.load_from_saved_model(\"final/model114\", custom_objects={\n",
        "        \"MultiplyConstant\": MultiplyConstant,\n",
        "        \"PositionalEncoding\": PositionalEncoding,\n",
        "        \"PaddingMask\": PaddingMask,\n",
        "        \"PaddingAndLookaheadMask\": PaddingAndLookaheadMask,\n",
        "        \"Attention\": Attention,\n",
        "        \"ConditionalNormalization\": ConditionalNormalization,\n",
        "        \"Gelu\": Gelu,\n",
        "    })"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/initializers.py:119: calling RandomUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling Ones.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FgKgIm625jRP",
        "colab_type": "text"
      },
      "source": [
        "### English to French"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XH1XFVga5gZ5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"This is a problem that we need to solve.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cN0jhGry5odk",
        "colab_type": "code",
        "outputId": "4d9d334e-61f8-4e79-ead0-2d560f36ffe8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        }
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensor2tensor/utils/beam_search.py:727: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use `tf.cast` instead.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:2403: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8MsOxURV5qH6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "66oTPUsR5req",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o883Aol85xNy",
        "colab_type": "code",
        "outputId": "d2b65a38-6bff-4200-9f69-aba5617a2be8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([\"Il s'agit d'un problème que nous devons résoudre.\",\n",
              "  \"C'est un problème que nous devons résoudre.\",\n",
              "  'C’est un problème que nous devons résoudre.',\n",
              "  'Ce problème est que nous devons résoudre.',\n",
              "  \"Il s'agit d'un problème que nous avons besoin de résoudre.\"],\n",
              " array([0.35213375, 0.33331218, 0.2737111 , 0.2399587 , 0.21996328],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UpaALHvr56x0",
        "colab_type": "text"
      },
      "source": [
        "### English to German"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QgjkgisG5yV1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"This is a problem that we need to solve.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tXYizgI06C_O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 0.0, 1.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "en6IYi_M6D5-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nmlqX_lw6FI6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c1dl-hAE6GLG",
        "colab_type": "code",
        "outputId": "d80b8abb-2e96-4835-927d-ff28e1d68a68",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Das ist ein Problem, das wir lösen müssen.',\n",
              "  'Dies ist ein Problem, das wir lösen müssen.',\n",
              "  'Das ist ein Problem, der wir lösen müssen.',\n",
              "  'Dies ist ein Problem, der wir lösen müssen.',\n",
              "  'Das ist ein Problem, das wir lösen musst.'],\n",
              " array([0.6566717 , 0.4704884 , 0.24058492, 0.21603352, 0.20849238],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gt3aez4k6Lzv",
        "colab_type": "text"
      },
      "source": [
        "### French to English"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDy9PKeP6HbX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"c'est un problème que nous devons résoudre.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULV7SQu76S7f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yBceuuwB6Uaz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Waq32kLs6Vyy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DXXxpcLk6XBL",
        "colab_type": "code",
        "outputId": "81c92d67-6d6f-49f6-be07-0d6bfebbbf36",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([\"That's a problem we need to resolve it.\",\n",
              "  \"That's a problem we need to solve it.\",\n",
              "  'This is a problem we need to resolve it.',\n",
              "  'That’s a problem we need to resolve it.',\n",
              "  'That’s a problem we need to solve it.'],\n",
              " array([0.20478117, 0.19986515, 0.19323574, 0.18281902, 0.18169725],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LgWiupN-6b-N",
        "colab_type": "text"
      },
      "source": [
        "### German to English"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FWkLJP9Y6YX_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Hier finden Sie alle Antworten, die Sie brauchen.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k33TP3wZ6e7v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([1.0, 0.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F2eNZu0K6f9Y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_YeRSjk36hL0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2Armf6bX6icz",
        "colab_type": "code",
        "outputId": "27d6b954-870b-4a03-ca13-45377de2e14f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        }
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Here you will find all the answers you need.',\n",
              "  'Here you will find all the answer you need.',\n",
              "  'Here you find all the answers you need.',\n",
              "  'You will find all the answers you need.',\n",
              "  'You will find all the answers you need here.'],\n",
              " array([0.43479517, 0.2945328 , 0.23725006, 0.19760388, 0.18721606],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wOXUw8D2AC8e",
        "colab_type": "text"
      },
      "source": [
        "### French to German\n",
        "\n",
        "__NOTE:__ This is zero-shot, training was done for en-fr and en-de"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jtRMi1z8AAcv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"c'est un problème que nous devons résoudre.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EYJOTUKAAFYg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 0.0, 1.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4oI86nyZAIDQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kw1RYMp5AJ2J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8z8tUfyNARRk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "786b69a8-58dc-4a9e-fb0a-3e27c77cbc3e"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Das ist ein Problem, das wir beheben müssen.',\n",
              "  'Es ist ein Problem, das wir beheben müssen.',\n",
              "  'Das ist ein Problem, den wir beheben müssen.',\n",
              "  'Das ist ein Problem, das wir lösen müssen.',\n",
              "  'Das ist ein Problem, das wir zu lösen müssen.'],\n",
              " array([0.43660545, 0.32214886, 0.19762374, 0.17219017, 0.10422194],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g7eildiIAUf9",
        "colab_type": "text"
      },
      "source": [
        "### German to French\n",
        "\n",
        "__NOTE:__ This is zero-shot, training was done for en-fr and en-de"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MFlaF9qUASCc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "a = tfs.encode([\"Hier finden Sie alle Antworten, die Sie brauchen.\"], model_file=encoding_model_file, add_bos=True, add_eos=True)[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DYbeerUpAXRZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids, probs = predict(\n",
        "    model=model,\n",
        "    inputs=a,\n",
        "    inpf=tf.constant([0.0, 0.0, 1.0, 0.0]),\n",
        "    tarf=tf.constant([0.0, 1.0, 0.0, 0.0]),\n",
        "    bos_id=1,\n",
        "    eos_id=2,\n",
        "    beam_size=5,\n",
        "    vocab_size=40960,\n",
        "    alpha=1.0,\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uOc2M156AZVt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mask = tf.cast(tf.not_equal(ids, 0), tf.int32)\n",
        "seq_len = tf.reduce_sum(mask, axis=-1)\n",
        "probs = tf.math.exp(probs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Za-CAH6ZAb-Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "ids_, probs_, seq_len_ = sess.run([ids, probs, seq_len])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Dlm8_NkAdy6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "1a519550-a04b-49ed-ce0b-3c9d5c61bd06"
      },
      "source": [
        "[x.decode(\"utf-8\") for x in sess.run(tfs.decode(ids_[0], seq_len_[0], model_file=encoding_model_file))], probs_[0]"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Vous trouverez ici toutes les réponses que vous avez besoin.',\n",
              "  'Vous trouverez ici toutes les réponses dont vous avez besoin.',\n",
              "  'Vous trouverez tous les réponses que vous avez besoin.',\n",
              "  'Vous trouverez tous les réponses dont vous avez besoin.',\n",
              "  'Ici vous trouver toutes les réponses que vous avez besoin.'],\n",
              " array([0.33657932, 0.3270293 , 0.32325435, 0.29828098, 0.28019544],\n",
              "       dtype=float32))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2WKbUxHh6jlB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sess.close()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}