Ex_Self_Attention.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Exemplo de Self Attention.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPjv1KW0Eqgx4wfu5Askdfo",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/finardi/tutos/blob/master/Ex_Self_Attention.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cdXdt1fRs52A"
      },
      "source": [
        "# Exemplo de self attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a8tpHOAtMz-W"
      },
      "source": [
        "import torch \n",
        "import numpy as np\n",
        "import seaborn as sns"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oQNdgEQ3MIy_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bbd74a00-9064-416e-d46a-8f795e9fff0f"
      },
      "source": [
        "# 00 -- ENTRADA:\n",
        "text = 'Quero um cartão'\n",
        "text_tokens = ['Quero', 'um', 'cartão']\n",
        "\n",
        "# 01 -- IDs:\n",
        "text_ids = [27,11,83]\n",
        "VOCAB = 100 # Quantidade de palavras\n",
        "\n",
        "# 02 -- DIMENSÃO DOS EMBEDDINGS = 4:\n",
        "D = 4\n",
        "\n",
        "# 03 -- COMPRIMENTO DA SEQUÊNCIA:\n",
        "L = 3\n",
        "\n",
        "# 04 -- MATRIZ DE EMBEDDINGS LxD:\n",
        "text_emb = torch.tensor([\n",
        "                         [0,1,2,0], \n",
        "                         [1,0,1,1],\n",
        "                         [1,0,1,2],\n",
        "                        ]).long()\n",
        "\n",
        "# 05 -- PRINT - SANITY CHECK:\n",
        "for i in zip(text_tokens, text_ids, text_emb):\n",
        "    print(f'Palavra: {i[0] :<6} -- ID: {i[1]} -- Embedding: {i[2]}')"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Palavra: Quero  -- ID: 27 -- Embedding: tensor([0, 1, 2, 0])\n",
            "Palavra: um     -- ID: 11 -- Embedding: tensor([1, 0, 1, 1])\n",
            "Palavra: cartão -- ID: 83 -- Embedding: tensor([1, 0, 1, 2])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7tnY8t3RROt2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0805c780-7cb0-4f80-dd62-e22683bf4973"
      },
      "source": [
        "# 06 -- USAR O nn.EMBEDDING PyTorch para CONSTRUIR AS MATRIZ DE PESOS Q, K, V:\n",
        "\n",
        "embs  = torch.nn.Embedding(VOCAB, D)\n",
        "\n",
        "# tensor com zeros_like\n",
        "embeddings = torch.zeros(len(text_ids) * D).reshape(len(text_ids), D) \n",
        "\n",
        "print(f'Shape da matriz de embeddings: {embeddings.shape}\\n')\n",
        "\n",
        "print('Texto\\t\\t\\t     Matriz de embeddings\\n')\n",
        "for i , token in enumerate(text_ids):\n",
        "    embeddings[i] = embs(torch.tensor(token))\n",
        "embeddings"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape da matriz de embeddings: torch.Size([3, 4])\n",
            "\n",
            "Texto\t\t\t     Matriz de embeddings\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.4409, -0.7220, -0.8106, -0.3540],\n",
              "        [-0.4031,  0.1947,  0.3793,  0.6797],\n",
              "        [ 0.0089,  0.4216, -1.2296, -2.1588]], grad_fn=<AsStridedBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5FBA15gsMIwS"
      },
      "source": [
        "# 06 -- MATRIZ DE PESOS, Q, K, V --> DxD (INICIADOS ARBITRARIAMENTE):\n",
        "\n",
        "Qw = torch.tensor([\n",
        "                   [1,  0, -1,  1], \n",
        "                   [0,  1,  0, -2],\n",
        "                   [3,  1, -2,  0],\n",
        "                   [0, -2,  1,  3],\n",
        "                ])\n",
        "\n",
        "Kw = torch.tensor([\n",
        "                   [3, -1,  0, -1], \n",
        "                   [0,  1, -1,  0],\n",
        "                   [3,  1,  0, -1],\n",
        "                   [-1, 0,  1,  3],\n",
        "                 ])\n",
        "\n",
        "Vw = torch.tensor([\n",
        "                   [-2, 0,  3,  0], \n",
        "                   [-1, 1, -2, -1],\n",
        "                   [-2, 1,  2,  3],\n",
        "                   [3,  2,  1, -2],\n",
        "                 ])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OS5NVaPXMItv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0a505528-34b1-4261-ab38-81b7931629a8"
      },
      "source": [
        "# 07 -- OBTER Q, K e V:\n",
        "print(f'Shape da entrada: {text_emb.size()} -- Shape das Matrizes de pesos Qw, Kw e Vw: {Qw.size()}')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Shape da entrada: torch.Size([3, 4]) -- Shape das Matrizes de pesos Qw, Kw e Vw: torch.Size([4, 4])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "daPRGnNWMIq1"
      },
      "source": [
        "K =  torch.einsum('ab, bc -> ac', text_emb, Kw) # text_emb @ Kw\n",
        "V =  torch.einsum('ab, bc -> ac', text_emb, Vw) # text_emb @ Vw\n",
        "Q =  torch.einsum('ab, bc -> ac', text_emb, Qw) # text_emb @ Qw"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cY7HNYRTMIoW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "66b957b0-7dcf-4a75-af59-095db1777f05"
      },
      "source": [
        "K"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 6,  3, -1, -2],\n",
              "        [ 5,  0,  1,  1],\n",
              "        [ 4,  0,  2,  4]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BgEZOvSZMIl0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "93c99767-92e6-435e-a035-ea7f4a4ee613"
      },
      "source": [
        "V"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-5,  3,  2,  5],\n",
              "        [-1,  3,  6,  1],\n",
              "        [ 2,  5,  7, -1]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GSj4Xs47MIjP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "255b6d0f-4ba8-43f4-d314-d60b04ac396b"
      },
      "source": [
        "Q"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 6,  3, -4, -2],\n",
              "        [ 4, -1, -2,  4],\n",
              "        [ 4, -3, -1,  7]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ss7IYIZFMIgq"
      },
      "source": [
        "# 08 -- CALCULAR O SCORE:\n",
        "QKT = torch.einsum('ab, cb -> ac', Q, K) # Q @ K.T\n",
        "QKT\n",
        "\n",
        "# 08.1 -- DIVIDIR POR SQRT(D)\n",
        "QKT = QKT/torch.sqrt(torch.tensor(D))"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZyFL9sryMIeR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "145ad14f-235e-4bf8-a874-6457cfcf544d"
      },
      "source": [
        "# 09 -- SOFTMAX:\n",
        "S = torch.nn.Softmax(dim=-1)\n",
        "S(QKT.float())"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1.0000e+00, 5.0435e-07, 1.6919e-10],\n",
              "        [1.4301e-03, 4.7358e-02, 9.5121e-01],\n",
              "        [2.0605e-09, 3.3535e-04, 9.9966e-01]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "upZSHypsMIbw"
      },
      "source": [
        "# 10 -- SIMPLIFICAR S - POR SIMPLICIDADE DE ENTENDIMENTO:\n",
        "Softmax  = torch.tensor([\n",
        "                         [1.00, 0.00,  0.00], \n",
        "                         [0.06, 0.04,  0.90],\n",
        "                         [0.00, 0.00,  1.00],\n",
        "                 ])"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hjbUXARcYveN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "21fbefa2-5d95-4265-bd4b-e95fc671c7b6"
      },
      "source": [
        "# 11 -- SOFTMAX * V (EXTRATOR DE FEAT):\n",
        "SV = torch.einsum('ab, bc -> ac', Softmax,V.float())\n",
        "SV "
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-5.0000,  3.0000,  2.0000,  5.0000],\n",
              "        [ 1.4600,  4.8000,  6.6600, -0.5600],\n",
              "        [ 2.0000,  5.0000,  7.0000, -1.0000]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EjxQz4s8Yvbh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5d392975-93dd-491c-8acd-84e90610739c"
      },
      "source": [
        "# 12 -- MULTIPLICAR PELA MATRIZ DE PROJEÇÃO Wo\n",
        "\n",
        "# PESOS DA Wo (INICIADOS ARBITRARIAMENTE)\n",
        "Wo = torch.tensor([\n",
        "                   [2,  0, -1,  1], \n",
        "                   [1, -1, -2,  1],\n",
        "                   [2,  0,  1,  0],\n",
        "                   [0, -2,  1, -1],\n",
        "                 ])\n",
        "Wo.shape"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([4, 4])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JkXnSgXwb29Z",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "861eecef-5835-4529-dc58-13836670b42c"
      },
      "source": [
        "## 13 -- SV @ Wo = MATRIZ DE ATENÇÃO:\n",
        "\n",
        "ATT = torch.einsum('ab, bc -> ac', SV, Wo.float())/8 # SV @ Wo\n",
        "ATT"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.3750, -1.6250,  0.7500, -0.8750],\n",
              "        [ 2.6300, -0.4600, -0.6200,  0.8525],\n",
              "        [ 2.8750, -0.3750, -0.7500,  1.0000]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        },
        "id": "WdjObeR-5GUC",
        "outputId": "524d402a-8b04-4300-8a0b-9050c5be61d1"
      },
      "source": [
        "# Embs sem ATT\n",
        "sns.heatmap(embeddings.detach().numpy(), center=0)"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7feebe506090>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAD7CAYAAAChScXIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQUklEQVR4nO3df+xdd13H8ef9frsBYmKlssFmOyqsb7MxwQJZCIGgixPxR8C5bMWx+QfIECdNHGgIbt0WEUVNgyvphLC1TKqZLBuBYeIScGrAhJQmlIX3KvvRrnMwO8rE4Bjf79c/vmfw3be333tve29Pz/s+H8lJ7/2c8z3nc/553Xff59x7egsLC0iSumGm7QlIkoZnaEtShxjaktQhhrYkdYihLUkdYmhLUoesOpEH2/jy3/D+QmDNmle1PYWTxinPfkHbUzhp7H/wk21P4aSy92uf7x3P3//hea8bOm/+6qv3HNexTqQTGtqSdKLMF/0OiqEtqaQ5aoa2PW1J6hArbUklVa20DW1JJc3Z05ak7qhaadvTlqQOsdKWVJK3/ElSh1RtjxjakjSCiNgA7ADWAIeAyzNz37JttgC/BzzSDP17Zr5rHMc3tCWV9NTC/KR2vR3Ylpm3RsRlwE3AL/bZbmdmXj3ug3shUpKGFBGnARuBXc3QLmBjRDz/RM3BSltSSaP0tCNiNbC6z6rDmXl4yfu1wMHMnAPIzLmIeKQZf2zZ314aERcCjwLXZuYXR5n/0VhpSyppbmFh6AXYDDzQZ9l8jIffDqzPzJ8DPgTcGRFrjv+sDG1JAtgKrO+zbF223QHgzIiYBWj+PaMZ/6HMfDQzn2pe/3Oz/qXjmKjtEUkljXIhsmmBHB5iu29FxB5gE3Br8+9XMvMZrZGIODMzDzavXw68CMihJ7QCQ1uSRnMlsCMirgG+DVwOEBF3Addk5peBD0TEK4A54PvAWzPz0XEc3NCWVNKkvhGZmV8Hzu8z/sYlr6+YyMExtCUVVfUbkV6IlKQOsdKWVNL85L4R2SpDW1JJ80XbI4a2pJJ8co0kdUjVStsLkZLUIVbakkryyTWS1CE17x0xtCUVZaUtSR3ihUhJUuustCWVZHtEkjqkantkqNBuHpOztnl7IDMPTW5KknT8pjK0I+LFwN+y+PThR5rhMyJiN3BlZu6b8Pwk6ZjM18zsgRcidwIfB9Zk5rmZeS6wBri5WSdJOoEGtUfWZObfLR3IzHng1oh4/+SmJUnHZyrbI8DjEbEJ+PvMXACIiB7wFoZ4CKYktWVaQ/sKYDuwLSIONmNnAnuadZKkE2jF0G4uNF4QEc/nmXePPLbCn0lS66b697SbkDaoJallfrlGUkk162xDW1JRVS9E+oNRktQhVtqSSqpZZxvakooytCWpQ6r2tA1tSSXVjGxDW5JGEhEbgB0s/njeIeDy5b94GhGzwIeBN7D4+fHBzPzYOI7v3SOSSloYYRnRdmBbZm4AtgE39dnmt4GXAGcDrwa2RMSLRj/UkQxtSSVNIrQj4jQWny+wqxnaBWxsfupjqUuAj2bmfPON8juAi4/xVJ7B9oikkkYM49XA6j6rDmfm0l80XQsczMw5gMyci4hHmvGlP/WxDnhoyfv9/Oj3m46LlbYkwWbggT7L5jYn1Y+hLUmwFVjfZ9m6bLsDwJnNhcanLzie0YwvtR84a8n7dX22OSa2RyQV1Rt6y6YFMvDBLpn5rYjYA2wCbm3+/Uqfn6u+DXh7RNzO4l0mbwJeO/SEVmClLUmjuRK4KiLuA65q3hMRd0XEK5ttPgHcD+wDvgRcn5kPjOPgVtqSippMTZqZXwfO7zP+xiWv54B3TuL4VtqS1CFW2pJqGr6l3SmGtqSiajYSap6VJBVlpS2ppF7R/oihLammnqEtSZ1hpS1JnVLzkl3Ns5Kkoqy0JZXUs6ctSR3Sq9lIMLQlldQr2v01tCWVVLU9UvOjSJKKstKWVJM97eP3g7n/O5GHO2l98xs3tz2Fk8Zpq57T9hROGr/8nDVtT6GUnqEtSd3hhUhJ6pCqlXbNs5Kkoqy0JZXU6822PYWJMLQllWR7RJLUOittSSXNzJzS9hQmwkpbkjrESltSSVV72oa2pJKq3j1S86NIkoqy0pZU0qwXIiVJbbPSllRS1Z62oS2ppLZCOyJ+DLgZeAXwA+DqzPxMn+1eD9wF3NcMPZmZ5w/av6EtSeN1NfBEZr4kIs4G/jUiXpKZ3+2z7b2Z+cpRdm5oSypplEo7IlYDq/usOpyZh0c89CXAFQCZuS8ivgz8CnDbiPvpy9CWVFJvZqT2yGbg2j7j1wFbRjz0OuChJe/3A2uPsu2GiNgNPAV8JDN3DNq5oS2ppJnRetpbgVv6jB9RZTchu+4o+zl9hGPuBtZm5nciYj1wd0QczMy7V/ojQ1vS1GtaIEO1QTJz40rrI2I/cBbwWDO0Dvh8n/08seT1AxFxB/AaYMXQ9j5tSSX1erNDL2N2G/AOgOZC5KuAf1q+UUS8MCJ6zevnARcCewbt3EpbUkkt3qf9IeCWiPhPYA743cz8H4CIuB54JDO3AxcB74yIp1jM4h2ZeeegnRvakkrq9dqJt8z8X+Dio6y7ZsnrG4EbR92/oS2ppBEvRHaGPW1J6hArbUkljXifdmcY2pJKaqunPWm2RySpQ2p+FEmaejMzz2p7ChNhpS1JHWKlLamkqrf8GdqSSurN1Iw32yOS1CE1P4okTb2qt/zVPCtJU6/qg32PuT0SEV8d50QkaZx6vVVDL12y4mwj4pwVVq8Z81wkaWyqXogcdFZ7gQeBXp91PzX22UiSVjQotB8EXpuZB5eviIgDE5mRJI1B19oewxp0Vp9i8VlnR4Q2cPv4pyNJYzKNoZ2Z71lh3bvHPx1JGo9p7WlLUidVbY/4jUhJ6pCaH0WSZHtEkjqk6DciDW1JJXkhUpK6xAuRkqS21fwokjT1FmyPSFKHzNS8EGl7RJI6xEpbUkkLq05tewoTYWhL0hhFxGXAe4FzgM2ZeeMK274d+CMWf/76c8AfZOb8Svu3PSKppIWZ2aGXMdsDXAp8cqWNImI9cC3wauDsZrls0M4NbUk1zcwOv4xRZu7NzHuBFStm4LeAOzLzsaa6/ihwyaD92x6RNPUiYjWwus+qw5l5eEKHXQc8tOT9fmDtoD8ytCWVNL/qlFE238xiq2K564AtSwciYjeLgdvP6Zk5N8qBR2VoSxJsBW7pM35ElZ2ZG8d0zP0sPhnsaeuAgY9xNLQllTQ/O/wlu6YFMqk2yNF8CrgnIq4DDgFvZ8DFS/BCpKSiFmZmhl7GKSI2RcTDwMXADRHxcESc06y7PiKuBMjM+4EbgC8B+4D7gVsH7d9KW5LGKDN3AbuOsu6aZe9vAm4aZf+GtqSSxl1BnywMbUklzRvaktQdCyNciOySmmclSUVZaUsqaWGm1/YUJsLQllTS/GzN0LY9IkkdYqUtqaT5VTVr0ppnJUlFWWlLKmmh5nN9T2xoP/FEnsjDnbTOfVa/n+2dTofesrPtKZw0fvXK57Y9hVKq3j1ie0SSOsT2iKSaRnoGQndYaUtSh1hpS6qpaElqaEuqqWhoFz0tSarJSltSSb2iJamhLamk3sxC21OYCENbUklW2pLUITNFv8Ze9LNIkmqy0pZUUtHn+hrakmryQqQkdYiVtiR1SNXQLnpaklSTlbakkqpW2oa2pJIMbUnqkLZCOyIuA94LnANszswbj7Ld64G7gPuaoScz8/xB+ze0JZU0294tf3uAS4E/HmLbezPzlaPs3NCWpDHKzL0AETE/if0b2pJKGqU9EhGrgdV9Vh3OzMPjmlMfGyJiN/AU8JHM3DHoDwxtSSWN2NPeDFzbZ/w6YMvSgSZk1x1lP6dn5tyQx9wNrM3M70TEeuDuiDiYmXev9EeGtiTBVuCWPuNHVNmZuXEcB8zMJ5a8fiAi7gBeAxjakqbPKSP8NGvTAplkG+QIEfFC4NHMXIiI5wEXAu8f9HdF72SUpHZExKaIeBi4GLghIh6OiHOadddHxJXNphcBeyNiD3APsDMz7xy0fyttSSXNtlSSZuYuYNdR1l2z5PWNQN97uFdiaEsqqeo3IoueliTVZKUtqaRTi6ablbYkdUjRzyJJ066tC5GTVvS0JKkmK21JJVW9e2TF0I6INcCfs/g9+zszc9uSdZ/KzIsmPD9JOiazvbZnMBmDPotuAh4HtgNviojbI+LpoP+Zic5Mko7D7MzwS5cMmu7ZmfnezLydxe/F/xfwmYh49uSnJknHblpD+9SnX2TmQma+C/gq8FnA4JakE2xQaN8fEa9bOpCZ7wH+A9gwsVlJ0nFaNdMbeumSQaH9VhYr62fIzPcB501kRpI0BlXbIyvePZKZj6+w7t7xT0eStBLv05ZU0qkjPAShSzr2HwNJmm5W2pJK6lqveliGtqSSZjt2V8iwin4WSVJNVtqSSvJCpCSpdVbakkqq2tM2tCWVVPXukaKnJUk1WWlLKsn2iCR1SNX2iKEtqaSZnpW2JHVG1Uq76GlJUk1W2pJKautCZERsAy4AngS+C7w7M798lG3/BPid5u0tmXnDoP1baUsqqcUn13wOOC8zXwb8GfAP/TZqHuV4MfDSZrl4+eMd+7HSllRSW5V2Zn5mydsvAj8dETOZOb9s00uAnZn5PYCI2NmM3bPS/g1tSSWNEtoRsRpY3WfV4cw8fBzT+H3gs30CG2Ad8IUl7/cDVtqSNITNwLV9xq8DtiwdiIjdLAZuP6dn5lyz3aXAWxgiiEdhaEsqacRe9Vbglj7jR1TZmblx0M4i4s3AnwIXZOY3j7LZfuCsJe/XAQcG7dvQllTSzAjtkaYFcjxtkB+KiF8D/hr4pcx8cIVNbwM+3NxtAnA5cNWg/Rvakkpq8bdHbga+D/xjRDw9dkFmHoqIjwGfzsxPZ+YXIuJ24GvNNjsz818G7dzQllRSW9+IzMznr7Dubcveb2FZz3wQ79OWpA6x0pZUkj/NKkkdMsqFyC6xPSJJHWKlLamkU1dZaUuSWmalLamkqhciewsLC23PQZI0JNsjktQhhrYkdYihLUkdYmhLUocY2pLUIYa2JHWIoS1JHWJoS1KHGNqS1CFT8zX2iNgA7ADWAIeAyzNzX7uzakdE/CVwEfAi4LzM3NvujNoREWuATwAvZvHxUPuAd2TmY61OrCURcQewHpgHvgtclZl72p2VlpumSns7sC0zNwDbgJtank+b7gBeBzzU9kRatgD8RWZGZp4HfAP4YMtzatMVmfmyzPx54C+Bj7c9IR1pKkI7Ik4DNgK7mqFdwMaIOOqz3CrLzH/LzANtz6Ntmfl4Zn5hydCXgLNamk7rMvM7S97+BIsVt04y09IeWQsczMw5gMyci4hHmvGp/K+wnikiZoB3Ap9uey5tap4WfiHQA97Q8nTUx1RU2tIQ/obFPu6NbU+kTZn5tsxcB7wP+FDb89GRpiW0DwBnRsQsQPPvGc24plxzYfZs4JLMtCUAZOYngF9oLtbqJDIVoZ2Z3wL2AJuaoU3AV6b1LgH9SER8AHgF8KbMfLLt+bQlIn48ItYuef/rwOPNopPI1DwEISJ+lsVb/n4S+DaLt/xlu7NqR0R8GPhN4AXAfwOHMvPcdmd14kXEucBe4D7ge83wA5n55vZm1Y6IOB24E3guMMdiWF+dmbtbnZiOMDWhLUkVTEV7RJKqMLQlqUMMbUnqEENbkjrE0JakDjG0JalDDG1J6hBDW5I65P8BoFRIDDKfeyUAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        },
        "id": "UccmwfhU48pn",
        "outputId": "f1c83bdf-3456-4ab4-9b45-c1d2ac3f8d1f"
      },
      "source": [
        "# ATT Embs\n",
        "sns.heatmap(ATT.detach().numpy(), center=0)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7feebe459b50>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAD7CAYAAAChScXIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAATJ0lEQVR4nO3df6xedX3A8fe9ZXNxy3a1Igq20NH2Y0QcKxhHjEanMsW4ocigEyGLQ6gMdrfgos4gYkaIkKyDYsokswW2LiFgMYwsSibOH8OM1EaB+aFKpbUNsRYrYhxtn/vsj3sKT2+fe+/z9D5PT8/3vl/JyT3P93yfc77PP59++jnfc74j7XYbSVIzjNY9AElS7wzaktQgBm1JahCDtiQ1iEFbkhrEoC1JDXLMkbxYLH+98wuBl5x/W91DOGq8+u4P1z2Eo8a2Y99a9xCOKv/5tetH5nSCR+/uPd6ccu7crnUEmWlLUoMc0Uxbko6UdqvVc9/GpNkYtCWVqrW/7hEMhUFbUpHaE70HbTNtSapbH+WRJjFoSypS2/KIJDWIQVuSmqOfmnaTOE9bkhrETFtSmbwRKUnN0d77f3UPYSgsj0hSg5hpSypSe8LyiCQ1xjDmaUfEQuAO4GRgL7AFuDQzd03ptw54O/DTqumuzPz7QYzBoC2pTMOZp90GPpuZDwJExA3A9cCHuvS9PjPXDHoABm1JRRpGeSQznwYe7Gh6CFg18AvNwKAtqUx9ZNoRMQaMdTm0JzP3TPOdUSYD9pemOe3fRMSlwA+Bj2fm//Y8oBk4e0RSkdqt/T1vwDiwtcs2PsMlbgaeBbqVQP4OWJqZpwL3AP8REQsG8bvMtCWVqb+a9mpgXZf26bLsG4FlwHsyc2Lq8czc0bF/e0T8A/Aq4Ml+BtWNQVtSkfqpaVclkK4BeqqIuA44HXh3Zj43TZ8TDgTuiPgjoAXs6Na3XwZtSepRRJwCfBx4HPhWRABszcz3RsRm4OzM3Amsj4jjgAngGeCPM3Mg01kM2pKK1N7bNQmek8x8lGkWusnM0zr23z7wi1cM2pKKVOoiCM4ekaQGMdOWVKZCF0EwaEsqUtv3aUtSgxi0Jak5Sr0RadCWVCYzbUlqDmvaktQgrlwjSU0ynzPtaomdRdXH7Zm5e3hDkqS5m5flkYg4GfgnYAWws2o+PiI2AZdl5pYhj0+SDku7dcgbU4swW6Z9O/A54B0H3hlbrdbwZ9WxM4c7PEk6TPM0aC/MzH/pbKiC950R8cnhDUuS1M1sQfvpiFgJ/FtmtgEiYoTJTLunF4ZLUh3mZU0buBhYC9wSEQdWXTgB2Fwdk6SjUrvVHvg5q0kZdwAnA3uBLcClmblrSr8XA19gcoWb/cBVmXnfIMYwY9CubjS+LSKO5eDZI7tm+Jok1W5i71AeY28Dn83MBwEi4gbgeuBDU/pdBTyTmUsjYhnw9YhYmpnPznUAPU35q4K0gVrSvJaZTwMPdjQ9BKzq0vV8qmpEZm6JiIeBdwF3zXUMPlwjqUjtid7LIxExBox1ObSnWvS323dGmQzYX+pyeDEHr7y+jReqFXPiyjWSitRutXvegHFga5dtfIZL3Aw8C6wZ8k85iJm2pCK1+5s8shpY16V9uiz7RmAZ8J4Dz7BMsQ04kRfKyouBr/Y1omkYtCUVqZ/ZI1UJpKdpzBFxHZOzQt6dmdMt+X4XcCnwcHUj8vXAyp4HNAPLI5KKNDHR+9ariDgF+DhwPPCtiNgcEV+sjm2OiOOrrjcAYxHxA+A+4MOZ+YtB/C4zbUlF6rM80pPMfBQYmebYaR37vwTOG/wIDNqSCjWMoH00sDwiSQ1ipi2pSP3UqpvEoC2pSK19XUvPjWfQllSkUjNta9qS1CBm2pKKVOrsEYO2pCJNTFjTlqTGmDDTlqTmMNOWpAZpG7QlqTlKnfJn0JZUpFLLI87TlqQGMdOWVKRSM22DtqQi7dtfZiHBoC2pSBPt4WTa1fqQ5wInAadm5iNd+lwDfATYWTV9MzMvH8T1DdqS1J+NwD8CX5+l3+2ZedWgL27QllSkPtd+HAPGuhzaUy36+7zM/Eb1nbkM77CVWfSRNO+12iM9b8A4sLXLNj6HIVwQEd+NiC9HxJlz/kEVg7akIu1rjfa8AauBJV221Yd5+bXAksx8HZMrs98bEQvn/qssj0gSVQlkz6wdez/fUx37X4mI7cBrga/N9dxm2pKKNNEe6XkbtIg4oWP/NCZnmuQgzm2mLalIreFN+bsJeB/wCuCBiNidmadExP3A1Zn5MHBdRJwOtIC9wAc7s++5MGhLKlKrPZzzZuaVwJVd2s/u2L94OFc/wkH7kZvPPZKXO2qd9tdX1D2Eo8b3XnR83UM4avzB7tmm/Upm2pIKtW+izFt2Bm1JRRpWeaRuBm1JRWpR5lv+yvz/gyQVykxbUpEsj0hSg7TqHsCQGLQlFWmvNW1JUt3MtCUVqdUus6ht0JZUJGvaktQgBm1JahCDtiQ1SAtr2pLUGGbaktQgw5o9EhE3AucyuRrNqZn5SJc+C4CbgHcCbeD6zLxtENd3nrYk9Wcj8GbgyRn6fABYCiwDzgSuiYiTBnFxM21JReqnPBIRY8BYl0N7qkV/n5eZ36i+M9Mpzwc+n5kTwK6I2Aicx+TK7HNipi2pSHuZ6HkDxoGtXbbxw7z8Yg7OxLcBiw7/17zATFtSkfq8EbkaWNelfU+XtloZtCXNe1UJZJABehtwIvA/1eepmfdhM2hLKlLN7x65C7gkIu4BFgLnAG8axImtaUsqUot2z1s/IuKmiPgx8CrggYh4tGq/PyLOqLrdATwBbAEeAq7NzK2D+F1m2pKKNKwnIjPzSuDKLu1nd+y3gFXDuL5BW1KRJnw1qyQ1h+8ekaQGMWhLUoOUunKNs0ckqUHMtCUVaV97ou4hDIVBW1KRrGlLUoOUGrStaUtSg5hpSyqSD9dIUoOUWh4xaEsqkvO0p4iI7w1yIJI0SBO0e96aZMZMOyJeM8PhhQMeiyQNTKmZ9mzlkUeAHwEjXY69bOCjkaQBma83In8EvCkzd0w9EBHbhzIiSRqAYd2IjIjlwHomqw27gYsyc8uUPtcAHwF2Vk3fzMzLB3H92YL23Uyuc3ZI0AbuGcQAJGkYJob3GPta4JbMvDMiLgRuBf6wS7/bM/OqQV98xqCdmR+d4dhfDXowkjQow7jBGBEvB1YA76iaNgBrIuLYzNw18At24ZQ/SUXq50ZkRIwBY10O7alWaj9gEbCjWk6MzGxFxM6qfWrQviAizgKeAj6Vmf/dz/in42PskorU55S/cWBrl238MC+/FliSma8DbgDujYiBzLgzaEsSrAaWdNlWT+m3HTghIhYAVH+Pr9qfl5lPZea+av8r1fHXDmKglkckFamfKX9VCWRPD/1+EhGbgZXAndXf70ytZ0fECQdm3UXEacBJQPY8oBkYtCUVaf/w5mlfBqyPiKuBnwEXAUTE/cDVmfkwcF1EnA60gL3ABzPzqUFc3KAtSX3IzO8Db+jSfnbH/sXDur5BW1KRmvZOkV4ZtCUVab4+xi5JjWSmLUkNYtCWpAaZKDNmG7QllclMW5IaxKAtSQ1S6OQR3z0iSU1ipi2pSMNauaZuBm1JRSq1pm15RJIaxExbUpHKzLMN2pIKZdCWpAYptaZt0JZUpGGF7IhYDqwHFgK7gYsyc8uUPguAm4B3VkO5PjNvG8T1vREpqUjtPrY+rQVuyczlwC3ArV36fABYCiwDzgSuiYiT+r/UoQzakoo0jKAdES8HVgAbqqYNwIqIOHZK1/OBz2fmRLV+5EbgvMP8KQexPCKpSH0G4zFgrMuhPdWivwcsAnZkZgsgM1sRsbNq71zcdzHwZMfnbVWfOTPTliQYB7Z22cbrHFQ3Bm1JgtXAki7b6in9tgMnVDcaD9xwPL5q77QNOLHj8+IufQ6L5RFJheo9J61KIHt66PeTiNgMrATurP5+p6pbd7oLuCQi7mFylsk5wJt6HtAMzLQlFWqkj60vlwFXRMTjwBXVZyLi/og4o+pzB/AEsAV4CLg2M7fO5dccYKYtSX3IzO8Db+jSfnbHfgtYNYzrH9Gg/WuvWnYkL3fU2r//V3UP4ajxot9eWPcQjhova836v3P1o+8EuhnMtCUVqszqr0FbUpFGRsoM2mX+KkkqlJm2pCKNFFrUNmhLKlSZhQSDtqQijYyYaUtScxR6I9KgLalII5ZHJKk5LI9IUpMUWh4p81dJUqHMtCUVqdQnIg3akoo0OlJmeCvzV0ma90ZGFtQ9hKEwaEvSAEXEi4EvAKcD+4GrMvO+Lv3eAtwPPF41PZeZh7yneyqDtqQi1VjTvgp4JjOXRsQy4OsRsTQzn+3S97HMPKNL+7TKrNRLmvdGRkZ73gbsfOBWgMzcAjwMvGtQJzfTllSkfmraETEGjHU5tKda9Lcfi4EnOz5vAxZN03d5RGwC9gGfy8z1s53coC2pSH1m0OPAp7q0fxq4prOhCrKLpznPcX1ccxOwKDN/HhFLgAciYkdmPjDTlwzakorU5+yR1cC6Lu2HZNmZuWKmE0XENuBEYFfVtBj4apfzPNOxvzUiNgJvBAzakuaffoJ2VQIZ1MrKdwGXAg9XNyJfD6yc2ikiXgk8lZntiHgpcBbwydlObtCWVKTR+maP3ACsi4gfAC3gw5n5C4CIuBbYmZlrgXOBVRGxj8lYvD4z753t5AZtSRqgzPwlcN40x67u2F8DrOn3/AZtSUXyiUhJapDR0V+vewhDYdCWVKSR0TIzbZ+IlKQGMdOWVKRRa9qS1BzeiJSkBhkdfVHdQxgKa9qS1CBm2pKKZE1bkhqk1Cl/Bm1JRRopdGHfGWvaEbEwIm6LiC9HxOVTjt093KFJkqaa7Z+iW4EnmFx8clVEvA3408zcD/zusAcnSYer1NkjswXtZZn5foCI+CKTb6S6LyLOGfrIJGkOSr0ROduUv+ffuJKZ7cy8HPge8O/AbwxzYJI0FyOjx/S8NclsQfuJiHhzZ0NmfhT4NrB8aKOSpIaKiAsj4rsRsT8i/nKWvpdExA8i4ocRsSYiZn12ZrYOH2Qysz5IZn4COHW2k0tSXUZGjul5G7DNwAXAv87UqVrM91PAmcCyartwtpPPONrMfHqGY4/NdnJJqktd7x7JzEcAImJilq7vBzZm5q6q/+eBPwdun+lLzSrmSFKPRhf0ftstIsaAsS6H9lSL/g7DYuDJjs/bgEWzfcmgLUkwzmSpYqpPA9d0NkTEJiYDbjfHZWZrsEM7mEFbUpH6rFWvBtZ1aT8ky87MFYc5pKm2ASd2fF4MbJ/tSwZtSWXqI2hXJZBhlUGmczfwXxHxaWA3cAmz3LwEX80qqVB1zdOOiJUR8WPgPOAzEfHjiHhNdezaiLgMIDOfAD4DPARsYfLp8ztnO7+ZtqQi1fXCqMzcAGyY5tjVUz7fyuTrQnpm0JZUpoY96dirMn+VJBX67hGDtqQiNe2dIr3yRqQkNUiZ/xRJUqEr15T5qyTNe+1j5uciCJLUTIUu7GtNW5IaxExbUpHahWbaBm1JRTJoS1KTGLQlqTnao2XesjNoSyqS5RFJapCJBWbaktQYpZZHyvxVklQoM21JRWodU094i4gLgb8FXgOMZ+aaafq9BbgfeLxqei4z3zDb+Q3akorUrq+mvRm4APhYD30fy8wz+jm5QVtSkdqjIz33jYgxYKzLoT3Vor89y8xHqnNO9PO9XlnTliQYB7Z22caHfN3lEbEpIr4dERf38gUzbUlFmljQe6YNrAbWdWk/JMuOiE3A4mnOc1xmtnq85iZgUWb+PCKWAA9ExI7MfGCmLxm0JRWpn/JIVQLpqQySmSsOd0xTzvNMx/7WiNgIvBGYMWhbHpFUpPboSM9bHSLilRExUu2/FDiLyZuYMzLTllSkdk1PsUfESuAG4CXAn0TEx4CzMvOxiLgW2JmZa4FzgVURsY/JWLw+M++d7fwGbUlFqiuDzswNwIZpjl3dsb8G6DqHeyYj7Xb78EcnSTqirGlLUoMYtCWpQQzaktQgBm1JahCDtiQ1iEFbkhrEoC1JDWLQlqQGMWhLUoPMm8fYI2I5sB5YCOwGLsrMLfWOqh4RcSOT7z04CTj1wEvb55uIWAjcAZwM7AW2AJdm5q5aB1aT6i1zS4AJ4Fngisyc9QVGOrLmU6a9FrglM5cDtwC31jyeOm0E3gw8WfdAatYGPpuZkZmnAj8Erq95THW6ODN/LzN/H7gR+Oe6B6RDzYugHREvB1bwwktcNgArIuLY+kZVn8z8RmZur3scdcvMpzPzwY6mh4ATaxpO7TLz5x0ff4fJjFtHmflSHlkE7DiwokRmtiJiZ9U+L/8rrINFxCiwCvhS3WOpU0TcxuR7nUeAd9Y8HHUxLzJtqQc3M1nH7ftVmSXJzL/IzMXAJ5h8J7SOMvMlaG8HToiIBQDV3+Ords1z1Y3ZZcD5mWlJAMjMO4C3VjdrdRSZF0E7M3/C5DI+K6umlcB35ussAb0gIq4DTgfOyczn6h5PXSLityJiUcfn9wBPV5uOIvNmEYSIeDWTU/5eAvyMySl/We+o6hERNwHvA14B/BTYnZmn1DuqIy8iTgEeAR4HflU1b83M99Y3qnpExHHAvcBvAi0mg/VVmbmp1oHpEPMmaEtSCeZFeUSSSmHQlqQGMWhLUoMYtCWpQQzaktQgBm1JahCDtiQ1iEFbkhrk/wGb+qcbPFWzmAAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z5EI8ia0gLJK"
      },
      "source": [
        "# Fim do Notebook"
      ]
    }
  ]
}