TrainDataEncodetextsBertParagraphEncoder.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TrainDataEncodetextsBertParagraphEncoder.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/andrewk1/pytorch-deep-bayesian-bandits/blob/master/TrainDataEncodetextsBertParagraphEncoder.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3i5hbi52REZ_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 311
        },
        "outputId": "4a5482c6-4c91-4104-fa66-db3f25d01544"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fri Aug  9 02:46:19 2019       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 418.67       Driver Version: 410.79       CUDA Version: 10.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   30C    P8    26W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yWMr-iR8WlIW",
        "colab_type": "code",
        "outputId": "293aee83-4696-4a22-dd30-b0d2c2b152ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!pip install regex requests\n",
        "import torch\n",
        "roberta = torch.hub.load('pytorch/fairseq', 'roberta.base')\n",
        "roberta.eval() \n",
        "roberta.cuda()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting regex\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6f/4e/1b178c38c9a1a184288f72065a65ca01f3154df43c6ad898624149b8b4e0/regex-2019.06.08.tar.gz (651kB)\n",
            "\u001b[K     |████████████████████████████████| 655kB 2.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (2.21.0)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests) (1.24.3)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests) (2.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests) (2019.6.16)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests) (3.0.4)\n",
            "Building wheels for collected packages: regex\n",
            "  Building wheel for regex (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for regex: filename=regex-2019.6.8-cp36-cp36m-linux_x86_64.whl size=604152 sha256=653aae866aeb0a1f26d8f2d8007615f682c1a3c597017ba4680a7b31e9b484fa\n",
            "  Stored in directory: /root/.cache/pip/wheels/35/e4/80/abf3b33ba89cf65cd262af8a22a5a999cc28fbfabea6b38473\n",
            "Successfully built regex\n",
            "Installing collected packages: regex\n",
            "Successfully installed regex-2019.6.8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://github.com/pytorch/fairseq/archive/master.zip\" to /root/.cache/torch/hub/master.zip\n",
            "100%|██████████| 231160875/231160875 [00:08<00:00, 27313562.62B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "loading archive file http://dl.fbaipublicfiles.com/fairseq/models/roberta.base.tar.gz from cache at /root/.cache/torch/pytorch_fairseq/37d2bc14cf6332d61ed5abeb579948e6054e46cc724c7d23426382d11a31b2d6.ae5852b4abc6bf762e0b6b30f19e741aa05562471e9eb8f4a6ae261f04f9b350\n",
            "extracting archive file /root/.cache/torch/pytorch_fairseq/37d2bc14cf6332d61ed5abeb579948e6054e46cc724c7d23426382d11a31b2d6.ae5852b4abc6bf762e0b6b30f19e741aa05562471e9eb8f4a6ae261f04f9b350 to temp dir /tmp/tmplk7rvyfn\n",
            "| dictionary: 50264 types\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "1042301B [00:00, 2273765.11B/s]\n",
            "456318B [00:00, 17587447.74B/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RobertaHubInterface(\n",
              "  (model): RobertaModel(\n",
              "    (decoder): RobertaEncoder(\n",
              "      (sentence_encoder): TransformerSentenceEncoder(\n",
              "        (embed_tokens): Embedding(50265, 768, padding_idx=1)\n",
              "        (embed_positions): LearnedPositionalEmbedding(514, 768, padding_idx=1)\n",
              "        (layers): ModuleList(\n",
              "          (0): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (1): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (2): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (3): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (4): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (5): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (6): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (7): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (8): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (9): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (10): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "          (11): TransformerSentenceEncoderLayer(\n",
              "            (self_attn): MultiheadAttention(\n",
              "              (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
              "            )\n",
              "            (self_attn_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (final_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "          )\n",
              "        )\n",
              "        (emb_layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "      )\n",
              "      (lm_head): RobertaLMHead(\n",
              "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "        (layer_norm): LayerNorm(torch.Size([768]), eps=1e-05, elementwise_affine=True)\n",
              "      )\n",
              "    )\n",
              "    (classification_heads): ModuleDict()\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qd0tG8OhWu_o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def ParaGraphEncoderAverage(inputText):\n",
        "    tokens = roberta.encode(inputText)\n",
        "    tokens = tokens[0:512] #max input is 512\n",
        "    all_layers = roberta.extract_features(tokens, return_all_hiddens=True)\n",
        "    second_to_last = all_layers[-2]\n",
        "    return torch.mean(second_to_last[0], 0).cpu().detach().numpy()\n",
        "\n",
        "def ParaGraphEncoderMaxPooled(inputText):\n",
        "    tokens = roberta.encode(inputText)\n",
        "    tokens = tokens[0:512] #max input is 512\n",
        "    all_layers = roberta.extract_features(tokens, return_all_hiddens=True)\n",
        "    second_to_last = all_layers[-2]\n",
        "    return torch.max(second_to_last[0], 0).cpu().detach().numpy()\n",
        "\n",
        "def ParaGraphEncoderConcatMaxMinPooled(inputText):\n",
        "    tokens = roberta.encode(inputText)\n",
        "    tokens = tokens[0:512] #max input is 512\n",
        "    all_layers = roberta.extract_features(tokens, return_all_hiddens=True)\n",
        "    second_to_last = all_layers[-2]\n",
        "    maxedd = torch.max(second_to_last[0], 0)\n",
        "    minedd = torch.min(second_to_last[0], 0)\n",
        "    return torch.cat( (maxedd[0], minedd[0]), 0 ).cpu().detach().numpy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ddJoTWkwXryJ",
        "colab_type": "code",
        "outputId": "22cb69a2-5ab1-438b-9058-ba3bc98b09ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "paragraphEmbeddingAveraged = ParaGraphEncoderAverage('Hello world!')\n",
        "print(paragraphEmbeddingAveraged)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[ 8.73666927e-02 -1.99637026e-01  1.22691663e-02  2.06399903e-01\n",
            "  1.24098711e-01  8.76930133e-02 -3.39047387e-02 -1.87423870e-01\n",
            " -1.17235914e-01  2.35696077e-01 -4.36867714e-01 -2.00869903e-01\n",
            " -5.86469769e-02 -4.52387542e-01 -3.40143472e-01 -9.85440910e-02\n",
            " -3.05399507e-01 -4.54312056e-01  4.33829017e-02  2.85150856e-01\n",
            " -3.25825699e-02 -2.03095600e-02 -9.98308286e-02 -3.05308878e-01\n",
            "  3.34835619e-01 -1.61100924e-03  2.77123988e-01 -1.18258797e-01\n",
            "  3.31543803e-01  2.86710232e-01  4.62458283e-02 -2.41641030e-01\n",
            "  2.56850064e-01 -3.56712550e-01  1.53611138e-01  1.61433697e-01\n",
            "  2.97572076e-01  2.75653452e-01  7.67790794e-01  1.20076492e-01\n",
            "  2.96198040e-01 -5.88729501e-01 -4.82953200e-03  1.88496001e-02\n",
            "  1.50541082e-01 -4.33758609e-02 -4.64460999e-01 -2.89933115e-01\n",
            "  8.85296986e-02 -1.65153906e-01  1.42797083e-02  1.43214747e-01\n",
            " -1.69136655e-02  1.76706519e-02  1.63483605e-01  3.05359930e-01\n",
            " -2.35360309e-01  5.56727588e-01 -9.95012075e-02 -3.16942036e-02\n",
            " -1.17717467e-01  4.33179915e-01 -4.02339846e-01  1.08776264e-01\n",
            "  2.27870807e-01  9.79527012e-02  1.45521229e-02  6.37788236e-01\n",
            "  7.54483864e-02  2.27323517e-01 -3.90673764e-02 -2.79406428e-01\n",
            "  1.77474529e-01  1.35113537e-01 -6.87303692e-02 -2.65483886e-01\n",
            " -1.67565554e-01 -6.28978109e+00  3.53177994e-01  2.99336195e-01\n",
            "  8.74640793e-03 -1.04465701e-01  1.34101665e+00 -1.91177614e-02\n",
            " -1.29776284e-01  2.53018796e-01  3.20588760e-02  2.56804407e-01\n",
            "  2.12969705e-01 -1.02147818e-01  2.07508072e-01 -1.16752975e-01\n",
            " -1.06911018e-01 -1.28097773e-01  7.40606859e-02 -4.67544800e-04\n",
            "  2.02339422e-02  7.68555999e-01 -2.29974999e-03 -9.71042551e-03\n",
            " -1.67864546e-01 -6.64709210e-02  4.37050462e-02 -5.41746728e-02\n",
            " -1.09841051e-02  7.59393796e-02 -2.33788975e-02 -1.37068003e-01\n",
            " -5.08264080e-02  1.57635048e-01  2.37820625e-01  2.07694415e-02\n",
            "  8.78502429e-03 -2.07782224e-01  8.94916877e-02  2.73828596e-01\n",
            "  1.18472233e-01 -2.69576069e-02 -1.22306325e-01  4.21861410e-01\n",
            " -7.89186805e-02  2.81430669e-02 -3.52027982e-01 -7.00584799e-02\n",
            " -4.93729442e-01  4.13509995e-01 -2.36713409e-01 -1.89062521e-01\n",
            " -3.18654507e-01 -4.57350090e-02 -9.92444083e-02 -8.45107198e-01\n",
            " -1.85426831e-01  7.54969148e-03 -2.56520659e-01  6.50349632e-02\n",
            " -2.60180067e-02 -2.48741671e-01 -8.77987221e-02 -3.01976889e-01\n",
            " -1.58176854e-01  6.67965040e-02  5.64961806e-02 -9.48536918e-02\n",
            " -4.03686315e-02  1.87856540e-01 -1.47359401e-01 -1.49545804e-01\n",
            "  1.97126791e-01  2.66108125e-01 -2.21610352e-01  2.80313402e-01\n",
            "  1.77136421e-01 -2.62812227e-01 -2.86400206e-02  7.38346696e-01\n",
            "  1.96814924e-01  5.26624382e-01 -1.70812011e-02  4.70474690e-01\n",
            "  1.40663654e-01  2.86068674e-02  2.79235244e-02 -4.05746460e-01\n",
            " -7.84279034e-02  2.58121461e-01  1.25685677e-01  1.06293492e-01\n",
            " -8.24123546e-02 -1.35768503e-01  5.34895062e-02  8.66183639e-02\n",
            "  7.34874308e-02 -6.71594813e-02  2.55797595e-01  3.68030131e-01\n",
            " -1.75722837e-01 -4.25611846e-02  2.59613425e-01  3.22243392e-01\n",
            " -1.72252640e-01  2.17250094e-01 -2.09704503e-01  1.29294857e-01\n",
            " -6.38325838e-03  7.60299489e-02 -1.90243628e-02  1.54953435e-01\n",
            " -2.87292719e-01  3.82400006e-01 -1.68540534e-02  3.36306100e-03\n",
            " -1.06427521e-02  3.91952872e-01  2.11649612e-02  5.60884237e-01\n",
            "  1.69399932e-01 -2.23372027e-01  4.32692438e-01 -1.77241489e-01\n",
            "  1.93668127e-01  1.75697803e-02  4.99226093e-01 -4.10803795e-01\n",
            " -4.03797632e-04 -1.93519965e-01  2.70878803e-02  5.35834990e-02\n",
            " -2.34591633e-01  5.36371842e-02 -3.16218823e-01 -2.43578717e-01\n",
            "  2.84162015e-01  1.42241433e-01  4.03742224e-01  6.79153353e-02\n",
            "  1.03277162e-01 -1.35695195e+00  1.02352522e-01 -6.59626484e-01\n",
            "  9.06629935e-02 -4.44426537e-01  6.27383068e-02 -1.24192551e-01\n",
            "  1.87352896e-01  1.78192690e-01  1.99305430e-01 -3.26015837e-02\n",
            "  4.78810638e-01 -1.45660296e-01  2.00136498e-01 -2.10931212e-01\n",
            "  1.37432098e-01 -3.48216901e-03  1.98298082e-01 -7.87858516e-02\n",
            " -1.15567055e-02  2.44615987e-01 -1.24032550e-01  1.17247766e-02\n",
            "  4.88317788e-01  2.08886787e-01  9.26957056e-02  5.60811870e-02\n",
            " -8.89966413e-02 -2.61419863e-01 -7.42821917e-02  2.11906910e-01\n",
            "  3.02339822e-01 -3.35788161e-01 -1.16893291e-01 -1.58768103e-01\n",
            "  3.36382277e-02  2.31627524e-02 -5.80808580e-01 -2.84624487e-01\n",
            " -1.51666269e-01 -7.00675249e-02 -3.72917652e-02 -3.15370828e-01\n",
            " -9.48616266e-02  2.44190529e-01  9.35694054e-02 -1.46813452e-01\n",
            " -1.32255539e-01  3.35274518e-01  3.04135233e-01 -3.14186998e-02\n",
            "  1.67987868e-01 -3.84573430e-01  3.61562558e-02 -2.72792410e-02\n",
            "  5.28583646e-01  1.45536706e-01 -1.95431605e-01 -6.94258809e-02\n",
            "  3.02277952e-01 -1.20902196e-01  7.33579248e-02  6.32392690e-02\n",
            "  4.01643058e-03 -2.62486428e-01  6.32817268e-01 -6.61447272e-02\n",
            "  2.11514980e-01  4.51638132e-01  2.06798702e-01 -2.17928693e-01\n",
            " -1.69905514e-01 -2.87921485e-02  2.62589365e-01  3.28497857e-01\n",
            " -1.76780611e-01 -7.16122836e-02  3.27328257e-02 -2.04244137e-01\n",
            " -1.93425789e-02  2.81708688e-01 -1.93854049e-01  3.18306684e-01\n",
            "  1.22476891e-01  5.02528727e-01  1.29298076e-01 -1.70746192e-01\n",
            " -2.19617367e-01  1.21232055e-01  1.71994135e-01  1.74851399e-02\n",
            "  9.35484748e-03  3.53836268e-01 -7.32642412e-02 -1.33317024e-01\n",
            "  3.48792262e-02  1.71288773e-01  2.03093052e-01 -1.16799608e-01\n",
            " -7.18573341e-03  6.94776550e-02 -5.70155075e-03  2.58958220e-01\n",
            " -1.41078040e-01 -6.78190291e-02  8.89410600e-02 -4.44583595e-02\n",
            "  1.73934370e-01 -2.36201867e-01  3.04214567e-01 -9.97560006e-03\n",
            " -6.57249317e-02  2.84370989e-01  2.58269161e-01 -1.28138289e-01\n",
            " -8.48640513e-04 -7.28279874e-02 -7.04867244e-02 -1.06368743e-01\n",
            "  2.74827494e-03  3.23925093e-02  1.77581877e-01  9.32253152e-02\n",
            " -4.63593930e-01  3.46119106e-01 -3.64882290e-01  2.13369783e-02\n",
            "  1.57609712e-02  2.28898719e-01 -2.55306929e-01 -1.47461355e-01\n",
            "  9.34298858e-02  3.88509244e-01  1.23919562e-01 -1.84590265e-01\n",
            "  1.77805603e-01  6.28398657e-02 -2.74655789e-01 -7.42660090e-02\n",
            "  2.46609554e-01  3.83946270e-01 -1.58247426e-01 -8.23410153e-02\n",
            "  2.78262764e-01  9.89066720e-01  6.51132986e-02  4.82466668e-02\n",
            " -2.07439452e-01 -1.26662165e-01 -1.29404530e-01 -1.12664752e-01\n",
            "  1.33124173e-01  7.33843893e-02  5.29899359e-01 -2.24696562e-01\n",
            " -1.11655429e-01 -2.06438407e-01 -8.47372338e-02 -1.77066505e-01\n",
            "  2.68617004e-01  8.93865451e-02  2.54415601e-01  3.06344002e-01\n",
            "  2.69480180e-02 -1.06295824e-01  2.10802201e-02  5.03538139e-02\n",
            "  1.26414940e-01  2.01686099e-02  1.86572164e-01  3.04527313e-01\n",
            "  3.16318184e-01 -4.41336110e-02 -8.48862678e-02  1.44159541e-01\n",
            "  5.29922433e-02 -1.70398369e-01 -3.72615047e-02  1.52752832e-01\n",
            " -3.39466661e-01 -1.52279675e-01 -1.83460265e-02  2.67002106e-01\n",
            " -2.33679097e-02  1.94102988e-01 -4.20150766e-03  1.79577157e-01\n",
            "  6.39828965e-02  1.60673007e-01  6.57387003e-02 -9.51342732e-02\n",
            " -6.36258543e-01 -2.00581431e-01 -2.38456801e-02 -1.10309720e-01\n",
            " -5.56792282e-02 -1.72633473e-02 -1.14123657e-01  6.87930733e-02\n",
            " -6.95692673e-02 -4.18411583e-01 -1.28244339e-02 -7.27136061e-02\n",
            " -1.52616918e-01  2.83094078e-01 -1.87993228e-01  4.81993631e-02\n",
            "  4.20902342e-01  4.55760919e-02  5.33763826e-01 -2.79155821e-01\n",
            "  2.24063918e-01  1.10225059e-01  3.19393307e-01  1.81637794e-01\n",
            "  3.37936580e-01 -1.22254170e-01  5.49028218e-01 -9.99593362e-02\n",
            "  7.68923312e-02 -7.86026269e-02  4.33821715e-02  1.18374839e-01\n",
            " -1.32635236e-03  3.30052935e-02 -7.83448815e-02 -2.31123611e-01\n",
            " -1.66742459e-01 -1.33389443e-01 -4.70133603e-01  8.51581916e-02\n",
            "  3.10965449e-01  2.26266533e-01  1.39094755e-01  2.03959569e-01\n",
            " -3.49796861e-01 -1.61760795e+00 -6.79071918e-02  6.47972152e-02\n",
            " -2.32882742e-02 -1.95509598e-01 -3.35051984e-01 -1.01717032e-01\n",
            " -3.31353039e-01 -2.69670159e-01  2.70368811e-02 -1.48144308e-02\n",
            " -2.08613515e-01 -1.32694608e-02  1.73680291e-01 -2.54185408e-01\n",
            " -4.09530371e-01 -5.04766479e-02 -1.72261074e-01  4.35961504e-03\n",
            " -1.38942068e-02 -1.83696076e-01 -1.41306728e-01 -6.47151992e-02\n",
            "  5.79041354e-02  4.34346676e-01 -5.83865717e-02  1.21689118e-01\n",
            "  1.05776787e-01  2.45936856e-01  1.59324944e-01  7.79636875e-02\n",
            "  3.85522835e-05 -6.55732527e-02 -1.13002978e-01 -9.66713503e-02\n",
            "  7.64696002e-02 -3.57279986e-01  1.51718214e-01 -5.39560258e-01\n",
            " -2.70934433e-01  1.73668310e-01 -2.08265424e-01 -4.64865156e-02\n",
            " -7.98138559e-01 -1.80191286e-02 -2.32875347e-03  8.48113447e-02\n",
            "  1.80608973e-01  3.20350111e-01  4.40748423e-01  3.50252777e-01\n",
            " -5.17183952e-02  1.30572349e-01  3.27181828e-04 -1.49038121e-01\n",
            "  3.48015912e-02  9.40598268e-03  1.24514632e-01  3.99827689e-01\n",
            "  1.01426482e-01 -2.73354053e-01  2.35813975e-01  1.06513120e-01\n",
            " -6.22660458e-01 -1.87311515e-01 -5.59052289e-01 -8.86527356e-03\n",
            " -6.72102580e-03  3.32429558e-01 -1.06132850e-02 -4.92227040e-02\n",
            "  7.00883940e-02 -1.39784142e-01  4.67951745e-01 -1.45630345e-01\n",
            "  3.75515223e-02 -1.79393768e-01  3.03934604e-01  1.48687348e-01\n",
            " -2.12576091e-02  2.77415793e-02 -1.93132073e-01 -7.60256639e-03\n",
            " -2.12926134e-01 -6.76173568e-02 -4.69300181e-01  2.39542767e-01\n",
            " -2.48657420e-01  1.61809444e-01  1.31235525e-01  1.79269001e-01\n",
            "  8.17512423e-02  3.15006167e-01  2.64160484e-01 -1.68231085e-01\n",
            " -1.56044960e-01 -2.89640009e-01 -2.50689626e-01 -3.67295146e-01\n",
            "  2.29359731e-01  2.50592947e-01  1.34807453e-01  9.53461751e-02\n",
            "  2.94159412e-01 -5.37194610e-02  5.83698740e-03 -4.08717357e-02\n",
            " -2.01327607e-01  3.29520553e-01 -1.73884302e-01 -8.16064999e-02\n",
            " -3.40375364e-01  2.01257333e-01 -1.89619958e-01  3.39611530e-01\n",
            "  2.44919688e-01  2.99428646e-02  3.75716478e-01 -6.77929744e-02\n",
            " -2.20062599e-01 -5.70103109e-01 -4.15750742e-02  5.94894402e-02\n",
            "  2.82041520e-01  2.03774497e-01 -1.46707147e-02  1.24926269e-01\n",
            " -7.11996794e-01 -2.26106290e-02  1.50421724e-01  2.11558253e-01\n",
            "  2.43302863e-02 -2.67764535e-02 -4.03059572e-01 -7.50856221e-01\n",
            "  2.05142059e+01 -2.56232113e-01  1.07699111e-01  4.66969423e-02\n",
            "  6.03337996e-02  6.43466115e-02  2.31183246e-01  1.02472596e-01\n",
            " -2.89072573e-01  1.63373396e-01  1.33069875e-02  3.97687644e-01\n",
            " -5.15090339e-02 -1.08926706e-01  2.31480911e-01 -2.21881345e-01\n",
            " -2.11066395e-01 -7.25655779e-02 -2.97964811e-01  2.40352880e-02\n",
            " -4.87300344e-02  2.10046485e-01 -1.61684811e-01  1.32202613e+00\n",
            "  6.36040494e-02 -2.56776482e-01  6.37829155e-02  1.09947041e-01\n",
            " -8.06970149e-02 -5.26524745e-02 -2.77652174e-01  6.13488071e-02\n",
            "  6.05893694e-02 -5.07568121e-02  1.37797460e-01 -1.52483046e-01\n",
            "  4.97192591e-01  1.15600288e-01  2.15865448e-01  5.28861821e-01\n",
            "  9.92074758e-02 -1.72364756e-01 -2.49918196e-02 -9.43522677e-02\n",
            "  9.39104613e-03  2.61100680e-01  4.56084758e-02 -3.16583604e-01\n",
            " -9.64765102e-02  2.28472035e-02 -2.45185476e-02 -3.81110720e-02\n",
            " -1.32402629e-02  2.07565710e-01 -6.92958757e-02  2.66624838e-01\n",
            "  1.57097932e-02  3.29565890e-02  1.00318566e-02 -6.61138356e-01\n",
            "  3.15422505e-01  1.69693515e-01 -1.28888264e-01  6.91120565e-01\n",
            "  4.29547345e-03  2.52962559e-01  7.03761503e-02  1.36842176e-01\n",
            " -3.98207098e-01 -1.59125060e-01 -3.25247675e-01  1.10950880e-01\n",
            " -1.98324800e-01  5.49932659e-01  2.84125507e-01  2.68875748e-01\n",
            " -6.61397874e-01  9.30323638e-03 -2.05843717e-01  1.53732166e-01\n",
            "  2.45323833e-02 -8.51286873e-02  3.81035171e-02  4.16427106e-01\n",
            " -1.77439213e-01  2.67549425e-01  2.21587136e-01  1.19953237e-01\n",
            " -5.22530563e-02  4.40035552e-01 -1.18440323e-01  1.08880043e-01\n",
            "  9.44328029e-03  9.85304490e-02  4.13141638e-01 -1.67330354e-01\n",
            "  2.19455995e-02 -3.58256668e-01  3.03196043e-01 -2.51716465e-01\n",
            " -2.43850276e-01 -6.23613894e-02 -3.21277291e-01  6.00578897e-02\n",
            "  5.09087443e-02  1.30346045e-01  3.83292213e-02 -3.14501151e-02\n",
            "  4.21860674e-03  6.40215501e-02 -2.97155362e-02  4.65697497e-01\n",
            " -1.13415301e-01 -3.47940102e-02 -2.26185754e-01 -3.67886238e-02\n",
            "  6.38301298e-02  1.81196123e-01 -1.52971983e-01  2.62398303e-01\n",
            "  3.78773287e-02  6.00625835e-02 -1.67911723e-01  1.49945959e-01\n",
            " -3.81432958e-02 -1.63960144e-01  1.32392496e-01 -2.87879701e-03\n",
            "  2.56655157e-01  1.01604760e-02 -2.75379866e-01  1.05471902e-01\n",
            "  5.06354272e-01 -1.08895279e-01  1.03343204e-01 -1.98759481e-01\n",
            " -1.63869619e-01  2.40282103e-01 -3.74268234e-01 -6.99222237e-02\n",
            " -2.34632477e-01  1.14650860e-01  3.38736176e-01 -1.34221077e+00\n",
            "  2.35253841e-01 -2.61469573e-01  4.52172846e-01 -1.12485223e-01\n",
            " -8.12343806e-02 -1.33031920e-01 -9.62378755e-02 -1.40510753e-01\n",
            " -1.24439098e-01  3.64177883e-01  2.06723567e-02  3.90369892e-01\n",
            "  1.06524609e-01 -1.92722484e-01  3.45594697e-02 -6.18635304e-02\n",
            "  7.50783011e-02 -7.92787373e-01  3.70656043e-01  1.06842183e-01\n",
            "  4.55146223e-01  4.66249064e-02 -2.59080976e-01  3.05921827e-02\n",
            "  2.46675014e-01 -1.73242018e-01  1.62800416e-01  1.90283641e-01\n",
            "  2.37317141e-02  1.62162751e-01  2.05906868e-01  3.10916632e-01\n",
            "  3.46273899e-01  9.70843956e-02 -1.45187780e-01  2.84845024e-01]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G6YwsuprX-eu",
        "colab_type": "code",
        "outputId": "d7c2b48a-61b8-47b4-b971-c04618d815af",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "# paragraphEmbeddingMaxPooled = ParaGraphEncoderMaxPooled('Hello world!')\n",
        "# print(paragraphEmbeddingMaxPooled[0])"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-7-840d5ff528ec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mparagraphEmbeddingMaxPooled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mParaGraphEncoderMaxPooled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Hello world!'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparagraphEmbeddingMaxPooled\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-5-53cf7222112a>\u001b[0m in \u001b[0;36mParaGraphEncoderMaxPooled\u001b[0;34m(inputText)\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mall_layers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mroberta\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextract_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_all_hiddens\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0msecond_to_last\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mall_layers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msecond_to_last\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mParaGraphEncoderConcatMaxMinPooled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputText\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'torch.return_types.max' object has no attribute 'cpu'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fONlMIUuZzk8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# paragraphEmbeddingMaxMinConcat = ParaGraphEncoderConcatMaxMinPooled('Hello world!')\n",
        "# print(paragraphEmbeddingMaxMinConcat)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "we-LUmHS0Z5B",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import requests\n",
        "\n",
        "def download_file_from_google_drive(id, destination):\n",
        "    URL = \"https://docs.google.com/uc?export=download\"\n",
        "\n",
        "    session = requests.Session()\n",
        "\n",
        "    response = session.get(URL, params = { 'id' : id }, stream = True)\n",
        "    token = get_confirm_token(response)\n",
        "\n",
        "    if token:\n",
        "        params = { 'id' : id, 'confirm' : token }\n",
        "        response = session.get(URL, params = params, stream = True)\n",
        "\n",
        "    save_response_content(response, destination)    \n",
        "\n",
        "def get_confirm_token(response):\n",
        "    for key, value in response.cookies.items():\n",
        "        if key.startswith('download_warning'):\n",
        "            return value\n",
        "\n",
        "    return None\n",
        "\n",
        "def save_response_content(response, destination):\n",
        "    CHUNK_SIZE = 32768\n",
        "\n",
        "    with open(destination, \"wb\") as f:\n",
        "        for chunk in response.iter_content(CHUNK_SIZE):\n",
        "            if chunk: # filter out keep-alive new chunks\n",
        "                f.write(chunk)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lv4blz300eSJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Download yelp data\n",
        "\n",
        "download_file_from_google_drive('0Bz8a_Dbh9QhbNUpYQ2N3SGlFaDg', 'yelp.tar.gz')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yx8DYF_k0e2J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tarfile\n",
        "tf = tarfile.open(\"yelp.tar.gz\")\n",
        "tf.extractall()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CNz3EwFl1jb4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p8T8opKr0ijh",
        "colab_type": "code",
        "outputId": "5afe1b26-1d4b-4a8c-aafc-e4b949d0bb06",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "os.listdir('yelp_review_polarity_csv')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['test.csv', 'train.csv', 'readme.txt']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_o9qDtfJ0kVB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_DYBq8AN0ljR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pd.read_csv('yelp_review_polarity_csv/train.csv', header =None, nrows = 62000)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HKPuBnXk1xO4",
        "colab_type": "code",
        "outputId": "a3dcadd6-9e36-45e3-ec29-b24d06175e9e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        }
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>Unfortunately, the frustration of being Dr. Go...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>Been going to Dr. Goldberg for over 10 years. ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>I don't know what Dr. Goldberg was like before...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>I'm writing this review to give you a heads up...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>All the food is great here. But the best thing...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   0                                                  1\n",
              "0  1  Unfortunately, the frustration of being Dr. Go...\n",
              "1  2  Been going to Dr. Goldberg for over 10 years. ...\n",
              "2  1  I don't know what Dr. Goldberg was like before...\n",
              "3  1  I'm writing this review to give you a heads up...\n",
              "4  2  All the food is great here. But the best thing..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ILi3xvdE0tEa",
        "colab_type": "code",
        "outputId": "9384311d-f0d5-44f9-df86-d8d14832f7ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df.shape"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(62000, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hoe5uqHe133A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.columns = ['label', 'text']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BgrqupHJ1-Fg",
        "colab_type": "code",
        "outputId": "f39f71e3-b92b-4f3f-8c7d-780c0092b54b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        }
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>Unfortunately, the frustration of being Dr. Go...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>Been going to Dr. Goldberg for over 10 years. ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>I don't know what Dr. Goldberg was like before...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>I'm writing this review to give you a heads up...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>All the food is great here. But the best thing...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   label                                               text\n",
              "0      1  Unfortunately, the frustration of being Dr. Go...\n",
              "1      2  Been going to Dr. Goldberg for over 10 years. ...\n",
              "2      1  I don't know what Dr. Goldberg was like before...\n",
              "3      1  I'm writing this review to give you a heads up...\n",
              "4      2  All the food is great here. But the best thing..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ol76DIG64ohQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oNlwPdPl4dso",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df[\"averagePooled\"] = ''\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rablg-Rk4rru",
        "colab_type": "code",
        "outputId": "71c3fe6b-4067-4cfb-c643-95a3a3b53117",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "df.shape[0]"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "62000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RbmSe9ux413t",
        "colab_type": "code",
        "outputId": "7bba1e2a-17c0-4bf9-da96-a8aff772c0a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "df.loc[8]['text']"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"Before I finally made it over to this range I heard the same thing from most people - it's just fine to go work on your swing. I had such a low expectation I was pleasantly surprised. \\\\n\\\\nIt's a fairly big range - if you are familiar with Scally's in Moon, it seems like it has almost as many tees, though its not nearly as nice a facility. \\\\n\\\\nThe guys in the pro shop were two of the friendlier guys I've come across at ranges or at courses. Yards were indeed marked and there are some targets to aim for, and even some hazards to aim away from. \\\\n\\\\nA big red flag to me was the extra charge ($3) to hit off the grass. I am no range expert, but this is the 4th one I've been to and the first I've seen of that sort of nickel and diming....\\\\n\\\\nPrice for the golf balls was reasonable and I do plan to be back every week until they close up in October for the season. Hopefully, since its for sale, it will reopen as a golf facility again.\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uo97Q_zk7SUO",
        "colab_type": "code",
        "outputId": "8fbc8db8-3575-4c82-e5e7-f020846d700c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "len(ParaGraphEncoderAverage(df.loc[0]['text']) )"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "768"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fJ45pNKv5LC-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df.at[0, 'averagePooled'] = ParaGraphEncoderAverage(df.loc[0]['text'] )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xUhjTcYO5PH_",
        "colab_type": "code",
        "outputId": "69aea496-a466-4f39-f2a5-0d4eb95c7f7c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        }
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "      <th>averagePooled</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>Unfortunately, the frustration of being Dr. Go...</td>\n",
              "      <td>[0.120301716, 0.10964332, -0.0901535, -0.18390...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>Been going to Dr. Goldberg for over 10 years. ...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>I don't know what Dr. Goldberg was like before...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>I'm writing this review to give you a heads up...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2</td>\n",
              "      <td>All the food is great here. But the best thing...</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   label  ...                                      averagePooled\n",
              "0      1  ...  [0.120301716, 0.10964332, -0.0901535, -0.18390...\n",
              "1      2  ...                                                   \n",
              "2      1  ...                                                   \n",
              "3      1  ...                                                   \n",
              "4      2  ...                                                   \n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8DqWY-LZAK-p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['averagePooledValues'] = df.text.apply(ParaGraphEncoderAverage)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}